162 lines
5.9 KiB
Python
162 lines
5.9 KiB
Python
"""五次探测:明文薪资来源(SSR 内嵌 JSON / 滚动 joblist XHR)+ 详情页/聊天页结构"""
|
||
import json
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import websocket
|
||
|
||
from client.browser.cdp_lite import LiteCDP
|
||
from client.browser.manager import BrowserManager
|
||
|
||
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe5.txt"
|
||
|
||
_lines: list[str] = []
|
||
|
||
|
||
def log(*a):
|
||
s = " ".join(str(x) for x in a)
|
||
_lines.append(s)
|
||
OUT.write_text("\n".join(_lines), encoding="utf-8")
|
||
|
||
|
||
class Session:
|
||
def __init__(self, ws_url):
|
||
self.ws = websocket.create_connection(ws_url, timeout=30)
|
||
self._id = 0
|
||
|
||
def call(self, method, params=None, timeout=10):
|
||
self._id += 1
|
||
mid = self._id
|
||
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
|
||
self.ws.settimeout(timeout)
|
||
while True:
|
||
msg = json.loads(self.ws.recv())
|
||
if msg.get("id") == mid:
|
||
return msg.get("result") or {}
|
||
|
||
def eval_js(self, expr, timeout=10):
|
||
return (self.call("Runtime.evaluate",
|
||
{"expression": expr, "returnByValue": True}, timeout)
|
||
.get("result", {}).get("value"))
|
||
|
||
def pump(self, duration):
|
||
self.ws.settimeout(0.5)
|
||
end = time.time() + duration
|
||
while time.time() < end:
|
||
try:
|
||
yield json.loads(self.ws.recv())
|
||
except Exception:
|
||
continue
|
||
|
||
def close(self):
|
||
try:
|
||
self.ws.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def main():
|
||
BrowserManager().ensure_browser_started()
|
||
lite = LiteCDP()
|
||
|
||
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
|
||
s = Session(tab["webSocketDebuggerUrl"])
|
||
s.call("Network.enable")
|
||
s.eval_js("""(() => {
|
||
const inp = document.querySelector('input[placeholder*="搜索"]');
|
||
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
|
||
setter.call(inp, 'java');
|
||
inp.dispatchEvent(new Event('input', {bubbles: true}));
|
||
document.querySelector('.search-btn').click();
|
||
return true;
|
||
})()""")
|
||
time.sleep(5)
|
||
|
||
# 1) SSR 内嵌 JSON 是否含明文薪资
|
||
r1 = s.eval_js("""(() => {
|
||
const scripts = [...document.querySelectorAll('script')].map(x => x.textContent || '');
|
||
const hit = scripts.find(t => t.includes('"salary"') && t.length > 500);
|
||
if (!hit) return JSON.stringify({found: false, lens: scripts.map(t => t.length).filter(l => l > 200)});
|
||
const i = hit.indexOf('"salary"');
|
||
return JSON.stringify({found: true, snippet: hit.slice(Math.max(0, i - 200), i + 300)});
|
||
})()""")
|
||
log("== 1) SSR 内嵌薪资 ==", r1)
|
||
|
||
# 2) 滚动触发 joblist XHR,被动捕获明文
|
||
req_urls: dict[str, str] = {}
|
||
finished: list[str] = []
|
||
s.eval_js("window.scrollTo(0, document.body.scrollHeight)")
|
||
for msg in s.pump(7):
|
||
m = msg.get("method")
|
||
if m == "Network.responseReceived":
|
||
u = msg["params"]["response"]["url"]
|
||
if "joblist" in u:
|
||
req_urls[msg["params"]["requestId"]] = u
|
||
elif m == "Network.loadingFinished":
|
||
rid = msg["params"]["requestId"]
|
||
if rid in req_urls:
|
||
finished.append(rid)
|
||
log("== 2) joblist XHR ==", [req_urls[r] for r in finished])
|
||
for rid in finished[:1]:
|
||
body = s.call("Network.getResponseBody", {"requestId": rid}).get("body", "")
|
||
try:
|
||
data = json.loads(body)
|
||
zp = data.get("zpData", {})
|
||
jobs = zp.get("jobList") or []
|
||
log("== 2b) 明文字段 ==", json.dumps({
|
||
"keys": list(zp.keys()),
|
||
"first": {k: jobs[0].get(k) for k in
|
||
("encryptJobId", "jobName", "salaryDesc", "salary", "companyName", "cityName", "jobDegree", "experience")} if jobs else None,
|
||
"allKeysOfJob": list(jobs[0].keys()) if jobs else [],
|
||
}, ensure_ascii=False)[:2500])
|
||
except Exception as e:
|
||
log("== 2b) 解析失败 ==", e, body[:300])
|
||
|
||
# 3) 详情页 + 聊天页
|
||
href = s.eval_js("(document.querySelector('.job-card-wrap a.job-name') || {getAttribute: () => ''}).getAttribute('href')")
|
||
s.close()
|
||
lite.close_tab(tab["id"])
|
||
log("== 3) 首职位 ==", href)
|
||
if not href:
|
||
log("== DONE ==")
|
||
return
|
||
|
||
dtab = lite.open_tab("https://www.zhipin.com" + href)
|
||
d = Session(dtab["webSocketDebuggerUrl"])
|
||
time.sleep(5)
|
||
r4 = d.eval_js("""(() => {
|
||
const btns = [...document.querySelectorAll('a,button')]
|
||
.filter(e => /立即沟通|投递简历/.test(e.innerText || '') && e.offsetHeight > 0)
|
||
.map(e => ({tag: e.tagName, cls: e.className}));
|
||
const cands = {};
|
||
['.job-detail-section', '.job-sec-text', '.detail-content', '.job-detail .text']
|
||
.forEach(sel => { const el = document.querySelector(sel); cands[sel] = el ? el.innerText.slice(0, 80) : null; });
|
||
return JSON.stringify({url: location.href, btns, cands});
|
||
})()""")
|
||
log("== 4) 详情页 ==", r4)
|
||
|
||
d.eval_js("""(() => {
|
||
const btn = [...document.querySelectorAll('a,button')].find(e => /立即沟通/.test(e.innerText || '') && e.offsetHeight > 0);
|
||
if (btn) btn.click();
|
||
return !!btn;
|
||
})()""")
|
||
time.sleep(4)
|
||
r5 = d.eval_js("""(() => {
|
||
const ta = document.querySelector('textarea');
|
||
const sends = [...document.querySelectorAll('a,button,div')]
|
||
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
|
||
.map(e => ({tag: e.tagName, cls: e.className}));
|
||
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
|
||
inChatInput: ta ? !!ta.closest('.chat-input') : false, sends: sends.slice(0, 5)});
|
||
})()""")
|
||
log("== 5) 聊天页 ==", r5)
|
||
|
||
d.close()
|
||
lite.close_tab(dtab["id"])
|
||
log("== DONE ==")
|
||
print("probe5 done ->", OUT)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|