147 lines
5.3 KiB
Python
147 lines
5.3 KiB
Python
"""六次探测:滚动分页 XHR 真实端点 + 立即沟通新标签页的聊天结构"""
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import websocket
|
|
|
|
from client.browser.cdp_lite import LiteCDP
|
|
from client.browser.manager import BrowserManager
|
|
|
|
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe6.txt"
|
|
|
|
_lines: list[str] = []
|
|
|
|
|
|
def log(*a):
|
|
s = " ".join(str(x) for x in a)
|
|
_lines.append(s)
|
|
OUT.write_text("\n".join(_lines), encoding="utf-8")
|
|
|
|
|
|
class Session:
|
|
def __init__(self, ws_url):
|
|
self.ws = websocket.create_connection(ws_url, timeout=30)
|
|
self._id = 0
|
|
|
|
def call(self, method, params=None, timeout=10):
|
|
self._id += 1
|
|
mid = self._id
|
|
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
|
|
self.ws.settimeout(timeout)
|
|
while True:
|
|
msg = json.loads(self.ws.recv())
|
|
if msg.get("id") == mid:
|
|
return msg.get("result") or {}
|
|
|
|
def eval_js(self, expr, timeout=10):
|
|
return (self.call("Runtime.evaluate",
|
|
{"expression": expr, "returnByValue": True}, timeout)
|
|
.get("result", {}).get("value"))
|
|
|
|
def pump(self, duration):
|
|
self.ws.settimeout(0.5)
|
|
end = time.time() + duration
|
|
while time.time() < end:
|
|
try:
|
|
yield json.loads(self.ws.recv())
|
|
except Exception:
|
|
continue
|
|
|
|
def close(self):
|
|
try:
|
|
self.ws.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def main():
|
|
BrowserManager().ensure_browser_started()
|
|
lite = LiteCDP()
|
|
|
|
# ---------- A. 滚动分页端点 ----------
|
|
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
|
|
s = Session(tab["webSocketDebuggerUrl"])
|
|
s.call("Network.enable")
|
|
s.eval_js("""(() => {
|
|
const inp = document.querySelector('input[placeholder*="搜索"]');
|
|
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
|
|
setter.call(inp, 'java');
|
|
inp.dispatchEvent(new Event('input', {bubbles: true}));
|
|
document.querySelector('.search-btn').click();
|
|
return true;
|
|
})()""")
|
|
time.sleep(5)
|
|
|
|
seen: list[str] = []
|
|
finished: dict[str, str] = {}
|
|
for _ in range(3): # 滚三轮
|
|
s.eval_js("window.scrollTo(0, document.body.scrollHeight)")
|
|
for msg in s.pump(5):
|
|
m = msg.get("method")
|
|
if m == "Network.responseReceived":
|
|
u = msg["params"]["response"]["url"]
|
|
if "/wapi/" in u and u not in seen:
|
|
seen.append(u)
|
|
rid = msg["params"]["requestId"]
|
|
finished[rid] = u
|
|
elif m == "Network.loadingFinished":
|
|
rid = msg["params"]["requestId"]
|
|
if rid in finished and "page=" in finished[rid]:
|
|
body = s.call("Network.getResponseBody", {"requestId": rid}).get("body", "")
|
|
try:
|
|
zp = json.loads(body).get("zpData", {})
|
|
jobs = zp.get("jobList") or []
|
|
log("== A2) 分页响应 ==", finished[rid][:150],
|
|
"jobs:", len(jobs),
|
|
"firstKeys:", list(jobs[0].keys())[:25] if jobs else [],
|
|
"salaryField:", (jobs[0].get("salaryDesc") or jobs[0].get("salary")) if jobs else None)
|
|
except Exception as e:
|
|
log("== A2) 解析失败 ==", e)
|
|
log("== A1) 滚动期间全部 wapi 请求 ==", seen)
|
|
cards = s.eval_js("document.querySelectorAll('.job-card-wrap').length")
|
|
log("== A3) 三轮滚动后卡片数 ==", cards)
|
|
|
|
# ---------- B. 立即沟通的新标签页 ----------
|
|
href = s.eval_js("(document.querySelector('.job-card-wrap a.job-name') || {getAttribute: () => ''}).getAttribute('href')")
|
|
s.close()
|
|
lite.close_tab(tab["id"])
|
|
|
|
before = {t["id"] for t in lite.list_tabs()}
|
|
dtab = lite.open_tab("https://www.zhipin.com" + href)
|
|
d = Session(dtab["webSocketDebuggerUrl"])
|
|
time.sleep(5)
|
|
d.eval_js("""(() => {
|
|
const btn = document.querySelector('.btn-startchat');
|
|
if (btn) btn.click();
|
|
return !!btn;
|
|
})()""")
|
|
time.sleep(4)
|
|
after = lite.list_tabs()
|
|
new_tabs = [t for t in after if t["id"] not in before and t["id"] != dtab["id"]]
|
|
log("== B1) 点击后新增标签页 ==", [(t.get("url"), t.get("id")) for t in new_tabs])
|
|
|
|
if new_tabs:
|
|
c = Session(new_tabs[0]["webSocketDebuggerUrl"])
|
|
time.sleep(3)
|
|
r = c.eval_js("""(() => {
|
|
const ta = document.querySelector('textarea');
|
|
const sends = [...document.querySelectorAll('a,button,div')]
|
|
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
|
|
.map(e => ({tag: e.tagName, cls: e.className}));
|
|
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
|
|
inChatInput: ta ? !!ta.closest('.chat-input') : false, sends: sends.slice(0, 5)});
|
|
})()""")
|
|
log("== B2) 聊天页结构 ==", r)
|
|
c.close()
|
|
lite.close_tab(new_tabs[0]["id"])
|
|
|
|
d.close()
|
|
lite.close_tab(dtab["id"])
|
|
log("== DONE ==")
|
|
print("probe6 done ->", OUT)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|