基本完成

This commit is contained in:
2026-08-25 22:53:57 +08:00
parent 05fa686749
commit 252d5b53ac
39 changed files with 3853 additions and 447 deletions
+239
View File
@@ -0,0 +1,239 @@
"""
Boss直聘 低痕迹 CDP 客户端
背景:Boss直聘 风控会探测"标签页被 CDP attach",一旦被 Playwright 接管,
页面会被重置为 about:blank。因此所有浏览器操作走无 attach 痕迹的原生通道:
- 导航:DevTools HTTP 协议(/json/new),由浏览器自身加载页面
- 读 Cookie:CDP WebSocket 的 Network.getCookies(不激活任何调试域)
- 读 DOM:CDP WebSocket 的 Runtime.evaluate(一次性表达式求值)
"""
import json
import time
from html.parser import HTMLParser
import requests
import websocket
from loguru import logger
from ..config import client_settings
class TextExtractor(HTMLParser):
"""从 HTML 片段中提取可见文本(跳过 script/style)"""
SKIP = {"script", "style", "noscript", "svg", "iframe"}
def __init__(self):
super().__init__()
self.parts: list[str] = []
self._skip_depth = 0
def handle_starttag(self, tag, attrs):
if tag in self.SKIP:
self._skip_depth += 1
elif tag in ("div", "p", "li", "br", "tr", "section", "h1", "h2", "h3", "h4"):
self.parts.append("\n")
def handle_endtag(self, tag):
if tag in self.SKIP and self._skip_depth > 0:
self._skip_depth -= 1
def handle_data(self, data):
if self._skip_depth == 0 and data.strip():
self.parts.append(data.strip())
def text(self) -> str:
lines = [ln.strip() for ln in "".join(self.parts).splitlines()]
return "\n".join(ln for ln in lines if ln)
def extract_text(html: str) -> str:
parser = TextExtractor()
parser.feed(html)
return parser.text()
class CDPSession:
"""单个标签页上的持久 CDP 会话:命令调用 / 事件泵 / 可信鼠标点击。
只启用必要域(Network 被动监听站点自身 XHR,Input 产生浏览器级可信点击),
不 attach 页面、不注入脚本,保持低痕迹。"""
def __init__(self, ws_url: str):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method: str, params: dict | None = None, timeout: float = 10) -> dict:
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expression: str, timeout: float = 10):
"""一次性表达式求值,返回结果值"""
return (self.call("Runtime.evaluate",
{"expression": expression, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration: float):
"""在 duration 秒内持续产出 CDP 事件(用于被动监听网络响应)"""
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def click_at(self, x: float, y: float) -> None:
"""浏览器级可信鼠标点击(isTrusted=true,与真人鼠标无异)"""
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": int(x), "y": int(y),
"button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": int(x), "y": int(y),
"button": "left", "clickCount": 1})
def close(self) -> None:
try:
self.ws.close()
except Exception:
pass
class LiteCDP:
"""无 Playwright 依赖的浏览器操作封装"""
def __init__(self):
self.debug_port = client_settings.BROWSER_DEBUG_PORT
# ------------------------------------------------------------------
# DevTools HTTP 协议
# ------------------------------------------------------------------
def _endpoint(self) -> str:
return f"http://127.0.0.1:{self.debug_port}"
def is_running(self) -> bool:
try:
resp = requests.get(f"{self._endpoint()}/json/version", timeout=3)
return resp.status_code == 200
except Exception:
return False
def list_tabs(self) -> list[dict]:
try:
return requests.get(f"{self._endpoint()}/json", timeout=5).json()
except Exception:
return []
def open_tab(self, url: str) -> dict:
"""原生方式新开标签页并导航(浏览器自身加载,无自动化痕迹)"""
r = requests.put(f"{self._endpoint()}/json/new?{url}", timeout=10)
if r.status_code == 405: # 部分 Chrome 版本只接受 GET
r = requests.get(f"{self._endpoint()}/json/new?{url}", timeout=10)
r.raise_for_status()
return r.json()
def close_tab(self, tab_id: str) -> None:
try:
requests.get(f"{self._endpoint()}/json/close/{tab_id}", timeout=5)
except Exception:
pass
def activate_tab(self, tab_id: str) -> None:
"""重新激活指定标签页。开/关临时标签会夺走焦点,
后台标签上的可信鼠标点击不生效,必须恢复前台。"""
try:
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
except Exception:
pass
def activate_tab(self, tab_id: str) -> None:
"""重新激活指定标签页(开/关临时标签会夺走焦点,
后台标签上的可信鼠标点击会失效,需恢复前台)"""
try:
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
except Exception:
pass
def tab_url(self, tab_id: str) -> str | None:
for t in self.list_tabs():
if t.get("id") == tab_id:
return t.get("url")
return None
# ------------------------------------------------------------------
# CDP WebSocket(只发必要的命令,不激活 Page/Runtime/Debugger 域监听)
# ------------------------------------------------------------------
def evaluate(self, ws_url: str, expression: str, timeout: float = 20):
"""一次性表达式求值,返回结果值;失败返回 None"""
ws = websocket.create_connection(ws_url, timeout=timeout)
try:
ws.send(json.dumps({
"id": 1,
"method": "Runtime.evaluate",
"params": {"expression": expression, "returnByValue": True},
}))
while True:
msg = json.loads(ws.recv())
if msg.get("id") == 1:
return msg.get("result", {}).get("result", {}).get("value")
finally:
try:
ws.close()
except Exception:
pass
def get_cookie_names(self) -> set[str]:
"""读取 zhipin 域 Cookie 名称集合(取任一标签页的 ws 通道即可)"""
tabs = [t for t in self.list_tabs() if t.get("webSocketDebuggerUrl")]
if not tabs:
return set()
ws = websocket.create_connection(tabs[0]["webSocketDebuggerUrl"], timeout=10)
try:
ws.send(json.dumps({
"id": 1,
"method": "Network.getCookies",
"params": {"urls": ["https://www.zhipin.com/"]},
}))
while True:
msg = json.loads(ws.recv())
if msg.get("id") == 1:
cookies = msg.get("result", {}).get("cookies", [])
return {c["name"] for c in cookies}
finally:
try:
ws.close()
except Exception:
pass
# ------------------------------------------------------------------
# 组合操作:开页 → 等渲染 → 取 DOM → 关页
# ------------------------------------------------------------------
def fetch_rendered_html(self, url: str, wait_s: float = 8,
max_wait_s: float = 20) -> str | None:
"""打开目标页等待 SPA 渲染,取回完整 HTML 后关闭标签页"""
tab = self.open_tab(url)
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
try:
deadline = time.time() + max_wait_s
waited = 0.0
html = ""
while time.time() < deadline:
time.sleep(2)
waited += 2
try:
html = self.evaluate(
ws_url, "document.documentElement.outerHTML") or ""
except Exception as e:
logger.debug(f"取 DOM 失败({waited}s):{e}")
continue
# 渲染完成的经验判断:内容足够长且非空壳
if len(html) > 50000 and waited >= wait_s:
return html
return html or None
finally:
self.close_tab(tab_id)