基本完成
This commit is contained in:
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Boss直聘 低痕迹 CDP 客户端
|
||||
背景:Boss直聘 风控会探测"标签页被 CDP attach",一旦被 Playwright 接管,
|
||||
页面会被重置为 about:blank。因此所有浏览器操作走无 attach 痕迹的原生通道:
|
||||
- 导航:DevTools HTTP 协议(/json/new),由浏览器自身加载页面
|
||||
- 读 Cookie:CDP WebSocket 的 Network.getCookies(不激活任何调试域)
|
||||
- 读 DOM:CDP WebSocket 的 Runtime.evaluate(一次性表达式求值)
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
from html.parser import HTMLParser
|
||||
|
||||
import requests
|
||||
import websocket
|
||||
from loguru import logger
|
||||
|
||||
from ..config import client_settings
|
||||
|
||||
|
||||
class TextExtractor(HTMLParser):
|
||||
"""从 HTML 片段中提取可见文本(跳过 script/style)"""
|
||||
|
||||
SKIP = {"script", "style", "noscript", "svg", "iframe"}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.parts: list[str] = []
|
||||
self._skip_depth = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in self.SKIP:
|
||||
self._skip_depth += 1
|
||||
elif tag in ("div", "p", "li", "br", "tr", "section", "h1", "h2", "h3", "h4"):
|
||||
self.parts.append("\n")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in self.SKIP and self._skip_depth > 0:
|
||||
self._skip_depth -= 1
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._skip_depth == 0 and data.strip():
|
||||
self.parts.append(data.strip())
|
||||
|
||||
def text(self) -> str:
|
||||
lines = [ln.strip() for ln in "".join(self.parts).splitlines()]
|
||||
return "\n".join(ln for ln in lines if ln)
|
||||
|
||||
|
||||
def extract_text(html: str) -> str:
|
||||
parser = TextExtractor()
|
||||
parser.feed(html)
|
||||
return parser.text()
|
||||
|
||||
|
||||
class CDPSession:
|
||||
"""单个标签页上的持久 CDP 会话:命令调用 / 事件泵 / 可信鼠标点击。
|
||||
只启用必要域(Network 被动监听站点自身 XHR,Input 产生浏览器级可信点击),
|
||||
不 attach 页面、不注入脚本,保持低痕迹。"""
|
||||
|
||||
def __init__(self, ws_url: str):
|
||||
self.ws = websocket.create_connection(ws_url, timeout=30)
|
||||
self._id = 0
|
||||
|
||||
def call(self, method: str, params: dict | None = None, timeout: float = 10) -> dict:
|
||||
self._id += 1
|
||||
mid = self._id
|
||||
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
|
||||
self.ws.settimeout(timeout)
|
||||
while True:
|
||||
msg = json.loads(self.ws.recv())
|
||||
if msg.get("id") == mid:
|
||||
return msg.get("result") or {}
|
||||
|
||||
def eval_js(self, expression: str, timeout: float = 10):
|
||||
"""一次性表达式求值,返回结果值"""
|
||||
return (self.call("Runtime.evaluate",
|
||||
{"expression": expression, "returnByValue": True}, timeout)
|
||||
.get("result", {}).get("value"))
|
||||
|
||||
def pump(self, duration: float):
|
||||
"""在 duration 秒内持续产出 CDP 事件(用于被动监听网络响应)"""
|
||||
self.ws.settimeout(0.5)
|
||||
end = time.time() + duration
|
||||
while time.time() < end:
|
||||
try:
|
||||
yield json.loads(self.ws.recv())
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
def click_at(self, x: float, y: float) -> None:
|
||||
"""浏览器级可信鼠标点击(isTrusted=true,与真人鼠标无异)"""
|
||||
self.call("Input.dispatchMouseEvent", {
|
||||
"type": "mousePressed", "x": int(x), "y": int(y),
|
||||
"button": "left", "clickCount": 1})
|
||||
time.sleep(0.08)
|
||||
self.call("Input.dispatchMouseEvent", {
|
||||
"type": "mouseReleased", "x": int(x), "y": int(y),
|
||||
"button": "left", "clickCount": 1})
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self.ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class LiteCDP:
|
||||
"""无 Playwright 依赖的浏览器操作封装"""
|
||||
|
||||
def __init__(self):
|
||||
self.debug_port = client_settings.BROWSER_DEBUG_PORT
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# DevTools HTTP 协议
|
||||
# ------------------------------------------------------------------
|
||||
def _endpoint(self) -> str:
|
||||
return f"http://127.0.0.1:{self.debug_port}"
|
||||
|
||||
def is_running(self) -> bool:
|
||||
try:
|
||||
resp = requests.get(f"{self._endpoint()}/json/version", timeout=3)
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_tabs(self) -> list[dict]:
|
||||
try:
|
||||
return requests.get(f"{self._endpoint()}/json", timeout=5).json()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def open_tab(self, url: str) -> dict:
|
||||
"""原生方式新开标签页并导航(浏览器自身加载,无自动化痕迹)"""
|
||||
r = requests.put(f"{self._endpoint()}/json/new?{url}", timeout=10)
|
||||
if r.status_code == 405: # 部分 Chrome 版本只接受 GET
|
||||
r = requests.get(f"{self._endpoint()}/json/new?{url}", timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def close_tab(self, tab_id: str) -> None:
|
||||
try:
|
||||
requests.get(f"{self._endpoint()}/json/close/{tab_id}", timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def activate_tab(self, tab_id: str) -> None:
|
||||
"""重新激活指定标签页。开/关临时标签会夺走焦点,
|
||||
后台标签上的可信鼠标点击不生效,必须恢复前台。"""
|
||||
try:
|
||||
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def activate_tab(self, tab_id: str) -> None:
|
||||
"""重新激活指定标签页(开/关临时标签会夺走焦点,
|
||||
后台标签上的可信鼠标点击会失效,需恢复前台)"""
|
||||
try:
|
||||
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def tab_url(self, tab_id: str) -> str | None:
|
||||
for t in self.list_tabs():
|
||||
if t.get("id") == tab_id:
|
||||
return t.get("url")
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CDP WebSocket(只发必要的命令,不激活 Page/Runtime/Debugger 域监听)
|
||||
# ------------------------------------------------------------------
|
||||
def evaluate(self, ws_url: str, expression: str, timeout: float = 20):
|
||||
"""一次性表达式求值,返回结果值;失败返回 None"""
|
||||
ws = websocket.create_connection(ws_url, timeout=timeout)
|
||||
try:
|
||||
ws.send(json.dumps({
|
||||
"id": 1,
|
||||
"method": "Runtime.evaluate",
|
||||
"params": {"expression": expression, "returnByValue": True},
|
||||
}))
|
||||
while True:
|
||||
msg = json.loads(ws.recv())
|
||||
if msg.get("id") == 1:
|
||||
return msg.get("result", {}).get("result", {}).get("value")
|
||||
finally:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_cookie_names(self) -> set[str]:
|
||||
"""读取 zhipin 域 Cookie 名称集合(取任一标签页的 ws 通道即可)"""
|
||||
tabs = [t for t in self.list_tabs() if t.get("webSocketDebuggerUrl")]
|
||||
if not tabs:
|
||||
return set()
|
||||
ws = websocket.create_connection(tabs[0]["webSocketDebuggerUrl"], timeout=10)
|
||||
try:
|
||||
ws.send(json.dumps({
|
||||
"id": 1,
|
||||
"method": "Network.getCookies",
|
||||
"params": {"urls": ["https://www.zhipin.com/"]},
|
||||
}))
|
||||
while True:
|
||||
msg = json.loads(ws.recv())
|
||||
if msg.get("id") == 1:
|
||||
cookies = msg.get("result", {}).get("cookies", [])
|
||||
return {c["name"] for c in cookies}
|
||||
finally:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 组合操作:开页 → 等渲染 → 取 DOM → 关页
|
||||
# ------------------------------------------------------------------
|
||||
def fetch_rendered_html(self, url: str, wait_s: float = 8,
|
||||
max_wait_s: float = 20) -> str | None:
|
||||
"""打开目标页等待 SPA 渲染,取回完整 HTML 后关闭标签页"""
|
||||
tab = self.open_tab(url)
|
||||
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
|
||||
try:
|
||||
deadline = time.time() + max_wait_s
|
||||
waited = 0.0
|
||||
html = ""
|
||||
while time.time() < deadline:
|
||||
time.sleep(2)
|
||||
waited += 2
|
||||
try:
|
||||
html = self.evaluate(
|
||||
ws_url, "document.documentElement.outerHTML") or ""
|
||||
except Exception as e:
|
||||
logger.debug(f"取 DOM 失败({waited}s):{e}")
|
||||
continue
|
||||
# 渲染完成的经验判断:内容足够长且非空壳
|
||||
if len(html) > 50000 and waited >= wait_s:
|
||||
return html
|
||||
return html or None
|
||||
finally:
|
||||
self.close_tab(tab_id)
|
||||
Reference in New Issue
Block a user