252 lines
9.5 KiB
Python
252 lines
9.5 KiB
Python
"""
|
||
Boss直聘 低痕迹 CDP 客户端
|
||
背景:Boss直聘 风控会探测"标签页被 CDP attach",一旦被 Playwright 接管,
|
||
页面会被重置为 about:blank。因此所有浏览器操作走无 attach 痕迹的原生通道:
|
||
- 导航:DevTools HTTP 协议(/json/new),由浏览器自身加载页面
|
||
- 读 Cookie:CDP WebSocket 的 Network.getCookies(不激活任何调试域)
|
||
- 读 DOM:CDP WebSocket 的 Runtime.evaluate(一次性表达式求值)
|
||
"""
|
||
import json
|
||
import random
|
||
import time
|
||
from html.parser import HTMLParser
|
||
|
||
import requests
|
||
import websocket
|
||
from loguru import logger
|
||
|
||
from ..config import client_settings
|
||
|
||
|
||
class TextExtractor(HTMLParser):
|
||
"""从 HTML 片段中提取可见文本(跳过 script/style)"""
|
||
|
||
SKIP = {"script", "style", "noscript", "svg", "iframe"}
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.parts: list[str] = []
|
||
self._skip_depth = 0
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
if tag in self.SKIP:
|
||
self._skip_depth += 1
|
||
elif tag in ("div", "p", "li", "br", "tr", "section", "h1", "h2", "h3", "h4"):
|
||
self.parts.append("\n")
|
||
|
||
def handle_endtag(self, tag):
|
||
if tag in self.SKIP and self._skip_depth > 0:
|
||
self._skip_depth -= 1
|
||
|
||
def handle_data(self, data):
|
||
if self._skip_depth == 0 and data.strip():
|
||
self.parts.append(data.strip())
|
||
|
||
def text(self) -> str:
|
||
lines = [ln.strip() for ln in "".join(self.parts).splitlines()]
|
||
return "\n".join(ln for ln in lines if ln)
|
||
|
||
|
||
def extract_text(html: str) -> str:
|
||
parser = TextExtractor()
|
||
parser.feed(html)
|
||
return parser.text()
|
||
|
||
|
||
class CDPSession:
|
||
"""单个标签页上的持久 CDP 会话:命令调用 / 事件泵 / 可信鼠标点击。
|
||
只启用必要域(Network 被动监听站点自身 XHR,Input 产生浏览器级可信点击),
|
||
不 attach 页面、不注入脚本,保持低痕迹。"""
|
||
|
||
def __init__(self, ws_url: str):
|
||
self.ws = websocket.create_connection(ws_url, timeout=30)
|
||
self._id = 0
|
||
|
||
def call(self, method: str, params: dict | None = None, timeout: float = 10) -> dict:
|
||
self._id += 1
|
||
mid = self._id
|
||
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
|
||
self.ws.settimeout(timeout)
|
||
while True:
|
||
msg = json.loads(self.ws.recv())
|
||
if msg.get("id") == mid:
|
||
return msg.get("result") or {}
|
||
|
||
def eval_js(self, expression: str, timeout: float = 10):
|
||
"""一次性表达式求值,返回结果值"""
|
||
return (self.call("Runtime.evaluate",
|
||
{"expression": expression, "returnByValue": True}, timeout)
|
||
.get("result", {}).get("value"))
|
||
|
||
def pump(self, duration: float):
|
||
"""在 duration 秒内持续产出 CDP 事件(用于被动监听网络响应)"""
|
||
self.ws.settimeout(0.5)
|
||
end = time.time() + duration
|
||
while time.time() < end:
|
||
try:
|
||
yield json.loads(self.ws.recv())
|
||
except Exception:
|
||
continue
|
||
|
||
def click_at(self, x: float, y: float) -> None:
|
||
"""浏览器级可信鼠标点击(isTrusted=true,与真人鼠标无异)"""
|
||
self.call("Input.dispatchMouseEvent", {
|
||
"type": "mousePressed", "x": int(x), "y": int(y),
|
||
"button": "left", "clickCount": 1})
|
||
time.sleep(0.08)
|
||
self.call("Input.dispatchMouseEvent", {
|
||
"type": "mouseReleased", "x": int(x), "y": int(y),
|
||
"button": "left", "clickCount": 1})
|
||
|
||
def human_click(self, x: float, y: float) -> None:
|
||
"""拟人化可信点击:先从上次位置移动鼠标(带轨迹),
|
||
再以随机偏移落下,避免“无 mousemove 历史 + 永远点正中心”的机器特征"""
|
||
tx = x + random.uniform(-6, 6)
|
||
ty = y + random.uniform(-4, 4)
|
||
sx, sy = getattr(self, "_last_xy", (
|
||
tx + random.uniform(-260, 260), ty + random.uniform(-180, 180)))
|
||
steps = random.randint(3, 6)
|
||
for i in range(1, steps + 1):
|
||
t = i / steps
|
||
self.call("Input.dispatchMouseEvent", {
|
||
"type": "mouseMoved",
|
||
"x": int(sx + (tx - sx) * t + random.uniform(-3, 3)),
|
||
"y": int(sy + (ty - sy) * t + random.uniform(-3, 3)),
|
||
"button": "none"})
|
||
time.sleep(random.uniform(0.02, 0.06))
|
||
self.click_at(tx, ty)
|
||
self._last_xy = (tx, ty)
|
||
|
||
def close(self) -> None:
|
||
try:
|
||
self.ws.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
class LiteCDP:
|
||
"""无 Playwright 依赖的浏览器操作封装"""
|
||
|
||
def __init__(self):
|
||
self.debug_port = client_settings.BROWSER_DEBUG_PORT
|
||
|
||
# ------------------------------------------------------------------
|
||
# DevTools HTTP 协议
|
||
# ------------------------------------------------------------------
|
||
def _endpoint(self) -> str:
|
||
return f"http://127.0.0.1:{self.debug_port}"
|
||
|
||
def is_running(self) -> bool:
|
||
try:
|
||
resp = requests.get(f"{self._endpoint()}/json/version", timeout=3)
|
||
return resp.status_code == 200
|
||
except Exception:
|
||
return False
|
||
|
||
def list_tabs(self) -> list[dict]:
|
||
try:
|
||
return requests.get(f"{self._endpoint()}/json", timeout=5).json()
|
||
except Exception:
|
||
return []
|
||
|
||
def open_tab(self, url: str) -> dict:
|
||
"""原生方式新开标签页并导航(浏览器自身加载,无自动化痕迹)"""
|
||
r = requests.put(f"{self._endpoint()}/json/new?{url}", timeout=10)
|
||
if r.status_code == 405: # 部分 Chrome 版本只接受 GET
|
||
r = requests.get(f"{self._endpoint()}/json/new?{url}", timeout=10)
|
||
r.raise_for_status()
|
||
return r.json()
|
||
|
||
def close_tab(self, tab_id: str) -> None:
|
||
try:
|
||
requests.get(f"{self._endpoint()}/json/close/{tab_id}", timeout=5)
|
||
except Exception:
|
||
pass
|
||
|
||
def activate_tab(self, tab_id: str) -> None:
|
||
"""重新激活指定标签页。开/关临时标签会夺走焦点,
|
||
后台标签上的可信鼠标点击不生效,必须恢复前台。"""
|
||
try:
|
||
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
|
||
except Exception:
|
||
pass
|
||
|
||
def tab_url(self, tab_id: str) -> str | None:
|
||
for t in self.list_tabs():
|
||
if t.get("id") == tab_id:
|
||
return t.get("url")
|
||
return None
|
||
|
||
# ------------------------------------------------------------------
|
||
# CDP WebSocket(只发必要的命令,不激活 Page/Runtime/Debugger 域监听)
|
||
# ------------------------------------------------------------------
|
||
def evaluate(self, ws_url: str, expression: str, timeout: float = 20):
|
||
"""一次性表达式求值,返回结果值;失败返回 None"""
|
||
ws = websocket.create_connection(ws_url, timeout=timeout)
|
||
try:
|
||
ws.send(json.dumps({
|
||
"id": 1,
|
||
"method": "Runtime.evaluate",
|
||
"params": {"expression": expression, "returnByValue": True},
|
||
}))
|
||
while True:
|
||
msg = json.loads(ws.recv())
|
||
if msg.get("id") == 1:
|
||
return msg.get("result", {}).get("result", {}).get("value")
|
||
finally:
|
||
try:
|
||
ws.close()
|
||
except Exception:
|
||
pass
|
||
|
||
def get_cookie_names(self) -> set[str]:
|
||
"""读取 zhipin 域 Cookie 名称集合(取任一标签页的 ws 通道即可)"""
|
||
tabs = [t for t in self.list_tabs() if t.get("webSocketDebuggerUrl")]
|
||
if not tabs:
|
||
return set()
|
||
ws = websocket.create_connection(tabs[0]["webSocketDebuggerUrl"], timeout=10)
|
||
try:
|
||
ws.send(json.dumps({
|
||
"id": 1,
|
||
"method": "Network.getCookies",
|
||
"params": {"urls": ["https://www.zhipin.com/"]},
|
||
}))
|
||
while True:
|
||
msg = json.loads(ws.recv())
|
||
if msg.get("id") == 1:
|
||
cookies = msg.get("result", {}).get("cookies", [])
|
||
return {c["name"] for c in cookies}
|
||
finally:
|
||
try:
|
||
ws.close()
|
||
except Exception:
|
||
pass
|
||
|
||
# ------------------------------------------------------------------
|
||
# 组合操作:开页 → 等渲染 → 取 DOM → 关页
|
||
# ------------------------------------------------------------------
|
||
def fetch_rendered_html(self, url: str, wait_s: float = 8,
|
||
max_wait_s: float = 20) -> str | None:
|
||
"""打开目标页等待 SPA 渲染,取回完整 HTML 后关闭标签页"""
|
||
tab = self.open_tab(url)
|
||
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
|
||
try:
|
||
deadline = time.time() + max_wait_s
|
||
waited = 0.0
|
||
html = ""
|
||
while time.time() < deadline:
|
||
time.sleep(2)
|
||
waited += 2
|
||
try:
|
||
html = self.evaluate(
|
||
ws_url, "document.documentElement.outerHTML") or ""
|
||
except Exception as e:
|
||
logger.debug(f"取 DOM 失败({waited}s):{e}")
|
||
continue
|
||
# 渲染完成的经验判断:内容足够长且非空壳
|
||
if len(html) > 50000 and waited >= wait_s:
|
||
return html
|
||
return html or None
|
||
finally:
|
||
self.close_tab(tab_id)
|