""" Boss直聘 低痕迹 CDP 客户端 背景:Boss直聘 风控会探测"标签页被 CDP attach",一旦被 Playwright 接管, 页面会被重置为 about:blank。因此所有浏览器操作走无 attach 痕迹的原生通道: - 导航:DevTools HTTP 协议(/json/new),由浏览器自身加载页面 - 读 Cookie:CDP WebSocket 的 Network.getCookies(不激活任何调试域) - 读 DOM:CDP WebSocket 的 Runtime.evaluate(一次性表达式求值) """ import json import random import time from html.parser import HTMLParser import requests import websocket from loguru import logger from ..config import client_settings class TextExtractor(HTMLParser): """从 HTML 片段中提取可见文本(跳过 script/style)""" SKIP = {"script", "style", "noscript", "svg", "iframe"} def __init__(self): super().__init__() self.parts: list[str] = [] self._skip_depth = 0 def handle_starttag(self, tag, attrs): if tag in self.SKIP: self._skip_depth += 1 elif tag in ("div", "p", "li", "br", "tr", "section", "h1", "h2", "h3", "h4"): self.parts.append("\n") def handle_endtag(self, tag): if tag in self.SKIP and self._skip_depth > 0: self._skip_depth -= 1 def handle_data(self, data): if self._skip_depth == 0 and data.strip(): self.parts.append(data.strip()) def text(self) -> str: lines = [ln.strip() for ln in "".join(self.parts).splitlines()] return "\n".join(ln for ln in lines if ln) def extract_text(html: str) -> str: parser = TextExtractor() parser.feed(html) return parser.text() class CDPSession: """单个标签页上的持久 CDP 会话:命令调用 / 事件泵 / 可信鼠标点击。 只启用必要域(Network 被动监听站点自身 XHR,Input 产生浏览器级可信点击), 不 attach 页面、不注入脚本,保持低痕迹。""" def __init__(self, ws_url: str): self.ws = websocket.create_connection(ws_url, timeout=30) self._id = 0 def call(self, method: str, params: dict | None = None, timeout: float = 10) -> dict: self._id += 1 mid = self._id self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}})) self.ws.settimeout(timeout) while True: msg = json.loads(self.ws.recv()) if msg.get("id") == mid: return msg.get("result") or {} def eval_js(self, expression: str, timeout: float = 10): """一次性表达式求值,返回结果值""" return (self.call("Runtime.evaluate", {"expression": expression, "returnByValue": True}, timeout) .get("result", {}).get("value")) def pump(self, duration: float): """在 duration 秒内持续产出 CDP 事件(用于被动监听网络响应)""" self.ws.settimeout(0.5) end = time.time() + duration while time.time() < end: try: yield json.loads(self.ws.recv()) except Exception: continue def click_at(self, x: float, y: float) -> None: """浏览器级可信鼠标点击(isTrusted=true,与真人鼠标无异)""" self.call("Input.dispatchMouseEvent", { "type": "mousePressed", "x": int(x), "y": int(y), "button": "left", "clickCount": 1}) time.sleep(0.08) self.call("Input.dispatchMouseEvent", { "type": "mouseReleased", "x": int(x), "y": int(y), "button": "left", "clickCount": 1}) def human_click(self, x: float, y: float) -> None: """拟人化可信点击:先从上次位置移动鼠标(带轨迹), 再以随机偏移落下,避免“无 mousemove 历史 + 永远点正中心”的机器特征""" tx = x + random.uniform(-6, 6) ty = y + random.uniform(-4, 4) sx, sy = getattr(self, "_last_xy", ( tx + random.uniform(-260, 260), ty + random.uniform(-180, 180))) steps = random.randint(3, 6) for i in range(1, steps + 1): t = i / steps self.call("Input.dispatchMouseEvent", { "type": "mouseMoved", "x": int(sx + (tx - sx) * t + random.uniform(-3, 3)), "y": int(sy + (ty - sy) * t + random.uniform(-3, 3)), "button": "none"}) time.sleep(random.uniform(0.02, 0.06)) self.click_at(tx, ty) self._last_xy = (tx, ty) logger.debug(f"拟人点击:目标({x:.0f},{y:.0f}) → 落点({tx:.0f},{ty:.0f})") def close(self) -> None: try: self.ws.close() except Exception: pass class LiteCDP: """无 Playwright 依赖的浏览器操作封装""" def __init__(self): self.debug_port = client_settings.BROWSER_DEBUG_PORT # ------------------------------------------------------------------ # DevTools HTTP 协议 # ------------------------------------------------------------------ def _endpoint(self) -> str: return f"http://127.0.0.1:{self.debug_port}" def is_running(self) -> bool: try: resp = requests.get(f"{self._endpoint()}/json/version", timeout=3) return resp.status_code == 200 except Exception: return False def list_tabs(self) -> list[dict]: try: return requests.get(f"{self._endpoint()}/json", timeout=5).json() except Exception: return [] def open_tab(self, url: str) -> dict: """原生方式新开标签页并导航(浏览器自身加载,无自动化痕迹)""" r = requests.put(f"{self._endpoint()}/json/new?{url}", timeout=10) if r.status_code == 405: # 部分 Chrome 版本只接受 GET r = requests.get(f"{self._endpoint()}/json/new?{url}", timeout=10) r.raise_for_status() return r.json() def close_tab(self, tab_id: str) -> None: try: requests.get(f"{self._endpoint()}/json/close/{tab_id}", timeout=5) except Exception: pass def activate_tab(self, tab_id: str) -> None: """重新激活指定标签页。开/关临时标签会夺走焦点, 后台标签上的可信鼠标点击不生效,必须恢复前台。""" try: requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5) except Exception: pass def tab_url(self, tab_id: str) -> str | None: for t in self.list_tabs(): if t.get("id") == tab_id: return t.get("url") return None # ------------------------------------------------------------------ # CDP WebSocket(只发必要的命令,不激活 Page/Runtime/Debugger 域监听) # ------------------------------------------------------------------ def evaluate(self, ws_url: str, expression: str, timeout: float = 20): """一次性表达式求值,返回结果值;失败返回 None""" ws = websocket.create_connection(ws_url, timeout=timeout) try: ws.send(json.dumps({ "id": 1, "method": "Runtime.evaluate", "params": {"expression": expression, "returnByValue": True}, })) while True: msg = json.loads(ws.recv()) if msg.get("id") == 1: return msg.get("result", {}).get("result", {}).get("value") finally: try: ws.close() except Exception: pass def get_cookie_names(self) -> set[str]: """读取 zhipin 域 Cookie 名称集合(取任一标签页的 ws 通道即可)""" tabs = [t for t in self.list_tabs() if t.get("webSocketDebuggerUrl")] if not tabs: return set() ws = websocket.create_connection(tabs[0]["webSocketDebuggerUrl"], timeout=10) try: ws.send(json.dumps({ "id": 1, "method": "Network.getCookies", "params": {"urls": ["https://www.zhipin.com/"]}, })) while True: msg = json.loads(ws.recv()) if msg.get("id") == 1: cookies = msg.get("result", {}).get("cookies", []) return {c["name"] for c in cookies} finally: try: ws.close() except Exception: pass # ------------------------------------------------------------------ # 组合操作:开页 → 等渲染 → 取 DOM → 关页 # ------------------------------------------------------------------ def fetch_rendered_html(self, url: str, wait_s: float = 8, max_wait_s: float = 20) -> str | None: """打开目标页等待 SPA 渲染,取回完整 HTML 后关闭标签页""" tab = self.open_tab(url) tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"] try: deadline = time.time() + max_wait_s waited = 0.0 html = "" while time.time() < deadline: time.sleep(2) waited += 2 try: html = self.evaluate( ws_url, "document.documentElement.outerHTML") or "" except Exception as e: logger.debug(f"取 DOM 失败({waited}s):{e}") continue # 渲染完成的经验判断:内容足够长且非空壳 if len(html) > 50000 and waited >= wait_s: return html return html or None finally: self.close_tab(tab_id)