基本完成

This commit is contained in:
2026-08-25 22:53:57 +08:00
parent 05fa686749
commit 252d5b53ac
39 changed files with 3853 additions and 447 deletions
+6 -5
View File
@@ -139,9 +139,10 @@ docker compose down
## 二、运行客户端(本地 Windows)
### 1. 安装 AdsPower
- 前往 [adspower.net](https://www.adspower.net) 下载安装
- 创建一个浏览器 Profile,记住它的 **用户 ID**
### 1. 浏览器准备(默认无需安装)
默认 `BROWSER_MODE=chrome`:自动探测本机 Chrome(其次 Edge),
以独立 profile 目录启动并持久化 Boss直聘 登录态,免费且反检测最好。
如需使用指纹浏览器,在 `client/.env` 设 `BROWSER_MODE=adspower` 并填写 `ADSPOWER_USER_ID`。
### 2. 安装依赖
```powershell
@@ -154,7 +155,7 @@ playwright install chromium
复制 `.env.example` 为 `.env` 并填写:
```ini
SERVER_URL=http://你的服务器IP:8000 # 云端服务端地址
ADSPOWER_USER_ID=你的ProfileID # AdsPower 用户 ID(必填)
BROWSER_MODE=chrome # chrome=本机真实浏览器(默认)/ adspower=指纹浏览器
AI_API_KEY=你的大模型Key # AI 评分用
AI_BASE_URL=https://api.deepseek.com # OpenAI 兼容接口
AI_MODEL=deepseek-chat
@@ -185,7 +186,7 @@ chcp 65001
|------|---------|
| 客户端连不上服务端 | 检查 `SERVER_URL` 是否正确、云服务器安全组是否放行端口 |
| 注册一直失败 | 确认服务端已启动且 MySQL 可连接 |
| 无法打开浏览器 | 确认 AdsPower 已运行且 `ADSPOWER_USER_ID` 正确 |
| 无法打开浏览器 | chrome 模式:确认本机装有 Chrome/Edge;adspower 模式:确认 AdsPower 已运行且 `ADSPOWER_USER_ID` 正确 |
| 抓不到简历 | 确认已登录 Boss直聘;页面结构变化时需调整 `resume_scraper.py` 选择器 |
| AI 评分不生效 | 检查 `AI_API_KEY` 是否配置,`AI_BASE_URL`/`AI_MODEL` 是否匹配 |
| 投递按钮找不到 | Boss直聘 前端改版,需更新 `applicant.py` 中的选择器 |
+7 -2
View File
@@ -7,9 +7,14 @@ SERVER_URL=http://127.0.0.1:8000
# 本地控制面板端口
WEB_PORT=8088
# AdsPower 本地 API(一般不用改)
# 浏览器模式:chrome=本机真实 Chrome/Edge(默认,免费) | adspower=指纹浏览器
BROWSER_MODE=chrome
# chrome 模式可选配置(一般不用改:自动探测浏览器、默认 profile 目录)
#BROWSER_PATH=
#BROWSER_PROFILE_DIR=
#BROWSER_DEBUG_PORT=9222
# adspower 模式配置(仅 BROWSER_MODE=adspower 时需要)
ADSPOWER_API_URL=http://local.adspower.net:50325
# 你的 AdsPower Profile ID(在 AdsPower 界面查看,必填)
ADSPOWER_USER_ID=
# AI 大模型(OpenAI 兼容接口)
+352 -201
View File
@@ -1,85 +1,187 @@
"""
投递自动化模块
在后台线程运行,支持用户随时开始/结束。
向云端服务端上报投递记录与统计。
投递自动化模块(低痕迹 CDP 版 · 期望条件栏驱动)
真实页面行为(已实测):
- 列表页 /web/geek/jobs 顶部有"期望搜索条件"栏(如 Java(北京)),
点击某条期望即由站点自身发请求加载对应 关键词+城市 的职位列表;
找不到目标城市的期望时,告知"期望城市没有目标城市"。
- 分页为下拉加载:滚动触发站点自身 XHR(list.json),被动监听可得明文薪资
(DOM 中薪资为自定义字体加密,无法直接读取)。
- 投递:右侧详情面板点"立即沟通"→ 站点自动发送默认招呼语并弹出
"已向BOSS发送消息"对话框 → 点"留在此页"即完成一次投递。
- 沟通按钮需浏览器级可信点击(Input.dispatchMouseEvent),合成 el.click() 无效。
"""
import json
import random
import re
import time
import threading
from dataclasses import dataclass
from loguru import logger
from .cdp_lite import LiteCDP, CDPSession
from .manager import BrowserManager
from ..ai import AIClient
from ..server_api import ServerAPI
# 职位列表页(注意是 jobs 复数)
JOBS_PAGE = "https://www.zhipin.com/web/geek/jobs"
# 常见城市编码 → 名称(用于在期望条件栏匹配目标城市)
CITY_NAMES = {
"101010100": "北京", "101020100": "上海", "101280100": "广州",
"101280600": "深圳", "101190100": "杭州", "101030100": "天津",
"101200100": "武汉", "101110100": "西安", "101270100": "成都",
"101040100": "重庆", "101190400": "苏州", "101200200": "南京",
"101210100": "合肥", "101180100": "郑州", "101230100": "福州",
"101240100": "南昌", "101250100": "济南", "101260100": "贵阳",
"101290100": "昆明", "101300100": "兰州", "101050100": "哈尔滨",
"101060100": "长春", "101070100": "沈阳", "101080100": "呼和浩特",
"101090100": "石家庄", "101100100": "太原", "101120100": "青岛",
"101190500": "宁波", "101280300": "佛山", "101280200": "珠海",
"101120200": "大连", "101210200": "厦门", "101180200": "开封",
"101200300": "无锡", "101190200": "温州", "101270200": "绵阳",
}
@dataclass
class JobCard:
job_id: str
title: str
company: str
salary: str
location: str
hr_active: str
can_chat: bool
can_deliver: bool
tags: str = ""
salary: str = ""
jd_text: str = ""
boss_online: bool = False
# 读取列表卡片(新版 DOM 结构)
_CARDS_JS = """(() => [...document.querySelectorAll('.job-card-wrap')].map(el => {
const a = el.querySelector('a.job-name');
const m = /\\/job_detail\\/([^.]+)\\.html/.exec(a ? (a.getAttribute('href') || '') : '');
const t = s => { const n = el.querySelector(s); return n ? n.innerText.trim() : ''; };
return {job_id: m ? m[1] : '', title: t('.job-name'), company: t('.boss-name'),
location: t('.company-location'), tags: t('.tag-list'),
online: !!el.querySelector('.boss-online-icon')};
}))()"""
# JD 正文(详情页,明文)
_JD_JS = """(() => {
const el = document.querySelector('.job-sec-text') || document.querySelector('.job-detail-section');
return el ? el.innerText.trim() : '';
})()"""
def _select_expect_js(keyword: str, city_name: str) -> str:
"""在期望条件栏查找目标期望(优先 关键词(城市) 全匹配,其次城市匹配),返回坐标供可信点击"""
return ("((kw, city) => (() => {"
"const items = [...document.querySelectorAll('.expect-item')];"
"const txt = el => ((el.querySelector('.text-content') || el).innerText || '').trim();"
"let hit = city && items.find(el => txt(el) === kw + '(' + city + ')');"
"if (!hit && city) hit = items.find(el => txt(el).endsWith('(' + city + ')'));"
"if (!hit && kw) hit = items.find(el => txt(el).toLowerCase().includes(kw.toLowerCase()));"
"if (!hit) return JSON.stringify({ok: false, list: items.map(txt)});"
"const r = hit.getBoundingClientRect();"
"return JSON.stringify({ok: true, text: txt(hit),"
" x: r.x + r.width / 2, y: r.y + r.height / 2});"
"})())(" + json.dumps(keyword, ensure_ascii=False) + ","
+ json.dumps(city_name, ensure_ascii=False) + ")")
def _click_card_js(job_id: str) -> str:
"""定位指定职位卡片并滚动到可视区,返回坐标供可信点击(右侧面板随之加载该职位)"""
return ("((id) => (() => {"
"const el = [...document.querySelectorAll('.job-card-wrap')]"
".find(c => ((c.querySelector('a.job-name') || {getAttribute: () => ''})"
".getAttribute('href') || '').includes(id));"
"if (!el) return null;"
"el.scrollIntoView({block: 'center'});"
"const r = el.getBoundingClientRect();"
"return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});"
"})())(" + json.dumps(job_id) + ")")
def _btn_xy_js(selector: str) -> str:
return (f"(() => {{ const b = document.querySelector('{selector}'); if (!b) return null;"
"const r = b.getBoundingClientRect();"
"return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2}); })()")
_STAY_JS = """(() => {
const b = [...document.querySelectorAll('a,button,div,span')]
.find(e => (e.innerText || '').trim() === '留在此页' && e.offsetHeight > 0);
if (!b) return null;
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()"""
def _parse_salary(text: str) -> tuple[int, int] | None:
"""'20-30K·15薪' → (20, 30);'3K以下' → (0, 3);'50K以上' → (50, 999)"""
if not text:
return None
m = re.search(r"(\d+)\s*-\s*(\d+)\s*K", text)
if m:
return int(m.group(1)), int(m.group(2))
m = re.search(r"(\d+)\s*K以下", text)
if m:
return 0, int(m.group(1))
m = re.search(r"(\d+)\s*K以上", text)
if m:
return int(m.group(1)), 999
return None
class Applicant:
"""投递执行器(可启停)"""
BASE_URL = "https://www.zhipin.com"
def __init__(
self,
manager: BrowserManager,
server_api: ServerAPI,
ai_client: AIClient | None,
):
def __init__(self, manager: BrowserManager, server_api: ServerAPI,
ai_client: AIClient | None):
self.manager = manager
self.server = server_api
self.ai = ai_client
self.cdp = LiteCDP()
# 运行控制
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
self.is_running = False
# 运行时状态(供 Web 面板查询)
self.session_id: int | None = None
self.primary_resume_content: str = ""
self.stats = {"applied": 0, "skipped": 0, "current": ""}
# 列表页会话(整个任务期间保持一个标签页)
self._tab: dict | None = None
self._sess: CDPSession | None = None
self._salary_by_id: dict[str, str] = {}
# ------------------------------------------------------------------
# 启停控制
# ------------------------------------------------------------------
def start(self, keywords: list[str], city_code: str,
salary_min: int | None, salary_max: int | None,
greeting: str, min_score: int, delay_range: tuple[int, int]) -> None:
"""在新线程中启动投递任务"""
greeting: str, min_score: int, delay_range: tuple[int, int],
boss_online_only: bool = False, daily_limit: int = 0) -> None:
if self.is_running:
logger.warning("已有投递任务在运行")
return
# 创建服务端会话
session = self.server.start_session(
keywords=",".join(keywords),
city_code=city_code,
salary_min=salary_min,
salary_max=salary_max,
keywords=",".join(keywords), city_code=city_code,
salary_min=salary_min, salary_max=salary_max,
)
self.session_id = session["id"]
# 获取主简历内容
primary = self.server.get_primary_resume()
self.primary_resume_content = primary["content"] if primary else ""
self._stop_event.clear()
self.is_running = True
self.stats = {"applied": 0, "skipped": 0, "current": "初始化..."}
self.boss_online_only = boss_online_only
self.daily_limit = daily_limit
self._daily_base = 0
self._thread = threading.Thread(
target=self._run,
@@ -91,12 +193,20 @@ class Applicant:
logger.success(f"投递任务已启动,会话 ID:{self.session_id}")
def stop(self) -> None:
"""请求停止投递任务"""
logger.info("收到停止投递请求...")
self._stop_event.set()
def _should_stop(self) -> bool:
return self._stop_event.is_set()
def _sleep(self, min_s: float, max_s: float) -> None:
total = random.uniform(min_s, max_s)
elapsed = 0.0
while elapsed < total and not self._should_stop():
time.sleep(0.5)
elapsed += 0.5
def _finish(self) -> None:
"""任务结束后的收尾工作"""
self.is_running = False
if self.session_id:
try:
@@ -104,13 +214,8 @@ class Applicant:
except Exception as e:
logger.warning(f"结束会话失败:{e}")
self.stats["current"] = "已停止"
logger.info(
f"投递任务结束。本次投递 {self.stats['applied']} 个,"
f"跳过 {self.stats['skipped']} 个"
)
def _should_stop(self) -> bool:
return self._stop_event.is_set()
logger.info(f"投递任务结束。本次投递 {self.stats['applied']} 个,"
f"跳过 {self.stats['skipped']} 个")
# ------------------------------------------------------------------
# 主循环
@@ -118,44 +223,141 @@ class Applicant:
def _run(self, keywords, city_code, salary_min, salary_max,
greeting, min_score, delay_range):
try:
self.manager.ensure_browser_started()
# 整个任务只开一个列表标签页,模拟真人持续浏览
self._tab = self.cdp.open_tab(JOBS_PAGE)
time.sleep(8) # 等 SPA 挂载
self._sess = CDPSession(self._tab["webSocketDebuggerUrl"])
self._sess.call("Network.enable")
# 每日上限:任务开始时拉取当天已投数量
if self.daily_limit:
try:
self._daily_base = self.server.today_applied_count()
except Exception as e:
logger.warning(f"获取当天投递数量失败,上限判断跳过:{e}")
self._daily_base = 0
logger.info(f"当天已投递 {self._daily_base} 个,每日上限 {self.daily_limit}")
if self._daily_base >= self.daily_limit:
logger.info("已达当日投递上限,任务直接结束")
return
city_name = CITY_NAMES.get(str(city_code), "")
for keyword in keywords:
if self._should_stop():
break
self.stats["current"] = f"搜索关键词:{keyword}"
self._process_keyword(
keyword, city_code, salary_min, salary_max,
greeting, min_score, delay_range,
)
if not self._should_stop():
self._sleep(5, 9)
self.stats["current"] = f"匹配期望条件:{keyword}"
if not self._select_expect(keyword, city_name):
continue
self._process_list(keyword, salary_min, salary_max,
min_score, delay_range)
self._sleep(3, 6)
except Exception as e:
logger.error(f"投递任务异常:{e}")
finally:
if self._sess:
self._sess.close()
if self._tab:
self.cdp.close_tab(self._tab["id"])
self._finish()
def _process_keyword(self, keyword, city_code, salary_min, salary_max,
greeting, min_score, delay_range, max_per_keyword: int = 15):
page = 1
applied_in_kw = 0
while applied_in_kw < max_per_keyword and not self._should_stop():
cards = self._fetch_job_list(keyword, city_code, salary_min, salary_max, page)
if not cards:
break
for card in cards:
if self._should_stop():
return
if applied_in_kw >= max_per_keyword:
break
applied_in_kw += self._handle_card(card, greeting, min_score, delay_range)
page += 1
# 翻页前随机停顿,模拟真人翻页节奏,避免连续翻页被识别为机器行为
if applied_in_kw < max_per_keyword and not self._should_stop():
self._sleep(2, 5)
def _select_expect(self, keyword: str, city_name: str) -> bool:
"""可信点击目标期望;找不到目标城市时告知并跳过"""
r = self._sess.eval_js(_select_expect_js(keyword, city_name))
try:
info = json.loads(r or "{}")
except Exception:
info = {}
if not info.get("ok"):
logger.warning(f"期望城市没有目标城市:{keyword}"
f"({city_name or '未配置城市名'}),"
f"现有期望:{info.get('list') or '无'}")
return False
self._sess.click_at(info["x"], info["y"])
logger.info(f"已选中期望条件:{info.get('text')}")
time.sleep(2)
# 点击后站点自发请求第一页列表,被动监听捕获明文薪资
self._pump_list_responses(4)
time.sleep(2) # 列表渲染可能滞后,留出缓冲
return True
def _handle_card(self, card: JobCard, greeting: str, min_score: int,
delay_range: tuple[int, int]) -> int:
"""处理单个职位卡片,返回是否投递成功(1/0)"""
# 1. 去重(服务端查询)
# ------------------------------------------------------------------
# 列表读取与下拉分页
# ------------------------------------------------------------------
def _pump_list_responses(self, duration: float) -> None:
"""被动监听站点自身的 list.json 分页响应,收集明文薪资"""
reqs: dict[str, str] = {}
for msg in self._sess.pump(duration):
m = msg.get("method")
if m == "Network.responseReceived":
url = msg["params"]["response"]["url"]
if "list.json" in url:
reqs[msg["params"]["requestId"]] = url
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in reqs:
try:
body = self._sess.call(
"Network.getResponseBody", {"requestId": rid}).get("body", "")
jobs = json.loads(body).get("zpData", {}).get("jobList") or []
for j in jobs:
if j.get("encryptJobId"):
self._salary_by_id[j["encryptJobId"]] = j.get("salaryDesc") or ""
except Exception:
pass
def _read_cards(self) -> list[JobCard]:
raw = self._sess.eval_js(_CARDS_JS)
if isinstance(raw, str):
try:
raw = json.loads(raw or "[]")
except Exception:
raw = []
items = raw or []
cards = []
for it in items:
if not it.get("job_id"):
continue
it["salary"] = self._salary_by_id.get(it["job_id"], "")
cards.append(JobCard(
job_id=it.get("job_id", ""), title=it.get("title", ""),
company=it.get("company", ""), location=it.get("location", ""),
tags=it.get("tags", ""), salary=it.get("salary", ""),
boss_online=bool(it.get("online")),
))
return cards
def _process_list(self, keyword, salary_min, salary_max,
min_score, delay_range, max_per_keyword: int = 15):
processed: set[str] = set()
applied = 0
no_new = 0
while applied < max_per_keyword and not self._should_stop():
cards = self._read_cards()
new_cards = [c for c in cards if c.job_id not in processed]
if not new_cards:
no_new += 1
if no_new >= 2:
logger.info(f"关键词 {keyword} 已无更多职位")
break
else:
no_new = 0
for card in new_cards:
if self._should_stop() or applied >= max_per_keyword:
break
processed.add(card.job_id)
applied += self._handle_card(card, salary_min, salary_max,
min_score, delay_range)
# 下拉加载下一页:滚动到底,站点自发请求,被动监听
self._sess.eval_js("window.scrollTo(0, document.body.scrollHeight)")
self._pump_list_responses(3)
self._sleep(1, 3)
# ------------------------------------------------------------------
# 单职位处理:去重 → 薪资 → JD+AI → 投递
# ------------------------------------------------------------------
def _handle_card(self, card: JobCard, salary_min, salary_max,
min_score, delay_range) -> int:
try:
if self.server.check_applied(card.job_id):
return 0
@@ -164,50 +366,119 @@ class Applicant:
self.stats["current"] = f"评估:{card.company} - {card.title}"
# 2. 抓取 JD 详情(供 AI 评分与归档记录,统一前置)
if not card.jd_text:
card.jd_text = self._fetch_jd_detail(card)
# 薪资范围过滤(薪资来自站点分页响应明文)
if salary_min or salary_max:
rng = _parse_salary(card.salary)
if rng:
lo = salary_min or 0
hi = salary_max or 999
if rng[1] < lo or rng[0] > hi:
self._report_skip(card, f"薪资不符({card.salary})")
return 0
# 只投在线 Boss:卡片 logo 旁无绿点直接跳过
if self.boss_online_only and not card.boss_online:
self._report_skip(card, "Boss不在线")
return 0
# 每日上限:达到后结束整个任务
if self.daily_limit and self._daily_base + self.stats["applied"] >= self.daily_limit:
logger.info(f"已达当日投递上限({self.daily_limit}),任务结束")
self._stop_event.set()
return 0
# JD 详情(临时标签页,读完即关)供 AI 评分
card.jd_text = self._fetch_jd(card)
# 3. AI 匹配度评分
ai_score, ai_reason = -1, ""
if self.ai and self.ai.available() and self.primary_resume_content:
result = self.ai.score_job(card.jd_text, self.primary_resume_content)
ai_score, ai_reason = result.score, result.reason
logger.info(f"AI 评分 {ai_score}(及格线 {min_score}):"
f"{card.company} - {card.title} | {ai_reason}")
if not result.is_worth_applying(min_score):
self._report_skip(card, f"AI 评分低({ai_score})", ai_score, ai_reason)
return 0
else:
logger.debug(f"AI 评分不可用,跳过评分:{card.company} - {card.title}")
# 4. 执行投递
success, apply_type = self._apply(card, greeting)
if success:
# 打招呼语仅 chat 方式实际发送过,才归档
greeting_used = greeting if (apply_type == "chat" and greeting) else None
self._report_apply(card, apply_type, ai_score, ai_reason, greeting_used)
if self._apply(card):
self._report_apply(card, ai_score, ai_reason)
self._sleep(*delay_range)
return 1
return 0
def _fetch_jd(self, card: JobCard) -> str:
tab = None
try:
tab = self.cdp.open_tab(
f"https://www.zhipin.com/job_detail/{card.job_id}.html")
time.sleep(5)
s = CDPSession(tab["webSocketDebuggerUrl"])
try:
jd = s.eval_js(_JD_JS) or ""
finally:
s.close()
return jd.strip() or f"{card.title} {card.company}"
except Exception:
return f"{card.title} {card.company}"
finally:
if tab:
self.cdp.close_tab(tab["id"])
# 临时标签夺走了焦点,恢复列表标签前台,否则可信点击失效
if self._tab:
self.cdp.activate_tab(self._tab["id"])
time.sleep(1)
def _apply(self, card: JobCard) -> bool:
"""选中卡片 → 可信点击立即沟通 → 点留在此页,完成一次投递"""
# 1. 选中卡片(右侧面板加载该职位)
pos = self._sess.eval_js(_click_card_js(card.job_id))
if not pos:
logger.warning(f"未能在列表中选中卡片:{card.title}")
return False
p = json.loads(pos)
self._sess.click_at(p["x"], p["y"])
time.sleep(1.5)
# 2. 可信点击"立即沟通"
xy = self._sess.eval_js(_btn_xy_js(".op-btn-chat"))
if not xy:
logger.warning(f"右侧面板无立即沟通按钮:{card.title}")
return False
p = json.loads(xy)
self._sess.click_at(p["x"], p["y"])
# 3. 等待"已向BOSS发送消息"弹窗,点"留在此页"
for _ in range(10):
time.sleep(1)
stay = self._sess.eval_js(_STAY_JS)
if stay:
q = json.loads(stay)
self._sess.click_at(q["x"], q["y"])
return True
logger.warning(f"点击立即沟通后未等到发送消息弹窗:{card.title}")
return False
# ------------------------------------------------------------------
# 服务端上报
# ------------------------------------------------------------------
def _report_apply(self, card: JobCard, apply_type: str, ai_score: int, ai_reason: str,
greeting: str | None = None):
def _report_apply(self, card: JobCard, ai_score: int, ai_reason: str):
try:
self.server.report_application(self.session_id, {
"job_id": card.job_id,
"job_title": card.title,
"company": card.company,
"salary": card.salary,
"salary": card.salary or None,
"location": card.location,
"apply_type": apply_type,
"apply_type": "chat",
"ai_score": ai_score,
"ai_reason": ai_reason[:255] if ai_reason else None,
"jd_text": card.jd_text or None,
"greeting": greeting,
})
self.server.update_session_stats(self.session_id, applied_delta=1, skipped_delta=0)
self.stats["applied"] += 1
logger.success(f"[{self.stats['applied']}] 投递成功({apply_type}):{card.company} - {card.title}")
logger.success(f"[{self.stats['applied']}] 投递成功:{card.company} - {card.title}")
except Exception as e:
logger.warning(f"上报投递记录失败:{e}")
@@ -217,7 +488,8 @@ class Applicant:
"job_id": card.job_id,
"job_title": card.title,
"company": card.company,
"salary": card.salary,
"salary": card.salary or None,
"location": card.location,
"apply_type": "skipped",
"ai_score": ai_score,
"ai_reason": ai_reason[:255] if ai_reason else None,
@@ -229,124 +501,3 @@ class Applicant:
logger.info(f"跳过({reason}):{card.company} - {card.title}")
except Exception as e:
logger.debug(f"上报跳过记录失败:{e}")
# ------------------------------------------------------------------
# 页面操作
# ------------------------------------------------------------------
def _fetch_job_list(self, keyword, city_code, salary_min, salary_max, page) -> list[JobCard]:
page_obj = self.manager.ensure_connected()
url = f"{self.BASE_URL}/web/geek/job?query={keyword}&city={city_code}&page={page}"
if salary_min or salary_max:
url += f"&salary={(salary_min or 0) * 1000},{(salary_max or 0) * 1000}"
self.stats["current"] = f"加载职位列表 第{page}页"
try:
page_obj.goto(url, wait_until="domcontentloaded", timeout=30000)
page_obj.wait_for_timeout(2500)
page_obj.wait_for_selector(".job-card-wrapper", timeout=10000)
except Exception:
logger.warning(f"第 {page} 页无职位列表")
return []
cards = []
for el in page_obj.query_selector_all(".job-card-wrapper"):
try:
card = self._parse_card(el)
if card:
cards.append(card)
except Exception:
continue
logger.info(f"第 {page} 页解析到 {len(cards)} 个职位")
return cards
def _parse_card(self, el) -> JobCard | None:
def text(sel: str) -> str:
node = el.query_selector(sel)
return node.inner_text().strip() if node else ""
link = el.query_selector("a.job-card-left")
href = link.get_attribute("href") if link else ""
m = re.search(r"/job_detail/(\w+)\.html", href or "")
if not m:
return None
return JobCard(
job_id=m.group(1),
title=text(".job-name"),
company=text(".company-name"),
salary=text(".salary"),
location=text(".job-area"),
hr_active=text(".info-public"),
can_chat=bool(el.query_selector(".start-chat-btn:not([disabled])")),
can_deliver=bool(el.query_selector(".btn-deliver")),
)
def _fetch_jd_detail(self, card: JobCard) -> str:
page_obj = self.manager.page
try:
page_obj.goto(f"{self.BASE_URL}/job_detail/{card.job_id}.html",
wait_until="domcontentloaded", timeout=20000)
page_obj.wait_for_timeout(1500)
el = page_obj.query_selector(".job-detail-section")
jd = el.inner_text().strip() if el else f"{card.title} {card.company}"
page_obj.go_back()
page_obj.wait_for_timeout(1500)
return jd
except Exception:
return f"{card.title} {card.company}"
def _apply(self, card: JobCard, greeting: str) -> tuple[bool, str]:
if card.can_chat and self._do_chat(card, greeting):
return True, "chat"
if card.can_deliver and self._do_deliver(card):
return True, "deliver"
return False, ""
def _do_chat(self, card: JobCard, greeting: str) -> bool:
page_obj = self.manager.page
try:
btn = page_obj.query_selector(f"[data-job-id='{card.job_id}'] .start-chat-btn")
if not btn:
return False
btn.click()
page_obj.wait_for_timeout(2000)
if greeting:
inp = page_obj.query_selector(".chat-input textarea")
if inp:
inp.fill(greeting)
page_obj.wait_for_timeout(500)
send = page_obj.query_selector(".chat-input .btn-send")
if send:
send.click()
page_obj.wait_for_timeout(1500)
close = page_obj.query_selector(".dialog-container .close-btn")
if close:
close.click()
return True
except Exception as e:
logger.debug(f"立即沟通失败:{e}")
return False
def _do_deliver(self, card: JobCard) -> bool:
page_obj = self.manager.page
try:
btn = page_obj.query_selector(f"[data-job-id='{card.job_id}'] .btn-deliver")
if not btn:
return False
btn.click()
page_obj.wait_for_timeout(2000)
confirm = page_obj.query_selector(".dialog-confirm .btn-sure")
if confirm:
confirm.click()
page_obj.wait_for_timeout(1500)
return True
except Exception as e:
logger.debug(f"投递简历失败:{e}")
return False
def _sleep(self, min_s: float, max_s: float) -> None:
# 分段睡眠以便及时响应停止信号
total = random.uniform(min_s, max_s)
elapsed = 0.0
while elapsed < total and not self._should_stop():
time.sleep(0.5)
elapsed += 0.5
+239
View File
@@ -0,0 +1,239 @@
"""
Boss直聘 低痕迹 CDP 客户端
背景:Boss直聘 风控会探测"标签页被 CDP attach",一旦被 Playwright 接管,
页面会被重置为 about:blank。因此所有浏览器操作走无 attach 痕迹的原生通道:
- 导航:DevTools HTTP 协议(/json/new),由浏览器自身加载页面
- 读 Cookie:CDP WebSocket 的 Network.getCookies(不激活任何调试域)
- 读 DOM:CDP WebSocket 的 Runtime.evaluate(一次性表达式求值)
"""
import json
import time
from html.parser import HTMLParser
import requests
import websocket
from loguru import logger
from ..config import client_settings
class TextExtractor(HTMLParser):
"""从 HTML 片段中提取可见文本(跳过 script/style)"""
SKIP = {"script", "style", "noscript", "svg", "iframe"}
def __init__(self):
super().__init__()
self.parts: list[str] = []
self._skip_depth = 0
def handle_starttag(self, tag, attrs):
if tag in self.SKIP:
self._skip_depth += 1
elif tag in ("div", "p", "li", "br", "tr", "section", "h1", "h2", "h3", "h4"):
self.parts.append("\n")
def handle_endtag(self, tag):
if tag in self.SKIP and self._skip_depth > 0:
self._skip_depth -= 1
def handle_data(self, data):
if self._skip_depth == 0 and data.strip():
self.parts.append(data.strip())
def text(self) -> str:
lines = [ln.strip() for ln in "".join(self.parts).splitlines()]
return "\n".join(ln for ln in lines if ln)
def extract_text(html: str) -> str:
parser = TextExtractor()
parser.feed(html)
return parser.text()
class CDPSession:
"""单个标签页上的持久 CDP 会话:命令调用 / 事件泵 / 可信鼠标点击。
只启用必要域(Network 被动监听站点自身 XHR,Input 产生浏览器级可信点击),
不 attach 页面、不注入脚本,保持低痕迹。"""
def __init__(self, ws_url: str):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method: str, params: dict | None = None, timeout: float = 10) -> dict:
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expression: str, timeout: float = 10):
"""一次性表达式求值,返回结果值"""
return (self.call("Runtime.evaluate",
{"expression": expression, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration: float):
"""在 duration 秒内持续产出 CDP 事件(用于被动监听网络响应)"""
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def click_at(self, x: float, y: float) -> None:
"""浏览器级可信鼠标点击(isTrusted=true,与真人鼠标无异)"""
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": int(x), "y": int(y),
"button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": int(x), "y": int(y),
"button": "left", "clickCount": 1})
def close(self) -> None:
try:
self.ws.close()
except Exception:
pass
class LiteCDP:
"""无 Playwright 依赖的浏览器操作封装"""
def __init__(self):
self.debug_port = client_settings.BROWSER_DEBUG_PORT
# ------------------------------------------------------------------
# DevTools HTTP 协议
# ------------------------------------------------------------------
def _endpoint(self) -> str:
return f"http://127.0.0.1:{self.debug_port}"
def is_running(self) -> bool:
try:
resp = requests.get(f"{self._endpoint()}/json/version", timeout=3)
return resp.status_code == 200
except Exception:
return False
def list_tabs(self) -> list[dict]:
try:
return requests.get(f"{self._endpoint()}/json", timeout=5).json()
except Exception:
return []
def open_tab(self, url: str) -> dict:
"""原生方式新开标签页并导航(浏览器自身加载,无自动化痕迹)"""
r = requests.put(f"{self._endpoint()}/json/new?{url}", timeout=10)
if r.status_code == 405: # 部分 Chrome 版本只接受 GET
r = requests.get(f"{self._endpoint()}/json/new?{url}", timeout=10)
r.raise_for_status()
return r.json()
def close_tab(self, tab_id: str) -> None:
try:
requests.get(f"{self._endpoint()}/json/close/{tab_id}", timeout=5)
except Exception:
pass
def activate_tab(self, tab_id: str) -> None:
"""重新激活指定标签页。开/关临时标签会夺走焦点,
后台标签上的可信鼠标点击不生效,必须恢复前台。"""
try:
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
except Exception:
pass
def activate_tab(self, tab_id: str) -> None:
"""重新激活指定标签页(开/关临时标签会夺走焦点,
后台标签上的可信鼠标点击会失效,需恢复前台)"""
try:
requests.get(f"{self._endpoint()}/json/activate/{tab_id}", timeout=5)
except Exception:
pass
def tab_url(self, tab_id: str) -> str | None:
for t in self.list_tabs():
if t.get("id") == tab_id:
return t.get("url")
return None
# ------------------------------------------------------------------
# CDP WebSocket(只发必要的命令,不激活 Page/Runtime/Debugger 域监听)
# ------------------------------------------------------------------
def evaluate(self, ws_url: str, expression: str, timeout: float = 20):
"""一次性表达式求值,返回结果值;失败返回 None"""
ws = websocket.create_connection(ws_url, timeout=timeout)
try:
ws.send(json.dumps({
"id": 1,
"method": "Runtime.evaluate",
"params": {"expression": expression, "returnByValue": True},
}))
while True:
msg = json.loads(ws.recv())
if msg.get("id") == 1:
return msg.get("result", {}).get("result", {}).get("value")
finally:
try:
ws.close()
except Exception:
pass
def get_cookie_names(self) -> set[str]:
"""读取 zhipin 域 Cookie 名称集合(取任一标签页的 ws 通道即可)"""
tabs = [t for t in self.list_tabs() if t.get("webSocketDebuggerUrl")]
if not tabs:
return set()
ws = websocket.create_connection(tabs[0]["webSocketDebuggerUrl"], timeout=10)
try:
ws.send(json.dumps({
"id": 1,
"method": "Network.getCookies",
"params": {"urls": ["https://www.zhipin.com/"]},
}))
while True:
msg = json.loads(ws.recv())
if msg.get("id") == 1:
cookies = msg.get("result", {}).get("cookies", [])
return {c["name"] for c in cookies}
finally:
try:
ws.close()
except Exception:
pass
# ------------------------------------------------------------------
# 组合操作:开页 → 等渲染 → 取 DOM → 关页
# ------------------------------------------------------------------
def fetch_rendered_html(self, url: str, wait_s: float = 8,
max_wait_s: float = 20) -> str | None:
"""打开目标页等待 SPA 渲染,取回完整 HTML 后关闭标签页"""
tab = self.open_tab(url)
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
try:
deadline = time.time() + max_wait_s
waited = 0.0
html = ""
while time.time() < deadline:
time.sleep(2)
waited += 2
try:
html = self.evaluate(
ws_url, "document.documentElement.outerHTML") or ""
except Exception as e:
logger.debug(f"取 DOM 失败({waited}s):{e}")
continue
# 渲染完成的经验判断:内容足够长且非空壳
if len(html) > 50000 and waited >= wait_s:
return html
return html or None
finally:
self.close_tab(tab_id)
+215 -106
View File
@@ -1,33 +1,93 @@
"""
浏览器管理器
统一管理 AdsPower 浏览器启动与 Playwright CDP 连接。
统一收敛浏览器启动与 Playwright CDP 连接。
登录、抓取简历、投递共用同一个浏览器会话(复用登录态)。
支持两种模式(client/.env 的 BROWSER_MODE 切换):
- chrome :本机真实 Chrome/Edge + 独立 user-data-dir(默认,免费、反检测最好)
- adspower :AdsPower 指纹浏览器(需 Profile ID,部分 API 需付费)
"""
import os
import subprocess
import time
from pathlib import Path
import requests
from loguru import logger
from playwright.sync_api import sync_playwright, Playwright, Browser, BrowserContext, Page
from ..config import client_settings
from .cdp_lite import LiteCDP
BOSS_HOME = "https://www.zhipin.com/web/geek/job"
BOSS_RESUME_PAGE = "https://www.zhipin.com/web/geek/resume"
# Windows 下 Chrome / Edge 常见安装路径(按优先级自动探测)
_CANDIDATE_EXES = [
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe"),
r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
]
def find_browser_exe() -> str | None:
"""探测本机浏览器可执行文件:配置优先,其次 Chrome,最后 Edge"""
if client_settings.BROWSER_PATH:
p = Path(client_settings.BROWSER_PATH)
return str(p) if p.exists() else None
for p in _CANDIDATE_EXES:
if Path(p).exists():
return p
return None
class BrowserManager:
"""管理 AdsPower 浏览器 + Playwright 连接"""
"""管理浏览器启动 + Playwright 连接(按 BROWSER_MODE 分派)"""
def __init__(self):
self.mode = (client_settings.BROWSER_MODE or "chrome").strip().lower()
# adspower 模式配置
self.api_url = client_settings.ADSPOWER_API_URL.rstrip("/")
self.user_id = client_settings.ADSPOWER_USER_ID
self._playwright: Playwright | None = None
self._browser: Browser | None = None
self._context: BrowserContext | None = None
self.page: Page | None = None
# chrome 模式配置
self.debug_port = client_settings.BROWSER_DEBUG_PORT
self._proc: subprocess.Popen | None = None
self.cdp_url: str | None = None
# 低痕迹 CDP 通道:Boss直聘 风控会探测 CDP attach,
# Playwright 接管页面会把页面重置为 about:blank,
# 因此登录检测/开用户标签页均走原生 DevTools 协议
self.lite = LiteCDP()
# 取账号名的后台复核只做一次,避免轮询期间反复开标签页
self._account_cached = False
self._account_name: str | None = None
# ------------------------------------------------------------------
# AdsPower 本地 API
# 可用性与运行状态
# ------------------------------------------------------------------
def check_available(self) -> None:
"""启动前置校验:当前模式所需环境是否就绪,不满足直接抛错"""
if self.mode == "adspower":
if not self.user_id:
raise RuntimeError("未配置 AdsPower Profile ID(ADSPOWER_USER_ID)")
if not self.is_adspower_running():
raise RuntimeError("AdsPower 软件未运行,请先打开 AdsPower")
else:
if not find_browser_exe():
raise RuntimeError(
"未检测到 Chrome/Edge,请安装 Chrome 或在 .env 中配置 BROWSER_PATH"
)
def is_browser_running(self) -> bool:
"""浏览器是否已在运行(adspower=软件在线;chrome=调试端口可达)"""
if self.mode == "adspower":
return self.is_adspower_running()
try:
resp = requests.get(f"{self._cdp_endpoint()}/json/version", timeout=3)
return resp.status_code == 200
except Exception:
return False
def is_adspower_running(self) -> bool:
try:
resp = requests.get(f"{self.api_url}/status", timeout=5)
@@ -35,7 +95,19 @@ class BrowserManager:
except Exception:
return False
def start_adspower_browser(self) -> str:
# ------------------------------------------------------------------
# 浏览器启动(按模式分派)
# ------------------------------------------------------------------
def _cdp_endpoint(self) -> str:
return f"http://127.0.0.1:{self.debug_port}"
def start_browser(self) -> str:
"""启动浏览器并返回 CDP 地址"""
if self.mode == "adspower":
return self._start_adspower_browser()
return self._start_local_browser()
def _start_adspower_browser(self) -> str:
"""启动 AdsPower 浏览器,返回 CDP 地址"""
url = f"{self.api_url}/api/v1/browser/start"
resp = requests.get(url, params={"user_id": self.user_id, "open_tabs": 1}, timeout=30)
@@ -47,7 +119,140 @@ class BrowserManager:
logger.success(f"AdsPower 浏览器已启动,CDP: {self.cdp_url}")
return self.cdp_url
def stop_adspower_browser(self) -> None:
def _start_local_browser(self) -> str:
"""以 CDP 调试端口启动本机真实浏览器,独立 profile 目录持久化登录态"""
if self.is_browser_running():
logger.info("检测到浏览器调试端口已存在,直接复用")
return self._cdp_endpoint()
exe = find_browser_exe()
if not exe:
raise RuntimeError("未检测到 Chrome/Edge,请安装 Chrome 或在 .env 中配置 BROWSER_PATH")
profile_dir = client_settings.BROWSER_PROFILE_DIR or str(
Path(__file__).resolve().parents[1] / "data" / "chrome_profile"
)
Path(profile_dir).mkdir(parents=True, exist_ok=True)
self._proc = subprocess.Popen(
[
exe,
f"--remote-debugging-port={self.debug_port}",
f"--user-data-dir={profile_dir}",
"--no-first-run",
"--no-default-browser-check",
# 允许 WebSocket 连接 CDP(新版 Chrome 默认拒绝非白名单 origin)
"--remote-allow-origins=*",
# 首次启动直接带上目标网址:由 Chrome 自己打开首个标签页,
# 该标签页不归程序管,程序永远不会导航/关闭它
BOSS_HOME,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
# 等待调试端口就绪(最多 20 秒)
for _ in range(40):
if self.is_browser_running():
logger.success(f"浏览器已启动:{exe}(profile:{profile_dir})")
return self._cdp_endpoint()
time.sleep(0.5)
raise RuntimeError("浏览器启动超时,请检查调试端口是否被占用")
# ------------------------------------------------------------------
# 浏览器确保就绪(不 attach 任何页面,无风控痕迹)
# ------------------------------------------------------------------
def ensure_browser_started(self) -> str:
"""确保浏览器已启动,返回 CDP 地址(已运行则直接复用)"""
self.check_available()
return self.start_browser()
# ------------------------------------------------------------------
# 登录状态检测(全部走低痕迹通道,不 attach 页面)
# ------------------------------------------------------------------
def open_user_tab(self, url: str) -> None:
"""为用户打开一个目标页(浏览器已在运行、首启页被关掉的场景)。
走原生 DevTools 协议由浏览器自己加载,打开后程序不再触碰。"""
try:
self.lite.open_tab(url)
except Exception as e:
logger.warning(f"打开页面失败:{e}")
def has_login_cookie(self) -> bool:
"""轻量登录判断:只读 Cookie,不创建/导航任何标签页(零刷新)。
注意:只认 wt2(真正的登录凭证);__zp_stoken__ 是游客 token,
未登录时也存在,不能作为登录依据。"""
if not self.lite.is_running():
return False
try:
return "wt2" in self.lite.get_cookie_names()
except Exception as e:
logger.debug(f"读 Cookie 失败:{e}")
return False
def check_boss_login(self) -> bool:
"""
检测 Boss直聘 是否已登录。
只读 Cookie(wt2),不创建任何标签页,轮询期间零刷新;
全程不触碰用户标签页。
"""
logged_in = self.has_login_cookie()
if logged_in and not self._account_cached:
logger.success("检测到已登录 Boss直聘")
return logged_in
def get_boss_account_name(self) -> str | None:
"""获取登录用户名:从首页 _PAGE 变量提取(开一个后台标签页,
一次性表达式求值后立即关闭,结果缓存,只开这一次)"""
if self._account_cached:
return self._account_name
self._account_cached = True
tab = None
try:
tab = self.lite.open_tab("https://www.zhipin.com/")
time.sleep(5)
# 页面内嵌的 _PAGE 变量含登录用户名,比 DOM 选择器稳定
name = self.lite.evaluate(
tab["webSocketDebuggerUrl"],
"(typeof _PAGE !== 'undefined' && _PAGE.name) ? _PAGE.name : "
"((document.querySelector('.user-nav .name') || "
"document.querySelector('.header-login .name')) ? "
"(document.querySelector('.user-nav .name') || "
"document.querySelector('.header-login .name')).innerText.trim() : '')",
)
self._account_name = (name or "").strip() or None
except Exception as e:
logger.debug(f"获取账号名失败:{e}")
self._account_name = None
finally:
if tab:
self.lite.close_tab(tab["id"])
return self._account_name
# ------------------------------------------------------------------
# 关闭
# ------------------------------------------------------------------
def close(self) -> None:
"""兼容保留:低痕迹模式下无连接可释放,什么都不做"""
def close_all(self) -> None:
"""关闭连接并停止浏览器(仅停止本管理器启动的进程/AdsPower 窗口)"""
self.close()
if self.mode == "adspower":
self._stop_adspower_browser()
return
if self._proc:
try:
self._proc.terminate()
self._proc.wait(timeout=5)
except Exception:
try:
self._proc.kill()
except Exception:
pass
self._proc = None
logger.info("本地浏览器已关闭")
def _stop_adspower_browser(self) -> None:
try:
requests.get(
f"{self.api_url}/api/v1/browser/stop",
@@ -56,99 +261,3 @@ class BrowserManager:
logger.info("AdsPower 浏览器已关闭")
except Exception as e:
logger.warning(f"关闭 AdsPower 浏览器失败:{e}")
# ------------------------------------------------------------------
# Playwright 连接
# ------------------------------------------------------------------
def connect(self) -> Page:
"""启动浏览器并通过 CDP 连接,返回可操作的 page"""
if self.page is not None:
return self.page
if not self.user_id:
raise RuntimeError("未配置 AdsPower Profile ID(ADSPOWER_USER_ID)")
if not self.is_adspower_running():
raise RuntimeError("AdsPower 软件未运行,请先打开 AdsPower")
self.start_adspower_browser()
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.connect_over_cdp(self.cdp_url)
contexts = self._browser.contexts
self._context = contexts[0] if contexts else self._browser.new_context()
pages = self._context.pages
self.page = pages[0] if pages else self._context.new_page()
logger.info("Playwright 已连接到 AdsPower 浏览器")
return self.page
def ensure_connected(self) -> Page:
"""确保已连接,返回 page"""
if self.page is None:
return self.connect()
return self.page
# ------------------------------------------------------------------
# 登录状态检测
# ------------------------------------------------------------------
def check_boss_login(self) -> bool:
"""
检测 Boss直聘 是否已登录。
通过访问首页并检查是否存在登录后才有的元素判断。
"""
page = self.ensure_connected()
try:
page.goto(BOSS_HOME, wait_until="domcontentloaded", timeout=30000)
page.wait_for_timeout(2000)
# 已登录:页面上会显示用户头像/用户名;未登录:会显示"登录/注册"
logged_in_indicators = [
".user-nav .user-info",
".nav-figure .user",
"[class*='avatar']",
".header-login .name",
]
for sel in logged_in_indicators:
if page.query_selector(sel):
logger.success("检测到已登录 Boss直聘")
return True
# 检查是否出现登录弹窗/登录按钮
if page.query_selector(".btn-sign-up, .login-register, [class*='login-btn']"):
return False
# 兜底:检查 cookie 中是否有登录凭证
cookies = page.context.cookies()
cookie_names = {c["name"] for c in cookies}
return "__zp_stoken__" in cookie_names or "wt2" in cookie_names
except Exception as e:
logger.warning(f"登录状态检测异常:{e}")
return False
def get_boss_account_name(self) -> str | None:
"""尝试获取登录用户名"""
page = self.ensure_connected()
try:
for sel in [".user-nav .name", ".header-login .name", ".user-info .name"]:
el = page.query_selector(sel)
if el:
return el.inner_text().strip()
except Exception:
pass
return None
# ------------------------------------------------------------------
# 关闭
# ------------------------------------------------------------------
def close(self) -> None:
"""关闭 Playwright 连接(不关闭 AdsPower 浏览器,保留登录态)"""
try:
if self._playwright:
self._playwright.stop()
except Exception:
pass
self._playwright = None
self._browser = None
self._context = None
self.page = None
def close_all(self) -> None:
"""关闭连接并停止 AdsPower 浏览器"""
self.close()
self.stop_adspower_browser()
+76 -100
View File
@@ -1,126 +1,102 @@
"""
AI 简历抓取模块
从已登录的 Boss直聘 账号中抓取所有简历,
并用 AI 整理成结构化文本后返回。
AI 简历抓取模块(低痕迹 CDP 版)
Boss直聘 的简历页是一份"在线简历"的分板块编辑页(不是一份份的简历列表),
板块以 id 区分:userinfo/summary/purpose/history/project/education/
clubExp/professional-skill。
通过原生 CDP 通道(不 attach 页面)读取各板块 innerText,拼装成结构化简历。
"""
import json
from loguru import logger
from .manager import BrowserManager, BOSS_RESUME_PAGE
from .cdp_lite import LiteCDP
from .manager import BOSS_RESUME_PAGE
# 在页面上一次性抽取各板块文本(返回 JSON 字符串)
_SECTION_JS = """(() => {
const ids = ['userinfo', 'summary', 'purpose', 'history', 'project',
'education', 'clubExp', 'professional-skill'];
const out = {};
for (const id of ids) {
const el = document.getElementById(id);
if (el) out[id] = el.innerText;
}
return JSON.stringify(out);
})()"""
# 板块标题(用于拼装简历文本)
SECTION_TITLES = {
"userinfo": "基本信息",
"summary": "个人优势",
"purpose": "期望职位",
"history": "工作经历",
"project": "项目经历",
"education": "教育经历",
"clubExp": "社团/组织经历",
"professional-skill": "专业技能",
}
class ResumeScraper:
"""抓取 Boss直聘 账号内的所有简历"""
"""抓取 Boss直聘 账号内的在线简历"""
def __init__(self, manager: BrowserManager, ai_client=None):
def __init__(self, manager, ai_client=None):
self.manager = manager
self.ai = ai_client
self.cdp = LiteCDP()
def scrape_all(self) -> list[dict]:
"""
抓取所有简历。
抓取在线简历。
返回格式:[{boss_resume_id, title, position, content}, ...]
"""
page = self.manager.ensure_connected()
logger.info("正在打开 Boss直聘 简历页...")
page.goto(BOSS_RESUME_PAGE, wait_until="domcontentloaded", timeout=30000)
page.wait_for_timeout(3000)
# 浏览器可能已被手动关闭,先确保拉起(复用登录态 profile)
self.manager.ensure_browser_started()
tab = self.cdp.open_tab(BOSS_RESUME_PAGE)
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
try:
import time
time.sleep(8) # 等 SPA 渲染完成
raw = self.cdp.evaluate(ws_url, _SECTION_JS)
finally:
self.cdp.close_tab(tab_id)
resumes: list[dict] = []
if not raw:
logger.warning("未能从简历页读取到内容")
return []
# 尝试多种可能的简历卡片选择器(Boss直聘 页面结构可能变化)
card_selectors = [
".resume-item",
".resume-card",
"[class*='resume-list'] li",
".geek-resume-item",
]
card_els = []
for sel in card_selectors:
card_els = page.query_selector_all(sel)
if card_els:
logger.info(f"使用选择器 {sel} 找到 {len(card_els)} 份简历")
break
try:
sections = json.loads(raw)
except Exception as e:
logger.warning(f"简历板块数据解析失败:{e}")
return []
if not card_els:
logger.warning("未通过卡片选择器找到简历,尝试整页抓取")
return self._scrape_whole_page(page)
if not sections:
logger.warning("简历页未找到任何板块,请确认已登录")
return []
for i, card in enumerate(card_els):
try:
resume = self._parse_resume_card(card, i)
if resume:
resumes.append(resume)
except Exception as e:
logger.debug(f"解析第 {i} 份简历失败:{e}")
# 拼装结构化简历文本
parts = []
for key, title in SECTION_TITLES.items():
text = (sections.get(key) or "").strip()
if text:
parts.append(f"【{title}】\n{text}")
content = "\n\n".join(parts)
logger.success(f"共抓取到 {len(resumes)} 份简历")
return resumes
# 从基本信息板块首行提取姓名作为标题
userinfo = (sections.get("userinfo") or "").strip()
name = userinfo.splitlines()[0].strip() if userinfo else ""
title = f"{name}的在线简历" if name else "我的在线简历"
def _parse_resume_card(self, card, index: int) -> dict | None:
"""解析单个简历卡片"""
def text(sel: str) -> str:
el = card.query_selector(sel)
return el.inner_text().strip() if el else ""
title = (
text(".resume-title") or text(".title") or
text(".name") or f"简历 {index + 1}"
)
position = text(".position") or text(".job-name") or text(".expect-position")
# 尝试点开详情获取完整内容
content = self._extract_detail_content(card) or text(".resume-content") or text("p")
# 用 AI 整理内容
if content and self.ai:
# 用 AI 整理内容(可选)
if content and self.ai and self.ai.available():
content = self.ai.clean_resume(content)
return {
"boss_resume_id": None, # Boss直聘 前端通常不暴露,后续可从 URL 提取
logger.success(f"简历抓取完成:{title},{len(content)} 字符")
return [{
"boss_resume_id": None, # Boss直聘 前端不暴露简历 ID
"title": title,
"position": position or None,
"position": None,
"content": content or None,
}
def _extract_detail_content(self, card) -> str | None:
"""尝试点击简历卡片进入详情页抓取完整内容"""
page = self.manager.page
try:
clickable = card.query_selector("a") or card
clickable.click()
page.wait_for_timeout(2000)
detail_selectors = [
".resume-detail",
".geek-resume-detail",
"[class*='resume-content']",
".resume-preview",
]
for sel in detail_selectors:
el = page.query_selector(sel)
if el:
content = el.inner_text().strip()
page.go_back()
page.wait_for_timeout(1500)
return content
page.go_back()
page.wait_for_timeout(1500)
except Exception as e:
logger.debug(f"提取简历详情失败:{e}")
return None
def _scrape_whole_page(self, page) -> list[dict]:
"""兜底方案:抓取整个简历页主体文本作为一份简历"""
body_selectors = [".main-wrap", ".geek-wrap", "main", "#main"]
for sel in body_selectors:
el = page.query_selector(sel)
if el:
content = el.inner_text().strip()
if content:
content = self.ai.clean_resume(content) if self.ai else content
return [{
"boss_resume_id": None,
"title": "我的在线简历",
"position": None,
"content": content,
}]
return []
}]
+12 -2
View File
@@ -17,10 +17,20 @@ class ClientSettings(BaseSettings):
WEB_PORT: int = 8088
# -------------------------------------------------------
# AdsPower 本地 API
# 浏览器模式:chrome = 本机真实 Chrome/Edge(默认,免费)
# adspower = AdsPower 指纹浏览器
# -------------------------------------------------------
BROWSER_MODE: str = "chrome"
# chrome 模式:浏览器可执行文件路径(空=自动探测,优先 Chrome 其次 Edge)
BROWSER_PATH: str = ""
# chrome 模式:独立用户数据目录(空=默认 client/data/chrome_profile,持久化登录态)
BROWSER_PROFILE_DIR: str = ""
# chrome 模式:CDP 调试端口
BROWSER_DEBUG_PORT: int = 9222
# adspower 模式:本地 API 与 Profile ID
ADSPOWER_API_URL: str = "http://local.adspower.net:50325"
# 要使用的 AdsPower Profile ID(在 config.env 或面板中配置)
ADSPOWER_USER_ID: str = ""
# -------------------------------------------------------
+112
View File
@@ -0,0 +1,112 @@
"""端到端验证:期望条件点击 → 卡片读取 → 立即沟通 → 留在此页(真实投递一次)"""
import json
import time
from pathlib import Path
from client.browser.applicant import (
_STAY_JS, _btn_xy_js, _click_card_js, _select_expect_js, _CARDS_JS)
from client.browser.cdp_lite import LiteCDP, CDPSession
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "投递流程验证.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = CDPSession(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
# 1) 可信点击期望 Java(北京)
info = json.loads(s.eval_js(_select_expect_js("java", "北京")) or "{}")
log("1) 期望定位:", info)
if not info.get("ok"):
log("无目标期望,退出")
return
s.click_at(info["x"], info["y"])
time.sleep(2)
# 被动监听第一页薪资
reqs: dict[str, str] = {}
薪资: dict[str, str] = {}
for msg in s.pump(4):
m = msg.get("method")
if m == "Network.responseReceived" and "list.json" in msg["params"]["response"]["url"]:
reqs[msg["params"]["requestId"]] = 1
elif m == "Network.loadingFinished" and msg["params"]["requestId"] in reqs:
body = s.call("Network.getResponseBody",
{"requestId": msg["params"]["requestId"]}).get("body", "")
try:
for j in json.loads(body).get("zpData", {}).get("jobList") or []:
薪资[j.get("encryptJobId")] = j.get("salaryDesc")
except Exception:
pass
# 2) 读卡片(渲染可能滞后,重试一次)
time.sleep(2)
cards = s.eval_js(_CARDS_JS) or []
if not cards:
time.sleep(3)
cards = s.eval_js(_CARDS_JS) or []
log("2) 卡片数:", len(cards), " 首卡:", cards[:2], " 薪资明文条数:", len(薪资))
if not cards:
log("无卡片,退出")
return
job_id = cards[0]["job_id"]
# 2b) 模拟 JD 临时标签(夺焦点)→ 关 → 重新激活列表标签
tmp = lite.open_tab(f"https://www.zhipin.com/job_detail/{job_id}.html")
time.sleep(3)
lite.close_tab(tmp["id"])
lite.activate_tab(tab["id"])
time.sleep(1)
log("2b) 临时标签开关 + 重新激活完成")
# 3) 选中首卡
pos = s.eval_js(_click_card_js(job_id))
log("3) 卡片坐标:", pos)
if pos:
q = json.loads(pos)
s.click_at(q["x"], q["y"])
time.sleep(1.5)
# 4) 可信点击立即沟通
xy = s.eval_js(_btn_xy_js(".op-btn-chat"))
log("4) 立即沟通坐标:", xy)
if xy:
p = json.loads(xy)
s.click_at(p["x"], p["y"])
# 5) 等弹窗点留在此页
ok = False
for i in range(10):
time.sleep(1)
stay = s.eval_js(_STAY_JS)
if stay:
q = json.loads(stay)
s.click_at(q["x"], q["y"])
ok = True
log("5) 已点留在此页,投递完成")
break
if not ok:
log("5) 未等到弹窗")
s.close()
lite.close_tab(tab["id"])
log("完成")
print("验证完成 ->", OUT)
if __name__ == "__main__":
main()
+54
View File
@@ -0,0 +1,54 @@
"""排查:点击期望后卡片容器的真实 DOM 状态"""
import json
import time
from pathlib import Path
from client.browser.applicant import _select_expect_js
from client.browser.cdp_lite import LiteCDP, CDPSession
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "卡片容器排查.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = CDPSession(tab["webSocketDebuggerUrl"])
log("0) 初始卡片数:", s.eval_js("document.querySelectorAll('.job-card-wrap').length"))
log("1) 期望点击:", s.eval_js(_select_expect_js("java", "北京")))
for i in range(6):
time.sleep(1)
log(f"2) 第{i + 1}秒:", s.eval_js("""JSON.stringify({
wrap: document.querySelectorAll('.job-card-wrap').length,
body: document.querySelectorAll('.job-card-body').length,
left: document.querySelectorAll('.job-card-left').length,
列表容器: (() => { const c = document.querySelector('.job-list-content, .job-list, .search-job-result, .recommend-job-list, .card-list'); return c ? String(c.className) : null; })()})"""))
log("3) 列表区域片段:", s.eval_js("""(() => {
const c = document.querySelector('.job-card-wrap') || document.querySelector('.job-list-content') || document.querySelector('.job-list');
let p = c; for (let i = 0; i < 2 && p && p.parentElement; i++) p = p.parentElement;
const el = p || document.body;
return el.outerHTML.replace(/\\s+/g, ' ').slice(0, 1200);
})()"""))
s.close()
lite.close_tab(tab["id"])
log("完成")
print("排查完成 ->", OUT)
if __name__ == "__main__":
main()
+65
View File
@@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
"""
验证低 CDP 痕迹方案:
1. /json/new 创建标签页并导航(浏览器自身导航,无自动化 attach 痕迹)
2. 纯 CDP WebSocket 的 Network.getCookies 读取登录 Cookie(不激活 Page/Runtime)
用法:python -m client.debug_cdp_lite
"""
import json
import time
import requests
import websocket
CDP = "http://127.0.0.1:9222"
RESUME_URL = "https://www.zhipin.com/web/geek/resume"
def main():
# 0. 确保浏览器在运行(未运行则拉起,profile 持久化登录态)
try:
requests.get(f"{CDP}/json/version", timeout=3)
except Exception:
from .browser.manager import BrowserManager
BrowserManager().start_browser()
time.sleep(2)
# 1. 用 DevTools HTTP 协议新开标签页(自动加载目标 URL)
r = requests.put(f"{CDP}/json/new?{RESUME_URL}", timeout=10)
if r.status_code == 405: # 部分 Chrome 版本只接受 GET
r = requests.get(f"{CDP}/json/new?{RESUME_URL}", timeout=10)
tab = r.json()
ws_url = tab["webSocketDebuggerUrl"]
tab_id = tab["id"]
print(f"[1] 新标签页已创建:{tab_id}", flush=True)
# 2. 等 SPA 加载,轮询标签页元信息观察 URL 是否被风控重置为 about:blank
for i in range(10):
time.sleep(3)
tabs = {t["id"]: t for t in requests.get(f"{CDP}/json", timeout=5).json()}
t = tabs.get(tab_id)
if t is None:
print(f"[2 {(i + 1) * 3}s] 标签页已消失", flush=True)
return
print(f"[2 {(i + 1) * 3}s] URL={t['url']} title={t.get('title')!r}", flush=True)
# 3. 纯 CDP WebSocket 读 Cookie(只发 Network.getCookies,不激活任何调试域)
ws = websocket.create_connection(ws_url, timeout=10)
ws.send(json.dumps({"id": 1, "method": "Network.getCookies",
"params": {"urls": ["https://www.zhipin.com/"]}}))
while True:
msg = json.loads(ws.recv())
if msg.get("id") == 1:
names = {c["name"] for c in msg["result"]["cookies"]}
print(f"[3] Cookie 数量:{len(names)},含 wt2:{'wt2' in names},"
f"含 __zp_stoken__:{'__zp_stoken__' in names}", flush=True)
break
ws.close()
# 4. 关掉调试标签页
requests.get(f"{CDP}/json/close/{tab_id}", timeout=5)
print("[4] 调试标签页已关闭", flush=True)
if __name__ == "__main__":
main()
+74
View File
@@ -0,0 +1,74 @@
"""监听用户手动点击"立即沟通"后打开的聊天标签页,只读抓取聊天页结构。
用户操作浏览器,本脚本轮询标签列表,发现 /web/geek/chat 标签后抓取结构并退出。"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
OUT = Path(__file__).resolve().parent / "data" / "debug" / "chat_page_probe.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
lite = LiteCDP()
print("等待聊天页标签(最长 3 分钟),请手动点击立即沟通...")
deadline = time.time() + 180
chat_tab = None
while time.time() < deadline:
for t in lite.list_tabs():
if "/web/geek/chat" in (t.get("url") or "") and t.get("webSocketDebuggerUrl"):
chat_tab = t
break
if chat_tab:
break
time.sleep(2)
if not chat_tab:
log("== 超时未发现聊天标签 ==")
print("timeout")
return
log("== 1) 聊天页 URL ==", chat_tab["url"])
time.sleep(4) # 等 SPA 渲染
ws = websocket.create_connection(chat_tab["webSocketDebuggerUrl"], timeout=30)
ws.send(json.dumps({"id": 1, "method": "Runtime.evaluate", "params": {
"expression": """(() => {
const ta = document.querySelector('textarea');
const wrap = ta ? (ta.closest('.chat-input') || ta.parentElement) : null;
const sendBtn = document.querySelector('.chat-input .btn-send, .btn-send');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: String(e.className).slice(0, 60)}));
return JSON.stringify({
url: location.href,
taClass: ta ? ta.className : null,
taPlaceholder: ta ? ta.placeholder : null,
wrapClass: wrap ? wrap.className : null,
wrapHTML: wrap ? wrap.outerHTML.replace(/\\s+/g, ' ').slice(0, 1500) : null,
sendBtnClass: sendBtn ? sendBtn.className : null,
sends: sends.slice(0, 5)
});
})()""", "returnByValue": True}}))
ws.settimeout(15)
while True:
msg = json.loads(ws.recv())
if msg.get("id") == 1:
log("== 2) 聊天页结构 ==", msg.get("result", {}).get("result", {}).get("value"))
break
ws.close()
log("== DONE ==")
print("captured ->", OUT)
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
"""验证城市弹窗选择是否生效:打开弹窗 → 选上海 → 确认切换 → 切回北京"""
import json
import time
from pathlib import Path
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "城市选择验证.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class 会话:
def __init__(self, ws_url):
import websocket
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def 求值(self, expr, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": "Runtime.evaluate",
"params": {"expression": expr, "returnByValue": True}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return (msg.get("result") or {}).get("result", {}).get("value")
def 关闭(self):
try:
self.ws.close()
except Exception:
pass
def 选城市脚本(名称: str) -> str:
return "((name) => (() => {" \
"const dlg = document.querySelector('.city-select-dialog');" \
"if (!dlg || dlg.offsetHeight === 0) return JSON.stringify({ok: false, reason: '弹窗未打开'});" \
"let li = [...dlg.querySelectorAll('.city-list-hot li, .city-list li')].find(l => (l.innerText || '').trim() === name);" \
"if (!li) { for (const t of [...dlg.querySelectorAll('.city-char-list li')]) { t.click();" \
" li = [...dlg.querySelectorAll('.city-list-hot li, .city-list li')].find(l => (l.innerText || '').trim() === name);" \
" if (li) break; } }" \
"if (!li) return JSON.stringify({ok: false, reason: '未找到城市'});" \
"li.click(); return JSON.stringify({ok: true});" \
"})())(" + json.dumps(名称, ensure_ascii=False) + ")"
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = 会话(tab["webSocketDebuggerUrl"])
log("1) 当前城市:", s.求值("(document.querySelector('.cur-city-label') || {}).innerText || ''"))
s.求值("(document.querySelector('.city-label') || {click: () => {}}).click()")
time.sleep(1.5)
log("2) 弹窗已打开:", s.求值(
"!!document.querySelector('.city-select-dialog') && document.querySelector('.city-select-dialog').offsetHeight > 0"))
log("3) 点击上海:", s.求值(选城市脚本("上海")))
time.sleep(4)
log("3b) 切换后:", s.求值("""JSON.stringify({
城市: (document.querySelector('.cur-city-label') || {}).innerText || '',
地址: location.href,
卡片数: document.querySelectorAll('.job-card-wrap').length})"""))
# 切回北京,恢复用户原状态
s.求值("(document.querySelector('.city-label') || {click: () => {}}).click()")
time.sleep(1.5)
log("4) 点击北京:", s.求值(选城市脚本("北京")))
time.sleep(2)
s.关闭()
lite.close_tab(tab["id"])
log("完成")
print("验证完成 ->", OUT)
if __name__ == "__main__":
main()
+77
View File
@@ -0,0 +1,77 @@
# -*- coding: utf-8 -*-
"""
验证纯 CDP DOM 读取链路(不启用 Page/Runtime/Debugger 域,规避风控探测):
Runtime.evaluate(表达式,不开调试域) + DOM.getDocument/getOuterHTML 读简历页 DOM。
用法:python -m client.debug_dom_dump
"""
import json
import time
from pathlib import Path
import requests
import websocket
CDP = "http://127.0.0.1:9222"
RESUME_URL = "https://www.zhipin.com/web/geek/resume"
OUT_DIR = Path(__file__).resolve().parent / "data" / "debug"
_msg_id = 0
def send(ws, method, params=None):
global _msg_id
_msg_id += 1
ws.send(json.dumps({"id": _msg_id, "method": method, "params": params or {}}))
while True:
msg = json.loads(ws.recv())
if msg.get("id") == _msg_id:
return msg
def main():
OUT_DIR.mkdir(parents=True, exist_ok=True)
# 0. 确保浏览器在运行
try:
requests.get(f"{CDP}/json/version", timeout=3)
except Exception:
from .browser.manager import BrowserManager
BrowserManager().start_browser()
time.sleep(2)
# 1. 原生方式开标签页
r = requests.put(f"{CDP}/json/new?{RESUME_URL}", timeout=10)
if r.status_code == 405:
r = requests.get(f"{CDP}/json/new?{RESUME_URL}", timeout=10)
tab = r.json()
tab_id = tab["id"]
print(f"[1] 标签页 {tab_id}", flush=True)
time.sleep(8) # 等 SPA 渲染
tabs = {t["id"]: t for t in requests.get(f"{CDP}/json", timeout=5).json()}
print(f"[2] 8s 后 URL={tabs[tab_id]['url']}", flush=True)
ws = websocket.create_connection(tab["webSocketDebuggerUrl"], timeout=15)
try:
# 2. Runtime.evaluate 取渲染后 HTML(纯表达式求值,不开调试域)
resp = send(ws, "Runtime.evaluate", {
"expression": "document.documentElement.outerHTML",
"returnByValue": True,
})
html = resp.get("result", {}).get("result", {}).get("value", "")
print(f"[3] Runtime.evaluate 取到 HTML {len(html)} 字符", flush=True)
if html:
(OUT_DIR / "resume_pure_cdp.html").write_text(html, encoding="utf-8")
# 3. 观察 attach 后 20 秒页面是否被风控重置
for i in range(4):
time.sleep(5)
resp = send(ws, "Runtime.evaluate", {"expression": "location.href", "returnByValue": True})
url = resp.get("result", {}).get("result", {}).get("value", "")
print(f"[4 {(i + 1) * 5}s] URL={url}", flush=True)
finally:
ws.close()
requests.get(f"{CDP}/json/close/{tab_id}", timeout=5)
print("[5] 完成,产物 client/data/debug/resume_pure_cdp.html", flush=True)
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
"""验证顶部"期望搜索条件"栏:结构 / 点击 Java(北京) 后列表是否加载 / 滚动分页"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "期望条件验证.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class 会话:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def 调用(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def 求值(self, expr, timeout=10):
return (self.调用("Runtime.evaluate",
{"expression": expr, "returnByValue": True})
.get("result", {}).get("value"))
def 泵(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def 关闭(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = 会话(tab["webSocketDebuggerUrl"])
s.调用("Network.enable")
# 1) 期望条件栏结构
r1 = s.求值("""(() => {
const items = [...document.querySelectorAll('a,div,span,li')]
.filter(e => /^[^()()]{1,20}[((][^()()]{1,10}[))]$/.test((e.innerText || '').trim())
&& e.offsetHeight > 0 && e.children.length === 0)
.map(e => ({tag: e.tagName, cls: String(e.className).slice(0, 60), text: e.innerText.trim()}));
const first = items[0];
let bar = null;
if (first) {
const el = [...document.querySelectorAll('*')].find(d => d.className &&
String(d.className).includes(first.cls.split(' ')[0]) && d.querySelectorAll('*').length > 2);
}
const wrap = items.length ? items[0] && (() => {
let p = document.querySelector('.' + (first.cls.split(' ')[0] || 'x'));
for (let i = 0; i < 3 && p && p.parentElement; i++) p = p.parentElement;
return p ? {cls: String(p.className).slice(0, 80), html: p.outerHTML.replace(/\\s+/g, ' ').slice(0, 1500)} : null;
})() : null;
return JSON.stringify({items: items, wrap: wrap});
})()""")
log("1) 期望条件栏:", r1)
# 2) 点击 Java(北京)
r2 = s.求值("""(() => {
const el = [...document.querySelectorAll('a,div,span,li')]
.find(e => (e.innerText || '').trim().includes('(北京)') && e.offsetHeight > 0 && e.children.length === 0);
if (!el) return JSON.stringify({ok: false});
el.click();
return JSON.stringify({ok: true, text: el.innerText.trim()});
})()""")
log("2) 点击:", r2)
time.sleep(5)
log("2b) 点击后:", s.求值("""JSON.stringify({
地址: location.href,
卡片数: document.querySelectorAll('.job-card-wrap').length})"""))
# 3) 滚动分页 + 薪资明文
请求: dict[str, str] = {}
s.求值("window.scrollTo(0, document.body.scrollHeight)")
薪资 = []
for msg in s.泵(6):
m = msg.get("method")
if m == "Network.responseReceived" and "list.json" in msg["params"]["response"]["url"]:
请求[msg["params"]["requestId"]] = msg["params"]["response"]["url"]
elif m == "Network.loadingFinished" and msg["params"]["requestId"] in 请求:
body = s.调用("Network.getResponseBody",
{"requestId": msg["params"]["requestId"]}).get("body", "")
try:
jobs = json.loads(body).get("zpData", {}).get("jobList") or []
薪资.extend(j.get("salaryDesc") for j in jobs[:5])
except Exception:
pass
log("3) 滚动后卡片数:", s.求值("document.querySelectorAll('.job-card-wrap').length"),
" 分页薪资明文:", 薪资)
s.关闭()
lite.close_tab(tab["id"])
log("完成")
print("验证完成 ->", OUT)
if __name__ == "__main__":
main()
+72
View File
@@ -0,0 +1,72 @@
"""排查二:可信点击期望项后的真实视图状态(期望栏 class / 可见卡片 / 容器)"""
import json
import time
from pathlib import Path
from client.browser.cdp_lite import LiteCDP, CDPSession
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "期望切换排查.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def 状态(s):
return s.eval_js("""JSON.stringify({
地址: location.href,
期望栏: [...document.querySelectorAll('.expect-item')].map(e => ({
t: (e.querySelector('.text-content') || {}).innerText,
cls: String(e.className)})),
推荐tab: String((document.querySelector('.synthesis') || {className: ''}).className),
可见wrap: [...document.querySelectorAll('.job-card-wrap')].filter(e => e.offsetParent).length,
全部wrap: document.querySelectorAll('.job-card-wrap').length,
可见列表容器: (() => {
const cands = ['.rec-job-list', '.search-job-result', '.expect-job-list', '.job-list'];
return cands.map(c => { const e = document.querySelector(c);
return e && e.offsetParent ? c : null; }).filter(Boolean);
})()})""")
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = CDPSession(tab["webSocketDebuggerUrl"])
log("1) 点击前:", 状态(s))
xy = s.eval_js("""(() => {
const el = [...document.querySelectorAll('.expect-item')]
.find(e => ((e.querySelector('.text-content') || {}).innerText || '').endsWith('(北京)'));
if (!el) return null;
const r = el.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
log("2) 期望坐标:", xy)
p = json.loads(xy)
s.click_at(p["x"], p["y"])
time.sleep(5)
log("3) 可信点击后:", 状态(s))
log("4) 可见列表片段:", s.eval_js("""(() => {
const el = [...document.querySelectorAll('.job-card-wrap')].find(e => e.offsetParent)
|| document.querySelector('.rec-job-list') || document.body;
return el.outerHTML.replace(/\\s+/g, ' ').slice(0, 900);
})()"""))
s.close()
lite.close_tab(tab["id"])
log("完成")
print("排查完成 ->", OUT)
if __name__ == "__main__":
main()
+135
View File
@@ -0,0 +1,135 @@
"""探测职位列表页真实结构:搜索框 / 筛选栏 / 下拉加载 / 城市下拉。
输出写入 data/debug/job_page_probe.txt(UTF-8,避免控制台乱码)。"""
import json
import time
from pathlib import Path
from client.browser.cdp_lite import LiteCDP
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe.txt"
OUT.parent.mkdir(parents=True, exist_ok=True)
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
ws = tab["webSocketDebuggerUrl"]
time.sleep(7)
# ---- 1) 页面结构 ----
r1 = lite.evaluate(ws, """(() => {
const q = s => document.querySelector(s);
const inp = q('.ipt-search') || q('.input-wrap input') || q('input[placeholder*="搜索"]');
const cards = document.querySelectorAll('.job-card-wrapper');
const list = q('.job-list-box') || q('.job-list') || (cards[0] ? cards[0].parentElement : null);
const fb = q('.filter-bar') || q('.search-condition') || q('[class*="condition"]') || q('[class*="filter"]');
return JSON.stringify({
url: location.href,
title: document.title,
searchInputClass: inp ? inp.className : null,
searchInputPlaceholder: inp ? inp.placeholder : null,
hasSearchBtn: !!q('.btn-search'),
cardCount: cards.length,
listClass: list ? list.className : null,
filterClass: fb ? fb.className : null,
filterHTML: fb ? fb.outerHTML.replace(/\\s+/g, ' ').slice(0, 5000) : null
});
})()""")
log("== 1) 页面结构 ==")
log(r1)
# ---- 2) 搜索交互:输入 java 点搜索 ----
r2 = lite.evaluate(ws, """(() => {
const inp = document.querySelector('.ipt-search') || document.querySelector('.input-wrap input') || document.querySelector('input[placeholder*="搜索"]');
if (!inp) return JSON.stringify({ok: false, reason: 'no input'});
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
const btn = document.querySelector('.btn-search');
if (btn) btn.click();
return JSON.stringify({ok: true, clicked: !!btn});
})()""")
log("== 2) 搜索操作 ==", r2)
time.sleep(5)
r2b = lite.evaluate(ws, """(() => JSON.stringify({
url: location.href,
cardCount: document.querySelectorAll('.job-card-wrapper').length,
firstTitle: (document.querySelector('.job-card-wrapper .job-name') || {}).innerText || ''
}))()""")
log("== 2b) 搜索后 ==", r2b)
# ---- 3) 下拉加载:滚到底看卡片数是否增加 ----
r3 = lite.evaluate(ws, """(() => {
const before = document.querySelectorAll('.job-card-wrapper').length;
window.scrollTo(0, document.body.scrollHeight);
return JSON.stringify({before: before, scrollY: window.scrollY, bodyH: document.body.scrollHeight});
})()""")
log("== 3) 滚动前 ==", r3)
time.sleep(4)
r3b = lite.evaluate(ws, """(() => {
const after1 = document.querySelectorAll('.job-card-wrapper').length;
window.scrollTo(0, document.body.scrollHeight);
return JSON.stringify({after1: after1});
})()""")
time.sleep(4)
r3c = lite.evaluate(ws, """(() => JSON.stringify({
after2: document.querySelectorAll('.job-card-wrapper').length,
url: location.href
}))()""")
log("== 3b) 滚动两次后 ==", r3b, r3c)
# ---- 4) 城市下拉结构 ----
r4 = lite.evaluate(ws, """(() => {
const fb = document.querySelector('.filter-bar') || document.querySelector('[class*="condition"]') || document.querySelector('[class*="filter"]');
if (!fb) return JSON.stringify({ok: false});
// 找到城市筛选节点(含“城市”或定位图标的第一个 dropdown)
const nodes = [...fb.querySelectorAll('li, .dropdown, [class*="city"]')];
const cityNode = nodes.find(n => /城市|北京|上海|广州|深圳/.test(n.innerText || '') && n.innerText.length < 20);
if (cityNode) cityNode.click();
return JSON.stringify({ok: true, cityNodeClass: cityNode ? cityNode.className : null, cityText: cityNode ? cityNode.innerText : null});
})()""")
log("== 4) 城市节点点击 ==", r4)
time.sleep(1.5)
r4b = lite.evaluate(ws, """(() => {
const panel = document.querySelector('.city-select, .dropdown-menu, [class*="city"][class*="select"], .position-city');
const anyPanel = panel || [...document.querySelectorAll('div')].find(d => d.className && /city/i.test(d.className) && d.offsetHeight > 100);
return JSON.stringify({
panelClass: anyPanel ? anyPanel.className : null,
panelHTML: anyPanel ? anyPanel.outerHTML.replace(/\\s+/g, ' ').slice(0, 4000) : null
});
})()""")
log("== 4b) 城市下拉面板 ==", r4b)
# ---- 5) 薪资下拉结构 ----
r5 = lite.evaluate(ws, """(() => {
const fb = document.querySelector('.filter-bar') || document.querySelector('[class*="condition"]');
if (!fb) return JSON.stringify({ok: false});
const nodes = [...fb.querySelectorAll('li, .dropdown')];
const salaryNode = nodes.find(n => /薪资待遇/.test(n.innerText || ''));
if (salaryNode) salaryNode.click();
return JSON.stringify({ok: true, found: !!salaryNode});
})()""")
log("== 5) 薪资节点点击 ==", r5)
time.sleep(1.5)
r5b = lite.evaluate(ws, """(() => {
const panels = [...document.querySelectorAll('div,ul')].filter(d => d.offsetHeight > 50 && /面议|K以下|K/.test(d.innerText || '') && (d.innerText || '').length < 300);
const p = panels[panels.length - 1];
return JSON.stringify({panelClass: p ? p.className : null, panelText: p ? p.innerText.replace(/\\s+/g, ' ').slice(0, 500) : null});
})()""")
log("== 5b) 薪资下拉面板 ==", r5b)
lite.close_tab(tab["id"])
log("== DONE ==")
print("probe done ->", OUT)
if __name__ == "__main__":
main()
+178
View File
@@ -0,0 +1,178 @@
"""十次探测(修正版):等加载→回车搜索→点卡片捕 XHR→面板结构→可信点击沟通(查新标签+同页跳转)"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe10.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def click_at(self, x, y):
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1})
def close(self):
try:
self.ws.close()
except Exception:
pass
SEARCH_JS = """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
if (!inp) return 'NO_INPUT';
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
return 'OK';
})()"""
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8) # 等 SPA 挂载
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
log("== 1) 搜索触发 ==", s.eval_js(SEARCH_JS))
time.sleep(5)
log("== 1b) 搜索后 URL ==", s.eval_js("location.href"))
# 点第二张卡片,捕获所有 wapi XHR
s.eval_js("""(() => {
const c = document.querySelectorAll('.job-card-wrap')[1];
if (c) c.click();
return !!c;
})()""")
req_urls: dict[str, str] = {}
bodies: dict[str, str] = {}
for msg in s.pump(4):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "/wapi/" in u:
req_urls[msg["params"]["requestId"]] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in req_urls:
bodies[req_urls[rid]] = s.call(
"Network.getResponseBody", {"requestId": rid}).get("body", "")
log("== 2) 点卡片后的 XHR ==", list(bodies.keys()))
for u, body in bodies.items():
if "detail" in u:
try:
zp = json.loads(body).get("zpData", {})
ji = zp.get("jobInfo") or zp
log("== 2b) detail 字段 ==", json.dumps({
"salary": ji.get("salaryDesc") or ji.get("salary"),
"jdHead": (ji.get("jobDesc") or "")[:80],
}, ensure_ascii=False)[:1200])
except Exception as e:
log("== 2b) 解析失败 ==", e)
# 右侧面板:沟通按钮祖先链 + JD
r3 = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
if (!b) return JSON.stringify({ok: false});
const chain = [];
let p = b;
for (let i = 0; i < 7 && p; i++) { chain.push(p.className || p.tagName); p = p.parentElement; }
const root = b.closest('.job-detail, .detail-content, .job-info-wrap, .recommend-result-job') || b.parentElement.parentElement.parentElement;
const jd = root ? [...root.querySelectorAll('div,section')].find(d => (d.innerText || '').trim().startsWith('职位描述')) : null;
return JSON.stringify({ok: true, chain: chain, jdClass: jd ? jd.className : null,
jdHead: jd ? jd.innerText.slice(0, 80) : null});
})()""")
log("== 3) 面板祖先链/JD ==", r3)
# 可信点击沟通:查新标签 + 同页跳转
xy = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
if (!b) return null;
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
before = {t["id"] for t in lite.list_tabs()}
if xy:
p = json.loads(xy)
s.click_at(int(p["x"]), int(p["y"]))
time.sleep(5)
new_tabs = [t for t in lite.list_tabs() if t["id"] not in before and t["id"] != tab["id"]]
same_url = s.eval_js("location.href")
log("== 4) 可信点击后 新标签 ==", [t.get("url") for t in new_tabs], " 同页URL ==", same_url)
target_ws = None
target_tab = None
if new_tabs:
target_ws, target_tab = new_tabs[0]["webSocketDebuggerUrl"], new_tabs[0]["id"]
elif same_url and "chat" in same_url:
target_ws, target_tab = tab["webSocketDebuggerUrl"], None
if target_ws:
c = Session(target_ws)
time.sleep(3)
r5 = c.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sendBtn = document.querySelector('.chat-input .btn-send, .btn-send');
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false,
sendBtnClass: sendBtn ? sendBtn.className : null});
})()""")
log("== 5) 聊天页 ==", r5)
c.close()
if target_tab:
lite.close_tab(target_tab)
s.close()
lite.close_tab(tab["id"])
log("== DONE ==")
print("probe10 done ->", OUT)
if __name__ == "__main__":
main()
+137
View File
@@ -0,0 +1,137 @@
"""十一探测:右侧面板是否 iframe / 可信点击是否生效 / 沟通点击后的弹窗"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe11.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def click_at(self, x, y):
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1})
def close(self):
try:
self.ws.close()
except Exception:
pass
SEARCH_JS = """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
if (!inp) return 'NO_INPUT';
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
return 'OK';
})()"""
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = Session(tab["webSocketDebuggerUrl"])
s.eval_js(SEARCH_JS)
time.sleep(5)
# 1) iframe 检查
r1 = s.eval_js("""(() => [...document.querySelectorAll('iframe')]
.map(f => ({src: (f.src || '').slice(0, 120), cls: f.className, w: f.offsetWidth, h: f.offsetHeight})))()""")
log("== 1) iframes ==", r1)
log("== 1b) targets ==", [(t.get("type"), t.get("url", "")[:100]) for t in lite.list_tabs()])
# 2) 可信点击第三张卡片,验证 Input 域生效
xy = s.eval_js("""(() => {
const c = document.querySelectorAll('.job-card-wrap')[2];
if (!c) return null;
c.scrollIntoView({block: 'center'});
const r = c.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
time.sleep(0.6)
xy = s.eval_js("""(() => {
const c = document.querySelectorAll('.job-card-wrap')[2];
const r = c.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
p = json.loads(xy)
s.click_at(int(p["x"]), int(p["y"]))
time.sleep(2)
r2 = s.eval_js("""(() => {
const cards = [...document.querySelectorAll('.job-card-wrap')];
return JSON.stringify({activeIndex: cards.findIndex(c => c.className.includes('active'))});
})()""")
log("== 2) 可信点击卡片后 activeIndex(应为2) ==", r2)
# 3) 可信点击沟通按钮 + 弹窗检查
xy2 = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
if (!b) return null;
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
log("== 3) 沟通按钮坐标 ==", xy2)
before = {t["id"] for t in lite.list_tabs()}
if xy2:
q = json.loads(xy2)
s.click_at(int(q["x"]), int(q["y"]))
time.sleep(4)
r3 = s.eval_js("""(() => {
const dlg = document.querySelector('.dialog-container, .toast-container, [class*="dialog-wrap"]');
return JSON.stringify({url: location.href,
dialog: dlg && dlg.offsetHeight > 0 ? dlg.innerText.slice(0, 200) : null});
})()""")
new_tabs = [t for t in lite.list_tabs() if t["id"] not in before and t["id"] != tab["id"]]
log("== 3b) 点击后 ==", r3, " 新标签 ==", [t.get("url") for t in new_tabs])
log("== 3c) targets after ==", [(t.get("type"), t.get("url", "")[:100]) for t in lite.list_tabs()])
s.close()
lite.close_tab(tab["id"])
for t in new_tabs:
lite.close_tab(t["id"])
log("== DONE ==")
print("probe11 done ->", OUT)
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
"""十二探测:详情页可信点击 btn-startchat 是否开聊天标签 + 聊天页结构"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe12.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def click_at(self, x, y):
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1})
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
# 先拿一个职位链接
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s0 = Session(tab["webSocketDebuggerUrl"])
s0.eval_js("""(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
return true;
})()""")
time.sleep(5)
href = s0.eval_js("(document.querySelector('.job-card-wrap a.job-name') || {getAttribute: () => ''}).getAttribute('href')")
s0.close()
lite.close_tab(tab["id"])
log("== 1) 职位 ==", href)
if not href:
log("== DONE ==")
return
dtab = lite.open_tab("https://www.zhipin.com" + href)
time.sleep(6)
d = Session(dtab["webSocketDebuggerUrl"])
# 按钮坐标 + elementFromPoint 校验
info = d.eval_js("""(() => {
const b = document.querySelector('.btn-startchat');
if (!b) return null;
const r = b.getBoundingClientRect();
const x = r.x + r.width / 2, y = r.y + r.height / 2;
const top = document.elementFromPoint(x, y);
return JSON.stringify({x, y, topCls: top ? top.className : null,
topIsBtn: top === b || (top && b.contains(top))});
})()""")
log("== 2) 按钮坐标/顶层元素 ==", info)
before = {t["id"] for t in lite.list_tabs()}
p = json.loads(info)
d.click_at(int(p["x"]), int(p["y"]))
time.sleep(5)
new_tabs = [t for t in lite.list_tabs() if t["id"] not in before and t["id"] != dtab["id"]]
log("== 3) 新标签 ==", [t.get("url") for t in new_tabs], " 同页 ==", d.eval_js("location.href"))
if new_tabs:
c = Session(new_tabs[0]["webSocketDebuggerUrl"])
time.sleep(4)
r5 = c.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sendBtn = document.querySelector('.chat-input .btn-send, .btn-send');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false,
sendBtnClass: sendBtn ? sendBtn.className : null, sends: sends.slice(0, 5)});
})()""")
log("== 4) 聊天页 ==", r5)
c.close()
lite.close_tab(new_tabs[0]["id"])
d.close()
lite.close_tab(dtab["id"])
log("== DONE ==")
print("probe12 done ->", OUT)
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
"""十三探测:点击事件是否到达 handler + 点击后全部网络请求/弹窗"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe13.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def click_at(self, x, y):
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1})
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s0 = Session(tab["webSocketDebuggerUrl"])
s0.eval_js("""(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
return true;
})()""")
time.sleep(5)
href = s0.eval_js("(document.querySelector('.job-card-wrap a.job-name') || {getAttribute: () => ''}).getAttribute('href')")
s0.close()
lite.close_tab(tab["id"])
dtab = lite.open_tab("https://www.zhipin.com" + href)
time.sleep(6)
d = Session(dtab["webSocketDebuggerUrl"])
d.call("Network.enable")
# 挂点击计数监听
d.eval_js("""(() => {
window.__clicks = 0;
const b = document.querySelector('.btn-startchat');
if (b) b.addEventListener('click', e => { window.__clicks++; window.__lastTrusted = e.isTrusted; });
return !!b;
})()""")
xy = d.eval_js("""(() => {
const b = document.querySelector('.btn-startchat');
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
p = json.loads(xy)
d.click_at(int(p["x"]), int(p["y"]))
reqs: list[str] = []
for msg in d.pump(5):
if msg.get("method") == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "/wapi/" in u or "chat" in u:
reqs.append(f"{msg['params']['response']['status']} {u[:140]}")
log("== 1) 点击计数/trusted ==", d.eval_js("JSON.stringify({c: window.__clicks, t: window.__lastTrusted})"))
log("== 2) 点击后请求 ==", reqs)
log("== 3) 弹窗/toast ==", d.eval_js("""(() => {
const els = [...document.querySelectorAll('.toast, .dialog-container, [class*="toast"], [class*="dialog"]')]
.filter(e => e.offsetHeight > 0);
return JSON.stringify(els.map(e => ({cls: e.className.slice(0, 60), txt: (e.innerText || '').slice(0, 100)})));
})()""")
log("== 4) 同页URL ==", d.eval_js("location.href"))
d.close()
lite.close_tab(dtab["id"])
log("== DONE ==")
print("probe13 done ->", OUT)
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
"""二次探测:职位卡片真实类名 / 搜索按钮 / 搜索触发方式"""
import time
from pathlib import Path
from client.browser.cdp_lite import LiteCDP
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe2.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
ws = tab["webSocketDebuggerUrl"]
time.sleep(7)
# 1) 搜索区域结构(input 的上层容器)
r1 = lite.evaluate(ws, """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
let box = inp; for (let i = 0; i < 3 && box; i++) box = box.parentElement;
const btns = [...document.querySelectorAll('a,button,div,span')]
.filter(e => (e.innerText || '').trim() === '搜索' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className, ka: e.getAttribute('ka')}));
return JSON.stringify({
searchBoxHTML: box ? box.outerHTML.replace(/\\s+/g, ' ').slice(0, 2500) : null,
searchBtns: btns
});
})()""")
log("== 1) 搜索区域 ==", r1)
# 2) 卡片真实类名:找所有 class 含 job/card 的元素
r2 = lite.evaluate(ws, """(() => {
const set = {};
document.querySelectorAll('[class*="job-card"], [class*="card-"]').forEach(e => {
set[e.className] = (set[e.className] || 0) + 1;
});
const list = document.querySelector('.job-list-box, .job-list, .search-job-result, .job-content');
const main = document.querySelector('main') || document.querySelector('.page-content') || document.querySelector('#wrap');
return JSON.stringify({
cardClasses: Object.keys(set).slice(0, 20),
listClass: list ? list.className : null,
mainSnippet: (main || document.body).outerHTML.replace(/\\s+/g, ' ').slice(0, 3500)
});
})()""")
log("== 2) 卡片类名/主区域 ==", r2)
# 3) 输入 java + 回车触发搜索
r3 = lite.evaluate(ws, """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
if (!inp) return JSON.stringify({ok: false});
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keyup', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
return JSON.stringify({ok: true});
})()""")
log("== 3) 回车搜索 ==", r3)
time.sleep(5)
r3b = lite.evaluate(ws, """(() => {
const cards = document.querySelectorAll('[class*="job-card"]');
return JSON.stringify({
url: location.href,
cardCount: cards.length,
firstCardHTML: cards[0] ? cards[0].outerHTML.replace(/\\s+/g, ' ').slice(0, 2500) : null
});
})()""")
log("== 3b) 回车后 ==", r3b)
# 4) 再滚一次看增量
lite.evaluate(ws, "window.scrollTo(0, document.body.scrollHeight)")
time.sleep(4)
r4 = lite.evaluate(ws, """(() => JSON.stringify({
cardCount: document.querySelectorAll('[class*="job-card"]').length,
url: location.href
}))()""")
log("== 4) 滚动后 ==", r4)
lite.close_tab(tab["id"])
log("== DONE ==")
print("probe2 done ->", OUT)
if __name__ == "__main__":
main()
+106
View File
@@ -0,0 +1,106 @@
"""三次探测:薪资反爬字符 / 卡片数据属性 / 右侧详情面板沟通按钮 / 薪资下拉交互"""
import time
from pathlib import Path
from client.browser.cdp_lite import LiteCDP
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe3.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
ws = tab["webSocketDebuggerUrl"]
time.sleep(7)
# 先搜索 java
lite.evaluate(ws, """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
document.querySelector('.search-btn').click();
return true;
})()""")
time.sleep(5)
# 1) 薪资文本码点 + 卡片 dataset
r1 = lite.evaluate(ws, """(() => {
const sal = document.querySelector('.job-salary');
const card = document.querySelector('.job-card-wrap');
const li = document.querySelector('.job-card-box');
return JSON.stringify({
salaryText: sal ? sal.innerText : null,
salaryCodes: sal ? [...sal.innerText].map(c => c.codePointAt(0).toString(16)) : null,
cardDataset: card ? Object.assign({}, card.dataset) : null,
liDataset: li ? Object.assign({}, li.dataset) : null,
liAttrs: li ? [...li.attributes].map(a => a.name + '=' + a.value).slice(0, 10) : null
});
})()""")
log("== 1) 薪资码点/卡片属性 ==", r1)
# 2) 点击第一张卡片,看右侧详情面板
r2 = lite.evaluate(ws, """(() => {
const card = document.querySelector('.job-card-wrap');
if (!card) return JSON.stringify({ok: false});
card.click();
return JSON.stringify({ok: true});
})()""")
log("== 2) 点击卡片 ==", r2)
time.sleep(3)
r2b = lite.evaluate(ws, """(() => {
const btns = [...document.querySelectorAll('a,button')]
.filter(e => /立即沟通|投递简历|收藏/.test(e.innerText || '') && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className, ka: e.getAttribute('ka'), text: (e.innerText || '').trim()}));
const jd = document.querySelector('.job-detail-section, .job-sec-text, [class*="job-detail"], [class*="desc"]');
return JSON.stringify({
btns: btns,
jdClass: jd ? jd.className : null,
jdTextHead: jd ? jd.innerText.slice(0, 120) : null
});
})()""")
log("== 2b) 详情面板 ==", r2b)
# 3) 薪资下拉:hover+click 打开,点 10-20K
r3 = lite.evaluate(ws, """(() => {
const sels = [...document.querySelectorAll('.condition-filter-select')];
const salary = sels.find(s => /薪资待遇/.test(s.innerText || ''));
if (!salary) return JSON.stringify({ok: false, reason: 'no salary select'});
const cur = salary.querySelector('.current-select');
cur.dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}));
cur.dispatchEvent(new MouseEvent('mouseover', {bubbles: true}));
cur.click();
return JSON.stringify({ok: true});
})()""")
log("== 3) 打开薪资下拉 ==", r3)
time.sleep(1.5)
r3b = lite.evaluate(ws, """(() => {
const li = [...document.querySelectorAll('.condition-filter-select li')]
.find(l => /10-20K/.test(l.innerText || ''));
if (!li) return JSON.stringify({ok: false, reason: 'no option visible'});
li.click();
return JSON.stringify({ok: true});
})()""")
log("== 3b) 点击10-20K ==", r3b)
time.sleep(4)
r3c = lite.evaluate(ws, """(() => JSON.stringify({
url: location.href,
cardCount: document.querySelectorAll('.job-card-wrap').length
}))()""")
log("== 3c) 薪资筛选后 ==", r3c)
lite.close_tab(tab["id"])
log("== DONE ==")
print("probe3 done ->", OUT)
if __name__ == "__main__":
main()
+166
View File
@@ -0,0 +1,166 @@
"""四次探测:
A. Network 域被动监听站点自己的 joblist XHR,验证能否拿到明文薪资(页面无感知)
B. 职位详情页结构:沟通/投递按钮、JD 容器、聊天输入框
"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe4.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
"""极简 CDP 会话:支持命令调用与事件泵"""
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result")
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
# ---------- A. 列表页 XHR 监听 ----------
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
s.call("Runtime.evaluate", {"expression": """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
document.querySelector('.search-btn').click();
return true;
})()"""})
req_urls: dict[str, str] = {}
finished: list[str] = []
for msg in s.pump(8):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "joblist" in u or "search" in u:
req_urls[msg["params"]["requestId"]] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in req_urls:
finished.append(rid)
log("== A1) 捕获的 XHR ==", {req_urls[r] for r in finished} or req_urls.values())
for rid in finished[:2]:
body = s.call("Network.getResponseBody", {"requestId": rid}) or {}
try:
data = json.loads(body.get("body", ""))
zp = data.get("zpData", {})
jobs = zp.get("jobList") or zp.get("dataList") or []
log("== A2) joblist 响应 ==", json.dumps({
"keys": list(zp.keys())[:15],
"jobCount": len(jobs),
"firstJob": jobs[0] if jobs else None,
}, ensure_ascii=False)[:3000])
except Exception as e:
log("== A2) 解析失败 ==", e, body.get("body", "")[:300])
# 列表页稳定性(Network.enable 后是否还被风控重置)
stable = s.call("Runtime.evaluate", {
"expression": "JSON.stringify({url: location.href, cards: document.querySelectorAll('.job-card-wrap').length})"
})
log("== A3) 列表页稳定性 ==", stable)
# 取第一个职位 ID 供 B 使用
first_href = s.call("Runtime.evaluate", {
"expression": "(document.querySelector('.job-card-wrap a.job-name') || {}).getAttribute ? document.querySelector('.job-card-wrap a.job-name').getAttribute('href') : ''"
})
s.close()
lite.close_tab(tab["id"])
log("== A4) 首个职位 href ==", first_href)
# ---------- B. 详情页结构 ----------
if not first_href:
log("== DONE(无职位,跳过 B)==")
return
dtab = lite.open_tab("https://www.zhipin.com" + first_href)
d = Session(dtab["webSocketDebuggerUrl"])
time.sleep(5)
r1 = d.call("Runtime.evaluate", {"expression": """(() => {
const btns = [...document.querySelectorAll('a,button')]
.filter(e => /立即沟通|投递简历|收藏/.test(e.innerText || '') && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className, href: e.getAttribute('href') || ''}));
const cands = {};
['.job-detail-section', '.job-sec-text', '.detail-content', '.job-detail .text', '[class*="job-detail"]']
.forEach(sel => { const el = document.querySelector(sel); cands[sel] = el ? el.innerText.slice(0, 60) : null; });
return JSON.stringify({url: location.href, btns: btns, cands: cands});
})()"""})
log("== B1) 详情页按钮/JD ==", r1)
# 点击立即沟通,看聊天结构
r2 = d.call("Runtime.evaluate", {"expression": """(() => {
const btn = [...document.querySelectorAll('a,button')].find(e => /立即沟通/.test(e.innerText || '') && e.offsetHeight > 0);
if (!btn) return JSON.stringify({ok: false});
btn.click();
return JSON.stringify({ok: true});
})()"""})
log("== B2) 点击立即沟通 ==", r2)
time.sleep(4)
r3 = d.call("Runtime.evaluate", {"expression": """(() => {
const ta = document.querySelector('textarea');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => /发送/.test(e.innerText || '') && (e.innerText || '').length < 6 && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
return JSON.stringify({
url: location.href,
taClass: ta ? ta.className : null,
taSelector: ta ? (ta.closest('.chat-input') ? '.chat-input textarea' : 'textarea') : null,
sends: sends.slice(0, 5)
});
})()"""})
log("== B3) 聊天页结构 ==", r3)
d.close()
lite.close_tab(dtab["id"])
log("== DONE ==")
print("probe4 done ->", OUT)
if __name__ == "__main__":
main()
+161
View File
@@ -0,0 +1,161 @@
"""五次探测:明文薪资来源(SSR 内嵌 JSON / 滚动 joblist XHR)+ 详情页/聊天页结构"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe5.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
s.eval_js("""(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
document.querySelector('.search-btn').click();
return true;
})()""")
time.sleep(5)
# 1) SSR 内嵌 JSON 是否含明文薪资
r1 = s.eval_js("""(() => {
const scripts = [...document.querySelectorAll('script')].map(x => x.textContent || '');
const hit = scripts.find(t => t.includes('"salary"') && t.length > 500);
if (!hit) return JSON.stringify({found: false, lens: scripts.map(t => t.length).filter(l => l > 200)});
const i = hit.indexOf('"salary"');
return JSON.stringify({found: true, snippet: hit.slice(Math.max(0, i - 200), i + 300)});
})()""")
log("== 1) SSR 内嵌薪资 ==", r1)
# 2) 滚动触发 joblist XHR,被动捕获明文
req_urls: dict[str, str] = {}
finished: list[str] = []
s.eval_js("window.scrollTo(0, document.body.scrollHeight)")
for msg in s.pump(7):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "joblist" in u:
req_urls[msg["params"]["requestId"]] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in req_urls:
finished.append(rid)
log("== 2) joblist XHR ==", [req_urls[r] for r in finished])
for rid in finished[:1]:
body = s.call("Network.getResponseBody", {"requestId": rid}).get("body", "")
try:
data = json.loads(body)
zp = data.get("zpData", {})
jobs = zp.get("jobList") or []
log("== 2b) 明文字段 ==", json.dumps({
"keys": list(zp.keys()),
"first": {k: jobs[0].get(k) for k in
("encryptJobId", "jobName", "salaryDesc", "salary", "companyName", "cityName", "jobDegree", "experience")} if jobs else None,
"allKeysOfJob": list(jobs[0].keys()) if jobs else [],
}, ensure_ascii=False)[:2500])
except Exception as e:
log("== 2b) 解析失败 ==", e, body[:300])
# 3) 详情页 + 聊天页
href = s.eval_js("(document.querySelector('.job-card-wrap a.job-name') || {getAttribute: () => ''}).getAttribute('href')")
s.close()
lite.close_tab(tab["id"])
log("== 3) 首职位 ==", href)
if not href:
log("== DONE ==")
return
dtab = lite.open_tab("https://www.zhipin.com" + href)
d = Session(dtab["webSocketDebuggerUrl"])
time.sleep(5)
r4 = d.eval_js("""(() => {
const btns = [...document.querySelectorAll('a,button')]
.filter(e => /立即沟通|投递简历/.test(e.innerText || '') && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
const cands = {};
['.job-detail-section', '.job-sec-text', '.detail-content', '.job-detail .text']
.forEach(sel => { const el = document.querySelector(sel); cands[sel] = el ? el.innerText.slice(0, 80) : null; });
return JSON.stringify({url: location.href, btns, cands});
})()""")
log("== 4) 详情页 ==", r4)
d.eval_js("""(() => {
const btn = [...document.querySelectorAll('a,button')].find(e => /立即沟通/.test(e.innerText || '') && e.offsetHeight > 0);
if (btn) btn.click();
return !!btn;
})()""")
time.sleep(4)
r5 = d.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false, sends: sends.slice(0, 5)});
})()""")
log("== 5) 聊天页 ==", r5)
d.close()
lite.close_tab(dtab["id"])
log("== DONE ==")
print("probe5 done ->", OUT)
if __name__ == "__main__":
main()
+146
View File
@@ -0,0 +1,146 @@
"""六次探测:滚动分页 XHR 真实端点 + 立即沟通新标签页的聊天结构"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe6.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
# ---------- A. 滚动分页端点 ----------
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
s.eval_js("""(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
document.querySelector('.search-btn').click();
return true;
})()""")
time.sleep(5)
seen: list[str] = []
finished: dict[str, str] = {}
for _ in range(3): # 滚三轮
s.eval_js("window.scrollTo(0, document.body.scrollHeight)")
for msg in s.pump(5):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "/wapi/" in u and u not in seen:
seen.append(u)
rid = msg["params"]["requestId"]
finished[rid] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in finished and "page=" in finished[rid]:
body = s.call("Network.getResponseBody", {"requestId": rid}).get("body", "")
try:
zp = json.loads(body).get("zpData", {})
jobs = zp.get("jobList") or []
log("== A2) 分页响应 ==", finished[rid][:150],
"jobs:", len(jobs),
"firstKeys:", list(jobs[0].keys())[:25] if jobs else [],
"salaryField:", (jobs[0].get("salaryDesc") or jobs[0].get("salary")) if jobs else None)
except Exception as e:
log("== A2) 解析失败 ==", e)
log("== A1) 滚动期间全部 wapi 请求 ==", seen)
cards = s.eval_js("document.querySelectorAll('.job-card-wrap').length")
log("== A3) 三轮滚动后卡片数 ==", cards)
# ---------- B. 立即沟通的新标签页 ----------
href = s.eval_js("(document.querySelector('.job-card-wrap a.job-name') || {getAttribute: () => ''}).getAttribute('href')")
s.close()
lite.close_tab(tab["id"])
before = {t["id"] for t in lite.list_tabs()}
dtab = lite.open_tab("https://www.zhipin.com" + href)
d = Session(dtab["webSocketDebuggerUrl"])
time.sleep(5)
d.eval_js("""(() => {
const btn = document.querySelector('.btn-startchat');
if (btn) btn.click();
return !!btn;
})()""")
time.sleep(4)
after = lite.list_tabs()
new_tabs = [t for t in after if t["id"] not in before and t["id"] != dtab["id"]]
log("== B1) 点击后新增标签页 ==", [(t.get("url"), t.get("id")) for t in new_tabs])
if new_tabs:
c = Session(new_tabs[0]["webSocketDebuggerUrl"])
time.sleep(3)
r = c.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false, sends: sends.slice(0, 5)});
})()""")
log("== B2) 聊天页结构 ==", r)
c.close()
lite.close_tab(new_tabs[0]["id"])
d.close()
lite.close_tab(dtab["id"])
log("== DONE ==")
print("probe6 done ->", OUT)
if __name__ == "__main__":
main()
+151
View File
@@ -0,0 +1,151 @@
"""七次探测:点卡片后 detail.json 明文字段 / 右侧面板沟通按钮 href / 聊天页结构"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe7.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
s.eval_js("""(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
document.querySelector('.search-btn').click();
return true;
})()""")
time.sleep(5)
# 点击第一张卡片,捕获 detail.json
s.eval_js("""(() => {
const c = document.querySelector('.job-card-wrap');
if (c) c.click();
return !!c;
})()""")
req_urls: dict[str, str] = {}
detail_bodies: list[str] = []
for msg in s.pump(5):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "detail.json" in u:
req_urls[msg["params"]["requestId"]] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in req_urls:
detail_bodies.append(
s.call("Network.getResponseBody", {"requestId": rid}).get("body", ""))
for body in detail_bodies[:1]:
try:
zp = json.loads(body).get("zpData", {}).get("jobInfo") or json.loads(body).get("zpData", {})
log("== 1) detail.json 字段 ==", json.dumps({
"keys": list(zp.keys())[:40],
"salary": zp.get("salaryDesc") or zp.get("salary"),
"jdHead": (zp.get("jobDesc") or "")[:100],
}, ensure_ascii=False)[:2000])
except Exception as e:
log("== 1) 解析失败 ==", e, body[:300])
# 右侧面板:沟通按钮 href + JD 容器
r2 = s.eval_js("""(() => {
const chat = document.querySelector('.op-btn-chat');
const deliver = [...document.querySelectorAll('a,button')].find(e => /投递简历/.test(e.innerText || ''));
const jd = document.querySelector('.job-sec-text, .job-detail-section');
return JSON.stringify({
chatHref: chat ? chat.getAttribute('href') : null,
chatClass: chat ? chat.className : null,
deliverHref: deliver ? deliver.getAttribute('href') : null,
jdClass: jd ? jd.className : null,
jdHead: jd ? jd.innerText.slice(0, 60) : null
});
})()""")
log("== 2) 右侧面板 ==", r2)
s.close()
lite.close_tab(tab["id"])
# 聊天页:用 href 原生打开
chat_href = (json.loads(r2) if r2 else {}).get("chatHref")
if chat_href:
before = {t["id"] for t in lite.list_tabs()}
ctab = lite.open_tab(chat_href if chat_href.startswith("http") else "https://www.zhipin.com" + chat_href)
c = Session(ctab["webSocketDebuggerUrl"])
time.sleep(5)
r3 = c.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
const sendBtn = document.querySelector('.chat-input .btn-send, .btn-send');
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false,
sends: sends.slice(0, 5), sendBtnClass: sendBtn ? sendBtn.className : null});
})()""")
log("== 3) 聊天页 ==", r3)
c.close()
lite.close_tab(ctab["id"])
else:
log("== 3) 无 chatHref,跳过 ==")
log("== DONE ==")
print("probe7 done ->", OUT)
if __name__ == "__main__":
main()
+172
View File
@@ -0,0 +1,172 @@
"""八次探测:detail.json 明文字段 / 可信点击立即沟通 / 聊天页结构 / 右侧面板 JD 容器"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe8.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def click_at(self, x, y):
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1})
def close(self):
try:
self.ws.close()
except Exception:
pass
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
s.eval_js("""(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
document.querySelector('.search-btn').click();
return true;
})()""")
time.sleep(5)
# 点第二张卡片(避免首张已选中不触发请求)
s.eval_js("""(() => {
const c = document.querySelectorAll('.job-card-wrap')[1];
if (c) c.click();
return !!c;
})()""")
req_urls: dict[str, str] = {}
bodies: list[str] = []
for msg in s.pump(5):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "detail.json" in u:
req_urls[msg["params"]["requestId"]] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in req_urls:
bodies.append(s.call("Network.getResponseBody", {"requestId": rid}).get("body", ""))
for body in bodies[:1]:
try:
zp = json.loads(body).get("zpData", {})
ji = zp.get("jobInfo") or zp
log("== 1) detail.json ==", json.dumps({
"zpKeys": list(zp.keys())[:20],
"jiKeys": list(ji.keys())[:40],
"salary": ji.get("salaryDesc") or ji.get("salary"),
"jdHead": (ji.get("jobDesc") or "")[:80],
}, ensure_ascii=False)[:2000])
except Exception as e:
log("== 1) 解析失败 ==", e, body[:300])
# 右侧面板 JD 容器
r2 = s.eval_js("""(() => {
const hits = [...document.querySelectorAll('div,section')]
.filter(d => (d.innerText || '').trim().startsWith('职位描述') && d.innerText.length < 5000)
.map(d => ({cls: d.className, len: d.innerText.length}))
.sort((a, b) => a.len - b.len);
return JSON.stringify(hits.slice(0, 4));
})()""")
log("== 2) 右侧JD容器 ==", r2)
# 可信点击 .op-btn-chat
box = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
if (!b) return null;
b.scrollIntoView({block: 'center'});
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
log("== 3) 沟通按钮坐标 ==", box)
before = {t["id"] for t in lite.list_tabs()}
if box:
xy = json.loads(box)
time.sleep(0.5)
box2 = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
xy = json.loads(box2)
s.click_at(int(xy["x"]), int(xy["y"]))
time.sleep(5)
after = lite.list_tabs()
new_tabs = [t for t in after if t["id"] not in before and t["id"] != tab["id"]]
log("== 4) 可信点击后新标签 ==", [(t.get("url")) for t in new_tabs])
if new_tabs:
c = Session(new_tabs[0]["webSocketDebuggerUrl"])
time.sleep(3)
r5 = c.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sendBtn = document.querySelector('.chat-input .btn-send, .btn-send');
const sends = [...document.querySelectorAll('a,button,div')]
.filter(e => (e.innerText || '').trim() === '发送' && e.offsetHeight > 0)
.map(e => ({tag: e.tagName, cls: e.className}));
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false,
sendBtnClass: sendBtn ? sendBtn.className : null, sends: sends.slice(0, 5)});
})()""")
log("== 5) 聊天页结构 ==", r5)
c.close()
lite.close_tab(new_tabs[0]["id"])
s.close()
lite.close_tab(tab["id"])
log("== DONE ==")
print("probe8 done ->", OUT)
if __name__ == "__main__":
main()
+165
View File
@@ -0,0 +1,165 @@
"""九次探测:回车搜索 / 点卡片捕获 detail.json / 右侧面板全量结构 / 可信点击沟通"""
import json
import time
from pathlib import Path
import websocket
from client.browser.cdp_lite import LiteCDP
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "job_page_probe9.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
class Session:
def __init__(self, ws_url):
self.ws = websocket.create_connection(ws_url, timeout=30)
self._id = 0
def call(self, method, params=None, timeout=10):
self._id += 1
mid = self._id
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
self.ws.settimeout(timeout)
while True:
msg = json.loads(self.ws.recv())
if msg.get("id") == mid:
return msg.get("result") or {}
def eval_js(self, expr, timeout=10):
return (self.call("Runtime.evaluate",
{"expression": expr, "returnByValue": True}, timeout)
.get("result", {}).get("value"))
def pump(self, duration):
self.ws.settimeout(0.5)
end = time.time() + duration
while time.time() < end:
try:
yield json.loads(self.ws.recv())
except Exception:
continue
def click_at(self, x, y):
self.call("Input.dispatchMouseEvent", {
"type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1})
time.sleep(0.08)
self.call("Input.dispatchMouseEvent", {
"type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1})
def close(self):
try:
self.ws.close()
except Exception:
pass
SEARCH_JS = """(() => {
const inp = document.querySelector('input[placeholder*="搜索"]');
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value').set;
setter.call(inp, 'java');
inp.dispatchEvent(new Event('input', {bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
inp.dispatchEvent(new KeyboardEvent('keyup', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
return true;
})()"""
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
s = Session(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
s.eval_js(SEARCH_JS)
time.sleep(5)
log("== 1) 搜索后 URL ==", s.eval_js("location.href"))
# 点第二张卡片 + 捕获 detail.json
s.eval_js("""(() => {
const c = document.querySelectorAll('.job-card-wrap')[1];
if (c) c.click();
return !!c;
})()""")
req_urls: dict[str, str] = {}
bodies: list[str] = []
for msg in s.pump(4):
m = msg.get("method")
if m == "Network.responseReceived":
u = msg["params"]["response"]["url"]
if "detail.json" in u:
req_urls[msg["params"]["requestId"]] = u
elif m == "Network.loadingFinished":
rid = msg["params"]["requestId"]
if rid in req_urls:
bodies.append(s.call("Network.getResponseBody", {"requestId": rid}).get("body", ""))
log("== 2) detail.json 数量 ==", len(bodies))
for body in bodies[:1]:
try:
zp = json.loads(body).get("zpData", {})
ji = zp.get("jobInfo") or zp
log("== 2b) 字段 ==", json.dumps({
"salary": ji.get("salaryDesc") or ji.get("salary"),
"jdHead": (ji.get("jobDesc") or "")[:80],
"jiKeys": list(ji.keys())[:35],
}, ensure_ascii=False)[:1800])
except Exception as e:
log("== 2b) 解析失败 ==", e, body[:300])
# 右侧面板全量结构:op-btn-chat 的祖先面板
r3 = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
if (!b) return JSON.stringify({ok: false});
let p = b;
for (let i = 0; i < 5 && p.parentElement; i++) p = p.parentElement;
return JSON.stringify({ok: true, panelClass: p.className,
html: p.outerHTML.replace(/\\s+/g, ' ').slice(0, 3000)});
})()""")
log("== 3) 右侧面板 ==", r3)
# 可信点击沟通按钮
xy = s.eval_js("""(() => {
const b = document.querySelector('.op-btn-chat');
if (!b) return null;
const r = b.getBoundingClientRect();
return JSON.stringify({x: r.x + r.width / 2, y: r.y + r.height / 2});
})()""")
before = {t["id"] for t in lite.list_tabs()}
if xy:
p = json.loads(xy)
s.click_at(int(p["x"]), int(p["y"]))
time.sleep(5)
new_tabs = [t for t in lite.list_tabs() if t["id"] not in before and t["id"] != tab["id"]]
log("== 4) 新标签 ==", [t.get("url") for t in new_tabs])
if new_tabs:
c = Session(new_tabs[0]["webSocketDebuggerUrl"])
time.sleep(3)
r5 = c.eval_js("""(() => {
const ta = document.querySelector('textarea');
const sendBtn = document.querySelector('.chat-input .btn-send, .btn-send');
return JSON.stringify({url: location.href, taClass: ta ? ta.className : null,
inChatInput: ta ? !!ta.closest('.chat-input') : false,
sendBtnClass: sendBtn ? sendBtn.className : null});
})()""")
log("== 5) 聊天页 ==", r5)
c.close()
lite.close_tab(new_tabs[0]["id"])
s.close()
lite.close_tab(tab["id"])
log("== DONE ==")
print("probe9 done ->", OUT)
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
"""探测卡片上 boss 在线绿点的 DOM 选择器"""
import json
import time
from pathlib import Path
from client.browser.applicant import _select_expect_js
from client.browser.cdp_lite import LiteCDP, CDPSession
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "在线标记排查.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = CDPSession(tab["webSocketDebuggerUrl"])
info = json.loads(s.eval_js(_select_expect_js("java", "北京")) or "{}")
if info.get("ok"):
s.click_at(info["x"], info["y"])
time.sleep(6)
log("1) 前 4 张卡片 footer 结构:", s.eval_js("""JSON.stringify(
[...document.querySelectorAll('.job-card-wrap')].slice(0, 4).map(el => {
const f = el.querySelector('.job-card-footer') || el;
return f.outerHTML.replace(/\\s+/g, ' ').slice(0, 700);
}))"""))
log("2) 疑似在线标记元素:", s.eval_js("""JSON.stringify(
[...document.querySelectorAll('.job-card-wrap')].slice(0, 6).map(el => {
const dots = [...el.querySelectorAll('*')].filter(d => {
const cls = String(d.className || '');
return /online|active|green|status/i.test(cls);
}).map(d => String(d.className));
const boss = el.querySelector('.boss-info');
return {dots: dots, boss_html: boss ? boss.outerHTML.replace(/\\s+/g, ' ').slice(0, 400) : null};
}))"""))
s.close()
lite.close_tab(tab["id"])
log("完成")
print("排查完成 ->", OUT)
if __name__ == "__main__":
main()
+58
View File
@@ -0,0 +1,58 @@
"""排查三:隔离变量——可信点击后不泵事件,对比数量与 _CARDS_JS 结果"""
import json
import time
from pathlib import Path
from client.browser.applicant import _select_expect_js, _CARDS_JS
from client.browser.cdp_lite import LiteCDP, CDPSession
from client.browser.manager import BrowserManager
OUT = Path(__file__).resolve().parent / "data" / "debug" / "读卡片排查.txt"
_lines: list[str] = []
def log(*a):
s = " ".join(str(x) for x in a)
_lines.append(s)
OUT.write_text("\n".join(_lines), encoding="utf-8")
def main():
BrowserManager().ensure_browser_started()
lite = LiteCDP()
tab = lite.open_tab("https://www.zhipin.com/web/geek/jobs")
time.sleep(8)
s = CDPSession(tab["webSocketDebuggerUrl"])
s.call("Network.enable")
info = json.loads(s.eval_js(_select_expect_js("java", "北京")) or "{}")
log("1) 期望定位:", info)
s.click_at(info["x"], info["y"])
time.sleep(7)
log("2) 数量:", s.eval_js("document.querySelectorAll('.job-card-wrap').length"))
raw = s.eval_js(_CARDS_JS)
log("3) _CARDS_JS 原始返回类型:", type(raw), " 长度:", len(raw) if raw else 0)
log("4) 前 300 字符:", (raw or "")[:300])
# 直接用 CDP 原始返回看异常详情
s._id += 1
s.ws.send(json.dumps({"id": s._id, "method": "Runtime.evaluate",
"params": {"expression": _CARDS_JS}}))
s.ws.settimeout(10)
while True:
msg = json.loads(s.ws.recv())
if msg.get("id") == s._id:
log("5) 原始结果:", json.dumps(msg.get("result"), ensure_ascii=False)[:500])
break
s.close()
lite.close_tab(tab["id"])
log("完成")
print("排查完成 ->", OUT)
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
# -*- coding: utf-8 -*-
"""
验证:Playwright attach + 反调试注入 后能否稳定抓取简历页 HTML。
用法:python -m client.debug_resume_dom
"""
import time
from pathlib import Path
from playwright.sync_api import sync_playwright
from .browser.manager import BrowserManager
OUT_DIR = Path(__file__).resolve().parent / "data" / "debug"
# 反调试注入:Boss 风控用 debugger 陷阱/堆栈特征检测 CDP,逐一屏蔽
STEALTH_JS = """
(() => {
// 1. 屏蔽 debugger 陷阱(构造器形式与直接语句)
const _Function = Function;
const handler = {
construct(target, args) {
if (args.length && /debugger/.test(String(args[args.length - 1]))) {
args[args.length - 1] = 'return null;';
}
return new (Function.prototype.bind.apply(target, [null, ...args]))();
},
};
try { window.Function = new Proxy(_Function, handler); } catch (e) {}
// 2. 抹掉 webdriver 特征
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
// 3. 拦截跳往 about:blank 的风控重置
const origAssign = window.location.assign, origReplace = window.location.replace;
try {
window.location.assign = function (u) { if (/about:blank/.test(u)) return; return origAssign.call(window.location, u); };
window.location.replace = function (u) { if (/about:blank/.test(u)) return; return origReplace.call(window.location, u); };
} catch (e) {}
})();
"""
def main():
OUT_DIR.mkdir(parents=True, exist_ok=True)
manager = BrowserManager()
with sync_playwright() as pw:
browser = pw.chromium.connect_over_cdp(manager.cdp_url or "http://127.0.0.1:9222")
context = browser.contexts[0]
context.add_init_script(STEALTH_JS)
page = context.new_page()
try:
print("[1] 带反调试注入打开简历页", flush=True)
page.goto("https://www.zhipin.com/web/geek/resume",
wait_until="domcontentloaded", timeout=30000)
for wait_s in (5, 10, 15):
time.sleep(5)
print(f"[2 {wait_s}s] URL={page.url}", flush=True)
if page.url == "about:blank":
print(" !! 仍被风控重置为 about:blank", flush=True)
continue
try:
html = page.content()
(OUT_DIR / f"resume_dom_{wait_s}s.html").write_text(html, encoding="utf-8")
page.screenshot(path=str(OUT_DIR / f"resume_dom_{wait_s}s.png"), full_page=True)
print(f" HTML {len(html)} 字符已保存", flush=True)
except Exception as e:
print(f" content 失败:{e}", flush=True)
finally:
try:
page.close()
except Exception:
pass
print("[3] 完成", flush=True)
if __name__ == "__main__":
main()
+60
View File
@@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
"""
调试脚本:打开 Boss直聘 简历页,保存真实页面截图 + HTML,
用于校准简历抓取选择器。跑完即退出,不影响客户端。
用法:python -m client.debug_resume_page
"""
import time
from pathlib import Path
from playwright.sync_api import sync_playwright
from .browser.manager import BrowserManager
OUT_DIR = Path(__file__).resolve().parent / "data" / "debug"
def main():
OUT_DIR.mkdir(parents=True, exist_ok=True)
manager = BrowserManager()
with sync_playwright() as pw:
cdp = manager.start_browser()
browser = pw.chromium.connect_over_cdp(cdp)
context = browser.contexts[0]
page = context.new_page()
try:
url = "https://www.zhipin.com/web/geek/resume"
print(f"[1] goto {url}")
page.goto(url, wait_until="domcontentloaded", timeout=30000)
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
pass
for wait_s in (3, 5, 8):
time.sleep(3)
print(f"[3] {wait_s}s:当前 URL = {page.url}")
for attempt in range(5):
try:
html = page.content()
break
except Exception as e:
print(f" content 失败({attempt + 1}):{e}")
time.sleep(2)
else:
continue
shot = OUT_DIR / f"resume_{wait_s}s.png"
page.screenshot(path=str(shot), full_page=True)
(OUT_DIR / f"resume_{wait_s}s.html").write_text(html, encoding="utf-8")
print(f" 截图 {shot.name},HTML {len(html)} 字符")
finally:
try:
page.close()
except Exception:
pass
# 注意:CDP 连接下 browser.close() 会杀掉 Chrome 进程,只断开连接即可
browser = None
print("[4] 完成,产物在 client/data/debug/")
if __name__ == "__main__":
main()
+58
View File
@@ -0,0 +1,58 @@
# -*- coding: utf-8 -*-
"""
调试脚本:观察浏览器稳定性。
阶段1:只开首页,不做任何导航,观察 60 秒浏览器是否存活、页面是否被重置;
阶段2:如果阶段1存活,再访问简历页,记录 URL 变化轨迹。
用法:python -m client.debug_watch_browser
"""
import time
from playwright.sync_api import sync_playwright
from .browser.manager import BrowserManager
def main():
manager = BrowserManager()
cdp = manager.start_browser()
print(f"[0] CDP: {cdp}", flush=True)
pw = sync_playwright().start()
browser = pw.chromium.connect_over_cdp(cdp)
context = browser.contexts[0]
page = context.new_page()
try:
print("[1] 打开 zhipin 首页(不导航其他页面)", flush=True)
page.goto("https://www.zhipin.com/", wait_until="domcontentloaded", timeout=30000)
for i in range(12): # 观察 60 秒
time.sleep(5)
try:
print(f"[阶段1 {(i + 1) * 5}s] URL={page.url} title={page.title()!r}", flush=True)
except Exception as e:
print(f"[阶段1 {(i + 1) * 5}s] 页面异常:{e}", flush=True)
break
print("[2] 现在访问简历页", flush=True)
try:
page.goto("https://www.zhipin.com/web/geek/resume",
wait_until="domcontentloaded", timeout=30000)
except Exception as e:
print(f"[2] goto 失败:{e}", flush=True)
for i in range(6): # 观察 30 秒
time.sleep(5)
try:
print(f"[阶段2 {(i + 1) * 5}s] URL={page.url} title={page.title()!r}", flush=True)
except Exception as e:
print(f"[阶段2 {(i + 1) * 5}s] 页面异常:{e}", flush=True)
break
finally:
try:
page.close()
except Exception:
pass
# 注意:CDP 连接下 browser.close() 会杀掉 Chrome 进程,只断开连接即可
pw.stop()
print("[3] 结束。如果浏览器窗口还活着,说明进程未崩;请观察窗口状态。", flush=True)
if __name__ == "__main__":
main()
+11 -2
View File
@@ -48,8 +48,17 @@ def main() -> None:
logger.info(f"机器指纹:{machine['client_code']}")
logger.info(f"云端服务端地址:{client_settings.SERVER_URL}")
if not client_settings.ADSPOWER_USER_ID:
logger.warning("尚未配置 AdsPower Profile ID(ADSPOWER_USER_ID),请在 client/.env 中填写")
# 浏览器模式就绪检查(不启动浏览器,仅校验环境)
from .browser.manager import BrowserManager, find_browser_exe
browser = BrowserManager()
if browser.mode == "chrome":
exe = find_browser_exe()
if exe:
logger.info(f"浏览器模式:chrome({exe})")
else:
logger.warning("浏览器模式:chrome,但未检测到 Chrome/Edge,请安装或在 .env 配置 BROWSER_PATH")
elif not client_settings.ADSPOWER_USER_ID:
logger.warning("浏览器模式:adspower,尚未配置 Profile ID(ADSPOWER_USER_ID)")
if not client_settings.AI_API_KEY:
logger.warning("尚未配置 AI_API_KEY,AI 评分功能将不可用")
+6
View File
@@ -218,6 +218,12 @@ class ServerAPI:
resp.raise_for_status()
return resp.json()
def today_applied_count(self) -> int:
"""当天(北京时间)实际投递数量"""
resp = requests.get(f"{self.base_url}/api/jobs/today-count", headers=self._headers(), timeout=15)
resp.raise_for_status()
return int(resp.json().get("count", 0))
def get_history(self, limit: int = 50, apply_type: str | None = None) -> list[dict]:
params = {"limit": limit}
if apply_type:
+45 -21
View File
@@ -112,6 +112,8 @@ class StartApplyReq(BaseModel):
min_score: int = 65
delay_min: int = 3
delay_max: int = 8
boss_online_only: bool = False
daily_limit: int = 0
# =======================================================
@@ -147,7 +149,7 @@ def get_status():
"server_connected": state.connected,
"server_reachable": state.server_api.ping(),
"rpa_online": me.get("is_online", False) if me else False,
"adspower_running": state.browser.is_adspower_running(),
"browser_running": state.browser.is_browser_running(),
"ai_configured": state.ai.available(),
"login_status": me.get("login_status", "unknown") if me else "unknown",
"boss_account": me.get("boss_account") if me else None,
@@ -221,14 +223,23 @@ def logout():
# =======================================================
@app.post("/api/start-login")
def start_login():
"""打开 AdsPower 浏览器并跳转到 Boss直聘,等待用户手动登录"""
"""拉起浏览器引导登录 Boss直聘,等待用户手动登录。
首次启动时浏览器自带 Boss直聘 首启标签页(由 Chrome 自己创建,程序不碰);
浏览器已在运行但首启页被关掉了,才补开一个用户标签页。"""
_require_auth()
try:
state.server_api.update_login_status("logging_in")
page = state.browser.connect()
page.goto("https://www.zhipin.com/", wait_until="domcontentloaded", timeout=30000)
# 启动后台轮询检测登录
threading.Thread(target=_poll_login, daemon=True).start()
already_running = state.browser.is_browser_running()
if not already_running:
# 首次启动:拉起浏览器(启动参数自带目标网址,由 Chrome 自己打开)
state.browser.ensure_browser_started()
else:
# 浏览器已在运行:无法确定用户是否还留着登录页,补开一个
# (原生 DevTools 协议开页,无自动化痕迹,风控不拦截)
state.browser.open_user_tab("https://www.zhipin.com/")
# 启动后台轮询检测登录(防止重复点击叠加多个轮询线程)
if not state.login_polling:
threading.Thread(target=_poll_login, daemon=True).start()
return {"ok": True, "msg": "浏览器已打开,请在浏览器中完成登录"}
except Exception as e:
state.server_api.update_login_status("not_login")
@@ -236,26 +247,33 @@ def start_login():
def _poll_login():
"""后台轮询检测登录状态,最长等待 5 分钟"""
"""后台轮询检测登录状态,最长等待 5 分钟。
检测仅读 Cookie(低痕迹通道,零刷新零开页),不打扰用户标签页。"""
state.login_polling = True
import time
for _ in range(150): # 150 * 2s = 5 分钟
time.sleep(2)
try:
if state.browser.check_boss_login():
account = state.browser.get_boss_account_name()
state.server_api.update_login_status("logged_in", account)
logger.success(f"登录成功:{account}")
break
except Exception as e:
logger.debug(f"登录轮询异常:{e}")
state.login_polling = False
try:
for _ in range(150): # 150 * 2s = 5 分钟
time.sleep(2)
try:
if state.browser.check_boss_login():
account = state.browser.get_boss_account_name()
state.server_api.update_login_status("logged_in", account)
logger.success(f"登录成功:{account}")
break
except Exception as e:
logger.debug(f"登录轮询异常:{e}")
finally:
state.login_polling = False
@app.post("/api/detect-login")
def detect_login():
"""活体检测 Boss直聘 登录状态(进入登录步骤时自动调用)"""
"""活体检测 Boss直聘 登录状态(进入登录步骤时自动调用;仅读 Cookie + 一次性后台复核)"""
_require_auth()
# 浏览器未运行时直接判未登录,检测行为不应负责拉起浏览器
if not state.browser.is_browser_running():
state.server_api.update_login_status("not_login")
return {"logged_in": False}
try:
logged_in = state.browser.check_boss_login()
except Exception as e:
@@ -292,6 +310,7 @@ def scrape_resumes():
"""AI 抓取账号内所有简历并上传到云端"""
_require_auth()
try:
# 抓取走低痕迹 CDP 通道(无 Playwright attach,风控不拦截)
scraper = ResumeScraper(state.browser, state.ai)
resumes = scraper.scrape_all()
if not resumes:
@@ -325,8 +344,11 @@ def start_apply(req: StartApplyReq):
_require_rpa_online() # 在线预检:与服务端 require_online_user 前置校验呼应
if not state.server_api.get_primary_resume():
raise HTTPException(status_code=400, detail="请先设置主简历")
if not state.browser.is_adspower_running():
raise HTTPException(status_code=400, detail="AdsPower 未运行")
try:
# 浏览器可能被用户手动关闭,投递前确保已拉起(复用登录态 profile)
state.browser.ensure_browser_started()
except RuntimeError as e:
raise HTTPException(status_code=400, detail=str(e))
applicant = state.ensure_applicant()
if applicant.is_running:
@@ -340,6 +362,8 @@ def start_apply(req: StartApplyReq):
greeting=req.greeting,
min_score=req.min_score,
delay_range=(req.delay_min, req.delay_max),
boss_online_only=req.boss_online_only,
daily_limit=req.daily_limit,
)
return {"ok": True, "session_id": applicant.session_id}
+6 -4
View File
@@ -58,9 +58,9 @@ async function refreshStatus() {
s.server_reachable ? '可连接' : '无法连接',
s.server_reachable ? 'ok' : 'err'
);
$('adspowerStatus').innerHTML = badge(
s.adspower_running ? '运行中' : '未运行',
s.adspower_running ? 'ok' : 'warn'
$('browserStatus').innerHTML = badge(
s.browser_running ? '运行中' : '未运行',
s.browser_running ? 'ok' : 'warn'
);
// 顶部徽章与用户区
let badges = badge(s.server_connected ? '已连接' : '未连接', s.server_connected ? 'ok' : '');
@@ -194,7 +194,7 @@ $('btnStartLogin').addEventListener('click', async () => {
btn.disabled = true;
try {
await api('/api/start-login', { method: 'POST' });
toast('浏览器已打开,请在 AdsPower 浏览器中完成登录');
toast('浏览器已打开,请在浏览器中完成登录');
$('loginSpinner').classList.remove('hidden');
startLoginPoll();
} catch (e) {
@@ -293,6 +293,8 @@ $('btnStartApply').addEventListener('click', async () => {
salary_max: parseInt($('salaryMax').value) || null,
greeting: $('greeting').value,
min_score: parseInt($('minScore').value) || 65,
boss_online_only: $('bossOnlineOnly').checked,
daily_limit: parseInt($('dailyLimit').value) || 0,
}),
});
toast('投递任务已启动');
+14 -3
View File
@@ -56,12 +56,12 @@
<!-- Step 2: 登录 Boss直聘 -->
<section class="panel hidden" id="panel-2">
<h2>登录 Boss直聘</h2>
<p class="hint">进入本步骤会自动检测指纹浏览器中的登录状态:若已登录可直接跳过;否则点击下方按钮打开 Boss直聘 手动登录(扫码或账号密码)。</p>
<p class="hint">进入本步骤会自动检测浏览器中的登录状态:若已登录可直接跳过;否则点击下方按钮打开 Boss直聘 手动登录(扫码或账号密码)。</p>
<div class="info-box">
<div><strong>AdsPower 状态:</strong><span id="adspowerStatus" class="badge">检测中</span></div>
<div><strong>浏览器状态:</strong><span id="browserStatus" class="badge">检测中</span></div>
<div><strong>登录状态:</strong><span id="loginStatus" class="badge">未登录</span></div>
</div>
<div class="spinner hidden" id="detectSpinner">正在检测指纹浏览器中的登录状态...</div>
<div class="spinner hidden" id="detectSpinner">正在检测浏览器中的登录状态...</div>
<div class="action-buttons">
<button class="btn btn-primary" id="btnStartLogin">打开浏览器登录</button>
<button class="btn hidden" id="btnSkipLogin">已登录,直接下一步</button>
@@ -114,6 +114,17 @@
<input type="number" id="salaryMax" value="40" min="0">
</div>
</div>
<div class="form-row">
<div class="form-group">
<label>每日投递上限(0=不限)</label>
<input type="number" id="dailyLimit" value="30" min="0">
</div>
<div class="form-group" style="display:flex;align-items:flex-end;padding-bottom:6px;">
<label style="display:flex;align-items:center;gap:6px;margin:0;cursor:pointer;">
<input type="checkbox" id="bossOnlineOnly" style="width:auto;"> 只投递 Boss 在线的
</label>
</div>
</div>
<div class="form-group">
<label>打招呼语(留空使用默认)</label>
<textarea id="greeting" rows="2" placeholder="您好,我对贵司岗位很感兴趣..."></textarea>
+17 -1
View File
@@ -1,7 +1,7 @@
"""
投递记录路由:上报投递结果、查询历史、统计
"""
from datetime import datetime
from datetime import datetime, timedelta
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
@@ -82,6 +82,22 @@ def check_applied(
return {"job_id": job_id, "applied": record is not None}
@router.get("/today-count", response_model=dict)
def today_count(
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
"""当前用户当天(北京时间)实际投递数量,用于客户端每日上限判断"""
now_cn = datetime.utcnow() + timedelta(hours=8)
start_utc = now_cn.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(hours=8)
count = db.query(JobApplication).filter(
JobApplication.user_id == current_user.id,
JobApplication.apply_type.in_(["chat", "deliver"]),
JobApplication.applied_at >= start_utc,
).count()
return {"count": count}
@router.get("/history", response_model=list[ApplicationOut])
def list_applications(
apply_type: Optional[str] = Query(None, description="过滤投递类型"),