基本完成

This commit is contained in:
2026-08-25 22:53:57 +08:00
parent 05fa686749
commit 252d5b53ac
39 changed files with 3853 additions and 447 deletions
+76 -100
View File
@@ -1,126 +1,102 @@
"""
AI 简历抓取模块
从已登录的 Boss直聘 账号中抓取所有简历,
并用 AI 整理成结构化文本后返回。
AI 简历抓取模块(低痕迹 CDP 版)
Boss直聘 的简历页是一份"在线简历"的分板块编辑页(不是一份份的简历列表),
板块以 id 区分:userinfo/summary/purpose/history/project/education/
clubExp/professional-skill。
通过原生 CDP 通道(不 attach 页面)读取各板块 innerText,拼装成结构化简历。
"""
import json
from loguru import logger
from .manager import BrowserManager, BOSS_RESUME_PAGE
from .cdp_lite import LiteCDP
from .manager import BOSS_RESUME_PAGE
# 在页面上一次性抽取各板块文本(返回 JSON 字符串)
_SECTION_JS = """(() => {
const ids = ['userinfo', 'summary', 'purpose', 'history', 'project',
'education', 'clubExp', 'professional-skill'];
const out = {};
for (const id of ids) {
const el = document.getElementById(id);
if (el) out[id] = el.innerText;
}
return JSON.stringify(out);
})()"""
# 板块标题(用于拼装简历文本)
SECTION_TITLES = {
"userinfo": "基本信息",
"summary": "个人优势",
"purpose": "期望职位",
"history": "工作经历",
"project": "项目经历",
"education": "教育经历",
"clubExp": "社团/组织经历",
"professional-skill": "专业技能",
}
class ResumeScraper:
"""抓取 Boss直聘 账号内的所有简历"""
"""抓取 Boss直聘 账号内的在线简历"""
def __init__(self, manager: BrowserManager, ai_client=None):
def __init__(self, manager, ai_client=None):
self.manager = manager
self.ai = ai_client
self.cdp = LiteCDP()
def scrape_all(self) -> list[dict]:
"""
抓取所有简历。
抓取在线简历。
返回格式:[{boss_resume_id, title, position, content}, ...]
"""
page = self.manager.ensure_connected()
logger.info("正在打开 Boss直聘 简历页...")
page.goto(BOSS_RESUME_PAGE, wait_until="domcontentloaded", timeout=30000)
page.wait_for_timeout(3000)
# 浏览器可能已被手动关闭,先确保拉起(复用登录态 profile)
self.manager.ensure_browser_started()
tab = self.cdp.open_tab(BOSS_RESUME_PAGE)
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
try:
import time
time.sleep(8) # 等 SPA 渲染完成
raw = self.cdp.evaluate(ws_url, _SECTION_JS)
finally:
self.cdp.close_tab(tab_id)
resumes: list[dict] = []
if not raw:
logger.warning("未能从简历页读取到内容")
return []
# 尝试多种可能的简历卡片选择器(Boss直聘 页面结构可能变化)
card_selectors = [
".resume-item",
".resume-card",
"[class*='resume-list'] li",
".geek-resume-item",
]
card_els = []
for sel in card_selectors:
card_els = page.query_selector_all(sel)
if card_els:
logger.info(f"使用选择器 {sel} 找到 {len(card_els)} 份简历")
break
try:
sections = json.loads(raw)
except Exception as e:
logger.warning(f"简历板块数据解析失败:{e}")
return []
if not card_els:
logger.warning("未通过卡片选择器找到简历,尝试整页抓取")
return self._scrape_whole_page(page)
if not sections:
logger.warning("简历页未找到任何板块,请确认已登录")
return []
for i, card in enumerate(card_els):
try:
resume = self._parse_resume_card(card, i)
if resume:
resumes.append(resume)
except Exception as e:
logger.debug(f"解析第 {i} 份简历失败:{e}")
# 拼装结构化简历文本
parts = []
for key, title in SECTION_TITLES.items():
text = (sections.get(key) or "").strip()
if text:
parts.append(f"【{title}】\n{text}")
content = "\n\n".join(parts)
logger.success(f"共抓取到 {len(resumes)} 份简历")
return resumes
# 从基本信息板块首行提取姓名作为标题
userinfo = (sections.get("userinfo") or "").strip()
name = userinfo.splitlines()[0].strip() if userinfo else ""
title = f"{name}的在线简历" if name else "我的在线简历"
def _parse_resume_card(self, card, index: int) -> dict | None:
"""解析单个简历卡片"""
def text(sel: str) -> str:
el = card.query_selector(sel)
return el.inner_text().strip() if el else ""
title = (
text(".resume-title") or text(".title") or
text(".name") or f"简历 {index + 1}"
)
position = text(".position") or text(".job-name") or text(".expect-position")
# 尝试点开详情获取完整内容
content = self._extract_detail_content(card) or text(".resume-content") or text("p")
# 用 AI 整理内容
if content and self.ai:
# 用 AI 整理内容(可选)
if content and self.ai and self.ai.available():
content = self.ai.clean_resume(content)
return {
"boss_resume_id": None, # Boss直聘 前端通常不暴露,后续可从 URL 提取
logger.success(f"简历抓取完成:{title},{len(content)} 字符")
return [{
"boss_resume_id": None, # Boss直聘 前端不暴露简历 ID
"title": title,
"position": position or None,
"position": None,
"content": content or None,
}
def _extract_detail_content(self, card) -> str | None:
"""尝试点击简历卡片进入详情页抓取完整内容"""
page = self.manager.page
try:
clickable = card.query_selector("a") or card
clickable.click()
page.wait_for_timeout(2000)
detail_selectors = [
".resume-detail",
".geek-resume-detail",
"[class*='resume-content']",
".resume-preview",
]
for sel in detail_selectors:
el = page.query_selector(sel)
if el:
content = el.inner_text().strip()
page.go_back()
page.wait_for_timeout(1500)
return content
page.go_back()
page.wait_for_timeout(1500)
except Exception as e:
logger.debug(f"提取简历详情失败:{e}")
return None
def _scrape_whole_page(self, page) -> list[dict]:
"""兜底方案:抓取整个简历页主体文本作为一份简历"""
body_selectors = [".main-wrap", ".geek-wrap", "main", "#main"]
for sel in body_selectors:
el = page.query_selector(sel)
if el:
content = el.inner_text().strip()
if content:
content = self.ai.clean_resume(content) if self.ai else content
return [{
"boss_resume_id": None,
"title": "我的在线简历",
"position": None,
"content": content,
}]
return []
}]