基本完成
This commit is contained in:
@@ -1,126 +1,102 @@
|
||||
"""
|
||||
AI 简历抓取模块
|
||||
从已登录的 Boss直聘 账号中抓取所有简历,
|
||||
并用 AI 整理成结构化文本后返回。
|
||||
AI 简历抓取模块(低痕迹 CDP 版)
|
||||
Boss直聘 的简历页是一份"在线简历"的分板块编辑页(不是一份份的简历列表),
|
||||
板块以 id 区分:userinfo/summary/purpose/history/project/education/
|
||||
clubExp/professional-skill。
|
||||
通过原生 CDP 通道(不 attach 页面)读取各板块 innerText,拼装成结构化简历。
|
||||
"""
|
||||
import json
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from .manager import BrowserManager, BOSS_RESUME_PAGE
|
||||
from .cdp_lite import LiteCDP
|
||||
from .manager import BOSS_RESUME_PAGE
|
||||
|
||||
# 在页面上一次性抽取各板块文本(返回 JSON 字符串)
|
||||
_SECTION_JS = """(() => {
|
||||
const ids = ['userinfo', 'summary', 'purpose', 'history', 'project',
|
||||
'education', 'clubExp', 'professional-skill'];
|
||||
const out = {};
|
||||
for (const id of ids) {
|
||||
const el = document.getElementById(id);
|
||||
if (el) out[id] = el.innerText;
|
||||
}
|
||||
return JSON.stringify(out);
|
||||
})()"""
|
||||
|
||||
# 板块标题(用于拼装简历文本)
|
||||
SECTION_TITLES = {
|
||||
"userinfo": "基本信息",
|
||||
"summary": "个人优势",
|
||||
"purpose": "期望职位",
|
||||
"history": "工作经历",
|
||||
"project": "项目经历",
|
||||
"education": "教育经历",
|
||||
"clubExp": "社团/组织经历",
|
||||
"professional-skill": "专业技能",
|
||||
}
|
||||
|
||||
|
||||
class ResumeScraper:
|
||||
"""抓取 Boss直聘 账号内的所有简历"""
|
||||
"""抓取 Boss直聘 账号内的在线简历"""
|
||||
|
||||
def __init__(self, manager: BrowserManager, ai_client=None):
|
||||
def __init__(self, manager, ai_client=None):
|
||||
self.manager = manager
|
||||
self.ai = ai_client
|
||||
self.cdp = LiteCDP()
|
||||
|
||||
def scrape_all(self) -> list[dict]:
|
||||
"""
|
||||
抓取所有简历。
|
||||
抓取在线简历。
|
||||
返回格式:[{boss_resume_id, title, position, content}, ...]
|
||||
"""
|
||||
page = self.manager.ensure_connected()
|
||||
logger.info("正在打开 Boss直聘 简历页...")
|
||||
page.goto(BOSS_RESUME_PAGE, wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
# 浏览器可能已被手动关闭,先确保拉起(复用登录态 profile)
|
||||
self.manager.ensure_browser_started()
|
||||
tab = self.cdp.open_tab(BOSS_RESUME_PAGE)
|
||||
tab_id, ws_url = tab["id"], tab["webSocketDebuggerUrl"]
|
||||
try:
|
||||
import time
|
||||
time.sleep(8) # 等 SPA 渲染完成
|
||||
raw = self.cdp.evaluate(ws_url, _SECTION_JS)
|
||||
finally:
|
||||
self.cdp.close_tab(tab_id)
|
||||
|
||||
resumes: list[dict] = []
|
||||
if not raw:
|
||||
logger.warning("未能从简历页读取到内容")
|
||||
return []
|
||||
|
||||
# 尝试多种可能的简历卡片选择器(Boss直聘 页面结构可能变化)
|
||||
card_selectors = [
|
||||
".resume-item",
|
||||
".resume-card",
|
||||
"[class*='resume-list'] li",
|
||||
".geek-resume-item",
|
||||
]
|
||||
card_els = []
|
||||
for sel in card_selectors:
|
||||
card_els = page.query_selector_all(sel)
|
||||
if card_els:
|
||||
logger.info(f"使用选择器 {sel} 找到 {len(card_els)} 份简历")
|
||||
break
|
||||
try:
|
||||
sections = json.loads(raw)
|
||||
except Exception as e:
|
||||
logger.warning(f"简历板块数据解析失败:{e}")
|
||||
return []
|
||||
|
||||
if not card_els:
|
||||
logger.warning("未通过卡片选择器找到简历,尝试整页抓取")
|
||||
return self._scrape_whole_page(page)
|
||||
if not sections:
|
||||
logger.warning("简历页未找到任何板块,请确认已登录")
|
||||
return []
|
||||
|
||||
for i, card in enumerate(card_els):
|
||||
try:
|
||||
resume = self._parse_resume_card(card, i)
|
||||
if resume:
|
||||
resumes.append(resume)
|
||||
except Exception as e:
|
||||
logger.debug(f"解析第 {i} 份简历失败:{e}")
|
||||
# 拼装结构化简历文本
|
||||
parts = []
|
||||
for key, title in SECTION_TITLES.items():
|
||||
text = (sections.get(key) or "").strip()
|
||||
if text:
|
||||
parts.append(f"【{title}】\n{text}")
|
||||
content = "\n\n".join(parts)
|
||||
|
||||
logger.success(f"共抓取到 {len(resumes)} 份简历")
|
||||
return resumes
|
||||
# 从基本信息板块首行提取姓名作为标题
|
||||
userinfo = (sections.get("userinfo") or "").strip()
|
||||
name = userinfo.splitlines()[0].strip() if userinfo else ""
|
||||
title = f"{name}的在线简历" if name else "我的在线简历"
|
||||
|
||||
def _parse_resume_card(self, card, index: int) -> dict | None:
|
||||
"""解析单个简历卡片"""
|
||||
def text(sel: str) -> str:
|
||||
el = card.query_selector(sel)
|
||||
return el.inner_text().strip() if el else ""
|
||||
|
||||
title = (
|
||||
text(".resume-title") or text(".title") or
|
||||
text(".name") or f"简历 {index + 1}"
|
||||
)
|
||||
position = text(".position") or text(".job-name") or text(".expect-position")
|
||||
|
||||
# 尝试点开详情获取完整内容
|
||||
content = self._extract_detail_content(card) or text(".resume-content") or text("p")
|
||||
|
||||
# 用 AI 整理内容
|
||||
if content and self.ai:
|
||||
# 用 AI 整理内容(可选)
|
||||
if content and self.ai and self.ai.available():
|
||||
content = self.ai.clean_resume(content)
|
||||
|
||||
return {
|
||||
"boss_resume_id": None, # Boss直聘 前端通常不暴露,后续可从 URL 提取
|
||||
logger.success(f"简历抓取完成:{title},{len(content)} 字符")
|
||||
return [{
|
||||
"boss_resume_id": None, # Boss直聘 前端不暴露简历 ID
|
||||
"title": title,
|
||||
"position": position or None,
|
||||
"position": None,
|
||||
"content": content or None,
|
||||
}
|
||||
|
||||
def _extract_detail_content(self, card) -> str | None:
|
||||
"""尝试点击简历卡片进入详情页抓取完整内容"""
|
||||
page = self.manager.page
|
||||
try:
|
||||
clickable = card.query_selector("a") or card
|
||||
clickable.click()
|
||||
page.wait_for_timeout(2000)
|
||||
detail_selectors = [
|
||||
".resume-detail",
|
||||
".geek-resume-detail",
|
||||
"[class*='resume-content']",
|
||||
".resume-preview",
|
||||
]
|
||||
for sel in detail_selectors:
|
||||
el = page.query_selector(sel)
|
||||
if el:
|
||||
content = el.inner_text().strip()
|
||||
page.go_back()
|
||||
page.wait_for_timeout(1500)
|
||||
return content
|
||||
page.go_back()
|
||||
page.wait_for_timeout(1500)
|
||||
except Exception as e:
|
||||
logger.debug(f"提取简历详情失败:{e}")
|
||||
return None
|
||||
|
||||
def _scrape_whole_page(self, page) -> list[dict]:
|
||||
"""兜底方案:抓取整个简历页主体文本作为一份简历"""
|
||||
body_selectors = [".main-wrap", ".geek-wrap", "main", "#main"]
|
||||
for sel in body_selectors:
|
||||
el = page.query_selector(sel)
|
||||
if el:
|
||||
content = el.inner_text().strip()
|
||||
if content:
|
||||
content = self.ai.clean_resume(content) if self.ai else content
|
||||
return [{
|
||||
"boss_resume_id": None,
|
||||
"title": "我的在线简历",
|
||||
"position": None,
|
||||
"content": content,
|
||||
}]
|
||||
return []
|
||||
}]
|
||||
|
||||
Reference in New Issue
Block a user