This commit is contained in:
2026-08-25 19:03:41 +08:00
commit e1416994bc
34 changed files with 3266 additions and 0 deletions
+126
View File
@@ -0,0 +1,126 @@
"""
AI 简历抓取模块
从已登录的 Boss直聘 账号中抓取所有简历,
并用 AI 整理成结构化文本后返回。
"""
from loguru import logger
from .manager import BrowserManager, BOSS_RESUME_PAGE
class ResumeScraper:
"""抓取 Boss直聘 账号内的所有简历"""
def __init__(self, manager: BrowserManager, ai_client=None):
self.manager = manager
self.ai = ai_client
def scrape_all(self) -> list[dict]:
"""
抓取所有简历。
返回格式:[{boss_resume_id, title, position, content}, ...]
"""
page = self.manager.ensure_connected()
logger.info("正在打开 Boss直聘 简历页...")
page.goto(BOSS_RESUME_PAGE, wait_until="domcontentloaded", timeout=30000)
page.wait_for_timeout(3000)
resumes: list[dict] = []
# 尝试多种可能的简历卡片选择器(Boss直聘 页面结构可能变化)
card_selectors = [
".resume-item",
".resume-card",
"[class*='resume-list'] li",
".geek-resume-item",
]
card_els = []
for sel in card_selectors:
card_els = page.query_selector_all(sel)
if card_els:
logger.info(f"使用选择器 {sel} 找到 {len(card_els)} 份简历")
break
if not card_els:
logger.warning("未通过卡片选择器找到简历,尝试整页抓取")
return self._scrape_whole_page(page)
for i, card in enumerate(card_els):
try:
resume = self._parse_resume_card(card, i)
if resume:
resumes.append(resume)
except Exception as e:
logger.debug(f"解析第 {i} 份简历失败:{e}")
logger.success(f"共抓取到 {len(resumes)} 份简历")
return resumes
def _parse_resume_card(self, card, index: int) -> dict | None:
"""解析单个简历卡片"""
def text(sel: str) -> str:
el = card.query_selector(sel)
return el.inner_text().strip() if el else ""
title = (
text(".resume-title") or text(".title") or
text(".name") or f"简历 {index + 1}"
)
position = text(".position") or text(".job-name") or text(".expect-position")
# 尝试点开详情获取完整内容
content = self._extract_detail_content(card) or text(".resume-content") or text("p")
# 用 AI 整理内容
if content and self.ai:
content = self.ai.clean_resume(content)
return {
"boss_resume_id": None, # Boss直聘 前端通常不暴露,后续可从 URL 提取
"title": title,
"position": position or None,
"content": content or None,
}
def _extract_detail_content(self, card) -> str | None:
"""尝试点击简历卡片进入详情页抓取完整内容"""
page = self.manager.page
try:
clickable = card.query_selector("a") or card
clickable.click()
page.wait_for_timeout(2000)
detail_selectors = [
".resume-detail",
".geek-resume-detail",
"[class*='resume-content']",
".resume-preview",
]
for sel in detail_selectors:
el = page.query_selector(sel)
if el:
content = el.inner_text().strip()
page.go_back()
page.wait_for_timeout(1500)
return content
page.go_back()
page.wait_for_timeout(1500)
except Exception as e:
logger.debug(f"提取简历详情失败:{e}")
return None
def _scrape_whole_page(self, page) -> list[dict]:
"""兜底方案:抓取整个简历页主体文本作为一份简历"""
body_selectors = [".main-wrap", ".geek-wrap", "main", "#main"]
for sel in body_selectors:
el = page.query_selector(sel)
if el:
content = el.inner_text().strip()
if content:
content = self.ai.clean_resume(content) if self.ai else content
return [{
"boss_resume_id": None,
"title": "我的在线简历",
"position": None,
"content": content,
}]
return []