127 lines
4.4 KiB
Python
127 lines
4.4 KiB
Python
"""
|
|
AI 简历抓取模块
|
|
从已登录的 Boss直聘 账号中抓取所有简历,
|
|
并用 AI 整理成结构化文本后返回。
|
|
"""
|
|
from loguru import logger
|
|
|
|
from .manager import BrowserManager, BOSS_RESUME_PAGE
|
|
|
|
|
|
class ResumeScraper:
|
|
"""抓取 Boss直聘 账号内的所有简历"""
|
|
|
|
def __init__(self, manager: BrowserManager, ai_client=None):
|
|
self.manager = manager
|
|
self.ai = ai_client
|
|
|
|
def scrape_all(self) -> list[dict]:
|
|
"""
|
|
抓取所有简历。
|
|
返回格式:[{boss_resume_id, title, position, content}, ...]
|
|
"""
|
|
page = self.manager.ensure_connected()
|
|
logger.info("正在打开 Boss直聘 简历页...")
|
|
page.goto(BOSS_RESUME_PAGE, wait_until="domcontentloaded", timeout=30000)
|
|
page.wait_for_timeout(3000)
|
|
|
|
resumes: list[dict] = []
|
|
|
|
# 尝试多种可能的简历卡片选择器(Boss直聘 页面结构可能变化)
|
|
card_selectors = [
|
|
".resume-item",
|
|
".resume-card",
|
|
"[class*='resume-list'] li",
|
|
".geek-resume-item",
|
|
]
|
|
card_els = []
|
|
for sel in card_selectors:
|
|
card_els = page.query_selector_all(sel)
|
|
if card_els:
|
|
logger.info(f"使用选择器 {sel} 找到 {len(card_els)} 份简历")
|
|
break
|
|
|
|
if not card_els:
|
|
logger.warning("未通过卡片选择器找到简历,尝试整页抓取")
|
|
return self._scrape_whole_page(page)
|
|
|
|
for i, card in enumerate(card_els):
|
|
try:
|
|
resume = self._parse_resume_card(card, i)
|
|
if resume:
|
|
resumes.append(resume)
|
|
except Exception as e:
|
|
logger.debug(f"解析第 {i} 份简历失败:{e}")
|
|
|
|
logger.success(f"共抓取到 {len(resumes)} 份简历")
|
|
return resumes
|
|
|
|
def _parse_resume_card(self, card, index: int) -> dict | None:
|
|
"""解析单个简历卡片"""
|
|
def text(sel: str) -> str:
|
|
el = card.query_selector(sel)
|
|
return el.inner_text().strip() if el else ""
|
|
|
|
title = (
|
|
text(".resume-title") or text(".title") or
|
|
text(".name") or f"简历 {index + 1}"
|
|
)
|
|
position = text(".position") or text(".job-name") or text(".expect-position")
|
|
|
|
# 尝试点开详情获取完整内容
|
|
content = self._extract_detail_content(card) or text(".resume-content") or text("p")
|
|
|
|
# 用 AI 整理内容
|
|
if content and self.ai:
|
|
content = self.ai.clean_resume(content)
|
|
|
|
return {
|
|
"boss_resume_id": None, # Boss直聘 前端通常不暴露,后续可从 URL 提取
|
|
"title": title,
|
|
"position": position or None,
|
|
"content": content or None,
|
|
}
|
|
|
|
def _extract_detail_content(self, card) -> str | None:
|
|
"""尝试点击简历卡片进入详情页抓取完整内容"""
|
|
page = self.manager.page
|
|
try:
|
|
clickable = card.query_selector("a") or card
|
|
clickable.click()
|
|
page.wait_for_timeout(2000)
|
|
detail_selectors = [
|
|
".resume-detail",
|
|
".geek-resume-detail",
|
|
"[class*='resume-content']",
|
|
".resume-preview",
|
|
]
|
|
for sel in detail_selectors:
|
|
el = page.query_selector(sel)
|
|
if el:
|
|
content = el.inner_text().strip()
|
|
page.go_back()
|
|
page.wait_for_timeout(1500)
|
|
return content
|
|
page.go_back()
|
|
page.wait_for_timeout(1500)
|
|
except Exception as e:
|
|
logger.debug(f"提取简历详情失败:{e}")
|
|
return None
|
|
|
|
def _scrape_whole_page(self, page) -> list[dict]:
|
|
"""兜底方案:抓取整个简历页主体文本作为一份简历"""
|
|
body_selectors = [".main-wrap", ".geek-wrap", "main", "#main"]
|
|
for sel in body_selectors:
|
|
el = page.query_selector(sel)
|
|
if el:
|
|
content = el.inner_text().strip()
|
|
if content:
|
|
content = self.ai.clean_resume(content) if self.ai else content
|
|
return [{
|
|
"boss_resume_id": None,
|
|
"title": "我的在线简历",
|
|
"position": None,
|
|
"content": content,
|
|
}]
|
|
return []
|