init
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
AI 简历抓取模块
|
||||
从已登录的 Boss直聘 账号中抓取所有简历,
|
||||
并用 AI 整理成结构化文本后返回。
|
||||
"""
|
||||
from loguru import logger
|
||||
|
||||
from .manager import BrowserManager, BOSS_RESUME_PAGE
|
||||
|
||||
|
||||
class ResumeScraper:
|
||||
"""抓取 Boss直聘 账号内的所有简历"""
|
||||
|
||||
def __init__(self, manager: BrowserManager, ai_client=None):
|
||||
self.manager = manager
|
||||
self.ai = ai_client
|
||||
|
||||
def scrape_all(self) -> list[dict]:
|
||||
"""
|
||||
抓取所有简历。
|
||||
返回格式:[{boss_resume_id, title, position, content}, ...]
|
||||
"""
|
||||
page = self.manager.ensure_connected()
|
||||
logger.info("正在打开 Boss直聘 简历页...")
|
||||
page.goto(BOSS_RESUME_PAGE, wait_until="domcontentloaded", timeout=30000)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
resumes: list[dict] = []
|
||||
|
||||
# 尝试多种可能的简历卡片选择器(Boss直聘 页面结构可能变化)
|
||||
card_selectors = [
|
||||
".resume-item",
|
||||
".resume-card",
|
||||
"[class*='resume-list'] li",
|
||||
".geek-resume-item",
|
||||
]
|
||||
card_els = []
|
||||
for sel in card_selectors:
|
||||
card_els = page.query_selector_all(sel)
|
||||
if card_els:
|
||||
logger.info(f"使用选择器 {sel} 找到 {len(card_els)} 份简历")
|
||||
break
|
||||
|
||||
if not card_els:
|
||||
logger.warning("未通过卡片选择器找到简历,尝试整页抓取")
|
||||
return self._scrape_whole_page(page)
|
||||
|
||||
for i, card in enumerate(card_els):
|
||||
try:
|
||||
resume = self._parse_resume_card(card, i)
|
||||
if resume:
|
||||
resumes.append(resume)
|
||||
except Exception as e:
|
||||
logger.debug(f"解析第 {i} 份简历失败:{e}")
|
||||
|
||||
logger.success(f"共抓取到 {len(resumes)} 份简历")
|
||||
return resumes
|
||||
|
||||
def _parse_resume_card(self, card, index: int) -> dict | None:
|
||||
"""解析单个简历卡片"""
|
||||
def text(sel: str) -> str:
|
||||
el = card.query_selector(sel)
|
||||
return el.inner_text().strip() if el else ""
|
||||
|
||||
title = (
|
||||
text(".resume-title") or text(".title") or
|
||||
text(".name") or f"简历 {index + 1}"
|
||||
)
|
||||
position = text(".position") or text(".job-name") or text(".expect-position")
|
||||
|
||||
# 尝试点开详情获取完整内容
|
||||
content = self._extract_detail_content(card) or text(".resume-content") or text("p")
|
||||
|
||||
# 用 AI 整理内容
|
||||
if content and self.ai:
|
||||
content = self.ai.clean_resume(content)
|
||||
|
||||
return {
|
||||
"boss_resume_id": None, # Boss直聘 前端通常不暴露,后续可从 URL 提取
|
||||
"title": title,
|
||||
"position": position or None,
|
||||
"content": content or None,
|
||||
}
|
||||
|
||||
def _extract_detail_content(self, card) -> str | None:
|
||||
"""尝试点击简历卡片进入详情页抓取完整内容"""
|
||||
page = self.manager.page
|
||||
try:
|
||||
clickable = card.query_selector("a") or card
|
||||
clickable.click()
|
||||
page.wait_for_timeout(2000)
|
||||
detail_selectors = [
|
||||
".resume-detail",
|
||||
".geek-resume-detail",
|
||||
"[class*='resume-content']",
|
||||
".resume-preview",
|
||||
]
|
||||
for sel in detail_selectors:
|
||||
el = page.query_selector(sel)
|
||||
if el:
|
||||
content = el.inner_text().strip()
|
||||
page.go_back()
|
||||
page.wait_for_timeout(1500)
|
||||
return content
|
||||
page.go_back()
|
||||
page.wait_for_timeout(1500)
|
||||
except Exception as e:
|
||||
logger.debug(f"提取简历详情失败:{e}")
|
||||
return None
|
||||
|
||||
def _scrape_whole_page(self, page) -> list[dict]:
|
||||
"""兜底方案:抓取整个简历页主体文本作为一份简历"""
|
||||
body_selectors = [".main-wrap", ".geek-wrap", "main", "#main"]
|
||||
for sel in body_selectors:
|
||||
el = page.query_selector(sel)
|
||||
if el:
|
||||
content = el.inner_text().strip()
|
||||
if content:
|
||||
content = self.ai.clean_resume(content) if self.ai else content
|
||||
return [{
|
||||
"boss_resume_id": None,
|
||||
"title": "我的在线简历",
|
||||
"position": None,
|
||||
"content": content,
|
||||
}]
|
||||
return []
|
||||
Reference in New Issue
Block a user