feat: initialize bangwen recruitment aggregation and search engine
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
This commit is contained in:
bitfennec
@@ -0,0 +1,291 @@
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
from common.content import sanitize_body_text
|
||||
from common.fetcher import PoliteFetcher
|
||||
from common.protocol import (
|
||||
AnnouncementItem,
|
||||
AttachmentItem,
|
||||
CollectorMeta,
|
||||
IngestPayload,
|
||||
)
|
||||
from common.security import is_safe_url
|
||||
from selectolax.lexbor import LexborHTMLParser
|
||||
|
||||
|
||||
def crawl_channel(
|
||||
channel_cfg: dict,
|
||||
site_cfg: dict,
|
||||
template_cfg: dict,
|
||||
max_items: int = 10,
|
||||
) -> IngestPayload:
|
||||
source_id = channel_cfg["source_id"]
|
||||
name = channel_cfg["name"]
|
||||
province = site_cfg.get("province", "")
|
||||
city = site_cfg.get("city", "")
|
||||
start_url = channel_cfg["start_url"]
|
||||
start_host = urlsplit(start_url).netloc.split(":")[0].lower()
|
||||
allowed_domains = channel_cfg.get(
|
||||
"allowed_domains",
|
||||
site_cfg.get("allowed_domains", [start_host]),
|
||||
)
|
||||
|
||||
item_selector = channel_cfg.get(
|
||||
"item_selector", template_cfg.get("item_selector", "a")
|
||||
)
|
||||
url_pattern = channel_cfg.get(
|
||||
"url_pattern", template_cfg.get("url_pattern")
|
||||
)
|
||||
keywords = channel_cfg.get(
|
||||
"keywords",
|
||||
template_cfg.get(
|
||||
"keywords", ["招聘", "招考", "公告", "公示", "选拔", "录用", "选调"]
|
||||
),
|
||||
)
|
||||
title_selectors = channel_cfg.get(
|
||||
"title_selector",
|
||||
template_cfg.get("title_selector", "h1, h2, title"),
|
||||
)
|
||||
body_selectors = channel_cfg.get(
|
||||
"body_selector",
|
||||
template_cfg.get(
|
||||
"body_selector",
|
||||
"div#content, div.content, div.article, div.article-content, div.con",
|
||||
),
|
||||
)
|
||||
date_regex = channel_cfg.get(
|
||||
"date_regex",
|
||||
template_cfg.get("date_regex", r"(\d{4}[-/年]\d{1,2}[-/月]\d{1,2})"),
|
||||
)
|
||||
att_extensions = channel_cfg.get(
|
||||
"attachment_extensions",
|
||||
template_cfg.get(
|
||||
"attachment_extensions",
|
||||
[".xlsx", ".xls", ".pdf", ".docx", ".doc"],
|
||||
),
|
||||
)
|
||||
|
||||
fetcher = PoliteFetcher()
|
||||
start_time = time.time()
|
||||
error_msg = None
|
||||
items: list[AnnouncementItem] = []
|
||||
|
||||
try:
|
||||
html = fetcher.get_text(start_url, allowed_domains, start_host)
|
||||
tree = LexborHTMLParser(html)
|
||||
|
||||
candidates = []
|
||||
for elem in tree.css(item_selector):
|
||||
text = elem.text(strip=True)
|
||||
href = elem.attributes.get("href", "").strip()
|
||||
if not text or not href or href.startswith("javascript:"):
|
||||
continue
|
||||
|
||||
if url_pattern and not re.search(url_pattern, href):
|
||||
continue
|
||||
|
||||
if keywords and not any(k in text for k in keywords):
|
||||
continue
|
||||
|
||||
full_url = urljoin(start_url, href)
|
||||
if not is_safe_url(full_url, allowed_domains, start_host):
|
||||
continue
|
||||
candidates.append((text, full_url))
|
||||
|
||||
seen_urls = set()
|
||||
for i, (title, url) in enumerate(candidates):
|
||||
if url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
if i > 0:
|
||||
time.sleep(0.3)
|
||||
|
||||
body_text = None
|
||||
attachments = []
|
||||
pub_date = None
|
||||
|
||||
try:
|
||||
detail_html = fetcher.get_text(url, allowed_domains, start_host)
|
||||
detail_tree = LexborHTMLParser(detail_html)
|
||||
detail_tree.strip_tags(["script", "style", "noscript", "iframe", "svg"])
|
||||
for crumb in detail_tree.css(".breadcrumb, .nav, .crumbs, .location, .pos, .head_top, .footer, .header"):
|
||||
crumb.decompose()
|
||||
|
||||
title_node = detail_tree.css_first(title_selectors)
|
||||
if title_node:
|
||||
parsed_title = title_node.text(strip=True)
|
||||
site_name = site_cfg.get("name", "")
|
||||
if (
|
||||
len(parsed_title) >= 6
|
||||
and parsed_title != site_name
|
||||
and not any(k == parsed_title for k in [site_name, "人才网", "人民政府", "人事考试网", "人事考试中心"])
|
||||
):
|
||||
title = parsed_title
|
||||
|
||||
body_node = detail_tree.css_first(body_selectors)
|
||||
if body_node:
|
||||
extracted = body_node.text(strip=True)
|
||||
extracted = re.sub(
|
||||
r"^(?:首页|当前位置[：:]\s*首页|您当前所在的位置[：:]\s*首页)[^\n\r]{0,100}(?:>|»|/|·|正文|内容|详情)?\s*",
|
||||
"",
|
||||
extracted,
|
||||
)
|
||||
body_text = sanitize_body_text(extracted)
|
||||
|
||||
if body_text and date_regex:
|
||||
m = re.search(date_regex, body_text)
|
||||
if m:
|
||||
pub_date = (
|
||||
m.group(1)
|
||||
.replace("年", "-")
|
||||
.replace("月", "-")
|
||||
.replace("/", "-")
|
||||
)
|
||||
|
||||
seen_att_urls = set()
|
||||
for att_a in detail_tree.css("a"):
|
||||
if len(attachments) >= 50:
|
||||
break
|
||||
att_href = att_a.attributes.get("href", "").strip()
|
||||
att_text = att_a.text(strip=True)
|
||||
if any(att_href.lower().endswith(ext) for ext in att_extensions):
|
||||
full_att_url = urljoin(url, att_href)
|
||||
if (
|
||||
is_safe_url(full_att_url, allowed_domains, start_host)
|
||||
and full_att_url not in seen_att_urls
|
||||
):
|
||||
seen_att_urls.add(full_att_url)
|
||||
attachments.append(
|
||||
AttachmentItem(name=att_text or "附件", url=full_att_url)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not body_text and not attachments:
|
||||
continue
|
||||
|
||||
items.append(
|
||||
AnnouncementItem(
|
||||
url=url,
|
||||
title=title,
|
||||
kind="招聘" if "公示" not in title else "拟聘用",
|
||||
employer=site_cfg.get("employer", f"{name} / 用人单位"),
|
||||
employer_type=site_cfg.get("employer_type", "事业单位"),
|
||||
published_at=pub_date,
|
||||
region_hint=f"{province} {city}".strip(),
|
||||
body_text=body_text,
|
||||
attachments=attachments,
|
||||
positions=[],
|
||||
)
|
||||
)
|
||||
|
||||
if len(items) >= max_items:
|
||||
break
|
||||
|
||||
except Exception as exc:
|
||||
error_msg = str(exc)
|
||||
finally:
|
||||
fetcher.close()
|
||||
|
||||
elapsed_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
meta = CollectorMeta(
|
||||
name=f"{name}_declarative",
|
||||
runtime="declarative/v1",
|
||||
auth_status="none",
|
||||
duration_ms=elapsed_ms,
|
||||
error_message=error_msg,
|
||||
)
|
||||
|
||||
return IngestPayload(
|
||||
source_id=source_id,
|
||||
batch_id=f"{source_id}-{int(time.time())}",
|
||||
collector=meta,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
def load_declarative_collectors(
|
||||
config_path: Path | str, base_dir: Path | None = None
|
||||
) -> list[dict]:
|
||||
cfg_file = Path(config_path)
|
||||
with open(cfg_file, "rb") as f:
|
||||
cfg = tomllib.load(f)
|
||||
|
||||
site_cfg = cfg.get("site", {})
|
||||
template_cfg = cfg.get("template", {})
|
||||
if not template_cfg:
|
||||
template_cfg = {**cfg.get("list", {}), **cfg.get("detail", {})}
|
||||
|
||||
site_id = site_cfg.get("id", f"{cfg_file.parent.name}.{cfg_file.stem}")
|
||||
site_name = site_cfg.get("name", site_id)
|
||||
province = site_cfg.get("province", cfg_file.parent.name)
|
||||
city = site_cfg.get("city", "")
|
||||
site_tags = site_cfg.get("tags", [])
|
||||
|
||||
channels = cfg.get("channels", [])
|
||||
if not channels and "start_url" in site_cfg:
|
||||
channels = [
|
||||
{
|
||||
"id": "",
|
||||
"name": site_name,
|
||||
"start_url": site_cfg["start_url"],
|
||||
"tags": site_tags,
|
||||
}
|
||||
]
|
||||
|
||||
rel_file = (
|
||||
str(cfg_file.relative_to(base_dir)) if base_dir else str(cfg_file)
|
||||
)
|
||||
|
||||
result = []
|
||||
for ch in channels:
|
||||
ch_id = ch.get("id", "").strip()
|
||||
full_source_id = f"{site_id}.{ch_id}" if ch_id else site_id
|
||||
ch_name = ch.get("name", ch_id)
|
||||
full_name = f"{site_name} - {ch_name}" if ch_id and ch_name != site_name else site_name
|
||||
ch_tags = list(set(site_tags + ch.get("tags", [])))
|
||||
|
||||
channel_entry = dict(ch)
|
||||
channel_entry["source_id"] = full_source_id
|
||||
channel_entry["name"] = full_name
|
||||
channel_entry["start_url"] = ch.get("start_url", site_cfg.get("start_url", ""))
|
||||
|
||||
if not channel_entry["start_url"]:
|
||||
continue
|
||||
|
||||
domain = urlsplit(channel_entry["start_url"]).netloc.lower()
|
||||
meta = {
|
||||
"source_id": full_source_id,
|
||||
"name": full_name,
|
||||
"province": province,
|
||||
"city": city,
|
||||
"tags": ch_tags,
|
||||
"domain": domain,
|
||||
}
|
||||
|
||||
crawl_fn = (
|
||||
lambda ch_e=channel_entry, s_c=site_cfg, t_c=template_cfg: (
|
||||
lambda max_items=10: crawl_channel(
|
||||
ch_e, s_c, t_c, max_items=max_items
|
||||
)
|
||||
)
|
||||
)()
|
||||
|
||||
result.append({
|
||||
"meta": meta,
|
||||
"crawl_fn": crawl_fn,
|
||||
"file": rel_file,
|
||||
"type": "toml",
|
||||
"domain": domain,
|
||||
})
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user