bitfennec
1059be1b62
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
292 lines
9.5 KiB
Python
292 lines
9.5 KiB
Python
from pathlib import Path
|
||
import re
|
||
import sys
|
||
import time
|
||
from urllib.parse import urljoin, urlsplit
|
||
|
||
try:
|
||
import tomllib
|
||
except ModuleNotFoundError:
|
||
import tomli as tomllib
|
||
|
||
from common.content import sanitize_body_text
|
||
from common.fetcher import PoliteFetcher
|
||
from common.protocol import (
|
||
AnnouncementItem,
|
||
AttachmentItem,
|
||
CollectorMeta,
|
||
IngestPayload,
|
||
)
|
||
from common.security import is_safe_url
|
||
from selectolax.lexbor import LexborHTMLParser
|
||
|
||
|
||
def crawl_channel(
|
||
channel_cfg: dict,
|
||
site_cfg: dict,
|
||
template_cfg: dict,
|
||
max_items: int = 10,
|
||
) -> IngestPayload:
|
||
source_id = channel_cfg["source_id"]
|
||
name = channel_cfg["name"]
|
||
province = site_cfg.get("province", "")
|
||
city = site_cfg.get("city", "")
|
||
start_url = channel_cfg["start_url"]
|
||
start_host = urlsplit(start_url).netloc.split(":")[0].lower()
|
||
allowed_domains = channel_cfg.get(
|
||
"allowed_domains",
|
||
site_cfg.get("allowed_domains", [start_host]),
|
||
)
|
||
|
||
item_selector = channel_cfg.get(
|
||
"item_selector", template_cfg.get("item_selector", "a")
|
||
)
|
||
url_pattern = channel_cfg.get(
|
||
"url_pattern", template_cfg.get("url_pattern")
|
||
)
|
||
keywords = channel_cfg.get(
|
||
"keywords",
|
||
template_cfg.get(
|
||
"keywords", ["招聘", "招考", "公告", "公示", "选拔", "录用", "选调"]
|
||
),
|
||
)
|
||
title_selectors = channel_cfg.get(
|
||
"title_selector",
|
||
template_cfg.get("title_selector", "h1, h2, title"),
|
||
)
|
||
body_selectors = channel_cfg.get(
|
||
"body_selector",
|
||
template_cfg.get(
|
||
"body_selector",
|
||
"div#content, div.content, div.article, div.article-content, div.con",
|
||
),
|
||
)
|
||
date_regex = channel_cfg.get(
|
||
"date_regex",
|
||
template_cfg.get("date_regex", r"(\d{4}[-/年]\d{1,2}[-/月]\d{1,2})"),
|
||
)
|
||
att_extensions = channel_cfg.get(
|
||
"attachment_extensions",
|
||
template_cfg.get(
|
||
"attachment_extensions",
|
||
[".xlsx", ".xls", ".pdf", ".docx", ".doc"],
|
||
),
|
||
)
|
||
|
||
fetcher = PoliteFetcher()
|
||
start_time = time.time()
|
||
error_msg = None
|
||
items: list[AnnouncementItem] = []
|
||
|
||
try:
|
||
html = fetcher.get_text(start_url, allowed_domains, start_host)
|
||
tree = LexborHTMLParser(html)
|
||
|
||
candidates = []
|
||
for elem in tree.css(item_selector):
|
||
text = elem.text(strip=True)
|
||
href = elem.attributes.get("href", "").strip()
|
||
if not text or not href or href.startswith("javascript:"):
|
||
continue
|
||
|
||
if url_pattern and not re.search(url_pattern, href):
|
||
continue
|
||
|
||
if keywords and not any(k in text for k in keywords):
|
||
continue
|
||
|
||
full_url = urljoin(start_url, href)
|
||
if not is_safe_url(full_url, allowed_domains, start_host):
|
||
continue
|
||
candidates.append((text, full_url))
|
||
|
||
seen_urls = set()
|
||
for i, (title, url) in enumerate(candidates):
|
||
if url in seen_urls:
|
||
continue
|
||
seen_urls.add(url)
|
||
if i > 0:
|
||
time.sleep(0.3)
|
||
|
||
body_text = None
|
||
attachments = []
|
||
pub_date = None
|
||
|
||
try:
|
||
detail_html = fetcher.get_text(url, allowed_domains, start_host)
|
||
detail_tree = LexborHTMLParser(detail_html)
|
||
detail_tree.strip_tags(["script", "style", "noscript", "iframe", "svg"])
|
||
for crumb in detail_tree.css(".breadcrumb, .nav, .crumbs, .location, .pos, .head_top, .footer, .header"):
|
||
crumb.decompose()
|
||
|
||
title_node = detail_tree.css_first(title_selectors)
|
||
if title_node:
|
||
parsed_title = title_node.text(strip=True)
|
||
site_name = site_cfg.get("name", "")
|
||
if (
|
||
len(parsed_title) >= 6
|
||
and parsed_title != site_name
|
||
and not any(k == parsed_title for k in [site_name, "人才网", "人民政府", "人事考试网", "人事考试中心"])
|
||
):
|
||
title = parsed_title
|
||
|
||
body_node = detail_tree.css_first(body_selectors)
|
||
if body_node:
|
||
extracted = body_node.text(strip=True)
|
||
extracted = re.sub(
|
||
r"^(?:首页|当前位置[：:]\s*首页|您当前所在的位置[：:]\s*首页)[^\n\r]{0,100}(?:>|»|/|·|正文|内容|详情)?\s*",
|
||
"",
|
||
extracted,
|
||
)
|
||
body_text = sanitize_body_text(extracted)
|
||
|
||
if body_text and date_regex:
|
||
m = re.search(date_regex, body_text)
|
||
if m:
|
||
pub_date = (
|
||
m.group(1)
|
||
.replace("年", "-")
|
||
.replace("月", "-")
|
||
.replace("/", "-")
|
||
)
|
||
|
||
seen_att_urls = set()
|
||
for att_a in detail_tree.css("a"):
|
||
if len(attachments) >= 50:
|
||
break
|
||
att_href = att_a.attributes.get("href", "").strip()
|
||
att_text = att_a.text(strip=True)
|
||
if any(att_href.lower().endswith(ext) for ext in att_extensions):
|
||
full_att_url = urljoin(url, att_href)
|
||
if (
|
||
is_safe_url(full_att_url, allowed_domains, start_host)
|
||
and full_att_url not in seen_att_urls
|
||
):
|
||
seen_att_urls.add(full_att_url)
|
||
attachments.append(
|
||
AttachmentItem(name=att_text or "附件", url=full_att_url)
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
if not body_text and not attachments:
|
||
continue
|
||
|
||
items.append(
|
||
AnnouncementItem(
|
||
url=url,
|
||
title=title,
|
||
kind="招聘" if "公示" not in title else "拟聘用",
|
||
employer=site_cfg.get("employer", f"{name} / 用人单位"),
|
||
employer_type=site_cfg.get("employer_type", "事业单位"),
|
||
published_at=pub_date,
|
||
region_hint=f"{province} {city}".strip(),
|
||
body_text=body_text,
|
||
attachments=attachments,
|
||
positions=[],
|
||
)
|
||
)
|
||
|
||
if len(items) >= max_items:
|
||
break
|
||
|
||
except Exception as exc:
|
||
error_msg = str(exc)
|
||
finally:
|
||
fetcher.close()
|
||
|
||
elapsed_ms = int((time.time() - start_time) * 1000)
|
||
|
||
meta = CollectorMeta(
|
||
name=f"{name}_declarative",
|
||
runtime="declarative/v1",
|
||
auth_status="none",
|
||
duration_ms=elapsed_ms,
|
||
error_message=error_msg,
|
||
)
|
||
|
||
return IngestPayload(
|
||
source_id=source_id,
|
||
batch_id=f"{source_id}-{int(time.time())}",
|
||
collector=meta,
|
||
items=items,
|
||
)
|
||
|
||
|
||
def load_declarative_collectors(
|
||
config_path: Path | str, base_dir: Path | None = None
|
||
) -> list[dict]:
|
||
cfg_file = Path(config_path)
|
||
with open(cfg_file, "rb") as f:
|
||
cfg = tomllib.load(f)
|
||
|
||
site_cfg = cfg.get("site", {})
|
||
template_cfg = cfg.get("template", {})
|
||
if not template_cfg:
|
||
template_cfg = {**cfg.get("list", {}), **cfg.get("detail", {})}
|
||
|
||
site_id = site_cfg.get("id", f"{cfg_file.parent.name}.{cfg_file.stem}")
|
||
site_name = site_cfg.get("name", site_id)
|
||
province = site_cfg.get("province", cfg_file.parent.name)
|
||
city = site_cfg.get("city", "")
|
||
site_tags = site_cfg.get("tags", [])
|
||
|
||
channels = cfg.get("channels", [])
|
||
if not channels and "start_url" in site_cfg:
|
||
channels = [
|
||
{
|
||
"id": "",
|
||
"name": site_name,
|
||
"start_url": site_cfg["start_url"],
|
||
"tags": site_tags,
|
||
}
|
||
]
|
||
|
||
rel_file = (
|
||
str(cfg_file.relative_to(base_dir)) if base_dir else str(cfg_file)
|
||
)
|
||
|
||
result = []
|
||
for ch in channels:
|
||
ch_id = ch.get("id", "").strip()
|
||
full_source_id = f"{site_id}.{ch_id}" if ch_id else site_id
|
||
ch_name = ch.get("name", ch_id)
|
||
full_name = f"{site_name} - {ch_name}" if ch_id and ch_name != site_name else site_name
|
||
ch_tags = list(set(site_tags + ch.get("tags", [])))
|
||
|
||
channel_entry = dict(ch)
|
||
channel_entry["source_id"] = full_source_id
|
||
channel_entry["name"] = full_name
|
||
channel_entry["start_url"] = ch.get("start_url", site_cfg.get("start_url", ""))
|
||
|
||
if not channel_entry["start_url"]:
|
||
continue
|
||
|
||
domain = urlsplit(channel_entry["start_url"]).netloc.lower()
|
||
meta = {
|
||
"source_id": full_source_id,
|
||
"name": full_name,
|
||
"province": province,
|
||
"city": city,
|
||
"tags": ch_tags,
|
||
"domain": domain,
|
||
}
|
||
|
||
crawl_fn = (
|
||
lambda ch_e=channel_entry, s_c=site_cfg, t_c=template_cfg: (
|
||
lambda max_items=10: crawl_channel(
|
||
ch_e, s_c, t_c, max_items=max_items
|
||
)
|
||
)
|
||
)()
|
||
|
||
result.append({
|
||
"meta": meta,
|
||
"crawl_fn": crawl_fn,
|
||
"file": rel_file,
|
||
"type": "toml",
|
||
"domain": domain,
|
||
})
|
||
|
||
return result
|