feat: initialize bangwen recruitment aggregation and search engine
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
This commit is contained in:
bitfennec
@@ -0,0 +1,29 @@
|
||||
from .content import (
|
||||
ATTACHMENT_EXTENSIONS,
|
||||
is_anti_crawler,
|
||||
is_attachment_name,
|
||||
sanitize_body_text,
|
||||
)
|
||||
from .fetcher import PoliteFetcher
|
||||
from .protocol import (
|
||||
AnnouncementItem,
|
||||
AttachmentItem,
|
||||
CollectorMeta,
|
||||
IngestPayload,
|
||||
PositionItem,
|
||||
emit_payload,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ATTACHMENT_EXTENSIONS",
|
||||
"AnnouncementItem",
|
||||
"AttachmentItem",
|
||||
"CollectorMeta",
|
||||
"IngestPayload",
|
||||
"PoliteFetcher",
|
||||
"PositionItem",
|
||||
"emit_payload",
|
||||
"is_anti_crawler",
|
||||
"is_attachment_name",
|
||||
"sanitize_body_text",
|
||||
]
|
||||
@@ -0,0 +1,56 @@
|
||||
from typing import Optional
|
||||
|
||||
ATTACHMENT_EXTENSIONS = (
|
||||
".pdf",
|
||||
".docx",
|
||||
".doc",
|
||||
".xlsx",
|
||||
".xls",
|
||||
".zip",
|
||||
".rar",
|
||||
".7z",
|
||||
".tar",
|
||||
".gz",
|
||||
)
|
||||
|
||||
ANTI_CRAWLER_KEYWORDS = (
|
||||
"环境异常",
|
||||
"完成验证",
|
||||
"访问过于频繁",
|
||||
"请进行验证",
|
||||
"滑动验证",
|
||||
"安全验证",
|
||||
"robot check",
|
||||
"access denied",
|
||||
)
|
||||
|
||||
MIN_BODY_LENGTH = 60
|
||||
MAX_ATTACHMENT_FILENAME_LENGTH = 120
|
||||
MAX_ANTI_CRAWLER_TEXT_LENGTH = 300
|
||||
|
||||
|
||||
def is_attachment_name(text: str) -> bool:
|
||||
s = text.strip()
|
||||
if not s or len(s) >= MAX_ATTACHMENT_FILENAME_LENGTH:
|
||||
return False
|
||||
return s.lower().endswith(ATTACHMENT_EXTENSIONS)
|
||||
|
||||
|
||||
def is_anti_crawler(text: str) -> bool:
|
||||
s = text.strip()
|
||||
if not s or len(s) >= MAX_ANTI_CRAWLER_TEXT_LENGTH:
|
||||
return False
|
||||
return any(kw in s for kw in ANTI_CRAWLER_KEYWORDS)
|
||||
|
||||
|
||||
def sanitize_body_text(raw_text: Optional[str]) -> Optional[str]:
|
||||
if not raw_text:
|
||||
return None
|
||||
cleaned = raw_text.strip()
|
||||
if len(cleaned) < MIN_BODY_LENGTH:
|
||||
return None
|
||||
if is_anti_crawler(cleaned):
|
||||
return None
|
||||
if is_attachment_name(cleaned):
|
||||
return None
|
||||
return cleaned
|
||||
@@ -0,0 +1,291 @@
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
from common.content import sanitize_body_text
|
||||
from common.fetcher import PoliteFetcher
|
||||
from common.protocol import (
|
||||
AnnouncementItem,
|
||||
AttachmentItem,
|
||||
CollectorMeta,
|
||||
IngestPayload,
|
||||
)
|
||||
from common.security import is_safe_url
|
||||
from selectolax.lexbor import LexborHTMLParser
|
||||
|
||||
|
||||
def crawl_channel(
|
||||
channel_cfg: dict,
|
||||
site_cfg: dict,
|
||||
template_cfg: dict,
|
||||
max_items: int = 10,
|
||||
) -> IngestPayload:
|
||||
source_id = channel_cfg["source_id"]
|
||||
name = channel_cfg["name"]
|
||||
province = site_cfg.get("province", "")
|
||||
city = site_cfg.get("city", "")
|
||||
start_url = channel_cfg["start_url"]
|
||||
start_host = urlsplit(start_url).netloc.split(":")[0].lower()
|
||||
allowed_domains = channel_cfg.get(
|
||||
"allowed_domains",
|
||||
site_cfg.get("allowed_domains", [start_host]),
|
||||
)
|
||||
|
||||
item_selector = channel_cfg.get(
|
||||
"item_selector", template_cfg.get("item_selector", "a")
|
||||
)
|
||||
url_pattern = channel_cfg.get(
|
||||
"url_pattern", template_cfg.get("url_pattern")
|
||||
)
|
||||
keywords = channel_cfg.get(
|
||||
"keywords",
|
||||
template_cfg.get(
|
||||
"keywords", ["招聘", "招考", "公告", "公示", "选拔", "录用", "选调"]
|
||||
),
|
||||
)
|
||||
title_selectors = channel_cfg.get(
|
||||
"title_selector",
|
||||
template_cfg.get("title_selector", "h1, h2, title"),
|
||||
)
|
||||
body_selectors = channel_cfg.get(
|
||||
"body_selector",
|
||||
template_cfg.get(
|
||||
"body_selector",
|
||||
"div#content, div.content, div.article, div.article-content, div.con",
|
||||
),
|
||||
)
|
||||
date_regex = channel_cfg.get(
|
||||
"date_regex",
|
||||
template_cfg.get("date_regex", r"(\d{4}[-/年]\d{1,2}[-/月]\d{1,2})"),
|
||||
)
|
||||
att_extensions = channel_cfg.get(
|
||||
"attachment_extensions",
|
||||
template_cfg.get(
|
||||
"attachment_extensions",
|
||||
[".xlsx", ".xls", ".pdf", ".docx", ".doc"],
|
||||
),
|
||||
)
|
||||
|
||||
fetcher = PoliteFetcher()
|
||||
start_time = time.time()
|
||||
error_msg = None
|
||||
items: list[AnnouncementItem] = []
|
||||
|
||||
try:
|
||||
html = fetcher.get_text(start_url, allowed_domains, start_host)
|
||||
tree = LexborHTMLParser(html)
|
||||
|
||||
candidates = []
|
||||
for elem in tree.css(item_selector):
|
||||
text = elem.text(strip=True)
|
||||
href = elem.attributes.get("href", "").strip()
|
||||
if not text or not href or href.startswith("javascript:"):
|
||||
continue
|
||||
|
||||
if url_pattern and not re.search(url_pattern, href):
|
||||
continue
|
||||
|
||||
if keywords and not any(k in text for k in keywords):
|
||||
continue
|
||||
|
||||
full_url = urljoin(start_url, href)
|
||||
if not is_safe_url(full_url, allowed_domains, start_host):
|
||||
continue
|
||||
candidates.append((text, full_url))
|
||||
|
||||
seen_urls = set()
|
||||
for i, (title, url) in enumerate(candidates):
|
||||
if url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
if i > 0:
|
||||
time.sleep(0.3)
|
||||
|
||||
body_text = None
|
||||
attachments = []
|
||||
pub_date = None
|
||||
|
||||
try:
|
||||
detail_html = fetcher.get_text(url, allowed_domains, start_host)
|
||||
detail_tree = LexborHTMLParser(detail_html)
|
||||
detail_tree.strip_tags(["script", "style", "noscript", "iframe", "svg"])
|
||||
for crumb in detail_tree.css(".breadcrumb, .nav, .crumbs, .location, .pos, .head_top, .footer, .header"):
|
||||
crumb.decompose()
|
||||
|
||||
title_node = detail_tree.css_first(title_selectors)
|
||||
if title_node:
|
||||
parsed_title = title_node.text(strip=True)
|
||||
site_name = site_cfg.get("name", "")
|
||||
if (
|
||||
len(parsed_title) >= 6
|
||||
and parsed_title != site_name
|
||||
and not any(k == parsed_title for k in [site_name, "人才网", "人民政府", "人事考试网", "人事考试中心"])
|
||||
):
|
||||
title = parsed_title
|
||||
|
||||
body_node = detail_tree.css_first(body_selectors)
|
||||
if body_node:
|
||||
extracted = body_node.text(strip=True)
|
||||
extracted = re.sub(
|
||||
r"^(?:首页|当前位置[：:]\s*首页|您当前所在的位置[：:]\s*首页)[^\n\r]{0,100}(?:>|»|/|·|正文|内容|详情)?\s*",
|
||||
"",
|
||||
extracted,
|
||||
)
|
||||
body_text = sanitize_body_text(extracted)
|
||||
|
||||
if body_text and date_regex:
|
||||
m = re.search(date_regex, body_text)
|
||||
if m:
|
||||
pub_date = (
|
||||
m.group(1)
|
||||
.replace("年", "-")
|
||||
.replace("月", "-")
|
||||
.replace("/", "-")
|
||||
)
|
||||
|
||||
seen_att_urls = set()
|
||||
for att_a in detail_tree.css("a"):
|
||||
if len(attachments) >= 50:
|
||||
break
|
||||
att_href = att_a.attributes.get("href", "").strip()
|
||||
att_text = att_a.text(strip=True)
|
||||
if any(att_href.lower().endswith(ext) for ext in att_extensions):
|
||||
full_att_url = urljoin(url, att_href)
|
||||
if (
|
||||
is_safe_url(full_att_url, allowed_domains, start_host)
|
||||
and full_att_url not in seen_att_urls
|
||||
):
|
||||
seen_att_urls.add(full_att_url)
|
||||
attachments.append(
|
||||
AttachmentItem(name=att_text or "附件", url=full_att_url)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not body_text and not attachments:
|
||||
continue
|
||||
|
||||
items.append(
|
||||
AnnouncementItem(
|
||||
url=url,
|
||||
title=title,
|
||||
kind="招聘" if "公示" not in title else "拟聘用",
|
||||
employer=site_cfg.get("employer", f"{name} / 用人单位"),
|
||||
employer_type=site_cfg.get("employer_type", "事业单位"),
|
||||
published_at=pub_date,
|
||||
region_hint=f"{province} {city}".strip(),
|
||||
body_text=body_text,
|
||||
attachments=attachments,
|
||||
positions=[],
|
||||
)
|
||||
)
|
||||
|
||||
if len(items) >= max_items:
|
||||
break
|
||||
|
||||
except Exception as exc:
|
||||
error_msg = str(exc)
|
||||
finally:
|
||||
fetcher.close()
|
||||
|
||||
elapsed_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
meta = CollectorMeta(
|
||||
name=f"{name}_declarative",
|
||||
runtime="declarative/v1",
|
||||
auth_status="none",
|
||||
duration_ms=elapsed_ms,
|
||||
error_message=error_msg,
|
||||
)
|
||||
|
||||
return IngestPayload(
|
||||
source_id=source_id,
|
||||
batch_id=f"{source_id}-{int(time.time())}",
|
||||
collector=meta,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
def load_declarative_collectors(
|
||||
config_path: Path | str, base_dir: Path | None = None
|
||||
) -> list[dict]:
|
||||
cfg_file = Path(config_path)
|
||||
with open(cfg_file, "rb") as f:
|
||||
cfg = tomllib.load(f)
|
||||
|
||||
site_cfg = cfg.get("site", {})
|
||||
template_cfg = cfg.get("template", {})
|
||||
if not template_cfg:
|
||||
template_cfg = {**cfg.get("list", {}), **cfg.get("detail", {})}
|
||||
|
||||
site_id = site_cfg.get("id", f"{cfg_file.parent.name}.{cfg_file.stem}")
|
||||
site_name = site_cfg.get("name", site_id)
|
||||
province = site_cfg.get("province", cfg_file.parent.name)
|
||||
city = site_cfg.get("city", "")
|
||||
site_tags = site_cfg.get("tags", [])
|
||||
|
||||
channels = cfg.get("channels", [])
|
||||
if not channels and "start_url" in site_cfg:
|
||||
channels = [
|
||||
{
|
||||
"id": "",
|
||||
"name": site_name,
|
||||
"start_url": site_cfg["start_url"],
|
||||
"tags": site_tags,
|
||||
}
|
||||
]
|
||||
|
||||
rel_file = (
|
||||
str(cfg_file.relative_to(base_dir)) if base_dir else str(cfg_file)
|
||||
)
|
||||
|
||||
result = []
|
||||
for ch in channels:
|
||||
ch_id = ch.get("id", "").strip()
|
||||
full_source_id = f"{site_id}.{ch_id}" if ch_id else site_id
|
||||
ch_name = ch.get("name", ch_id)
|
||||
full_name = f"{site_name} - {ch_name}" if ch_id and ch_name != site_name else site_name
|
||||
ch_tags = list(set(site_tags + ch.get("tags", [])))
|
||||
|
||||
channel_entry = dict(ch)
|
||||
channel_entry["source_id"] = full_source_id
|
||||
channel_entry["name"] = full_name
|
||||
channel_entry["start_url"] = ch.get("start_url", site_cfg.get("start_url", ""))
|
||||
|
||||
if not channel_entry["start_url"]:
|
||||
continue
|
||||
|
||||
domain = urlsplit(channel_entry["start_url"]).netloc.lower()
|
||||
meta = {
|
||||
"source_id": full_source_id,
|
||||
"name": full_name,
|
||||
"province": province,
|
||||
"city": city,
|
||||
"tags": ch_tags,
|
||||
"domain": domain,
|
||||
}
|
||||
|
||||
crawl_fn = (
|
||||
lambda ch_e=channel_entry, s_c=site_cfg, t_c=template_cfg: (
|
||||
lambda max_items=10: crawl_channel(
|
||||
ch_e, s_c, t_c, max_items=max_items
|
||||
)
|
||||
)
|
||||
)()
|
||||
|
||||
result.append({
|
||||
"meta": meta,
|
||||
"crawl_fn": crawl_fn,
|
||||
"file": rel_file,
|
||||
"type": "toml",
|
||||
"domain": domain,
|
||||
})
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,61 @@
|
||||
from typing import Optional
|
||||
from urllib.parse import urljoin
|
||||
import httpx
|
||||
from common.security import is_safe_url
|
||||
|
||||
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 bangwen/1.0"
|
||||
MAX_REDIRECTS = 5
|
||||
|
||||
|
||||
class PoliteFetcher:
|
||||
def __init__(self, timeout: float = 15.0, verify: bool | str = True):
|
||||
self.headers = {
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
self.client = httpx.Client(
|
||||
headers=self.headers,
|
||||
timeout=timeout,
|
||||
follow_redirects=False,
|
||||
verify=verify,
|
||||
)
|
||||
|
||||
def get_text(
|
||||
self,
|
||||
url: str,
|
||||
allowed_domains: Optional[list[str]] = None,
|
||||
start_host: Optional[str] = None,
|
||||
) -> str:
|
||||
current_url = url
|
||||
for _ in range(MAX_REDIRECTS):
|
||||
if not is_safe_url(current_url, allowed_domains, start_host):
|
||||
raise ValueError(f"Unsafe request URL blocked: {current_url}")
|
||||
|
||||
resp = self.client.get(current_url)
|
||||
if resp.is_redirect and "location" in resp.headers:
|
||||
next_url = urljoin(current_url, resp.headers["location"])
|
||||
current_url = next_url
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
|
||||
content_type = resp.headers.get("content-type", "").lower()
|
||||
if (
|
||||
"gbk" in content_type
|
||||
or "gb2312" in content_type
|
||||
or "gb18030" in content_type
|
||||
):
|
||||
return resp.content.decode("gb18030", errors="replace")
|
||||
|
||||
try:
|
||||
return resp.text
|
||||
except UnicodeDecodeError:
|
||||
return resp.content.decode("gb18030", errors="replace")
|
||||
|
||||
raise httpx.TooManyRedirects(
|
||||
f"Exceeded max redirects ({MAX_REDIRECTS}) for {url}"
|
||||
)
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,101 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class AttachmentItem(BaseModel):
|
||||
name: str | None = None
|
||||
url: str
|
||||
mime: str | None = None
|
||||
|
||||
|
||||
class PositionItem(BaseModel):
|
||||
post_code: str | None = None
|
||||
dept: str | None = None
|
||||
post_name: str
|
||||
headcount: int | None = None
|
||||
education_min: str | None = None
|
||||
degree_required: bool | None = None
|
||||
majors: list[str] | None = None
|
||||
major_unlimited: bool | None = False
|
||||
age_max: int | None = None
|
||||
bianzhi: str | None = None
|
||||
salary_min: int | None = None
|
||||
salary_max: int | None = None
|
||||
work_city: str | None = None
|
||||
other_req: str | None = None
|
||||
extract_method: str = "edge_python"
|
||||
confidence: float | None = 1.0
|
||||
|
||||
|
||||
class AnnouncementItem(BaseModel):
|
||||
url: str
|
||||
url_hash: str | None = None
|
||||
title: str
|
||||
kind: str = "招聘"
|
||||
employer: str | None = None
|
||||
employer_type: str | None = None
|
||||
published_at: str | None = None
|
||||
apply_start: str | None = None
|
||||
apply_end: str | None = None
|
||||
region_hint: str | None = None
|
||||
body_text: str | None = None
|
||||
attachments: list[AttachmentItem] = Field(default_factory=list)
|
||||
positions: list[PositionItem] = Field(default_factory=list)
|
||||
reconcile_positions: bool | None = None
|
||||
|
||||
|
||||
class CollectorMeta(BaseModel):
|
||||
name: str
|
||||
runtime: str | None = "python/3.12"
|
||||
auth_status: str = "none"
|
||||
duration_ms: int | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
class IngestPayload(BaseModel):
|
||||
protocol_version: str = "bangwen/v1"
|
||||
source_id: str
|
||||
batch_id: str
|
||||
collector: CollectorMeta | None = None
|
||||
items: list[AnnouncementItem]
|
||||
|
||||
|
||||
def emit_payload(
|
||||
payload: IngestPayload,
|
||||
push_url: str | None = None,
|
||||
token: str | None = None,
|
||||
) -> None:
|
||||
data = payload.model_dump(mode="json")
|
||||
if push_url:
|
||||
token = token or os.environ.get("INGEST_TOKEN")
|
||||
headers = {}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token.strip()}"
|
||||
max_retries = 3
|
||||
last_exc = None
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
resp = httpx.post(push_url, json=data, headers=headers, timeout=30.0)
|
||||
resp.raise_for_status()
|
||||
sys.stderr.write(
|
||||
f"Pushed {len(payload.items)} items to {push_url}: {resp.text}\n"
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
sys.stderr.write(
|
||||
f"Attempt {attempt}/{max_retries} failed to push to {push_url}: {e}\n"
|
||||
)
|
||||
if attempt < max_retries:
|
||||
time.sleep(2 ** attempt)
|
||||
if last_exc:
|
||||
raise last_exc
|
||||
else:
|
||||
json_output = json.dumps(data, ensure_ascii=False)
|
||||
sys.stdout.write(json_output + "\n")
|
||||
sys.stdout.flush()
|
||||
@@ -0,0 +1,35 @@
|
||||
import ipaddress
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
def is_safe_url(
|
||||
url: str,
|
||||
allowed_domains: list[str] | None = None,
|
||||
default_host: str = "",
|
||||
) -> bool:
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
if parts.scheme not in ("http", "https"):
|
||||
return False
|
||||
|
||||
host = (parts.hostname or "").lower()
|
||||
if not host or host in ("localhost", "127.0.0.1", "::1"):
|
||||
return False
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
domains = [d.lower() for d in (allowed_domains or []) if d]
|
||||
if not domains and default_host:
|
||||
domains = [default_host.lower()]
|
||||
|
||||
if not domains:
|
||||
return True
|
||||
|
||||
return any(host == d or host.endswith("." + d) for d in domains)
|
||||
except Exception:
|
||||
return False
|
||||
Reference in New Issue
Block a user