bitfennec
1059be1b62
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
102 lines
2.8 KiB
Python
102 lines
2.8 KiB
Python
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import httpx
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class AttachmentItem(BaseModel):
|
|
name: str | None = None
|
|
url: str
|
|
mime: str | None = None
|
|
|
|
|
|
class PositionItem(BaseModel):
|
|
post_code: str | None = None
|
|
dept: str | None = None
|
|
post_name: str
|
|
headcount: int | None = None
|
|
education_min: str | None = None
|
|
degree_required: bool | None = None
|
|
majors: list[str] | None = None
|
|
major_unlimited: bool | None = False
|
|
age_max: int | None = None
|
|
bianzhi: str | None = None
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
work_city: str | None = None
|
|
other_req: str | None = None
|
|
extract_method: str = "edge_python"
|
|
confidence: float | None = 1.0
|
|
|
|
|
|
class AnnouncementItem(BaseModel):
|
|
url: str
|
|
url_hash: str | None = None
|
|
title: str
|
|
kind: str = "招聘"
|
|
employer: str | None = None
|
|
employer_type: str | None = None
|
|
published_at: str | None = None
|
|
apply_start: str | None = None
|
|
apply_end: str | None = None
|
|
region_hint: str | None = None
|
|
body_text: str | None = None
|
|
attachments: list[AttachmentItem] = Field(default_factory=list)
|
|
positions: list[PositionItem] = Field(default_factory=list)
|
|
reconcile_positions: bool | None = None
|
|
|
|
|
|
class CollectorMeta(BaseModel):
|
|
name: str
|
|
runtime: str | None = "python/3.12"
|
|
auth_status: str = "none"
|
|
duration_ms: int | None = None
|
|
error_message: str | None = None
|
|
|
|
|
|
class IngestPayload(BaseModel):
|
|
protocol_version: str = "bangwen/v1"
|
|
source_id: str
|
|
batch_id: str
|
|
collector: CollectorMeta | None = None
|
|
items: list[AnnouncementItem]
|
|
|
|
|
|
def emit_payload(
|
|
payload: IngestPayload,
|
|
push_url: str | None = None,
|
|
token: str | None = None,
|
|
) -> None:
|
|
data = payload.model_dump(mode="json")
|
|
if push_url:
|
|
token = token or os.environ.get("INGEST_TOKEN")
|
|
headers = {}
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token.strip()}"
|
|
max_retries = 3
|
|
last_exc = None
|
|
for attempt in range(1, max_retries + 1):
|
|
try:
|
|
resp = httpx.post(push_url, json=data, headers=headers, timeout=30.0)
|
|
resp.raise_for_status()
|
|
sys.stderr.write(
|
|
f"Pushed {len(payload.items)} items to {push_url}: {resp.text}\n"
|
|
)
|
|
return
|
|
except Exception as e:
|
|
last_exc = e
|
|
sys.stderr.write(
|
|
f"Attempt {attempt}/{max_retries} failed to push to {push_url}: {e}\n"
|
|
)
|
|
if attempt < max_retries:
|
|
time.sleep(2 ** attempt)
|
|
if last_exc:
|
|
raise last_exc
|
|
else:
|
|
json_output = json.dumps(data, ensure_ascii=False)
|
|
sys.stdout.write(json_output + "\n")
|
|
sys.stdout.flush()
|