feat: initialize bangwen recruitment aggregation and search engine

- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
2026-10-23 23:44:10 +08:00
commit 1059be1b62
61 changed files with 10489 additions and 0 deletions
@@ -0,0 +1,61 @@
+from typing import Optional
+from urllib.parse import urljoin
+import httpx
+from common.security import is_safe_url
+
+DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 bangwen/1.0"
+MAX_REDIRECTS = 5
+
+
+class PoliteFetcher:
+    def __init__(self, timeout: float = 15.0, verify: bool | str = True):
+        self.headers = {
+            "User-Agent": DEFAULT_USER_AGENT,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+        }
+        self.client = httpx.Client(
+            headers=self.headers,
+            timeout=timeout,
+            follow_redirects=False,
+            verify=verify,
+        )
+
+    def get_text(
+        self,
+        url: str,
+        allowed_domains: Optional[list[str]] = None,
+        start_host: Optional[str] = None,
+    ) -> str:
+        current_url = url
+        for _ in range(MAX_REDIRECTS):
+            if not is_safe_url(current_url, allowed_domains, start_host):
+                raise ValueError(f"Unsafe request URL blocked: {current_url}")
+
+            resp = self.client.get(current_url)
+            if resp.is_redirect and "location" in resp.headers:
+                next_url = urljoin(current_url, resp.headers["location"])
+                current_url = next_url
+                continue
+
+            resp.raise_for_status()
+
+            content_type = resp.headers.get("content-type", "").lower()
+            if (
+                "gbk" in content_type
+                or "gb2312" in content_type
+                or "gb18030" in content_type
+            ):
+                return resp.content.decode("gb18030", errors="replace")
+
+            try:
+                return resp.text
+            except UnicodeDecodeError:
+                return resp.content.decode("gb18030", errors="replace")
+
+        raise httpx.TooManyRedirects(
+            f"Exceeded max redirects ({MAX_REDIRECTS}) for {url}"
+        )
+
+    def close(self):
+        self.client.close()