bitfennec
1059be1b62
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
62 lines
2.0 KiB
Python
62 lines
2.0 KiB
Python
from typing import Optional
|
|
from urllib.parse import urljoin
|
|
import httpx
|
|
from common.security import is_safe_url
|
|
|
|
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 bangwen/1.0"
|
|
MAX_REDIRECTS = 5
|
|
|
|
|
|
class PoliteFetcher:
|
|
def __init__(self, timeout: float = 15.0, verify: bool | str = True):
|
|
self.headers = {
|
|
"User-Agent": DEFAULT_USER_AGENT,
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
}
|
|
self.client = httpx.Client(
|
|
headers=self.headers,
|
|
timeout=timeout,
|
|
follow_redirects=False,
|
|
verify=verify,
|
|
)
|
|
|
|
def get_text(
|
|
self,
|
|
url: str,
|
|
allowed_domains: Optional[list[str]] = None,
|
|
start_host: Optional[str] = None,
|
|
) -> str:
|
|
current_url = url
|
|
for _ in range(MAX_REDIRECTS):
|
|
if not is_safe_url(current_url, allowed_domains, start_host):
|
|
raise ValueError(f"Unsafe request URL blocked: {current_url}")
|
|
|
|
resp = self.client.get(current_url)
|
|
if resp.is_redirect and "location" in resp.headers:
|
|
next_url = urljoin(current_url, resp.headers["location"])
|
|
current_url = next_url
|
|
continue
|
|
|
|
resp.raise_for_status()
|
|
|
|
content_type = resp.headers.get("content-type", "").lower()
|
|
if (
|
|
"gbk" in content_type
|
|
or "gb2312" in content_type
|
|
or "gb18030" in content_type
|
|
):
|
|
return resp.content.decode("gb18030", errors="replace")
|
|
|
|
try:
|
|
return resp.text
|
|
except UnicodeDecodeError:
|
|
return resp.content.decode("gb18030", errors="replace")
|
|
|
|
raise httpx.TooManyRedirects(
|
|
f"Exceeded max redirects ({MAX_REDIRECTS}) for {url}"
|
|
)
|
|
|
|
def close(self):
|
|
self.client.close()
|