bitfennec
1059be1b62
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
57 lines
1.2 KiB
Python
57 lines
1.2 KiB
Python
from typing import Optional
|
|
|
|
ATTACHMENT_EXTENSIONS = (
|
|
".pdf",
|
|
".docx",
|
|
".doc",
|
|
".xlsx",
|
|
".xls",
|
|
".zip",
|
|
".rar",
|
|
".7z",
|
|
".tar",
|
|
".gz",
|
|
)
|
|
|
|
ANTI_CRAWLER_KEYWORDS = (
|
|
"环境异常",
|
|
"完成验证",
|
|
"访问过于频繁",
|
|
"请进行验证",
|
|
"滑动验证",
|
|
"安全验证",
|
|
"robot check",
|
|
"access denied",
|
|
)
|
|
|
|
MIN_BODY_LENGTH = 60
|
|
MAX_ATTACHMENT_FILENAME_LENGTH = 120
|
|
MAX_ANTI_CRAWLER_TEXT_LENGTH = 300
|
|
|
|
|
|
def is_attachment_name(text: str) -> bool:
|
|
s = text.strip()
|
|
if not s or len(s) >= MAX_ATTACHMENT_FILENAME_LENGTH:
|
|
return False
|
|
return s.lower().endswith(ATTACHMENT_EXTENSIONS)
|
|
|
|
|
|
def is_anti_crawler(text: str) -> bool:
|
|
s = text.strip()
|
|
if not s or len(s) >= MAX_ANTI_CRAWLER_TEXT_LENGTH:
|
|
return False
|
|
return any(kw in s for kw in ANTI_CRAWLER_KEYWORDS)
|
|
|
|
|
|
def sanitize_body_text(raw_text: Optional[str]) -> Optional[str]:
|
|
if not raw_text:
|
|
return None
|
|
cleaned = raw_text.strip()
|
|
if len(cleaned) < MIN_BODY_LENGTH:
|
|
return None
|
|
if is_anti_crawler(cleaned):
|
|
return None
|
|
if is_attachment_name(cleaned):
|
|
return None
|
|
return cleaned
|