from typing import Optional ATTACHMENT_EXTENSIONS = ( ".pdf", ".docx", ".doc", ".xlsx", ".xls", ".zip", ".rar", ".7z", ".tar", ".gz", ) ANTI_CRAWLER_KEYWORDS = ( "环境异常", "完成验证", "访问过于频繁", "请进行验证", "滑动验证", "安全验证", "robot check", "access denied", ) MIN_BODY_LENGTH = 60 MAX_ATTACHMENT_FILENAME_LENGTH = 120 MAX_ANTI_CRAWLER_TEXT_LENGTH = 300 def is_attachment_name(text: str) -> bool: s = text.strip() if not s or len(s) >= MAX_ATTACHMENT_FILENAME_LENGTH: return False return s.lower().endswith(ATTACHMENT_EXTENSIONS) def is_anti_crawler(text: str) -> bool: s = text.strip() if not s or len(s) >= MAX_ANTI_CRAWLER_TEXT_LENGTH: return False return any(kw in s for kw in ANTI_CRAWLER_KEYWORDS) def sanitize_body_text(raw_text: Optional[str]) -> Optional[str]: if not raw_text: return None cleaned = raw_text.strip() if len(cleaned) < MIN_BODY_LENGTH: return None if is_anti_crawler(cleaned): return None if is_attachment_name(cleaned): return None return cleaned