feat: initialize bangwen recruitment aggregation and search engine

- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
2026-10-23 23:44:10 +08:00
commit 1059be1b62
61 changed files with 10489 additions and 0 deletions
@@ -0,0 +1,113 @@
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+CREATE TABLE IF NOT EXISTS source (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    tier SMALLINT NOT NULL DEFAULT 1,
+    region_code CHAR(6),
+    collector_type TEXT NOT NULL DEFAULT 'python_script',
+    auth_mode TEXT NOT NULL DEFAULT 'none',
+    auth_status TEXT NOT NULL DEFAULT 'none',
+    health TEXT NOT NULL DEFAULT 'ok',
+    last_ingest_at TIMESTAMPTZ,
+    last_count INT,
+    fail_streak INT NOT NULL DEFAULT 0,
+    config JSONB NOT NULL DEFAULT '{}'::jsonb
+);
+
+CREATE TABLE IF NOT EXISTS raw_page (
+    sha256 CHAR(64) PRIMARY KEY,
+    url TEXT NOT NULL,
+    status INT,
+    content_type TEXT,
+    fetched_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS announcement (
+    id BIGSERIAL PRIMARY KEY,
+    source_id TEXT REFERENCES source(id),
+    url TEXT NOT NULL UNIQUE,
+    url_hash CHAR(16) NOT NULL,
+    content_hash CHAR(16),
+    simhash BIGINT,
+    canonical_id BIGINT REFERENCES announcement(id),
+    title TEXT NOT NULL,
+    kind TEXT NOT NULL DEFAULT 'job_posting',
+    employer TEXT,
+    employer_type TEXT,
+    province_code CHAR(6),
+    city_code CHAR(6),
+    county_code CHAR(6),
+    published_at DATE,
+    apply_start TIMESTAMPTZ,
+    apply_end TIMESTAMPTZ,
+    body_text TEXT,
+    raw_sha256 CHAR(64) REFERENCES raw_page(sha256),
+    first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_announcement_kind_published ON announcement (kind, published_at DESC);
+CREATE INDEX IF NOT EXISTS idx_announcement_city_apply_end ON announcement (city_code, apply_end);
+CREATE INDEX IF NOT EXISTS idx_announcement_title_trgm ON announcement USING gin (title gin_trgm_ops);
+
+CREATE TABLE IF NOT EXISTS attachment (
+    id BIGSERIAL PRIMARY KEY,
+    announcement_id BIGINT NOT NULL REFERENCES announcement(id) ON DELETE CASCADE,
+    url TEXT NOT NULL,
+    filename TEXT,
+    mime TEXT,
+    sha256 CHAR(64),
+    parse_status TEXT NOT NULL DEFAULT 'pending',
+    parser TEXT,
+    error TEXT
+);
+
+CREATE TABLE IF NOT EXISTS position (
+    id BIGSERIAL PRIMARY KEY,
+    announcement_id BIGINT NOT NULL REFERENCES announcement(id) ON DELETE CASCADE,
+    attachment_id BIGINT REFERENCES attachment(id),
+    post_code TEXT,
+    dept TEXT,
+    post_name TEXT NOT NULL,
+    headcount INT,
+    education TEXT,
+    edu_min SMALLINT,
+    degree_required BOOLEAN,
+    majors_raw TEXT,
+    majors TEXT[],
+    major_unlimited BOOLEAN NOT NULL DEFAULT false,
+    age_max SMALLINT,
+    bianzhi TEXT,
+    salary_min INT,
+    salary_max INT,
+    work_city_code CHAR(6),
+    other_req TEXT,
+    extract_method TEXT NOT NULL DEFAULT 'edge_python',
+    confidence REAL,
+    raw JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_position_city_edu_age ON position (work_city_code, edu_min, age_max);
+CREATE INDEX IF NOT EXISTS idx_position_majors ON position USING gin (majors);
+
+CREATE TABLE IF NOT EXISTS embedding (
+    owner_type TEXT NOT NULL,
+    owner_id BIGINT NOT NULL,
+    model TEXT NOT NULL,
+    chunk SMALLINT NOT NULL DEFAULT 0,
+    text_hash CHAR(16) NOT NULL,
+    vec vector(1024) NOT NULL,
+    PRIMARY KEY (owner_type, owner_id, model, chunk)
+);
+
+CREATE INDEX IF NOT EXISTS idx_embedding_vec_hnsw ON embedding USING hnsw (vec vector_cosine_ops);
+
+CREATE TABLE IF NOT EXISTS notify_sent (
+    channel TEXT NOT NULL,
+    owner_type TEXT NOT NULL,
+    owner_id BIGINT NOT NULL,
+    sent_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (channel, owner_type, owner_id)
+);
@@ -0,0 +1,32 @@
+-- Migration: Fix deduplication constraints and clean up duplicate rows
+
+-- 1. Deduplicate attachments
+DELETE FROM attachment a
+USING attachment b
+WHERE a.announcement_id = b.announcement_id
+  AND a.url = b.url
+  AND a.id > b.id;
+
+-- 2. Unique constraint for attachment idempotency
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_constraint WHERE conname = 'uq_attachment_announcement_url'
+    ) THEN
+        ALTER TABLE attachment ADD CONSTRAINT uq_attachment_announcement_url UNIQUE (announcement_id, url);
+    END IF;
+END $$;
+
+-- 3. Fix invalid self-referencing canonical_id
+UPDATE announcement SET canonical_id = NULL WHERE canonical_id = id;
+
+-- 4. Clean duplicate positions and add unique index
+DELETE FROM position a
+USING position b
+WHERE a.announcement_id = b.announcement_id
+  AND a.post_name = b.post_name
+  AND COALESCE(a.dept, '') = COALESCE(b.dept, '')
+  AND a.id > b.id;
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_position_announcement_post_dept 
+ON position (announcement_id, post_name, COALESCE(dept, ''));
@@ -0,0 +1,26 @@
+-- Migration: Clean up excessive attachments, dirty bodies, and orphan test sources
+
+-- 1. Cap excessive historical attachments per announcement (keep newest 50 per announcement)
+DELETE FROM attachment
+WHERE id IN (
+    SELECT id FROM (
+        SELECT id, ROW_NUMBER() OVER (PARTITION BY announcement_id ORDER BY id DESC) AS rn
+        FROM attachment
+    ) t
+    WHERE t.rn > 50
+)
+AND id NOT IN (SELECT attachment_id FROM "position" WHERE attachment_id IS NOT NULL);
+
+-- 2. Clean dirty, empty, or anti-crawler body texts and clear associated hashes
+UPDATE announcement 
+SET body_text = NULL, simhash = NULL, content_hash = NULL 
+WHERE body_text = '' 
+   OR (LENGTH(body_text) < 120 AND body_text ~* '\.(pdf|docx?|xlsx?|zip)$')
+   OR LENGTH(body_text) < 60
+   OR body_text LIKE '%环境异常%' 
+   OR body_text LIKE '%完成验证%';
+
+-- 3. Delete orphan test sources with 0 announcements
+DELETE FROM source 
+WHERE id IN ('test_auth', 'yn.ynrsksw.special8')
+  AND NOT EXISTS (SELECT 1 FROM announcement WHERE announcement.source_id = source.id);