feat: initialize bangwen recruitment aggregation and search engine

- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
2026-10-23 23:44:10 +08:00
commit 1059be1b62
61 changed files with 10489 additions and 0 deletions
@@ -0,0 +1,113 @@
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+CREATE TABLE IF NOT EXISTS source (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    tier SMALLINT NOT NULL DEFAULT 1,
+    region_code CHAR(6),
+    collector_type TEXT NOT NULL DEFAULT 'python_script',
+    auth_mode TEXT NOT NULL DEFAULT 'none',
+    auth_status TEXT NOT NULL DEFAULT 'none',
+    health TEXT NOT NULL DEFAULT 'ok',
+    last_ingest_at TIMESTAMPTZ,
+    last_count INT,
+    fail_streak INT NOT NULL DEFAULT 0,
+    config JSONB NOT NULL DEFAULT '{}'::jsonb
+);
+
+CREATE TABLE IF NOT EXISTS raw_page (
+    sha256 CHAR(64) PRIMARY KEY,
+    url TEXT NOT NULL,
+    status INT,
+    content_type TEXT,
+    fetched_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS announcement (
+    id BIGSERIAL PRIMARY KEY,
+    source_id TEXT REFERENCES source(id),
+    url TEXT NOT NULL UNIQUE,
+    url_hash CHAR(16) NOT NULL,
+    content_hash CHAR(16),
+    simhash BIGINT,
+    canonical_id BIGINT REFERENCES announcement(id),
+    title TEXT NOT NULL,
+    kind TEXT NOT NULL DEFAULT 'job_posting',
+    employer TEXT,
+    employer_type TEXT,
+    province_code CHAR(6),
+    city_code CHAR(6),
+    county_code CHAR(6),
+    published_at DATE,
+    apply_start TIMESTAMPTZ,
+    apply_end TIMESTAMPTZ,
+    body_text TEXT,
+    raw_sha256 CHAR(64) REFERENCES raw_page(sha256),
+    first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_announcement_kind_published ON announcement (kind, published_at DESC);
+CREATE INDEX IF NOT EXISTS idx_announcement_city_apply_end ON announcement (city_code, apply_end);
+CREATE INDEX IF NOT EXISTS idx_announcement_title_trgm ON announcement USING gin (title gin_trgm_ops);
+
+CREATE TABLE IF NOT EXISTS attachment (
+    id BIGSERIAL PRIMARY KEY,
+    announcement_id BIGINT NOT NULL REFERENCES announcement(id) ON DELETE CASCADE,
+    url TEXT NOT NULL,
+    filename TEXT,
+    mime TEXT,
+    sha256 CHAR(64),
+    parse_status TEXT NOT NULL DEFAULT 'pending',
+    parser TEXT,
+    error TEXT
+);
+
+CREATE TABLE IF NOT EXISTS position (
+    id BIGSERIAL PRIMARY KEY,
+    announcement_id BIGINT NOT NULL REFERENCES announcement(id) ON DELETE CASCADE,
+    attachment_id BIGINT REFERENCES attachment(id),
+    post_code TEXT,
+    dept TEXT,
+    post_name TEXT NOT NULL,
+    headcount INT,
+    education TEXT,
+    edu_min SMALLINT,
+    degree_required BOOLEAN,
+    majors_raw TEXT,
+    majors TEXT[],
+    major_unlimited BOOLEAN NOT NULL DEFAULT false,
+    age_max SMALLINT,
+    bianzhi TEXT,
+    salary_min INT,
+    salary_max INT,
+    work_city_code CHAR(6),
+    other_req TEXT,
+    extract_method TEXT NOT NULL DEFAULT 'edge_python',
+    confidence REAL,
+    raw JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_position_city_edu_age ON position (work_city_code, edu_min, age_max);
+CREATE INDEX IF NOT EXISTS idx_position_majors ON position USING gin (majors);
+
+CREATE TABLE IF NOT EXISTS embedding (
+    owner_type TEXT NOT NULL,
+    owner_id BIGINT NOT NULL,
+    model TEXT NOT NULL,
+    chunk SMALLINT NOT NULL DEFAULT 0,
+    text_hash CHAR(16) NOT NULL,
+    vec vector(1024) NOT NULL,
+    PRIMARY KEY (owner_type, owner_id, model, chunk)
+);
+
+CREATE INDEX IF NOT EXISTS idx_embedding_vec_hnsw ON embedding USING hnsw (vec vector_cosine_ops);
+
+CREATE TABLE IF NOT EXISTS notify_sent (
+    channel TEXT NOT NULL,
+    owner_type TEXT NOT NULL,
+    owner_id BIGINT NOT NULL,
+    sent_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (channel, owner_type, owner_id)
+);