bitfennec
1059be1b62
- Rust core engine with HTTP ingestion gateway, hybrid search, and white-box resume matching - Python edge collectors supporting declarative TOML and specialized adapters - PostgreSQL schema migrations with pgvector embedding support - Hardened security controls: strict CSP, terminal sanitization, and input boundary validation - Multi-service container orchestration with dedicated non-root execution
114 lines
3.5 KiB
SQL
114 lines
3.5 KiB
SQL
CREATE EXTENSION IF NOT EXISTS vector;
|
|
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
|
|
|
CREATE TABLE IF NOT EXISTS source (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
tier SMALLINT NOT NULL DEFAULT 1,
|
|
region_code CHAR(6),
|
|
collector_type TEXT NOT NULL DEFAULT 'python_script',
|
|
auth_mode TEXT NOT NULL DEFAULT 'none',
|
|
auth_status TEXT NOT NULL DEFAULT 'none',
|
|
health TEXT NOT NULL DEFAULT 'ok',
|
|
last_ingest_at TIMESTAMPTZ,
|
|
last_count INT,
|
|
fail_streak INT NOT NULL DEFAULT 0,
|
|
config JSONB NOT NULL DEFAULT '{}'::jsonb
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS raw_page (
|
|
sha256 CHAR(64) PRIMARY KEY,
|
|
url TEXT NOT NULL,
|
|
status INT,
|
|
content_type TEXT,
|
|
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS announcement (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
source_id TEXT REFERENCES source(id),
|
|
url TEXT NOT NULL UNIQUE,
|
|
url_hash CHAR(16) NOT NULL,
|
|
content_hash CHAR(16),
|
|
simhash BIGINT,
|
|
canonical_id BIGINT REFERENCES announcement(id),
|
|
title TEXT NOT NULL,
|
|
kind TEXT NOT NULL DEFAULT 'job_posting',
|
|
employer TEXT,
|
|
employer_type TEXT,
|
|
province_code CHAR(6),
|
|
city_code CHAR(6),
|
|
county_code CHAR(6),
|
|
published_at DATE,
|
|
apply_start TIMESTAMPTZ,
|
|
apply_end TIMESTAMPTZ,
|
|
body_text TEXT,
|
|
raw_sha256 CHAR(64) REFERENCES raw_page(sha256),
|
|
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_announcement_kind_published ON announcement (kind, published_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_announcement_city_apply_end ON announcement (city_code, apply_end);
|
|
CREATE INDEX IF NOT EXISTS idx_announcement_title_trgm ON announcement USING gin (title gin_trgm_ops);
|
|
|
|
CREATE TABLE IF NOT EXISTS attachment (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
announcement_id BIGINT NOT NULL REFERENCES announcement(id) ON DELETE CASCADE,
|
|
url TEXT NOT NULL,
|
|
filename TEXT,
|
|
mime TEXT,
|
|
sha256 CHAR(64),
|
|
parse_status TEXT NOT NULL DEFAULT 'pending',
|
|
parser TEXT,
|
|
error TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS position (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
announcement_id BIGINT NOT NULL REFERENCES announcement(id) ON DELETE CASCADE,
|
|
attachment_id BIGINT REFERENCES attachment(id),
|
|
post_code TEXT,
|
|
dept TEXT,
|
|
post_name TEXT NOT NULL,
|
|
headcount INT,
|
|
education TEXT,
|
|
edu_min SMALLINT,
|
|
degree_required BOOLEAN,
|
|
majors_raw TEXT,
|
|
majors TEXT[],
|
|
major_unlimited BOOLEAN NOT NULL DEFAULT false,
|
|
age_max SMALLINT,
|
|
bianzhi TEXT,
|
|
salary_min INT,
|
|
salary_max INT,
|
|
work_city_code CHAR(6),
|
|
other_req TEXT,
|
|
extract_method TEXT NOT NULL DEFAULT 'edge_python',
|
|
confidence REAL,
|
|
raw JSONB
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_position_city_edu_age ON position (work_city_code, edu_min, age_max);
|
|
CREATE INDEX IF NOT EXISTS idx_position_majors ON position USING gin (majors);
|
|
|
|
CREATE TABLE IF NOT EXISTS embedding (
|
|
owner_type TEXT NOT NULL,
|
|
owner_id BIGINT NOT NULL,
|
|
model TEXT NOT NULL,
|
|
chunk SMALLINT NOT NULL DEFAULT 0,
|
|
text_hash CHAR(16) NOT NULL,
|
|
vec vector(1024) NOT NULL,
|
|
PRIMARY KEY (owner_type, owner_id, model, chunk)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_embedding_vec_hnsw ON embedding USING hnsw (vec vector_cosine_ops);
|
|
|
|
CREATE TABLE IF NOT EXISTS notify_sent (
|
|
channel TEXT NOT NULL,
|
|
owner_type TEXT NOT NULL,
|
|
owner_id BIGINT NOT NULL,
|
|
sent_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (channel, owner_type, owner_id)
|
|
);
|