feat: Add v2 architecture with multi-state support and orchestrator services
Major additions: - Multi-state expansion: states table, StateSelector, NationalDashboard, StateHeatmap, CrossStateCompare - Orchestrator services: trace service, error taxonomy, retry manager, proxy rotator - Discovery system: dutchie discovery service, geo validation, city seeding scripts - Analytics infrastructure: analytics v2 routes, brand/pricing/stores intelligence pages - Local development: setup-local.sh starts all 5 services (postgres, backend, cannaiq, findadispo, findagram) - Migrations 037-056: crawler profiles, states, analytics indexes, worker metadata Frontend pages added: - Discovery, ChainsDashboard, IntelligenceBrands, IntelligencePricing, IntelligenceStores - StateHeatmap, CrossStateCompare, SyncInfoPanel Components added: - StateSelector, OrchestratorTraceModal, WorkflowStepper 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
@@ -1,23 +1,63 @@
|
||||
/**
|
||||
* Database Migration Script (CLI-ONLY)
|
||||
*
|
||||
* This file is for running migrations via CLI only:
|
||||
* npx tsx src/db/migrate.ts
|
||||
*
|
||||
* DO NOT import this file from runtime code.
|
||||
* Runtime code should import from src/db/pool.ts instead.
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
// Consolidated DB connection:
|
||||
// - Prefer CRAWLSY_DATABASE_URL (e.g., crawlsy_local, crawlsy_prod)
|
||||
// - Then DATABASE_URL (default)
|
||||
const DATABASE_URL =
|
||||
process.env.CRAWLSY_DATABASE_URL ||
|
||||
process.env.DATABASE_URL ||
|
||||
'postgresql://dutchie:dutchie_local_pass@localhost:54320/crawlsy_local';
|
||||
// Load .env BEFORE any env var access
|
||||
dotenv.config();
|
||||
|
||||
const pool = new Pool({
|
||||
connectionString: DATABASE_URL,
|
||||
});
|
||||
/**
|
||||
* Get the database connection string from environment variables.
|
||||
* Strict validation - will throw if required vars are missing.
|
||||
*/
|
||||
function getConnectionString(): string {
|
||||
// Priority 1: Full connection URL
|
||||
if (process.env.CANNAIQ_DB_URL) {
|
||||
return process.env.CANNAIQ_DB_URL;
|
||||
}
|
||||
|
||||
// Priority 2: Build from individual env vars (all required)
|
||||
const required = ['CANNAIQ_DB_HOST', 'CANNAIQ_DB_PORT', 'CANNAIQ_DB_NAME', 'CANNAIQ_DB_USER', 'CANNAIQ_DB_PASS'];
|
||||
const missing = required.filter((key) => !process.env[key]);
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`[Migrate] Missing required environment variables: ${missing.join(', ')}\n` +
|
||||
`Either set CANNAIQ_DB_URL or all of: CANNAIQ_DB_HOST, CANNAIQ_DB_PORT, CANNAIQ_DB_NAME, CANNAIQ_DB_USER, CANNAIQ_DB_PASS`
|
||||
);
|
||||
}
|
||||
|
||||
const host = process.env.CANNAIQ_DB_HOST!;
|
||||
const port = process.env.CANNAIQ_DB_PORT!;
|
||||
const name = process.env.CANNAIQ_DB_NAME!;
|
||||
const user = process.env.CANNAIQ_DB_USER!;
|
||||
const pass = process.env.CANNAIQ_DB_PASS!;
|
||||
|
||||
return `postgresql://${user}:${pass}@${host}:${port}/${name}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all database migrations
|
||||
*/
|
||||
async function runMigrations() {
|
||||
// Create pool only when migrations are actually run
|
||||
const pool = new Pool({
|
||||
connectionString: getConnectionString(),
|
||||
});
|
||||
|
||||
export async function runMigrations() {
|
||||
const client = await pool.connect();
|
||||
|
||||
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
|
||||
// Users table
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
@@ -29,7 +69,7 @@ export async function runMigrations() {
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Stores table
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS stores (
|
||||
@@ -44,7 +84,7 @@ export async function runMigrations() {
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Categories table (shop, brands, specials)
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
@@ -59,7 +99,7 @@ export async function runMigrations() {
|
||||
UNIQUE(store_id, slug)
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Products table
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS products (
|
||||
@@ -90,7 +130,7 @@ export async function runMigrations() {
|
||||
UNIQUE(store_id, dutchie_product_id)
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Campaigns table
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS campaigns (
|
||||
@@ -106,7 +146,7 @@ export async function runMigrations() {
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Add variant column to products table (for different sizes/options of same product)
|
||||
await client.query(`
|
||||
ALTER TABLE products ADD COLUMN IF NOT EXISTS variant VARCHAR(255);
|
||||
@@ -226,7 +266,7 @@ export async function runMigrations() {
|
||||
UNIQUE(campaign_id, product_id)
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Click tracking
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS clicks (
|
||||
@@ -239,14 +279,14 @@ export async function runMigrations() {
|
||||
clicked_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Create index on clicked_at for analytics queries
|
||||
await client.query(`
|
||||
CREATE INDEX IF NOT EXISTS idx_clicks_clicked_at ON clicks(clicked_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_clicks_product_id ON clicks(product_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_clicks_campaign_id ON clicks(campaign_id);
|
||||
`);
|
||||
|
||||
|
||||
// Proxies table
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS proxies (
|
||||
@@ -310,7 +350,7 @@ export async function runMigrations() {
|
||||
CREATE INDEX IF NOT EXISTS idx_proxy_test_jobs_status ON proxy_test_jobs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_proxy_test_jobs_created_at ON proxy_test_jobs(created_at DESC);
|
||||
`);
|
||||
|
||||
|
||||
// Settings table
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
@@ -320,7 +360,7 @@ export async function runMigrations() {
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
|
||||
// Insert default settings
|
||||
await client.query(`
|
||||
INSERT INTO settings (key, value, description) VALUES
|
||||
@@ -331,7 +371,7 @@ export async function runMigrations() {
|
||||
('proxy_test_url', 'https://httpbin.org/ip', 'URL to test proxies against')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
`);
|
||||
|
||||
|
||||
await client.query('COMMIT');
|
||||
console.log('✅ Migrations completed successfully');
|
||||
} catch (error) {
|
||||
@@ -340,12 +380,12 @@ export async function runMigrations() {
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
export { pool };
|
||||
|
||||
// Run migrations if this file is executed directly
|
||||
// Only run when executed directly (CLI mode)
|
||||
// DO NOT export pool - runtime code must use src/db/pool.ts
|
||||
if (require.main === module) {
|
||||
runMigrations()
|
||||
.then(() => process.exit(0))
|
||||
|
||||
Reference in New Issue
Block a user