feat: Add v2 architecture with multi-state support and orchestrator services

Major additions:
- Multi-state expansion: states table, StateSelector, NationalDashboard, StateHeatmap, CrossStateCompare
- Orchestrator services: trace service, error taxonomy, retry manager, proxy rotator
- Discovery system: dutchie discovery service, geo validation, city seeding scripts
- Analytics infrastructure: analytics v2 routes, brand/pricing/stores intelligence pages
- Local development: setup-local.sh starts all 5 services (postgres, backend, cannaiq, findadispo, findagram)
- Migrations 037-056: crawler profiles, states, analytics indexes, worker metadata

Frontend pages added:
- Discovery, ChainsDashboard, IntelligenceBrands, IntelligencePricing, IntelligenceStores
- StateHeatmap, CrossStateCompare, SyncInfoPanel

Components added:
- StateSelector, OrchestratorTraceModal, WorkflowStepper

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-07 11:30:57 -07:00
parent 8ac64ba077
commit b4a2fb7d03
248 changed files with 60714 additions and 666 deletions

View File

@@ -60,11 +60,22 @@ import versionRoutes from './routes/version';
import publicApiRoutes from './routes/public-api';
import usersRoutes from './routes/users';
import staleProcessesRoutes from './routes/stale-processes';
import orchestratorAdminRoutes from './routes/orchestrator-admin';
import adminRoutes from './routes/admin';
import { dutchieAZRouter, startScheduler as startDutchieAZScheduler, initializeDefaultSchedules } from './dutchie-az';
import { getPool } from './dutchie-az/db/connection';
import { createAnalyticsRouter } from './dutchie-az/routes/analytics';
import { createMultiStateRoutes } from './multi-state';
import { trackApiUsage, checkRateLimit } from './middleware/apiTokenTracker';
import { startCrawlScheduler } from './services/crawl-scheduler';
import { validateWordPressPermissions } from './middleware/wordpressPermissions';
import { markTrustedDomains } from './middleware/trustedDomains';
import { createSystemRouter, createPrometheusRouter } from './system/routes';
import { createPortalRoutes } from './portals';
import { createStatesRouter } from './routes/states';
import { createAnalyticsV2Router } from './routes/analytics-v2';
import { createDiscoveryRoutes } from './discovery';
import { createDutchieDiscoveryRoutes, promoteDiscoveryLocation } from './dutchie-az/discovery';
// Mark requests from trusted domains (cannaiq.co, findagram.co, findadispo.com)
// These domains can access the API without authentication
@@ -98,15 +109,124 @@ app.use('/api/crawler-sandbox', crawlerSandboxRoutes);
app.use('/api/version', versionRoutes);
app.use('/api/users', usersRoutes);
app.use('/api/stale-processes', staleProcessesRoutes);
// Admin routes - operator actions (crawl triggers, health checks)
app.use('/api/admin', adminRoutes);
app.use('/api/admin/orchestrator', orchestratorAdminRoutes);
// Vendor-agnostic AZ data pipeline routes (new public surface)
app.use('/api/az', dutchieAZRouter);
// Legacy alias (kept temporarily for backward compatibility)
app.use('/api/dutchie-az', dutchieAZRouter);
// Phase 3: Analytics Dashboards - price trends, penetration, category growth, etc.
try {
const analyticsRouter = createAnalyticsRouter(getPool());
app.use('/api/az/analytics', analyticsRouter);
console.log('[Analytics] Routes registered at /api/az/analytics');
} catch (error) {
console.warn('[Analytics] Failed to register routes:', error);
}
// Phase 3: Analytics V2 - Enhanced analytics with rec/med state segmentation
try {
const analyticsV2Router = createAnalyticsV2Router(getPool());
app.use('/api/analytics/v2', analyticsV2Router);
console.log('[AnalyticsV2] Routes registered at /api/analytics/v2');
} catch (error) {
console.warn('[AnalyticsV2] Failed to register routes:', error);
}
// Public API v1 - External consumer endpoints (WordPress, etc.)
// Uses dutchie_az data pipeline with per-dispensary API key auth
app.use('/api/v1', publicApiRoutes);
// Multi-state API routes - national analytics and cross-state comparisons
// Phase 4: Multi-State Expansion
try {
const multiStateRoutes = createMultiStateRoutes(getPool());
app.use('/api', multiStateRoutes);
console.log('[MultiState] Routes registered');
} catch (error) {
console.warn('[MultiState] Failed to register routes (DB may not be configured):', error);
}
// States API routes - cannabis legalization status and targeting
try {
const statesRouter = createStatesRouter(getPool());
app.use('/api/states', statesRouter);
console.log('[States] Routes registered at /api/states');
} catch (error) {
console.warn('[States] Failed to register routes:', error);
}
// Phase 5: Production Sync + Monitoring
// System orchestrator, DLQ, integrity checks, auto-fix routines, alerts
try {
const systemRouter = createSystemRouter(getPool());
const prometheusRouter = createPrometheusRouter(getPool());
app.use('/api/system', systemRouter);
app.use('/metrics', prometheusRouter);
console.log('[System] Routes registered at /api/system and /metrics');
} catch (error) {
console.warn('[System] Failed to register routes:', error);
}
// Phase 6 & 7: Portals (Brand, Buyer), Intelligence, Orders, Inventory, Pricing
try {
const portalRoutes = createPortalRoutes(getPool());
app.use('/api/portal', portalRoutes);
console.log('[Portals] Routes registered at /api/portal');
} catch (error) {
console.warn('[Portals] Failed to register routes:', error);
}
// Discovery Pipeline - Store discovery from Dutchie with verification workflow
try {
const discoveryRoutes = createDiscoveryRoutes(getPool());
app.use('/api/discovery', discoveryRoutes);
console.log('[Discovery] Routes registered at /api/discovery');
} catch (error) {
console.warn('[Discovery] Failed to register routes:', error);
}
// Platform-specific Discovery Routes
// Uses neutral slugs to avoid trademark issues in URLs:
// dt = Dutchie, jn = Jane, wm = Weedmaps, etc.
// Routes: /api/discovery/platforms/:platformSlug/*
try {
const dtDiscoveryRoutes = createDutchieDiscoveryRoutes(getPool());
app.use('/api/discovery/platforms/dt', dtDiscoveryRoutes);
console.log('[Discovery] Platform routes registered at /api/discovery/platforms/dt');
} catch (error) {
console.warn('[Discovery] Failed to register platform routes:', error);
}
// Orchestrator promotion endpoint (platform-agnostic)
// Route: /api/orchestrator/platforms/:platformSlug/promote/:id
app.post('/api/orchestrator/platforms/:platformSlug/promote/:id', async (req, res) => {
try {
const { platformSlug, id } = req.params;
// Validate platform slug
const validPlatforms = ['dt']; // dt = Dutchie
if (!validPlatforms.includes(platformSlug)) {
return res.status(400).json({
success: false,
error: `Invalid platform slug: ${platformSlug}. Valid slugs: ${validPlatforms.join(', ')}`
});
}
const result = await promoteDiscoveryLocation(getPool(), parseInt(id, 10));
if (result.success) {
res.json(result);
} else {
res.status(400).json(result);
}
} catch (error: any) {
console.error('[Orchestrator] Promotion error:', error);
res.status(500).json({ success: false, error: error.message });
}
});
async function startServer() {
try {
logger.info('system', 'Starting server...');