fix(ci): Fix buildx cache_from syntax for array format

Plugin was splitting comma-separated values incorrectly.
Use array format with quoted strings instead.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.woodpecker.yml

@@ -1,6 +1,3 @@
-when:
-  - event: [push, pull_request]
-
 steps:
   # ===========================================
   # PR VALIDATION: Parallel type checks (PRs only)
@@ -45,6 +42,31 @@ steps:
     when:
       event: pull_request
 
+  # ===========================================
+  # AUTO-MERGE: Merge PR after all checks pass
+  # ===========================================
+  auto-merge:
+    image: alpine:latest
+    environment:
+      GITEA_TOKEN:
+        from_secret: gitea_token
+    commands:
+      - apk add --no-cache curl
+      - |
+        echo "Merging PR #${CI_COMMIT_PULL_REQUEST}..."
+        curl -s -X POST \
+          -H "Authorization: token $GITEA_TOKEN" \
+          -H "Content-Type: application/json" \
+          -d '{"Do":"merge"}' \
+          "https://code.cannabrands.app/api/v1/repos/Creationshop/dispensary-scraper/pulls/${CI_COMMIT_PULL_REQUEST}/merge"
+    depends_on:
+      - typecheck-backend
+      - typecheck-cannaiq
+      - typecheck-findadispo
+      - typecheck-findagram
+    when:
+      event: pull_request
+
   # ===========================================
   # MASTER DEPLOY: Parallel Docker builds
   # ===========================================
@@ -64,11 +86,15 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache,mode=max"
       build_args:
-        - APP_BUILD_VERSION=${CI_COMMIT_SHA}
+        APP_BUILD_VERSION: ${CI_COMMIT_SHA:0:8}
-        - APP_GIT_SHA=${CI_COMMIT_SHA}
+        APP_GIT_SHA: ${CI_COMMIT_SHA}
-        - APP_BUILD_TIME=${CI_PIPELINE_CREATED}
+        APP_BUILD_TIME: ${CI_PIPELINE_CREATED}
-        - CONTAINER_IMAGE_TAG=${CI_COMMIT_SHA:0:8}
+        CONTAINER_IMAGE_TAG: ${CI_COMMIT_SHA:0:8}
     depends_on: []
     when:
       branch: master
@@ -90,6 +116,10 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache,mode=max"
     depends_on: []
     when:
       branch: master
@@ -111,6 +141,10 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache,mode=max"
     depends_on: []
     when:
       branch: master
@@ -132,13 +166,17 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache,mode=max"
     depends_on: []
     when:
       branch: master
       event: push
 
   # ===========================================
-  # STAGE 3: Deploy (after Docker builds)
+  # STAGE 3: Deploy and Run Migrations
   # ===========================================
   deploy:
     image: bitnami/kubectl:latest
@@ -149,12 +187,17 @@ steps:
       - mkdir -p ~/.kube
       - echo "$KUBECONFIG_CONTENT" | tr -d '[:space:]' | base64 -d > ~/.kube/config
       - chmod 600 ~/.kube/config
+      # Deploy backend first
       - kubectl set image deployment/scraper scraper=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
+      # Note: Migrations run automatically at startup via auto-migrate
+      # Deploy remaining services
+      # Resilience: ensure workers are scaled up if at 0
+      - REPLICAS=$(kubectl get deployment scraper-worker -n dispensary-scraper -o jsonpath='{.spec.replicas}'); if [ "$REPLICAS" = "0" ]; then echo "Scaling workers from 0 to 5"; kubectl scale deployment/scraper-worker --replicas=5 -n dispensary-scraper; fi
       - kubectl set image deployment/scraper-worker worker=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/cannaiq-frontend cannaiq-frontend=code.cannabrands.app/creationshop/cannaiq-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/findadispo-frontend findadispo-frontend=code.cannabrands.app/creationshop/findadispo-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/findagram-frontend findagram-frontend=code.cannabrands.app/creationshop/findagram-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
-      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
       - kubectl rollout status deployment/cannaiq-frontend -n dispensary-scraper --timeout=120s
     depends_on:
       - docker-backend

.woodpecker/ci.yml

@@ -0,0 +1,191 @@
+steps:
+  # ===========================================
+  # PR VALIDATION: Only typecheck changed projects
+  # ===========================================
+  typecheck-backend:
+    image: code.cannabrands.app/creationshop/node:20
+    commands:
+      - npm config set cache /npm-cache/backend --global
+      - cd backend
+      - npm ci --prefer-offline
+      - npx tsc --noEmit
+    volumes:
+      - npm-cache:/npm-cache
+    depends_on: []
+    when:
+      event: pull_request
+      path:
+        include: ['backend/**']
+
+  typecheck-cannaiq:
+    image: code.cannabrands.app/creationshop/node:20
+    commands:
+      - npm config set cache /npm-cache/cannaiq --global
+      - cd cannaiq
+      - npm ci --prefer-offline
+      - npx tsc --noEmit
+    volumes:
+      - npm-cache:/npm-cache
+    depends_on: []
+    when:
+      event: pull_request
+      path:
+        include: ['cannaiq/**']
+
+  # findadispo/findagram typechecks skipped - they have || true anyway
+
+  # ===========================================
+  # AUTO-MERGE: Merge PR after all checks pass
+  # ===========================================
+  auto-merge:
+    image: alpine:latest
+    environment:
+      GITEA_TOKEN:
+        from_secret: gitea_token
+    commands:
+      - apk add --no-cache curl
+      - |
+        echo "Merging PR #${CI_COMMIT_PULL_REQUEST}..."
+        curl -s -X POST \
+          -H "Authorization: token $GITEA_TOKEN" \
+          -H "Content-Type: application/json" \
+          -d '{"Do":"merge"}' \
+          "https://code.cannabrands.app/api/v1/repos/Creationshop/dispensary-scraper/pulls/${CI_COMMIT_PULL_REQUEST}/merge"
+    depends_on:
+      - typecheck-backend
+      - typecheck-cannaiq
+    when:
+      event: pull_request
+
+  # ===========================================
+  # MASTER DEPLOY: Parallel Docker builds
+  # ===========================================
+  docker-backend:
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      registry: code.cannabrands.app
+      repo: code.cannabrands.app/creationshop/dispensary-scraper
+      tags:
+        - latest
+        - ${CI_COMMIT_SHA:0:8}
+      dockerfile: backend/Dockerfile
+      context: backend
+      username:
+        from_secret: registry_username
+      password:
+        from_secret: registry_password
+      platforms: linux/amd64
+      provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache,mode=max
+      build_args:
+        APP_BUILD_VERSION: ${CI_COMMIT_SHA:0:8}
+        APP_GIT_SHA: ${CI_COMMIT_SHA}
+        APP_BUILD_TIME: ${CI_PIPELINE_CREATED}
+        CONTAINER_IMAGE_TAG: ${CI_COMMIT_SHA:0:8}
+    depends_on: []
+    when:
+      branch: master
+      event: push
+
+  docker-cannaiq:
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      registry: code.cannabrands.app
+      repo: code.cannabrands.app/creationshop/cannaiq-frontend
+      tags:
+        - latest
+        - ${CI_COMMIT_SHA:0:8}
+      dockerfile: cannaiq/Dockerfile
+      context: cannaiq
+      username:
+        from_secret: registry_username
+      password:
+        from_secret: registry_password
+      platforms: linux/amd64
+      provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache,mode=max
+    depends_on: []
+    when:
+      branch: master
+      event: push
+
+  docker-findadispo:
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      registry: code.cannabrands.app
+      repo: code.cannabrands.app/creationshop/findadispo-frontend
+      tags:
+        - latest
+        - ${CI_COMMIT_SHA:0:8}
+      dockerfile: findadispo/frontend/Dockerfile
+      context: findadispo/frontend
+      username:
+        from_secret: registry_username
+      password:
+        from_secret: registry_password
+      platforms: linux/amd64
+      provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache,mode=max
+    depends_on: []
+    when:
+      branch: master
+      event: push
+
+  docker-findagram:
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      registry: code.cannabrands.app
+      repo: code.cannabrands.app/creationshop/findagram-frontend
+      tags:
+        - latest
+        - ${CI_COMMIT_SHA:0:8}
+      dockerfile: findagram/frontend/Dockerfile
+      context: findagram/frontend
+      username:
+        from_secret: registry_username
+      password:
+        from_secret: registry_password
+      platforms: linux/amd64
+      provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache,mode=max
+    depends_on: []
+    when:
+      branch: master
+      event: push
+
+  # ===========================================
+  # STAGE 3: Deploy and Run Migrations
+  # ===========================================
+  deploy:
+    image: bitnami/kubectl:latest
+    environment:
+      KUBECONFIG_CONTENT:
+        from_secret: kubeconfig_data
+    commands:
+      - mkdir -p ~/.kube
+      - echo "$KUBECONFIG_CONTENT" | tr -d '[:space:]' | base64 -d > ~/.kube/config
+      - chmod 600 ~/.kube/config
+      # Deploy backend first
+      - kubectl set image deployment/scraper scraper=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
+      # Note: Migrations run automatically at startup via auto-migrate
+      # Deploy remaining services
+      # Resilience: ensure workers are scaled up if at 0
+      - REPLICAS=$(kubectl get deployment scraper-worker -n dispensary-scraper -o jsonpath='{.spec.replicas}'); if [ "$REPLICAS" = "0" ]; then echo "Scaling workers from 0 to 5"; kubectl scale deployment/scraper-worker --replicas=5 -n dispensary-scraper; fi
+      - kubectl set image deployment/scraper-worker worker=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl set image deployment/cannaiq-frontend cannaiq-frontend=code.cannabrands.app/creationshop/cannaiq-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl set image deployment/findadispo-frontend findadispo-frontend=code.cannabrands.app/creationshop/findadispo-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl set image deployment/findagram-frontend findagram-frontend=code.cannabrands.app/creationshop/findagram-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl rollout status deployment/cannaiq-frontend -n dispensary-scraper --timeout=120s
+    depends_on:
+      - docker-backend
+      - docker-cannaiq
+      - docker-findadispo
+      - docker-findagram
+    when:
+      branch: master
+      event: push

CLAUDE.md

@@ -119,7 +119,42 @@ npx tsx src/db/migrate.ts
 - Importing it at runtime causes startup crashes if env vars aren't perfect
 - `pool.ts` uses lazy initialization - only validates when first query is made
 
-### 6. LOCAL DEVELOPMENT BY DEFAULT
+### 6. ALL API ROUTES REQUIRE AUTHENTICATION — NO EXCEPTIONS
+
+**Every API router MUST apply `authMiddleware` at the router level.**
+
+```typescript
+import { authMiddleware } from '../auth/middleware';
+
+const router = Router();
+router.use(authMiddleware);  // REQUIRED - first line after router creation
+```
+
+**Authentication flow (see `src/auth/middleware.ts`):**
+1. Check Bearer token (JWT or API token) → grant access if valid
+2. Check trusted origins (cannaiq.co, findadispo.com, localhost, etc.) → grant access
+3. Check trusted IPs (127.0.0.1, ::1, internal pod IPs) → grant access
+4. **Return 401 Unauthorized** if none of the above
+
+**NEVER create API routes without auth middleware:**
+- No "public" endpoints that bypass authentication
+- No "read-only" exceptions
+- No "analytics-only" exceptions
+- If an endpoint exists under `/api/*`, it MUST be protected
+
+**When creating new route files:**
+1. Import `authMiddleware` from `../auth/middleware`
+2. Add `router.use(authMiddleware)` immediately after creating the router
+3. Document security requirements in file header comments
+
+**Trusted origins (defined in middleware):**
+- `https://cannaiq.co`
+- `https://findadispo.com`
+- `https://findagram.co`
+- `*.cannabrands.app` domains
+- `localhost:*` for development
+
+### 7. LOCAL DEVELOPMENT BY DEFAULT
 
 **Quick Start:**
 ```bash
@@ -213,22 +248,23 @@ CannaiQ has **TWO databases** with distinct purposes:
 | Table | Purpose | Row Count |
 |-------|---------|-----------|
 | `dispensaries` | Store/dispensary records | ~188+ rows |
-| `dutchie_products` | Product catalog | ~37,000+ rows |
+| `store_products` | Product catalog | ~37,000+ rows |
-| `dutchie_product_snapshots` | Price/stock history | ~millions |
+| `store_product_snapshots` | Price/stock history | ~millions |
-| `store_products` | Canonical product schema | ~37,000+ rows |
-| `store_product_snapshots` | Canonical snapshot schema | growing |
 
 **LEGACY TABLES (EMPTY - DO NOT USE):**
 
 | Table | Status | Action |
 |-------|--------|--------|
 | `stores` | EMPTY (0 rows) | Use `dispensaries` instead |
-| `products` | EMPTY (0 rows) | Use `dutchie_products` or `store_products` |
+| `products` | EMPTY (0 rows) | Use `store_products` instead |
+| `dutchie_products` | LEGACY (0 rows) | Use `store_products` instead |
+| `dutchie_product_snapshots` | LEGACY (0 rows) | Use `store_product_snapshots` instead |
 | `categories` | EMPTY (0 rows) | Categories stored in product records |
 
 **Code must NEVER:**
 - Query the `stores` table (use `dispensaries`)
-- Query the `products` table (use `dutchie_products` or `store_products`)
+- Query the `products` table (use `store_products`)
+- Query the `dutchie_products` table (use `store_products`)
 - Query the `categories` table (categories are in product records)
 
 **CRITICAL RULES:**
@@ -343,23 +379,23 @@ npx tsx src/scripts/etl/042_legacy_import.ts
 - SCHEMA ONLY - no data inserts from legacy tables
 
 **ETL Script 042** (`backend/src/scripts/etl/042_legacy_import.ts`):
-- Copies data from `dutchie_products` → `store_products`
+- Copies data from legacy `dutchie_legacy.dutchie_products` → `store_products`
-- Copies data from `dutchie_product_snapshots` → `store_product_snapshots`
+- Copies data from legacy `dutchie_legacy.dutchie_product_snapshots` → `store_product_snapshots`
 - Extracts brands from product data into `brands` table
 - Links dispensaries to chains and states
 - INSERT-ONLY and IDEMPOTENT (uses ON CONFLICT DO NOTHING)
 - Run manually: `cd backend && npx tsx src/scripts/etl/042_legacy_import.ts`
 
 **Tables touched by ETL:**
-| Source Table | Target Table |
+| Source Table (dutchie_legacy) | Target Table (dutchie_menus) |
-|--------------|--------------|
+|-------------------------------|------------------------------|
 | `dutchie_products` | `store_products` |
 | `dutchie_product_snapshots` | `store_product_snapshots` |
 | (brand names extracted) | `brands` |
 | (state codes mapped) | `dispensaries.state_id` |
 | (chain names matched) | `dispensaries.chain_id` |
 
-**Legacy tables remain intact** - `dutchie_products` and `dutchie_product_snapshots` are not modified.
+**Note:** The legacy `dutchie_products` and `dutchie_product_snapshots` tables in `dutchie_legacy` are read-only sources. All new crawl data goes directly to `store_products` and `store_product_snapshots`.
 
 **Migration 045** (`backend/migrations/045_add_image_columns.sql`):
 - Adds `thumbnail_url` to `store_products` and `store_product_snapshots`
@@ -451,6 +487,7 @@ const result = await pool.query(`
 16. **Running `lsof -ti:PORT | xargs kill`** or similar process-killing commands
 17. **Using hardcoded database names** in code or comments
 18. **Creating or connecting to a second database**
+19. **Creating API routes without authMiddleware** (all `/api/*` routes MUST be protected)
 
 ---
 
@@ -881,7 +918,7 @@ export default defineConfig({
 
 18) **Dashboard Architecture**
     - **Frontend**: Rebuild the frontend with `VITE_API_URL` pointing to the correct backend and redeploy.
-    - **Backend**: `/api/dashboard/stats` MUST use the canonical DB pool. Use the correct tables: `dutchie_products`, `dispensaries`, and views like `v_dashboard_stats`, `v_latest_snapshots`.
+    - **Backend**: `/api/dashboard/stats` MUST use the canonical DB pool. Use the correct tables: `store_products`, `dispensaries`, and views like `v_dashboard_stats`, `v_latest_snapshots`.
 
 19) **Deployment (Gitea + Kubernetes)**
     - **Registry**: Gitea at `code.cannabrands.app/creationshop/dispensary-scraper`
@@ -902,7 +939,8 @@ export default defineConfig({
 
 20) **Crawler Architecture**
     - **Scraper pod (1 replica)**: Runs the Express API server + scheduler.
-    - **Scraper-worker pods (5 replicas)**: Each worker runs `dist/dutchie-az/services/worker.js`, polling the job queue.
+    - **Scraper-worker pods (25 replicas)**: Each runs `dist/tasks/task-worker.js`, polling the job queue.
+    - **Worker naming**: Pods use fantasy names (Aethelgard, Xylos, Kryll, Coriolis, etc.) - see `k8s/scraper-worker.yaml` ConfigMap. Worker IDs: `{PodName}-worker-{n}`
     - **Job types**: `menu_detection`, `menu_detection_single`, `dutchie_product_crawl`
     - **Job schedules** (managed in `job_schedules` table):
       - `dutchie_az_menu_detection`: Runs daily with 60-min jitter

backend/.gitignore

@@ -0,0 +1,3 @@
+
+# IP2Location database (downloaded separately)
+data/ip2location/

backend/Dockerfile

@@ -5,7 +5,7 @@ FROM code.cannabrands.app/creationshop/node:20-slim AS builder
 WORKDIR /app
 
 COPY package*.json ./
-RUN npm ci
+RUN npm install
 
 COPY . .
 RUN npm run build
@@ -25,8 +25,9 @@ ENV APP_GIT_SHA=${APP_GIT_SHA}
 ENV APP_BUILD_TIME=${APP_BUILD_TIME}
 ENV CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}
 
-# Install Chromium dependencies
+# Install Chromium dependencies and curl for HTTP requests
 RUN apt-get update && apt-get install -y \
+    curl \
     chromium \
     fonts-liberation \
     libnss3 \
@@ -43,10 +44,13 @@ ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 WORKDIR /app
 
 COPY package*.json ./
-RUN npm ci --omit=dev
+RUN npm install --omit=dev
 
 COPY --from=builder /app/dist ./dist
 
+# Copy migrations for auto-migrate on startup
+COPY migrations ./migrations
+
 # Create local images directory for when MinIO is not configured
 RUN mkdir -p /app/public/images/products
 

backend/docs/BRAND_INTELLIGENCE_API.md

@@ -0,0 +1,394 @@
+# Brand Intelligence API
+
+## Endpoint
+
+```
+GET /api/analytics/v2/brand/:name/intelligence
+```
+
+## Query Parameters
+
+| Param | Type | Default | Description |
+|-------|------|---------|-------------|
+| `window` | `7d\|30d\|90d` | `30d` | Time window for trend calculations |
+| `state` | string | - | Filter by state code (e.g., `AZ`) |
+| `category` | string | - | Filter by category (e.g., `Flower`) |
+
+## Response Payload Schema
+
+```typescript
+interface BrandIntelligenceResult {
+  brand_name: string;
+  window: '7d' | '30d' | '90d';
+  generated_at: string;  // ISO timestamp when data was computed
+
+  performance_snapshot: PerformanceSnapshot;
+  alerts: Alerts;
+  sku_performance: SkuPerformance[];
+  retail_footprint: RetailFootprint;
+  competitive_landscape: CompetitiveLandscape;
+  inventory_health: InventoryHealth;
+  promo_performance: PromoPerformance;
+}
+```
+
+---
+
+## Section 1: Performance Snapshot
+
+Summary cards with key brand metrics.
+
+```typescript
+interface PerformanceSnapshot {
+  active_skus: number;              // Total products in catalog
+  total_revenue_30d: number | null; // Estimated from qty × price
+  total_stores: number;             // Active retail partners
+  new_stores_30d: number;           // New distribution in window
+  market_share: number | null;      // % of category SKUs
+  avg_wholesale_price: number | null;
+  price_position: 'premium' | 'value' | 'competitive';
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label | Helper Text |
+|-------|-------------------|-------------|
+| `active_skus` | Active Products | X total in catalog |
+| `total_revenue_30d` | Monthly Revenue | Estimated from sales |
+| `total_stores` | Retail Distribution | Active retail partners |
+| `new_stores_30d` | New Opportunities | X new in last 30 days |
+| `market_share` | Category Position | % of category |
+| `avg_wholesale_price` | Avg Wholesale | Per unit |
+| `price_position` | Pricing Tier | Premium/Value/Market Rate |
+
+---
+
+## Section 2: Alerts
+
+Issues requiring attention.
+
+```typescript
+interface Alerts {
+  lost_stores_30d_count: number;
+  lost_skus_30d_count: number;
+  competitor_takeover_count: number;
+  avg_oos_duration_days: number | null;
+  avg_reorder_lag_days: number | null;
+  items: AlertItem[];
+}
+
+interface AlertItem {
+  type: 'lost_store' | 'delisted_sku' | 'shelf_loss' | 'extended_oos';
+  severity: 'critical' | 'warning';
+  store_name?: string;
+  product_name?: string;
+  competitor_brand?: string;
+  days_since?: number;
+  state_code?: string;
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label |
+|-------|-------------------|
+| `lost_stores_30d_count` | Accounts at Risk |
+| `lost_skus_30d_count` | Delisted SKUs |
+| `competitor_takeover_count` | Shelf Losses |
+| `avg_oos_duration_days` | Avg Stockout Length |
+| `avg_reorder_lag_days` | Avg Restock Time |
+| `severity: critical` | Urgent |
+| `severity: warning` | Watch |
+
+---
+
+## Section 3: SKU Performance (Product Velocity)
+
+How fast each SKU sells.
+
+```typescript
+interface SkuPerformance {
+  store_product_id: number;
+  product_name: string;
+  category: string | null;
+  daily_velocity: number;        // Units/day estimate
+  velocity_status: 'hot' | 'steady' | 'slow' | 'stale';
+  retail_price: number | null;
+  on_sale: boolean;
+  stores_carrying: number;
+  stock_status: 'in_stock' | 'low_stock' | 'out_of_stock';
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label |
+|-------|-------------------|
+| `daily_velocity` | Daily Rate |
+| `velocity_status` | Momentum |
+| `velocity_status: hot` | Hot |
+| `velocity_status: steady` | Steady |
+| `velocity_status: slow` | Slow |
+| `velocity_status: stale` | Stale |
+| `retail_price` | Retail Price |
+| `on_sale` | Promo (badge) |
+
+**Velocity Thresholds:**
+- `hot`: >= 5 units/day
+- `steady`: >= 1 unit/day
+- `slow`: >= 0.1 units/day
+- `stale`: < 0.1 units/day
+
+---
+
+## Section 4: Retail Footprint
+
+Store placement and coverage.
+
+```typescript
+interface RetailFootprint {
+  total_stores: number;
+  in_stock_count: number;
+  out_of_stock_count: number;
+  penetration_by_region: RegionPenetration[];
+  whitespace_stores: WhitespaceStore[];
+}
+
+interface RegionPenetration {
+  state_code: string;
+  store_count: number;
+  percent_reached: number;    // % of state's dispensaries
+  in_stock: number;
+  out_of_stock: number;
+}
+
+interface WhitespaceStore {
+  store_id: number;
+  store_name: string;
+  state_code: string;
+  city: string | null;
+  category_fit: number;       // How many competing brands they carry
+  competitor_brands: string[];
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label |
+|-------|-------------------|
+| `penetration_by_region` | Market Coverage by Region |
+| `percent_reached` | X% reached |
+| `in_stock` | X stocked |
+| `out_of_stock` | X out |
+| `whitespace_stores` | Expansion Opportunities |
+| `category_fit` | X fit |
+
+---
+
+## Section 5: Competitive Landscape
+
+Market positioning vs competitors.
+
+```typescript
+interface CompetitiveLandscape {
+  brand_price_position: 'premium' | 'value' | 'competitive';
+  market_share_trend: MarketSharePoint[];
+  competitors: Competitor[];
+  head_to_head_skus: HeadToHead[];
+}
+
+interface MarketSharePoint {
+  date: string;
+  share_percent: number;
+}
+
+interface Competitor {
+  brand_name: string;
+  store_overlap_percent: number;
+  price_position: 'premium' | 'value' | 'competitive';
+  avg_price: number | null;
+  sku_count: number;
+}
+
+interface HeadToHead {
+  product_name: string;
+  brand_price: number;
+  competitor_brand: string;
+  competitor_price: number;
+  price_diff_percent: number;
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label |
+|-------|-------------------|
+| `price_position: premium` | Premium Tier |
+| `price_position: value` | Value Leader |
+| `price_position: competitive` | Market Rate |
+| `market_share_trend` | Share of Shelf Trend |
+| `head_to_head_skus` | Price Comparison |
+| `store_overlap_percent` | X% store overlap |
+
+---
+
+## Section 6: Inventory Health
+
+Stock projections and risk levels.
+
+```typescript
+interface InventoryHealth {
+  critical_count: number;      // <7 days stock
+  warning_count: number;       // 7-14 days stock
+  healthy_count: number;       // 14-90 days stock
+  overstocked_count: number;   // >90 days stock
+  skus: InventorySku[];
+  overstock_alert: OverstockItem[];
+}
+
+interface InventorySku {
+  store_product_id: number;
+  product_name: string;
+  store_name: string;
+  days_of_stock: number | null;
+  risk_level: 'critical' | 'elevated' | 'moderate' | 'healthy';
+  current_quantity: number | null;
+  daily_sell_rate: number | null;
+}
+
+interface OverstockItem {
+  product_name: string;
+  store_name: string;
+  excess_units: number;
+  days_of_stock: number;
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label |
+|-------|-------------------|
+| `risk_level: critical` | Reorder Now |
+| `risk_level: elevated` | Low Stock |
+| `risk_level: moderate` | Monitor |
+| `risk_level: healthy` | Healthy |
+| `critical_count` | Urgent (<7 days) |
+| `warning_count` | Low (7-14 days) |
+| `overstocked_count` | Excess (>90 days) |
+| `days_of_stock` | X days remaining |
+| `overstock_alert` | Overstock Alert |
+| `excess_units` | X excess units |
+
+---
+
+## Section 7: Promotion Effectiveness
+
+How promotions impact sales.
+
+```typescript
+interface PromoPerformance {
+  avg_baseline_velocity: number | null;
+  avg_promo_velocity: number | null;
+  avg_velocity_lift: number | null;     // % increase during promo
+  avg_efficiency_score: number | null;  // ROI proxy
+  promotions: Promotion[];
+}
+
+interface Promotion {
+  product_name: string;
+  store_name: string;
+  status: 'active' | 'scheduled' | 'ended';
+  start_date: string;
+  end_date: string | null;
+  regular_price: number;
+  promo_price: number;
+  discount_percent: number;
+  baseline_velocity: number | null;
+  promo_velocity: number | null;
+  velocity_lift: number | null;
+  efficiency_score: number | null;
+}
+```
+
+**UI Label Mapping:**
+| Field | User-Facing Label |
+|-------|-------------------|
+| `avg_baseline_velocity` | Normal Rate |
+| `avg_promo_velocity` | During Promos |
+| `avg_velocity_lift` | Avg Sales Lift |
+| `avg_efficiency_score` | ROI Score |
+| `velocity_lift` | Sales Lift |
+| `efficiency_score` | ROI Score |
+| `status: active` | Live |
+| `status: scheduled` | Scheduled |
+| `status: ended` | Ended |
+
+---
+
+## Example Queries
+
+### Get full payload
+```javascript
+const response = await fetch('/api/analytics/v2/brand/Wyld/intelligence?window=30d');
+const data = await response.json();
+```
+
+### Extract summary cards (flattened)
+```javascript
+const { performance_snapshot: ps, alerts } = data;
+
+const summaryCards = {
+  activeProducts: ps.active_skus,
+  monthlyRevenue: ps.total_revenue_30d,
+  retailDistribution: ps.total_stores,
+  newOpportunities: ps.new_stores_30d,
+  categoryPosition: ps.market_share,
+  avgWholesale: ps.avg_wholesale_price,
+  pricingTier: ps.price_position,
+  accountsAtRisk: alerts.lost_stores_30d_count,
+  delistedSkus: alerts.lost_skus_30d_count,
+  shelfLosses: alerts.competitor_takeover_count,
+};
+```
+
+### Get top 10 fastest selling SKUs
+```javascript
+const topSkus = data.sku_performance
+  .filter(sku => sku.velocity_status === 'hot' || sku.velocity_status === 'steady')
+  .sort((a, b) => b.daily_velocity - a.daily_velocity)
+  .slice(0, 10);
+```
+
+### Get critical inventory alerts only
+```javascript
+const criticalInventory = data.inventory_health.skus
+  .filter(sku => sku.risk_level === 'critical');
+```
+
+### Get states with <50% penetration
+```javascript
+const underPenetrated = data.retail_footprint.penetration_by_region
+  .filter(region => region.percent_reached < 50)
+  .sort((a, b) => a.percent_reached - b.percent_reached);
+```
+
+### Get active promotions with positive lift
+```javascript
+const effectivePromos = data.promo_performance.promotions
+  .filter(p => p.status === 'active' && p.velocity_lift > 0)
+  .sort((a, b) => b.velocity_lift - a.velocity_lift);
+```
+
+### Build chart data for market share trend
+```javascript
+const chartData = data.competitive_landscape.market_share_trend.map(point => ({
+  x: new Date(point.date),
+  y: point.share_percent,
+}));
+```
+
+---
+
+## Notes for Frontend Implementation
+
+1. **All fields are snake_case** - transform to camelCase if needed
+2. **Null values are possible** - handle gracefully in UI
+3. **Arrays may be empty** - show appropriate empty states
+4. **Timestamps are ISO format** - parse with `new Date()`
+5. **Percentages are already computed** - no need to multiply by 100
+6. **The `window` parameter affects trend calculations** - 7d/30d/90d

backend/docs/CRAWL_PIPELINE.md

@@ -275,6 +275,22 @@ Store metadata:
 
 ---
 
+## Worker Roles
+
+Workers pull tasks from the `worker_tasks` queue based on their assigned role.
+
+| Role | Name | Description | Handler |
+|------|------|-------------|---------|
+| `product_resync` | Product Resync | Re-crawl dispensary products for price/stock changes | `handleProductResync` |
+| `product_discovery` | Product Discovery | Initial product discovery for new dispensaries | `handleProductDiscovery` |
+| `store_discovery` | Store Discovery | Discover new dispensary locations | `handleStoreDiscovery` |
+| `entry_point_discovery` | Entry Point Discovery | Resolve platform IDs from menu URLs | `handleEntryPointDiscovery` |
+| `analytics_refresh` | Analytics Refresh | Refresh materialized views and analytics | `handleAnalyticsRefresh` |
+
+**API Endpoint:** `GET /api/worker-registry/roles`
+
+---
+
 ## Scheduling
 
 Crawls are scheduled via `worker_tasks` table:
@@ -282,8 +298,220 @@ Crawls are scheduled via `worker_tasks` table:
 | Role | Frequency | Description |
 |------|-----------|-------------|
 | `product_resync` | Every 4 hours | Regular product refresh |
+| `product_discovery` | On-demand | First crawl for new stores |
 | `entry_point_discovery` | On-demand | New store setup |
 | `store_discovery` | Daily | Find new stores |
+| `analytics_refresh` | Daily | Refresh analytics materialized views |
+
+---
+
+## Priority & On-Demand Tasks
+
+Tasks are claimed by workers in order of **priority DESC, created_at ASC**.
+
+### Priority Levels
+
+| Priority | Use Case | Example |
+|----------|----------|---------|
+| 0 | Scheduled/batch tasks | Daily product_resync generation |
+| 10 | On-demand/chained tasks | entry_point → product_discovery |
+| Higher | Urgent/manual triggers | Admin-triggered immediate crawl |
+
+### Task Chaining
+
+When a task completes, the system automatically creates follow-up tasks:
+
+```
+store_discovery (completed)
+    └─► entry_point_discovery (priority: 10) for each new store
+
+entry_point_discovery (completed, success)
+    └─► product_discovery (priority: 10) for that store
+
+product_discovery (completed)
+    └─► [no chain] Store enters regular resync schedule
+```
+
+### On-Demand Task Creation
+
+Use the task service to create high-priority tasks:
+
+```typescript
+// Create immediate product resync for a store
+await taskService.createTask({
+  role: 'product_resync',
+  dispensary_id: 123,
+  platform: 'dutchie',
+  priority: 20, // Higher than batch tasks
+});
+
+// Convenience methods with default high priority (10)
+await taskService.createEntryPointTask(dispensaryId, 'dutchie');
+await taskService.createProductDiscoveryTask(dispensaryId, 'dutchie');
+await taskService.createStoreDiscoveryTask('dutchie', 'AZ');
+```
+
+### Claim Function
+
+The `claim_task()` SQL function atomically claims tasks:
+- Respects priority ordering (higher = first)
+- Uses `FOR UPDATE SKIP LOCKED` for concurrency
+- Prevents multiple active tasks per store
+
+---
+
+## Image Storage
+
+Images are downloaded from Dutchie's AWS S3 and stored locally with on-demand resizing.
+
+### Storage Path
+```
+/storage/images/products/<state>/<store>/<brand>/<product_id>/image-<hash>.webp
+/storage/images/brands/<brand>/logo-<hash>.webp
+```
+
+**Example:**
+```
+/storage/images/products/az/az-deeply-rooted/bud-bros/6913e3cd444eac3935e928b9/image-ae38b1f9.webp
+```
+
+### Image Proxy API
+Served via `/img/*` with on-demand resizing using **sharp**:
+
+```
+GET /img/products/az/az-deeply-rooted/bud-bros/6913e3cd444eac3935e928b9/image-ae38b1f9.webp?w=200
+```
+
+| Param | Description |
+|-------|-------------|
+| `w` | Width in pixels (max 4000) |
+| `h` | Height in pixels (max 4000) |
+| `q` | Quality 1-100 (default 80) |
+| `fit` | cover, contain, fill, inside, outside |
+| `blur` | Blur sigma (0.3-1000) |
+| `gray` | Grayscale (1 = enabled) |
+| `format` | webp, jpeg, png, avif (default webp) |
+
+### Key Files
+| File | Purpose |
+|------|---------|
+| `src/utils/image-storage.ts` | Download & save images to local filesystem |
+| `src/routes/image-proxy.ts` | On-demand resize/transform at `/img/*` |
+
+### Download Rules
+
+| Scenario | Image Action |
+|----------|--------------|
+| **New product (first crawl)** | Download if `primaryImageUrl` exists |
+| **Existing product (refresh)** | Download only if `local_image_path` is NULL (backfill) |
+| **Product already has local image** | Skip download entirely |
+
+**Logic:**
+- Images are downloaded **once** and never re-downloaded on subsequent crawls
+- `skipIfExists: true` - filesystem check prevents re-download even if queued
+- First crawl: all products get images
+- Refresh crawl: only new products or products missing local images
+
+### Storage Rules
+- **NO MinIO** - local filesystem only (`STORAGE_DRIVER=local`)
+- Store full resolution, resize on-demand via `/img` proxy
+- Convert to webp for consistency using **sharp**
+- Preserve original Dutchie URL as fallback in `image_url` column
+- Local path stored in `local_image_path` column
+
+---
+
+## Stealth & Anti-Detection
+
+**PROXIES ARE REQUIRED** - Workers will fail to start if no active proxies are available in the database. All HTTP requests to Dutchie go through a proxy.
+
+Workers automatically initialize anti-detection systems on startup.
+
+### Components
+
+| Component | Purpose | Source |
+|-----------|---------|--------|
+| **CrawlRotator** | Coordinates proxy + UA rotation | `src/services/crawl-rotator.ts` |
+| **ProxyRotator** | Round-robin proxy selection, health tracking | `src/services/crawl-rotator.ts` |
+| **UserAgentRotator** | Cycles through realistic browser fingerprints | `src/services/crawl-rotator.ts` |
+| **Dutchie Client** | Curl-based HTTP with auto-retry on 403 | `src/platforms/dutchie/client.ts` |
+
+### Initialization Flow
+
+```
+Worker Start
+    │
+    ├─► initializeStealth()
+    │       │
+    │       ├─► CrawlRotator.initialize()
+    │       │       └─► Load proxies from `proxies` table
+    │       │
+    │       └─► setCrawlRotator(rotator)
+    │               └─► Wire to Dutchie client
+    │
+    └─► Process tasks...
+```
+
+### Stealth Session (per task)
+
+Each crawl task starts a stealth session:
+
+```typescript
+// In product-refresh.ts, entry-point-discovery.ts
+const session = startSession(dispensary.state || 'AZ', 'America/Phoenix');
+```
+
+This creates a new identity with:
+- **Random fingerprint:** Chrome/Firefox/Safari/Edge on Win/Mac/Linux
+- **Accept-Language:** Matches timezone (e.g., `America/Phoenix` → `en-US,en;q=0.9`)
+- **sec-ch-ua headers:** Proper Client Hints for the browser profile
+
+### On 403 Block
+
+When Dutchie returns 403, the client automatically:
+
+1. Records failure on current proxy (increments `failure_count`)
+2. If proxy has 5+ failures, deactivates it
+3. Rotates to next healthy proxy
+4. Rotates fingerprint
+5. Retries the request
+
+### Proxy Table Schema
+
+```sql
+CREATE TABLE proxies (
+  id SERIAL PRIMARY KEY,
+  host VARCHAR(255) NOT NULL,
+  port INTEGER NOT NULL,
+  username VARCHAR(100),
+  password VARCHAR(100),
+  protocol VARCHAR(10) DEFAULT 'http',  -- http, https, socks5
+  is_active BOOLEAN DEFAULT true,
+  last_used_at TIMESTAMPTZ,
+  failure_count INTEGER DEFAULT 0,
+  success_count INTEGER DEFAULT 0,
+  avg_response_time_ms INTEGER,
+  last_failure_at TIMESTAMPTZ,
+  last_error TEXT
+);
+```
+
+### Configuration
+
+Proxies are mandatory. There is no environment variable to disable them. Workers will refuse to start without active proxies in the database.
+
+### User-Agent Generation
+
+See `workflow-12102025.md` for full specification.
+
+**Summary:**
+- Uses `intoli/user-agents` library (daily-updated market share data)
+- Device distribution: Mobile 62%, Desktop 36%, Tablet 2%
+- Browser whitelist: Chrome, Safari, Edge, Firefox only
+- UA sticks until IP rotates (403 or manual rotation)
+- Failure = alert admin + stop crawl (no fallback)
+
+Each fingerprint includes proper `sec-ch-ua`, `sec-ch-ua-platform`, and `sec-ch-ua-mobile` headers.
 
 ---
 
@@ -293,6 +521,7 @@ Crawls are scheduled via `worker_tasks` table:
 - **Normalization errors:** Logged as warnings, continue with valid products
 - **Image download errors:** Non-fatal, logged, continue
 - **Database errors:** Task fails, will be retried
+- **403 blocks:** Auto-rotate proxy + fingerprint, retry (up to 3 retries)
 
 ---
 
@@ -305,4 +534,6 @@ Crawls are scheduled via `worker_tasks` table:
 | `src/platforms/dutchie/index.ts` | GraphQL client, session management |
 | `src/hydration/normalizers/dutchie.ts` | Payload normalization |
 | `src/hydration/canonical-upsert.ts` | Database upsert logic |
+| `src/utils/image-storage.ts` | Image download and local storage |
+| `src/routes/image-proxy.ts` | On-demand image resizing |
 | `migrations/075_consecutive_misses.sql` | OOS tracking column |

backend/docs/TASK_WORKFLOW_2024-12-10.md

@@ -0,0 +1,584 @@
+# Task Workflow Documentation
+**Date: 2024-12-10**
+
+This document describes the complete task/job processing architecture after the 2024-12-10 rewrite.
+
+---
+
+## Complete Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────────────────┐
+│                              KUBERNETES CLUSTER                                  │
+├─────────────────────────────────────────────────────────────────────────────────┤
+│                                                                                  │
+│  ┌─────────────────────────────────────────────────────────────────────────┐    │
+│  │                         API SERVER POD (scraper)                         │    │
+│  │                                                                          │    │
+│  │   ┌──────────────────┐     ┌────────────────────────────────────────┐   │    │
+│  │   │   Express API    │     │         TaskScheduler                   │   │    │
+│  │   │                  │     │   (src/services/task-scheduler.ts)      │   │    │
+│  │   │  /api/job-queue  │     │                                         │   │    │
+│  │   │  /api/tasks      │     │   • Polls every 60s                     │   │    │
+│  │   │  /api/schedules  │     │   • Checks task_schedules table         │   │    │
+│  │   └────────┬─────────┘     │   • SELECT FOR UPDATE SKIP LOCKED       │   │    │
+│  │            │               │   • Generates tasks when due            │   │    │
+│  │            │               └──────────────────┬─────────────────────┘   │    │
+│  │            │                                  │                          │    │
+│  └────────────┼──────────────────────────────────┼──────────────────────────┘    │
+│               │                                  │                               │
+│               │         ┌────────────────────────┘                               │
+│               │         │                                                        │
+│               ▼         ▼                                                        │
+│  ┌─────────────────────────────────────────────────────────────────────────┐    │
+│  │                          POSTGRESQL DATABASE                             │    │
+│  │                                                                          │    │
+│  │   ┌─────────────────────┐        ┌─────────────────────┐                │    │
+│  │   │   task_schedules    │        │    worker_tasks     │                │    │
+│  │   │                     │        │                     │                │    │
+│  │   │ • product_refresh   │───────►│ • pending tasks     │                │    │
+│  │   │ • store_discovery   │ create │ • claimed tasks     │                │    │
+│  │   │ • analytics_refresh │ tasks  │ • running tasks     │                │    │
+│  │   │                     │        │ • completed tasks   │                │    │
+│  │   │ next_run_at         │        │                     │                │    │
+│  │   │ last_run_at         │        │ role, dispensary_id │                │    │
+│  │   │ interval_hours      │        │ priority, status    │                │    │
+│  │   └─────────────────────┘        └──────────┬──────────┘                │    │
+│  │                                             │                            │    │
+│  └─────────────────────────────────────────────┼────────────────────────────┘    │
+│                                                │                                  │
+│                         ┌──────────────────────┘                                  │
+│                         │ Workers poll for tasks                                  │
+│                         │ (SELECT FOR UPDATE SKIP LOCKED)                         │
+│                         ▼                                                         │
+│  ┌─────────────────────────────────────────────────────────────────────────┐    │
+│  │                    WORKER PODS (StatefulSet: scraper-worker)             │    │
+│  │                                                                          │    │
+│  │   ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐    │    │
+│  │   │  Worker 0   │  │  Worker 1   │  │  Worker 2   │  │  Worker N   │    │    │
+│  │   │             │  │             │  │             │  │             │    │    │
+│  │   │ task-worker │  │ task-worker │  │ task-worker │  │ task-worker │    │    │
+│  │   │     .ts     │  │     .ts     │  │     .ts     │  │     .ts     │    │    │
+│  │   └─────────────┘  └─────────────┘  └─────────────┘  └─────────────┘    │    │
+│  │                                                                          │    │
+│  └──────────────────────────────────────────────────────────────────────────┘    │
+│                                                                                  │
+└──────────────────────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Startup Sequence
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                        API SERVER STARTUP                                    │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                              │
+│   1. Express app initializes                                                 │
+│                    │                                                         │
+│                    ▼                                                         │
+│   2. runAutoMigrations()                                                     │
+│      • Runs pending migrations (including 079_task_schedules.sql)           │
+│                    │                                                         │
+│                    ▼                                                         │
+│   3. initializeMinio() / initializeImageStorage()                           │
+│                    │                                                         │
+│                    ▼                                                         │
+│   4. cleanupOrphanedJobs()                                                   │
+│                    │                                                         │
+│                    ▼                                                         │
+│   5. taskScheduler.start()  ◄─── NEW (per TASK_WORKFLOW_2024-12-10.md)      │
+│      │                                                                       │
+│      ├── Recover stale tasks (workers that died)                            │
+│      ├── Ensure default schedules exist in task_schedules                   │
+│      ├── Check and run any due schedules immediately                        │
+│      └── Start 60-second poll interval                                      │
+│                    │                                                         │
+│                    ▼                                                         │
+│   6. app.listen(PORT)                                                        │
+│                                                                              │
+└─────────────────────────────────────────────────────────────────────────────┘
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                        WORKER POD STARTUP                                    │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                              │
+│   1. K8s starts pod from StatefulSet                                        │
+│                    │                                                         │
+│                    ▼                                                         │
+│   2. TaskWorker.constructor()                                               │
+│      • Create DB pool                                                        │
+│      • Create CrawlRotator                                                   │
+│                    │                                                         │
+│                    ▼                                                         │
+│   3. initializeStealth()                                                    │
+│      • Load proxies from DB (REQUIRED - fails if none)                      │
+│      • Wire rotator to Dutchie client                                       │
+│                    │                                                         │
+│                    ▼                                                         │
+│   4. register() with API                                                    │
+│      • Optional - continues if fails                                         │
+│                    │                                                         │
+│                    ▼                                                         │
+│   5. startRegistryHeartbeat() every 30s                                     │
+│                    │                                                         │
+│                    ▼                                                         │
+│   6. processNextTask() loop                                                 │
+│      │                                                                       │
+│      ├── Poll for pending task (FOR UPDATE SKIP LOCKED)                     │
+│      ├── Claim task atomically                                              │
+│      ├── Execute handler (product_refresh, store_discovery, etc.)           │
+│      ├── Mark complete/failed                                               │
+│      ├── Chain next task if applicable                                      │
+│      └── Loop                                                               │
+│                                                                              │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Schedule Flow
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                     SCHEDULER POLL (every 60 seconds)                        │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                              │
+│   BEGIN TRANSACTION                                                          │
+│         │                                                                    │
+│         ▼                                                                    │
+│   SELECT * FROM task_schedules                                              │
+│   WHERE enabled = true AND next_run_at <= NOW()                             │
+│   FOR UPDATE SKIP LOCKED  ◄─── Prevents duplicate execution across replicas │
+│         │                                                                    │
+│         ▼                                                                    │
+│   For each due schedule:                                                     │
+│         │                                                                    │
+│         ├── product_refresh_all                                             │
+│         │   └─► Query dispensaries needing crawl                            │
+│         │   └─► Create product_refresh tasks in worker_tasks                │
+│         │                                                                    │
+│         ├── store_discovery_dutchie                                         │
+│         │   └─► Create single store_discovery task                          │
+│         │                                                                    │
+│         └── analytics_refresh                                                │
+│             └─► Create single analytics_refresh task                        │
+│         │                                                                    │
+│         ▼                                                                    │
+│   UPDATE task_schedules SET                                                  │
+│     last_run_at = NOW(),                                                     │
+│     next_run_at = NOW() + interval_hours                                    │
+│         │                                                                    │
+│         ▼                                                                    │
+│   COMMIT                                                                     │
+│                                                                              │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Task Lifecycle
+
+```
+                                    ┌──────────┐
+                                    │ SCHEDULE │
+                                    │   DUE    │
+                                    └────┬─────┘
+                                         │
+                                         ▼
+┌──────────────┐    claim    ┌──────────────┐    start    ┌──────────────┐
+│   PENDING    │────────────►│   CLAIMED    │────────────►│   RUNNING    │
+└──────────────┘             └──────────────┘             └──────┬───────┘
+       ▲                                                        │
+       │                                         ┌──────────────┼──────────────┐
+       │ retry                                   │              │              │
+       │ (if retries < max)                      ▼              ▼              ▼
+       │                                  ┌──────────┐   ┌──────────┐   ┌──────────┐
+       └──────────────────────────────────│  FAILED  │   │ COMPLETED│   │  STALE   │
+                                          └──────────┘   └──────────┘   └────┬─────┘
+                                                                              │
+                                                              recover_stale_tasks()
+                                                                              │
+                                                                              ▼
+                                                                        ┌──────────┐
+                                                                        │ PENDING  │
+                                                                        └──────────┘
+```
+
+---
+
+## Database Tables
+
+### task_schedules (NEW - migration 079)
+
+Stores schedule definitions. Survives restarts.
+
+```sql
+CREATE TABLE task_schedules (
+  id SERIAL PRIMARY KEY,
+  name VARCHAR(100) NOT NULL UNIQUE,
+  role VARCHAR(50) NOT NULL,        -- product_refresh, store_discovery, etc.
+  enabled BOOLEAN DEFAULT TRUE,
+  interval_hours INTEGER NOT NULL,  -- How often to run
+  priority INTEGER DEFAULT 0,       -- Task priority when created
+  state_code VARCHAR(2),            -- Optional filter
+  last_run_at TIMESTAMPTZ,          -- When it last ran
+  next_run_at TIMESTAMPTZ,          -- When it's due next
+  last_task_count INTEGER,          -- Tasks created last run
+  last_error TEXT                   -- Error message if failed
+);
+```
+
+### worker_tasks (migration 074)
+
+The task queue. Workers pull from here.
+
+```sql
+CREATE TABLE worker_tasks (
+  id SERIAL PRIMARY KEY,
+  role task_role NOT NULL,          -- What type of work
+  dispensary_id INTEGER,            -- Which store (if applicable)
+  platform VARCHAR(50),             -- Which platform
+  status task_status DEFAULT 'pending',
+  priority INTEGER DEFAULT 0,       -- Higher = process first
+  scheduled_for TIMESTAMP,          -- Don't process before this time
+  worker_id VARCHAR(100),           -- Which worker claimed it
+  claimed_at TIMESTAMP,
+  started_at TIMESTAMP,
+  completed_at TIMESTAMP,
+  last_heartbeat_at TIMESTAMP,      -- For stale detection
+  result JSONB,
+  error_message TEXT,
+  retry_count INTEGER DEFAULT 0,
+  max_retries INTEGER DEFAULT 3
+);
+```
+
+---
+
+## Default Schedules
+
+| Name | Role | Interval | Priority | Description |
+|------|------|----------|----------|-------------|
+| `payload_fetch_all` | payload_fetch | 4 hours | 0 | Fetch payloads from Dutchie API (chains to product_refresh) |
+| `store_discovery_dutchie` | store_discovery | 24 hours | 5 | Find new Dutchie stores |
+| `analytics_refresh` | analytics_refresh | 6 hours | 0 | Refresh MVs |
+
+---
+
+## Task Roles
+
+| Role | Description | Creates Tasks For |
+|------|-------------|-------------------|
+| `payload_fetch` | **NEW** - Fetch from Dutchie API, save to disk | Each dispensary needing crawl |
+| `product_refresh` | **CHANGED** - Read local payload, normalize, upsert to DB | Chained from payload_fetch |
+| `store_discovery` | Find new dispensaries, returns newStoreIds[] | Single task per platform |
+| `entry_point_discovery` | **DEPRECATED** - Resolve platform IDs | No longer used |
+| `product_discovery` | Initial product fetch for new stores | Chained from store_discovery |
+| `analytics_refresh` | Refresh MVs | Single global task |
+
+### Payload/Refresh Separation (2024-12-10)
+
+The crawl workflow is now split into two phases:
+
+```
+payload_fetch (scheduled every 4h)
+  └─► Hit Dutchie GraphQL API
+  └─► Save raw JSON to /storage/payloads/{year}/{month}/{day}/store_{id}_{ts}.json.gz
+  └─► Record metadata in raw_crawl_payloads table
+  └─► Queue product_refresh task with payload_id
+
+product_refresh (chained from payload_fetch)
+  └─► Load payload from filesystem (NOT from API)
+  └─► Normalize via DutchieNormalizer
+  └─► Upsert to store_products
+  └─► Create snapshots
+  └─► Track missing products
+  └─► Download images
+```
+
+**Benefits:**
+- **Retry-friendly**: If normalize fails, re-run product_refresh without re-crawling
+- **Replay-able**: Run product_refresh against any historical payload
+- **Faster refreshes**: Local file read vs network call
+- **Historical diffs**: Compare payloads to see what changed between crawls
+- **Less API pressure**: Only payload_fetch hits Dutchie
+
+---
+
+## Task Chaining
+
+Tasks automatically queue follow-up tasks upon successful completion. This creates two main flows:
+
+### Discovery Flow (New Stores)
+
+When `store_discovery` finds new dispensaries, they automatically get their initial product data:
+
+```
+store_discovery
+  └─► Discovers new locations via Dutchie GraphQL
+  └─► Auto-promotes valid locations to dispensaries table
+  └─► Collects newDispensaryIds[] from promotions
+  └─► Returns { newStoreIds: [...] } in result
+
+chainNextTask() detects newStoreIds
+  └─► Creates product_discovery task for each new store
+
+product_discovery
+  └─► Calls handlePayloadFetch() internally
+  └─► payload_fetch hits Dutchie API
+  └─► Saves raw JSON to /storage/payloads/
+  └─► Queues product_refresh task with payload_id
+
+product_refresh
+  └─► Loads payload from filesystem
+  └─► Normalizes and upserts to store_products
+  └─► Creates snapshots, downloads images
+```
+
+**Complete Discovery Chain:**
+```
+store_discovery → product_discovery → payload_fetch → product_refresh
+                        (internal call)    (queues next)
+```
+
+### Scheduled Flow (Existing Stores)
+
+For existing stores, `payload_fetch_all` schedule runs every 4 hours:
+
+```
+TaskScheduler (every 60s)
+  └─► Checks task_schedules for due schedules
+  └─► payload_fetch_all is due
+  └─► Generates payload_fetch task for each dispensary
+
+payload_fetch
+  └─► Hits Dutchie GraphQL API
+  └─► Saves raw JSON to /storage/payloads/
+  └─► Queues product_refresh task with payload_id
+
+product_refresh
+  └─► Loads payload from filesystem (NOT API)
+  └─► Normalizes via DutchieNormalizer
+  └─► Upserts to store_products
+  └─► Creates snapshots
+```
+
+**Complete Scheduled Chain:**
+```
+payload_fetch → product_refresh
+  (queues)        (reads local)
+```
+
+### Chaining Implementation
+
+Task chaining is handled in two places:
+
+1. **Internal chaining (handler calls handler):**
+   - `product_discovery` calls `handlePayloadFetch()` directly
+
+2. **External chaining (chainNextTask() in task-service.ts):**
+   - Called after task completion
+   - `store_discovery` → queues `product_discovery` for each newStoreId
+
+3. **Queue-based chaining (taskService.createTask):**
+   - `payload_fetch` queues `product_refresh` with `payload: { payload_id }`
+
+---
+
+## Payload API Endpoints
+
+Raw crawl payloads can be accessed via the Payloads API:
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `GET /api/payloads` | GET | List payload metadata (paginated) |
+| `GET /api/payloads/:id` | GET | Get payload metadata by ID |
+| `GET /api/payloads/:id/data` | GET | Get full payload JSON (decompressed) |
+| `GET /api/payloads/store/:dispensaryId` | GET | List payloads for a store |
+| `GET /api/payloads/store/:dispensaryId/latest` | GET | Get latest payload for a store |
+| `GET /api/payloads/store/:dispensaryId/diff` | GET | Diff two payloads for changes |
+
+### Payload Diff Response
+
+The diff endpoint returns:
+```json
+{
+  "success": true,
+  "from": { "id": 123, "fetchedAt": "...", "productCount": 100 },
+  "to": { "id": 456, "fetchedAt": "...", "productCount": 105 },
+  "diff": {
+    "added": 10,
+    "removed": 5,
+    "priceChanges": 8,
+    "stockChanges": 12
+  },
+  "details": {
+    "added": [...],
+    "removed": [...],
+    "priceChanges": [...],
+    "stockChanges": [...]
+  }
+}
+```
+
+---
+
+## API Endpoints
+
+### Schedules (NEW)
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `GET /api/schedules` | GET | List all schedules |
+| `PUT /api/schedules/:id` | PUT | Update schedule |
+| `POST /api/schedules/:id/trigger` | POST | Run schedule immediately |
+
+### Task Creation (rewired 2024-12-10)
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `POST /api/job-queue/enqueue` | POST | Create single task |
+| `POST /api/job-queue/enqueue-batch` | POST | Create batch tasks |
+| `POST /api/job-queue/enqueue-state` | POST | Create tasks for state |
+| `POST /api/tasks` | POST | Direct task creation |
+
+### Task Management
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `GET /api/tasks` | GET | List tasks |
+| `GET /api/tasks/:id` | GET | Get single task |
+| `GET /api/tasks/counts` | GET | Task counts by status |
+| `POST /api/tasks/recover-stale` | POST | Recover stale tasks |
+
+---
+
+## Key Files
+
+| File | Purpose |
+|------|---------|
+| `src/services/task-scheduler.ts` | **NEW** - DB-driven scheduler |
+| `src/tasks/task-worker.ts` | Worker that processes tasks |
+| `src/tasks/task-service.ts` | Task CRUD operations |
+| `src/tasks/handlers/payload-fetch.ts` | **NEW** - Fetches from API, saves to disk |
+| `src/tasks/handlers/product-refresh.ts` | **CHANGED** - Reads from disk, processes to DB |
+| `src/utils/payload-storage.ts` | **NEW** - Payload save/load utilities |
+| `src/routes/tasks.ts` | Task API endpoints |
+| `src/routes/job-queue.ts` | Job Queue UI endpoints (rewired) |
+| `migrations/079_task_schedules.sql` | Schedule table |
+| `migrations/080_raw_crawl_payloads.sql` | Payload metadata table |
+| `migrations/081_payload_fetch_columns.sql` | payload, last_fetch_at columns |
+| `migrations/074_worker_task_queue.sql` | Task queue table |
+
+---
+
+## Legacy Code (DEPRECATED)
+
+| File | Status | Replacement |
+|------|--------|-------------|
+| `src/services/scheduler.ts` | DEPRECATED | `task-scheduler.ts` |
+| `dispensary_crawl_jobs` table | ORPHANED | `worker_tasks` |
+| `job_schedules` table | LEGACY | `task_schedules` |
+
+---
+
+## Dashboard Integration
+
+Both pages remain wired to the dashboard:
+
+| Page | Data Source | Actions |
+|------|-------------|---------|
+| **Job Queue** | `worker_tasks`, `task_schedules` | Create tasks, view schedules |
+| **Task Queue** | `worker_tasks` | View tasks, recover stale |
+
+---
+
+## Multi-Replica Safety
+
+The scheduler uses `SELECT FOR UPDATE SKIP LOCKED` to ensure:
+
+1. **Only one replica** executes a schedule at a time
+2. **No duplicate tasks** created
+3. **Survives pod restarts** - state in DB, not memory
+4. **Self-healing** - recovers stale tasks on startup
+
+```sql
+-- This query is atomic across all API server replicas
+SELECT * FROM task_schedules
+WHERE enabled = true AND next_run_at <= NOW()
+FOR UPDATE SKIP LOCKED
+```
+
+---
+
+## Worker Scaling (K8s)
+
+Workers run as a StatefulSet in Kubernetes. You can scale from the admin UI or CLI.
+
+### From Admin UI
+
+The Workers page (`/admin/workers`) provides:
+- Current replica count display
+- Scale up/down buttons
+- Target replica input
+
+### API Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `GET /api/workers/k8s/replicas` | GET | Get current/desired replica counts |
+| `POST /api/workers/k8s/scale` | POST | Scale to N replicas (body: `{ replicas: N }`) |
+
+### From CLI
+
+```bash
+# View current replicas
+kubectl get statefulset scraper-worker -n dispensary-scraper
+
+# Scale to 10 workers
+kubectl scale statefulset scraper-worker -n dispensary-scraper --replicas=10
+
+# Scale down to 3 workers
+kubectl scale statefulset scraper-worker -n dispensary-scraper --replicas=3
+```
+
+### Configuration
+
+Environment variables for the API server:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `K8S_NAMESPACE` | `dispensary-scraper` | Kubernetes namespace |
+| `K8S_WORKER_STATEFULSET` | `scraper-worker` | StatefulSet name |
+
+### RBAC Requirements
+
+The API server pod needs these K8s permissions:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: worker-scaler
+  namespace: dispensary-scraper
+rules:
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: scraper-worker-scaler
+  namespace: dispensary-scraper
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: dispensary-scraper
+roleRef:
+  kind: Role
+  name: worker-scaler
+  apiGroup: rbac.authorization.k8s.io
+```

backend/docs/WORKER_TASK_ARCHITECTURE.md

@@ -362,6 +362,148 @@ SET status = 'pending', retry_count = retry_count + 1
 WHERE status = 'failed' AND retry_count < max_retries;
 ```
 
+## Concurrent Task Processing (Added 2024-12)
+
+Workers can now process multiple tasks concurrently within a single worker instance. This improves throughput by utilizing async I/O efficiently.
+
+### Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                         Pod (K8s)                           │
+│                                                             │
+│  ┌─────────────────────────────────────────────────────┐   │
+│  │                    TaskWorker                        │   │
+│  │                                                      │   │
+│  │  ┌─────────┐  ┌─────────┐  ┌─────────┐             │   │
+│  │  │ Task 1  │  │ Task 2  │  │ Task 3  │  (concurrent)│   │
+│  │  └─────────┘  └─────────┘  └─────────┘             │   │
+│  │                                                      │   │
+│  │  Resource Monitor                                    │   │
+│  │  ├── Memory: 65% (threshold: 85%)                   │   │
+│  │  ├── CPU: 45% (threshold: 90%)                      │   │
+│  │  └── Status: Normal                                  │   │
+│  └─────────────────────────────────────────────────────┘   │
+└─────────────────────────────────────────────────────────────┘
+```
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MAX_CONCURRENT_TASKS` | 3 | Maximum tasks a worker will run concurrently |
+| `MEMORY_BACKOFF_THRESHOLD` | 0.85 | Back off when heap memory exceeds 85% |
+| `CPU_BACKOFF_THRESHOLD` | 0.90 | Back off when CPU exceeds 90% |
+| `BACKOFF_DURATION_MS` | 10000 | How long to wait when backing off (10s) |
+
+### How It Works
+
+1. **Main Loop**: Worker continuously tries to fill up to `MAX_CONCURRENT_TASKS`
+2. **Resource Monitoring**: Before claiming a new task, worker checks memory and CPU
+3. **Backoff**: If resources exceed thresholds, worker pauses and stops claiming new tasks
+4. **Concurrent Execution**: Tasks run in parallel using `Promise` - they don't block each other
+5. **Graceful Shutdown**: On SIGTERM/decommission, worker stops claiming but waits for active tasks
+
+### Resource Monitoring
+
+```typescript
+// ResourceStats interface
+interface ResourceStats {
+  memoryPercent: number;    // Current heap usage as decimal (0.0-1.0)
+  memoryMb: number;         // Current heap used in MB
+  memoryTotalMb: number;    // Total heap available in MB
+  cpuPercent: number;       // CPU usage as percentage (0-100)
+  isBackingOff: boolean;    // True if worker is in backoff state
+  backoffReason: string;    // Why the worker is backing off
+}
+```
+
+### Heartbeat Data
+
+Workers report the following in their heartbeat:
+
+```json
+{
+  "worker_id": "worker-abc123",
+  "current_task_id": 456,
+  "current_task_ids": [456, 457, 458],
+  "active_task_count": 3,
+  "max_concurrent_tasks": 3,
+  "status": "active",
+  "resources": {
+    "memory_mb": 256,
+    "memory_total_mb": 512,
+    "memory_rss_mb": 320,
+    "memory_percent": 50,
+    "cpu_user_ms": 12500,
+    "cpu_system_ms": 3200,
+    "cpu_percent": 45,
+    "is_backing_off": false,
+    "backoff_reason": null
+  }
+}
+```
+
+### Backoff Behavior
+
+When resources exceed thresholds:
+
+1. Worker logs the backoff reason:
+   ```
+   [TaskWorker] MyWorker backing off: Memory at 87.3% (threshold: 85%)
+   ```
+
+2. Worker stops claiming new tasks but continues existing tasks
+
+3. After `BACKOFF_DURATION_MS`, worker rechecks resources
+
+4. When resources return to normal:
+   ```
+   [TaskWorker] MyWorker resuming normal operation
+   ```
+
+### UI Display
+
+The Workers Dashboard shows:
+
+- **Tasks Column**: `2/3 tasks` (active/max concurrent)
+- **Resources Column**: Memory % and CPU % with color coding
+  - Green: < 50%
+  - Yellow: 50-74%
+  - Amber: 75-89%
+  - Red: 90%+
+- **Backing Off**: Orange warning badge when worker is in backoff state
+
+### Task Count Badge Details
+
+```
+┌─────────────────────────────────────────────┐
+│ Worker: "MyWorker"                          │
+│ Tasks: 2/3 tasks  #456, #457                │
+│ Resources: 🧠 65%  💻 45%                    │
+│ Status: ● Active                            │
+└─────────────────────────────────────────────┘
+```
+
+### Best Practices
+
+1. **Start Conservative**: Use `MAX_CONCURRENT_TASKS=3` initially
+2. **Monitor Resources**: Watch for frequent backoffs in logs
+3. **Tune Per Workload**: I/O-bound tasks benefit from higher concurrency
+4. **Scale Horizontally**: Add more pods rather than cranking concurrency too high
+
+### Code References
+
+| File | Purpose |
+|------|---------|
+| `src/tasks/task-worker.ts:68-71` | Concurrency environment variables |
+| `src/tasks/task-worker.ts:104-111` | ResourceStats interface |
+| `src/tasks/task-worker.ts:149-179` | getResourceStats() method |
+| `src/tasks/task-worker.ts:184-196` | shouldBackOff() method |
+| `src/tasks/task-worker.ts:462-516` | mainLoop() with concurrent claiming |
+| `src/routes/worker-registry.ts:148-195` | Heartbeat endpoint handling |
+| `cannaiq/src/pages/WorkersDashboard.tsx:233-305` | UI components for resources |
+
 ## Monitoring
 
 ### Logs

backend/k8s/cronjob-ip2location.yaml

@@ -0,0 +1,69 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: ip2location-update
+  namespace: default
+spec:
+  # Run on the 1st of every month at 3am UTC
+  schedule: "0 3 1 * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: ip2location-updater
+            image: curlimages/curl:latest
+            command:
+            - /bin/sh
+            - -c
+            - |
+              set -e
+              echo "Downloading IP2Location LITE DB5..."
+
+              # Download to temp
+              cd /tmp
+              curl -L -o ip2location.zip "https://www.ip2location.com/download/?token=${IP2LOCATION_TOKEN}&file=DB5LITEBIN"
+
+              # Extract
+              unzip -o ip2location.zip
+
+              # Find and copy the BIN file
+              BIN_FILE=$(ls *.BIN 2>/dev/null | head -1)
+              if [ -z "$BIN_FILE" ]; then
+                echo "ERROR: No BIN file found"
+                exit 1
+              fi
+
+              # Copy to shared volume
+              cp "$BIN_FILE" /data/IP2LOCATION-LITE-DB5.BIN
+
+              echo "Done! Database updated: /data/IP2LOCATION-LITE-DB5.BIN"
+            env:
+            - name: IP2LOCATION_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: dutchie-backend-secret
+                  key: IP2LOCATION_TOKEN
+            volumeMounts:
+            - name: ip2location-data
+              mountPath: /data
+          restartPolicy: OnFailure
+          volumes:
+          - name: ip2location-data
+            persistentVolumeClaim:
+              claimName: ip2location-pvc
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ip2location-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Mi

backend/k8s/deployment.yaml

@@ -26,6 +26,12 @@ spec:
             name: dutchie-backend-config
         - secretRef:
             name: dutchie-backend-secret
+        env:
+        - name: IP2LOCATION_DB_PATH
+          value: /data/ip2location/IP2LOCATION-LITE-DB5.BIN
+        volumeMounts:
+        - name: ip2location-data
+          mountPath: /data/ip2location
         resources:
           requests:
             memory: "256Mi"
@@ -45,3 +51,7 @@ spec:
             port: 3010
           initialDelaySeconds: 5
           periodSeconds: 5
+      volumes:
+      - name: ip2location-data
+        persistentVolumeClaim:
+          claimName: ip2location-pvc

backend/migrations/074_worker_commands.sql

@@ -0,0 +1,27 @@
+-- Migration: Worker Commands Table
+-- Purpose: Store commands for workers (decommission, etc.)
+-- Workers poll this table after each task to check for commands
+
+CREATE TABLE IF NOT EXISTS worker_commands (
+  id SERIAL PRIMARY KEY,
+  worker_id TEXT NOT NULL,
+  command TEXT NOT NULL,  -- 'decommission', 'pause', 'resume'
+  reason TEXT,
+  issued_by TEXT,
+  issued_at TIMESTAMPTZ DEFAULT NOW(),
+  acknowledged_at TIMESTAMPTZ,
+  executed_at TIMESTAMPTZ,
+  status TEXT DEFAULT 'pending'  -- 'pending', 'acknowledged', 'executed', 'cancelled'
+);
+
+-- Index for worker lookups
+CREATE INDEX IF NOT EXISTS idx_worker_commands_worker_id ON worker_commands(worker_id);
+CREATE INDEX IF NOT EXISTS idx_worker_commands_pending ON worker_commands(worker_id, status) WHERE status = 'pending';
+
+-- Add decommission_requested column to worker_registry for quick checks
+ALTER TABLE worker_registry ADD COLUMN IF NOT EXISTS decommission_requested BOOLEAN DEFAULT FALSE;
+ALTER TABLE worker_registry ADD COLUMN IF NOT EXISTS decommission_reason TEXT;
+ALTER TABLE worker_registry ADD COLUMN IF NOT EXISTS decommission_requested_at TIMESTAMPTZ;
+
+-- Comment
+COMMENT ON TABLE worker_commands IS 'Commands issued to workers (decommission after task, pause, etc.)';

backend/migrations/076_visitor_analytics.sql

@@ -0,0 +1,71 @@
+-- Visitor location analytics for Findagram
+-- Tracks visitor locations to understand popular areas
+
+CREATE TABLE IF NOT EXISTS visitor_locations (
+  id SERIAL PRIMARY KEY,
+
+  -- Location data (from IP lookup)
+  ip_hash VARCHAR(64),           -- Hashed IP for privacy (SHA256)
+  city VARCHAR(100),
+  state VARCHAR(100),
+  state_code VARCHAR(10),
+  country VARCHAR(100),
+  country_code VARCHAR(10),
+  latitude DECIMAL(10, 7),
+  longitude DECIMAL(10, 7),
+
+  -- Visit metadata
+  domain VARCHAR(50) NOT NULL,    -- 'findagram.co', 'findadispo.com', etc.
+  page_path VARCHAR(255),         -- '/products', '/dispensaries/123', etc.
+  referrer VARCHAR(500),
+  user_agent VARCHAR(500),
+
+  -- Session tracking
+  session_id VARCHAR(64),         -- For grouping page views in a session
+
+  -- Timestamps
+  created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Indexes for analytics queries
+CREATE INDEX IF NOT EXISTS idx_visitor_locations_domain ON visitor_locations(domain);
+CREATE INDEX IF NOT EXISTS idx_visitor_locations_city_state ON visitor_locations(city, state_code);
+CREATE INDEX IF NOT EXISTS idx_visitor_locations_created_at ON visitor_locations(created_at);
+CREATE INDEX IF NOT EXISTS idx_visitor_locations_session ON visitor_locations(session_id);
+
+-- Aggregated daily stats (materialized for performance)
+CREATE TABLE IF NOT EXISTS visitor_location_stats (
+  id SERIAL PRIMARY KEY,
+  date DATE NOT NULL,
+  domain VARCHAR(50) NOT NULL,
+  city VARCHAR(100),
+  state VARCHAR(100),
+  state_code VARCHAR(10),
+  country_code VARCHAR(10),
+
+  -- Metrics
+  visit_count INTEGER DEFAULT 0,
+  unique_sessions INTEGER DEFAULT 0,
+
+  UNIQUE(date, domain, city, state_code, country_code)
+);
+
+CREATE INDEX IF NOT EXISTS idx_visitor_stats_date ON visitor_location_stats(date);
+CREATE INDEX IF NOT EXISTS idx_visitor_stats_domain ON visitor_location_stats(domain);
+CREATE INDEX IF NOT EXISTS idx_visitor_stats_state ON visitor_location_stats(state_code);
+
+-- View for easy querying of top locations
+CREATE OR REPLACE VIEW v_top_visitor_locations AS
+SELECT
+  domain,
+  city,
+  state,
+  state_code,
+  country_code,
+  COUNT(*) as total_visits,
+  COUNT(DISTINCT session_id) as unique_sessions,
+  MAX(created_at) as last_visit
+FROM visitor_locations
+WHERE created_at > NOW() - INTERVAL '30 days'
+GROUP BY domain, city, state, state_code, country_code
+ORDER BY total_visits DESC;

backend/migrations/076_worker_registry.sql

@@ -0,0 +1,141 @@
+-- Migration 076: Worker Registry for Dynamic Workers
+-- Workers register on startup, receive a friendly name, and report heartbeats
+
+-- Name pool for workers (expandable, no hardcoding)
+CREATE TABLE IF NOT EXISTS worker_name_pool (
+  id SERIAL PRIMARY KEY,
+  name VARCHAR(50) UNIQUE NOT NULL,
+  in_use BOOLEAN DEFAULT FALSE,
+  assigned_to VARCHAR(100), -- worker_id
+  assigned_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Seed with initial names (can add more via API)
+INSERT INTO worker_name_pool (name) VALUES
+  ('Alice'), ('Bella'), ('Clara'), ('Diana'), ('Elena'),
+  ('Fiona'), ('Grace'), ('Hazel'), ('Iris'), ('Julia'),
+  ('Katie'), ('Luna'), ('Mia'), ('Nora'), ('Olive'),
+  ('Pearl'), ('Quinn'), ('Rosa'), ('Sara'), ('Tara'),
+  ('Uma'), ('Vera'), ('Wendy'), ('Xena'), ('Yuki'), ('Zara'),
+  ('Amber'), ('Blake'), ('Coral'), ('Dawn'), ('Echo'),
+  ('Fleur'), ('Gem'), ('Haven'), ('Ivy'), ('Jade'),
+  ('Kira'), ('Lotus'), ('Maple'), ('Nova'), ('Onyx'),
+  ('Pixel'), ('Quest'), ('Raven'), ('Sage'), ('Terra'),
+  ('Unity'), ('Violet'), ('Willow'), ('Xylo'), ('Yara'), ('Zen')
+ON CONFLICT (name) DO NOTHING;
+
+-- Worker registry - tracks active workers
+CREATE TABLE IF NOT EXISTS worker_registry (
+  id SERIAL PRIMARY KEY,
+  worker_id VARCHAR(100) UNIQUE NOT NULL,  -- e.g., "pod-abc123" or uuid
+  friendly_name VARCHAR(50),                -- assigned from pool
+  role VARCHAR(50) NOT NULL,                -- task role
+  pod_name VARCHAR(100),                    -- k8s pod name
+  hostname VARCHAR(100),                    -- machine hostname
+  ip_address VARCHAR(50),                   -- worker IP
+  status VARCHAR(20) DEFAULT 'starting',    -- starting, active, idle, offline, terminated
+  started_at TIMESTAMPTZ DEFAULT NOW(),
+  last_heartbeat_at TIMESTAMPTZ DEFAULT NOW(),
+  last_task_at TIMESTAMPTZ,
+  tasks_completed INTEGER DEFAULT 0,
+  tasks_failed INTEGER DEFAULT 0,
+  current_task_id INTEGER,
+  metadata JSONB DEFAULT '{}',
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  updated_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Indexes for worker registry
+CREATE INDEX IF NOT EXISTS idx_worker_registry_status ON worker_registry(status);
+CREATE INDEX IF NOT EXISTS idx_worker_registry_role ON worker_registry(role);
+CREATE INDEX IF NOT EXISTS idx_worker_registry_heartbeat ON worker_registry(last_heartbeat_at);
+
+-- Function to assign a name to a new worker
+CREATE OR REPLACE FUNCTION assign_worker_name(p_worker_id VARCHAR(100))
+RETURNS VARCHAR(50) AS $$
+DECLARE
+  v_name VARCHAR(50);
+BEGIN
+  -- Try to get an unused name
+  UPDATE worker_name_pool
+  SET in_use = TRUE, assigned_to = p_worker_id, assigned_at = NOW()
+  WHERE id = (
+    SELECT id FROM worker_name_pool
+    WHERE in_use = FALSE
+    ORDER BY RANDOM()
+    LIMIT 1
+    FOR UPDATE SKIP LOCKED
+  )
+  RETURNING name INTO v_name;
+
+  -- If no names available, generate one
+  IF v_name IS NULL THEN
+    v_name := 'Worker-' || SUBSTRING(p_worker_id FROM 1 FOR 8);
+  END IF;
+
+  RETURN v_name;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to release a worker's name back to the pool
+CREATE OR REPLACE FUNCTION release_worker_name(p_worker_id VARCHAR(100))
+RETURNS VOID AS $$
+BEGIN
+  UPDATE worker_name_pool
+  SET in_use = FALSE, assigned_to = NULL, assigned_at = NULL
+  WHERE assigned_to = p_worker_id;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to mark stale workers as offline
+CREATE OR REPLACE FUNCTION mark_stale_workers(stale_threshold_minutes INTEGER DEFAULT 5)
+RETURNS INTEGER AS $$
+DECLARE
+  v_count INTEGER;
+BEGIN
+  UPDATE worker_registry
+  SET status = 'offline', updated_at = NOW()
+  WHERE status IN ('active', 'idle', 'starting')
+    AND last_heartbeat_at < NOW() - (stale_threshold_minutes || ' minutes')::INTERVAL
+  RETURNING COUNT(*) INTO v_count;
+
+  -- Release names from offline workers
+  PERFORM release_worker_name(worker_id)
+  FROM worker_registry
+  WHERE status = 'offline'
+    AND last_heartbeat_at < NOW() - INTERVAL '30 minutes';
+
+  RETURN COALESCE(v_count, 0);
+END;
+$$ LANGUAGE plpgsql;
+
+-- View for dashboard
+CREATE OR REPLACE VIEW v_active_workers AS
+SELECT
+  wr.id,
+  wr.worker_id,
+  wr.friendly_name,
+  wr.role,
+  wr.status,
+  wr.pod_name,
+  wr.hostname,
+  wr.started_at,
+  wr.last_heartbeat_at,
+  wr.last_task_at,
+  wr.tasks_completed,
+  wr.tasks_failed,
+  wr.current_task_id,
+  EXTRACT(EPOCH FROM (NOW() - wr.last_heartbeat_at)) as seconds_since_heartbeat,
+  CASE
+    WHEN wr.status = 'offline' THEN 'offline'
+    WHEN wr.last_heartbeat_at < NOW() - INTERVAL '2 minutes' THEN 'stale'
+    WHEN wr.current_task_id IS NOT NULL THEN 'busy'
+    ELSE 'ready'
+  END as health_status
+FROM worker_registry wr
+WHERE wr.status != 'terminated'
+ORDER BY wr.status = 'active' DESC, wr.last_heartbeat_at DESC;
+
+COMMENT ON TABLE worker_registry IS 'Tracks all workers that have registered with the system';
+COMMENT ON TABLE worker_name_pool IS 'Pool of friendly names for workers - expandable via API';

backend/migrations/077_click_events_location.sql

@@ -0,0 +1,35 @@
+-- Migration: Add visitor location and dispensary name to click events
+-- Captures where visitors are clicking from and which dispensary
+
+-- Add visitor location columns
+ALTER TABLE product_click_events
+ADD COLUMN IF NOT EXISTS visitor_city VARCHAR(100);
+
+ALTER TABLE product_click_events
+ADD COLUMN IF NOT EXISTS visitor_state VARCHAR(10);
+
+ALTER TABLE product_click_events
+ADD COLUMN IF NOT EXISTS visitor_lat DECIMAL(10, 7);
+
+ALTER TABLE product_click_events
+ADD COLUMN IF NOT EXISTS visitor_lng DECIMAL(10, 7);
+
+-- Add dispensary name for easier reporting
+ALTER TABLE product_click_events
+ADD COLUMN IF NOT EXISTS dispensary_name VARCHAR(255);
+
+-- Create index for location-based analytics
+CREATE INDEX IF NOT EXISTS idx_product_click_events_visitor_state
+ON product_click_events(visitor_state)
+WHERE visitor_state IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_product_click_events_visitor_city
+ON product_click_events(visitor_city)
+WHERE visitor_city IS NOT NULL;
+
+-- Add comments
+COMMENT ON COLUMN product_click_events.visitor_city IS 'City where the visitor is located (from IP geolocation)';
+COMMENT ON COLUMN product_click_events.visitor_state IS 'State where the visitor is located (from IP geolocation)';
+COMMENT ON COLUMN product_click_events.visitor_lat IS 'Visitor latitude (from IP geolocation)';
+COMMENT ON COLUMN product_click_events.visitor_lng IS 'Visitor longitude (from IP geolocation)';
+COMMENT ON COLUMN product_click_events.dispensary_name IS 'Name of the dispensary (denormalized for easier reporting)';

backend/migrations/078_proxy_consecutive_403.sql

@@ -0,0 +1,8 @@
+-- Migration 078: Add consecutive_403_count to proxies table
+-- Per workflow-12102025.md: Track consecutive 403s per proxy
+-- After 3 consecutive 403s with different fingerprints → disable proxy
+
+ALTER TABLE proxies ADD COLUMN IF NOT EXISTS consecutive_403_count INTEGER DEFAULT 0;
+
+-- Add comment explaining the column
+COMMENT ON COLUMN proxies.consecutive_403_count IS 'Tracks consecutive 403 blocks. Reset to 0 on success. Proxy disabled at 3.';

backend/migrations/079_task_schedules.sql

@@ -0,0 +1,49 @@
+-- Migration 079: Task Schedules for Database-Driven Scheduler
+-- Per TASK_WORKFLOW_2024-12-10.md: Replaces node-cron with DB-driven scheduling
+--
+-- 2024-12-10: Created for reliable, multi-replica-safe task scheduling
+
+-- task_schedules: Stores schedule definitions and state
+CREATE TABLE IF NOT EXISTS task_schedules (
+  id SERIAL PRIMARY KEY,
+  name VARCHAR(100) NOT NULL UNIQUE,
+  role VARCHAR(50) NOT NULL,  -- TaskRole: product_refresh, store_discovery, etc.
+  description TEXT,
+
+  -- Schedule configuration
+  enabled BOOLEAN DEFAULT TRUE,
+  interval_hours INTEGER NOT NULL DEFAULT 4,
+  priority INTEGER DEFAULT 0,
+
+  -- Optional scope filters
+  state_code VARCHAR(2),  -- NULL = all states
+  platform VARCHAR(50),   -- NULL = all platforms
+
+  -- Execution state (updated by scheduler)
+  last_run_at TIMESTAMPTZ,
+  next_run_at TIMESTAMPTZ,
+  last_task_count INTEGER DEFAULT 0,
+  last_error TEXT,
+
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  updated_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Indexes for scheduler queries
+CREATE INDEX IF NOT EXISTS idx_task_schedules_enabled ON task_schedules(enabled) WHERE enabled = TRUE;
+CREATE INDEX IF NOT EXISTS idx_task_schedules_next_run ON task_schedules(next_run_at) WHERE enabled = TRUE;
+
+-- Insert default schedules
+INSERT INTO task_schedules (name, role, interval_hours, priority, description, next_run_at)
+VALUES
+  ('product_refresh_all', 'product_refresh', 4, 0, 'Generate product refresh tasks for all crawl-enabled stores every 4 hours', NOW()),
+  ('store_discovery_dutchie', 'store_discovery', 24, 5, 'Discover new Dutchie stores daily', NOW()),
+  ('analytics_refresh', 'analytics_refresh', 6, 0, 'Refresh analytics materialized views every 6 hours', NOW())
+ON CONFLICT (name) DO NOTHING;
+
+-- Comment for documentation
+COMMENT ON TABLE task_schedules IS 'Database-driven task scheduler configuration. Per TASK_WORKFLOW_2024-12-10.md:
+- Schedules persist in DB (survive restarts)
+- Uses SELECT FOR UPDATE SKIP LOCKED for multi-replica safety
+- Scheduler polls every 60s and executes due schedules
+- Creates tasks in worker_tasks for task-worker.ts to process';

backend/migrations/080_raw_crawl_payloads.sql

@@ -0,0 +1,58 @@
+-- Migration 080: Raw Crawl Payloads Metadata Table
+-- Per TASK_WORKFLOW_2024-12-10.md: Store full GraphQL payloads for historical analysis
+--
+-- Design Pattern: Metadata/Payload Separation
+-- - Metadata (this table): Small, indexed, queryable
+-- - Payload (filesystem): Gzipped JSON at storage_path
+--
+-- Benefits:
+-- - Compare any two crawls to see what changed
+-- - Replay/re-normalize historical data if logic changes
+-- - Debug issues by seeing exactly what the API returned
+-- - DB stays small, backups stay fast
+--
+-- Storage location: /storage/payloads/{year}/{month}/{day}/store_{id}_{timestamp}.json.gz
+-- Compression: ~90% reduction (1.5MB -> 150KB per crawl)
+
+CREATE TABLE IF NOT EXISTS raw_crawl_payloads (
+  id SERIAL PRIMARY KEY,
+
+  -- Links to crawl tracking
+  crawl_run_id INTEGER REFERENCES crawl_runs(id) ON DELETE SET NULL,
+  dispensary_id INTEGER NOT NULL REFERENCES dispensaries(id) ON DELETE CASCADE,
+
+  -- File location (gzipped JSON)
+  storage_path TEXT NOT NULL,
+
+  -- Metadata for quick queries without loading file
+  product_count INTEGER NOT NULL DEFAULT 0,
+  size_bytes INTEGER,                    -- Compressed size
+  size_bytes_raw INTEGER,                -- Uncompressed size
+
+  -- Timestamps
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  -- Optional: checksum for integrity verification
+  checksum_sha256 VARCHAR(64)
+);
+
+-- Indexes for common queries
+CREATE INDEX IF NOT EXISTS idx_raw_crawl_payloads_dispensary
+  ON raw_crawl_payloads(dispensary_id);
+
+CREATE INDEX IF NOT EXISTS idx_raw_crawl_payloads_dispensary_fetched
+  ON raw_crawl_payloads(dispensary_id, fetched_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_raw_crawl_payloads_fetched
+  ON raw_crawl_payloads(fetched_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_raw_crawl_payloads_crawl_run
+  ON raw_crawl_payloads(crawl_run_id)
+  WHERE crawl_run_id IS NOT NULL;
+
+-- Comments
+COMMENT ON TABLE raw_crawl_payloads IS 'Metadata for raw GraphQL payloads stored on filesystem. Per TASK_WORKFLOW_2024-12-10.md: Full payloads enable historical diffs and replay.';
+COMMENT ON COLUMN raw_crawl_payloads.storage_path IS 'Path to gzipped JSON file, e.g. /storage/payloads/2024/12/10/store_123_1702234567.json.gz';
+COMMENT ON COLUMN raw_crawl_payloads.size_bytes IS 'Compressed file size in bytes';
+COMMENT ON COLUMN raw_crawl_payloads.size_bytes_raw IS 'Uncompressed payload size in bytes';

backend/migrations/081_payload_fetch_columns.sql

@@ -0,0 +1,37 @@
+-- Migration 081: Payload Fetch Columns
+-- Per TASK_WORKFLOW_2024-12-10.md: Separates API fetch from data processing
+--
+-- New architecture:
+-- - payload_fetch: Hits Dutchie API, saves raw payload to disk
+-- - product_refresh: Reads local payload, normalizes, upserts to DB
+--
+-- This migration adds:
+-- 1. payload column to worker_tasks (for task chaining data)
+-- 2. processed_at column to raw_crawl_payloads (track when payload was processed)
+-- 3. last_fetch_at column to dispensaries (track when last payload was fetched)
+
+-- Add payload column to worker_tasks for task chaining
+-- Used by payload_fetch to pass payload_id to product_refresh
+ALTER TABLE worker_tasks
+ADD COLUMN IF NOT EXISTS payload JSONB DEFAULT NULL;
+
+COMMENT ON COLUMN worker_tasks.payload IS 'Per TASK_WORKFLOW_2024-12-10.md: Task chaining data (e.g., payload_id from payload_fetch to product_refresh)';
+
+-- Add processed_at to raw_crawl_payloads
+-- Tracks when the payload was processed by product_refresh
+ALTER TABLE raw_crawl_payloads
+ADD COLUMN IF NOT EXISTS processed_at TIMESTAMPTZ DEFAULT NULL;
+
+COMMENT ON COLUMN raw_crawl_payloads.processed_at IS 'When this payload was processed by product_refresh handler';
+
+-- Index for finding unprocessed payloads
+CREATE INDEX IF NOT EXISTS idx_raw_crawl_payloads_unprocessed
+  ON raw_crawl_payloads(dispensary_id, fetched_at DESC)
+  WHERE processed_at IS NULL;
+
+-- Add last_fetch_at to dispensaries
+-- Tracks when the last payload was fetched (separate from last_crawl_at which is when processing completed)
+ALTER TABLE dispensaries
+ADD COLUMN IF NOT EXISTS last_fetch_at TIMESTAMPTZ DEFAULT NULL;
+
+COMMENT ON COLUMN dispensaries.last_fetch_at IS 'Per TASK_WORKFLOW_2024-12-10.md: When last payload was fetched from API (separate from last_crawl_at which is when processing completed)';

backend/migrations/082_proxy_notification_trigger.sql

@@ -0,0 +1,27 @@
+-- Migration: 082_proxy_notification_trigger
+-- Date: 2024-12-11
+-- Description: Add PostgreSQL NOTIFY trigger to alert workers when proxies are added
+
+-- Create function to notify workers when active proxy is added/activated
+CREATE OR REPLACE FUNCTION notify_proxy_added()
+RETURNS TRIGGER AS $$
+BEGIN
+  -- Only notify if proxy is active
+  IF NEW.active = true THEN
+    PERFORM pg_notify('proxy_added', NEW.id::text);
+  END IF;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Drop existing trigger if any
+DROP TRIGGER IF EXISTS proxy_added_trigger ON proxies;
+
+-- Create trigger on insert and update of active column
+CREATE TRIGGER proxy_added_trigger
+AFTER INSERT OR UPDATE OF active ON proxies
+FOR EACH ROW
+EXECUTE FUNCTION notify_proxy_added();
+
+COMMENT ON FUNCTION notify_proxy_added() IS
+'Sends PostgreSQL NOTIFY to proxy_added channel when an active proxy is added or activated. Workers LISTEN on this channel to wake up immediately.';

backend/migrations/083_discovery_runs.sql

@@ -0,0 +1,88 @@
+-- Migration 083: Discovery Run Tracking
+-- Tracks progress of store discovery runs step-by-step
+
+-- Main discovery runs table
+CREATE TABLE IF NOT EXISTS discovery_runs (
+  id SERIAL PRIMARY KEY,
+  platform VARCHAR(50) NOT NULL DEFAULT 'dutchie',
+  status VARCHAR(20) NOT NULL DEFAULT 'running', -- running, completed, failed
+  started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  finished_at TIMESTAMPTZ,
+  task_id INTEGER REFERENCES worker_task_queue(id),
+
+  -- Totals
+  states_total INTEGER DEFAULT 0,
+  states_completed INTEGER DEFAULT 0,
+  locations_discovered INTEGER DEFAULT 0,
+  locations_promoted INTEGER DEFAULT 0,
+  new_store_ids INTEGER[] DEFAULT '{}',
+
+  -- Error info
+  error_message TEXT,
+
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Per-state progress within a run
+CREATE TABLE IF NOT EXISTS discovery_run_states (
+  id SERIAL PRIMARY KEY,
+  run_id INTEGER NOT NULL REFERENCES discovery_runs(id) ON DELETE CASCADE,
+  state_code VARCHAR(2) NOT NULL,
+  status VARCHAR(20) NOT NULL DEFAULT 'pending', -- pending, running, completed, failed
+  started_at TIMESTAMPTZ,
+  finished_at TIMESTAMPTZ,
+
+  -- Results
+  cities_found INTEGER DEFAULT 0,
+  locations_found INTEGER DEFAULT 0,
+  locations_upserted INTEGER DEFAULT 0,
+  new_dispensary_ids INTEGER[] DEFAULT '{}',
+
+  -- Error info
+  error_message TEXT,
+
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  UNIQUE(run_id, state_code)
+);
+
+-- Step-by-step log for detailed progress tracking
+CREATE TABLE IF NOT EXISTS discovery_run_steps (
+  id SERIAL PRIMARY KEY,
+  run_id INTEGER NOT NULL REFERENCES discovery_runs(id) ON DELETE CASCADE,
+  state_code VARCHAR(2),
+  step_name VARCHAR(100) NOT NULL,
+  status VARCHAR(20) NOT NULL DEFAULT 'started', -- started, completed, failed
+  started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  finished_at TIMESTAMPTZ,
+
+  -- Details (JSON for flexibility)
+  details JSONB DEFAULT '{}',
+
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes for querying
+CREATE INDEX IF NOT EXISTS idx_discovery_runs_status ON discovery_runs(status);
+CREATE INDEX IF NOT EXISTS idx_discovery_runs_platform ON discovery_runs(platform);
+CREATE INDEX IF NOT EXISTS idx_discovery_runs_started_at ON discovery_runs(started_at DESC);
+CREATE INDEX IF NOT EXISTS idx_discovery_run_states_run_id ON discovery_run_states(run_id);
+CREATE INDEX IF NOT EXISTS idx_discovery_run_steps_run_id ON discovery_run_steps(run_id);
+
+-- View for latest run status per platform
+CREATE OR REPLACE VIEW v_latest_discovery_runs AS
+SELECT DISTINCT ON (platform)
+  id,
+  platform,
+  status,
+  started_at,
+  finished_at,
+  states_total,
+  states_completed,
+  locations_discovered,
+  locations_promoted,
+  array_length(new_store_ids, 1) as new_stores_count,
+  error_message,
+  EXTRACT(EPOCH FROM (COALESCE(finished_at, NOW()) - started_at)) as duration_seconds
+FROM discovery_runs
+ORDER BY platform, started_at DESC;

backend/node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "dutchie-menus-backend",
-  "version": "1.5.1",
+  "version": "1.6.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
@@ -46,6 +46,97 @@
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz",
       "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ=="
     },
+    "node_modules/@jsep-plugin/assignment": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@jsep-plugin/assignment/-/assignment-1.3.0.tgz",
+      "integrity": "sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==",
+      "engines": {
+        "node": ">= 10.16.0"
+      },
+      "peerDependencies": {
+        "jsep": "^0.4.0||^1.0.0"
+      }
+    },
+    "node_modules/@jsep-plugin/regex": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@jsep-plugin/regex/-/regex-1.0.4.tgz",
+      "integrity": "sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==",
+      "engines": {
+        "node": ">= 10.16.0"
+      },
+      "peerDependencies": {
+        "jsep": "^0.4.0||^1.0.0"
+      }
+    },
+    "node_modules/@kubernetes/client-node": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-1.4.0.tgz",
+      "integrity": "sha512-Zge3YvF7DJi264dU1b3wb/GmzR99JhUpqTvp+VGHfwZT+g7EOOYNScDJNZwXy9cszyIGPIs0VHr+kk8e95qqrA==",
+      "dependencies": {
+        "@types/js-yaml": "^4.0.1",
+        "@types/node": "^24.0.0",
+        "@types/node-fetch": "^2.6.13",
+        "@types/stream-buffers": "^3.0.3",
+        "form-data": "^4.0.0",
+        "hpagent": "^1.2.0",
+        "isomorphic-ws": "^5.0.0",
+        "js-yaml": "^4.1.0",
+        "jsonpath-plus": "^10.3.0",
+        "node-fetch": "^2.7.0",
+        "openid-client": "^6.1.3",
+        "rfc4648": "^1.3.0",
+        "socks-proxy-agent": "^8.0.4",
+        "stream-buffers": "^3.0.2",
+        "tar-fs": "^3.0.9",
+        "ws": "^8.18.2"
+      }
+    },
+    "node_modules/@kubernetes/client-node/node_modules/@types/node": {
+      "version": "24.10.3",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.3.tgz",
+      "integrity": "sha512-gqkrWUsS8hcm0r44yn7/xZeV1ERva/nLgrLxFRUGb7aoNMIJfZJ3AC261zDQuOAKC7MiXai1WCpYc48jAHoShQ==",
+      "dependencies": {
+        "undici-types": "~7.16.0"
+      }
+    },
+    "node_modules/@kubernetes/client-node/node_modules/tar-fs": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz",
+      "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==",
+      "dependencies": {
+        "pump": "^3.0.0",
+        "tar-stream": "^3.1.5"
+      },
+      "optionalDependencies": {
+        "bare-fs": "^4.0.1",
+        "bare-path": "^3.0.0"
+      }
+    },
+    "node_modules/@kubernetes/client-node/node_modules/undici-types": {
+      "version": "7.16.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
+      "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="
+    },
+    "node_modules/@kubernetes/client-node/node_modules/ws": {
+      "version": "8.18.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+      "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@mapbox/node-pre-gyp": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
@@ -251,6 +342,11 @@
       "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==",
       "dev": true
     },
+    "node_modules/@types/js-yaml": {
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz",
+      "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg=="
+    },
     "node_modules/@types/jsonwebtoken": {
       "version": "9.0.10",
       "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.10.tgz",
@@ -276,7 +372,6 @@
       "version": "20.19.25",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
       "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
-      "devOptional": true,
       "dependencies": {
         "undici-types": "~6.21.0"
       }
@@ -287,6 +382,15 @@
       "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==",
       "dev": true
     },
+    "node_modules/@types/node-fetch": {
+      "version": "2.6.13",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz",
+      "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==",
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^4.0.4"
+      }
+    },
     "node_modules/@types/pg": {
       "version": "8.15.6",
       "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz",
@@ -340,6 +444,14 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/stream-buffers": {
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.8.tgz",
+      "integrity": "sha512-J+7VaHKNvlNPJPEJXX/fKa9DZtR/xPMwuIbe+yNOwp1YB+ApUOBv2aUpEoBJEi8nJgbgs1x8e73ttg0r1rSUdw==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/uuid": {
       "version": "9.0.8",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz",
@@ -520,6 +632,78 @@
         }
       }
     },
+    "node_modules/bare-fs": {
+      "version": "4.5.2",
+      "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.2.tgz",
+      "integrity": "sha512-veTnRzkb6aPHOvSKIOy60KzURfBdUflr5VReI+NSaPL6xf+XLdONQgZgpYvUuZLVQ8dCqxpBAudaOM1+KpAUxw==",
+      "optional": true,
+      "dependencies": {
+        "bare-events": "^2.5.4",
+        "bare-path": "^3.0.0",
+        "bare-stream": "^2.6.4",
+        "bare-url": "^2.2.2",
+        "fast-fifo": "^1.3.2"
+      },
+      "engines": {
+        "bare": ">=1.16.0"
+      },
+      "peerDependencies": {
+        "bare-buffer": "*"
+      },
+      "peerDependenciesMeta": {
+        "bare-buffer": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/bare-os": {
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz",
+      "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==",
+      "optional": true,
+      "engines": {
+        "bare": ">=1.14.0"
+      }
+    },
+    "node_modules/bare-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz",
+      "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==",
+      "optional": true,
+      "dependencies": {
+        "bare-os": "^3.0.1"
+      }
+    },
+    "node_modules/bare-stream": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz",
+      "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==",
+      "optional": true,
+      "dependencies": {
+        "streamx": "^2.21.0"
+      },
+      "peerDependencies": {
+        "bare-buffer": "*",
+        "bare-events": "*"
+      },
+      "peerDependenciesMeta": {
+        "bare-buffer": {
+          "optional": true
+        },
+        "bare-events": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/bare-url": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz",
+      "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==",
+      "optional": true,
+      "dependencies": {
+        "bare-path": "^3.0.0"
+      }
+    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -1026,6 +1210,17 @@
         "url": "https://github.com/sponsors/fb55"
       }
     },
+    "node_modules/csv-parser": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.2.0.tgz",
+      "integrity": "sha512-fgKbp+AJbn1h2dcAHKIdKNSSjfp43BZZykXsCjzALjKy80VXQNHPFJ6T9Afwdzoj24aMkq8GwDS7KGcDPpejrA==",
+      "bin": {
+        "csv-parser": "bin/csv-parser"
+      },
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/data-uri-to-buffer": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
@@ -2008,6 +2203,14 @@
         "node": ">=16.0.0"
       }
     },
+    "node_modules/hpagent": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz",
+      "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/htmlparser2": {
       "version": "10.0.0",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz",
@@ -2235,6 +2438,14 @@
         "node": ">= 12"
       }
     },
+    "node_modules/ip2location-nodejs": {
+      "version": "9.7.0",
+      "resolved": "https://registry.npmjs.org/ip2location-nodejs/-/ip2location-nodejs-9.7.0.tgz",
+      "integrity": "sha512-eQ4T5TXm1cx0+pQcRycPiuaiRuoDEMd9O89Be7Ugk555qi9UY9enXSznkkqr3kQRyUaXx7zj5dORC5LGTPOttA==",
+      "dependencies": {
+        "csv-parser": "^3.0.0"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz",
@@ -2363,6 +2574,22 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/isomorphic-ws": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-5.0.0.tgz",
+      "integrity": "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==",
+      "peerDependencies": {
+        "ws": "*"
+      }
+    },
+    "node_modules/jose": {
+      "version": "6.1.3",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz",
+      "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==",
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -2379,6 +2606,14 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/jsep": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz",
+      "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==",
+      "engines": {
+        "node": ">= 10.16.0"
+      }
+    },
     "node_modules/json-parse-even-better-errors": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
@@ -2400,6 +2635,23 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/jsonpath-plus": {
+      "version": "10.3.0",
+      "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-10.3.0.tgz",
+      "integrity": "sha512-8TNmfeTCk2Le33A3vRRwtuworG/L5RrgMvdjhKZxvyShO+mBu2fP50OWUjRLNtvw344DdDarFh9buFAZs5ujeA==",
+      "dependencies": {
+        "@jsep-plugin/assignment": "^1.3.0",
+        "@jsep-plugin/regex": "^1.0.4",
+        "jsep": "^1.4.0"
+      },
+      "bin": {
+        "jsonpath": "bin/jsonpath-cli.js",
+        "jsonpath-plus": "bin/jsonpath-cli.js"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/jsonwebtoken": {
       "version": "9.0.2",
       "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz",
@@ -2474,6 +2726,11 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
+    "node_modules/lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ=="
+    },
     "node_modules/lodash.defaults": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@@ -2923,6 +3180,14 @@
         "url": "https://github.com/fb55/nth-check?sponsor=1"
       }
     },
+    "node_modules/oauth4webapi": {
+      "version": "3.8.3",
+      "resolved": "https://registry.npmjs.org/oauth4webapi/-/oauth4webapi-3.8.3.tgz",
+      "integrity": "sha512-pQ5BsX3QRTgnt5HxgHwgunIRaDXBdkT23tf8dfzmtTIL2LTpdmxgbpbBm0VgFWAIDlezQvQCTgnVIUmHupXHxw==",
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -2961,6 +3226,18 @@
         "wrappy": "1"
       }
     },
+    "node_modules/openid-client": {
+      "version": "6.8.1",
+      "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-6.8.1.tgz",
+      "integrity": "sha512-VoYT6enBo6Vj2j3Q5Ec0AezS+9YGzQo1f5Xc42lreMGlfP4ljiXPKVDvCADh+XHCV/bqPu/wWSiCVXbJKvrODw==",
+      "dependencies": {
+        "jose": "^6.1.0",
+        "oauth4webapi": "^3.8.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/pac-proxy-agent": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
@@ -3864,6 +4141,11 @@
         "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
       }
     },
+    "node_modules/rfc4648": {
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/rfc4648/-/rfc4648-1.5.4.tgz",
+      "integrity": "sha512-rRg/6Lb+IGfJqO05HZkN50UtY7K/JhxJag1kP23+zyMfrvoB0B7RWv06MbOzoc79RgCdNTiUaNsTT1AJZ7Z+cg=="
+    },
     "node_modules/rimraf": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -4294,6 +4576,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/stream-buffers": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.3.tgz",
+      "integrity": "sha512-pqMqwQCso0PBJt2PQmDO0cFj0lyqmiwOMiMSkVtRokl7e+ZTRYgDHKnuZNbqjiJXgsg4nuqtD/zxuo9KqTp0Yw==",
+      "engines": {
+        "node": ">= 0.10.0"
+      }
+    },
     "node_modules/streamx": {
       "version": "2.23.0",
       "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz",
@@ -4513,8 +4803,7 @@
     "node_modules/undici-types": {
       "version": "6.21.0",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
-      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="
-      "devOptional": true
     },
     "node_modules/universalify": {
       "version": "2.0.1",
@@ -4537,6 +4826,14 @@
       "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz",
       "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg=="
     },
+    "node_modules/user-agents": {
+      "version": "1.1.669",
+      "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.669.tgz",
+      "integrity": "sha512-pbIzG+AOqCaIpySKJ4IAm1l0VyE4jMnK4y1thV8lm8PYxI+7X5uWcppOK7zY79TCKKTAnJH3/4gaVIZHsjrmJA==",
+      "dependencies": {
+        "lodash.clonedeep": "^4.5.0"
+      }
+    },
     "node_modules/util": {
       "version": "0.12.5",
       "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz",

backend/package-lock.json

@@ -1,13 +1,14 @@
 {
   "name": "dutchie-menus-backend",
-  "version": "1.5.1",
+  "version": "1.6.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "dutchie-menus-backend",
-      "version": "1.5.1",
+      "version": "1.6.0",
       "dependencies": {
+        "@kubernetes/client-node": "^1.4.0",
         "@types/bcryptjs": "^3.0.0",
         "axios": "^1.6.2",
         "bcrypt": "^5.1.1",
@@ -21,6 +22,7 @@
         "helmet": "^7.1.0",
         "https-proxy-agent": "^7.0.2",
         "ioredis": "^5.8.2",
+        "ip2location-nodejs": "^9.7.0",
         "ipaddr.js": "^2.2.0",
         "jsonwebtoken": "^9.0.2",
         "minio": "^7.1.3",
@@ -33,6 +35,7 @@
         "puppeteer-extra-plugin-stealth": "^2.11.2",
         "sharp": "^0.32.0",
         "socks-proxy-agent": "^8.0.2",
+        "user-agents": "^1.1.669",
         "uuid": "^9.0.1",
         "zod": "^3.22.4"
       },
@@ -491,6 +494,97 @@
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz",
       "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ=="
     },
+    "node_modules/@jsep-plugin/assignment": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@jsep-plugin/assignment/-/assignment-1.3.0.tgz",
+      "integrity": "sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==",
+      "engines": {
+        "node": ">= 10.16.0"
+      },
+      "peerDependencies": {
+        "jsep": "^0.4.0||^1.0.0"
+      }
+    },
+    "node_modules/@jsep-plugin/regex": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@jsep-plugin/regex/-/regex-1.0.4.tgz",
+      "integrity": "sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==",
+      "engines": {
+        "node": ">= 10.16.0"
+      },
+      "peerDependencies": {
+        "jsep": "^0.4.0||^1.0.0"
+      }
+    },
+    "node_modules/@kubernetes/client-node": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-1.4.0.tgz",
+      "integrity": "sha512-Zge3YvF7DJi264dU1b3wb/GmzR99JhUpqTvp+VGHfwZT+g7EOOYNScDJNZwXy9cszyIGPIs0VHr+kk8e95qqrA==",
+      "dependencies": {
+        "@types/js-yaml": "^4.0.1",
+        "@types/node": "^24.0.0",
+        "@types/node-fetch": "^2.6.13",
+        "@types/stream-buffers": "^3.0.3",
+        "form-data": "^4.0.0",
+        "hpagent": "^1.2.0",
+        "isomorphic-ws": "^5.0.0",
+        "js-yaml": "^4.1.0",
+        "jsonpath-plus": "^10.3.0",
+        "node-fetch": "^2.7.0",
+        "openid-client": "^6.1.3",
+        "rfc4648": "^1.3.0",
+        "socks-proxy-agent": "^8.0.4",
+        "stream-buffers": "^3.0.2",
+        "tar-fs": "^3.0.9",
+        "ws": "^8.18.2"
+      }
+    },
+    "node_modules/@kubernetes/client-node/node_modules/@types/node": {
+      "version": "24.10.3",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.3.tgz",
+      "integrity": "sha512-gqkrWUsS8hcm0r44yn7/xZeV1ERva/nLgrLxFRUGb7aoNMIJfZJ3AC261zDQuOAKC7MiXai1WCpYc48jAHoShQ==",
+      "dependencies": {
+        "undici-types": "~7.16.0"
+      }
+    },
+    "node_modules/@kubernetes/client-node/node_modules/tar-fs": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz",
+      "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==",
+      "dependencies": {
+        "pump": "^3.0.0",
+        "tar-stream": "^3.1.5"
+      },
+      "optionalDependencies": {
+        "bare-fs": "^4.0.1",
+        "bare-path": "^3.0.0"
+      }
+    },
+    "node_modules/@kubernetes/client-node/node_modules/undici-types": {
+      "version": "7.16.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
+      "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="
+    },
+    "node_modules/@kubernetes/client-node/node_modules/ws": {
+      "version": "8.18.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+      "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@mapbox/node-pre-gyp": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
@@ -756,6 +850,11 @@
       "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==",
       "dev": true
     },
+    "node_modules/@types/js-yaml": {
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz",
+      "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg=="
+    },
     "node_modules/@types/jsonwebtoken": {
       "version": "9.0.10",
       "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.10.tgz",
@@ -781,7 +880,6 @@
       "version": "20.19.25",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
       "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
-      "devOptional": true,
       "dependencies": {
         "undici-types": "~6.21.0"
       }
@@ -792,6 +890,15 @@
       "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==",
       "dev": true
     },
+    "node_modules/@types/node-fetch": {
+      "version": "2.6.13",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz",
+      "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==",
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^4.0.4"
+      }
+    },
     "node_modules/@types/pg": {
       "version": "8.15.6",
       "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz",
@@ -845,6 +952,14 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/stream-buffers": {
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.8.tgz",
+      "integrity": "sha512-J+7VaHKNvlNPJPEJXX/fKa9DZtR/xPMwuIbe+yNOwp1YB+ApUOBv2aUpEoBJEi8nJgbgs1x8e73ttg0r1rSUdw==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/uuid": {
       "version": "9.0.8",
       "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz",
@@ -1025,6 +1140,78 @@
         }
       }
     },
+    "node_modules/bare-fs": {
+      "version": "4.5.2",
+      "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.2.tgz",
+      "integrity": "sha512-veTnRzkb6aPHOvSKIOy60KzURfBdUflr5VReI+NSaPL6xf+XLdONQgZgpYvUuZLVQ8dCqxpBAudaOM1+KpAUxw==",
+      "optional": true,
+      "dependencies": {
+        "bare-events": "^2.5.4",
+        "bare-path": "^3.0.0",
+        "bare-stream": "^2.6.4",
+        "bare-url": "^2.2.2",
+        "fast-fifo": "^1.3.2"
+      },
+      "engines": {
+        "bare": ">=1.16.0"
+      },
+      "peerDependencies": {
+        "bare-buffer": "*"
+      },
+      "peerDependenciesMeta": {
+        "bare-buffer": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/bare-os": {
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz",
+      "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==",
+      "optional": true,
+      "engines": {
+        "bare": ">=1.14.0"
+      }
+    },
+    "node_modules/bare-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz",
+      "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==",
+      "optional": true,
+      "dependencies": {
+        "bare-os": "^3.0.1"
+      }
+    },
+    "node_modules/bare-stream": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz",
+      "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==",
+      "optional": true,
+      "dependencies": {
+        "streamx": "^2.21.0"
+      },
+      "peerDependencies": {
+        "bare-buffer": "*",
+        "bare-events": "*"
+      },
+      "peerDependenciesMeta": {
+        "bare-buffer": {
+          "optional": true
+        },
+        "bare-events": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/bare-url": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz",
+      "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==",
+      "optional": true,
+      "dependencies": {
+        "bare-path": "^3.0.0"
+      }
+    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -1531,6 +1718,17 @@
         "url": "https://github.com/sponsors/fb55"
       }
     },
+    "node_modules/csv-parser": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.2.0.tgz",
+      "integrity": "sha512-fgKbp+AJbn1h2dcAHKIdKNSSjfp43BZZykXsCjzALjKy80VXQNHPFJ6T9Afwdzoj24aMkq8GwDS7KGcDPpejrA==",
+      "bin": {
+        "csv-parser": "bin/csv-parser"
+      },
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/data-uri-to-buffer": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
@@ -2527,6 +2725,14 @@
         "node": ">=16.0.0"
       }
     },
+    "node_modules/hpagent": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz",
+      "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/htmlparser2": {
       "version": "10.0.0",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz",
@@ -2754,6 +2960,14 @@
         "node": ">= 12"
       }
     },
+    "node_modules/ip2location-nodejs": {
+      "version": "9.7.0",
+      "resolved": "https://registry.npmjs.org/ip2location-nodejs/-/ip2location-nodejs-9.7.0.tgz",
+      "integrity": "sha512-eQ4T5TXm1cx0+pQcRycPiuaiRuoDEMd9O89Be7Ugk555qi9UY9enXSznkkqr3kQRyUaXx7zj5dORC5LGTPOttA==",
+      "dependencies": {
+        "csv-parser": "^3.0.0"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz",
@@ -2882,6 +3096,22 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/isomorphic-ws": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-5.0.0.tgz",
+      "integrity": "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==",
+      "peerDependencies": {
+        "ws": "*"
+      }
+    },
+    "node_modules/jose": {
+      "version": "6.1.3",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz",
+      "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==",
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -2898,6 +3128,14 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/jsep": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz",
+      "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==",
+      "engines": {
+        "node": ">= 10.16.0"
+      }
+    },
     "node_modules/json-parse-even-better-errors": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
@@ -2919,6 +3157,23 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/jsonpath-plus": {
+      "version": "10.3.0",
+      "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-10.3.0.tgz",
+      "integrity": "sha512-8TNmfeTCk2Le33A3vRRwtuworG/L5RrgMvdjhKZxvyShO+mBu2fP50OWUjRLNtvw344DdDarFh9buFAZs5ujeA==",
+      "dependencies": {
+        "@jsep-plugin/assignment": "^1.3.0",
+        "@jsep-plugin/regex": "^1.0.4",
+        "jsep": "^1.4.0"
+      },
+      "bin": {
+        "jsonpath": "bin/jsonpath-cli.js",
+        "jsonpath-plus": "bin/jsonpath-cli.js"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/jsonwebtoken": {
       "version": "9.0.2",
       "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz",
@@ -2993,6 +3248,11 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
+    "node_modules/lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ=="
+    },
     "node_modules/lodash.defaults": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@@ -3442,6 +3702,14 @@
         "url": "https://github.com/fb55/nth-check?sponsor=1"
       }
     },
+    "node_modules/oauth4webapi": {
+      "version": "3.8.3",
+      "resolved": "https://registry.npmjs.org/oauth4webapi/-/oauth4webapi-3.8.3.tgz",
+      "integrity": "sha512-pQ5BsX3QRTgnt5HxgHwgunIRaDXBdkT23tf8dfzmtTIL2LTpdmxgbpbBm0VgFWAIDlezQvQCTgnVIUmHupXHxw==",
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -3480,6 +3748,18 @@
         "wrappy": "1"
       }
     },
+    "node_modules/openid-client": {
+      "version": "6.8.1",
+      "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-6.8.1.tgz",
+      "integrity": "sha512-VoYT6enBo6Vj2j3Q5Ec0AezS+9YGzQo1f5Xc42lreMGlfP4ljiXPKVDvCADh+XHCV/bqPu/wWSiCVXbJKvrODw==",
+      "dependencies": {
+        "jose": "^6.1.0",
+        "oauth4webapi": "^3.8.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/panva"
+      }
+    },
     "node_modules/pac-proxy-agent": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
@@ -4396,6 +4676,11 @@
         "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
       }
     },
+    "node_modules/rfc4648": {
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/rfc4648/-/rfc4648-1.5.4.tgz",
+      "integrity": "sha512-rRg/6Lb+IGfJqO05HZkN50UtY7K/JhxJag1kP23+zyMfrvoB0B7RWv06MbOzoc79RgCdNTiUaNsTT1AJZ7Z+cg=="
+    },
     "node_modules/rimraf": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -4826,6 +5111,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/stream-buffers": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.3.tgz",
+      "integrity": "sha512-pqMqwQCso0PBJt2PQmDO0cFj0lyqmiwOMiMSkVtRokl7e+ZTRYgDHKnuZNbqjiJXgsg4nuqtD/zxuo9KqTp0Yw==",
+      "engines": {
+        "node": ">= 0.10.0"
+      }
+    },
     "node_modules/streamx": {
       "version": "2.23.0",
       "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz",
@@ -5045,8 +5338,7 @@
     "node_modules/undici-types": {
       "version": "6.21.0",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
-      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="
-      "devOptional": true
     },
     "node_modules/universalify": {
       "version": "2.0.1",
@@ -5069,6 +5361,14 @@
       "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz",
       "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg=="
     },
+    "node_modules/user-agents": {
+      "version": "1.1.669",
+      "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.669.tgz",
+      "integrity": "sha512-pbIzG+AOqCaIpySKJ4IAm1l0VyE4jMnK4y1thV8lm8PYxI+7X5uWcppOK7zY79TCKKTAnJH3/4gaVIZHsjrmJA==",
+      "dependencies": {
+        "lodash.clonedeep": "^4.5.0"
+      }
+    },
     "node_modules/util": {
       "version": "0.12.5",
       "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz",

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dutchie-menus-backend",
-  "version": "1.5.1",
+  "version": "1.6.0",
   "description": "Backend API for Dutchie Menus scraper and management",
   "main": "dist/index.js",
   "scripts": {
@@ -22,6 +22,7 @@
     "seed:dt:cities:bulk": "tsx src/scripts/seed-dt-cities-bulk.ts"
   },
   "dependencies": {
+    "@kubernetes/client-node": "^1.4.0",
     "@types/bcryptjs": "^3.0.0",
     "axios": "^1.6.2",
     "bcrypt": "^5.1.1",
@@ -35,6 +36,7 @@
     "helmet": "^7.1.0",
     "https-proxy-agent": "^7.0.2",
     "ioredis": "^5.8.2",
+    "ip2location-nodejs": "^9.7.0",
     "ipaddr.js": "^2.2.0",
     "jsonwebtoken": "^9.0.2",
     "minio": "^7.1.3",
@@ -47,6 +49,7 @@
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sharp": "^0.32.0",
     "socks-proxy-agent": "^8.0.2",
+    "user-agents": "^1.1.669",
     "uuid": "^9.0.1",
     "zod": "^3.22.4"
   },

backend/public/downloads/cannaiq-menus-latest.zip

@@ -0,0 +1 @@
+cannaiq-menus-1.6.0.zip

backend/scripts/download-ip2location.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# Download IP2Location LITE DB3 (City-level) database
+# Free for commercial use with attribution
+# https://lite.ip2location.com/database/db3-ip-country-region-city
+
+set -e
+
+DATA_DIR="${1:-./data/ip2location}"
+DB_FILE="IP2LOCATION-LITE-DB3.BIN"
+
+mkdir -p "$DATA_DIR"
+cd "$DATA_DIR"
+
+echo "Downloading IP2Location LITE DB3 database..."
+
+# IP2Location LITE DB3 - includes city, region, country, lat/lng
+# You need to register at https://lite.ip2location.com/ to get a download token
+# Then set IP2LOCATION_TOKEN environment variable
+
+if [ -z "$IP2LOCATION_TOKEN" ]; then
+  echo ""
+  echo "ERROR: IP2LOCATION_TOKEN not set"
+  echo ""
+  echo "To download the database:"
+  echo "1. Register free at https://lite.ip2location.com/"
+  echo "2. Get your download token from the dashboard"
+  echo "3. Run: IP2LOCATION_TOKEN=your_token ./scripts/download-ip2location.sh"
+  echo ""
+  exit 1
+fi
+
+# Download DB3.LITE (IPv4 + City)
+DOWNLOAD_URL="https://www.ip2location.com/download/?token=${IP2LOCATION_TOKEN}&file=DB3LITEBIN"
+
+echo "Downloading from IP2Location..."
+curl -L -o ip2location.zip "$DOWNLOAD_URL"
+
+echo "Extracting..."
+unzip -o ip2location.zip
+
+# Rename to standard name
+if [ -f "IP2LOCATION-LITE-DB3.BIN" ]; then
+  echo "Database ready: $DATA_DIR/IP2LOCATION-LITE-DB3.BIN"
+elif [ -f "IP-COUNTRY-REGION-CITY.BIN" ]; then
+  mv "IP-COUNTRY-REGION-CITY.BIN" "$DB_FILE"
+  echo "Database ready: $DATA_DIR/$DB_FILE"
+else
+  # Find whatever BIN file was extracted
+  BIN_FILE=$(ls *.BIN 2>/dev/null | head -1)
+  if [ -n "$BIN_FILE" ]; then
+    mv "$BIN_FILE" "$DB_FILE"
+    echo "Database ready: $DATA_DIR/$DB_FILE"
+  else
+    echo "ERROR: No BIN file found in archive"
+    ls -la
+    exit 1
+  fi
+fi
+
+# Cleanup
+rm -f ip2location.zip *.txt LICENSE* README*
+
+echo ""
+echo "Done! Database saved to: $DATA_DIR/$DB_FILE"
+echo "Update monthly by re-running this script."

backend/src/auth/middleware.ts

@@ -32,6 +32,7 @@ const TRUSTED_ORIGINS = [
 // Pattern-based trusted origins (wildcards)
 const TRUSTED_ORIGIN_PATTERNS = [
   /^https:\/\/.*\.cannabrands\.app$/,  // *.cannabrands.app
+  /^https:\/\/.*\.cannaiq\.co$/,       // *.cannaiq.co
 ];
 
 // Trusted IPs for internal pod-to-pod communication
@@ -152,22 +153,10 @@ export async function authenticateUser(email: string, password: string): Promise
 }
 
 export async function authMiddleware(req: AuthRequest, res: Response, next: NextFunction) {
-  // Allow trusted origins/IPs to bypass auth (internal services, same-origin)
-  if (isTrustedRequest(req)) {
-    req.user = {
-      id: 0,
-      email: 'internal@system',
-      role: 'internal'
-    };
-    return next();
-  }
-
   const authHeader = req.headers.authorization;
 
-  if (!authHeader || !authHeader.startsWith('Bearer ')) {
+  // If a Bearer token is provided, always try to use it first (logged-in user)
-    return res.status(401).json({ error: 'No token provided' });
+  if (authHeader && authHeader.startsWith('Bearer ')) {
-  }
-
     const token = authHeader.substring(7);
 
     // Try JWT first
@@ -186,56 +175,44 @@ export async function authMiddleware(req: AuthRequest, res: Response, next: Next
         WHERE token = $1
       `, [token]);
 
-    if (result.rows.length === 0) {
+      if (result.rows.length > 0) {
+        const apiToken = result.rows[0];
+        if (!apiToken.active) {
+          return res.status(401).json({ error: 'API token is inactive' });
+        }
+        if (apiToken.expires_at && new Date(apiToken.expires_at) < new Date()) {
+          return res.status(401).json({ error: 'API token has expired' });
+        }
+        req.user = {
+          id: 0,
+          email: `api:${apiToken.name}`,
+          role: 'api_token'
+        };
+        req.apiToken = apiToken;
+        return next();
+      }
+    } catch (err) {
+      console.error('API token lookup error:', err);
+    }
+
+    // Token provided but invalid
     return res.status(401).json({ error: 'Invalid token' });
   }
 
-    const apiToken = result.rows[0];
+  // No token provided - check trusted origins for API access (WordPress, etc.)
-
+  if (isTrustedRequest(req)) {
-    // Check if token is active
-    if (!apiToken.active) {
-      return res.status(401).json({ error: 'Token is disabled' });
-    }
-
-    // Check if token is expired
-    if (apiToken.expires_at && new Date(apiToken.expires_at) < new Date()) {
-      return res.status(401).json({ error: 'Token has expired' });
-    }
-
-    // Check allowed endpoints
-    if (apiToken.allowed_endpoints && apiToken.allowed_endpoints.length > 0) {
-      const isAllowed = apiToken.allowed_endpoints.some((pattern: string) => {
-        // Simple wildcard matching
-        const regex = new RegExp('^' + pattern.replace('*', '.*') + '$');
-        return regex.test(req.path);
-      });
-
-      if (!isAllowed) {
-        return res.status(403).json({ error: 'Endpoint not allowed for this token' });
-      }
-    }
-
-    // Set API token on request for tracking
-    req.apiToken = {
-      id: apiToken.id,
-      name: apiToken.name,
-      rate_limit: apiToken.rate_limit
-    };
-
-    // Set a generic user for compatibility with existing code
     req.user = {
-      id: apiToken.id,
+      id: 0,
-      email: `api-token-${apiToken.id}@system`,
+      email: 'internal@system',
-      role: 'api'
+      role: 'internal'
     };
-
+    return next();
-    next();
-  } catch (error) {
-    console.error('Error verifying API token:', error);
-    return res.status(500).json({ error: 'Authentication failed' });
   }
+
+  return res.status(401).json({ error: 'No token provided' });
 }
 
+
 /**
  * Require specific role(s) to access endpoint.
  *

backend/src/db/auto-migrate.ts

@@ -0,0 +1,141 @@
+/**
+ * Auto-Migration System
+ *
+ * Runs SQL migration files from the migrations/ folder automatically on server startup.
+ * Uses a schema_migrations table to track which migrations have been applied.
+ *
+ * Safe to run multiple times - only applies new migrations.
+ */
+
+import { Pool } from 'pg';
+import fs from 'fs';
+import path from 'path';
+
+const MIGRATIONS_DIR = path.join(__dirname, '../../migrations');
+
+/**
+ * Ensure schema_migrations table exists
+ */
+async function ensureMigrationsTable(pool: Pool): Promise<void> {
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS schema_migrations (
+      id SERIAL PRIMARY KEY,
+      name VARCHAR(255) UNIQUE NOT NULL,
+      applied_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+    )
+  `);
+}
+
+/**
+ * Get list of already-applied migrations
+ */
+async function getAppliedMigrations(pool: Pool): Promise<Set<string>> {
+  const result = await pool.query('SELECT name FROM schema_migrations');
+  return new Set(result.rows.map(row => row.name));
+}
+
+/**
+ * Get list of migration files from disk
+ */
+function getMigrationFiles(): string[] {
+  if (!fs.existsSync(MIGRATIONS_DIR)) {
+    console.log('[AutoMigrate] No migrations directory found');
+    return [];
+  }
+
+  return fs.readdirSync(MIGRATIONS_DIR)
+    .filter(f => f.endsWith('.sql'))
+    .sort(); // Sort alphabetically (001_, 002_, etc.)
+}
+
+/**
+ * Run a single migration file
+ */
+async function runMigration(pool: Pool, filename: string): Promise<void> {
+  const filepath = path.join(MIGRATIONS_DIR, filename);
+  const sql = fs.readFileSync(filepath, 'utf8');
+
+  const client = await pool.connect();
+  try {
+    await client.query('BEGIN');
+
+    // Run the migration SQL
+    await client.query(sql);
+
+    // Record that this migration was applied
+    await client.query(
+      'INSERT INTO schema_migrations (name) VALUES ($1) ON CONFLICT (name) DO NOTHING',
+      [filename]
+    );
+
+    await client.query('COMMIT');
+    console.log(`[AutoMigrate] ✓ Applied: ${filename}`);
+  } catch (error: any) {
+    await client.query('ROLLBACK');
+    console.error(`[AutoMigrate] ✗ Failed: ${filename}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * Run all pending migrations
+ *
+ * @param pool - Database connection pool
+ * @returns Number of migrations applied
+ */
+export async function runAutoMigrations(pool: Pool): Promise<number> {
+  console.log('[AutoMigrate] Checking for pending migrations...');
+
+  try {
+    // Ensure migrations table exists
+    await ensureMigrationsTable(pool);
+
+    // Get applied and available migrations
+    const applied = await getAppliedMigrations(pool);
+    const available = getMigrationFiles();
+
+    // Find pending migrations
+    const pending = available.filter(f => !applied.has(f));
+
+    if (pending.length === 0) {
+      console.log('[AutoMigrate] No pending migrations');
+      return 0;
+    }
+
+    console.log(`[AutoMigrate] Found ${pending.length} pending migrations`);
+
+    // Run each pending migration in order
+    for (const filename of pending) {
+      await runMigration(pool, filename);
+    }
+
+    console.log(`[AutoMigrate] Successfully applied ${pending.length} migrations`);
+    return pending.length;
+
+  } catch (error: any) {
+    console.error('[AutoMigrate] Migration failed:', error.message);
+    // Don't crash the server - log and continue
+    // The specific failing migration will have been rolled back
+    return -1;
+  }
+}
+
+/**
+ * Check migration status without running anything
+ */
+export async function checkMigrationStatus(pool: Pool): Promise<{
+  applied: string[];
+  pending: string[];
+}> {
+  await ensureMigrationsTable(pool);
+
+  const applied = await getAppliedMigrations(pool);
+  const available = getMigrationFiles();
+
+  return {
+    applied: available.filter(f => applied.has(f)),
+    pending: available.filter(f => !applied.has(f)),
+  };
+}

backend/src/discovery/discovery-crawler.ts

@@ -172,6 +172,9 @@ export async function runFullDiscovery(
     console.log(`Errors: ${totalErrors}`);
   }
 
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  let newDispensaryIds: number[] = [];
+
   // Step 4: Auto-validate and promote discovered locations
   if (!dryRun && totalLocationsUpserted > 0) {
     console.log('\n[Discovery] Step 4: Auto-promoting discovered locations...');
@@ -180,6 +183,13 @@ export async function runFullDiscovery(
     console.log(`  Created: ${promotionResult.created} new dispensaries`);
     console.log(`  Updated: ${promotionResult.updated} existing dispensaries`);
     console.log(`  Rejected: ${promotionResult.rejected} (validation failed)`);
+
+    // Per TASK_WORKFLOW_2024-12-10.md: Capture new IDs for task chaining
+    newDispensaryIds = promotionResult.newDispensaryIds;
+    if (newDispensaryIds.length > 0) {
+      console.log(`  New store IDs for crawl: [${newDispensaryIds.join(', ')}]`);
+    }
+
     if (promotionResult.rejectedRecords.length > 0) {
       console.log(`  Rejection reasons:`);
       promotionResult.rejectedRecords.slice(0, 5).forEach(r => {
@@ -191,12 +201,132 @@ export async function runFullDiscovery(
     }
   }
 
+  // Step 5: Detect dropped stores (in DB but not in discovery results)
+  if (!dryRun) {
+    console.log('\n[Discovery] Step 5: Detecting dropped stores...');
+    const droppedResult = await detectDroppedStores(pool, stateCode);
+    if (droppedResult.droppedCount > 0) {
+      console.log(`[Discovery] Found ${droppedResult.droppedCount} dropped stores:`);
+      droppedResult.droppedStores.slice(0, 10).forEach(s => {
+        console.log(`    - ${s.name} (${s.city}, ${s.state}) - last seen: ${s.lastSeenAt}`);
+      });
+      if (droppedResult.droppedCount > 10) {
+        console.log(`    ... and ${droppedResult.droppedCount - 10} more`);
+      }
+    } else {
+      console.log(`[Discovery] No dropped stores detected`);
+    }
+  }
+
   return {
     cities: cityResult,
     locations: locationResults,
     totalLocationsFound,
     totalLocationsUpserted,
     durationMs,
+    // Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
+    newDispensaryIds,
+  };
+}
+
+// ============================================================
+// DROPPED STORE DETECTION
+// ============================================================
+
+export interface DroppedStoreResult {
+  droppedCount: number;
+  droppedStores: Array<{
+    id: number;
+    name: string;
+    city: string;
+    state: string;
+    platformDispensaryId: string;
+    lastSeenAt: string;
+  }>;
+}
+
+/**
+ * Detect stores that exist in dispensaries but were not found in discovery.
+ * Marks them as status='dropped' for manual review.
+ *
+ * A store is considered "dropped" if:
+ * 1. It has a platform_dispensary_id (was verified via Dutchie)
+ * 2. It was NOT seen in the latest discovery crawl (last_seen_at in discovery < 24h ago)
+ * 3. It's currently marked as 'open' status
+ */
+export async function detectDroppedStores(
+  pool: Pool,
+  stateCode?: string
+): Promise<DroppedStoreResult> {
+  // Find dispensaries that:
+  // 1. Have platform_dispensary_id (verified Dutchie stores)
+  // 2. Are currently 'open' status
+  // 3. Have a linked discovery record that wasn't seen in the last discovery run
+  //    (last_seen_at in dutchie_discovery_locations is older than 24 hours)
+  const params: any[] = [];
+  let stateFilter = '';
+
+  if (stateCode) {
+    stateFilter = ` AND d.state = $1`;
+    params.push(stateCode);
+  }
+
+  const query = `
+    WITH recently_seen AS (
+      SELECT DISTINCT platform_location_id
+      FROM dutchie_discovery_locations
+      WHERE last_seen_at > NOW() - INTERVAL '24 hours'
+        AND active = true
+    )
+    SELECT
+      d.id,
+      d.name,
+      d.city,
+      d.state,
+      d.platform_dispensary_id,
+      d.updated_at as last_seen_at
+    FROM dispensaries d
+    WHERE d.platform_dispensary_id IS NOT NULL
+      AND d.platform = 'dutchie'
+      AND (d.status = 'open' OR d.status IS NULL)
+      AND d.crawl_enabled = true
+      AND d.platform_dispensary_id NOT IN (SELECT platform_location_id FROM recently_seen)
+      ${stateFilter}
+    ORDER BY d.name
+  `;
+
+  const result = await pool.query(query, params);
+  const droppedStores = result.rows;
+
+  // Mark these stores as 'dropped' status
+  if (droppedStores.length > 0) {
+    const ids = droppedStores.map(s => s.id);
+    await pool.query(`
+      UPDATE dispensaries
+      SET status = 'dropped', updated_at = NOW()
+      WHERE id = ANY($1::int[])
+    `, [ids]);
+
+    // Log to promotion log for audit
+    for (const store of droppedStores) {
+      await pool.query(`
+        INSERT INTO dutchie_promotion_log
+        (dispensary_id, action, state_code, store_name, triggered_by)
+        VALUES ($1, 'dropped', $2, $3, 'discovery_detection')
+      `, [store.id, store.state, store.name]);
+    }
+  }
+
+  return {
+    droppedCount: droppedStores.length,
+    droppedStores: droppedStores.map(s => ({
+      id: s.id,
+      name: s.name,
+      city: s.city,
+      state: s.state,
+      platformDispensaryId: s.platform_dispensary_id,
+      lastSeenAt: s.last_seen_at,
+    })),
   };
 }
 

backend/src/discovery/promotion.ts

@@ -127,6 +127,8 @@ export interface PromotionSummary {
     errors: string[];
   }>;
   durationMs: number;
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  newDispensaryIds: number[];
 }
 
 /**
@@ -469,6 +471,8 @@ export async function promoteDiscoveredLocations(
 
   const results: PromotionResult[] = [];
   const rejectedRecords: PromotionSummary['rejectedRecords'] = [];
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  const newDispensaryIds: number[] = [];
   let created = 0;
   let updated = 0;
   let skipped = 0;
@@ -525,6 +529,8 @@ export async function promoteDiscoveredLocations(
 
       if (promotionResult.action === 'created') {
         created++;
+        // Per TASK_WORKFLOW_2024-12-10.md: Track new IDs for task chaining
+        newDispensaryIds.push(promotionResult.dispensaryId);
       } else {
         updated++;
       }
@@ -548,6 +554,8 @@ export async function promoteDiscoveredLocations(
     results,
     rejectedRecords,
     durationMs: Date.now() - startTime,
+    // Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
+    newDispensaryIds,
   };
 }
 

backend/src/discovery/types.ts

@@ -211,6 +211,8 @@ export interface FullDiscoveryResult {
   totalLocationsFound: number;
   totalLocationsUpserted: number;
   durationMs: number;
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  newDispensaryIds?: number[];
 }
 
 // ============================================================

backend/src/hydration/canonical-upsert.ts

@@ -90,7 +90,7 @@ export async function upsertStoreProducts(
             name_raw, brand_name_raw, category_raw, subcategory_raw,
             price_rec, price_med, price_rec_special, price_med_special,
             is_on_special, discount_percent,
-            is_in_stock, stock_status,
+            is_in_stock, stock_status, stock_quantity, total_quantity_available,
             thc_percent, cbd_percent,
             image_url,
             first_seen_at, last_seen_at, updated_at
@@ -99,9 +99,9 @@ export async function upsertStoreProducts(
             $5, $6, $7, $8,
             $9, $10, $11, $12,
             $13, $14,
-            $15, $16,
+            $15, $16, $17, $17,
-            $17, $18,
+            $18, $19,
-            $19,
+            $20,
             NOW(), NOW(), NOW()
           )
           ON CONFLICT (dispensary_id, provider, provider_product_id)
@@ -118,6 +118,8 @@ export async function upsertStoreProducts(
             discount_percent = EXCLUDED.discount_percent,
             is_in_stock = EXCLUDED.is_in_stock,
             stock_status = EXCLUDED.stock_status,
+            stock_quantity = EXCLUDED.stock_quantity,
+            total_quantity_available = EXCLUDED.total_quantity_available,
             thc_percent = EXCLUDED.thc_percent,
             cbd_percent = EXCLUDED.cbd_percent,
             image_url = EXCLUDED.image_url,
@@ -141,6 +143,7 @@ export async function upsertStoreProducts(
             productPricing?.discountPercent,
             productAvailability?.inStock ?? true,
             productAvailability?.stockStatus || 'unknown',
+            productAvailability?.quantity ?? null,  // stock_quantity and total_quantity_available
             // Clamp THC/CBD to valid percentage range (0-100) - some products report mg as %
             product.thcPercent !== null && product.thcPercent <= 100 ? product.thcPercent : null,
             product.cbdPercent !== null && product.cbdPercent <= 100 ? product.cbdPercent : null,

backend/src/index.ts

@@ -6,6 +6,10 @@ import { initializeMinio, isMinioEnabled } from './utils/minio';
 import { initializeImageStorage } from './utils/image-storage';
 import { logger } from './services/logger';
 import { cleanupOrphanedJobs } from './services/proxyTestQueue';
+// Per TASK_WORKFLOW_2024-12-10.md: Database-driven task scheduler
+import { taskScheduler } from './services/task-scheduler';
+import { runAutoMigrations } from './db/auto-migrate';
+import { getPool } from './db/pool';
 import healthRoutes from './routes/health';
 import imageProxyRoutes from './routes/image-proxy';
 
@@ -127,7 +131,6 @@ import { createStatesRouter } from './routes/states';
 import { createAnalyticsV2Router } from './routes/analytics-v2';
 import { createDiscoveryRoutes } from './discovery';
 import pipelineRoutes from './routes/pipeline';
-import { getPool } from './db/pool';
 
 // Consumer API routes (findadispo.com, findagram.co)
 import consumerAuthRoutes from './routes/consumer-auth';
@@ -140,6 +143,10 @@ import clickAnalyticsRoutes from './routes/click-analytics';
 import seoRoutes from './routes/seo';
 import priceAnalyticsRoutes from './routes/price-analytics';
 import tasksRoutes from './routes/tasks';
+import workerRegistryRoutes from './routes/worker-registry';
+// Per TASK_WORKFLOW_2024-12-10.md: Raw payload access API
+import payloadsRoutes from './routes/payloads';
+import k8sRoutes from './routes/k8s';
 
 // Mark requests from trusted domains (cannaiq.co, findagram.co, findadispo.com)
 // These domains can access the API without authentication
@@ -216,6 +223,18 @@ console.log('[Workers] Routes registered at /api/workers, /api/monitor, and /api
 app.use('/api/tasks', tasksRoutes);
 console.log('[Tasks] Routes registered at /api/tasks');
 
+// Worker registry - dynamic worker registration, heartbeats, and name management
+app.use('/api/worker-registry', workerRegistryRoutes);
+console.log('[WorkerRegistry] Routes registered at /api/worker-registry');
+
+// Per TASK_WORKFLOW_2024-12-10.md: Raw payload access API
+app.use('/api/payloads', payloadsRoutes);
+console.log('[Payloads] Routes registered at /api/payloads');
+
+// K8s control routes - worker scaling from admin UI
+app.use('/api/k8s', k8sRoutes);
+console.log('[K8s] Routes registered at /api/k8s');
+
 // Phase 3: Analytics V2 - Enhanced analytics with rec/med state segmentation
 try {
   const analyticsV2Router = createAnalyticsV2Router(getPool());
@@ -302,6 +321,17 @@ async function startServer() {
   try {
     logger.info('system', 'Starting server...');
 
+    // Run auto-migrations before anything else
+    const pool = getPool();
+    const migrationsApplied = await runAutoMigrations(pool);
+    if (migrationsApplied > 0) {
+      logger.info('system', `Applied ${migrationsApplied} database migrations`);
+    } else if (migrationsApplied === 0) {
+      logger.info('system', 'Database schema up to date');
+    } else {
+      logger.warn('system', 'Some migrations failed - check logs');
+    }
+
     await initializeMinio();
     await initializeImageStorage();
     logger.info('system', isMinioEnabled() ? 'MinIO storage initialized' : 'Local filesystem storage initialized');
@@ -309,6 +339,17 @@ async function startServer() {
     // Clean up any orphaned proxy test jobs from previous server runs
     await cleanupOrphanedJobs();
 
+    // Per TASK_WORKFLOW_2024-12-10.md: Start database-driven task scheduler
+    // This replaces node-cron - schedules are stored in DB and survive restarts
+    // Uses SELECT FOR UPDATE SKIP LOCKED for multi-replica safety
+    try {
+      await taskScheduler.start();
+      logger.info('system', 'Task scheduler started');
+    } catch (err: any) {
+      // Non-fatal - scheduler can recover on next poll
+      logger.warn('system', `Task scheduler startup warning: ${err.message}`);
+    }
+
     app.listen(PORT, () => {
       logger.info('system', `Server running on port ${PORT}`);
       console.log(`🚀 Server running on port ${PORT}`);

backend/src/middleware/trustedDomains.ts

@@ -5,8 +5,8 @@ import { Request, Response, NextFunction } from 'express';
  * These are our own frontends that should have unrestricted access.
  */
 const TRUSTED_DOMAINS = [
-  'cannaiq.co',
+  '*.cannaiq.co',
-  'www.cannaiq.co',
+  '*.cannabrands.app',
   'findagram.co',
   'www.findagram.co',
   'findadispo.com',
@@ -32,6 +32,24 @@ function extractDomain(header: string): string | null {
   }
 }
 
+/**
+ * Checks if a domain matches any trusted domain (supports *.domain.com wildcards)
+ */
+function isTrustedDomain(domain: string): boolean {
+  for (const trusted of TRUSTED_DOMAINS) {
+    if (trusted.startsWith('*.')) {
+      // Wildcard: *.example.com matches example.com and any subdomain
+      const baseDomain = trusted.slice(2);
+      if (domain === baseDomain || domain.endsWith('.' + baseDomain)) {
+        return true;
+      }
+    } else if (domain === trusted) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Checks if the request comes from a trusted domain
  */
@@ -42,7 +60,7 @@ function isRequestFromTrustedDomain(req: Request): boolean {
   // Check Origin header first (preferred for CORS requests)
   if (origin) {
     const domain = extractDomain(origin);
-    if (domain && TRUSTED_DOMAINS.includes(domain)) {
+    if (domain && isTrustedDomain(domain)) {
       return true;
     }
   }
@@ -50,7 +68,7 @@ function isRequestFromTrustedDomain(req: Request): boolean {
   // Fallback to Referer header
   if (referer) {
     const domain = extractDomain(referer);
-    if (domain && TRUSTED_DOMAINS.includes(domain)) {
+    if (domain && isTrustedDomain(domain)) {
       return true;
     }
   }

backend/src/multi-state/state-query-service.ts

@@ -702,12 +702,10 @@ export class StateQueryService {
   async getNationalSummary(): Promise<NationalSummary> {
     const stateMetrics = await this.getAllStateMetrics();
 
+    // Get all states count and aggregate metrics
     const result = await this.pool.query(`
       SELECT
         COUNT(DISTINCT s.code) AS total_states,
-        COUNT(DISTINCT CASE WHEN EXISTS (
-          SELECT 1 FROM dispensaries d WHERE d.state = s.code AND d.menu_type IS NOT NULL
-        ) THEN s.code END) AS active_states,
         (SELECT COUNT(*) FROM dispensaries WHERE state IS NOT NULL) AS total_stores,
         (SELECT COUNT(*) FROM store_products sp
          JOIN dispensaries d ON sp.dispensary_id = d.id
@@ -725,7 +723,7 @@ export class StateQueryService {
 
     return {
       totalStates: parseInt(data.total_states),
-      activeStates: parseInt(data.active_states),
+      activeStates: parseInt(data.total_states), // Same as totalStates - all states shown
       totalStores: parseInt(data.total_stores),
       totalProducts: parseInt(data.total_products),
       totalBrands: parseInt(data.total_brands),

backend/src/platforms/dutchie/client.ts

@@ -5,22 +5,35 @@
  *
  * DO NOT MODIFY THIS FILE WITHOUT EXPLICIT AUTHORIZATION.
  *
- * This is the canonical HTTP client for all Dutchie communication.
+ * Updated: 2025-12-10 per workflow-12102025.md
- * All Dutchie workers (Alice, Bella, etc.) MUST use this client.
+ *
+ * KEY BEHAVIORS (per workflow-12102025.md):
+ * 1. startSession() gets identity from PROXY LOCATION, not task params
+ * 2. On 403: immediately get new IP + new fingerprint, then retry
+ * 3. After 3 consecutive 403s on same proxy → disable it (burned)
+ * 4. Language is always English (en-US)
  *
  * IMPLEMENTATION:
  * - Uses curl via child_process.execSync (bypasses TLS fingerprinting)
  * - NO Puppeteer, NO axios, NO fetch
- * - Fingerprint rotation on 403
+ * - Uses intoli/user-agents via CrawlRotator for realistic fingerprints
  * - Residential IP compatible
  *
  * USAGE:
- *   import { curlPost, curlGet, executeGraphQL } from '@dutchie/client';
+ *   import { curlPost, curlGet, executeGraphQL, startSession } from '@dutchie/client';
  *
  * ============================================================
  */
 
 import { execSync } from 'child_process';
+import {
+  buildOrderedHeaders,
+  buildRefererFromMenuUrl,
+  getCurlBinary,
+  isCurlImpersonateAvailable,
+  HeaderContext,
+  BrowserType,
+} from '../../services/http-fingerprint';
 
 // ============================================================
 // TYPES
@@ -32,6 +45,8 @@ export interface CurlResponse {
   error?: string;
 }
 
+// Per workflow-12102025.md: fingerprint comes from CrawlRotator's BrowserFingerprint
+// We keep a simplified interface here for header building
 export interface Fingerprint {
   userAgent: string;
   acceptLanguage: string;
@@ -57,15 +72,13 @@ export const DUTCHIE_CONFIG = {
 
 // ============================================================
 // PROXY SUPPORT
-// ============================================================
+// Per workflow-12102025.md:
-// Integrates with the CrawlRotator system from proxy-rotator.ts
+// - On 403: recordBlock() → increment consecutive_403_count
-// On 403 errors:
+// - After 3 consecutive 403s → proxy disabled
-// 1. Record failure on current proxy
+// - Immediately rotate to new IP + new fingerprint on 403
-// 2. Rotate to next proxy
-// 3. Retry with new proxy
 // ============================================================
 
-import type { CrawlRotator, Proxy } from '../../services/crawl-rotator';
+import type { CrawlRotator, BrowserFingerprint } from '../../services/crawl-rotator';
 
 let currentProxy: string | null = null;
 let crawlRotator: CrawlRotator | null = null;
@@ -92,13 +105,12 @@ export function getProxy(): string | null {
 
 /**
  * Set CrawlRotator for proxy rotation on 403s
- * This enables automatic proxy rotation when blocked
+ * Per workflow-12102025.md: enables automatic rotation when blocked
  */
 export function setCrawlRotator(rotator: CrawlRotator | null): void {
   crawlRotator = rotator;
   if (rotator) {
     console.log('[Dutchie Client] CrawlRotator attached - proxy rotation enabled');
-    // Set initial proxy from rotator
     const proxy = rotator.proxy.getCurrent();
     if (proxy) {
       currentProxy = rotator.proxy.getProxyUrl(proxy);
@@ -115,30 +127,41 @@ export function getCrawlRotator(): CrawlRotator | null {
 }
 
 /**
- * Rotate to next proxy (called on 403)
+ * Handle 403 block - per workflow-12102025.md:
+ * 1. Record block on current proxy (increments consecutive_403_count)
+ * 2. Immediately rotate to new proxy (new IP)
+ * 3. Rotate fingerprint
+ * Returns false if no more proxies available
  */
-async function rotateProxyOn403(error?: string): Promise<boolean> {
+async function handle403Block(): Promise<boolean> {
   if (!crawlRotator) {
+    console.warn('[Dutchie Client] No CrawlRotator - cannot handle 403');
     return false;
   }
 
-  // Record failure on current proxy
+  // Per workflow-12102025.md: record block (tracks consecutive 403s)
-  await crawlRotator.recordFailure(error || '403 Forbidden');
+  const wasDisabled = await crawlRotator.recordBlock();
+  if (wasDisabled) {
+    console.log('[Dutchie Client] Current proxy was disabled (3 consecutive 403s)');
+  }
+
+  // Per workflow-12102025.md: immediately get new IP + new fingerprint
+  const { proxy: nextProxy, fingerprint } = crawlRotator.rotateBoth();
 
-  // Rotate to next proxy
-  const nextProxy = crawlRotator.rotateProxy();
   if (nextProxy) {
     currentProxy = crawlRotator.proxy.getProxyUrl(nextProxy);
-    console.log(`[Dutchie Client] Rotated proxy: ${currentProxy.replace(/:[^:@]+@/, ':***@')}`);
+    console.log(`[Dutchie Client] Rotated to new proxy: ${currentProxy.replace(/:[^:@]+@/, ':***@')}`);
+    console.log(`[Dutchie Client] New fingerprint: ${fingerprint.userAgent.slice(0, 50)}...`);
     return true;
   }
 
-  console.warn('[Dutchie Client] No more proxies available');
+  console.error('[Dutchie Client] No more proxies available!');
   return false;
 }
 
 /**
  * Record success on current proxy
+ * Per workflow-12102025.md: resets consecutive_403_count
  */
 async function recordProxySuccess(responseTimeMs?: number): Promise<void> {
   if (crawlRotator) {
@@ -162,163 +185,69 @@ export const GRAPHQL_HASHES = {
   GetAllCitiesByState: 'ae547a0466ace5a48f91e55bf6699eacd87e3a42841560f0c0eabed5a0a920e6',
 };
 
-// ============================================================
-// FINGERPRINTS - Browser profiles for anti-detect
-// ============================================================
-
-const FINGERPRINTS: Fingerprint[] = [
-  // Chrome Windows (latest) - typical residential user, use first
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"Windows"',
-    secChUaMobile: '?0',
-  },
-  // Chrome Mac (latest)
-  {
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"macOS"',
-    secChUaMobile: '?0',
-  },
-  // Chrome Windows (120)
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Chromium";v="120", "Google Chrome";v="120", "Not-A.Brand";v="99"',
-    secChUaPlatform: '"Windows"',
-    secChUaMobile: '?0',
-  },
-  // Firefox Windows
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
-    acceptLanguage: 'en-US,en;q=0.5',
-  },
-  // Safari Mac
-  {
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
-    acceptLanguage: 'en-US,en;q=0.9',
-  },
-  // Edge Windows
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Microsoft Edge";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"Windows"',
-    secChUaMobile: '?0',
-  },
-];
-
-let currentFingerprintIndex = 0;
-
-// Forward declaration for session (actual CrawlSession interface defined later)
-let currentSession: {
-  sessionId: string;
-  fingerprint: Fingerprint;
-  proxyUrl: string | null;
-  stateCode?: string;
-  timezone?: string;
-  startedAt: Date;
-} | null = null;
-
-/**
- * Get current fingerprint - returns session fingerprint if active, otherwise default
- */
-export function getFingerprint(): Fingerprint {
-  // Use session fingerprint if a session is active
-  if (currentSession) {
-    return currentSession.fingerprint;
-  }
-  return FINGERPRINTS[currentFingerprintIndex];
-}
-
-export function rotateFingerprint(): Fingerprint {
-  currentFingerprintIndex = (currentFingerprintIndex + 1) % FINGERPRINTS.length;
-  const fp = FINGERPRINTS[currentFingerprintIndex];
-  console.log(`[Dutchie Client] Rotated to fingerprint: ${fp.userAgent.slice(0, 50)}...`);
-  return fp;
-}
-
-export function resetFingerprint(): void {
-  currentFingerprintIndex = 0;
-}
-
-/**
- * Get a random fingerprint from the pool
- */
-export function getRandomFingerprint(): Fingerprint {
-  const index = Math.floor(Math.random() * FINGERPRINTS.length);
-  return FINGERPRINTS[index];
-}
-
 // ============================================================
 // SESSION MANAGEMENT
-// Per-session fingerprint rotation for stealth
+// Per workflow-12102025.md:
+// - Session identity comes from PROXY LOCATION
+// - NOT from task params (no stateCode/timezone params)
+// - Language is always English
 // ============================================================
 
 export interface CrawlSession {
   sessionId: string;
-  fingerprint: Fingerprint;
+  fingerprint: BrowserFingerprint;
   proxyUrl: string | null;
-  stateCode?: string;
+  proxyTimezone?: string;
-  timezone?: string;
+  proxyState?: string;
   startedAt: Date;
+  // Per workflow-12102025.md: Dynamic Referer per dispensary
+  menuUrl?: string;
+  referer: string;
 }
 
-// Note: currentSession variable declared earlier in file for proper scoping
+let currentSession: CrawlSession | null = null;
 
 /**
- * Timezone to Accept-Language mapping
+ * Start a new crawl session
- * US timezones all use en-US but this can be extended for international
+ *
+ * Per workflow-12102025.md:
+ * - NO state/timezone params - identity comes from proxy location
+ * - Gets fingerprint from CrawlRotator (uses intoli/user-agents)
+ * - Language is always English (en-US)
+ * - Dynamic Referer per dispensary (from menuUrl)
+ *
+ * @param menuUrl - The dispensary's menu URL for dynamic Referer header
  */
-const TIMEZONE_TO_LOCALE: Record<string, string> = {
+export function startSession(menuUrl?: string): CrawlSession {
-  'America/Phoenix': 'en-US,en;q=0.9',
+  if (!crawlRotator) {
-  'America/Los_Angeles': 'en-US,en;q=0.9',
+    throw new Error('[Dutchie Client] Cannot start session without CrawlRotator');
-  'America/Denver': 'en-US,en;q=0.9',
+  }
-  'America/Chicago': 'en-US,en;q=0.9',
-  'America/New_York': 'en-US,en;q=0.9',
-  'America/Detroit': 'en-US,en;q=0.9',
-  'America/Anchorage': 'en-US,en;q=0.9',
-  'Pacific/Honolulu': 'en-US,en;q=0.9',
-};
 
-/**
+  // Per workflow-12102025.md: get identity from proxy location
- * Get Accept-Language header for a given timezone
+  const proxyLocation = crawlRotator.getProxyLocation();
- */
+  const fingerprint = crawlRotator.userAgent.getCurrent();
-export function getLocaleForTimezone(timezone?: string): string {
-  if (!timezone) return 'en-US,en;q=0.9';
-  return TIMEZONE_TO_LOCALE[timezone] || 'en-US,en;q=0.9';
-}
 
-/**
+  // Per workflow-12102025.md: Dynamic Referer per dispensary
- * Start a new crawl session with a random fingerprint
+  const referer = buildRefererFromMenuUrl(menuUrl);
- * Call this before crawling a store to get a fresh identity
- */
-export function startSession(stateCode?: string, timezone?: string): CrawlSession {
-  const baseFp = getRandomFingerprint();
-
-  // Override Accept-Language based on timezone for geographic consistency
-  const fingerprint: Fingerprint = {
-    ...baseFp,
-    acceptLanguage: getLocaleForTimezone(timezone),
-  };
 
   currentSession = {
     sessionId: `session_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
     fingerprint,
     proxyUrl: currentProxy,
-    stateCode,
+    proxyTimezone: proxyLocation?.timezone,
-    timezone,
+    proxyState: proxyLocation?.state,
     startedAt: new Date(),
+    menuUrl,
+    referer,
   };
 
   console.log(`[Dutchie Client] Started session ${currentSession.sessionId}`);
-  console.log(`[Dutchie Client]   Fingerprint: ${fingerprint.userAgent.slice(0, 50)}...`);
+  console.log(`[Dutchie Client]   Browser: ${fingerprint.browserName} (${fingerprint.deviceCategory})`);
-  console.log(`[Dutchie Client]   Accept-Language: ${fingerprint.acceptLanguage}`);
+  console.log(`[Dutchie Client]   DNT: ${fingerprint.httpFingerprint.hasDNT ? 'enabled' : 'disabled'}`);
-  if (timezone) {
+  console.log(`[Dutchie Client]   TLS: ${fingerprint.httpFingerprint.curlImpersonateBinary}`);
-    console.log(`[Dutchie Client]   Timezone: ${timezone}`);
+  console.log(`[Dutchie Client]   Referer: ${referer}`);
+  if (proxyLocation?.timezone) {
+    console.log(`[Dutchie Client]   Proxy: ${proxyLocation.state || 'unknown'} (${proxyLocation.timezone})`);
   }
 
   return currentSession;
@@ -347,48 +276,80 @@ export function getCurrentSession(): CrawlSession | null {
 // ============================================================
 
 /**
- * Build headers for Dutchie requests
+ * Per workflow-12102025.md: Build headers using HTTP fingerprint system
+ * Returns headers in browser-specific order with all natural variations
  */
-export function buildHeaders(refererPath: string, fingerprint?: Fingerprint): Record<string, string> {
+export function buildHeaders(isPost: boolean, contentLength?: number): { headers: Record<string, string>; orderedHeaders: string[] } {
-  const fp = fingerprint || getFingerprint();
+  if (!currentSession || !crawlRotator) {
-  const refererUrl = `https://dutchie.com${refererPath}`;
+    throw new Error('[Dutchie Client] Cannot build headers without active session');
-
-  const headers: Record<string, string> = {
-    'accept': 'application/json, text/plain, */*',
-    'accept-language': fp.acceptLanguage,
-    'content-type': 'application/json',
-    'origin': 'https://dutchie.com',
-    'referer': refererUrl,
-    'user-agent': fp.userAgent,
-    'apollographql-client-name': 'Marketplace (production)',
-  };
-
-  if (fp.secChUa) {
-    headers['sec-ch-ua'] = fp.secChUa;
-    headers['sec-ch-ua-mobile'] = fp.secChUaMobile || '?0';
-    headers['sec-ch-ua-platform'] = fp.secChUaPlatform || '"Windows"';
-    headers['sec-fetch-dest'] = 'empty';
-    headers['sec-fetch-mode'] = 'cors';
-    headers['sec-fetch-site'] = 'same-site';
   }
 
-  return headers;
+  const fp = currentSession.fingerprint;
+  const httpFp = fp.httpFingerprint;
+
+  // Per workflow-12102025.md: Build context for ordered headers
+  const context: HeaderContext = {
+    userAgent: fp.userAgent,
+    secChUa: fp.secChUa,
+    secChUaPlatform: fp.secChUaPlatform,
+    secChUaMobile: fp.secChUaMobile,
+    referer: currentSession.referer,
+    isPost,
+    contentLength,
+  };
+
+  // Per workflow-12102025.md: Get ordered headers from HTTP fingerprint service
+  return buildOrderedHeaders(httpFp, context);
 }
 
 /**
- * Execute HTTP POST using curl (bypasses TLS fingerprinting)
+ * Per workflow-12102025.md: Get curl binary for current session's browser
+ * Uses curl-impersonate for TLS fingerprint matching
  */
-export function curlPost(url: string, body: any, headers: Record<string, string>, timeout = 30000): CurlResponse {
+function getCurlBinaryForSession(): string {
-  const filteredHeaders = Object.entries(headers)
+  if (!currentSession) {
-    .filter(([k]) => k.toLowerCase() !== 'accept-encoding')
+    return 'curl';  // Fallback to standard curl
-    .map(([k, v]) => `-H '${k}: ${v}'`)
+  }
+
+  const browserType = currentSession.fingerprint.browserName as BrowserType;
+
+  // Per workflow-12102025.md: Check if curl-impersonate is available
+  if (isCurlImpersonateAvailable(browserType)) {
+    return getCurlBinary(browserType);
+  }
+
+  // Fallback to standard curl with warning
+  console.warn(`[Dutchie Client] curl-impersonate not available for ${browserType}, using standard curl`);
+  return 'curl';
+}
+
+/**
+ * Per workflow-12102025.md: Execute HTTP POST using curl/curl-impersonate
+ * - Uses browser-specific TLS fingerprint via curl-impersonate
+ * - Headers sent in browser-specific order
+ * - Dynamic Referer per dispensary
+ */
+export function curlPost(url: string, body: any, timeout = 30000): CurlResponse {
+  const bodyJson = JSON.stringify(body);
+
+  // Per workflow-12102025.md: Build ordered headers for POST request
+  const { headers, orderedHeaders } = buildHeaders(true, bodyJson.length);
+
+  // Per workflow-12102025.md: Build header args in browser-specific order
+  const headerArgs = orderedHeaders
+    .filter(h => h !== 'Host' && h !== 'Content-Length')  // curl handles these
+    .map(h => `-H '${h}: ${headers[h]}'`)
     .join(' ');
 
-  const bodyJson = JSON.stringify(body).replace(/'/g, "'\\''");
+  const bodyEscaped = bodyJson.replace(/'/g, "'\\''");
   const timeoutSec = Math.ceil(timeout / 1000);
   const separator = '___HTTP_STATUS___';
   const proxyArg = getProxyArg();
-  const cmd = `curl -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${filteredHeaders} -d '${bodyJson}' '${url}'`;
+
+  // Per workflow-12102025.md: Use curl-impersonate for TLS fingerprint matching
+  const curlBinary = getCurlBinaryForSession();
+
+  const cmd = `${curlBinary} -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${headerArgs} -d '${bodyEscaped}' '${url}'`;
 
   try {
     const output = execSync(cmd, {
@@ -427,19 +388,29 @@ export function curlPost(url: string, body: any, headers: Record<string, string>
 }
 
 /**
- * Execute HTTP GET using curl (bypasses TLS fingerprinting)
+ * Per workflow-12102025.md: Execute HTTP GET using curl/curl-impersonate
- * Returns HTML or JSON depending on response content-type
+ * - Uses browser-specific TLS fingerprint via curl-impersonate
+ * - Headers sent in browser-specific order
+ * - Dynamic Referer per dispensary
  */
-export function curlGet(url: string, headers: Record<string, string>, timeout = 30000): CurlResponse {
+export function curlGet(url: string, timeout = 30000): CurlResponse {
-  const filteredHeaders = Object.entries(headers)
+  // Per workflow-12102025.md: Build ordered headers for GET request
-    .filter(([k]) => k.toLowerCase() !== 'accept-encoding')
+  const { headers, orderedHeaders } = buildHeaders(false);
-    .map(([k, v]) => `-H '${k}: ${v}'`)
+
+  // Per workflow-12102025.md: Build header args in browser-specific order
+  const headerArgs = orderedHeaders
+    .filter(h => h !== 'Host' && h !== 'Content-Length')  // curl handles these
+    .map(h => `-H '${h}: ${headers[h]}'`)
     .join(' ');
 
   const timeoutSec = Math.ceil(timeout / 1000);
   const separator = '___HTTP_STATUS___';
   const proxyArg = getProxyArg();
-  const cmd = `curl -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${filteredHeaders} '${url}'`;
+
+  // Per workflow-12102025.md: Use curl-impersonate for TLS fingerprint matching
+  const curlBinary = getCurlBinaryForSession();
+
+  const cmd = `${curlBinary} -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${headerArgs} '${url}'`;
 
   try {
     const output = execSync(cmd, {
@@ -459,7 +430,6 @@ export function curlGet(url: string, headers: Record<string, string>, timeout =
     const responseBody = output.slice(0, separatorIndex);
     const statusCode = parseInt(output.slice(separatorIndex + separator.length).trim(), 10);
 
-    // Try to parse as JSON, otherwise return as string (HTML)
     try {
       return { status: statusCode, data: JSON.parse(responseBody) };
     } catch {
@@ -476,16 +446,22 @@ export function curlGet(url: string, headers: Record<string, string>, timeout =
 
 // ============================================================
 // GRAPHQL EXECUTION
+// Per workflow-12102025.md:
+// - On 403: immediately rotate IP + fingerprint (no delay first)
+// - Then retry
 // ============================================================
 
 export interface ExecuteGraphQLOptions {
   maxRetries?: number;
   retryOn403?: boolean;
-  cName?: string;  // Optional - used for Referer header, defaults to 'cities'
+  cName?: string;
 }
 
 /**
- * Execute GraphQL query with curl (bypasses TLS fingerprinting)
+ * Per workflow-12102025.md: Execute GraphQL query with curl/curl-impersonate
+ * - Uses browser-specific TLS fingerprint
+ * - Headers in browser-specific order
+ * - On 403: immediately rotate IP + fingerprint, then retry
  */
 export async function executeGraphQL(
   operationName: string,
@@ -493,7 +469,12 @@ export async function executeGraphQL(
   hash: string,
   options: ExecuteGraphQLOptions
 ): Promise<any> {
-  const { maxRetries = 3, retryOn403 = true, cName = 'cities' } = options;
+  const { maxRetries = 3, retryOn403 = true } = options;
+
+  // Per workflow-12102025.md: Session must be active for requests
+  if (!currentSession) {
+    throw new Error('[Dutchie Client] Cannot execute GraphQL without active session - call startSession() first');
+  }
 
   const body = {
     operationName,
@@ -507,14 +488,14 @@ export async function executeGraphQL(
   let attempt = 0;
 
   while (attempt <= maxRetries) {
-    const fingerprint = getFingerprint();
-    const headers = buildHeaders(`/embedded-menu/${cName}`, fingerprint);
-
     console.log(`[Dutchie Client] curl POST ${operationName} (attempt ${attempt + 1}/${maxRetries + 1})`);
 
-    const response = curlPost(DUTCHIE_CONFIG.graphqlEndpoint, body, headers, DUTCHIE_CONFIG.timeout);
+    const startTime = Date.now();
+    // Per workflow-12102025.md: curlPost now uses ordered headers and curl-impersonate
+    const response = curlPost(DUTCHIE_CONFIG.graphqlEndpoint, body, DUTCHIE_CONFIG.timeout);
+    const responseTime = Date.now() - startTime;
 
-    console.log(`[Dutchie Client] Response status: ${response.status}`);
+    console.log(`[Dutchie Client] Response status: ${response.status} (${responseTime}ms)`);
 
     if (response.error) {
       console.error(`[Dutchie Client] curl error: ${response.error}`);
@@ -527,6 +508,9 @@ export async function executeGraphQL(
     }
 
     if (response.status === 200) {
+      // Per workflow-12102025.md: success resets consecutive 403 count
+      await recordProxySuccess(responseTime);
+
       if (response.data?.errors?.length > 0) {
         console.warn(`[Dutchie Client] GraphQL errors: ${JSON.stringify(response.data.errors[0])}`);
       }
@@ -534,10 +518,20 @@ export async function executeGraphQL(
     }
 
     if (response.status === 403 && retryOn403) {
-      console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`);
+      // Per workflow-12102025.md: immediately rotate IP + fingerprint
-      rotateFingerprint();
+      console.warn(`[Dutchie Client] 403 blocked - immediately rotating proxy + fingerprint...`);
+      const hasMoreProxies = await handle403Block();
+
+      if (!hasMoreProxies) {
+        throw new Error('All proxies exhausted - no more IPs available');
+      }
+
+      // Per workflow-12102025.md: Update session referer after rotation
+      currentSession.referer = buildRefererFromMenuUrl(currentSession.menuUrl);
+
       attempt++;
-      await sleep(1000 * attempt);
+      // Per workflow-12102025.md: small backoff after rotation
+      await sleep(500);
       continue;
     }
 
@@ -566,8 +560,10 @@ export interface FetchPageOptions {
 }
 
 /**
- * Fetch HTML page from Dutchie (for city pages, dispensary pages, etc.)
+ * Per workflow-12102025.md: Fetch HTML page from Dutchie
- * Returns raw HTML string
+ * - Uses browser-specific TLS fingerprint
+ * - Headers in browser-specific order
+ * - Same 403 handling as GraphQL
  */
 export async function fetchPage(
   path: string,
@@ -576,32 +572,22 @@ export async function fetchPage(
   const { maxRetries = 3, retryOn403 = true } = options;
   const url = `${DUTCHIE_CONFIG.baseUrl}${path}`;
 
+  // Per workflow-12102025.md: Session must be active for requests
+  if (!currentSession) {
+    throw new Error('[Dutchie Client] Cannot fetch page without active session - call startSession() first');
+  }
+
   let attempt = 0;
 
   while (attempt <= maxRetries) {
-    const fingerprint = getFingerprint();
+    // Per workflow-12102025.md: curlGet now uses ordered headers and curl-impersonate
-    const headers: Record<string, string> = {
-      'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-      'accept-language': fingerprint.acceptLanguage,
-      'user-agent': fingerprint.userAgent,
-    };
-
-    if (fingerprint.secChUa) {
-      headers['sec-ch-ua'] = fingerprint.secChUa;
-      headers['sec-ch-ua-mobile'] = fingerprint.secChUaMobile || '?0';
-      headers['sec-ch-ua-platform'] = fingerprint.secChUaPlatform || '"Windows"';
-      headers['sec-fetch-dest'] = 'document';
-      headers['sec-fetch-mode'] = 'navigate';
-      headers['sec-fetch-site'] = 'none';
-      headers['sec-fetch-user'] = '?1';
-      headers['upgrade-insecure-requests'] = '1';
-    }
-
     console.log(`[Dutchie Client] curl GET ${path} (attempt ${attempt + 1}/${maxRetries + 1})`);
 
-    const response = curlGet(url, headers, DUTCHIE_CONFIG.timeout);
+    const startTime = Date.now();
+    const response = curlGet(url, DUTCHIE_CONFIG.timeout);
+    const responseTime = Date.now() - startTime;
 
-    console.log(`[Dutchie Client] Response status: ${response.status}`);
+    console.log(`[Dutchie Client] Response status: ${response.status} (${responseTime}ms)`);
 
     if (response.error) {
       console.error(`[Dutchie Client] curl error: ${response.error}`);
@@ -613,14 +599,26 @@ export async function fetchPage(
     }
 
     if (response.status === 200) {
+      // Per workflow-12102025.md: success resets consecutive 403 count
+      await recordProxySuccess(responseTime);
       return { html: response.data, status: response.status };
     }
 
     if (response.status === 403 && retryOn403) {
-      console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`);
+      // Per workflow-12102025.md: immediately rotate IP + fingerprint
-      rotateFingerprint();
+      console.warn(`[Dutchie Client] 403 blocked - immediately rotating proxy + fingerprint...`);
+      const hasMoreProxies = await handle403Block();
+
+      if (!hasMoreProxies) {
+        throw new Error('All proxies exhausted - no more IPs available');
+      }
+
+      // Per workflow-12102025.md: Update session after rotation
+      currentSession.referer = buildRefererFromMenuUrl(currentSession.menuUrl);
+
       attempt++;
-      await sleep(1000 * attempt);
+      // Per workflow-12102025.md: small backoff after rotation
+      await sleep(500);
       continue;
     }
 

backend/src/platforms/dutchie/index.ts

@@ -6,22 +6,17 @@
  */
 
 export {
-  // HTTP Client
+  // HTTP Client (per workflow-12102025.md: uses curl-impersonate + ordered headers)
   curlPost,
   curlGet,
   executeGraphQL,
   fetchPage,
   extractNextData,
 
-  // Headers & Fingerprints
+  // Headers (per workflow-12102025.md: browser-specific ordering)
   buildHeaders,
-  getFingerprint,
-  rotateFingerprint,
-  resetFingerprint,
-  getRandomFingerprint,
-  getLocaleForTimezone,
 
-  // Session Management (per-store fingerprint rotation)
+  // Session Management (per workflow-12102025.md: menuUrl for dynamic Referer)
   startSession,
   endSession,
   getCurrentSession,

backend/src/routes/analytics-v2.ts

@@ -7,15 +7,23 @@
  * Routes are prefixed with /api/analytics/v2
  *
  * Phase 3: Analytics Engine + Rec/Med by State
+ *
+ * SECURITY: All routes require authentication via authMiddleware.
+ * Access is granted to:
+ * - Trusted origins (cannaiq.co, findadispo.com, etc.)
+ * - Trusted IPs (localhost, internal pods)
+ * - Valid JWT or API tokens
  */
 
 import { Router, Request, Response } from 'express';
 import { Pool } from 'pg';
+import { authMiddleware } from '../auth/middleware';
 import { PriceAnalyticsService } from '../services/analytics/PriceAnalyticsService';
 import { BrandPenetrationService } from '../services/analytics/BrandPenetrationService';
 import { CategoryAnalyticsService } from '../services/analytics/CategoryAnalyticsService';
 import { StoreAnalyticsService } from '../services/analytics/StoreAnalyticsService';
 import { StateAnalyticsService } from '../services/analytics/StateAnalyticsService';
+import { BrandIntelligenceService } from '../services/analytics/BrandIntelligenceService';
 import { TimeWindow, LegalType } from '../services/analytics/types';
 
 function parseTimeWindow(window?: string): TimeWindow {
@@ -35,12 +43,17 @@ function parseLegalType(legalType?: string): LegalType {
 export function createAnalyticsV2Router(pool: Pool): Router {
   const router = Router();
 
+  // SECURITY: Apply auth middleware to ALL routes
+  // This gate ensures only authenticated requests can access analytics data
+  router.use(authMiddleware);
+
   // Initialize services
   const priceService = new PriceAnalyticsService(pool);
   const brandService = new BrandPenetrationService(pool);
   const categoryService = new CategoryAnalyticsService(pool);
   const storeService = new StoreAnalyticsService(pool);
   const stateService = new StateAnalyticsService(pool);
+  const brandIntelligenceService = new BrandIntelligenceService(pool);
 
   // ============================================================
   // PRICE ANALYTICS
@@ -231,6 +244,76 @@ export function createAnalyticsV2Router(pool: Pool): Router {
     }
   });
 
+  /**
+   * GET /brand/:name/promotions
+   * Get brand promotional history - tracks specials, discounts, duration, and sales estimates
+   *
+   * Query params:
+   *   - window: 7d|30d|90d (default: 90d)
+   *   - state: state code filter (e.g., AZ)
+   *   - category: category filter (e.g., Flower)
+   */
+  router.get('/brand/:name/promotions', async (req: Request, res: Response) => {
+    try {
+      const brandName = decodeURIComponent(req.params.name);
+      const window = parseTimeWindow(req.query.window as string) || '90d';
+      const stateCode = req.query.state as string | undefined;
+      const category = req.query.category as string | undefined;
+
+      const result = await brandService.getBrandPromotionalHistory(brandName, {
+        window,
+        stateCode,
+        category,
+      });
+      res.json(result);
+    } catch (error) {
+      console.error('[AnalyticsV2] Brand promotions error:', error);
+      res.status(500).json({ error: 'Failed to fetch brand promotional history' });
+    }
+  });
+
+  /**
+   * GET /brand/:name/intelligence
+   * Get comprehensive B2B brand intelligence dashboard data
+   *
+   * Returns all brand metrics in a single unified response:
+   *   - Performance Snapshot (active SKUs, revenue, stores, market share)
+   *   - Alerts/Slippage (lost stores, delisted SKUs, competitor takeovers)
+   *   - Product Velocity (daily rates, velocity status)
+   *   - Retail Footprint (penetration, whitespace opportunities)
+   *   - Competitive Landscape (price position, market share trend)
+   *   - Inventory Health (days of stock, risk levels)
+   *   - Promotion Effectiveness (baseline vs promo velocity, ROI)
+   *
+   * Query params:
+   *   - window: 7d|30d|90d (default: 30d)
+   *   - state: state code filter (e.g., AZ)
+   *   - category: category filter (e.g., Flower)
+   */
+  router.get('/brand/:name/intelligence', async (req: Request, res: Response) => {
+    try {
+      const brandName = decodeURIComponent(req.params.name);
+      const window = parseTimeWindow(req.query.window as string);
+      const stateCode = req.query.state as string | undefined;
+      const category = req.query.category as string | undefined;
+
+      const result = await brandIntelligenceService.getBrandIntelligence(brandName, {
+        window,
+        stateCode,
+        category,
+      });
+
+      if (!result) {
+        return res.status(404).json({ error: 'Brand not found' });
+      }
+
+      res.json(result);
+    } catch (error) {
+      console.error('[AnalyticsV2] Brand intelligence error:', error);
+      res.status(500).json({ error: 'Failed to fetch brand intelligence' });
+    }
+  });
+
   // ============================================================
   // CATEGORY ANALYTICS
   // ============================================================
@@ -400,6 +483,31 @@ export function createAnalyticsV2Router(pool: Pool): Router {
     }
   });
 
+  /**
+   * GET /store/:id/quantity-changes
+   * Get quantity changes for a store (increases/decreases)
+   * Useful for estimating sales (decreases) or restocks (increases)
+   *
+   * Query params:
+   *   - window: 7d|30d|90d (default: 7d)
+   *   - direction: increase|decrease|all (default: all)
+   *   - limit: number (default: 100)
+   */
+  router.get('/store/:id/quantity-changes', async (req: Request, res: Response) => {
+    try {
+      const dispensaryId = parseInt(req.params.id);
+      const window = parseTimeWindow(req.query.window as string);
+      const direction = (req.query.direction as 'increase' | 'decrease' | 'all') || 'all';
+      const limit = req.query.limit ? parseInt(req.query.limit as string) : 100;
+
+      const result = await storeService.getQuantityChanges(dispensaryId, { window, direction, limit });
+      res.json(result);
+    } catch (error) {
+      console.error('[AnalyticsV2] Store quantity changes error:', error);
+      res.status(500).json({ error: 'Failed to fetch store quantity changes' });
+    }
+  });
+
   /**
    * GET /store/:id/inventory
    * Get store inventory composition

backend/src/routes/auth.ts

@@ -47,4 +47,27 @@ router.post('/refresh', authMiddleware, async (req: AuthRequest, res) => {
   res.json({ token });
 });
 
+// Verify password for sensitive actions (requires current user to be authenticated)
+router.post('/verify-password', authMiddleware, async (req: AuthRequest, res) => {
+  try {
+    const { password } = req.body;
+
+    if (!password) {
+      return res.status(400).json({ error: 'Password required' });
+    }
+
+    // Re-authenticate the current user with the provided password
+    const user = await authenticateUser(req.user!.email, password);
+
+    if (!user) {
+      return res.status(401).json({ error: 'Invalid password', verified: false });
+    }
+
+    res.json({ verified: true });
+  } catch (error) {
+    console.error('Password verification error:', error);
+    res.status(500).json({ error: 'Internal server error' });
+  }
+});
+
 export default router;

backend/src/routes/categories.ts

@@ -5,31 +5,35 @@ import { pool } from '../db/pool';
 const router = Router();
 router.use(authMiddleware);
 
-// Get categories (flat list)
+// Get categories (flat list) - derived from actual product data
 router.get('/', async (req, res) => {
   try {
-    const { store_id } = req.query;
+    const { store_id, in_stock_only } = req.query;
 
     let query = `
       SELECT
-        c.*,
+        category_raw as name,
-        COUNT(DISTINCT p.id) as product_count,
+        category_raw as slug,
-        pc.name as parent_name
+        COUNT(*) as product_count,
-      FROM categories c
+        COUNT(*) FILTER (WHERE is_in_stock = true) as in_stock_count
-      LEFT JOIN store_products p ON c.name = p.category_raw
+      FROM store_products
-      LEFT JOIN categories pc ON c.parent_id = pc.id
+      WHERE category_raw IS NOT NULL
     `;
 
     const params: any[] = [];
 
     if (store_id) {
-      query += ' WHERE c.store_id = $1';
       params.push(store_id);
+      query += ` AND dispensary_id = $${params.length}`;
+    }
+
+    if (in_stock_only === 'true') {
+      query += ` AND is_in_stock = true`;
     }
 
     query += `
-      GROUP BY c.id, pc.name
+      GROUP BY category_raw
-      ORDER BY c.display_order, c.name
+      ORDER BY category_raw
     `;
 
     const result = await pool.query(query, params);
@@ -40,49 +44,85 @@ router.get('/', async (req, res) => {
   }
 });
 
-// Get category tree (hierarchical)
+// Get category tree (hierarchical) - category -> subcategory structure from product data
 router.get('/tree', async (req, res) => {
   try {
-    const { store_id } = req.query;
+    const { store_id, in_stock_only } = req.query;
 
-    if (!store_id) {
+    // Get category + subcategory combinations with counts
-      return res.status(400).json({ error: 'store_id is required' });
+    let query = `
-    }
-    
-    // Get all categories for the store
-    const result = await pool.query(`
       SELECT
-        c.*,
+        category_raw as category,
-        COUNT(DISTINCT p.id) as product_count
+        subcategory_raw as subcategory,
-      FROM categories c
+        COUNT(*) as product_count,
-      LEFT JOIN store_products p ON c.name = p.category_raw AND p.is_in_stock = true AND p.dispensary_id = $1
+        COUNT(*) FILTER (WHERE is_in_stock = true) as in_stock_count
-      WHERE c.store_id = $1
+      FROM store_products
-      GROUP BY c.id
+      WHERE category_raw IS NOT NULL
-      ORDER BY c.display_order, c.name
+    `;
-    `, [store_id]);
 
-    // Build tree structure
+    const params: any[] = [];
-    const categories = result.rows;
-    const categoryMap = new Map();
-    const tree: any[] = [];
 
-    // First pass: create map
+    if (store_id) {
-    categories.forEach((cat: { id: number; parent_id?: number }) => {
+      params.push(store_id);
-      categoryMap.set(cat.id, { ...cat, children: [] });
+      query += ` AND dispensary_id = $${params.length}`;
-    });
-
-    // Second pass: build tree
-    categories.forEach((cat: { id: number; parent_id?: number }) => {
-      const node = categoryMap.get(cat.id);
-      if (cat.parent_id) {
-        const parent = categoryMap.get(cat.parent_id);
-        if (parent) {
-          parent.children.push(node);
     }
-      } else {
+
-        tree.push(node);
+    if (in_stock_only === 'true') {
+      query += ` AND is_in_stock = true`;
     }
+
+    query += `
+      GROUP BY category_raw, subcategory_raw
+      ORDER BY category_raw, subcategory_raw
+    `;
+
+    const result = await pool.query(query, params);
+
+    // Build tree structure: category -> subcategories
+    const categoryMap = new Map<string, {
+      name: string;
+      slug: string;
+      product_count: number;
+      in_stock_count: number;
+      subcategories: Array<{
+        name: string;
+        slug: string;
+        product_count: number;
+        in_stock_count: number;
+      }>;
+    }>();
+
+    for (const row of result.rows) {
+      const category = row.category;
+      const subcategory = row.subcategory;
+      const count = parseInt(row.product_count);
+      const inStockCount = parseInt(row.in_stock_count);
+
+      if (!categoryMap.has(category)) {
+        categoryMap.set(category, {
+          name: category,
+          slug: category.toLowerCase().replace(/\s+/g, '-'),
+          product_count: 0,
+          in_stock_count: 0,
+          subcategories: []
         });
+      }
+
+      const cat = categoryMap.get(category)!;
+      cat.product_count += count;
+      cat.in_stock_count += inStockCount;
+
+      if (subcategory) {
+        cat.subcategories.push({
+          name: subcategory,
+          slug: subcategory.toLowerCase().replace(/\s+/g, '-'),
+          product_count: count,
+          in_stock_count: inStockCount
+        });
+      }
+    }
+
+    const tree = Array.from(categoryMap.values());
 
     res.json({ tree });
   } catch (error) {
@@ -91,4 +131,91 @@ router.get('/tree', async (req, res) => {
   }
 });
 
+// Get all unique subcategories for a category
+router.get('/:category/subcategories', async (req, res) => {
+  try {
+    const { category } = req.params;
+    const { store_id, in_stock_only } = req.query;
+
+    let query = `
+      SELECT
+        subcategory_raw as name,
+        subcategory_raw as slug,
+        COUNT(*) as product_count,
+        COUNT(*) FILTER (WHERE is_in_stock = true) as in_stock_count
+      FROM store_products
+      WHERE category_raw = $1
+        AND subcategory_raw IS NOT NULL
+    `;
+
+    const params: any[] = [category];
+
+    if (store_id) {
+      params.push(store_id);
+      query += ` AND dispensary_id = $${params.length}`;
+    }
+
+    if (in_stock_only === 'true') {
+      query += ` AND is_in_stock = true`;
+    }
+
+    query += `
+      GROUP BY subcategory_raw
+      ORDER BY subcategory_raw
+    `;
+
+    const result = await pool.query(query, params);
+    res.json({
+      category,
+      subcategories: result.rows
+    });
+  } catch (error) {
+    console.error('Error fetching subcategories:', error);
+    res.status(500).json({ error: 'Failed to fetch subcategories' });
+  }
+});
+
+// Get global category summary (across all stores)
+router.get('/summary', async (req, res) => {
+  try {
+    const { state } = req.query;
+
+    let query = `
+      SELECT
+        sp.category_raw as category,
+        COUNT(DISTINCT sp.id) as product_count,
+        COUNT(DISTINCT sp.dispensary_id) as store_count,
+        COUNT(*) FILTER (WHERE sp.is_in_stock = true) as in_stock_count
+      FROM store_products sp
+    `;
+
+    const params: any[] = [];
+
+    if (state) {
+      query += `
+        JOIN dispensaries d ON sp.dispensary_id = d.id
+        WHERE sp.category_raw IS NOT NULL
+          AND d.state = $1
+      `;
+      params.push(state);
+    } else {
+      query += ` WHERE sp.category_raw IS NOT NULL`;
+    }
+
+    query += `
+      GROUP BY sp.category_raw
+      ORDER BY product_count DESC
+    `;
+
+    const result = await pool.query(query, params);
+    res.json({
+      categories: result.rows,
+      total_categories: result.rows.length
+    });
+  } catch (error) {
+    console.error('Error fetching category summary:', error);
+    res.status(500).json({ error: 'Failed to fetch category summary' });
+  }
+});
+
 export default router;

backend/src/routes/dispensaries.ts

@@ -11,7 +11,7 @@ const VALID_MENU_TYPES = ['dutchie', 'treez', 'jane', 'weedmaps', 'leafly', 'mea
 // Get all dispensaries (with pagination)
 router.get('/', async (req, res) => {
   try {
-    const { menu_type, city, state, crawl_enabled, dutchie_verified, limit, offset, search } = req.query;
+    const { menu_type, city, state, crawl_enabled, dutchie_verified, status, limit, offset, search } = req.query;
     const pageLimit = Math.min(parseInt(limit as string) || 50, 500);
     const pageOffset = parseInt(offset as string) || 0;
 
@@ -100,6 +100,12 @@ router.get('/', async (req, res) => {
       }
     }
 
+    // Filter by status (e.g., 'dropped', 'open', 'closed')
+    if (status) {
+      conditions.push(`status = $${params.length + 1}`);
+      params.push(status);
+    }
+
     // Search filter (name, dba_name, city, company_name)
     if (search) {
       conditions.push(`(name ILIKE $${params.length + 1} OR dba_name ILIKE $${params.length + 1} OR city ILIKE $${params.length + 1})`);
@@ -161,6 +167,7 @@ router.get('/stats/crawl-status', async (req, res) => {
         COUNT(*) FILTER (WHERE crawl_enabled = false OR crawl_enabled IS NULL) as disabled_count,
         COUNT(*) FILTER (WHERE dutchie_verified = true) as verified_count,
         COUNT(*) FILTER (WHERE dutchie_verified = false OR dutchie_verified IS NULL) as unverified_count,
+        COUNT(*) FILTER (WHERE status = 'dropped') as dropped_count,
         COUNT(*) as total_count
       FROM dispensaries
     `;
@@ -190,6 +197,34 @@ router.get('/stats/crawl-status', async (req, res) => {
   }
 });
 
+// Get dropped stores count (for dashboard alert)
+router.get('/stats/dropped', async (req, res) => {
+  try {
+    const result = await pool.query(`
+      SELECT
+        COUNT(*) as dropped_count,
+        json_agg(json_build_object(
+          'id', id,
+          'name', name,
+          'city', city,
+          'state', state,
+          'dropped_at', updated_at
+        ) ORDER BY updated_at DESC) FILTER (WHERE status = 'dropped') as dropped_stores
+      FROM dispensaries
+      WHERE status = 'dropped'
+    `);
+
+    const row = result.rows[0];
+    res.json({
+      dropped_count: parseInt(row.dropped_count) || 0,
+      dropped_stores: row.dropped_stores || []
+    });
+  } catch (error) {
+    console.error('Error fetching dropped stores:', error);
+    res.status(500).json({ error: 'Failed to fetch dropped stores' });
+  }
+});
+
 // Get single dispensary by slug or ID
 router.get('/:slugOrId', async (req, res) => {
   try {

backend/src/routes/events.ts

@@ -22,11 +22,17 @@ interface ProductClickEventPayload {
   store_id?: string;
   brand_id?: string;
   campaign_id?: string;
+  dispensary_name?: string;
   action: 'view' | 'open_store' | 'open_product' | 'compare' | 'other';
   source: string;
   page_type?: string;  // Page where event occurred (e.g., StoreDetailPage, BrandsIntelligence)
   url_path?: string;   // URL path for debugging
   occurred_at?: string;
+  // Visitor location (from frontend IP geolocation)
+  visitor_city?: string;
+  visitor_state?: string;
+  visitor_lat?: number;
+  visitor_lng?: number;
 }
 
 /**
@@ -77,13 +83,14 @@ router.post('/product-click', optionalAuthMiddleware, async (req: Request, res:
     // Insert the event with enhanced fields
     await pool.query(
       `INSERT INTO product_click_events
-       (product_id, store_id, brand_id, campaign_id, action, source, user_id, ip_address, user_agent, occurred_at, event_type, page_type, url_path, device_type)
+       (product_id, store_id, brand_id, campaign_id, dispensary_name, action, source, user_id, ip_address, user_agent, occurred_at, event_type, page_type, url_path, device_type, visitor_city, visitor_state, visitor_lat, visitor_lng)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)`,
       [
         payload.product_id,
         payload.store_id || null,
         payload.brand_id || null,
         payload.campaign_id || null,
+        payload.dispensary_name || null,
         payload.action,
         payload.source,
         userId,
@@ -93,7 +100,11 @@ router.post('/product-click', optionalAuthMiddleware, async (req: Request, res:
         'product_click',  // event_type
         payload.page_type || null,
         payload.url_path || null,
-        deviceType
+        deviceType,
+        payload.visitor_city || null,
+        payload.visitor_state || null,
+        payload.visitor_lat || null,
+        payload.visitor_lng || null
       ]
     );
 

backend/src/routes/intelligence.ts

@@ -14,35 +14,56 @@ router.use(authMiddleware);
 /**
  * GET /api/admin/intelligence/brands
  * List all brands with state presence, store counts, and pricing
+ * Query params:
+ *   - state: Filter by state (e.g., "AZ")
+ *   - limit: Max results (default 500)
+ *   - offset: Pagination offset
  */
 router.get('/brands', async (req: Request, res: Response) => {
   try {
-    const { limit = '500', offset = '0' } = req.query;
+    const { limit = '500', offset = '0', state } = req.query;
     const limitNum = Math.min(parseInt(limit as string, 10), 1000);
     const offsetNum = parseInt(offset as string, 10);
 
+    // Build WHERE clause based on state filter
+    let stateFilter = '';
+    const params: any[] = [limitNum, offsetNum];
+    if (state && state !== 'all') {
+      stateFilter = 'AND d.state = $3';
+      params.push(state);
+    }
+
     const { rows } = await pool.query(`
       SELECT
         sp.brand_name_raw as brand_name,
         array_agg(DISTINCT d.state) FILTER (WHERE d.state IS NOT NULL) as states,
         COUNT(DISTINCT d.id) as store_count,
         COUNT(DISTINCT sp.id) as sku_count,
-        ROUND(AVG(sp.price_rec)::numeric, 2) FILTER (WHERE sp.price_rec > 0) as avg_price_rec,
+        ROUND(AVG(sp.price_rec) FILTER (WHERE sp.price_rec > 0)::numeric, 2) as avg_price_rec,
-        ROUND(AVG(sp.price_med)::numeric, 2) FILTER (WHERE sp.price_med > 0) as avg_price_med
+        ROUND(AVG(sp.price_med) FILTER (WHERE sp.price_med > 0)::numeric, 2) as avg_price_med
       FROM store_products sp
       JOIN dispensaries d ON sp.dispensary_id = d.id
       WHERE sp.brand_name_raw IS NOT NULL AND sp.brand_name_raw != ''
+        ${stateFilter}
       GROUP BY sp.brand_name_raw
       ORDER BY store_count DESC, sku_count DESC
       LIMIT $1 OFFSET $2
-    `, [limitNum, offsetNum]);
+    `, params);
 
-    // Get total count
+    // Get total count with same state filter
+    const countParams: any[] = [];
+    let countStateFilter = '';
+    if (state && state !== 'all') {
+      countStateFilter = 'AND d.state = $1';
+      countParams.push(state);
+    }
     const { rows: countRows } = await pool.query(`
-      SELECT COUNT(DISTINCT brand_name_raw) as total
+      SELECT COUNT(DISTINCT sp.brand_name_raw) as total
-      FROM store_products
+      FROM store_products sp
-      WHERE brand_name_raw IS NOT NULL AND brand_name_raw != ''
+      JOIN dispensaries d ON sp.dispensary_id = d.id
-    `);
+      WHERE sp.brand_name_raw IS NOT NULL AND sp.brand_name_raw != ''
+        ${countStateFilter}
+    `, countParams);
 
     res.json({
       brands: rows.map((r: any) => ({
@@ -147,29 +168,63 @@ router.get('/brands/:brandName/penetration', async (req: Request, res: Response)
 /**
  * GET /api/admin/intelligence/pricing
  * Get pricing analytics by category
+ * Query params:
+ *   - state: Filter by state (e.g., "AZ")
  */
 router.get('/pricing', async (req: Request, res: Response) => {
   try {
-    const { rows: categoryRows } = await pool.query(`
+    const { state } = req.query;
+
+    // Build WHERE clause based on state filter
+    let stateFilter = '';
+    const categoryParams: any[] = [];
+    const stateQueryParams: any[] = [];
+    const overallParams: any[] = [];
+
+    if (state && state !== 'all') {
+      stateFilter = 'AND d.state = $1';
+      categoryParams.push(state);
+      overallParams.push(state);
+    }
+
+    // Category pricing with optional state filter
+    const categoryQuery = state && state !== 'all'
+      ? `
         SELECT
           sp.category_raw as category,
           ROUND(AVG(sp.price_rec)::numeric, 2) as avg_price,
-        MIN(sp.price_rec) FILTER (WHERE sp.price_rec > 0) as min_price,
+          MIN(sp.price_rec) as min_price,
           MAX(sp.price_rec) as max_price,
-        ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec)::numeric, 2)
+          ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec)::numeric, 2) as median_price,
-          FILTER (WHERE sp.price_rec > 0) as median_price,
+          COUNT(*) as product_count
+        FROM store_products sp
+        JOIN dispensaries d ON sp.dispensary_id = d.id
+        WHERE sp.category_raw IS NOT NULL AND sp.price_rec > 0 ${stateFilter}
+        GROUP BY sp.category_raw
+        ORDER BY product_count DESC
+      `
+      : `
+        SELECT
+          sp.category_raw as category,
+          ROUND(AVG(sp.price_rec)::numeric, 2) as avg_price,
+          MIN(sp.price_rec) as min_price,
+          MAX(sp.price_rec) as max_price,
+          ROUND(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec)::numeric, 2) as median_price,
           COUNT(*) as product_count
         FROM store_products sp
         WHERE sp.category_raw IS NOT NULL AND sp.price_rec > 0
         GROUP BY sp.category_raw
         ORDER BY product_count DESC
-    `);
+      `;
 
+    const { rows: categoryRows } = await pool.query(categoryQuery, categoryParams);
+
+    // State pricing
     const { rows: stateRows } = await pool.query(`
       SELECT
         d.state,
         ROUND(AVG(sp.price_rec)::numeric, 2) as avg_price,
-        MIN(sp.price_rec) FILTER (WHERE sp.price_rec > 0) as min_price,
+        MIN(sp.price_rec) as min_price,
         MAX(sp.price_rec) as max_price,
         COUNT(DISTINCT sp.id) as product_count
       FROM store_products sp
@@ -179,6 +234,31 @@ router.get('/pricing', async (req: Request, res: Response) => {
       ORDER BY avg_price DESC
     `);
 
+    // Overall stats with optional state filter
+    const overallQuery = state && state !== 'all'
+      ? `
+        SELECT
+          ROUND(AVG(sp.price_rec)::numeric, 2) as avg_price,
+          MIN(sp.price_rec) as min_price,
+          MAX(sp.price_rec) as max_price,
+          COUNT(*) as total_products
+        FROM store_products sp
+        JOIN dispensaries d ON sp.dispensary_id = d.id
+        WHERE sp.price_rec > 0 ${stateFilter}
+      `
+      : `
+        SELECT
+          ROUND(AVG(sp.price_rec)::numeric, 2) as avg_price,
+          MIN(sp.price_rec) as min_price,
+          MAX(sp.price_rec) as max_price,
+          COUNT(*) as total_products
+        FROM store_products sp
+        WHERE sp.price_rec > 0
+      `;
+
+    const { rows: overallRows } = await pool.query(overallQuery, overallParams);
+    const overall = overallRows[0];
+
     res.json({
       byCategory: categoryRows.map((r: any) => ({
         category: r.category,
@@ -195,6 +275,12 @@ router.get('/pricing', async (req: Request, res: Response) => {
         maxPrice: r.max_price ? parseFloat(r.max_price) : null,
         productCount: parseInt(r.product_count, 10),
       })),
+      overall: {
+        avgPrice: overall?.avg_price ? parseFloat(overall.avg_price) : null,
+        minPrice: overall?.min_price ? parseFloat(overall.min_price) : null,
+        maxPrice: overall?.max_price ? parseFloat(overall.max_price) : null,
+        totalProducts: parseInt(overall?.total_products || '0', 10),
+      },
     });
   } catch (error: any) {
     console.error('[Intelligence] Error fetching pricing:', error.message);
@@ -205,9 +291,23 @@ router.get('/pricing', async (req: Request, res: Response) => {
 /**
  * GET /api/admin/intelligence/stores
  * Get store intelligence summary
+ * Query params:
+ *   - state: Filter by state (e.g., "AZ")
+ *   - limit: Max results (default 200)
  */
 router.get('/stores', async (req: Request, res: Response) => {
   try {
+    const { state, limit = '200' } = req.query;
+    const limitNum = Math.min(parseInt(limit as string, 10), 500);
+
+    // Build WHERE clause based on state filter
+    let stateFilter = '';
+    const params: any[] = [limitNum];
+    if (state && state !== 'all') {
+      stateFilter = 'AND d.state = $2';
+      params.push(state);
+    }
+
     const { rows: storeRows } = await pool.query(`
       SELECT
         d.id,
@@ -217,17 +317,22 @@ router.get('/stores', async (req: Request, res: Response) => {
         d.state,
         d.menu_type,
         d.crawl_enabled,
-        COUNT(DISTINCT sp.id) as product_count,
+        c.name as chain_name,
+        COUNT(DISTINCT sp.id) as sku_count,
         COUNT(DISTINCT sp.brand_name_raw) as brand_count,
         ROUND(AVG(sp.price_rec)::numeric, 2) as avg_price,
-        MAX(sp.updated_at) as last_product_update
+        MAX(sp.updated_at) as last_crawl,
+        (SELECT COUNT(*) FROM store_product_snapshots sps
+         WHERE sps.store_product_id IN (SELECT id FROM store_products WHERE dispensary_id = d.id)) as snapshot_count
       FROM dispensaries d
       LEFT JOIN store_products sp ON sp.dispensary_id = d.id
-      WHERE d.state IS NOT NULL
+      LEFT JOIN chains c ON d.chain_id = c.id
-      GROUP BY d.id, d.name, d.dba_name, d.city, d.state, d.menu_type, d.crawl_enabled
+      WHERE d.state IS NOT NULL AND d.crawl_enabled = true
-      ORDER BY product_count DESC
+        ${stateFilter}
-      LIMIT 200
+      GROUP BY d.id, d.name, d.dba_name, d.city, d.state, d.menu_type, d.crawl_enabled, c.name
-    `);
+      ORDER BY sku_count DESC
+      LIMIT $1
+    `, params);
 
     res.json({
       stores: storeRows.map((r: any) => ({
@@ -238,10 +343,13 @@ router.get('/stores', async (req: Request, res: Response) => {
         state: r.state,
         menuType: r.menu_type,
         crawlEnabled: r.crawl_enabled,
-        productCount: parseInt(r.product_count || '0', 10),
+        chainName: r.chain_name || null,
+        skuCount: parseInt(r.sku_count || '0', 10),
+        snapshotCount: parseInt(r.snapshot_count || '0', 10),
         brandCount: parseInt(r.brand_count || '0', 10),
         avgPrice: r.avg_price ? parseFloat(r.avg_price) : null,
-        lastProductUpdate: r.last_product_update,
+        lastCrawl: r.last_crawl,
+        crawlFrequencyHours: 4, // Default crawl frequency
       })),
       total: storeRows.length,
     });

backend/src/routes/job-queue.ts

@@ -543,6 +543,9 @@ router.post('/bulk-priority', async (req: Request, res: Response) => {
 
 /**
  * POST /api/job-queue/enqueue - Add a new job to the queue
+ *
+ * 2024-12-10: Rewired to use worker_tasks via taskService.
+ * Legacy dispensary_crawl_jobs code commented out below.
  */
 router.post('/enqueue', async (req: Request, res: Response) => {
   try {
@@ -552,6 +555,59 @@ router.post('/enqueue', async (req: Request, res: Response) => {
       return res.status(400).json({ success: false, error: 'dispensary_id is required' });
     }
 
+    // 2024-12-10: Map legacy job_type to new task role
+    const roleMap: Record<string, string> = {
+      'dutchie_product_crawl': 'product_refresh',
+      'menu_detection': 'entry_point_discovery',
+      'menu_detection_single': 'entry_point_discovery',
+      'product_discovery': 'product_discovery',
+      'store_discovery': 'store_discovery',
+    };
+    const role = roleMap[job_type] || 'product_refresh';
+
+    // 2024-12-10: Use taskService to create task in worker_tasks table
+    const { taskService } = await import('../tasks/task-service');
+
+    // Check if task already pending for this dispensary
+    const existingTasks = await taskService.listTasks({
+      dispensary_id,
+      role: role as any,
+      status: ['pending', 'claimed', 'running'],
+      limit: 1,
+    });
+
+    if (existingTasks.length > 0) {
+      return res.json({
+        success: true,
+        task_id: existingTasks[0].id,
+        message: 'Task already queued'
+      });
+    }
+
+    const task = await taskService.createTask({
+      role: role as any,
+      dispensary_id,
+      priority,
+    });
+
+    res.json({ success: true, task_id: task.id, message: 'Task enqueued' });
+  } catch (error: any) {
+    console.error('[JobQueue] Error enqueuing task:', error);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/*
+ * LEGACY CODE - 2024-12-10: Commented out, was using orphaned dispensary_crawl_jobs table
+ *
+router.post('/enqueue', async (req: Request, res: Response) => {
+  try {
+    const { dispensary_id, job_type = 'dutchie_product_crawl', priority = 0 } = req.body;
+
+    if (!dispensary_id) {
+      return res.status(400).json({ success: false, error: 'dispensary_id is required' });
+    }
+
     // Check if job already pending for this dispensary
     const existing = await pool.query(`
       SELECT id FROM dispensary_crawl_jobs
@@ -585,6 +641,7 @@ router.post('/enqueue', async (req: Request, res: Response) => {
     res.status(500).json({ success: false, error: error.message });
   }
 });
+*/
 
 /**
  * POST /api/job-queue/pause - Pause queue processing
@@ -612,6 +669,8 @@ router.get('/paused', async (_req: Request, res: Response) => {
 /**
  * POST /api/job-queue/enqueue-batch - Queue multiple dispensaries at once
  * Body: { dispensary_ids: number[], job_type?: string, priority?: number }
+ *
+ * 2024-12-10: Rewired to use worker_tasks via taskService.
  */
 router.post('/enqueue-batch', async (req: Request, res: Response) => {
   try {
@@ -625,35 +684,30 @@ router.post('/enqueue-batch', async (req: Request, res: Response) => {
       return res.status(400).json({ success: false, error: 'Maximum 500 dispensaries per batch' });
     }
 
-    // Insert jobs, skipping duplicates
+    // 2024-12-10: Map legacy job_type to new task role
-    const { rows } = await pool.query(`
+    const roleMap: Record<string, string> = {
-      INSERT INTO dispensary_crawl_jobs (dispensary_id, job_type, priority, trigger_type, status, created_at)
+      'dutchie_product_crawl': 'product_refresh',
-      SELECT
+      'menu_detection': 'entry_point_discovery',
-        d.id,
+      'product_discovery': 'product_discovery',
-        $2::text,
+    };
-        $3::integer,
+    const role = roleMap[job_type] || 'product_refresh';
-        'api_batch',
+
-        'pending',
+    // 2024-12-10: Use taskService to create tasks in worker_tasks table
-        NOW()
+    const { taskService } = await import('../tasks/task-service');
-      FROM dispensaries d
+
-      WHERE d.id = ANY($1::int[])
+    const tasks = dispensary_ids.map(dispensary_id => ({
-        AND d.crawl_enabled = true
+      role: role as any,
-        AND d.platform_dispensary_id IS NOT NULL
+      dispensary_id,
-        AND NOT EXISTS (
+      priority,
-          SELECT 1 FROM dispensary_crawl_jobs cj
+    }));
-          WHERE cj.dispensary_id = d.id
+
-          AND cj.job_type = $2::text
+    const createdCount = await taskService.createTasks(tasks);
-          AND cj.status IN ('pending', 'running')
-        )
-      RETURNING id, dispensary_id
-    `, [dispensary_ids, job_type, priority]);
 
     res.json({
       success: true,
-      queued: rows.length,
+      queued: createdCount,
       requested: dispensary_ids.length,
-      job_ids: rows.map(r => r.id),
+      message: `Queued ${createdCount} of ${dispensary_ids.length} dispensaries`
-      message: `Queued ${rows.length} of ${dispensary_ids.length} dispensaries`
     });
   } catch (error: any) {
     console.error('[JobQueue] Error batch enqueuing:', error);
@@ -664,6 +718,8 @@ router.post('/enqueue-batch', async (req: Request, res: Response) => {
 /**
  * POST /api/job-queue/enqueue-state - Queue all crawl-enabled dispensaries for a state
  * Body: { state_code: string, job_type?: string, priority?: number, limit?: number }
+ *
+ * 2024-12-10: Rewired to use worker_tasks via taskService.
  */
 router.post('/enqueue-state', async (req: Request, res: Response) => {
   try {
@@ -673,52 +729,55 @@ router.post('/enqueue-state', async (req: Request, res: Response) => {
       return res.status(400).json({ success: false, error: 'state_code is required (e.g., "AZ")' });
     }
 
-    // Get state_id and queue jobs
+    // 2024-12-10: Map legacy job_type to new task role
-    const { rows } = await pool.query(`
+    const roleMap: Record<string, string> = {
-      WITH target_state AS (
+      'dutchie_product_crawl': 'product_refresh',
-        SELECT id FROM states WHERE code = $1
+      'menu_detection': 'entry_point_discovery',
-      )
+      'product_discovery': 'product_discovery',
-      INSERT INTO dispensary_crawl_jobs (dispensary_id, job_type, priority, trigger_type, status, created_at)
+    };
-      SELECT
+    const role = roleMap[job_type] || 'product_refresh';
-        d.id,
+
-        $2::text,
+    // Get dispensary IDs for the state
-        $3::integer,
+    const dispensaryResult = await pool.query(`
-        'api_state',
+      SELECT d.id
-        'pending',
+      FROM dispensaries d
-        NOW()
+      JOIN states s ON s.id = d.state_id
-      FROM dispensaries d, target_state
+      WHERE s.code = $1
-      WHERE d.state_id = target_state.id
         AND d.crawl_enabled = true
         AND d.platform_dispensary_id IS NOT NULL
-        AND NOT EXISTS (
+      LIMIT $2
-          SELECT 1 FROM dispensary_crawl_jobs cj
+    `, [state_code.toUpperCase(), limit]);
-          WHERE cj.dispensary_id = d.id
+
-          AND cj.job_type = $2::text
+    const dispensary_ids = dispensaryResult.rows.map((r: any) => r.id);
-          AND cj.status IN ('pending', 'running')
+
-        )
+    // 2024-12-10: Use taskService to create tasks in worker_tasks table
-      LIMIT $4::integer
+    const { taskService } = await import('../tasks/task-service');
-      RETURNING id, dispensary_id
+
-    `, [state_code.toUpperCase(), job_type, priority, limit]);
+    const tasks = dispensary_ids.map((dispensary_id: number) => ({
+      role: role as any,
+      dispensary_id,
+      priority,
+    }));
+
+    const createdCount = await taskService.createTasks(tasks);
 
     // Get total available count
     const countResult = await pool.query(`
-      WITH target_state AS (
-        SELECT id FROM states WHERE code = $1
-      )
       SELECT COUNT(*) as total
-      FROM dispensaries d, target_state
+      FROM dispensaries d
-      WHERE d.state_id = target_state.id
+      JOIN states s ON s.id = d.state_id
+      WHERE s.code = $1
         AND d.crawl_enabled = true
         AND d.platform_dispensary_id IS NOT NULL
     `, [state_code.toUpperCase()]);
 
     res.json({
       success: true,
-      queued: rows.length,
+      queued: createdCount,
       total_available: parseInt(countResult.rows[0].total),
       state: state_code.toUpperCase(),
-      job_type,
+      role,
-      message: `Queued ${rows.length} dispensaries for ${state_code.toUpperCase()}`
+      message: `Queued ${createdCount} dispensaries for ${state_code.toUpperCase()}`
     });
   } catch (error: any) {
     console.error('[JobQueue] Error enqueuing state:', error);

backend/src/routes/k8s.ts

@@ -0,0 +1,140 @@
+/**
+ * Kubernetes Control Routes
+ *
+ * Provides admin UI control over k8s resources like worker scaling.
+ * Uses in-cluster config when running in k8s, or kubeconfig locally.
+ */
+
+import { Router, Request, Response } from 'express';
+import * as k8s from '@kubernetes/client-node';
+
+const router = Router();
+
+// K8s client setup - lazy initialization
+let appsApi: k8s.AppsV1Api | null = null;
+let k8sError: string | null = null;
+
+function getK8sClient(): k8s.AppsV1Api | null {
+  if (appsApi) return appsApi;
+  if (k8sError) return null;
+
+  try {
+    const kc = new k8s.KubeConfig();
+
+    // Try in-cluster config first (when running in k8s)
+    try {
+      kc.loadFromCluster();
+      console.log('[K8s] Loaded in-cluster config');
+    } catch {
+      // Fall back to default kubeconfig (local dev)
+      try {
+        kc.loadFromDefault();
+        console.log('[K8s] Loaded default kubeconfig');
+      } catch (e) {
+        k8sError = 'No k8s config available';
+        console.log('[K8s] No config available - k8s routes disabled');
+        return null;
+      }
+    }
+
+    appsApi = kc.makeApiClient(k8s.AppsV1Api);
+    return appsApi;
+  } catch (e: any) {
+    k8sError = e.message;
+    console.error('[K8s] Failed to initialize client:', e.message);
+    return null;
+  }
+}
+
+const NAMESPACE = process.env.K8S_NAMESPACE || 'dispensary-scraper';
+const WORKER_DEPLOYMENT = 'scraper-worker';
+
+/**
+ * GET /api/k8s/workers
+ * Get current worker deployment status
+ */
+router.get('/workers', async (_req: Request, res: Response) => {
+  const client = getK8sClient();
+
+  if (!client) {
+    return res.json({
+      success: true,
+      available: false,
+      error: k8sError || 'K8s not available',
+      replicas: 0,
+      readyReplicas: 0,
+    });
+  }
+
+  try {
+    const deployment = await client.readNamespacedDeployment({
+      name: WORKER_DEPLOYMENT,
+      namespace: NAMESPACE,
+    });
+
+    res.json({
+      success: true,
+      available: true,
+      replicas: deployment.spec?.replicas || 0,
+      readyReplicas: deployment.status?.readyReplicas || 0,
+      availableReplicas: deployment.status?.availableReplicas || 0,
+      updatedReplicas: deployment.status?.updatedReplicas || 0,
+    });
+  } catch (e: any) {
+    console.error('[K8s] Error getting deployment:', e.message);
+    res.status(500).json({
+      success: false,
+      error: e.message,
+    });
+  }
+});
+
+/**
+ * POST /api/k8s/workers/scale
+ * Scale worker deployment
+ * Body: { replicas: number }
+ */
+router.post('/workers/scale', async (req: Request, res: Response) => {
+  const client = getK8sClient();
+
+  if (!client) {
+    return res.status(503).json({
+      success: false,
+      error: k8sError || 'K8s not available',
+    });
+  }
+
+  const { replicas } = req.body;
+
+  if (typeof replicas !== 'number' || replicas < 0 || replicas > 50) {
+    return res.status(400).json({
+      success: false,
+      error: 'replicas must be a number between 0 and 50',
+    });
+  }
+
+  try {
+    // Patch the deployment to set replicas
+    await client.patchNamespacedDeploymentScale({
+      name: WORKER_DEPLOYMENT,
+      namespace: NAMESPACE,
+      body: { spec: { replicas } },
+    });
+
+    console.log(`[K8s] Scaled ${WORKER_DEPLOYMENT} to ${replicas} replicas`);
+
+    res.json({
+      success: true,
+      replicas,
+      message: `Scaled to ${replicas} workers`,
+    });
+  } catch (e: any) {
+    console.error('[K8s] Error scaling deployment:', e.message);
+    res.status(500).json({
+      success: false,
+      error: e.message,
+    });
+  }
+});
+
+export default router;

backend/src/routes/markets.ts

@@ -291,6 +291,107 @@ router.get('/stores/:id/summary', async (req: Request, res: Response) => {
   }
 });
 
+/**
+ * GET /api/markets/stores/:id/crawl-history
+ * Get crawl history for a specific store
+ */
+router.get('/stores/:id/crawl-history', async (req: Request, res: Response) => {
+  try {
+    const { id } = req.params;
+    const { limit = '50' } = req.query;
+    const dispensaryId = parseInt(id, 10);
+    const limitNum = Math.min(parseInt(limit as string, 10), 100);
+
+    // Get crawl history from crawl_orchestration_traces
+    const { rows: historyRows } = await pool.query(`
+      SELECT
+        id,
+        run_id,
+        profile_key,
+        crawler_module,
+        state_at_start,
+        state_at_end,
+        total_steps,
+        duration_ms,
+        success,
+        error_message,
+        products_found,
+        started_at,
+        completed_at
+      FROM crawl_orchestration_traces
+      WHERE dispensary_id = $1
+      ORDER BY started_at DESC
+      LIMIT $2
+    `, [dispensaryId, limitNum]);
+
+    // Get next scheduled crawl if available
+    const { rows: scheduleRows } = await pool.query(`
+      SELECT
+        js.id as schedule_id,
+        js.job_name,
+        js.enabled,
+        js.base_interval_minutes,
+        js.jitter_minutes,
+        js.next_run_at,
+        js.last_run_at,
+        js.last_status
+      FROM job_schedules js
+      WHERE js.enabled = true
+        AND js.job_config->>'dispensaryId' = $1::text
+      ORDER BY js.next_run_at
+      LIMIT 1
+    `, [dispensaryId.toString()]);
+
+    // Get dispensary info for slug
+    const { rows: dispRows } = await pool.query(`
+      SELECT
+        id,
+        name,
+        dba_name,
+        slug,
+        state,
+        city,
+        menu_type,
+        platform_dispensary_id,
+        last_menu_scrape
+      FROM dispensaries
+      WHERE id = $1
+    `, [dispensaryId]);
+
+    res.json({
+      dispensary: dispRows[0] || null,
+      history: historyRows.map(row => ({
+        id: row.id,
+        runId: row.run_id,
+        profileKey: row.profile_key,
+        crawlerModule: row.crawler_module,
+        stateAtStart: row.state_at_start,
+        stateAtEnd: row.state_at_end,
+        totalSteps: row.total_steps,
+        durationMs: row.duration_ms,
+        success: row.success,
+        errorMessage: row.error_message,
+        productsFound: row.products_found,
+        startedAt: row.started_at?.toISOString() || null,
+        completedAt: row.completed_at?.toISOString() || null,
+      })),
+      nextSchedule: scheduleRows[0] ? {
+        scheduleId: scheduleRows[0].schedule_id,
+        jobName: scheduleRows[0].job_name,
+        enabled: scheduleRows[0].enabled,
+        baseIntervalMinutes: scheduleRows[0].base_interval_minutes,
+        jitterMinutes: scheduleRows[0].jitter_minutes,
+        nextRunAt: scheduleRows[0].next_run_at?.toISOString() || null,
+        lastRunAt: scheduleRows[0].last_run_at?.toISOString() || null,
+        lastStatus: scheduleRows[0].last_status,
+      } : null,
+    });
+  } catch (error: any) {
+    console.error('[Markets] Error fetching crawl history:', error.message);
+    res.status(500).json({ error: error.message });
+  }
+});
+
 /**
  * GET /api/markets/stores/:id/products
  * Get products for a store with filtering and pagination

backend/src/routes/orchestrator-admin.ts

@@ -78,14 +78,14 @@ router.get('/metrics', async (_req: Request, res: Response) => {
 
 /**
  * GET /api/admin/orchestrator/states
- * Returns array of states with at least one known dispensary
+ * Returns array of states with at least one crawl-enabled dispensary
  */
 router.get('/states', async (_req: Request, res: Response) => {
   try {
     const { rows } = await pool.query(`
       SELECT DISTINCT state, COUNT(*) as store_count
       FROM dispensaries
-      WHERE state IS NOT NULL
+      WHERE state IS NOT NULL AND crawl_enabled = true
       GROUP BY state
       ORDER BY state
     `);

backend/src/routes/payloads.ts

@@ -0,0 +1,334 @@
+/**
+ * Payload Routes
+ *
+ * Per TASK_WORKFLOW_2024-12-10.md: API access to raw crawl payloads.
+ *
+ * Endpoints:
+ * - GET /api/payloads - List payload metadata (paginated)
+ * - GET /api/payloads/:id - Get payload metadata by ID
+ * - GET /api/payloads/:id/data - Get full payload JSON
+ * - GET /api/payloads/store/:dispensaryId - List payloads for a store
+ * - GET /api/payloads/store/:dispensaryId/latest - Get latest payload for a store
+ * - GET /api/payloads/store/:dispensaryId/diff - Diff two payloads
+ */
+
+import { Router, Request, Response } from 'express';
+import { getPool } from '../db/pool';
+import {
+  loadRawPayloadById,
+  getLatestPayload,
+  getRecentPayloads,
+  listPayloadMetadata,
+} from '../utils/payload-storage';
+import { Pool } from 'pg';
+
+const router = Router();
+
+// Get pool instance for queries
+const getDbPool = (): Pool => getPool() as unknown as Pool;
+
+/**
+ * GET /api/payloads
+ * List payload metadata (paginated)
+ */
+router.get('/', async (req: Request, res: Response) => {
+  try {
+    const pool = getDbPool();
+    const limit = Math.min(parseInt(req.query.limit as string) || 50, 100);
+    const offset = parseInt(req.query.offset as string) || 0;
+    const dispensaryId = req.query.dispensary_id ? parseInt(req.query.dispensary_id as string) : undefined;
+
+    const payloads = await listPayloadMetadata(pool, {
+      dispensaryId,
+      limit,
+      offset,
+    });
+
+    res.json({
+      success: true,
+      payloads,
+      pagination: { limit, offset },
+    });
+  } catch (error: any) {
+    console.error('[Payloads] List error:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/payloads/:id
+ * Get payload metadata by ID
+ */
+router.get('/:id', async (req: Request, res: Response) => {
+  try {
+    const pool = getDbPool();
+    const id = parseInt(req.params.id);
+
+    const result = await pool.query(`
+      SELECT
+        p.id,
+        p.dispensary_id,
+        p.crawl_run_id,
+        p.storage_path,
+        p.product_count,
+        p.size_bytes,
+        p.size_bytes_raw,
+        p.fetched_at,
+        p.processed_at,
+        p.checksum_sha256,
+        d.name as dispensary_name
+      FROM raw_crawl_payloads p
+      LEFT JOIN dispensaries d ON d.id = p.dispensary_id
+      WHERE p.id = $1
+    `, [id]);
+
+    if (result.rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Payload not found' });
+    }
+
+    res.json({
+      success: true,
+      payload: result.rows[0],
+    });
+  } catch (error: any) {
+    console.error('[Payloads] Get error:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/payloads/:id/data
+ * Get full payload JSON (decompressed from disk)
+ */
+router.get('/:id/data', async (req: Request, res: Response) => {
+  try {
+    const pool = getDbPool();
+    const id = parseInt(req.params.id);
+
+    const result = await loadRawPayloadById(pool, id);
+
+    if (!result) {
+      return res.status(404).json({ success: false, error: 'Payload not found' });
+    }
+
+    res.json({
+      success: true,
+      metadata: result.metadata,
+      data: result.payload,
+    });
+  } catch (error: any) {
+    console.error('[Payloads] Get data error:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/payloads/store/:dispensaryId
+ * List payloads for a specific store
+ */
+router.get('/store/:dispensaryId', async (req: Request, res: Response) => {
+  try {
+    const pool = getDbPool();
+    const dispensaryId = parseInt(req.params.dispensaryId);
+    const limit = Math.min(parseInt(req.query.limit as string) || 20, 100);
+    const offset = parseInt(req.query.offset as string) || 0;
+
+    const payloads = await listPayloadMetadata(pool, {
+      dispensaryId,
+      limit,
+      offset,
+    });
+
+    res.json({
+      success: true,
+      dispensaryId,
+      payloads,
+      pagination: { limit, offset },
+    });
+  } catch (error: any) {
+    console.error('[Payloads] Store list error:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/payloads/store/:dispensaryId/latest
+ * Get the latest payload for a store (with full data)
+ */
+router.get('/store/:dispensaryId/latest', async (req: Request, res: Response) => {
+  try {
+    const pool = getDbPool();
+    const dispensaryId = parseInt(req.params.dispensaryId);
+
+    const result = await getLatestPayload(pool, dispensaryId);
+
+    if (!result) {
+      return res.status(404).json({
+        success: false,
+        error: `No payloads found for dispensary ${dispensaryId}`,
+      });
+    }
+
+    res.json({
+      success: true,
+      metadata: result.metadata,
+      data: result.payload,
+    });
+  } catch (error: any) {
+    console.error('[Payloads] Latest error:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/payloads/store/:dispensaryId/diff
+ * Compare two payloads for a store
+ *
+ * Query params:
+ * - from: payload ID (older)
+ * - to: payload ID (newer) - optional, defaults to latest
+ */
+router.get('/store/:dispensaryId/diff', async (req: Request, res: Response) => {
+  try {
+    const pool = getDbPool();
+    const dispensaryId = parseInt(req.params.dispensaryId);
+    const fromId = req.query.from ? parseInt(req.query.from as string) : undefined;
+    const toId = req.query.to ? parseInt(req.query.to as string) : undefined;
+
+    let fromPayload: any;
+    let toPayload: any;
+
+    if (fromId && toId) {
+      // Load specific payloads
+      const [from, to] = await Promise.all([
+        loadRawPayloadById(pool, fromId),
+        loadRawPayloadById(pool, toId),
+      ]);
+      fromPayload = from;
+      toPayload = to;
+    } else {
+      // Load two most recent
+      const recent = await getRecentPayloads(pool, dispensaryId, 2);
+      if (recent.length < 2) {
+        return res.status(400).json({
+          success: false,
+          error: 'Need at least 2 payloads to diff. Only found ' + recent.length,
+        });
+      }
+      toPayload = recent[0];   // Most recent
+      fromPayload = recent[1]; // Previous
+    }
+
+    if (!fromPayload || !toPayload) {
+      return res.status(404).json({ success: false, error: 'One or both payloads not found' });
+    }
+
+    // Build product maps by ID
+    const fromProducts = new Map<string, any>();
+    const toProducts = new Map<string, any>();
+
+    for (const p of fromPayload.payload.products || []) {
+      const id = p._id || p.id;
+      if (id) fromProducts.set(id, p);
+    }
+
+    for (const p of toPayload.payload.products || []) {
+      const id = p._id || p.id;
+      if (id) toProducts.set(id, p);
+    }
+
+    // Find differences
+    const added: any[] = [];
+    const removed: any[] = [];
+    const priceChanges: any[] = [];
+    const stockChanges: any[] = [];
+
+    // Products in "to" but not in "from" = added
+    for (const [id, product] of toProducts) {
+      if (!fromProducts.has(id)) {
+        added.push({
+          id,
+          name: product.name,
+          brand: product.brand?.name,
+          price: product.Prices?.[0]?.price,
+        });
+      }
+    }
+
+    // Products in "from" but not in "to" = removed
+    for (const [id, product] of fromProducts) {
+      if (!toProducts.has(id)) {
+        removed.push({
+          id,
+          name: product.name,
+          brand: product.brand?.name,
+          price: product.Prices?.[0]?.price,
+        });
+      }
+    }
+
+    // Products in both - check for changes
+    for (const [id, toProduct] of toProducts) {
+      const fromProduct = fromProducts.get(id);
+      if (!fromProduct) continue;
+
+      const fromPrice = fromProduct.Prices?.[0]?.price;
+      const toPrice = toProduct.Prices?.[0]?.price;
+
+      if (fromPrice !== toPrice) {
+        priceChanges.push({
+          id,
+          name: toProduct.name,
+          brand: toProduct.brand?.name,
+          oldPrice: fromPrice,
+          newPrice: toPrice,
+          change: toPrice && fromPrice ? toPrice - fromPrice : null,
+        });
+      }
+
+      const fromStock = fromProduct.Status || fromProduct.status;
+      const toStock = toProduct.Status || toProduct.status;
+
+      if (fromStock !== toStock) {
+        stockChanges.push({
+          id,
+          name: toProduct.name,
+          brand: toProduct.brand?.name,
+          oldStatus: fromStock,
+          newStatus: toStock,
+        });
+      }
+    }
+
+    res.json({
+      success: true,
+      from: {
+        id: fromPayload.metadata.id,
+        fetchedAt: fromPayload.metadata.fetchedAt,
+        productCount: fromPayload.metadata.productCount,
+      },
+      to: {
+        id: toPayload.metadata.id,
+        fetchedAt: toPayload.metadata.fetchedAt,
+        productCount: toPayload.metadata.productCount,
+      },
+      diff: {
+        added: added.length,
+        removed: removed.length,
+        priceChanges: priceChanges.length,
+        stockChanges: stockChanges.length,
+      },
+      details: {
+        added,
+        removed,
+        priceChanges,
+        stockChanges,
+      },
+    });
+  } catch (error: any) {
+    console.error('[Payloads] Diff error:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+export default router;

backend/src/routes/products.ts

@@ -1,11 +1,29 @@
 import { Router } from 'express';
 import { authMiddleware } from '../auth/middleware';
 import { pool } from '../db/pool';
-import { getImageUrl } from '../utils/minio';
 
 const router = Router();
 router.use(authMiddleware);
 
+/**
+ * Convert local image path to proxy URL
+ * /images/products/... -> /img/products/...
+ */
+function getImageUrl(localPath: string): string {
+  if (!localPath) return '';
+  // If already a full URL, return as-is
+  if (localPath.startsWith('http')) return localPath;
+  // Convert /images/ path to /img/ proxy path
+  if (localPath.startsWith('/images/')) {
+    return '/img' + localPath.substring(7);
+  }
+  // Handle paths without leading slash
+  if (localPath.startsWith('images/')) {
+    return '/img/' + localPath.substring(7);
+  }
+  return '/img/' + localPath;
+}
+
 // Freshness threshold: data older than this is considered stale
 const STALE_THRESHOLD_HOURS = 4;
 

backend/src/routes/proxies.ts

@@ -2,7 +2,7 @@ import { Router } from 'express';
 import { authMiddleware, requireRole } from '../auth/middleware';
 import { pool } from '../db/pool';
 import { testProxy, addProxy, addProxiesFromList } from '../services/proxy';
-import { createProxyTestJob, getProxyTestJob, getActiveProxyTestJob, cancelProxyTestJob } from '../services/proxyTestQueue';
+import { createProxyTestJob, getProxyTestJob, getActiveProxyTestJob, cancelProxyTestJob, ProxyTestMode } from '../services/proxyTestQueue';
 
 const router = Router();
 router.use(authMiddleware);
@@ -11,9 +11,10 @@ router.use(authMiddleware);
 router.get('/', async (req, res) => {
   try {
     const result = await pool.query(`
-      SELECT id, host, port, protocol, active, is_anonymous,
+      SELECT id, host, port, protocol, username, password, active, is_anonymous,
              last_tested_at, test_result, response_time_ms, created_at,
-             city, state, country, country_code, location_updated_at
+             city, state, country, country_code, location_updated_at,
+             COALESCE(max_connections, 1) as max_connections
       FROM proxies
       ORDER BY created_at DESC
     `);
@@ -166,13 +167,39 @@ router.post('/:id/test', requireRole('superadmin', 'admin'), async (req, res) =>
 });
 
 // Start proxy test job
+// Query params: mode=all|failed|inactive, concurrency=10
 router.post('/test-all', requireRole('superadmin', 'admin'), async (req, res) => {
   try {
-    const jobId = await createProxyTestJob();
+    const mode = (req.query.mode as ProxyTestMode) || 'all';
-    res.json({ jobId, message: 'Proxy test job started' });
+    const concurrency = parseInt(req.query.concurrency as string) || 10;
-  } catch (error) {
+
+    // Validate mode
+    if (!['all', 'failed', 'inactive'].includes(mode)) {
+      return res.status(400).json({ error: 'Invalid mode. Use: all, failed, or inactive' });
+    }
+
+    // Validate concurrency (1-50)
+    if (concurrency < 1 || concurrency > 50) {
+      return res.status(400).json({ error: 'Concurrency must be between 1 and 50' });
+    }
+
+    const { jobId, totalProxies } = await createProxyTestJob(mode, concurrency);
+    res.json({ jobId, total: totalProxies, mode, concurrency, message: `Proxy test job started (mode: ${mode}, concurrency: ${concurrency})` });
+  } catch (error: any) {
     console.error('Error starting proxy test job:', error);
-    res.status(500).json({ error: 'Failed to start proxy test job' });
+    res.status(500).json({ error: error.message || 'Failed to start proxy test job' });
+  }
+});
+
+// Convenience endpoint: Test only failed proxies
+router.post('/test-failed', requireRole('superadmin', 'admin'), async (req, res) => {
+  try {
+    const concurrency = parseInt(req.query.concurrency as string) || 10;
+    const { jobId, totalProxies } = await createProxyTestJob('failed', concurrency);
+    res.json({ jobId, total: totalProxies, mode: 'failed', concurrency, message: 'Retesting failed proxies...' });
+  } catch (error: any) {
+    console.error('Error starting failed proxy test:', error);
+    res.status(500).json({ error: error.message || 'Failed to start proxy test job' });
   }
 });
 
@@ -197,7 +224,7 @@ router.post('/test-job/:jobId/cancel', requireRole('superadmin', 'admin'), async
 router.put('/:id', requireRole('superadmin', 'admin'), async (req, res) => {
   try {
     const { id } = req.params;
-    const { host, port, protocol, username, password, active } = req.body;
+    const { host, port, protocol, username, password, active, max_connections } = req.body;
 
     const result = await pool.query(`
       UPDATE proxies
@@ -207,10 +234,11 @@ router.put('/:id', requireRole('superadmin', 'admin'), async (req, res) => {
           username = COALESCE($4, username),
           password = COALESCE($5, password),
           active = COALESCE($6, active),
+          max_connections = COALESCE($7, max_connections),
           updated_at = CURRENT_TIMESTAMP
-      WHERE id = $7
+      WHERE id = $8
       RETURNING *
-    `, [host, port, protocol, username, password, active, id]);
+    `, [host, port, protocol, username, password, active, max_connections, id]);
     
     if (result.rows.length === 0) {
       return res.status(404).json({ error: 'Proxy not found' });

backend/src/routes/public-api.ts

@@ -130,6 +130,12 @@ const CONSUMER_TRUSTED_ORIGINS = [
   'http://localhost:3002',
 ];
 
+// Wildcard trusted origin patterns (*.domain.com)
+const CONSUMER_TRUSTED_PATTERNS = [
+  /^https:\/\/([a-z0-9-]+\.)?cannaiq\.co$/,
+  /^https:\/\/([a-z0-9-]+\.)?cannabrands\.app$/,
+];
+
 // Trusted IPs for local development (bypass API key auth)
 const TRUSTED_IPS = ['127.0.0.1', '::1', '::ffff:127.0.0.1'];
 
@@ -150,9 +156,18 @@ function isConsumerTrustedRequest(req: Request): boolean {
     return true;
   }
   const origin = req.headers.origin;
-  if (origin && CONSUMER_TRUSTED_ORIGINS.includes(origin)) {
+  if (origin) {
+    // Check exact matches
+    if (CONSUMER_TRUSTED_ORIGINS.includes(origin)) {
       return true;
     }
+    // Check wildcard patterns
+    for (const pattern of CONSUMER_TRUSTED_PATTERNS) {
+      if (pattern.test(origin)) {
+        return true;
+      }
+    }
+  }
   const referer = req.headers.referer;
   if (referer) {
     for (const trusted of CONSUMER_TRUSTED_ORIGINS) {
@@ -160,6 +175,18 @@ function isConsumerTrustedRequest(req: Request): boolean {
         return true;
       }
     }
+    // Check wildcard patterns against referer origin
+    try {
+      const refererUrl = new URL(referer);
+      const refererOrigin = refererUrl.origin;
+      for (const pattern of CONSUMER_TRUSTED_PATTERNS) {
+        if (pattern.test(refererOrigin)) {
+          return true;
+        }
+      }
+    } catch {
+      // Invalid referer URL, ignore
+    }
   }
   return false;
 }
@@ -463,7 +490,7 @@ router.get('/products', async (req: PublicApiRequest, res: Response) => {
 
     // Filter by on special
     if (on_special === 'true' || on_special === '1') {
-      whereClause += ` AND s.is_on_special = TRUE`;
+      whereClause += ` AND s.special = TRUE`;
     }
 
     // Search by name or brand
@@ -547,7 +574,7 @@ router.get('/products', async (req: PublicApiRequest, res: Response) => {
     const { rows: countRows } = await pool.query(`
       SELECT COUNT(*) as total FROM store_products p
       LEFT JOIN LATERAL (
-        SELECT rec_min_price_cents / 100.0 as price_rec, med_min_price_cents / 100.0 as price_med, special as is_on_special FROM v_product_snapshots
+        SELECT rec_min_price_cents / 100.0 as price_rec, med_min_price_cents / 100.0 as price_med, special FROM v_product_snapshots
         WHERE store_product_id = p.id
         ORDER BY crawled_at DESC
         LIMIT 1
@@ -1125,6 +1152,7 @@ router.get('/dispensaries', async (req: PublicApiRequest, res: Response) => {
       SELECT
         d.id,
         d.name,
+        d.slug,
         d.address1,
         d.address2,
         d.city,
@@ -1179,6 +1207,7 @@ router.get('/dispensaries', async (req: PublicApiRequest, res: Response) => {
     const transformedDispensaries = dispensaries.map((d) => ({
       id: d.id,
       name: d.name,
+      slug: d.slug || null,
       address1: d.address1,
       address2: d.address2,
       city: d.city,
@@ -1876,7 +1905,7 @@ router.get('/stats', async (req: PublicApiRequest, res: Response) => {
       SELECT
         (SELECT COUNT(*) FROM store_products) as product_count,
         (SELECT COUNT(DISTINCT brand_name_raw) FROM store_products WHERE brand_name_raw IS NOT NULL) as brand_count,
-        (SELECT COUNT(*) FROM dispensaries WHERE crawl_enabled = true AND product_count > 0) as dispensary_count
+        (SELECT COUNT(DISTINCT dispensary_id) FROM store_products) as dispensary_count
     `);
 
     const s = stats[0] || {};
@@ -1996,4 +2025,235 @@ router.get('/menu', async (req: PublicApiRequest, res: Response) => {
   }
 });
 
+// ============================================================
+// VISITOR TRACKING & GEOLOCATION
+// ============================================================
+
+import crypto from 'crypto';
+import { GeoLocation, lookupIP } from '../services/ip2location';
+
+/**
+ * Get location from IP using local IP2Location database
+ */
+function getLocationFromIP(ip: string): GeoLocation | null {
+  return lookupIP(ip);
+}
+
+/**
+ * Hash IP for privacy (we don't store raw IPs)
+ */
+function hashIP(ip: string): string {
+  return crypto.createHash('sha256').update(ip).digest('hex').substring(0, 16);
+}
+
+/**
+ * POST /api/v1/visitor/track
+ * Track visitor location for analytics
+ *
+ * Body:
+ *   - domain: string (required) - 'findagram.co', 'findadispo.com', etc.
+ *   - page_path: string (optional) - current page path
+ *   - session_id: string (optional) - client-generated session ID
+ *   - referrer: string (optional) - document.referrer
+ *
+ * Returns:
+ *   - location: { city, state, lat, lng } for client use
+ */
+router.post('/visitor/track', async (req: Request, res: Response) => {
+  try {
+    const { domain, page_path, session_id, referrer } = req.body;
+
+    if (!domain) {
+      return res.status(400).json({ error: 'domain is required' });
+    }
+
+    // Get client IP
+    const clientIp = (req.headers['x-forwarded-for'] as string)?.split(',')[0].trim() ||
+                     req.headers['x-real-ip'] as string ||
+                     req.ip ||
+                     req.socket.remoteAddress ||
+                     '';
+
+    // Get location from IP (local database lookup)
+    const location = getLocationFromIP(clientIp);
+
+    // Store visit (with hashed IP for privacy)
+    await pool.query(`
+      INSERT INTO visitor_locations (
+        ip_hash, city, state, state_code, country, country_code,
+        latitude, longitude, domain, page_path, referrer, user_agent, session_id
+      ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+    `, [
+      hashIP(clientIp),
+      location?.city || null,
+      location?.state || null,
+      location?.stateCode || null,
+      location?.country || null,
+      location?.countryCode || null,
+      location?.lat || null,
+      location?.lng || null,
+      domain,
+      page_path || null,
+      referrer || null,
+      req.headers['user-agent'] || null,
+      session_id || null
+    ]);
+
+    // Return location to client (for nearby dispensary feature)
+    res.json({
+      success: true,
+      location: location ? {
+        city: location.city,
+        state: location.state,
+        stateCode: location.stateCode,
+        lat: location.lat,
+        lng: location.lng
+      } : null
+    });
+  } catch (error: any) {
+    console.error('Visitor tracking error:', error);
+    // Don't fail the request - tracking is non-critical
+    res.json({
+      success: false,
+      location: null
+    });
+  }
+});
+
+/**
+ * GET /api/v1/visitor/location
+ * Get visitor location without tracking (just IP lookup)
+ */
+router.get('/visitor/location', (req: Request, res: Response) => {
+  try {
+    const clientIp = (req.headers['x-forwarded-for'] as string)?.split(',')[0].trim() ||
+                     req.headers['x-real-ip'] as string ||
+                     req.ip ||
+                     req.socket.remoteAddress ||
+                     '';
+
+    const location = getLocationFromIP(clientIp);
+
+    res.json({
+      success: true,
+      location: location ? {
+        city: location.city,
+        state: location.state,
+        stateCode: location.stateCode,
+        lat: location.lat,
+        lng: location.lng
+      } : null
+    });
+  } catch (error: any) {
+    console.error('Location lookup error:', error);
+    res.json({
+      success: false,
+      location: null
+    });
+  }
+});
+
+/**
+ * GET /api/v1/analytics/visitors
+ * Get visitor analytics (admin only - requires auth)
+ *
+ * Query params:
+ *   - domain: filter by domain
+ *   - days: number of days to look back (default: 30)
+ *   - limit: max results (default: 50)
+ */
+router.get('/analytics/visitors', async (req: PublicApiRequest, res: Response) => {
+  try {
+    const scope = req.scope;
+
+    // Only allow internal keys
+    if (!scope || scope.type !== 'internal') {
+      return res.status(403).json({ error: 'Access denied - internal key required' });
+    }
+
+    const { domain, days = '30', limit = '50' } = req.query;
+    const daysNum = Math.min(parseInt(days as string, 10) || 30, 90);
+    const limitNum = Math.min(parseInt(limit as string, 10) || 50, 200);
+
+    let whereClause = 'WHERE created_at > NOW() - $1::interval';
+    const params: any[] = [`${daysNum} days`];
+    let paramIndex = 2;
+
+    if (domain) {
+      whereClause += ` AND domain = $${paramIndex}`;
+      params.push(domain);
+      paramIndex++;
+    }
+
+    // Get top locations
+    const { rows: topLocations } = await pool.query(`
+      SELECT
+        city,
+        state,
+        state_code,
+        country_code,
+        COUNT(*) as visit_count,
+        COUNT(DISTINCT session_id) as unique_sessions,
+        MAX(created_at) as last_visit
+      FROM visitor_locations
+      ${whereClause}
+      GROUP BY city, state, state_code, country_code
+      ORDER BY visit_count DESC
+      LIMIT $${paramIndex}
+    `, [...params, limitNum]);
+
+    // Get daily totals
+    const { rows: dailyStats } = await pool.query(`
+      SELECT
+        DATE(created_at) as date,
+        COUNT(*) as visits,
+        COUNT(DISTINCT session_id) as unique_sessions
+      FROM visitor_locations
+      ${whereClause}
+      GROUP BY DATE(created_at)
+      ORDER BY date DESC
+      LIMIT 30
+    `, params);
+
+    // Get totals
+    const { rows: totals } = await pool.query(`
+      SELECT
+        COUNT(*) as total_visits,
+        COUNT(DISTINCT session_id) as total_sessions,
+        COUNT(DISTINCT city || state_code) as unique_locations
+      FROM visitor_locations
+      ${whereClause}
+    `, params);
+
+    res.json({
+      success: true,
+      period: {
+        days: daysNum,
+        domain: domain || 'all'
+      },
+      totals: totals[0],
+      top_locations: topLocations.map(l => ({
+        city: l.city,
+        state: l.state,
+        state_code: l.state_code,
+        country_code: l.country_code,
+        visits: parseInt(l.visit_count, 10),
+        unique_sessions: parseInt(l.unique_sessions, 10),
+        last_visit: l.last_visit
+      })),
+      daily_stats: dailyStats.map(d => ({
+        date: d.date,
+        visits: parseInt(d.visits, 10),
+        unique_sessions: parseInt(d.unique_sessions, 10)
+      }))
+    });
+  } catch (error: any) {
+    console.error('Visitor analytics error:', error);
+    res.status(500).json({
+      error: 'Failed to fetch visitor analytics',
+      message: error.message
+    });
+  }
+});
+
 export default router;

backend/src/routes/tasks.ts

@@ -13,6 +13,12 @@ import {
   TaskFilter,
 } from '../tasks/task-service';
 import { pool } from '../db/pool';
+import {
+  isTaskPoolPaused,
+  pauseTaskPool,
+  resumeTaskPool,
+  getTaskPoolStatus,
+} from '../tasks/task-pool-state';
 
 const router = Router();
 
@@ -145,6 +151,36 @@ router.get('/:id', async (req: Request, res: Response) => {
   }
 });
 
+/**
+ * DELETE /api/tasks/:id
+ * Delete a specific task by ID
+ * Only allows deletion of failed, completed, or pending tasks (not running)
+ */
+router.delete('/:id', async (req: Request, res: Response) => {
+  try {
+    const taskId = parseInt(req.params.id, 10);
+
+    // First check if task exists and its status
+    const task = await taskService.getTask(taskId);
+    if (!task) {
+      return res.status(404).json({ error: 'Task not found' });
+    }
+
+    // Don't allow deleting running tasks
+    if (task.status === 'running' || task.status === 'claimed') {
+      return res.status(400).json({ error: 'Cannot delete a running or claimed task' });
+    }
+
+    // Delete the task
+    await pool.query('DELETE FROM worker_tasks WHERE id = $1', [taskId]);
+
+    res.json({ success: true, message: `Task ${taskId} deleted` });
+  } catch (error: unknown) {
+    console.error('Error deleting task:', error);
+    res.status(500).json({ error: 'Failed to delete task' });
+  }
+});
+
 /**
  * POST /api/tasks
  * Create a new task
@@ -444,7 +480,7 @@ router.post('/migration/cancel-pending-crawl-jobs', async (_req: Request, res: R
 
 /**
  * POST /api/tasks/migration/create-resync-tasks
- * Create product_resync tasks for all crawl-enabled dispensaries
+ * Create product_refresh tasks for all crawl-enabled dispensaries
  */
 router.post('/migration/create-resync-tasks', async (req: Request, res: Response) => {
   try {
@@ -474,7 +510,7 @@ router.post('/migration/create-resync-tasks', async (req: Request, res: Response
       const hasActive = await taskService.hasActiveTask(disp.id);
       if (!hasActive) {
         await taskService.createTask({
-          role: 'product_resync',
+          role: 'product_refresh',
           dispensary_id: disp.id,
           platform: 'dutchie',
           priority,
@@ -562,4 +598,42 @@ router.post('/migration/full-migrate', async (req: Request, res: Response) => {
   }
 });
 
+/**
+ * GET /api/tasks/pool/status
+ * Check if task pool is paused
+ */
+router.get('/pool/status', async (_req: Request, res: Response) => {
+  const status = getTaskPoolStatus();
+  res.json({
+    success: true,
+    ...status,
+  });
+});
+
+/**
+ * POST /api/tasks/pool/pause
+ * Pause the task pool - workers won't pick up new tasks
+ */
+router.post('/pool/pause', async (_req: Request, res: Response) => {
+  pauseTaskPool();
+  res.json({
+    success: true,
+    paused: true,
+    message: 'Task pool paused - workers will not pick up new tasks',
+  });
+});
+
+/**
+ * POST /api/tasks/pool/resume
+ * Resume the task pool - workers will pick up tasks again
+ */
+router.post('/pool/resume', async (_req: Request, res: Response) => {
+  resumeTaskPool();
+  res.json({
+    success: true,
+    paused: false,
+    message: 'Task pool resumed - workers will pick up new tasks',
+  });
+});
+
 export default router;

backend/src/routes/users.ts

@@ -14,23 +14,36 @@ router.get('/', async (req: AuthRequest, res) => {
   try {
     const { search, domain } = req.query;
 
-    let query = `
+    // Check which columns exist (schema-tolerant)
-      SELECT id, email, role, first_name, last_name, phone, domain, created_at, updated_at
+    const columnsResult = await pool.query(`
-      FROM users
+      SELECT column_name FROM information_schema.columns
-      WHERE 1=1
+      WHERE table_name = 'users' AND column_name IN ('first_name', 'last_name', 'phone', 'domain')
-    `;
+    `);
+    const existingColumns = new Set(columnsResult.rows.map((r: any) => r.column_name));
+
+    // Build column list based on what exists
+    const selectCols = ['id', 'email', 'role', 'created_at', 'updated_at'];
+    if (existingColumns.has('first_name')) selectCols.push('first_name');
+    if (existingColumns.has('last_name')) selectCols.push('last_name');
+    if (existingColumns.has('phone')) selectCols.push('phone');
+    if (existingColumns.has('domain')) selectCols.push('domain');
+
+    let query = `SELECT ${selectCols.join(', ')} FROM users WHERE 1=1`;
     const params: any[] = [];
     let paramIndex = 1;
 
-    // Search by email, first_name, or last_name
+    // Search by email (and optionally first_name, last_name if they exist)
     if (search && typeof search === 'string') {
-      query += ` AND (email ILIKE $${paramIndex} OR first_name ILIKE $${paramIndex} OR last_name ILIKE $${paramIndex})`;
+      const searchClauses = ['email ILIKE $' + paramIndex];
+      if (existingColumns.has('first_name')) searchClauses.push('first_name ILIKE $' + paramIndex);
+      if (existingColumns.has('last_name')) searchClauses.push('last_name ILIKE $' + paramIndex);
+      query += ` AND (${searchClauses.join(' OR ')})`;
       params.push(`%${search}%`);
       paramIndex++;
     }
 
-    // Filter by domain
+    // Filter by domain (if column exists)
-    if (domain && typeof domain === 'string') {
+    if (domain && typeof domain === 'string' && existingColumns.has('domain')) {
       query += ` AND domain = $${paramIndex}`;
       params.push(domain);
       paramIndex++;
@@ -50,8 +63,22 @@ router.get('/', async (req: AuthRequest, res) => {
 router.get('/:id', async (req: AuthRequest, res) => {
   try {
     const { id } = req.params;
+
+    // Check which columns exist (schema-tolerant)
+    const columnsResult = await pool.query(`
+      SELECT column_name FROM information_schema.columns
+      WHERE table_name = 'users' AND column_name IN ('first_name', 'last_name', 'phone', 'domain')
+    `);
+    const existingColumns = new Set(columnsResult.rows.map((r: any) => r.column_name));
+
+    const selectCols = ['id', 'email', 'role', 'created_at', 'updated_at'];
+    if (existingColumns.has('first_name')) selectCols.push('first_name');
+    if (existingColumns.has('last_name')) selectCols.push('last_name');
+    if (existingColumns.has('phone')) selectCols.push('phone');
+    if (existingColumns.has('domain')) selectCols.push('domain');
+
     const result = await pool.query(`
-      SELECT id, email, role, first_name, last_name, phone, domain, created_at, updated_at
+      SELECT ${selectCols.join(', ')}
       FROM users
       WHERE id = $1
     `, [id]);

backend/src/routes/worker-registry.ts

@@ -0,0 +1,861 @@
+/**
+ * Worker Registry API Routes
+ *
+ * Dynamic worker management - workers register on startup, get assigned names,
+ * and report heartbeats. Everything is API-driven, no hardcoding.
+ *
+ * Endpoints:
+ *   POST /api/worker-registry/register      - Worker reports for duty
+ *   POST /api/worker-registry/heartbeat     - Worker heartbeat
+ *   POST /api/worker-registry/deregister    - Worker signing off
+ *   GET  /api/worker-registry/workers       - List all workers (for dashboard)
+ *   GET  /api/worker-registry/workers/:id   - Get specific worker
+ *   POST /api/worker-registry/cleanup       - Mark stale workers offline
+ *
+ *   GET  /api/worker-registry/names         - List all names in pool
+ *   POST /api/worker-registry/names         - Add names to pool
+ *   DELETE /api/worker-registry/names/:name - Remove name from pool
+ *
+ *   GET  /api/worker-registry/roles         - List available task roles
+ *   POST /api/worker-registry/roles         - Add a new role (future)
+ */
+
+import { Router, Request, Response } from 'express';
+import { pool } from '../db/pool';
+import os from 'os';
+
+const router = Router();
+
+// ============================================================
+// WORKER REGISTRATION
+// ============================================================
+
+/**
+ * POST /api/worker-registry/register
+ * Worker reports for duty - gets assigned a friendly name
+ *
+ * Body:
+ *   - role: string (optional) - task role, or null for role-agnostic workers
+ *   - worker_id: string (optional) - custom ID, auto-generated if not provided
+ *   - pod_name: string (optional) - k8s pod name
+ *   - hostname: string (optional) - machine hostname
+ *   - metadata: object (optional) - additional worker info
+ *
+ * Returns:
+ *   - worker_id: assigned worker ID
+ *   - friendly_name: assigned name from pool
+ *   - role: confirmed role (or null if agnostic)
+ *   - message: welcome message
+ */
+router.post('/register', async (req: Request, res: Response) => {
+  try {
+    const {
+      role = null,  // Role is now optional - null means agnostic
+      worker_id,
+      pod_name,
+      hostname,
+      ip_address,
+      metadata = {}
+    } = req.body;
+
+    // Generate worker_id if not provided
+    const finalWorkerId = worker_id || `worker-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const finalHostname = hostname || os.hostname();
+    const clientIp = ip_address || req.ip || req.socket.remoteAddress;
+
+    // Check if worker already registered
+    const existing = await pool.query(
+      'SELECT id, friendly_name, status FROM worker_registry WHERE worker_id = $1',
+      [finalWorkerId]
+    );
+
+    if (existing.rows.length > 0) {
+      // Re-activate existing worker - keep existing pod_name (fantasy name), don't overwrite with K8s name
+      const { rows } = await pool.query(`
+        UPDATE worker_registry
+        SET status = 'active',
+            role = $1,
+            hostname = $2,
+            ip_address = $3,
+            last_heartbeat_at = NOW(),
+            started_at = NOW(),
+            metadata = $4,
+            updated_at = NOW()
+        WHERE worker_id = $5
+        RETURNING id, worker_id, friendly_name, pod_name, role
+      `, [role, finalHostname, clientIp, metadata, finalWorkerId]);
+
+      const worker = rows[0];
+      const roleMsg = role ? `for ${role}` : 'as role-agnostic';
+      console.log(`[WorkerRegistry] Worker "${worker.friendly_name}" (${finalWorkerId}) re-registered ${roleMsg}`);
+
+      return res.json({
+        success: true,
+        worker_id: worker.worker_id,
+        friendly_name: worker.friendly_name,
+        role: worker.role,
+        message: role
+          ? `Welcome back, ${worker.friendly_name}! You are assigned to ${role}.`
+          : `Welcome back, ${worker.friendly_name}! You are ready to take any task.`
+      });
+    }
+
+    // Assign a friendly name
+    const nameResult = await pool.query('SELECT assign_worker_name($1) as name', [finalWorkerId]);
+    const friendlyName = nameResult.rows[0].name;
+
+    // Register the worker - use friendlyName as pod_name (not K8s name)
+    const { rows } = await pool.query(`
+      INSERT INTO worker_registry (
+        worker_id, friendly_name, role, pod_name, hostname, ip_address, status, metadata
+      ) VALUES ($1, $2, $3, $4, $5, $6, 'active', $7)
+      RETURNING id, worker_id, friendly_name, pod_name, role
+    `, [finalWorkerId, friendlyName, role, friendlyName, finalHostname, clientIp, metadata]);
+
+    const worker = rows[0];
+    const roleMsg = role ? `for ${role}` : 'as role-agnostic';
+    console.log(`[WorkerRegistry] New worker "${friendlyName}" (${finalWorkerId}) reporting for duty ${roleMsg}`);
+
+    res.json({
+      success: true,
+      worker_id: worker.worker_id,
+      friendly_name: worker.friendly_name,
+      role: worker.role,
+      message: role
+        ? `Hello ${friendlyName}! You are now registered for ${role}. Ready for work!`
+        : `Hello ${friendlyName}! You are ready to take any task from the pool.`
+    });
+  } catch (error: any) {
+    console.error('[WorkerRegistry] Registration error:', error);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/heartbeat
+ * Worker sends heartbeat to stay alive
+ *
+ * Body:
+ *   - worker_id: string (required)
+ *   - current_task_id: number (optional) - task currently being processed (primary task)
+ *   - current_task_ids: number[] (optional) - all tasks currently being processed (concurrent)
+ *   - active_task_count: number (optional) - number of tasks currently running
+ *   - max_concurrent_tasks: number (optional) - max concurrent tasks this worker can handle
+ *   - status: string (optional) - 'active', 'idle'
+ *   - resources: object (optional) - memory_mb, cpu_user_ms, cpu_system_ms, etc.
+ */
+router.post('/heartbeat', async (req: Request, res: Response) => {
+  try {
+    const {
+      worker_id,
+      current_task_id,
+      current_task_ids,
+      active_task_count,
+      max_concurrent_tasks,
+      status = 'active',
+      resources
+    } = req.body;
+
+    if (!worker_id) {
+      return res.status(400).json({ success: false, error: 'worker_id is required' });
+    }
+
+    // Build metadata object with all the new fields
+    const metadata: Record<string, unknown> = {};
+    if (resources) Object.assign(metadata, resources);
+    if (current_task_ids) metadata.current_task_ids = current_task_ids;
+    if (active_task_count !== undefined) metadata.active_task_count = active_task_count;
+    if (max_concurrent_tasks !== undefined) metadata.max_concurrent_tasks = max_concurrent_tasks;
+
+    // Store resources in metadata jsonb column
+    const { rows } = await pool.query(`
+      UPDATE worker_registry
+      SET last_heartbeat_at = NOW(),
+          current_task_id = $1,
+          status = $2,
+          metadata = COALESCE(metadata, '{}'::jsonb) || COALESCE($4::jsonb, '{}'::jsonb),
+          updated_at = NOW()
+      WHERE worker_id = $3
+      RETURNING id, friendly_name, status
+    `, [current_task_id || null, status, worker_id, Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null]);
+
+    if (rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found - please register first' });
+    }
+
+    res.json({
+      success: true,
+      worker: rows[0]
+    });
+  } catch (error: any) {
+    console.error('[WorkerRegistry] Heartbeat error:', error);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/task-completed
+ * Worker reports task completion
+ *
+ * Body:
+ *   - worker_id: string (required)
+ *   - success: boolean (required)
+ */
+router.post('/task-completed', async (req: Request, res: Response) => {
+  try {
+    const { worker_id, success } = req.body;
+
+    if (!worker_id) {
+      return res.status(400).json({ success: false, error: 'worker_id is required' });
+    }
+
+    const incrementField = success ? 'tasks_completed' : 'tasks_failed';
+
+    const { rows } = await pool.query(`
+      UPDATE worker_registry
+      SET ${incrementField} = ${incrementField} + 1,
+          last_task_at = NOW(),
+          current_task_id = NULL,
+          status = 'idle',
+          updated_at = NOW()
+      WHERE worker_id = $1
+      RETURNING id, friendly_name, tasks_completed, tasks_failed
+    `, [worker_id]);
+
+    if (rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found' });
+    }
+
+    res.json({ success: true, worker: rows[0] });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/deregister
+ * Worker signing off (graceful shutdown)
+ *
+ * Body:
+ *   - worker_id: string (required)
+ */
+router.post('/deregister', async (req: Request, res: Response) => {
+  try {
+    const { worker_id } = req.body;
+
+    if (!worker_id) {
+      return res.status(400).json({ success: false, error: 'worker_id is required' });
+    }
+
+    // Release the name back to the pool
+    await pool.query('SELECT release_worker_name($1)', [worker_id]);
+
+    // Mark as terminated
+    const { rows } = await pool.query(`
+      UPDATE worker_registry
+      SET status = 'terminated',
+          current_task_id = NULL,
+          updated_at = NOW()
+      WHERE worker_id = $1
+      RETURNING id, friendly_name
+    `, [worker_id]);
+
+    if (rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found' });
+    }
+
+    console.log(`[WorkerRegistry] Worker "${rows[0].friendly_name}" (${worker_id}) signed off`);
+
+    res.json({
+      success: true,
+      message: `Goodbye ${rows[0].friendly_name}! Thanks for your work.`
+    });
+  } catch (error: any) {
+    console.error('[WorkerRegistry] Deregister error:', error);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+// ============================================================
+// WORKER LISTING (for Dashboard)
+// ============================================================
+
+/**
+ * GET /api/worker-registry/workers
+ * List all workers (for dashboard)
+ *
+ * Query params:
+ *   - status: filter by status (active, idle, offline, all)
+ *   - role: filter by role
+ *   - include_terminated: include terminated workers (default: false)
+ */
+router.get('/workers', async (req: Request, res: Response) => {
+  try {
+    // Check if worker_registry table exists
+    const tableCheck = await pool.query(`
+      SELECT EXISTS (
+        SELECT FROM information_schema.tables
+        WHERE table_name = 'worker_registry'
+      ) as exists
+    `);
+
+    if (!tableCheck.rows[0].exists) {
+      // Return empty result if table doesn't exist yet
+      return res.json({
+        success: true,
+        workers: [],
+        summary: {
+          active_count: 0,
+          idle_count: 0,
+          offline_count: 0,
+          total_count: 0,
+          active_roles: 0
+        }
+      });
+    }
+
+    const { status, role, include_terminated = 'false' } = req.query;
+
+    let whereClause = include_terminated === 'true' ? 'WHERE 1=1' : "WHERE status != 'terminated'";
+    const params: any[] = [];
+    let paramIndex = 1;
+
+    if (status && status !== 'all') {
+      whereClause += ` AND status = $${paramIndex}`;
+      params.push(status);
+      paramIndex++;
+    }
+
+    if (role) {
+      whereClause += ` AND role = $${paramIndex}`;
+      params.push(role);
+      paramIndex++;
+    }
+
+    const { rows } = await pool.query(`
+      SELECT
+        id,
+        worker_id,
+        friendly_name,
+        role,
+        pod_name,
+        hostname,
+        ip_address,
+        status,
+        started_at,
+        last_heartbeat_at,
+        last_task_at,
+        tasks_completed,
+        tasks_failed,
+        current_task_id,
+        -- Concurrent task fields from metadata
+        (metadata->>'current_task_ids')::jsonb as current_task_ids,
+        (metadata->>'active_task_count')::int as active_task_count,
+        (metadata->>'max_concurrent_tasks')::int as max_concurrent_tasks,
+        -- Decommission fields
+        COALESCE(decommission_requested, false) as decommission_requested,
+        decommission_reason,
+        -- Full metadata for resources
+        metadata,
+        EXTRACT(EPOCH FROM (NOW() - last_heartbeat_at)) as seconds_since_heartbeat,
+        CASE
+          WHEN status = 'offline' OR status = 'terminated' THEN status
+          WHEN last_heartbeat_at < NOW() - INTERVAL '2 minutes' THEN 'stale'
+          WHEN current_task_id IS NOT NULL THEN 'busy'
+          WHEN (metadata->>'active_task_count')::int > 0 THEN 'busy'
+          ELSE 'ready'
+        END as health_status,
+        created_at
+      FROM worker_registry
+      ${whereClause}
+      ORDER BY
+        CASE status
+          WHEN 'active' THEN 1
+          WHEN 'idle' THEN 2
+          WHEN 'offline' THEN 3
+          ELSE 4
+        END,
+        last_heartbeat_at DESC
+    `, params);
+
+    // Get summary counts
+    const { rows: summary } = await pool.query(`
+      SELECT
+        COUNT(*) FILTER (WHERE status = 'active') as active_count,
+        COUNT(*) FILTER (WHERE status = 'idle') as idle_count,
+        COUNT(*) FILTER (WHERE status = 'offline') as offline_count,
+        COUNT(*) FILTER (WHERE status != 'terminated') as total_count,
+        COUNT(DISTINCT role) FILTER (WHERE status IN ('active', 'idle')) as active_roles
+      FROM worker_registry
+    `);
+
+    res.json({
+      success: true,
+      workers: rows,
+      summary: summary[0]
+    });
+  } catch (error: any) {
+    console.error('[WorkerRegistry] List workers error:', error);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/worker-registry/workers/:workerId
+ * Get specific worker details
+ */
+router.get('/workers/:workerId', async (req: Request, res: Response) => {
+  try {
+    const { workerId } = req.params;
+
+    const { rows } = await pool.query(`
+      SELECT * FROM worker_registry WHERE worker_id = $1
+    `, [workerId]);
+
+    if (rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found' });
+    }
+
+    res.json({ success: true, worker: rows[0] });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * DELETE /api/worker-registry/workers/:workerId
+ * Remove a worker (admin action)
+ */
+router.delete('/workers/:workerId', async (req: Request, res: Response) => {
+  try {
+    const { workerId } = req.params;
+
+    // Release name
+    await pool.query('SELECT release_worker_name($1)', [workerId]);
+
+    // Delete worker
+    const { rows } = await pool.query(`
+      DELETE FROM worker_registry WHERE worker_id = $1 RETURNING friendly_name
+    `, [workerId]);
+
+    if (rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found' });
+    }
+
+    res.json({ success: true, message: `Worker ${rows[0].friendly_name} removed` });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/cleanup
+ * Mark stale workers as offline
+ *
+ * Body:
+ *   - stale_threshold_minutes: number (default: 5)
+ */
+router.post('/cleanup', async (req: Request, res: Response) => {
+  try {
+    const { stale_threshold_minutes = 5 } = req.body;
+
+    const { rows } = await pool.query(
+      'SELECT mark_stale_workers($1) as count',
+      [stale_threshold_minutes]
+    );
+
+    res.json({
+      success: true,
+      stale_workers_marked: rows[0].count,
+      message: `Marked ${rows[0].count} stale workers as offline`
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+// ============================================================
+// NAME POOL MANAGEMENT
+// ============================================================
+
+/**
+ * GET /api/worker-registry/names
+ * List all names in the pool
+ */
+router.get('/names', async (_req: Request, res: Response) => {
+  try {
+    const { rows } = await pool.query(`
+      SELECT
+        id,
+        name,
+        in_use,
+        assigned_to,
+        assigned_at
+      FROM worker_name_pool
+      ORDER BY in_use DESC, name ASC
+    `);
+
+    const { rows: summary } = await pool.query(`
+      SELECT
+        COUNT(*) as total,
+        COUNT(*) FILTER (WHERE in_use = true) as in_use,
+        COUNT(*) FILTER (WHERE in_use = false) as available
+      FROM worker_name_pool
+    `);
+
+    res.json({
+      success: true,
+      names: rows,
+      summary: summary[0]
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/names
+ * Add names to the pool
+ *
+ * Body:
+ *   - names: string[] (required) - array of names to add
+ */
+router.post('/names', async (req: Request, res: Response) => {
+  try {
+    const { names } = req.body;
+
+    if (!names || !Array.isArray(names) || names.length === 0) {
+      return res.status(400).json({ success: false, error: 'names array is required' });
+    }
+
+    const values = names.map(n => `('${n.replace(/'/g, "''")}')`).join(', ');
+
+    const { rowCount } = await pool.query(`
+      INSERT INTO worker_name_pool (name)
+      VALUES ${values}
+      ON CONFLICT (name) DO NOTHING
+    `);
+
+    res.json({
+      success: true,
+      added: rowCount,
+      message: `Added ${rowCount} new names to the pool`
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * DELETE /api/worker-registry/names/:name
+ * Remove a name from the pool (only if not in use)
+ */
+router.delete('/names/:name', async (req: Request, res: Response) => {
+  try {
+    const { name } = req.params;
+
+    const { rows } = await pool.query(`
+      DELETE FROM worker_name_pool
+      WHERE name = $1 AND in_use = false
+      RETURNING name
+    `, [name]);
+
+    if (rows.length === 0) {
+      return res.status(400).json({
+        success: false,
+        error: 'Name not found or currently in use'
+      });
+    }
+
+    res.json({ success: true, message: `Name "${name}" removed from pool` });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+// ============================================================
+// ROLE MANAGEMENT
+// ============================================================
+
+/**
+ * GET /api/worker-registry/roles
+ * List available task roles
+ */
+router.get('/roles', async (_req: Request, res: Response) => {
+  // These are the roles the task handlers support
+  const roles = [
+    {
+      id: 'product_refresh',
+      name: 'Product Refresh',
+      description: 'Re-crawl dispensary products for price/stock changes',
+      handler: 'handleProductRefresh'
+    },
+    {
+      id: 'product_discovery',
+      name: 'Product Discovery',
+      description: 'Initial product discovery for new dispensaries',
+      handler: 'handleProductDiscovery'
+    },
+    {
+      id: 'store_discovery',
+      name: 'Store Discovery',
+      description: 'Discover new dispensary locations',
+      handler: 'handleStoreDiscovery'
+    },
+    {
+      id: 'entry_point_discovery',
+      name: 'Entry Point Discovery',
+      description: 'Resolve platform IDs from menu URLs',
+      handler: 'handleEntryPointDiscovery'
+    },
+    {
+      id: 'analytics_refresh',
+      name: 'Analytics Refresh',
+      description: 'Refresh materialized views and analytics',
+      handler: 'handleAnalyticsRefresh'
+    }
+  ];
+
+  // Get active worker counts per role
+  try {
+    const { rows } = await pool.query(`
+      SELECT role, COUNT(*) as worker_count
+      FROM worker_registry
+      WHERE status IN ('active', 'idle')
+      GROUP BY role
+    `);
+
+    const countMap = new Map(rows.map(r => [r.role, parseInt(r.worker_count)]));
+
+    const rolesWithCounts = roles.map(r => ({
+      ...r,
+      active_workers: countMap.get(r.id) || 0
+    }));
+
+    res.json({ success: true, roles: rolesWithCounts });
+  } catch {
+    // If table doesn't exist yet, just return roles without counts
+    res.json({ success: true, roles: roles.map(r => ({ ...r, active_workers: 0 })) });
+  }
+});
+
+/**
+ * GET /api/worker-registry/capacity
+ * Get capacity planning info
+ */
+router.get('/capacity', async (_req: Request, res: Response) => {
+  try {
+    // Get worker counts by role
+    const { rows: workerCounts } = await pool.query(`
+      SELECT role, COUNT(*) as count
+      FROM worker_registry
+      WHERE status IN ('active', 'idle')
+      GROUP BY role
+    `);
+
+    // Get pending task counts by role (if worker_tasks exists)
+    let taskCounts: any[] = [];
+    try {
+      const result = await pool.query(`
+        SELECT role, COUNT(*) as pending_count
+        FROM worker_tasks
+        WHERE status = 'pending'
+        GROUP BY role
+      `);
+      taskCounts = result.rows;
+    } catch {
+      // worker_tasks might not exist yet
+    }
+
+    // Get crawl-enabled store count
+    const storeCountResult = await pool.query(`
+      SELECT COUNT(*) as count
+      FROM dispensaries
+      WHERE crawl_enabled = true AND platform_dispensary_id IS NOT NULL
+    `);
+    const totalStores = parseInt(storeCountResult.rows[0].count);
+
+    const workerMap = new Map(workerCounts.map(r => [r.role, parseInt(r.count)]));
+    const taskMap = new Map(taskCounts.map(r => [r.role, parseInt(r.pending_count)]));
+
+    const roles = ['product_refresh', 'product_discovery', 'store_discovery', 'entry_point_discovery', 'analytics_refresh'];
+
+    const capacity = roles.map(role => ({
+      role,
+      active_workers: workerMap.get(role) || 0,
+      pending_tasks: taskMap.get(role) || 0,
+      // Rough estimate: 20 seconds per task, 4-hour cycle
+      tasks_per_worker_per_cycle: 720,
+      workers_needed_for_all_stores: Math.ceil(totalStores / 720)
+    }));
+
+    res.json({
+      success: true,
+      total_stores: totalStores,
+      capacity
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+// ============================================================
+// WORKER LIFECYCLE MANAGEMENT
+// ============================================================
+
+/**
+ * POST /api/worker-registry/workers/:workerId/decommission
+ * Request graceful decommission of a worker (will stop after current task)
+ */
+router.post('/workers/:workerId/decommission', async (req: Request, res: Response) => {
+  try {
+    const { workerId } = req.params;
+    const { reason, issued_by } = req.body;
+
+    // Update worker_registry to flag for decommission
+    const result = await pool.query(
+      `UPDATE worker_registry
+       SET decommission_requested = true,
+           decommission_reason = $2,
+           decommission_requested_at = NOW()
+       WHERE worker_id = $1
+       RETURNING friendly_name, status, current_task_id`,
+      [workerId, reason || 'Manual decommission from admin']
+    );
+
+    if (result.rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found' });
+    }
+
+    const worker = result.rows[0];
+
+    // Also log to worker_commands for audit trail
+    await pool.query(
+      `INSERT INTO worker_commands (worker_id, command, reason, issued_by)
+       VALUES ($1, 'decommission', $2, $3)
+       ON CONFLICT DO NOTHING`,
+      [workerId, reason || 'Manual decommission', issued_by || 'admin']
+    ).catch(() => {
+      // Table might not exist yet - ignore
+    });
+
+    res.json({
+      success: true,
+      message: worker.current_task_id
+        ? `Worker ${worker.friendly_name} will stop after completing task #${worker.current_task_id}`
+        : `Worker ${worker.friendly_name} will stop on next poll`,
+      worker: {
+        friendly_name: worker.friendly_name,
+        status: worker.status,
+        current_task_id: worker.current_task_id,
+        decommission_requested: true
+      }
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/workers/:workerId/cancel-decommission
+ * Cancel a pending decommission request
+ */
+router.post('/workers/:workerId/cancel-decommission', async (req: Request, res: Response) => {
+  try {
+    const { workerId } = req.params;
+
+    const result = await pool.query(
+      `UPDATE worker_registry
+       SET decommission_requested = false,
+           decommission_reason = NULL,
+           decommission_requested_at = NULL
+       WHERE worker_id = $1
+       RETURNING friendly_name`,
+      [workerId]
+    );
+
+    if (result.rows.length === 0) {
+      return res.status(404).json({ success: false, error: 'Worker not found' });
+    }
+
+    res.json({
+      success: true,
+      message: `Decommission cancelled for ${result.rows[0].friendly_name}`
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * POST /api/worker-registry/spawn
+ * Spawn a new worker in the current pod (only works in multi-worker-per-pod mode)
+ * For now, this is a placeholder - actual spawning requires the pod supervisor
+ */
+router.post('/spawn', async (req: Request, res: Response) => {
+  try {
+    const { pod_name, role } = req.body;
+
+    // For now, we can't actually spawn workers from the API
+    // This would require a supervisor process in each pod that listens for spawn commands
+    // Instead, return instructions for how to scale
+    res.json({
+      success: false,
+      error: 'Direct worker spawning not yet implemented',
+      instructions: 'To add workers, scale the K8s deployment: kubectl scale deployment/scraper-worker --replicas=N'
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+/**
+ * GET /api/worker-registry/pods
+ * Get workers grouped by pod
+ */
+router.get('/pods', async (_req: Request, res: Response) => {
+  try {
+    const { rows } = await pool.query(`
+      SELECT
+        COALESCE(pod_name, 'Unknown') as pod_name,
+        COUNT(*) as worker_count,
+        COUNT(*) FILTER (WHERE current_task_id IS NOT NULL) as busy_count,
+        COUNT(*) FILTER (WHERE current_task_id IS NULL) as idle_count,
+        SUM(tasks_completed) as total_completed,
+        SUM(tasks_failed) as total_failed,
+        SUM((metadata->>'memory_rss_mb')::int) as total_memory_mb,
+        array_agg(json_build_object(
+          'worker_id', worker_id,
+          'friendly_name', friendly_name,
+          'status', status,
+          'current_task_id', current_task_id,
+          'tasks_completed', tasks_completed,
+          'tasks_failed', tasks_failed,
+          'decommission_requested', COALESCE(decommission_requested, false),
+          'last_heartbeat_at', last_heartbeat_at
+        )) as workers
+      FROM worker_registry
+      WHERE status NOT IN ('offline', 'terminated')
+      GROUP BY pod_name
+      ORDER BY pod_name
+    `);
+
+    res.json({
+      success: true,
+      pods: rows.map(row => ({
+        pod_name: row.pod_name,
+        worker_count: parseInt(row.worker_count),
+        busy_count: parseInt(row.busy_count),
+        idle_count: parseInt(row.idle_count),
+        total_completed: parseInt(row.total_completed) || 0,
+        total_failed: parseInt(row.total_failed) || 0,
+        total_memory_mb: parseInt(row.total_memory_mb) || 0,
+        workers: row.workers
+      }))
+    });
+  } catch (error: any) {
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+export default router;

backend/src/routes/workers.ts

@@ -17,13 +17,234 @@
  *   GET /api/monitor/jobs              - Get recent job history
  *   GET /api/monitor/active-jobs       - Get currently running jobs
  *   GET /api/monitor/summary           - Get monitoring summary
+ *
+ * K8s Scaling (added 2024-12-10):
+ *   GET /api/workers/k8s/replicas      - Get current replica count
+ *   POST /api/workers/k8s/scale        - Scale worker replicas up/down
  */
 
 import { Router, Request, Response } from 'express';
 import { pool } from '../db/pool';
+import * as k8s from '@kubernetes/client-node';
 
 const router = Router();
 
+// ============================================================
+// K8S SCALING CONFIGURATION (added 2024-12-10)
+// Per TASK_WORKFLOW_2024-12-10.md: Admin can scale workers from UI
+// ============================================================
+
+const K8S_NAMESPACE = process.env.K8S_NAMESPACE || 'dispensary-scraper';
+const K8S_DEPLOYMENT_NAME = process.env.K8S_WORKER_DEPLOYMENT || 'scraper-worker';
+
+// Initialize K8s client - uses in-cluster config when running in K8s,
+// or kubeconfig when running locally
+let k8sAppsApi: k8s.AppsV1Api | null = null;
+
+function getK8sClient(): k8s.AppsV1Api | null {
+  if (k8sAppsApi) return k8sAppsApi;
+
+  try {
+    const kc = new k8s.KubeConfig();
+
+    // Try in-cluster config first (when running as a pod)
+    // Falls back to default kubeconfig (~/.kube/config) for local dev
+    try {
+      kc.loadFromCluster();
+    } catch {
+      kc.loadFromDefault();
+    }
+
+    k8sAppsApi = kc.makeApiClient(k8s.AppsV1Api);
+    return k8sAppsApi;
+  } catch (err: any) {
+    console.warn('[Workers] K8s client not available:', err.message);
+    return null;
+  }
+}
+
+// ============================================================
+// K8S SCALING ROUTES (added 2024-12-10)
+// Per TASK_WORKFLOW_2024-12-10.md: Admin can scale workers from UI
+// ============================================================
+
+/**
+ * GET /api/workers/k8s/replicas - Get current worker replica count
+ * Returns current and desired replica counts from the Deployment
+ */
+router.get('/k8s/replicas', async (_req: Request, res: Response) => {
+  const client = getK8sClient();
+
+  if (!client) {
+    return res.status(503).json({
+      success: false,
+      error: 'K8s client not available (not running in cluster or no kubeconfig)',
+      replicas: null,
+    });
+  }
+
+  try {
+    const response = await client.readNamespacedDeployment({
+      name: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+    });
+
+    const deployment = response;
+    res.json({
+      success: true,
+      replicas: {
+        current: deployment.status?.readyReplicas || 0,
+        desired: deployment.spec?.replicas || 0,
+        available: deployment.status?.availableReplicas || 0,
+        updated: deployment.status?.updatedReplicas || 0,
+      },
+      deployment: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+    });
+  } catch (err: any) {
+    console.error('[Workers] K8s replicas error:', err.body?.message || err.message);
+    res.status(500).json({
+      success: false,
+      error: err.body?.message || err.message,
+    });
+  }
+});
+
+/**
+ * POST /api/workers/k8s/scale - Scale worker replicas
+ * Body: { replicas: number } - desired replica count (0-20)
+ */
+router.post('/k8s/scale', async (req: Request, res: Response) => {
+  const client = getK8sClient();
+
+  if (!client) {
+    return res.status(503).json({
+      success: false,
+      error: 'K8s client not available (not running in cluster or no kubeconfig)',
+    });
+  }
+
+  const { replicas } = req.body;
+
+  // Validate replica count
+  if (typeof replicas !== 'number' || replicas < 0 || replicas > 20) {
+    return res.status(400).json({
+      success: false,
+      error: 'replicas must be a number between 0 and 20',
+    });
+  }
+
+  try {
+    // Get current state first
+    const currentResponse = await client.readNamespacedDeploymentScale({
+      name: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+    });
+    const currentReplicas = currentResponse.spec?.replicas || 0;
+
+    // Update scale using replaceNamespacedDeploymentScale
+    await client.replaceNamespacedDeploymentScale({
+      name: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+      body: {
+        apiVersion: 'autoscaling/v1',
+        kind: 'Scale',
+        metadata: {
+          name: K8S_DEPLOYMENT_NAME,
+          namespace: K8S_NAMESPACE,
+        },
+        spec: {
+          replicas: replicas,
+        },
+      },
+    });
+
+    console.log(`[Workers] Scaled ${K8S_DEPLOYMENT_NAME} from ${currentReplicas} to ${replicas} replicas`);
+
+    res.json({
+      success: true,
+      message: `Scaled from ${currentReplicas} to ${replicas} replicas`,
+      previous: currentReplicas,
+      desired: replicas,
+      deployment: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+    });
+  } catch (err: any) {
+    console.error('[Workers] K8s scale error:', err.body?.message || err.message);
+    res.status(500).json({
+      success: false,
+      error: err.body?.message || err.message,
+    });
+  }
+});
+
+/**
+ * POST /api/workers/k8s/scale-up - Scale up worker replicas by 1
+ * Convenience endpoint for adding a single worker
+ */
+router.post('/k8s/scale-up', async (_req: Request, res: Response) => {
+  const client = getK8sClient();
+
+  if (!client) {
+    return res.status(503).json({
+      success: false,
+      error: 'K8s client not available (not running in cluster or no kubeconfig)',
+    });
+  }
+
+  try {
+    // Get current replica count
+    const currentResponse = await client.readNamespacedDeploymentScale({
+      name: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+    });
+    const currentReplicas = currentResponse.spec?.replicas || 0;
+    const newReplicas = currentReplicas + 1;
+
+    // Cap at 20 replicas
+    if (newReplicas > 20) {
+      return res.status(400).json({
+        success: false,
+        error: 'Maximum replica count (20) reached',
+      });
+    }
+
+    // Scale up by 1
+    await client.replaceNamespacedDeploymentScale({
+      name: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+      body: {
+        apiVersion: 'autoscaling/v1',
+        kind: 'Scale',
+        metadata: {
+          name: K8S_DEPLOYMENT_NAME,
+          namespace: K8S_NAMESPACE,
+        },
+        spec: {
+          replicas: newReplicas,
+        },
+      },
+    });
+
+    console.log(`[Workers] Scaled up ${K8S_DEPLOYMENT_NAME} from ${currentReplicas} to ${newReplicas} replicas`);
+
+    res.json({
+      success: true,
+      message: `Added worker (${currentReplicas} → ${newReplicas} replicas)`,
+      previous: currentReplicas,
+      desired: newReplicas,
+      deployment: K8S_DEPLOYMENT_NAME,
+      namespace: K8S_NAMESPACE,
+    });
+  } catch (err: any) {
+    console.error('[Workers] K8s scale-up error:', err.body?.message || err.message);
+    res.status(500).json({
+      success: false,
+      error: err.body?.message || err.message,
+    });
+  }
+});
+
 // ============================================================
 // STATIC ROUTES (must come before parameterized routes)
 // ============================================================

backend/src/scripts/crawl-single-store.ts

@@ -16,10 +16,11 @@ import {
   executeGraphQL,
   startSession,
   endSession,
-  getFingerprint,
+  setCrawlRotator,
   GRAPHQL_HASHES,
   DUTCHIE_CONFIG,
 } from '../platforms/dutchie';
+import { CrawlRotator } from '../services/crawl-rotator';
 
 dotenv.config();
 
@@ -108,19 +109,27 @@ async function main() {
 
     // ============================================================
     // STEP 2: Start stealth session
+    // Per workflow-12102025.md: Initialize CrawlRotator and start session with menuUrl
     // ============================================================
     console.log('┌─────────────────────────────────────────────────────────────┐');
     console.log('│ STEP 2: Start Stealth Session                               │');
     console.log('└─────────────────────────────────────────────────────────────┘');
 
-    // Use Arizona timezone for this store
+    // Per workflow-12102025.md: Initialize CrawlRotator (required for sessions)
-    const session = startSession(disp.state || 'AZ', 'America/Phoenix');
+    const rotator = new CrawlRotator();
+    setCrawlRotator(rotator);
 
-    const fp = getFingerprint();
+    // Per workflow-12102025.md: startSession takes menuUrl for dynamic Referer
+    const session = startSession(disp.menu_url);
+
+    const fp = session.fingerprint;
     console.log(`  Session ID:         ${session.sessionId}`);
+    console.log(`  Browser:            ${fp.browserName} (${fp.deviceCategory})`);
     console.log(`  User-Agent:         ${fp.userAgent.slice(0, 60)}...`);
     console.log(`  Accept-Language:    ${fp.acceptLanguage}`);
-    console.log(`  Sec-CH-UA:          ${fp.secChUa || '(not set)'}`);
+    console.log(`  Referer:            ${session.referer}`);
+    console.log(`  DNT:                ${fp.httpFingerprint.hasDNT ? 'enabled' : 'disabled'}`);
+    console.log(`  TLS:                ${fp.httpFingerprint.curlImpersonateBinary}`);
     console.log('');
 
     // ============================================================

backend/src/scripts/test-stealth-session.ts

@@ -1,10 +1,10 @@
 /**
  * Test script for stealth session management
  *
- * Tests:
+ * Per workflow-12102025.md:
- * 1. Per-session fingerprint rotation
+ * - Tests HTTP fingerprinting (browser-specific headers + ordering)
- * 2. Geographic consistency (timezone → Accept-Language)
+ * - Tests UA generation (device distribution, browser filtering)
- * 3. Proxy location loading from database
+ * - Tests dynamic Referer per dispensary
  *
  * Usage:
  *   npx tsx src/scripts/test-stealth-session.ts
@@ -14,104 +14,142 @@ import {
   startSession,
   endSession,
   getCurrentSession,
-  getFingerprint,
-  getRandomFingerprint,
-  getLocaleForTimezone,
   buildHeaders,
+  setCrawlRotator,
 } from '../platforms/dutchie';
 
+import { CrawlRotator } from '../services/crawl-rotator';
+import {
+  generateHTTPFingerprint,
+  buildRefererFromMenuUrl,
+  BrowserType,
+} from '../services/http-fingerprint';
+
 console.log('='.repeat(60));
-console.log('STEALTH SESSION TEST');
+console.log('STEALTH SESSION TEST (per workflow-12102025.md)');
 console.log('='.repeat(60));
 
-// Test 1: Timezone to Locale mapping
+// Initialize CrawlRotator (required for sessions)
-console.log('\n[Test 1] Timezone to Locale Mapping:');
+console.log('\n[Setup] Initializing CrawlRotator...');
-const testTimezones = [
+const rotator = new CrawlRotator();
-  'America/Phoenix',
+setCrawlRotator(rotator);
-  'America/Los_Angeles',
+console.log('  CrawlRotator initialized');
-  'America/New_York',
+
-  'America/Chicago',
+// Test 1: HTTP Fingerprint Generation
+console.log('\n[Test 1] HTTP Fingerprint Generation:');
+const browsers: BrowserType[] = ['Chrome', 'Firefox', 'Safari', 'Edge'];
+
+for (const browser of browsers) {
+  const httpFp = generateHTTPFingerprint(browser);
+  console.log(`  ${browser}:`);
+  console.log(`    TLS binary: ${httpFp.curlImpersonateBinary}`);
+  console.log(`    DNT: ${httpFp.hasDNT ? 'enabled' : 'disabled'}`);
+  console.log(`    Header order: ${httpFp.headerOrder.slice(0, 5).join(', ')}...`);
+}
+
+// Test 2: Dynamic Referer from menu URLs
+console.log('\n[Test 2] Dynamic Referer from Menu URLs:');
+const testUrls = [
+  'https://dutchie.com/embedded-menu/harvest-of-tempe',
+  'https://dutchie.com/dispensary/zen-leaf-mesa',
+  '/embedded-menu/deeply-rooted',
+  '/dispensary/curaleaf-phoenix',
+  null,
   undefined,
-  'Invalid/Timezone',
 ];
 
-for (const tz of testTimezones) {
+for (const url of testUrls) {
-  const locale = getLocaleForTimezone(tz);
+  const referer = buildRefererFromMenuUrl(url);
-  console.log(`  ${tz || '(undefined)'} → ${locale}`);
+  console.log(`  ${url || '(null/undefined)'}`);
+  console.log(`    → ${referer}`);
 }
 
-// Test 2: Random fingerprint selection
+// Test 3: Session with Dynamic Referer
-console.log('\n[Test 2] Random Fingerprint Selection (5 samples):');
+console.log('\n[Test 3] Session with Dynamic Referer:');
-for (let i = 0; i < 5; i++) {
+const testMenuUrl = 'https://dutchie.com/dispensary/harvest-of-tempe';
-  const fp = getRandomFingerprint();
+console.log(`  Starting session with menuUrl: ${testMenuUrl}`);
-  console.log(`  ${i + 1}. ${fp.userAgent.slice(0, 60)}...`);
-}
 
-// Test 3: Session Management
+const session1 = startSession(testMenuUrl);
-console.log('\n[Test 3] Session Management:');
-
-// Before session - should use default fingerprint
-console.log('  Before session:');
-const beforeFp = getFingerprint();
-console.log(`    getFingerprint(): ${beforeFp.userAgent.slice(0, 50)}...`);
-console.log(`    getCurrentSession(): ${getCurrentSession()}`);
-
-// Start session with Arizona timezone
-console.log('\n  Starting session (AZ, America/Phoenix):');
-const session1 = startSession('AZ', 'America/Phoenix');
 console.log(`  Session ID: ${session1.sessionId}`);
-console.log(`    Fingerprint UA: ${session1.fingerprint.userAgent.slice(0, 50)}...`);
+console.log(`  Browser: ${session1.fingerprint.browserName}`);
-console.log(`    Accept-Language: ${session1.fingerprint.acceptLanguage}`);
+console.log(`  Device: ${session1.fingerprint.deviceCategory}`);
-console.log(`    Timezone: ${session1.timezone}`);
+console.log(`  Referer: ${session1.referer}`);
+console.log(`  DNT: ${session1.fingerprint.httpFingerprint.hasDNT ? 'enabled' : 'disabled'}`);
+console.log(`  TLS: ${session1.fingerprint.httpFingerprint.curlImpersonateBinary}`);
 
-// During session - should use session fingerprint
+// Test 4: Build Headers (browser-specific order)
-console.log('\n  During session:');
+console.log('\n[Test 4] Build Headers (browser-specific order):');
-const duringFp = getFingerprint();
+const { headers, orderedHeaders } = buildHeaders(true, 1000);
-console.log(`    getFingerprint(): ${duringFp.userAgent.slice(0, 50)}...`);
+console.log(`  Headers built for ${session1.fingerprint.browserName}:`);
-console.log(`    Same as session? ${duringFp.userAgent === session1.fingerprint.userAgent}`);
+console.log(`  Order: ${orderedHeaders.join(' → ')}`);
+console.log(`  Sample headers:`);
+console.log(`    User-Agent: ${headers['User-Agent']?.slice(0, 50)}...`);
+console.log(`    Accept: ${headers['Accept']}`);
+console.log(`    Accept-Language: ${headers['Accept-Language']}`);
+console.log(`    Referer: ${headers['Referer']}`);
+if (headers['sec-ch-ua']) {
+  console.log(`    sec-ch-ua: ${headers['sec-ch-ua']}`);
+}
+if (headers['DNT']) {
+  console.log(`    DNT: ${headers['DNT']}`);
+}
 
-// Test buildHeaders with session
-console.log('\n  buildHeaders() during session:');
-const headers = buildHeaders('/embedded-menu/test-store');
-console.log(`    User-Agent: ${headers['user-agent'].slice(0, 50)}...`);
-console.log(`    Accept-Language: ${headers['accept-language']}`);
-console.log(`    Origin: ${headers['origin']}`);
-console.log(`    Referer: ${headers['referer']}`);
-
-// End session
-console.log('\n  Ending session:');
 endSession();
-console.log(`    getCurrentSession(): ${getCurrentSession()}`);
 
-// Test 4: Multiple sessions should have different fingerprints
+// Test 5: Multiple Sessions (UA variety)
-console.log('\n[Test 4] Multiple Sessions (fingerprint variety):');
+console.log('\n[Test 5] Multiple Sessions (UA & fingerprint variety):');
-const fingerprints: string[] = [];
+const sessions: {
+  browser: string;
+  device: string;
+  hasDNT: boolean;
+}[] = [];
+
 for (let i = 0; i < 10; i++) {
-  const session = startSession('CA', 'America/Los_Angeles');
+  const session = startSession(`/dispensary/store-${i}`);
-  fingerprints.push(session.fingerprint.userAgent);
+  sessions.push({
+    browser: session.fingerprint.browserName,
+    device: session.fingerprint.deviceCategory,
+    hasDNT: session.fingerprint.httpFingerprint.hasDNT,
+  });
   endSession();
 }
 
-const uniqueCount = new Set(fingerprints).size;
+// Count distribution
-console.log(`  10 sessions created, ${uniqueCount} unique fingerprints`);
+const browserCounts: Record<string, number> = {};
-console.log(`  Variety: ${uniqueCount >= 3 ? '✅ Good' : '⚠️ Low - may need more fingerprint options'}`);
+const deviceCounts: Record<string, number> = {};
+let dntCount = 0;
 
-// Test 5: Geographic consistency check
+for (const s of sessions) {
-console.log('\n[Test 5] Geographic Consistency:');
+  browserCounts[s.browser] = (browserCounts[s.browser] || 0) + 1;
-const geoTests = [
+  deviceCounts[s.device] = (deviceCounts[s.device] || 0) + 1;
-  { state: 'AZ', tz: 'America/Phoenix' },
+  if (s.hasDNT) dntCount++;
-  { state: 'CA', tz: 'America/Los_Angeles' },
+}
-  { state: 'NY', tz: 'America/New_York' },
-  { state: 'IL', tz: 'America/Chicago' },
-];
 
-for (const { state, tz } of geoTests) {
+console.log(`  10 sessions created:`);
-  const session = startSession(state, tz);
+console.log(`    Browsers: ${JSON.stringify(browserCounts)}`);
-  const consistent = session.fingerprint.acceptLanguage.includes('en-US');
+console.log(`    Devices: ${JSON.stringify(deviceCounts)}`);
-  console.log(`  ${state} (${tz}): Accept-Language=${session.fingerprint.acceptLanguage} ${consistent ? '✅' : '❌'}`);
+console.log(`    DNT enabled: ${dntCount}/10 (expected ~30%)`);
+
+// Test 6: Device distribution check (per workflow-12102025.md: 62/36/2)
+console.log('\n[Test 6] Device Distribution (larger sample):');
+const deviceSamples: string[] = [];
+
+for (let i = 0; i < 100; i++) {
+  const session = startSession();
+  deviceSamples.push(session.fingerprint.deviceCategory);
   endSession();
 }
 
+const mobileCount = deviceSamples.filter(d => d === 'mobile').length;
+const desktopCount = deviceSamples.filter(d => d === 'desktop').length;
+const tabletCount = deviceSamples.filter(d => d === 'tablet').length;
+
+console.log(`  100 sessions (expected: 62% mobile, 36% desktop, 2% tablet):`);
+console.log(`    Mobile: ${mobileCount}%`);
+console.log(`    Desktop: ${desktopCount}%`);
+console.log(`    Tablet: ${tabletCount}%`);
+console.log(`    Distribution: ${Math.abs(mobileCount - 62) < 15 && Math.abs(desktopCount - 36) < 15 ? '✅ Reasonable' : '⚠️ Off target'}`);
+
 console.log('\n' + '='.repeat(60));
 console.log('TEST COMPLETE');
 console.log('='.repeat(60));

backend/src/services/analytics/BrandPenetrationService.ts

@@ -26,6 +26,8 @@ import {
   PenetrationDataPoint,
   BrandMarketPosition,
   BrandRecVsMedFootprint,
+  BrandPromotionalSummary,
+  BrandPromotionalEvent,
 } from './types';
 
 export class BrandPenetrationService {
@@ -44,16 +46,17 @@ export class BrandPenetrationService {
     // Get current brand presence
     const currentResult = await this.pool.query(`
       SELECT
-        sp.brand_name,
+        sp.brand_name_raw AS brand_name,
         COUNT(DISTINCT sp.dispensary_id) AS total_dispensaries,
         COUNT(*) AS total_skus,
         ROUND(COUNT(*)::NUMERIC / NULLIF(COUNT(DISTINCT sp.dispensary_id), 0), 2) AS avg_skus_per_dispensary,
         ARRAY_AGG(DISTINCT s.code) FILTER (WHERE s.code IS NOT NULL) AS states_present
       FROM store_products sp
-      LEFT JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
-      WHERE sp.brand_name = $1
+      LEFT JOIN states s ON s.id = d.state_id
+      WHERE sp.brand_name_raw = $1
         AND sp.is_in_stock = TRUE
-      GROUP BY sp.brand_name
+      GROUP BY sp.brand_name_raw
     `, [brandName]);
 
     if (currentResult.rows.length === 0) {
@@ -72,7 +75,7 @@ export class BrandPenetrationService {
           DATE(sps.captured_at) AS date,
           COUNT(DISTINCT sps.dispensary_id) AS dispensary_count
         FROM store_product_snapshots sps
-        WHERE sps.brand_name = $1
+        WHERE sps.brand_name_raw = $1
           AND sps.captured_at >= $2
           AND sps.captured_at <= $3
           AND sps.is_in_stock = TRUE
@@ -123,8 +126,9 @@ export class BrandPenetrationService {
           COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
           COUNT(*) AS sku_count
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
-        WHERE sp.brand_name = $1
+        JOIN states s ON s.id = d.state_id
+        WHERE sp.brand_name_raw = $1
           AND sp.is_in_stock = TRUE
         GROUP BY s.code, s.name, s.recreational_legal, s.medical_legal
       ),
@@ -133,7 +137,8 @@ export class BrandPenetrationService {
           s.code AS state_code,
           COUNT(DISTINCT sp.dispensary_id) AS total_dispensaries
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+        JOIN states s ON s.id = d.state_id
         WHERE sp.is_in_stock = TRUE
         GROUP BY s.code
       )
@@ -169,7 +174,7 @@ export class BrandPenetrationService {
     let filters = '';
 
     if (options.category) {
-      filters += ` AND sp.category = $${paramIdx}`;
+      filters += ` AND sp.category_raw = $${paramIdx}`;
       params.push(options.category);
       paramIdx++;
     }
@@ -183,31 +188,33 @@ export class BrandPenetrationService {
     const result = await this.pool.query(`
       WITH brand_metrics AS (
         SELECT
-          sp.brand_name,
+          sp.brand_name_raw AS brand_name,
-          sp.category,
+          sp.category_raw AS category,
           s.code AS state_code,
           COUNT(*) AS sku_count,
           COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
           AVG(sp.price_rec) AS avg_price
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
-        WHERE sp.brand_name = $1
+        JOIN states s ON s.id = d.state_id
+        WHERE sp.brand_name_raw = $1
           AND sp.is_in_stock = TRUE
-          AND sp.category IS NOT NULL
+          AND sp.category_raw IS NOT NULL
           ${filters}
-        GROUP BY sp.brand_name, sp.category, s.code
+        GROUP BY sp.brand_name_raw, sp.category_raw, s.code
       ),
       category_totals AS (
         SELECT
-          sp.category,
+          sp.category_raw AS category,
           s.code AS state_code,
           COUNT(*) AS total_skus,
           AVG(sp.price_rec) AS category_avg_price
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+        JOIN states s ON s.id = d.state_id
         WHERE sp.is_in_stock = TRUE
-          AND sp.category IS NOT NULL
+          AND sp.category_raw IS NOT NULL
-        GROUP BY sp.category, s.code
+        GROUP BY sp.category_raw, s.code
       )
       SELECT
         bm.*,
@@ -243,8 +250,9 @@ export class BrandPenetrationService {
           COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
           ROUND(COUNT(*)::NUMERIC / NULLIF(COUNT(DISTINCT sp.dispensary_id), 0), 2) AS avg_skus
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
-        WHERE sp.brand_name = $1
+        JOIN states s ON s.id = d.state_id
+        WHERE sp.brand_name_raw = $1
           AND sp.is_in_stock = TRUE
           AND s.recreational_legal = TRUE
       ),
@@ -255,8 +263,9 @@ export class BrandPenetrationService {
           COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
           ROUND(COUNT(*)::NUMERIC / NULLIF(COUNT(DISTINCT sp.dispensary_id), 0), 2) AS avg_skus
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
-        WHERE sp.brand_name = $1
+        JOIN states s ON s.id = d.state_id
+        WHERE sp.brand_name_raw = $1
           AND sp.is_in_stock = TRUE
           AND s.medical_legal = TRUE
           AND (s.recreational_legal = FALSE OR s.recreational_legal IS NULL)
@@ -311,23 +320,24 @@ export class BrandPenetrationService {
     }
 
     if (category) {
-      filters += ` AND sp.category = $${paramIdx}`;
+      filters += ` AND sp.category_raw = $${paramIdx}`;
       params.push(category);
       paramIdx++;
     }
 
     const result = await this.pool.query(`
       SELECT
-        sp.brand_name,
+        sp.brand_name_raw AS brand_name,
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
         COUNT(*) AS sku_count,
         COUNT(DISTINCT s.code) AS state_count
       FROM store_products sp
-      LEFT JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
-      WHERE sp.brand_name IS NOT NULL
+      LEFT JOIN states s ON s.id = d.state_id
+      WHERE sp.brand_name_raw IS NOT NULL
         AND sp.is_in_stock = TRUE
         ${filters}
-      GROUP BY sp.brand_name
+      GROUP BY sp.brand_name_raw
       ORDER BY dispensary_count DESC, sku_count DESC
       LIMIT $1
     `, params);
@@ -358,23 +368,23 @@ export class BrandPenetrationService {
     const result = await this.pool.query(`
       WITH start_counts AS (
         SELECT
-          brand_name,
+          brand_name_raw AS brand_name,
           COUNT(DISTINCT dispensary_id) AS dispensary_count
         FROM store_product_snapshots
         WHERE captured_at >= $1 AND captured_at < $1 + INTERVAL '1 day'
-          AND brand_name IS NOT NULL
+          AND brand_name_raw IS NOT NULL
           AND is_in_stock = TRUE
-        GROUP BY brand_name
+        GROUP BY brand_name_raw
       ),
       end_counts AS (
         SELECT
-          brand_name,
+          brand_name_raw AS brand_name,
           COUNT(DISTINCT dispensary_id) AS dispensary_count
         FROM store_product_snapshots
         WHERE captured_at >= $2 - INTERVAL '1 day' AND captured_at <= $2
-          AND brand_name IS NOT NULL
+          AND brand_name_raw IS NOT NULL
           AND is_in_stock = TRUE
-        GROUP BY brand_name
+        GROUP BY brand_name_raw
       )
       SELECT
         COALESCE(sc.brand_name, ec.brand_name) AS brand_name,
@@ -401,6 +411,225 @@ export class BrandPenetrationService {
       change_percent: row.change_percent ? parseFloat(row.change_percent) : 0,
     }));
   }
+
+  /**
+   * Get brand promotional history
+   *
+   * Tracks when products went on special, how long, what discount,
+   * and estimated quantity sold during the promotion.
+   */
+  async getBrandPromotionalHistory(
+    brandName: string,
+    options: { window?: TimeWindow; customRange?: DateRange; stateCode?: string; category?: string } = {}
+  ): Promise<BrandPromotionalSummary> {
+    const { window = '90d', customRange, stateCode, category } = options;
+    const { start, end } = getDateRangeFromWindow(window, customRange);
+
+    // Build filters
+    const params: any[] = [brandName, start, end];
+    let paramIdx = 4;
+    let filters = '';
+
+    if (stateCode) {
+      filters += ` AND s.code = $${paramIdx}`;
+      params.push(stateCode);
+      paramIdx++;
+    }
+
+    if (category) {
+      filters += ` AND sp.category_raw = $${paramIdx}`;
+      params.push(category);
+      paramIdx++;
+    }
+
+    // Find promotional events by detecting when is_on_special transitions to TRUE
+    // and tracking until it transitions back to FALSE
+    const eventsResult = await this.pool.query(`
+      WITH snapshot_with_lag AS (
+        SELECT
+          sps.id,
+          sps.store_product_id,
+          sps.dispensary_id,
+          sps.brand_name_raw,
+          sps.name_raw,
+          sps.category_raw,
+          sps.is_on_special,
+          sps.price_rec,
+          sps.price_rec_special,
+          sps.stock_quantity,
+          sps.captured_at,
+          LAG(sps.is_on_special) OVER (
+            PARTITION BY sps.store_product_id
+            ORDER BY sps.captured_at
+          ) AS prev_is_on_special,
+          LAG(sps.stock_quantity) OVER (
+            PARTITION BY sps.store_product_id
+            ORDER BY sps.captured_at
+          ) AS prev_stock_quantity
+        FROM store_product_snapshots sps
+        JOIN store_products sp ON sp.id = sps.store_product_id
+        JOIN dispensaries dd ON dd.id = sp.dispensary_id
+        LEFT JOIN states s ON s.id = dd.state_id
+        WHERE sps.brand_name_raw = $1
+          AND sps.captured_at >= $2
+          AND sps.captured_at <= $3
+          ${filters}
+      ),
+      special_starts AS (
+        -- Find when specials START (transition from not-on-special to on-special)
+        SELECT
+          store_product_id,
+          dispensary_id,
+          name_raw,
+          category_raw,
+          captured_at AS special_start,
+          price_rec AS regular_price,
+          price_rec_special AS special_price,
+          stock_quantity AS quantity_at_start
+        FROM snapshot_with_lag
+        WHERE is_on_special = TRUE
+          AND (prev_is_on_special = FALSE OR prev_is_on_special IS NULL)
+          AND price_rec_special IS NOT NULL
+          AND price_rec IS NOT NULL
+      ),
+      special_ends AS (
+        -- Find when specials END (transition from on-special to not-on-special)
+        SELECT
+          store_product_id,
+          captured_at AS special_end,
+          prev_stock_quantity AS quantity_at_end
+        FROM snapshot_with_lag
+        WHERE is_on_special = FALSE
+          AND prev_is_on_special = TRUE
+      ),
+      matched_events AS (
+        SELECT
+          ss.store_product_id,
+          ss.dispensary_id,
+          ss.name_raw AS product_name,
+          ss.category_raw AS category,
+          ss.special_start,
+          se.special_end,
+          ss.regular_price,
+          ss.special_price,
+          ss.quantity_at_start,
+          COALESCE(se.quantity_at_end, ss.quantity_at_start) AS quantity_at_end
+        FROM special_starts ss
+        LEFT JOIN special_ends se ON se.store_product_id = ss.store_product_id
+          AND se.special_end > ss.special_start
+          AND se.special_end = (
+            SELECT MIN(se2.special_end)
+            FROM special_ends se2
+            WHERE se2.store_product_id = ss.store_product_id
+              AND se2.special_end > ss.special_start
+          )
+      )
+      SELECT
+        me.store_product_id,
+        me.dispensary_id,
+        d.name AS dispensary_name,
+        s.code AS state_code,
+        me.product_name,
+        me.category,
+        me.special_start,
+        me.special_end,
+        EXTRACT(DAY FROM COALESCE(me.special_end, NOW()) - me.special_start)::INT AS duration_days,
+        me.regular_price,
+        me.special_price,
+        ROUND(((me.regular_price - me.special_price) / NULLIF(me.regular_price, 0)) * 100, 1) AS discount_percent,
+        me.quantity_at_start,
+        me.quantity_at_end,
+        GREATEST(0, COALESCE(me.quantity_at_start, 0) - COALESCE(me.quantity_at_end, 0)) AS quantity_sold_estimate
+      FROM matched_events me
+      JOIN dispensaries d ON d.id = me.dispensary_id
+      LEFT JOIN states s ON s.id = d.state_id
+      ORDER BY me.special_start DESC
+    `, params);
+
+    const events: BrandPromotionalEvent[] = eventsResult.rows.map((row: any) => ({
+      product_name: row.product_name,
+      store_product_id: parseInt(row.store_product_id),
+      dispensary_id: parseInt(row.dispensary_id),
+      dispensary_name: row.dispensary_name,
+      state_code: row.state_code || 'Unknown',
+      category: row.category,
+      special_start: row.special_start.toISOString().split('T')[0],
+      special_end: row.special_end ? row.special_end.toISOString().split('T')[0] : null,
+      duration_days: row.duration_days ? parseInt(row.duration_days) : null,
+      regular_price: parseFloat(row.regular_price) || 0,
+      special_price: parseFloat(row.special_price) || 0,
+      discount_percent: parseFloat(row.discount_percent) || 0,
+      quantity_at_start: row.quantity_at_start ? parseInt(row.quantity_at_start) : null,
+      quantity_at_end: row.quantity_at_end ? parseInt(row.quantity_at_end) : null,
+      quantity_sold_estimate: row.quantity_sold_estimate ? parseInt(row.quantity_sold_estimate) : null,
+    }));
+
+    // Calculate summary stats
+    const totalEvents = events.length;
+    const uniqueProducts = new Set(events.map(e => e.store_product_id)).size;
+    const uniqueDispensaries = new Set(events.map(e => e.dispensary_id)).size;
+    const uniqueStates = [...new Set(events.map(e => e.state_code))];
+
+    const avgDiscount = totalEvents > 0
+      ? events.reduce((sum, e) => sum + e.discount_percent, 0) / totalEvents
+      : 0;
+
+    const durations = events.filter(e => e.duration_days !== null).map(e => e.duration_days!);
+    const avgDuration = durations.length > 0
+      ? durations.reduce((sum, d) => sum + d, 0) / durations.length
+      : null;
+
+    const totalQuantitySold = events
+      .filter(e => e.quantity_sold_estimate !== null)
+      .reduce((sum, e) => sum + (e.quantity_sold_estimate || 0), 0);
+
+    // Calculate frequency
+    const windowDays = Math.ceil((end.getTime() - start.getTime()) / (1000 * 60 * 60 * 24));
+    const weeklyAvg = windowDays > 0 ? (totalEvents / windowDays) * 7 : 0;
+    const monthlyAvg = windowDays > 0 ? (totalEvents / windowDays) * 30 : 0;
+
+    // Group by category
+    const categoryMap = new Map<string, { count: number; discounts: number[]; quantity: number }>();
+    for (const event of events) {
+      const cat = event.category || 'Uncategorized';
+      if (!categoryMap.has(cat)) {
+        categoryMap.set(cat, { count: 0, discounts: [], quantity: 0 });
+      }
+      const entry = categoryMap.get(cat)!;
+      entry.count++;
+      entry.discounts.push(event.discount_percent);
+      if (event.quantity_sold_estimate !== null) {
+        entry.quantity += event.quantity_sold_estimate;
+      }
+    }
+
+    const byCategory = Array.from(categoryMap.entries()).map(([category, data]) => ({
+      category,
+      event_count: data.count,
+      avg_discount_percent: data.discounts.length > 0
+        ? Math.round((data.discounts.reduce((a, b) => a + b, 0) / data.discounts.length) * 10) / 10
+        : 0,
+      quantity_sold_estimate: data.quantity > 0 ? data.quantity : null,
+    })).sort((a, b) => b.event_count - a.event_count);
+
+    return {
+      brand_name: brandName,
+      window,
+      total_promotional_events: totalEvents,
+      total_products_on_special: uniqueProducts,
+      total_dispensaries_with_specials: uniqueDispensaries,
+      states_with_specials: uniqueStates,
+      avg_discount_percent: Math.round(avgDiscount * 10) / 10,
+      avg_duration_days: avgDuration !== null ? Math.round(avgDuration * 10) / 10 : null,
+      total_quantity_sold_estimate: totalQuantitySold > 0 ? totalQuantitySold : null,
+      promotional_frequency: {
+        weekly_avg: Math.round(weeklyAvg * 10) / 10,
+        monthly_avg: Math.round(monthlyAvg * 10) / 10,
+      },
+      by_category: byCategory,
+      events,
+    };
+  }
 }
 
 export default BrandPenetrationService;

backend/src/services/analytics/CategoryAnalyticsService.ts

@@ -43,14 +43,14 @@ export class CategoryAnalyticsService {
     // Get current category metrics
     const currentResult = await this.pool.query(`
       SELECT
-        sp.category,
+        sp.category_raw,
         COUNT(*) AS sku_count,
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
         AVG(sp.price_rec) AS avg_price
       FROM store_products sp
-      WHERE sp.category = $1
+      WHERE sp.category_raw = $1
         AND sp.is_in_stock = TRUE
-      GROUP BY sp.category
+      GROUP BY sp.category_raw
     `, [category]);
 
     if (currentResult.rows.length === 0) {
@@ -70,7 +70,7 @@ export class CategoryAnalyticsService {
         COUNT(DISTINCT sps.dispensary_id) AS dispensary_count,
         AVG(sps.price_rec) AS avg_price
       FROM store_product_snapshots sps
-      WHERE sps.category = $1
+      WHERE sps.category_raw = $1
         AND sps.captured_at >= $2
         AND sps.captured_at <= $3
         AND sps.is_in_stock = TRUE
@@ -111,8 +111,9 @@ export class CategoryAnalyticsService {
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
         AVG(sp.price_rec) AS avg_price
       FROM store_products sp
-      JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
-      WHERE sp.category = $1
+      JOIN states s ON s.id = d.state_id
+      WHERE sp.category_raw = $1
         AND sp.is_in_stock = TRUE
       GROUP BY s.code, s.name, s.recreational_legal
       ORDER BY sku_count DESC
@@ -154,24 +155,25 @@ export class CategoryAnalyticsService {
 
     const result = await this.pool.query(`
       SELECT
-        sp.category,
+        sp.category_raw,
         COUNT(*) AS sku_count,
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
-        COUNT(DISTINCT sp.brand_name) AS brand_count,
+        COUNT(DISTINCT sp.brand_name_raw) AS brand_count,
         AVG(sp.price_rec) AS avg_price,
         COUNT(DISTINCT s.code) AS state_count
       FROM store_products sp
-      LEFT JOIN states s ON s.id = sp.state_id
+      LEFT JOIN dispensaries d ON d.id = sp.dispensary_id
-      WHERE sp.category IS NOT NULL
+      JOIN states s ON s.id = d.state_id
+      WHERE sp.category_raw IS NOT NULL
         AND sp.is_in_stock = TRUE
         ${stateFilter}
-      GROUP BY sp.category
+      GROUP BY sp.category_raw
       ORDER BY sku_count DESC
       LIMIT $1
     `, params);
 
     return result.rows.map((row: any) => ({
-      category: row.category,
+      category: row.category_raw,
       sku_count: parseInt(row.sku_count),
       dispensary_count: parseInt(row.dispensary_count),
       brand_count: parseInt(row.brand_count),
@@ -188,14 +190,14 @@ export class CategoryAnalyticsService {
     let categoryFilter = '';
 
     if (category) {
-      categoryFilter = 'WHERE sp.category = $1';
+      categoryFilter = 'WHERE sp.category_raw = $1';
       params.push(category);
     }
 
     const result = await this.pool.query(`
       WITH category_stats AS (
         SELECT
-          sp.category,
+          sp.category_raw,
           CASE WHEN s.recreational_legal = TRUE THEN 'recreational' ELSE 'medical_only' END AS legal_type,
           COUNT(DISTINCT s.code) AS state_count,
           COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
@@ -203,13 +205,14 @@ export class CategoryAnalyticsService {
           AVG(sp.price_rec) AS avg_price,
           PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec) AS median_price
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
         ${categoryFilter}
-        ${category ? 'AND' : 'WHERE'} sp.category IS NOT NULL
+        ${category ? 'AND' : 'WHERE'} sp.category_raw IS NOT NULL
           AND sp.is_in_stock = TRUE
           AND sp.price_rec IS NOT NULL
           AND (s.recreational_legal = TRUE OR s.medical_legal = TRUE)
-        GROUP BY sp.category, CASE WHEN s.recreational_legal = TRUE THEN 'recreational' ELSE 'medical_only' END
+        GROUP BY sp.category_raw, CASE WHEN s.recreational_legal = TRUE THEN 'recreational' ELSE 'medical_only' END
       ),
       rec_stats AS (
         SELECT * FROM category_stats WHERE legal_type = 'recreational'
@@ -218,7 +221,7 @@ export class CategoryAnalyticsService {
         SELECT * FROM category_stats WHERE legal_type = 'medical_only'
       )
       SELECT
-        COALESCE(r.category, m.category) AS category,
+        COALESCE(r.category_raw, m.category_raw) AS category,
         r.state_count AS rec_state_count,
         r.dispensary_count AS rec_dispensary_count,
         r.sku_count AS rec_sku_count,
@@ -235,7 +238,7 @@ export class CategoryAnalyticsService {
           ELSE NULL
         END AS price_diff_percent
       FROM rec_stats r
-      FULL OUTER JOIN med_stats m ON r.category = m.category
+      FULL OUTER JOIN med_stats m ON r.category_raw = m.category_raw
       ORDER BY COALESCE(r.sku_count, 0) + COALESCE(m.sku_count, 0) DESC
     `, params);
 
@@ -282,7 +285,7 @@ export class CategoryAnalyticsService {
           COUNT(*) AS sku_count,
           COUNT(DISTINCT sps.dispensary_id) AS dispensary_count
         FROM store_product_snapshots sps
-        WHERE sps.category = $1
+        WHERE sps.category_raw = $1
           AND sps.captured_at >= $2
           AND sps.captured_at <= $3
           AND sps.is_in_stock = TRUE
@@ -335,31 +338,33 @@ export class CategoryAnalyticsService {
       WITH category_total AS (
         SELECT COUNT(*) AS total
         FROM store_products sp
-        LEFT JOIN states s ON s.id = sp.state_id
+        LEFT JOIN dispensaries d ON d.id = sp.dispensary_id
-        WHERE sp.category = $1
+      JOIN states s ON s.id = d.state_id
+        WHERE sp.category_raw = $1
           AND sp.is_in_stock = TRUE
-          AND sp.brand_name IS NOT NULL
+          AND sp.brand_name_raw IS NOT NULL
           ${stateFilter}
       )
       SELECT
-        sp.brand_name,
+        sp.brand_name_raw,
         COUNT(*) AS sku_count,
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count,
         AVG(sp.price_rec) AS avg_price,
         ROUND(COUNT(*)::NUMERIC * 100 / NULLIF((SELECT total FROM category_total), 0), 2) AS category_share_percent
       FROM store_products sp
-      LEFT JOIN states s ON s.id = sp.state_id
+      LEFT JOIN dispensaries d ON d.id = sp.dispensary_id
-      WHERE sp.category = $1
+      JOIN states s ON s.id = d.state_id
+      WHERE sp.category_raw = $1
         AND sp.is_in_stock = TRUE
-        AND sp.brand_name IS NOT NULL
+        AND sp.brand_name_raw IS NOT NULL
         ${stateFilter}
-      GROUP BY sp.brand_name
+      GROUP BY sp.brand_name_raw
       ORDER BY sku_count DESC
       LIMIT $2
     `, params);
 
     return result.rows.map((row: any) => ({
-      brand_name: row.brand_name,
+      brand_name: row.brand_name_raw,
       sku_count: parseInt(row.sku_count),
       dispensary_count: parseInt(row.dispensary_count),
       avg_price: row.avg_price ? parseFloat(row.avg_price) : null,
@@ -421,7 +426,7 @@ export class CategoryAnalyticsService {
     `, [start, end, limit]);
 
     return result.rows.map((row: any) => ({
-      category: row.category,
+      category: row.category_raw,
       start_sku_count: parseInt(row.start_sku_count),
       end_sku_count: parseInt(row.end_sku_count),
       growth: parseInt(row.growth),

backend/src/services/analytics/PriceAnalyticsService.ts

@@ -43,9 +43,9 @@ export class PriceAnalyticsService {
     const productResult = await this.pool.query(`
       SELECT
         sp.id,
-        sp.name,
+        sp.name_raw,
-        sp.brand_name,
+        sp.brand_name_raw,
-        sp.category,
+        sp.category_raw,
         sp.dispensary_id,
         sp.price_rec,
         sp.price_med,
@@ -53,7 +53,7 @@ export class PriceAnalyticsService {
         s.code AS state_code
       FROM store_products sp
       JOIN dispensaries d ON d.id = sp.dispensary_id
-      LEFT JOIN states s ON s.id = sp.state_id
+      JOIN states s ON s.id = d.state_id
       WHERE sp.id = $1
     `, [storeProductId]);
 
@@ -133,7 +133,7 @@ export class PriceAnalyticsService {
 
     const result = await this.pool.query(`
       SELECT
-        sp.category,
+        sp.category_raw,
         s.code AS state_code,
         s.name AS state_name,
         CASE
@@ -148,18 +148,18 @@ export class PriceAnalyticsService {
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count
       FROM store_products sp
       JOIN dispensaries d ON d.id = sp.dispensary_id
-      JOIN states s ON s.id = sp.state_id
+      JOIN states s ON s.id = d.state_id
-      WHERE sp.category = $1
+      WHERE sp.category_raw = $1
         AND sp.price_rec IS NOT NULL
         AND sp.is_in_stock = TRUE
         AND (s.recreational_legal = TRUE OR s.medical_legal = TRUE)
         ${stateFilter}
-      GROUP BY sp.category, s.code, s.name, s.recreational_legal
+      GROUP BY sp.category_raw, s.code, s.name, s.recreational_legal
       ORDER BY state_code
     `, params);
 
     return result.rows.map((row: any) => ({
-      category: row.category,
+      category: row.category_raw,
       state_code: row.state_code,
       state_name: row.state_name,
       legal_type: row.legal_type,
@@ -189,7 +189,7 @@ export class PriceAnalyticsService {
 
     const result = await this.pool.query(`
       SELECT
-        sp.brand_name AS category,
+        sp.brand_name_raw AS category,
         s.code AS state_code,
         s.name AS state_name,
         CASE
@@ -204,18 +204,18 @@ export class PriceAnalyticsService {
         COUNT(DISTINCT sp.dispensary_id) AS dispensary_count
       FROM store_products sp
       JOIN dispensaries d ON d.id = sp.dispensary_id
-      JOIN states s ON s.id = sp.state_id
+      JOIN states s ON s.id = d.state_id
-      WHERE sp.brand_name = $1
+      WHERE sp.brand_name_raw = $1
         AND sp.price_rec IS NOT NULL
         AND sp.is_in_stock = TRUE
         AND (s.recreational_legal = TRUE OR s.medical_legal = TRUE)
         ${stateFilter}
-      GROUP BY sp.brand_name, s.code, s.name, s.recreational_legal
+      GROUP BY sp.brand_name_raw, s.code, s.name, s.recreational_legal
       ORDER BY state_code
     `, params);
 
     return result.rows.map((row: any) => ({
-      category: row.category,
+      category: row.category_raw,
       state_code: row.state_code,
       state_name: row.state_name,
       legal_type: row.legal_type,
@@ -254,7 +254,7 @@ export class PriceAnalyticsService {
     }
 
     if (category) {
-      filters += ` AND sp.category = $${paramIdx}`;
+      filters += ` AND sp.category_raw = $${paramIdx}`;
       params.push(category);
       paramIdx++;
     }
@@ -288,15 +288,16 @@ export class PriceAnalyticsService {
       )
       SELECT
         v.store_product_id,
-        sp.name AS product_name,
+        sp.name_raw AS product_name,
-        sp.brand_name,
+        sp.brand_name_raw,
         v.change_count,
         v.avg_change_pct,
         v.max_change_pct,
         v.last_change_at
       FROM volatility v
       JOIN store_products sp ON sp.id = v.store_product_id
-      LEFT JOIN states s ON s.id = sp.state_id
+      LEFT JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
       WHERE 1=1 ${filters}
       ORDER BY v.change_count DESC, v.avg_change_pct DESC
       LIMIT $3
@@ -305,7 +306,7 @@ export class PriceAnalyticsService {
     return result.rows.map((row: any) => ({
       store_product_id: row.store_product_id,
       product_name: row.product_name,
-      brand_name: row.brand_name,
+      brand_name: row.brand_name_raw,
       change_count: parseInt(row.change_count),
       avg_change_percent: row.avg_change_pct ? parseFloat(row.avg_change_pct) : 0,
       max_change_percent: row.max_change_pct ? parseFloat(row.max_change_pct) : 0,
@@ -327,13 +328,13 @@ export class PriceAnalyticsService {
     let categoryFilter = '';
 
     if (category) {
-      categoryFilter = 'WHERE sp.category = $1';
+      categoryFilter = 'WHERE sp.category_raw = $1';
       params.push(category);
     }
 
     const result = await this.pool.query(`
       SELECT
-        sp.category,
+        sp.category_raw,
         AVG(sp.price_rec) FILTER (WHERE s.recreational_legal = TRUE) AS rec_avg,
         PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec)
           FILTER (WHERE s.recreational_legal = TRUE) AS rec_median,
@@ -343,17 +344,18 @@ export class PriceAnalyticsService {
         PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec)
           FILTER (WHERE s.medical_legal = TRUE AND (s.recreational_legal = FALSE OR s.recreational_legal IS NULL)) AS med_median
       FROM store_products sp
-      JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
       ${categoryFilter}
       ${category ? 'AND' : 'WHERE'} sp.price_rec IS NOT NULL
         AND sp.is_in_stock = TRUE
-        AND sp.category IS NOT NULL
+        AND sp.category_raw IS NOT NULL
-      GROUP BY sp.category
+      GROUP BY sp.category_raw
-      ORDER BY sp.category
+      ORDER BY sp.category_raw
     `, params);
 
     return result.rows.map((row: any) => ({
-      category: row.category,
+      category: row.category_raw,
       rec_avg: row.rec_avg ? parseFloat(row.rec_avg) : null,
       rec_median: row.rec_median ? parseFloat(row.rec_median) : null,
       med_avg: row.med_avg ? parseFloat(row.med_avg) : null,

backend/src/services/analytics/StateAnalyticsService.ts

@@ -108,14 +108,14 @@ export class StateAnalyticsService {
       SELECT
         COUNT(DISTINCT d.id) AS dispensary_count,
         COUNT(DISTINCT sp.id) AS product_count,
-        COUNT(DISTINCT sp.brand_name) FILTER (WHERE sp.brand_name IS NOT NULL) AS brand_count,
+        COUNT(DISTINCT sp.brand_name_raw) FILTER (WHERE sp.brand_name_raw IS NOT NULL) AS brand_count,
-        COUNT(DISTINCT sp.category) FILTER (WHERE sp.category IS NOT NULL) AS category_count,
+        COUNT(DISTINCT sp.category_raw) FILTER (WHERE sp.category_raw IS NOT NULL) AS category_count,
         COUNT(sps.id) AS snapshot_count,
         MAX(sps.captured_at) AS last_crawl_at
       FROM states s
       LEFT JOIN dispensaries d ON d.state_id = s.id
-      LEFT JOIN store_products sp ON sp.state_id = s.id AND sp.is_in_stock = TRUE
+      LEFT JOIN store_products sp ON sp.dispensary_id = d.id AND sp.is_in_stock = TRUE
-      LEFT JOIN store_product_snapshots sps ON sps.state_id = s.id
+      LEFT JOIN store_product_snapshots sps ON sps.dispensary_id = d.id
       WHERE s.code = $1
     `, [stateCode]);
 
@@ -129,7 +129,8 @@ export class StateAnalyticsService {
         MIN(price_rec) AS min_price,
         MAX(price_rec) AS max_price
       FROM store_products sp
-      JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
       WHERE s.code = $1
         AND sp.price_rec IS NOT NULL
         AND sp.is_in_stock = TRUE
@@ -140,14 +141,15 @@ export class StateAnalyticsService {
     // Get top categories
     const topCategoriesResult = await this.pool.query(`
       SELECT
-        sp.category,
+        sp.category_raw,
         COUNT(*) AS count
       FROM store_products sp
-      JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
       WHERE s.code = $1
-        AND sp.category IS NOT NULL
+        AND sp.category_raw IS NOT NULL
         AND sp.is_in_stock = TRUE
-      GROUP BY sp.category
+      GROUP BY sp.category_raw
       ORDER BY count DESC
       LIMIT 10
     `, [stateCode]);
@@ -155,14 +157,15 @@ export class StateAnalyticsService {
     // Get top brands
     const topBrandsResult = await this.pool.query(`
       SELECT
-        sp.brand_name AS brand,
+        sp.brand_name_raw AS brand,
         COUNT(*) AS count
       FROM store_products sp
-      JOIN states s ON s.id = sp.state_id
+      JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
       WHERE s.code = $1
-        AND sp.brand_name IS NOT NULL
+        AND sp.brand_name_raw IS NOT NULL
         AND sp.is_in_stock = TRUE
-      GROUP BY sp.brand_name
+      GROUP BY sp.brand_name_raw
       ORDER BY count DESC
       LIMIT 10
     `, [stateCode]);
@@ -191,7 +194,7 @@ export class StateAnalyticsService {
         max_price: pricing.max_price ? parseFloat(pricing.max_price) : null,
       },
       top_categories: topCategoriesResult.rows.map((row: any) => ({
-        category: row.category,
+        category: row.category_raw,
         count: parseInt(row.count),
       })),
       top_brands: topBrandsResult.rows.map((row: any) => ({
@@ -215,8 +218,8 @@ export class StateAnalyticsService {
         COUNT(sps.id) AS snapshot_count
       FROM states s
       LEFT JOIN dispensaries d ON d.state_id = s.id
-      LEFT JOIN store_products sp ON sp.state_id = s.id AND sp.is_in_stock = TRUE
+      LEFT JOIN store_products sp ON sp.dispensary_id = d.id AND sp.is_in_stock = TRUE
-      LEFT JOIN store_product_snapshots sps ON sps.state_id = s.id
+      LEFT JOIN store_product_snapshots sps ON sps.dispensary_id = d.id
       WHERE s.recreational_legal = TRUE
       GROUP BY s.code, s.name
       ORDER BY dispensary_count DESC
@@ -232,8 +235,8 @@ export class StateAnalyticsService {
         COUNT(sps.id) AS snapshot_count
       FROM states s
       LEFT JOIN dispensaries d ON d.state_id = s.id
-      LEFT JOIN store_products sp ON sp.state_id = s.id AND sp.is_in_stock = TRUE
+      LEFT JOIN store_products sp ON sp.dispensary_id = d.id AND sp.is_in_stock = TRUE
-      LEFT JOIN store_product_snapshots sps ON sps.state_id = s.id
+      LEFT JOIN store_product_snapshots sps ON sps.dispensary_id = d.id
       WHERE s.medical_legal = TRUE
         AND (s.recreational_legal = FALSE OR s.recreational_legal IS NULL)
       GROUP BY s.code, s.name
@@ -295,46 +298,48 @@ export class StateAnalyticsService {
     let groupBy = 'NULL';
 
     if (category) {
-      categoryFilter = 'AND sp.category = $1';
+      categoryFilter = 'AND sp.category_raw = $1';
       params.push(category);
-      groupBy = 'sp.category';
+      groupBy = 'sp.category_raw';
     } else {
-      groupBy = 'sp.category';
+      groupBy = 'sp.category_raw';
     }
 
     const result = await this.pool.query(`
       WITH rec_prices AS (
         SELECT
-          ${category ? 'sp.category' : 'sp.category'},
+          ${category ? 'sp.category_raw' : 'sp.category_raw'},
           COUNT(DISTINCT s.code) AS state_count,
           COUNT(*) AS product_count,
           AVG(sp.price_rec) AS avg_price,
           PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec) AS median_price
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
         WHERE s.recreational_legal = TRUE
           AND sp.price_rec IS NOT NULL
           AND sp.is_in_stock = TRUE
-          AND sp.category IS NOT NULL
+          AND sp.category_raw IS NOT NULL
           ${categoryFilter}
-        GROUP BY sp.category
+        GROUP BY sp.category_raw
       ),
       med_prices AS (
         SELECT
-          ${category ? 'sp.category' : 'sp.category'},
+          ${category ? 'sp.category_raw' : 'sp.category_raw'},
           COUNT(DISTINCT s.code) AS state_count,
           COUNT(*) AS product_count,
           AVG(sp.price_rec) AS avg_price,
           PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec) AS median_price
         FROM store_products sp
-        JOIN states s ON s.id = sp.state_id
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+      JOIN states s ON s.id = d.state_id
         WHERE s.medical_legal = TRUE
           AND (s.recreational_legal = FALSE OR s.recreational_legal IS NULL)
           AND sp.price_rec IS NOT NULL
           AND sp.is_in_stock = TRUE
-          AND sp.category IS NOT NULL
+          AND sp.category_raw IS NOT NULL
           ${categoryFilter}
-        GROUP BY sp.category
+        GROUP BY sp.category_raw
       )
       SELECT
         COALESCE(r.category, m.category) AS category,
@@ -357,7 +362,7 @@ export class StateAnalyticsService {
     `, params);
 
     return result.rows.map((row: any) => ({
-      category: row.category,
+      category: row.category_raw,
       recreational: {
         state_count: parseInt(row.rec_state_count) || 0,
         product_count: parseInt(row.rec_product_count) || 0,
@@ -395,12 +400,12 @@ export class StateAnalyticsService {
         COALESCE(s.medical_legal, FALSE) AS medical_legal,
         COUNT(DISTINCT d.id) AS dispensary_count,
         COUNT(DISTINCT sp.id) AS product_count,
-        COUNT(DISTINCT sp.brand_name) FILTER (WHERE sp.brand_name IS NOT NULL) AS brand_count,
+        COUNT(DISTINCT sp.brand_name_raw) FILTER (WHERE sp.brand_name_raw IS NOT NULL) AS brand_count,
         MAX(sps.captured_at) AS last_crawl_at
       FROM states s
       LEFT JOIN dispensaries d ON d.state_id = s.id
-      LEFT JOIN store_products sp ON sp.state_id = s.id AND sp.is_in_stock = TRUE
+      LEFT JOIN store_products sp ON sp.dispensary_id = d.id AND sp.is_in_stock = TRUE
-      LEFT JOIN store_product_snapshots sps ON sps.state_id = s.id
+      LEFT JOIN store_product_snapshots sps ON sps.dispensary_id = d.id
       GROUP BY s.code, s.name, s.recreational_legal, s.medical_legal
       ORDER BY dispensary_count DESC, s.name
     `);
@@ -451,8 +456,8 @@ export class StateAnalyticsService {
         END AS gap_reason
       FROM states s
       LEFT JOIN dispensaries d ON d.state_id = s.id
-      LEFT JOIN store_products sp ON sp.state_id = s.id AND sp.is_in_stock = TRUE
+      LEFT JOIN store_products sp ON sp.dispensary_id = d.id AND sp.is_in_stock = TRUE
-      LEFT JOIN store_product_snapshots sps ON sps.state_id = s.id
+      LEFT JOIN store_product_snapshots sps ON sps.dispensary_id = d.id
       WHERE s.recreational_legal = TRUE OR s.medical_legal = TRUE
       GROUP BY s.code, s.name, s.recreational_legal, s.medical_legal
       HAVING COUNT(DISTINCT d.id) = 0
@@ -499,7 +504,8 @@ export class StateAnalyticsService {
           PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY sp.price_rec) AS median_price,
           COUNT(*) AS product_count
         FROM states s
-        JOIN store_products sp ON sp.state_id = s.id
+        JOIN dispensaries d ON d.state_id = s.id
+        JOIN store_products sp ON sp.dispensary_id = d.id
         WHERE sp.price_rec IS NOT NULL
           AND sp.is_in_stock = TRUE
           AND (s.recreational_legal = TRUE OR s.medical_legal = TRUE)

backend/src/services/analytics/StoreAnalyticsService.ts

@@ -89,22 +89,22 @@ export class StoreAnalyticsService {
     // Get brands added/dropped
     const brandsResult = await this.pool.query(`
       WITH start_brands AS (
-        SELECT DISTINCT brand_name
+        SELECT DISTINCT brand_name_raw
         FROM store_product_snapshots
         WHERE dispensary_id = $1
-          AND captured_at >= $2 AND captured_at < $2 + INTERVAL '1 day'
+          AND captured_at >= $2::timestamp AND captured_at < $2::timestamp + INTERVAL '1 day'
-          AND brand_name IS NOT NULL
+          AND brand_name_raw IS NOT NULL
       ),
       end_brands AS (
-        SELECT DISTINCT brand_name
+        SELECT DISTINCT brand_name_raw
         FROM store_product_snapshots
         WHERE dispensary_id = $1
-          AND captured_at >= $3 - INTERVAL '1 day' AND captured_at <= $3
+          AND captured_at >= $3::timestamp - INTERVAL '1 day' AND captured_at <= $3::timestamp
-          AND brand_name IS NOT NULL
+          AND brand_name_raw IS NOT NULL
       )
       SELECT
-        ARRAY(SELECT brand_name FROM end_brands EXCEPT SELECT brand_name FROM start_brands) AS added,
+        ARRAY(SELECT brand_name_raw FROM end_brands EXCEPT SELECT brand_name_raw FROM start_brands) AS added,
-        ARRAY(SELECT brand_name FROM start_brands EXCEPT SELECT brand_name FROM end_brands) AS dropped
+        ARRAY(SELECT brand_name_raw FROM start_brands EXCEPT SELECT brand_name_raw FROM end_brands) AS dropped
     `, [dispensaryId, start, end]);
 
     const brands = brandsResult.rows[0] || { added: [], dropped: [] };
@@ -184,9 +184,9 @@ export class StoreAnalyticsService {
         -- Products added
         SELECT
           sp.id AS store_product_id,
-          sp.name AS product_name,
+          sp.name_raw AS product_name,
-          sp.brand_name,
+          sp.brand_name_raw,
-          sp.category,
+          sp.category_raw,
           'added' AS event_type,
           sp.first_seen_at AS event_date,
           NULL::TEXT AS old_value,
@@ -201,9 +201,9 @@ export class StoreAnalyticsService {
         -- Stock in/out from snapshots
         SELECT
           sps.store_product_id,
-          sp.name AS product_name,
+          sp.name_raw AS product_name,
-          sp.brand_name,
+          sp.brand_name_raw,
-          sp.category,
+          sp.category_raw,
           CASE
             WHEN sps.is_in_stock = TRUE AND LAG(sps.is_in_stock) OVER w = FALSE THEN 'stock_in'
             WHEN sps.is_in_stock = FALSE AND LAG(sps.is_in_stock) OVER w = TRUE THEN 'stock_out'
@@ -224,9 +224,9 @@ export class StoreAnalyticsService {
         -- Price changes from snapshots
         SELECT
           sps.store_product_id,
-          sp.name AS product_name,
+          sp.name_raw AS product_name,
-          sp.brand_name,
+          sp.brand_name_raw,
-          sp.category,
+          sp.category_raw,
           'price_change' AS event_type,
           sps.captured_at AS event_date,
           LAG(sps.price_rec::TEXT) OVER w AS old_value,
@@ -250,8 +250,8 @@ export class StoreAnalyticsService {
     return result.rows.map((row: any) => ({
       store_product_id: row.store_product_id,
       product_name: row.product_name,
-      brand_name: row.brand_name,
+      brand_name: row.brand_name_raw,
-      category: row.category,
+      category: row.category_raw,
       event_type: row.event_type,
       event_date: row.event_date ? row.event_date.toISOString() : null,
       old_value: row.old_value,
@@ -259,6 +259,122 @@ export class StoreAnalyticsService {
     }));
   }
 
+  /**
+   * Get quantity changes for a store (increases/decreases)
+   * Useful for estimating sales (decreases) or restocks (increases)
+   *
+   * @param direction - 'decrease' for likely sales, 'increase' for restocks, 'all' for both
+   */
+  async getQuantityChanges(
+    dispensaryId: number,
+    options: {
+      window?: TimeWindow;
+      customRange?: DateRange;
+      direction?: 'increase' | 'decrease' | 'all';
+      limit?: number;
+    } = {}
+  ): Promise<{
+    dispensary_id: number;
+    window: TimeWindow;
+    direction: string;
+    total_changes: number;
+    total_units_decreased: number;
+    total_units_increased: number;
+    changes: Array<{
+      store_product_id: number;
+      product_name: string;
+      brand_name: string | null;
+      category: string | null;
+      old_quantity: number;
+      new_quantity: number;
+      quantity_delta: number;
+      direction: 'increase' | 'decrease';
+      captured_at: string;
+    }>;
+  }> {
+    const { window = '7d', customRange, direction = 'all', limit = 100 } = options;
+    const { start, end } = getDateRangeFromWindow(window, customRange);
+
+    // Build direction filter
+    let directionFilter = '';
+    if (direction === 'decrease') {
+      directionFilter = 'AND qty_delta < 0';
+    } else if (direction === 'increase') {
+      directionFilter = 'AND qty_delta > 0';
+    }
+
+    const result = await this.pool.query(`
+      WITH qty_changes AS (
+        SELECT
+          sps.store_product_id,
+          sp.name_raw AS product_name,
+          sp.brand_name_raw AS brand_name,
+          sp.category_raw AS category,
+          LAG(sps.stock_quantity) OVER w AS old_quantity,
+          sps.stock_quantity AS new_quantity,
+          sps.stock_quantity - LAG(sps.stock_quantity) OVER w AS qty_delta,
+          sps.captured_at
+        FROM store_product_snapshots sps
+        JOIN store_products sp ON sp.id = sps.store_product_id
+        WHERE sps.dispensary_id = $1
+          AND sps.captured_at >= $2
+          AND sps.captured_at <= $3
+          AND sps.stock_quantity IS NOT NULL
+        WINDOW w AS (PARTITION BY sps.store_product_id ORDER BY sps.captured_at)
+      )
+      SELECT *
+      FROM qty_changes
+      WHERE old_quantity IS NOT NULL
+        AND qty_delta != 0
+        ${directionFilter}
+      ORDER BY captured_at DESC
+      LIMIT $4
+    `, [dispensaryId, start, end, limit]);
+
+    // Calculate totals
+    const totalsResult = await this.pool.query(`
+      WITH qty_changes AS (
+        SELECT
+          sps.stock_quantity - LAG(sps.stock_quantity) OVER w AS qty_delta
+        FROM store_product_snapshots sps
+        WHERE sps.dispensary_id = $1
+          AND sps.captured_at >= $2
+          AND sps.captured_at <= $3
+          AND sps.stock_quantity IS NOT NULL
+          AND sps.store_product_id IS NOT NULL
+        WINDOW w AS (PARTITION BY sps.store_product_id ORDER BY sps.captured_at)
+      )
+      SELECT
+        COUNT(*) FILTER (WHERE qty_delta != 0) AS total_changes,
+        COALESCE(SUM(ABS(qty_delta)) FILTER (WHERE qty_delta < 0), 0) AS units_decreased,
+        COALESCE(SUM(qty_delta) FILTER (WHERE qty_delta > 0), 0) AS units_increased
+      FROM qty_changes
+      WHERE qty_delta IS NOT NULL
+    `, [dispensaryId, start, end]);
+
+    const totals = totalsResult.rows[0] || {};
+
+    return {
+      dispensary_id: dispensaryId,
+      window,
+      direction,
+      total_changes: parseInt(totals.total_changes) || 0,
+      total_units_decreased: parseInt(totals.units_decreased) || 0,
+      total_units_increased: parseInt(totals.units_increased) || 0,
+      changes: result.rows.map((row: any) => ({
+        store_product_id: row.store_product_id,
+        product_name: row.product_name,
+        brand_name: row.brand_name_raw,
+        category: row.category_raw,
+        old_quantity: row.old_quantity,
+        new_quantity: row.new_quantity,
+        quantity_delta: row.qty_delta,
+        direction: row.qty_delta > 0 ? 'increase' : 'decrease',
+        captured_at: row.captured_at?.toISOString() || null,
+      })),
+    };
+  }
+
   /**
    * Get store inventory composition (categories and brands breakdown)
    */
@@ -299,14 +415,14 @@ export class StoreAnalyticsService {
     // Get top brands
     const brandsResult = await this.pool.query(`
       SELECT
-        brand_name AS brand,
+        brand_name_raw AS brand,
         COUNT(*) AS count,
         ROUND(COUNT(*)::NUMERIC * 100 / NULLIF($2, 0), 2) AS percent
       FROM store_products
       WHERE dispensary_id = $1
-        AND brand_name IS NOT NULL
+        AND brand_name_raw IS NOT NULL
         AND is_in_stock = TRUE
-      GROUP BY brand_name
+      GROUP BY brand_name_raw
       ORDER BY count DESC
       LIMIT 20
     `, [dispensaryId, totalProducts]);
@@ -316,7 +432,7 @@ export class StoreAnalyticsService {
       in_stock_count: parseInt(totals.in_stock) || 0,
       out_of_stock_count: parseInt(totals.out_of_stock) || 0,
       categories: categoriesResult.rows.map((row: any) => ({
-        category: row.category,
+        category: row.category_raw,
         count: parseInt(row.count),
         percent: parseFloat(row.percent) || 0,
       })),
@@ -458,23 +574,24 @@ export class StoreAnalyticsService {
       ),
       market_prices AS (
         SELECT
-          sp.category,
+          sp.category_raw,
           AVG(sp.price_rec) AS market_avg
         FROM store_products sp
-        WHERE sp.state_id = $2
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+        WHERE d.state_id = $2
           AND sp.price_rec IS NOT NULL
           AND sp.is_in_stock = TRUE
-          AND sp.category IS NOT NULL
+          AND sp.category_raw IS NOT NULL
-        GROUP BY sp.category
+        GROUP BY sp.category_raw
       )
       SELECT
-        sp.category,
+        sp.category_raw,
         sp.store_avg AS store_avg_price,
         mp.market_avg AS market_avg_price,
         ROUND(((sp.store_avg - mp.market_avg) / NULLIF(mp.market_avg, 0) * 100)::NUMERIC, 2) AS price_vs_market_percent,
         sp.product_count
       FROM store_prices sp
-      LEFT JOIN market_prices mp ON mp.category = sp.category
+      LEFT JOIN market_prices mp ON mp.category = sp.category_raw
       ORDER BY sp.product_count DESC
     `, [dispensaryId, dispensary.state_id]);
 
@@ -486,9 +603,10 @@ export class StoreAnalyticsService {
         WHERE dispensary_id = $1 AND price_rec IS NOT NULL AND is_in_stock = TRUE
       ),
       market_avg AS (
-        SELECT AVG(price_rec) AS avg
+        SELECT AVG(sp.price_rec) AS avg
-        FROM store_products
+        FROM store_products sp
-        WHERE state_id = $2 AND price_rec IS NOT NULL AND is_in_stock = TRUE
+        JOIN dispensaries d ON d.id = sp.dispensary_id
+        WHERE d.state_id = $2 AND sp.price_rec IS NOT NULL AND sp.is_in_stock = TRUE
       )
       SELECT
         ROUND(((sa.avg - ma.avg) / NULLIF(ma.avg, 0) * 100)::NUMERIC, 2) AS price_vs_market
@@ -499,7 +617,7 @@ export class StoreAnalyticsService {
       dispensary_id: dispensaryId,
       dispensary_name: dispensary.name,
       categories: result.rows.map((row: any) => ({
-        category: row.category,
+        category: row.category_raw,
         store_avg_price: parseFloat(row.store_avg_price),
         market_avg_price: row.market_avg_price ? parseFloat(row.market_avg_price) : 0,
         price_vs_market_percent: row.price_vs_market_percent ? parseFloat(row.price_vs_market_percent) : 0,

backend/src/services/analytics/index.ts

@@ -11,3 +11,4 @@ export { BrandPenetrationService } from './BrandPenetrationService';
 export { CategoryAnalyticsService } from './CategoryAnalyticsService';
 export { StoreAnalyticsService } from './StoreAnalyticsService';
 export { StateAnalyticsService } from './StateAnalyticsService';
+export { BrandIntelligenceService } from './BrandIntelligenceService';

backend/src/services/analytics/types.ts

@@ -322,3 +322,48 @@ export interface RecVsMedPriceComparison {
   };
   price_diff_percent: number | null;
 }
+
+// ============================================================
+// BRAND PROMOTIONAL ANALYTICS TYPES
+// ============================================================
+
+export interface BrandPromotionalEvent {
+  product_name: string;
+  store_product_id: number;
+  dispensary_id: number;
+  dispensary_name: string;
+  state_code: string;
+  category: string | null;
+  special_start: string;          // ISO date when special started
+  special_end: string | null;     // ISO date when special ended (null if ongoing)
+  duration_days: number | null;
+  regular_price: number;
+  special_price: number;
+  discount_percent: number;
+  quantity_at_start: number | null;
+  quantity_at_end: number | null;
+  quantity_sold_estimate: number | null;  // quantity_at_start - quantity_at_end
+}
+
+export interface BrandPromotionalSummary {
+  brand_name: string;
+  window: TimeWindow;
+  total_promotional_events: number;
+  total_products_on_special: number;
+  total_dispensaries_with_specials: number;
+  states_with_specials: string[];
+  avg_discount_percent: number;
+  avg_duration_days: number | null;
+  total_quantity_sold_estimate: number | null;
+  promotional_frequency: {
+    weekly_avg: number;
+    monthly_avg: number;
+  };
+  by_category: Array<{
+    category: string;
+    event_count: number;
+    avg_discount_percent: number;
+    quantity_sold_estimate: number | null;
+  }>;
+  events: BrandPromotionalEvent[];
+}

backend/src/services/crawl-rotator.ts

@@ -1,49 +1,53 @@
 /**
  * Crawl Rotator - Proxy & User Agent Rotation for Crawlers
  *
- * Manages rotation of proxies and user agents to avoid blocks.
+ * Updated: 2025-12-10 per workflow-12102025.md
- * Used by platform-specific crawlers (Dutchie, Jane, etc.)
+ *
+ * KEY BEHAVIORS (per workflow-12102025.md):
+ * 1. Task determines WHAT work to do, proxy determines SESSION IDENTITY
+ * 2. Proxy location (timezone) sets Accept-Language headers (always English)
+ * 3. On 403: immediately get new IP, new fingerprint, retry
+ * 4. After 3 consecutive 403s on same proxy with different fingerprints → disable proxy
+ *
+ * USER-AGENT GENERATION (per workflow-12102025.md):
+ * - Device distribution: Mobile 62%, Desktop 36%, Tablet 2%
+ * - Browser whitelist: Chrome, Safari, Edge, Firefox only
+ * - UA sticks until IP rotates
+ * - Failure = alert admin + stop crawl (no fallback)
+ *
+ * Uses intoli/user-agents for realistic UA generation with daily-updated data.
  *
  * Canonical location: src/services/crawl-rotator.ts
  */
 
 import { Pool } from 'pg';
+import UserAgent from 'user-agents';
+import {
+  HTTPFingerprint,
+  generateHTTPFingerprint,
+  BrowserType,
+} from './http-fingerprint';
 
 // ============================================================
-// USER AGENT CONFIGURATION
+// UA CONSTANTS (per workflow-12102025.md)
 // ============================================================
 
 /**
- * Modern browser user agents (Chrome, Firefox, Safari, Edge on various platforms)
+ * Per workflow-12102025.md: Device category distribution (hardcoded)
- * Updated: 2024
+ * Mobile: 62%, Desktop: 36%, Tablet: 2%
  */
-export const USER_AGENTS = [
+const DEVICE_WEIGHTS = {
-  // Chrome on Windows
+  mobile: 62,
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+  desktop: 36,
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
+  tablet: 2,
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
+} as const;
 
-  // Chrome on macOS
+/**
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+ * Per workflow-12102025.md: Browser whitelist
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
+ * Only Chrome (67%), Safari (20%), Edge (6%), Firefox (3%)
-
+ * Samsung Internet, Opera, and other niche browsers are filtered out
-  // Firefox on Windows
+ */
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
+const ALLOWED_BROWSERS = ['Chrome', 'Safari', 'Edge', 'Firefox'] as const;
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
-
-  // Firefox on macOS
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
-
-  // Safari on macOS
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
-
-  // Edge on Windows
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
-
-  // Chrome on Linux
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-];
 
 // ============================================================
 // PROXY TYPES
@@ -61,6 +65,18 @@ export interface Proxy {
   failureCount: number;
   successCount: number;
   avgResponseTimeMs: number | null;
+  maxConnections: number;
+  /**
+   * Per workflow-12102025.md: Track consecutive 403s with different fingerprints.
+   * After 3 consecutive 403s → disable proxy (it's burned).
+   */
+  consecutive403Count: number;
+  // Location info - determines session headers per workflow-12102025.md
+  city?: string;
+  state?: string;
+  country?: string;
+  countryCode?: string;
+  timezone?: string;
 }
 
 export interface ProxyStats {
@@ -70,6 +86,40 @@ export interface ProxyStats {
   avgSuccessRate: number;
 }
 
+// ============================================================
+// FINGERPRINT TYPE
+// Per workflow-12102025.md: Full browser fingerprint from user-agents
+// ============================================================
+
+export interface BrowserFingerprint {
+  userAgent: string;
+  platform: string;
+  screenWidth: number;
+  screenHeight: number;
+  viewportWidth: number;
+  viewportHeight: number;
+  deviceCategory: string;
+  browserName: string;  // Per workflow-12102025.md: for session logging
+  // Derived headers for anti-detect
+  acceptLanguage: string;
+  secChUa?: string;
+  secChUaPlatform?: string;
+  secChUaMobile?: string;
+  // Per workflow-12102025.md: HTTP Fingerprinting section
+  httpFingerprint: HTTPFingerprint;
+}
+
+/**
+ * Per workflow-12102025.md: Session log entry for debugging blocked sessions
+ */
+export interface UASessionLog {
+  deviceCategory: string;
+  browserName: string;
+  userAgent: string;
+  proxyIp: string | null;
+  sessionStartedAt: Date;
+}
+
 // ============================================================
 // PROXY ROTATOR CLASS
 // ============================================================
@@ -84,9 +134,6 @@ export class ProxyRotator {
     this.pool = pool || null;
   }
 
-  /**
-   * Initialize with database pool
-   */
   setPool(pool: Pool): void {
     this.pool = pool;
   }
@@ -109,20 +156,28 @@ export class ProxyRotator {
           username,
           password,
           protocol,
-          is_active as "isActive",
+          active as "isActive",
-          last_used_at as "lastUsedAt",
+          last_tested_at as "lastUsedAt",
           failure_count as "failureCount",
-          success_count as "successCount",
+          0 as "successCount",
-          avg_response_time_ms as "avgResponseTimeMs"
+          response_time_ms as "avgResponseTimeMs",
+          COALESCE(max_connections, 1) as "maxConnections",
+          COALESCE(consecutive_403_count, 0) as "consecutive403Count",
+          city,
+          state,
+          country,
+          country_code as "countryCode",
+          timezone
         FROM proxies
-        WHERE is_active = true
+        WHERE active = true
-        ORDER BY failure_count ASC, last_used_at ASC NULLS FIRST
+        ORDER BY failure_count ASC, last_tested_at ASC NULLS FIRST
       `);
 
       this.proxies = result.rows;
-      console.log(`[ProxyRotator] Loaded ${this.proxies.length} active proxies`);
+
+      const totalCapacity = this.proxies.reduce((sum, p) => sum + p.maxConnections, 0);
+      console.log(`[ProxyRotator] Loaded ${this.proxies.length} active proxies (${totalCapacity} max concurrent connections)`);
     } catch (error) {
-      // Table might not exist - that's okay
       console.warn(`[ProxyRotator] Could not load proxies: ${error}`);
       this.proxies = [];
     }
@@ -134,7 +189,6 @@ export class ProxyRotator {
   getNext(): Proxy | null {
     if (this.proxies.length === 0) return null;
 
-    // Round-robin rotation
     this.currentIndex = (this.currentIndex + 1) % this.proxies.length;
     this.lastRotation = new Date();
 
@@ -169,19 +223,27 @@ export class ProxyRotator {
   }
 
   /**
-   * Mark proxy as failed (temporarily remove from rotation)
+   * Mark proxy as blocked (403 received)
+   * Per workflow-12102025.md:
+   * - Increment consecutive_403_count
+   * - After 3 consecutive 403s with different fingerprints → disable proxy
+   * - This is separate from general failures (timeouts, etc.)
    */
-  async markFailed(proxyId: number, error?: string): Promise<void> {
+  async markBlocked(proxyId: number): Promise<boolean> {
-    // Update in-memory
     const proxy = this.proxies.find(p => p.id === proxyId);
-    if (proxy) {
+    let shouldDisable = false;
-      proxy.failureCount++;
 
-      // Deactivate if too many failures
+    if (proxy) {
-      if (proxy.failureCount >= 5) {
+      proxy.consecutive403Count++;
+
+      // Per workflow-12102025.md: 3 consecutive 403s → proxy is burned
+      if (proxy.consecutive403Count >= 3) {
         proxy.isActive = false;
         this.proxies = this.proxies.filter(p => p.id !== proxyId);
-        console.log(`[ProxyRotator] Proxy ${proxyId} deactivated after ${proxy.failureCount} failures`);
+        console.log(`[ProxyRotator] Proxy ${proxyId} DISABLED after ${proxy.consecutive403Count} consecutive 403s (burned)`);
+        shouldDisable = true;
+      } else {
+        console.log(`[ProxyRotator] Proxy ${proxyId} blocked (403 #${proxy.consecutive403Count}/3)`);
       }
     }
 
@@ -191,12 +253,49 @@ export class ProxyRotator {
         await this.pool.query(`
           UPDATE proxies
           SET
-            failure_count = failure_count + 1,
+            consecutive_403_count = COALESCE(consecutive_403_count, 0) + 1,
             last_failure_at = NOW(),
-            last_error = $2,
+            test_result = '403 Forbidden',
-            is_active = CASE WHEN failure_count >= 4 THEN false ELSE is_active END
+            active = CASE WHEN COALESCE(consecutive_403_count, 0) >= 2 THEN false ELSE active END,
+            updated_at = NOW()
           WHERE id = $1
-        `, [proxyId, error || null]);
+        `, [proxyId]);
+      } catch (err) {
+        console.error(`[ProxyRotator] Failed to update proxy ${proxyId}:`, err);
+      }
+    }
+
+    return shouldDisable;
+  }
+
+  /**
+   * Mark proxy as failed (general error - timeout, connection error, etc.)
+   * Separate from 403 blocking per workflow-12102025.md
+   */
+  async markFailed(proxyId: number, error?: string): Promise<void> {
+    const proxy = this.proxies.find(p => p.id === proxyId);
+    if (proxy) {
+      proxy.failureCount++;
+
+      // Deactivate if too many general failures
+      if (proxy.failureCount >= 5) {
+        proxy.isActive = false;
+        this.proxies = this.proxies.filter(p => p.id !== proxyId);
+        console.log(`[ProxyRotator] Proxy ${proxyId} deactivated after ${proxy.failureCount} general failures`);
+      }
+    }
+
+    if (this.pool) {
+      try {
+        await this.pool.query(`
+          UPDATE proxies
+          SET
+            failure_count = failure_count + 1,
+            updated_at = NOW(),
+            test_result = $2,
+            active = CASE WHEN failure_count >= 4 THEN false ELSE active END
+          WHERE id = $1
+        `, [proxyId, error || 'failed']);
       } catch (err) {
         console.error(`[ProxyRotator] Failed to update proxy ${proxyId}:`, err);
       }
@@ -204,34 +303,35 @@ export class ProxyRotator {
   }
 
   /**
-   * Mark proxy as successful
+   * Mark proxy as successful - resets consecutive 403 count
+   * Per workflow-12102025.md: successful request clears the 403 counter
    */
   async markSuccess(proxyId: number, responseTimeMs?: number): Promise<void> {
-    // Update in-memory
     const proxy = this.proxies.find(p => p.id === proxyId);
     if (proxy) {
       proxy.successCount++;
+      proxy.consecutive403Count = 0; // Reset on success per workflow-12102025.md
       proxy.lastUsedAt = new Date();
       if (responseTimeMs !== undefined) {
-        // Rolling average
         proxy.avgResponseTimeMs = proxy.avgResponseTimeMs
           ? (proxy.avgResponseTimeMs * 0.8) + (responseTimeMs * 0.2)
           : responseTimeMs;
       }
     }
 
-    // Update database
     if (this.pool) {
       try {
         await this.pool.query(`
           UPDATE proxies
           SET
-            success_count = success_count + 1,
+            last_tested_at = NOW(),
-            last_used_at = NOW(),
+            test_result = 'success',
-            avg_response_time_ms = CASE
+            consecutive_403_count = 0,
-              WHEN avg_response_time_ms IS NULL THEN $2
+            response_time_ms = CASE
-              ELSE (avg_response_time_ms * 0.8) + ($2 * 0.2)
+              WHEN response_time_ms IS NULL THEN $2
-            END
+              ELSE (response_time_ms * 0.8 + $2 * 0.2)::integer
+            END,
+            updated_at = NOW()
           WHERE id = $1
         `, [proxyId, responseTimeMs || null]);
       } catch (err) {
@@ -255,8 +355,8 @@ export class ProxyRotator {
    */
   getStats(): ProxyStats {
     const totalProxies = this.proxies.length;
-    const activeProxies = this.proxies.filter(p => p.isActive).length;
+    const activeProxies = this.proxies.reduce((sum, p) => sum + p.maxConnections, 0);
-    const blockedProxies = this.proxies.filter(p => p.failureCount >= 5).length;
+    const blockedProxies = this.proxies.filter(p => p.failureCount >= 5 || p.consecutive403Count >= 3).length;
 
     const successRates = this.proxies
       .filter(p => p.successCount + p.failureCount > 0)
@@ -274,9 +374,6 @@ export class ProxyRotator {
     };
   }
 
-  /**
-   * Check if proxy pool has available proxies
-   */
   hasAvailableProxies(): boolean {
     return this.proxies.length > 0;
   }
@@ -284,53 +381,194 @@ export class ProxyRotator {
 
 // ============================================================
 // USER AGENT ROTATOR CLASS
+// Per workflow-12102025.md: Uses intoli/user-agents for realistic fingerprints
 // ============================================================
 
 export class UserAgentRotator {
-  private userAgents: string[];
+  private currentFingerprint: BrowserFingerprint | null = null;
-  private currentIndex: number = 0;
+  private sessionLog: UASessionLog | null = null;
-  private lastRotation: Date = new Date();
 
-  constructor(userAgents: string[] = USER_AGENTS) {
+  constructor() {
-    this.userAgents = userAgents;
+    // Per workflow-12102025.md: Initialize with first fingerprint
-    // Start at random index to avoid patterns
+    this.rotate();
-    this.currentIndex = Math.floor(Math.random() * userAgents.length);
   }
 
   /**
-   * Get next user agent in rotation
+   * Per workflow-12102025.md: Roll device category based on distribution
+   * Mobile: 62%, Desktop: 36%, Tablet: 2%
    */
-  getNext(): string {
+  private rollDeviceCategory(): 'mobile' | 'desktop' | 'tablet' {
-    this.currentIndex = (this.currentIndex + 1) % this.userAgents.length;
+    const roll = Math.random() * 100;
-    this.lastRotation = new Date();
+    if (roll < DEVICE_WEIGHTS.mobile) {
-    return this.userAgents[this.currentIndex];
+      return 'mobile';
+    } else if (roll < DEVICE_WEIGHTS.mobile + DEVICE_WEIGHTS.desktop) {
+      return 'desktop';
+    } else {
+      return 'tablet';
+    }
   }
 
   /**
-   * Get current user agent without rotating
+   * Per workflow-12102025.md: Extract browser name from UA string
    */
-  getCurrent(): string {
+  private extractBrowserName(userAgent: string): string {
-    return this.userAgents[this.currentIndex];
+    if (userAgent.includes('Edg/')) return 'Edge';
+    if (userAgent.includes('Firefox/')) return 'Firefox';
+    if (userAgent.includes('Safari/') && !userAgent.includes('Chrome/')) return 'Safari';
+    if (userAgent.includes('Chrome/')) return 'Chrome';
+    return 'Unknown';
   }
 
   /**
-   * Get a random user agent
+   * Per workflow-12102025.md: Check if browser is in whitelist
    */
-  getRandom(): string {
+  private isAllowedBrowser(userAgent: string): boolean {
-    const index = Math.floor(Math.random() * this.userAgents.length);
+    const browserName = this.extractBrowserName(userAgent);
-    return this.userAgents[index];
+    return ALLOWED_BROWSERS.includes(browserName as typeof ALLOWED_BROWSERS[number]);
   }
 
   /**
-   * Get total available user agents
+   * Generate a new random fingerprint
+   * Per workflow-12102025.md:
+   * - Roll device category (62/36/2)
+   * - Filter to top 4 browsers only
+   * - Failure = alert admin + stop (no fallback)
    */
+  rotate(proxyIp?: string): BrowserFingerprint {
+    // Per workflow-12102025.md: Roll device category
+    const deviceCategory = this.rollDeviceCategory();
+
+    // Per workflow-12102025.md: Generate UA filtered to device category
+    const generator = new UserAgent({ deviceCategory });
+
+    // Per workflow-12102025.md: Try to get an allowed browser (max 50 attempts)
+    let ua: ReturnType<typeof generator>;
+    let attempts = 0;
+    const maxAttempts = 50;
+
+    do {
+      ua = generator();
+      attempts++;
+    } while (!this.isAllowedBrowser(ua.data.userAgent) && attempts < maxAttempts);
+
+    // Per workflow-12102025.md: If we can't get allowed browser, this is a failure
+    if (!this.isAllowedBrowser(ua.data.userAgent)) {
+      const errorMsg = `[UserAgentRotator] CRITICAL: Failed to generate allowed browser after ${maxAttempts} attempts. Device: ${deviceCategory}. Last UA: ${ua.data.userAgent}`;
+      console.error(errorMsg);
+      // Per workflow-12102025.md: Alert admin + stop crawl
+      // TODO: Post alert to admin dashboard
+      throw new Error(errorMsg);
+    }
+
+    const data = ua.data;
+    const browserName = this.extractBrowserName(data.userAgent);
+
+    // Build sec-ch-ua headers from user agent string
+    const secChUa = this.buildSecChUa(data.userAgent, deviceCategory);
+
+    // Per workflow-12102025.md: HTTP Fingerprinting - generate full HTTP fingerprint
+    const httpFingerprint = generateHTTPFingerprint(browserName as BrowserType);
+
+    this.currentFingerprint = {
+      userAgent: data.userAgent,
+      platform: data.platform,
+      screenWidth: data.screenWidth,
+      screenHeight: data.screenHeight,
+      viewportWidth: data.viewportWidth,
+      viewportHeight: data.viewportHeight,
+      deviceCategory: data.deviceCategory,
+      browserName,  // Per workflow-12102025.md: for session logging
+      // Per workflow-12102025.md: always English
+      acceptLanguage: 'en-US,en;q=0.9',
+      ...secChUa,
+      // Per workflow-12102025.md: HTTP Fingerprinting section
+      httpFingerprint,
+    };
+
+    // Per workflow-12102025.md: Log session data
+    this.sessionLog = {
+      deviceCategory,
+      browserName,
+      userAgent: data.userAgent,
+      proxyIp: proxyIp || null,
+      sessionStartedAt: new Date(),
+    };
+
+    console.log(`[UserAgentRotator] New fingerprint: device=${deviceCategory}, browser=${browserName}, UA=${data.userAgent.slice(0, 50)}...`);
+    return this.currentFingerprint;
+  }
+
+  /**
+   * Get current fingerprint without rotating
+   */
+  getCurrent(): BrowserFingerprint {
+    if (!this.currentFingerprint) {
+      return this.rotate();
+    }
+    return this.currentFingerprint;
+  }
+
+  /**
+   * Get a random fingerprint (rotates and returns)
+   */
+  getRandom(proxyIp?: string): BrowserFingerprint {
+    return this.rotate(proxyIp);
+  }
+
+  /**
+   * Per workflow-12102025.md: Get session log for debugging
+   */
+  getSessionLog(): UASessionLog | null {
+    return this.sessionLog;
+  }
+
+  /**
+   * Build sec-ch-ua headers from user agent string
+   * Per workflow-12102025.md: Include mobile indicator based on device category
+   */
+  private buildSecChUa(userAgent: string, deviceCategory: string): { secChUa?: string; secChUaPlatform?: string; secChUaMobile?: string } {
+    const isMobile = deviceCategory === 'mobile' || deviceCategory === 'tablet';
+
+    // Extract Chrome version if present
+    const chromeMatch = userAgent.match(/Chrome\/(\d+)/);
+    const edgeMatch = userAgent.match(/Edg\/(\d+)/);
+
+    if (edgeMatch) {
+      const version = edgeMatch[1];
+      return {
+        secChUa: `"Microsoft Edge";v="${version}", "Chromium";v="${version}", "Not_A Brand";v="24"`,
+        secChUaPlatform: userAgent.includes('Windows') ? '"Windows"' : userAgent.includes('Android') ? '"Android"' : '"macOS"',
+        secChUaMobile: isMobile ? '?1' : '?0',
+      };
+    }
+
+    if (chromeMatch) {
+      const version = chromeMatch[1];
+      let platform = '"Linux"';
+      if (userAgent.includes('Windows')) platform = '"Windows"';
+      else if (userAgent.includes('Mac')) platform = '"macOS"';
+      else if (userAgent.includes('Android')) platform = '"Android"';
+      else if (userAgent.includes('iPhone') || userAgent.includes('iPad')) platform = '"iOS"';
+
+      return {
+        secChUa: `"Google Chrome";v="${version}", "Chromium";v="${version}", "Not_A Brand";v="24"`,
+        secChUaPlatform: platform,
+        secChUaMobile: isMobile ? '?1' : '?0',
+      };
+    }
+
+    // Firefox/Safari don't send sec-ch-ua
+    return {};
+  }
+
   getCount(): number {
-    return this.userAgents.length;
+    return 1; // user-agents generates dynamically
   }
 }
 
 // ============================================================
-// COMBINED ROTATOR (for convenience)
+// COMBINED ROTATOR
+// Per workflow-12102025.md: Coordinates proxy + fingerprint rotation
 // ============================================================
 
 export class CrawlRotator {
@@ -342,49 +580,51 @@ export class CrawlRotator {
     this.userAgent = new UserAgentRotator();
   }
 
-  /**
-   * Initialize rotator (load proxies from DB)
-   */
   async initialize(): Promise<void> {
     await this.proxy.loadProxies();
   }
 
   /**
-   * Rotate proxy only
+   * Rotate proxy only (get new IP)
    */
   rotateProxy(): Proxy | null {
     return this.proxy.getNext();
   }
 
   /**
-   * Rotate user agent only
+   * Rotate fingerprint only (new UA, screen size, etc.)
    */
-  rotateUserAgent(): string {
+  rotateFingerprint(): BrowserFingerprint {
-    return this.userAgent.getNext();
+    return this.userAgent.rotate();
   }
 
   /**
-   * Rotate both proxy and user agent
+   * Rotate both proxy and fingerprint
+   * Per workflow-12102025.md: called on 403 for fresh identity
+   * Passes proxy IP to UA rotation for session logging
    */
-  rotateBoth(): { proxy: Proxy | null; userAgent: string } {
+  rotateBoth(): { proxy: Proxy | null; fingerprint: BrowserFingerprint } {
+    const proxy = this.proxy.getNext();
+    const proxyIp = proxy ? proxy.host : undefined;
     return {
-      proxy: this.proxy.getNext(),
+      proxy,
-      userAgent: this.userAgent.getNext(),
+      fingerprint: this.userAgent.rotate(proxyIp),
     };
   }
 
   /**
-   * Get current proxy and user agent without rotating
+   * Get current proxy and fingerprint without rotating
    */
-  getCurrent(): { proxy: Proxy | null; userAgent: string } {
+  getCurrent(): { proxy: Proxy | null; fingerprint: BrowserFingerprint } {
     return {
       proxy: this.proxy.getCurrent(),
-      userAgent: this.userAgent.getCurrent(),
+      fingerprint: this.userAgent.getCurrent(),
     };
   }
 
   /**
    * Record success for current proxy
+   * Per workflow-12102025.md: resets consecutive 403 count
    */
   async recordSuccess(responseTimeMs?: number): Promise<void> {
     const current = this.proxy.getCurrent();
@@ -394,7 +634,20 @@ export class CrawlRotator {
   }
 
   /**
-   * Record failure for current proxy
+   * Record 403 block for current proxy
+   * Per workflow-12102025.md: increments consecutive_403_count, disables after 3
+   * Returns true if proxy was disabled
+   */
+  async recordBlock(): Promise<boolean> {
+    const current = this.proxy.getCurrent();
+    if (current) {
+      return await this.proxy.markBlocked(current.id);
+    }
+    return false;
+  }
+
+  /**
+   * Record general failure (not 403)
    */
   async recordFailure(error?: string): Promise<void> {
     const current = this.proxy.getCurrent();
@@ -402,6 +655,34 @@ export class CrawlRotator {
       await this.proxy.markFailed(current.id, error);
     }
   }
+
+  /**
+   * Get current proxy location info
+   * Per workflow-12102025.md: proxy location determines session headers
+   */
+  getProxyLocation(): { city?: string; state?: string; country?: string; timezone?: string; isRotating: boolean } | null {
+    const current = this.proxy.getCurrent();
+    if (!current) return null;
+
+    const isRotating = current.maxConnections > 1;
+
+    return {
+      city: current.city,
+      state: current.state,
+      country: current.country,
+      timezone: current.timezone,
+      isRotating
+    };
+  }
+
+  /**
+   * Get timezone from current proxy
+   * Per workflow-12102025.md: used for Accept-Language header
+   */
+  getProxyTimezone(): string | undefined {
+    const current = this.proxy.getCurrent();
+    return current?.timezone;
+  }
 }
 
 // ============================================================

backend/src/services/http-fingerprint.ts

@@ -0,0 +1,315 @@
+/**
+ * HTTP Fingerprinting Service
+ *
+ * Per workflow-12102025.md - HTTP Fingerprinting section:
+ * - Full header set per browser type
+ * - Browser-specific header ordering
+ * - Natural randomization (DNT, Accept quality)
+ * - Dynamic Referer per dispensary
+ *
+ * Canonical location: src/services/http-fingerprint.ts
+ */
+
+// ============================================================
+// TYPES
+// ============================================================
+
+export type BrowserType = 'Chrome' | 'Firefox' | 'Safari' | 'Edge';
+
+/**
+ * Per workflow-12102025.md: Full HTTP fingerprint for a session
+ */
+export interface HTTPFingerprint {
+  browserType: BrowserType;
+  headers: Record<string, string>;
+  headerOrder: string[];
+  curlImpersonateBinary: string;
+  hasDNT: boolean;
+}
+
+/**
+ * Per workflow-12102025.md: Context for building headers
+ */
+export interface HeaderContext {
+  userAgent: string;
+  secChUa?: string;
+  secChUaPlatform?: string;
+  secChUaMobile?: string;
+  referer: string;
+  isPost: boolean;
+  contentLength?: number;
+}
+
+// ============================================================
+// CONSTANTS (per workflow-12102025.md)
+// ============================================================
+
+/**
+ * Per workflow-12102025.md: DNT header distribution (~30% of users)
+ */
+const DNT_PROBABILITY = 0.30;
+
+/**
+ * Per workflow-12102025.md: Accept header variations for natural traffic
+ */
+const ACCEPT_VARIATIONS = [
+  'application/json, text/plain, */*',
+  'application/json,text/plain,*/*',
+  '*/*',
+];
+
+/**
+ * Per workflow-12102025.md: Accept-Language variations
+ */
+const ACCEPT_LANGUAGE_VARIATIONS = [
+  'en-US,en;q=0.9',
+  'en-US,en;q=0.8',
+  'en-US;q=0.9,en;q=0.8',
+];
+
+/**
+ * Per workflow-12102025.md: curl-impersonate binaries per browser
+ */
+const CURL_IMPERSONATE_BINARIES: Record<BrowserType, string> = {
+  Chrome: 'curl_chrome131',
+  Edge: 'curl_chrome131',  // Edge uses Chromium
+  Firefox: 'curl_ff133',
+  Safari: 'curl_safari17',
+};
+
+// ============================================================
+// HEADER ORDERING (per workflow-12102025.md)
+// ============================================================
+
+/**
+ * Per workflow-12102025.md: Chrome header order for GraphQL requests
+ */
+const CHROME_HEADER_ORDER = [
+  'Host',
+  'Connection',
+  'Content-Length',
+  'sec-ch-ua',
+  'DNT',
+  'sec-ch-ua-mobile',
+  'User-Agent',
+  'sec-ch-ua-platform',
+  'Content-Type',
+  'Accept',
+  'Origin',
+  'sec-fetch-site',
+  'sec-fetch-mode',
+  'sec-fetch-dest',
+  'Referer',
+  'Accept-Encoding',
+  'Accept-Language',
+];
+
+/**
+ * Per workflow-12102025.md: Firefox header order for GraphQL requests
+ */
+const FIREFOX_HEADER_ORDER = [
+  'Host',
+  'User-Agent',
+  'Accept',
+  'Accept-Language',
+  'Accept-Encoding',
+  'Content-Type',
+  'Content-Length',
+  'Origin',
+  'DNT',
+  'Connection',
+  'Referer',
+  'sec-fetch-dest',
+  'sec-fetch-mode',
+  'sec-fetch-site',
+];
+
+/**
+ * Per workflow-12102025.md: Safari header order for GraphQL requests
+ */
+const SAFARI_HEADER_ORDER = [
+  'Host',
+  'Connection',
+  'Content-Length',
+  'Accept',
+  'User-Agent',
+  'Content-Type',
+  'Origin',
+  'Referer',
+  'Accept-Encoding',
+  'Accept-Language',
+];
+
+/**
+ * Per workflow-12102025.md: Edge uses Chrome order (Chromium-based)
+ */
+const HEADER_ORDERS: Record<BrowserType, string[]> = {
+  Chrome: CHROME_HEADER_ORDER,
+  Edge: CHROME_HEADER_ORDER,
+  Firefox: FIREFOX_HEADER_ORDER,
+  Safari: SAFARI_HEADER_ORDER,
+};
+
+// ============================================================
+// FINGERPRINT GENERATION
+// ============================================================
+
+/**
+ * Per workflow-12102025.md: Generate HTTP fingerprint for a session
+ * Randomization is done once per session for consistency
+ */
+export function generateHTTPFingerprint(browserType: BrowserType): HTTPFingerprint {
+  // Per workflow-12102025.md: DNT randomized per session (~30%)
+  const hasDNT = Math.random() < DNT_PROBABILITY;
+
+  return {
+    browserType,
+    headers: {},  // Built dynamically per request
+    headerOrder: HEADER_ORDERS[browserType],
+    curlImpersonateBinary: CURL_IMPERSONATE_BINARIES[browserType],
+    hasDNT,
+  };
+}
+
+/**
+ * Per workflow-12102025.md: Build complete headers for a request
+ * Returns headers in browser-specific order
+ */
+export function buildOrderedHeaders(
+  fingerprint: HTTPFingerprint,
+  context: HeaderContext
+): { headers: Record<string, string>; orderedHeaders: string[] } {
+  const { browserType, hasDNT, headerOrder } = fingerprint;
+  const { userAgent, secChUa, secChUaPlatform, secChUaMobile, referer, isPost, contentLength } = context;
+
+  // Per workflow-12102025.md: Natural randomization for Accept
+  const accept = ACCEPT_VARIATIONS[Math.floor(Math.random() * ACCEPT_VARIATIONS.length)];
+  const acceptLanguage = ACCEPT_LANGUAGE_VARIATIONS[Math.floor(Math.random() * ACCEPT_LANGUAGE_VARIATIONS.length)];
+
+  // Build all possible headers
+  const allHeaders: Record<string, string> = {
+    'Connection': 'keep-alive',
+    'User-Agent': userAgent,
+    'Accept': accept,
+    'Accept-Language': acceptLanguage,
+    'Accept-Encoding': 'gzip, deflate, br',
+  };
+
+  // Per workflow-12102025.md: POST-only headers
+  if (isPost) {
+    allHeaders['Content-Type'] = 'application/json';
+    allHeaders['Origin'] = 'https://dutchie.com';
+    if (contentLength !== undefined) {
+      allHeaders['Content-Length'] = String(contentLength);
+    }
+  }
+
+  // Per workflow-12102025.md: Dynamic Referer per dispensary
+  allHeaders['Referer'] = referer;
+
+  // Per workflow-12102025.md: DNT randomized per session
+  if (hasDNT) {
+    allHeaders['DNT'] = '1';
+  }
+
+  // Per workflow-12102025.md: Chromium-only headers (Chrome, Edge)
+  if (browserType === 'Chrome' || browserType === 'Edge') {
+    if (secChUa) allHeaders['sec-ch-ua'] = secChUa;
+    if (secChUaMobile) allHeaders['sec-ch-ua-mobile'] = secChUaMobile;
+    if (secChUaPlatform) allHeaders['sec-ch-ua-platform'] = secChUaPlatform;
+    allHeaders['sec-fetch-site'] = 'same-origin';
+    allHeaders['sec-fetch-mode'] = 'cors';
+    allHeaders['sec-fetch-dest'] = 'empty';
+  }
+
+  // Per workflow-12102025.md: Firefox has sec-fetch but no sec-ch
+  if (browserType === 'Firefox') {
+    allHeaders['sec-fetch-site'] = 'same-origin';
+    allHeaders['sec-fetch-mode'] = 'cors';
+    allHeaders['sec-fetch-dest'] = 'empty';
+  }
+
+  // Per workflow-12102025.md: Safari has no sec-* headers
+
+  // Filter to only headers that exist and order them
+  const orderedHeaders: string[] = [];
+  const headers: Record<string, string> = {};
+
+  for (const headerName of headerOrder) {
+    if (allHeaders[headerName]) {
+      orderedHeaders.push(headerName);
+      headers[headerName] = allHeaders[headerName];
+    }
+  }
+
+  return { headers, orderedHeaders };
+}
+
+/**
+ * Per workflow-12102025.md: Build curl command arguments for headers
+ * Headers are added in browser-specific order
+ */
+export function buildCurlHeaderArgs(
+  fingerprint: HTTPFingerprint,
+  context: HeaderContext
+): string[] {
+  const { headers, orderedHeaders } = buildOrderedHeaders(fingerprint, context);
+
+  const args: string[] = [];
+  for (const headerName of orderedHeaders) {
+    // Skip Host and Content-Length - curl handles these
+    if (headerName === 'Host' || headerName === 'Content-Length') continue;
+    args.push('-H', `${headerName}: ${headers[headerName]}`);
+  }
+
+  return args;
+}
+
+/**
+ * Per workflow-12102025.md: Extract Referer from dispensary menu_url
+ */
+export function buildRefererFromMenuUrl(menuUrl: string | null | undefined): string {
+  if (!menuUrl) {
+    return 'https://dutchie.com/';
+  }
+
+  // Extract slug from menu_url
+  // Formats: /embedded-menu/<slug> or /dispensary/<slug> or full URL
+  let slug: string | null = null;
+
+  const embeddedMatch = menuUrl.match(/\/embedded-menu\/([^/?]+)/);
+  const dispensaryMatch = menuUrl.match(/\/dispensary\/([^/?]+)/);
+
+  if (embeddedMatch) {
+    slug = embeddedMatch[1];
+  } else if (dispensaryMatch) {
+    slug = dispensaryMatch[1];
+  }
+
+  if (slug) {
+    return `https://dutchie.com/dispensary/${slug}`;
+  }
+
+  return 'https://dutchie.com/';
+}
+
+/**
+ * Per workflow-12102025.md: Get curl-impersonate binary for browser
+ */
+export function getCurlBinary(browserType: BrowserType): string {
+  return CURL_IMPERSONATE_BINARIES[browserType];
+}
+
+/**
+ * Per workflow-12102025.md: Check if curl-impersonate is available
+ */
+export function isCurlImpersonateAvailable(browserType: BrowserType): boolean {
+  const binary = CURL_IMPERSONATE_BINARIES[browserType];
+  try {
+    const { execSync } = require('child_process');
+    execSync(`which ${binary}`, { stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}

backend/src/services/ip2location.ts

@@ -0,0 +1,134 @@
+/**
+ * IP2Location Service
+ *
+ * Uses local IP2Location LITE DB3 database for IP geolocation.
+ * No external API calls, no rate limits.
+ *
+ * Database: IP2Location LITE DB3 (free, monthly updates)
+ * Fields: country, region, city, latitude, longitude
+ */
+
+import path from 'path';
+import fs from 'fs';
+
+// @ts-ignore - no types for ip2location-nodejs
+const { IP2Location } = require('ip2location-nodejs');
+
+const DB_PATH = process.env.IP2LOCATION_DB_PATH ||
+  path.join(__dirname, '../../data/ip2location/IP2LOCATION-LITE-DB5.BIN');
+
+let ip2location: any = null;
+let dbLoaded = false;
+
+/**
+ * Initialize IP2Location database
+ */
+export function initIP2Location(): boolean {
+  if (dbLoaded) return true;
+
+  try {
+    if (!fs.existsSync(DB_PATH)) {
+      console.warn(`IP2Location database not found at: ${DB_PATH}`);
+      console.warn('Run: ./scripts/download-ip2location.sh to download');
+      return false;
+    }
+
+    ip2location = new IP2Location();
+    ip2location.open(DB_PATH);
+    dbLoaded = true;
+    console.log('IP2Location database loaded successfully');
+    return true;
+  } catch (err) {
+    console.error('Failed to load IP2Location database:', err);
+    return false;
+  }
+}
+
+/**
+ * Close IP2Location database
+ */
+export function closeIP2Location(): void {
+  if (ip2location) {
+    ip2location.close();
+    ip2location = null;
+    dbLoaded = false;
+  }
+}
+
+export interface GeoLocation {
+  city: string | null;
+  state: string | null;
+  stateCode: string | null;
+  country: string | null;
+  countryCode: string | null;
+  lat: number | null;
+  lng: number | null;
+}
+
+/**
+ * Lookup IP address location
+ *
+ * @param ip - IPv4 or IPv6 address
+ * @returns Location data or null if not found
+ */
+export function lookupIP(ip: string): GeoLocation | null {
+  // Skip private/localhost IPs
+  if (!ip || ip === '127.0.0.1' || ip === '::1' ||
+      ip.startsWith('192.168.') || ip.startsWith('10.') ||
+      ip.startsWith('172.16.') || ip.startsWith('172.17.') ||
+      ip.startsWith('::ffff:127.') || ip.startsWith('::ffff:192.168.') ||
+      ip.startsWith('::ffff:10.')) {
+    return null;
+  }
+
+  // Strip IPv6 prefix if present
+  const cleanIP = ip.replace(/^::ffff:/, '');
+
+  // Initialize on first use if not already loaded
+  if (!dbLoaded) {
+    if (!initIP2Location()) {
+      return null;
+    }
+  }
+
+  try {
+    const result = ip2location.getAll(cleanIP);
+
+    if (!result || result.ip === '?' || result.countryShort === '-') {
+      return null;
+    }
+
+    // DB3 LITE doesn't include lat/lng - would need DB5+ for that
+    const lat = typeof result.latitude === 'number' && result.latitude !== 0 ? result.latitude : null;
+    const lng = typeof result.longitude === 'number' && result.longitude !== 0 ? result.longitude : null;
+
+    return {
+      city: result.city !== '-' ? result.city : null,
+      state: result.region !== '-' ? result.region : null,
+      stateCode: null, // DB3 doesn't include state codes
+      country: result.countryLong !== '-' ? result.countryLong : null,
+      countryCode: result.countryShort !== '-' ? result.countryShort : null,
+      lat,
+      lng,
+    };
+  } catch (err) {
+    console.error('IP2Location lookup error:', err);
+    return null;
+  }
+}
+
+/**
+ * Check if IP2Location database is available
+ */
+export function isIP2LocationAvailable(): boolean {
+  if (dbLoaded) return true;
+  return fs.existsSync(DB_PATH);
+}
+
+// Export singleton-style interface
+export default {
+  init: initIP2Location,
+  close: closeIP2Location,
+  lookup: lookupIP,
+  isAvailable: isIP2LocationAvailable,
+};

backend/src/services/proxy.ts

@@ -276,7 +276,6 @@ export async function addProxiesFromList(proxies: Array<{
       await pool.query(`
         INSERT INTO proxies (host, port, protocol, username, password, active)
         VALUES ($1, $2, $3, $4, $5, false)
-        ON CONFLICT (host, port, protocol) DO NOTHING
       `, [
         proxy.host,
         proxy.port,
@@ -285,28 +284,10 @@ export async function addProxiesFromList(proxies: Array<{
         proxy.password
       ]);
 
-      // Check if it was actually inserted
-      const result = await pool.query(`
-        SELECT id FROM proxies
-        WHERE host = $1 AND port = $2 AND protocol = $3
-      `, [proxy.host, proxy.port, proxy.protocol]);
-
-      if (result.rows.length > 0) {
-        // Check if it was just inserted (no last_tested_at means new)
-        const checkResult = await pool.query(`
-          SELECT last_tested_at FROM proxies
-          WHERE host = $1 AND port = $2 AND protocol = $3
-        `, [proxy.host, proxy.port, proxy.protocol]);
-
-        if (checkResult.rows[0].last_tested_at === null) {
       added++;
       if (added % 100 === 0) {
         console.log(`📥 Imported ${added} proxies...`);
       }
-        } else {
-          duplicates++;
-        }
-      }
     } catch (error: any) {
       failed++;
       const errorMsg = `${proxy.host}:${proxy.port} - ${error.message}`;

backend/src/services/proxyTestQueue.ts

@@ -8,8 +8,12 @@ interface ProxyTestJob {
   tested_proxies: number;
   passed_proxies: number;
   failed_proxies: number;
+  mode?: string;  // 'all' | 'failed' | 'inactive'
 }
 
+// Concurrency settings
+const DEFAULT_CONCURRENCY = 10;  // Test 10 proxies at a time
+
 // Simple in-memory queue - could be replaced with Bull/Bee-Queue for production
 const activeJobs = new Map<number, { cancelled: boolean }>();
 
@@ -33,18 +37,40 @@ export async function cleanupOrphanedJobs(): Promise<void> {
   }
 }
 
-export async function createProxyTestJob(): Promise<number> {
+export type ProxyTestMode = 'all' | 'failed' | 'inactive';
+
+export interface CreateJobResult {
+  jobId: number;
+  totalProxies: number;
+}
+
+export async function createProxyTestJob(mode: ProxyTestMode = 'all', concurrency: number = DEFAULT_CONCURRENCY): Promise<CreateJobResult> {
   // Check for existing running jobs first
   const existingJob = await getActiveProxyTestJob();
   if (existingJob) {
     throw new Error('A proxy test job is already running. Please cancel it first.');
   }
-  const result = await pool.query(`
-    SELECT COUNT(*) as count FROM proxies
-  `);
 
+  // Get count based on mode
+  let countQuery: string;
+  switch (mode) {
+    case 'failed':
+      countQuery = `SELECT COUNT(*) as count FROM proxies WHERE test_result = 'failed' OR active = false`;
+      break;
+    case 'inactive':
+      countQuery = `SELECT COUNT(*) as count FROM proxies WHERE active = false`;
+      break;
+    default:
+      countQuery = `SELECT COUNT(*) as count FROM proxies`;
+  }
+
+  const result = await pool.query(countQuery);
   const totalProxies = parseInt(result.rows[0].count);
 
+  if (totalProxies === 0) {
+    throw new Error(`No proxies to test with mode '${mode}'`);
+  }
+
   const jobResult = await pool.query(`
     INSERT INTO proxy_test_jobs (status, total_proxies)
     VALUES ('pending', $1)
@@ -53,12 +79,12 @@ export async function createProxyTestJob(): Promise<number> {
 
   const jobId = jobResult.rows[0].id;
 
-  // Start job in background
+  // Start job in background with mode and concurrency
-  runProxyTestJob(jobId).catch(err => {
+  runProxyTestJob(jobId, mode, concurrency).catch(err => {
     console.error(`❌ Proxy test job ${jobId} failed:`, err);
   });
 
-  return jobId;
+  return { jobId, totalProxies };
 }
 
 export async function getProxyTestJob(jobId: number): Promise<ProxyTestJob | null> {
@@ -111,7 +137,7 @@ export async function cancelProxyTestJob(jobId: number): Promise<boolean> {
   return result.rows.length > 0;
 }
 
-async function runProxyTestJob(jobId: number): Promise<void> {
+async function runProxyTestJob(jobId: number, mode: ProxyTestMode = 'all', concurrency: number = DEFAULT_CONCURRENCY): Promise<void> {
   // Register job as active
   activeJobs.set(jobId, { cancelled: false });
 
@@ -125,20 +151,30 @@ async function runProxyTestJob(jobId: number): Promise<void> {
       WHERE id = $1
     `, [jobId]);
 
-    console.log(`🔍 Starting proxy test job ${jobId}...`);
+    console.log(`🔍 Starting proxy test job ${jobId} (mode: ${mode}, concurrency: ${concurrency})...`);
 
-    // Get all proxies
+    // Get proxies based on mode
-    const result = await pool.query(`
+    let query: string;
-      SELECT id, host, port, protocol, username, password
+    switch (mode) {
-      FROM proxies
+      case 'failed':
-      ORDER BY id
+        query = `SELECT id, host, port, protocol, username, password FROM proxies WHERE test_result = 'failed' OR active = false ORDER BY id`;
-    `);
+        break;
+      case 'inactive':
+        query = `SELECT id, host, port, protocol, username, password FROM proxies WHERE active = false ORDER BY id`;
+        break;
+      default:
+        query = `SELECT id, host, port, protocol, username, password FROM proxies ORDER BY id`;
+    }
+
+    const result = await pool.query(query);
+    const proxies = result.rows;
 
     let tested = 0;
     let passed = 0;
     let failed = 0;
 
-    for (const proxy of result.rows) {
+    // Process proxies in batches for parallel testing
+    for (let i = 0; i < proxies.length; i += concurrency) {
       // Check if job was cancelled
       const jobControl = activeJobs.get(jobId);
       if (jobControl?.cancelled) {
@@ -146,7 +182,11 @@ async function runProxyTestJob(jobId: number): Promise<void> {
         break;
       }
 
-      // Test the proxy
+      const batch = proxies.slice(i, i + concurrency);
+
+      // Test batch in parallel
+      const batchResults = await Promise.all(
+        batch.map(async (proxy) => {
           const testResult = await testProxy(
             proxy.host,
             proxy.port,
@@ -158,12 +198,19 @@ async function runProxyTestJob(jobId: number): Promise<void> {
           // Save result
           await saveProxyTestResult(proxy.id, testResult);
 
+          return testResult.success;
+        })
+      );
+
+      // Count results
+      for (const success of batchResults) {
         tested++;
-      if (testResult.success) {
+        if (success) {
           passed++;
         } else {
           failed++;
         }
+      }
 
       // Update job progress
       await pool.query(`
@@ -175,10 +222,8 @@ async function runProxyTestJob(jobId: number): Promise<void> {
         WHERE id = $4
       `, [tested, passed, failed, jobId]);
 
-      // Log progress every 10 proxies
+      // Log progress
-      if (tested % 10 === 0) {
+      console.log(`📊 Job ${jobId}: ${tested}/${proxies.length} proxies tested (${passed} passed, ${failed} failed)`);
-        console.log(`📊 Job ${jobId}: ${tested}/${result.rows.length} proxies tested (${passed} passed, ${failed} failed)`);
-      }
     }
 
     // Mark job as completed

backend/src/services/scheduler.ts

@@ -1,116 +1,38 @@
-import cron from 'node-cron';
+/**
-import { pool } from '../db/pool';
+ * LEGACY SCHEDULER - DEPRECATED 2024-12-10
-import { scrapeStore, scrapeCategory } from '../scraper-v2';
+ *
-
+ * DO NOT USE THIS FILE.
-let scheduledJobs: cron.ScheduledTask[] = [];
+ *
-
+ * Per TASK_WORKFLOW_2024-12-10.md:
-async function getSettings(): Promise<{
+ * This node-cron scheduler has been replaced by the database-driven
-  scrapeIntervalHours: number;
+ * task scheduler in src/services/task-scheduler.ts
-  scrapeSpecialsTime: string;
+ *
-}> {
+ * The new scheduler:
-  const result = await pool.query(`
+ * - Stores schedules in PostgreSQL (survives restarts)
-    SELECT key, value FROM settings
+ * - Uses SELECT FOR UPDATE SKIP LOCKED (multi-replica safe)
-    WHERE key IN ('scrape_interval_hours', 'scrape_specials_time')
+ * - Creates tasks in worker_tasks table (processed by task-worker.ts)
-  `);
+ *
-  
+ * This file is kept for reference only. All exports are no-ops.
-  const settings: Record<string, string> = {};
+ * Legacy code has been removed - see git history for original implementation.
-  result.rows.forEach((row: { key: string; value: string }) => {
+ */
-    settings[row.key] = row.value;
-  });
-  
-  return {
-    scrapeIntervalHours: parseInt(settings.scrape_interval_hours || '4'),
-    scrapeSpecialsTime: settings.scrape_specials_time || '00:01'
-  };
-}
-
-async function scrapeAllStores(): Promise<void> {
-  console.log('🔄 Starting scheduled scrape for all stores...');
-  
-  const result = await pool.query(`
-    SELECT id, name FROM stores WHERE active = true AND scrape_enabled = true
-  `);
-  
-  for (const store of result.rows) {
-    try {
-      console.log(`Scraping store: ${store.name}`);
-      await scrapeStore(store.id);
-    } catch (error) {
-      console.error(`Failed to scrape store ${store.name}:`, error);
-    }
-  }
-  
-  console.log('✅ Scheduled scrape completed');
-}
-
-async function scrapeSpecials(): Promise<void> {
-  console.log('🌟 Starting scheduled specials scrape...');
-  
-  const result = await pool.query(`
-    SELECT s.id, s.name, c.id as category_id
-    FROM stores s
-    JOIN categories c ON c.store_id = s.id
-    WHERE s.active = true AND s.scrape_enabled = true
-      AND c.slug = 'specials' AND c.scrape_enabled = true
-  `);
-  
-  for (const row of result.rows) {
-    try {
-      console.log(`Scraping specials for: ${row.name}`);
-      await scrapeCategory(row.id, row.category_id);
-    } catch (error) {
-      console.error(`Failed to scrape specials for ${row.name}:`, error);
-    }
-  }
-  
-  console.log('✅ Specials scrape completed');
-}
 
+// 2024-12-10: All functions are now no-ops
 export async function startScheduler(): Promise<void> {
-  // Stop any existing jobs
+  console.warn('[DEPRECATED] startScheduler() called - use taskScheduler from task-scheduler.ts instead');
-  stopScheduler();
-  
-  const settings = await getSettings();
-  
-  // Schedule regular store scrapes (every N hours)
-  const scrapeIntervalCron = `0 */${settings.scrapeIntervalHours} * * *`;
-  const storeJob = cron.schedule(scrapeIntervalCron, scrapeAllStores);
-  scheduledJobs.push(storeJob);
-  console.log(`📅 Scheduled store scraping: every ${settings.scrapeIntervalHours} hours`);
-  
-  // Schedule specials scraping (daily at specified time)
-  const [hours, minutes] = settings.scrapeSpecialsTime.split(':');
-  const specialsCron = `${minutes} ${hours} * * *`;
-  const specialsJob = cron.schedule(specialsCron, scrapeSpecials);
-  scheduledJobs.push(specialsJob);
-  console.log(`📅 Scheduled specials scraping: daily at ${settings.scrapeSpecialsTime}`);
-  
-  // Initial scrape on startup (after 10 seconds)
-  setTimeout(() => {
-    console.log('🚀 Running initial scrape...');
-    scrapeAllStores().catch(console.error);
-  }, 10000);
 }
 
 export function stopScheduler(): void {
-  scheduledJobs.forEach(job => job.stop());
+  console.warn('[DEPRECATED] stopScheduler() called - use taskScheduler from task-scheduler.ts instead');
-  scheduledJobs = [];
-  console.log('🛑 Scheduler stopped');
 }
 
 export async function restartScheduler(): Promise<void> {
-  console.log('🔄 Restarting scheduler...');
+  console.warn('[DEPRECATED] restartScheduler() called - use taskScheduler from task-scheduler.ts instead');
-  stopScheduler();
-  await startScheduler();
 }
 
-// Manual trigger functions for admin
+export async function triggerStoreScrape(_storeId: number): Promise<void> {
-export async function triggerStoreScrape(storeId: number): Promise<void> {
+  console.warn('[DEPRECATED] triggerStoreScrape() called - use taskService.createTask() instead');
-  console.log(`🔧 Manual scrape triggered for store ID: ${storeId}`);
-  await scrapeStore(storeId);
 }
 
 export async function triggerAllStoresScrape(): Promise<void> {
-  console.log('🔧 Manual scrape triggered for all stores');
+  console.warn('[DEPRECATED] triggerAllStoresScrape() called - use taskScheduler.triggerSchedule() instead');
-  await scrapeAllStores();
 }

backend/src/services/scraper.ts

@@ -3,7 +3,7 @@ import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 import { Browser, Page } from 'puppeteer';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import { pool } from '../db/pool';
-import { uploadImageFromUrl, getImageUrl } from '../utils/minio';
+import { downloadProductImageLegacy } from '../utils/image-storage';
 import { logger } from './logger';
 import { registerScraper, updateScraperStats, completeScraper } from '../routes/scraper-monitor';
 import { incrementProxyFailure, getActiveProxy, isBotDetectionError, putProxyInTimeout } from './proxy';
@@ -767,7 +767,8 @@ export async function saveProducts(storeId: number, categoryId: number, products
         
         if (product.imageUrl && !localImagePath) {
           try {
-            localImagePath = await uploadImageFromUrl(product.imageUrl, productId);
+            const result = await downloadProductImageLegacy(product.imageUrl, 0, productId);
+            localImagePath = result.urls?.original || null;
             await client.query(`
               UPDATE products
               SET local_image_path = $1

backend/src/services/task-scheduler.ts

@@ -0,0 +1,375 @@
+/**
+ * Database-Driven Task Scheduler
+ *
+ * Per TASK_WORKFLOW_2024-12-10.md:
+ * - Schedules stored in DB (survives restarts)
+ * - Uses SELECT FOR UPDATE to prevent duplicate execution across replicas
+ * - Polls every 60s to check if schedules are due
+ * - Generates tasks into worker_tasks table for task-worker.ts to process
+ *
+ * 2024-12-10: Created to replace legacy node-cron scheduler
+ */
+
+import { pool } from '../db/pool';
+import { taskService, TaskRole } from '../tasks/task-service';
+
+// Per TASK_WORKFLOW_2024-12-10.md: Poll interval for checking schedules
+const POLL_INTERVAL_MS = 60_000; // 60 seconds
+
+interface TaskSchedule {
+  id: number;
+  name: string;
+  role: TaskRole;
+  enabled: boolean;
+  interval_hours: number;
+  last_run_at: Date | null;
+  next_run_at: Date | null;
+  state_code: string | null;
+  priority: number;
+}
+
+class TaskScheduler {
+  private pollTimer: NodeJS.Timeout | null = null;
+  private isRunning = false;
+
+  /**
+   * Start the scheduler
+   * Per TASK_WORKFLOW_2024-12-10.md: Called on API server startup
+   */
+  async start(): Promise<void> {
+    if (this.isRunning) {
+      console.log('[TaskScheduler] Already running');
+      return;
+    }
+
+    console.log('[TaskScheduler] Starting database-driven scheduler...');
+    this.isRunning = true;
+
+    // Per TASK_WORKFLOW_2024-12-10.md: On startup, recover stale tasks
+    try {
+      const recovered = await taskService.recoverStaleTasks(10);
+      if (recovered > 0) {
+        console.log(`[TaskScheduler] Recovered ${recovered} stale tasks from dead workers`);
+      }
+    } catch (err: any) {
+      console.error('[TaskScheduler] Failed to recover stale tasks:', err.message);
+    }
+
+    // Per TASK_WORKFLOW_2024-12-10.md: Ensure default schedules exist
+    await this.ensureDefaultSchedules();
+
+    // Per TASK_WORKFLOW_2024-12-10.md: Check immediately on startup
+    await this.checkAndRunDueSchedules();
+
+    // Per TASK_WORKFLOW_2024-12-10.md: Then poll every 60 seconds
+    this.pollTimer = setInterval(async () => {
+      await this.checkAndRunDueSchedules();
+    }, POLL_INTERVAL_MS);
+
+    console.log('[TaskScheduler] Started - polling every 60s');
+  }
+
+  /**
+   * Stop the scheduler
+   */
+  stop(): void {
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+      this.pollTimer = null;
+    }
+    this.isRunning = false;
+    console.log('[TaskScheduler] Stopped');
+  }
+
+  /**
+   * Ensure default schedules exist in the database
+   * Per TASK_WORKFLOW_2024-12-10.md: Creates schedules if they don't exist
+   */
+  private async ensureDefaultSchedules(): Promise<void> {
+    // Per TASK_WORKFLOW_2024-12-10.md: Default schedules for task generation
+    // NOTE: payload_fetch replaces direct product_refresh - it chains to product_refresh
+    const defaults = [
+      {
+        name: 'payload_fetch_all',
+        role: 'payload_fetch' as TaskRole,
+        interval_hours: 4,
+        priority: 0,
+        description: 'Fetch payloads from Dutchie API for all crawl-enabled stores every 4 hours. Chains to product_refresh.',
+      },
+      {
+        name: 'store_discovery_dutchie',
+        role: 'store_discovery' as TaskRole,
+        interval_hours: 24,
+        priority: 5,
+        description: 'Discover new Dutchie stores daily',
+      },
+      {
+        name: 'analytics_refresh',
+        role: 'analytics_refresh' as TaskRole,
+        interval_hours: 6,
+        priority: 0,
+        description: 'Refresh analytics materialized views every 6 hours',
+      },
+    ];
+
+    for (const sched of defaults) {
+      try {
+        await pool.query(`
+          INSERT INTO task_schedules (name, role, interval_hours, priority, description, enabled, next_run_at)
+          VALUES ($1, $2, $3, $4, $5, true, NOW())
+          ON CONFLICT (name) DO NOTHING
+        `, [sched.name, sched.role, sched.interval_hours, sched.priority, sched.description]);
+      } catch (err: any) {
+        // Table may not exist yet - will be created by migration
+        if (!err.message.includes('does not exist')) {
+          console.error(`[TaskScheduler] Failed to create default schedule ${sched.name}:`, err.message);
+        }
+      }
+    }
+  }
+
+  /**
+   * Check for and run any due schedules
+   * Per TASK_WORKFLOW_2024-12-10.md: Uses SELECT FOR UPDATE SKIP LOCKED to prevent duplicates
+   */
+  private async checkAndRunDueSchedules(): Promise<void> {
+    const client = await pool.connect();
+
+    try {
+      await client.query('BEGIN');
+
+      // Per TASK_WORKFLOW_2024-12-10.md: Atomic claim of due schedules
+      const result = await client.query<TaskSchedule>(`
+        SELECT *
+        FROM task_schedules
+        WHERE enabled = true
+          AND (next_run_at IS NULL OR next_run_at <= NOW())
+        FOR UPDATE SKIP LOCKED
+      `);
+
+      for (const schedule of result.rows) {
+        console.log(`[TaskScheduler] Running schedule: ${schedule.name} (${schedule.role})`);
+
+        try {
+          const tasksCreated = await this.executeSchedule(schedule);
+          console.log(`[TaskScheduler] Schedule ${schedule.name} created ${tasksCreated} tasks`);
+
+          // Per TASK_WORKFLOW_2024-12-10.md: Update last_run_at and calculate next_run_at
+          await client.query(`
+            UPDATE task_schedules
+            SET
+              last_run_at = NOW(),
+              next_run_at = NOW() + ($1 || ' hours')::interval,
+              last_task_count = $2,
+              updated_at = NOW()
+            WHERE id = $3
+          `, [schedule.interval_hours, tasksCreated, schedule.id]);
+
+        } catch (err: any) {
+          console.error(`[TaskScheduler] Schedule ${schedule.name} failed:`, err.message);
+
+          // Still update next_run_at to prevent infinite retry loop
+          await client.query(`
+            UPDATE task_schedules
+            SET
+              next_run_at = NOW() + ($1 || ' hours')::interval,
+              last_error = $2,
+              updated_at = NOW()
+            WHERE id = $3
+          `, [schedule.interval_hours, err.message, schedule.id]);
+        }
+      }
+
+      await client.query('COMMIT');
+    } catch (err: any) {
+      await client.query('ROLLBACK');
+      console.error('[TaskScheduler] Failed to check schedules:', err.message);
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Execute a schedule and create tasks
+   * Per TASK_WORKFLOW_2024-12-10.md: Different logic per role
+   */
+  private async executeSchedule(schedule: TaskSchedule): Promise<number> {
+    switch (schedule.role) {
+      case 'payload_fetch':
+        // Per TASK_WORKFLOW_2024-12-10.md: payload_fetch replaces direct product_refresh
+        return this.generatePayloadFetchTasks(schedule);
+
+      case 'product_refresh':
+        // Legacy - kept for manual triggers, but scheduled crawls use payload_fetch
+        return this.generatePayloadFetchTasks(schedule);
+
+      case 'store_discovery':
+        return this.generateStoreDiscoveryTasks(schedule);
+
+      case 'analytics_refresh':
+        return this.generateAnalyticsRefreshTasks(schedule);
+
+      default:
+        console.warn(`[TaskScheduler] Unknown role: ${schedule.role}`);
+        return 0;
+    }
+  }
+
+  /**
+   * Generate payload_fetch tasks for stores that need crawling
+   * Per TASK_WORKFLOW_2024-12-10.md: payload_fetch hits API, saves to disk, chains to product_refresh
+   */
+  private async generatePayloadFetchTasks(schedule: TaskSchedule): Promise<number> {
+    // Per TASK_WORKFLOW_2024-12-10.md: Find stores needing refresh
+    const result = await pool.query(`
+      SELECT d.id
+      FROM dispensaries d
+      WHERE d.crawl_enabled = true
+        AND d.platform_dispensary_id IS NOT NULL
+        -- No pending/running payload_fetch or product_refresh task already
+        AND NOT EXISTS (
+          SELECT 1 FROM worker_tasks t
+          WHERE t.dispensary_id = d.id
+            AND t.role IN ('payload_fetch', 'product_refresh')
+            AND t.status IN ('pending', 'claimed', 'running')
+        )
+        -- Never fetched OR last fetch > interval ago
+        AND (
+          d.last_fetch_at IS NULL
+          OR d.last_fetch_at < NOW() - ($1 || ' hours')::interval
+        )
+      ${schedule.state_code ? 'AND d.state_id = (SELECT id FROM states WHERE code = $2)' : ''}
+    `, schedule.state_code ? [schedule.interval_hours, schedule.state_code] : [schedule.interval_hours]);
+
+    const dispensaryIds = result.rows.map((r: { id: number }) => r.id);
+
+    if (dispensaryIds.length === 0) {
+      return 0;
+    }
+
+    // Per TASK_WORKFLOW_2024-12-10.md: Create payload_fetch tasks (they chain to product_refresh)
+    const tasks = dispensaryIds.map((id: number) => ({
+      role: 'payload_fetch' as TaskRole,
+      dispensary_id: id,
+      priority: schedule.priority,
+    }));
+
+    return taskService.createTasks(tasks);
+  }
+
+  /**
+   * Generate store_discovery tasks
+   * Per TASK_WORKFLOW_2024-12-10.md: One task per platform
+   */
+  private async generateStoreDiscoveryTasks(schedule: TaskSchedule): Promise<number> {
+    // Check if discovery task already pending
+    const existing = await taskService.listTasks({
+      role: 'store_discovery',
+      status: ['pending', 'claimed', 'running'],
+      limit: 1,
+    });
+
+    if (existing.length > 0) {
+      console.log('[TaskScheduler] Store discovery task already pending, skipping');
+      return 0;
+    }
+
+    await taskService.createTask({
+      role: 'store_discovery',
+      platform: 'dutchie',
+      priority: schedule.priority,
+    });
+
+    return 1;
+  }
+
+  /**
+   * Generate analytics_refresh tasks
+   * Per TASK_WORKFLOW_2024-12-10.md: Single task to refresh all MVs
+   */
+  private async generateAnalyticsRefreshTasks(schedule: TaskSchedule): Promise<number> {
+    // Check if analytics task already pending
+    const existing = await taskService.listTasks({
+      role: 'analytics_refresh',
+      status: ['pending', 'claimed', 'running'],
+      limit: 1,
+    });
+
+    if (existing.length > 0) {
+      console.log('[TaskScheduler] Analytics refresh task already pending, skipping');
+      return 0;
+    }
+
+    await taskService.createTask({
+      role: 'analytics_refresh',
+      priority: schedule.priority,
+    });
+
+    return 1;
+  }
+
+  /**
+   * Get all schedules for dashboard display
+   */
+  async getSchedules(): Promise<TaskSchedule[]> {
+    try {
+      const result = await pool.query(`
+        SELECT * FROM task_schedules ORDER BY name
+      `);
+      return result.rows as TaskSchedule[];
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Update a schedule
+   */
+  async updateSchedule(id: number, updates: Partial<TaskSchedule>): Promise<void> {
+    const setClauses: string[] = [];
+    const values: any[] = [];
+    let paramIndex = 1;
+
+    if (updates.enabled !== undefined) {
+      setClauses.push(`enabled = $${paramIndex++}`);
+      values.push(updates.enabled);
+    }
+    if (updates.interval_hours !== undefined) {
+      setClauses.push(`interval_hours = $${paramIndex++}`);
+      values.push(updates.interval_hours);
+    }
+    if (updates.priority !== undefined) {
+      setClauses.push(`priority = $${paramIndex++}`);
+      values.push(updates.priority);
+    }
+
+    if (setClauses.length === 0) return;
+
+    setClauses.push('updated_at = NOW()');
+    values.push(id);
+
+    await pool.query(`
+      UPDATE task_schedules
+      SET ${setClauses.join(', ')}
+      WHERE id = $${paramIndex}
+    `, values);
+  }
+
+  /**
+   * Trigger a schedule to run immediately
+   */
+  async triggerSchedule(id: number): Promise<number> {
+    const result = await pool.query(`
+      SELECT * FROM task_schedules WHERE id = $1
+    `, [id]);
+
+    if (result.rows.length === 0) {
+      throw new Error(`Schedule ${id} not found`);
+    }
+
+    return this.executeSchedule(result.rows[0] as TaskSchedule);
+  }
+}
+
+// Per TASK_WORKFLOW_2024-12-10.md: Singleton instance
+export const taskScheduler = new TaskScheduler();

backend/src/tasks/handlers/entry-point-discovery.ts

@@ -1,13 +1,21 @@
 /**
  * Entry Point Discovery Handler
  *
- * Detects menu type and resolves platform IDs for a discovered store.
+ * Resolves platform IDs for a discovered store using Dutchie GraphQL.
  * This is the step between store_discovery and product_discovery.
  *
- * TODO: Integrate with platform ID resolution when available
+ * Flow:
+ * 1. Load dispensary info from database
+ * 2. Extract slug from menu_url
+ * 3. Start stealth session (fingerprint + optional proxy)
+ * 4. Query Dutchie GraphQL to resolve slug → platform_dispensary_id
+ * 5. Update dispensary record with resolved ID
+ * 6. Queue product_discovery task if successful
  */
 
 import { TaskContext, TaskResult } from '../task-worker';
+import { startSession, endSession } from '../../platforms/dutchie';
+import { resolveDispensaryIdWithDetails } from '../../platforms/dutchie/queries';
 
 export async function handleEntryPointDiscovery(ctx: TaskContext): Promise<TaskResult> {
   const { pool, task } = ctx;
@@ -18,9 +26,11 @@ export async function handleEntryPointDiscovery(ctx: TaskContext): Promise<TaskR
   }
 
   try {
-    // Get dispensary info
+    // ============================================================
+    // STEP 1: Load dispensary info
+    // ============================================================
     const dispResult = await pool.query(`
-      SELECT id, name, menu_url, platform_dispensary_id, menu_type
+      SELECT id, name, menu_url, platform_dispensary_id, menu_type, state
       FROM dispensaries
       WHERE id = $1
     `, [dispensaryId]);
@@ -33,7 +43,7 @@ export async function handleEntryPointDiscovery(ctx: TaskContext): Promise<TaskR
 
     // If already has platform_dispensary_id, we're done
     if (dispensary.platform_dispensary_id) {
-      console.log(`[EntryPointDiscovery] Dispensary ${dispensaryId} already has platform ID`);
+      console.log(`[EntryPointDiscovery] Dispensary ${dispensaryId} already has platform ID: ${dispensary.platform_dispensary_id}`);
       return {
         success: true,
         alreadyResolved: true,
@@ -46,9 +56,12 @@ export async function handleEntryPointDiscovery(ctx: TaskContext): Promise<TaskR
       return { success: false, error: `Dispensary ${dispensaryId} has no menu_url` };
     }
 
-    console.log(`[EntryPointDiscovery] Would resolve platform ID for ${dispensary.name} from ${menuUrl}`);
+    console.log(`[EntryPointDiscovery] Resolving platform ID for ${dispensary.name}`);
+    console.log(`[EntryPointDiscovery] Menu URL: ${menuUrl}`);
 
-    // Extract slug from menu URL
+    // ============================================================
+    // STEP 2: Extract slug from menu URL
+    // ============================================================
     let slug: string | null = null;
 
     const embeddedMatch = menuUrl.match(/\/embedded-menu\/([^/?]+)/);
@@ -61,21 +74,110 @@ export async function handleEntryPointDiscovery(ctx: TaskContext): Promise<TaskR
     }
 
     if (!slug) {
+      // Mark as non-dutchie menu type
+      await pool.query(`
+        UPDATE dispensaries
+        SET menu_type = 'unknown', updated_at = NOW()
+        WHERE id = $1
+      `, [dispensaryId]);
+
       return {
         success: false,
         error: `Could not extract slug from menu_url: ${menuUrl}`,
       };
     }
 
-    // TODO: Integrate with actual platform ID resolution
+    console.log(`[EntryPointDiscovery] Extracted slug: ${slug}`);
-    // For now, mark the task as needing manual resolution
+
-    console.log(`[EntryPointDiscovery] Found slug: ${slug} - manual resolution needed`);
+    await ctx.heartbeat();
+
+    // ============================================================
+    // STEP 3: Start stealth session
+    // ============================================================
+    // Per workflow-12102025.md: session identity comes from proxy location, not task params
+    const session = startSession();
+    console.log(`[EntryPointDiscovery] Session started: ${session.sessionId}`);
+
+    try {
+      // ============================================================
+      // STEP 4: Resolve platform ID via GraphQL
+      // ============================================================
+      console.log(`[EntryPointDiscovery] Querying Dutchie GraphQL for slug: ${slug}`);
+
+      const result = await resolveDispensaryIdWithDetails(slug);
+
+      if (!result.dispensaryId) {
+        // Resolution failed - could be 403, 404, or invalid response
+        const reason = result.httpStatus
+          ? `HTTP ${result.httpStatus}`
+          : result.error || 'Unknown error';
+
+        console.log(`[EntryPointDiscovery] Failed to resolve ${slug}: ${reason}`);
+
+        // Mark as failed resolution but keep menu_type as dutchie
+        await pool.query(`
+          UPDATE dispensaries
+          SET
+            menu_type = CASE
+              WHEN $2 = 404 THEN 'removed'
+              WHEN $2 = 403 THEN 'blocked'
+              ELSE 'dutchie'
+            END,
+            updated_at = NOW()
+          WHERE id = $1
+        `, [dispensaryId, result.httpStatus || 0]);
+
+        return {
+          success: false,
+          error: `Could not resolve platform ID: ${reason}`,
+          slug,
+          httpStatus: result.httpStatus,
+        };
+      }
+
+      const platformId = result.dispensaryId;
+      console.log(`[EntryPointDiscovery] Resolved ${slug} -> ${platformId}`);
+
+      await ctx.heartbeat();
+
+      // ============================================================
+      // STEP 5: Update dispensary with resolved ID
+      // ============================================================
+      await pool.query(`
+        UPDATE dispensaries
+        SET
+          platform_dispensary_id = $2,
+          menu_type = 'dutchie',
+          crawl_enabled = true,
+          updated_at = NOW()
+        WHERE id = $1
+      `, [dispensaryId, platformId]);
+
+      console.log(`[EntryPointDiscovery] Updated dispensary ${dispensaryId} with platform ID`);
+
+      // ============================================================
+      // STEP 6: Queue product_discovery task
+      // ============================================================
+      await pool.query(`
+        INSERT INTO worker_tasks (role, dispensary_id, priority, scheduled_for)
+        VALUES ('product_discovery', $1, 5, NOW())
+        ON CONFLICT DO NOTHING
+      `, [dispensaryId]);
+
+      console.log(`[EntryPointDiscovery] Queued product_discovery task for dispensary ${dispensaryId}`);
 
       return {
         success: true,
-      message: 'Slug extracted, awaiting platform ID resolution',
+        platformId,
         slug,
+        queuedProductDiscovery: true,
       };
+
+    } finally {
+      // Always end session
+      endSession();
+    }
+
   } catch (error: unknown) {
     const errorMessage = error instanceof Error ? error.message : 'Unknown error';
     console.error(`[EntryPointDiscovery] Error for dispensary ${dispensaryId}:`, errorMessage);

backend/src/tasks/handlers/index.ts

@@ -4,8 +4,9 @@
  * Exports all task handlers for the task worker.
  */
 
-export { handleProductResync } from './product-resync';
+export { handleProductRefresh } from './product-refresh';
 export { handleProductDiscovery } from './product-discovery';
 export { handleStoreDiscovery } from './store-discovery';
 export { handleEntryPointDiscovery } from './entry-point-discovery';
 export { handleAnalyticsRefresh } from './analytics-refresh';
+export { handleProxyTest } from './proxy-test';

backend/src/tasks/handlers/payload-fetch.ts

@@ -0,0 +1,221 @@
+/**
+ * Payload Fetch Handler
+ *
+ * Per TASK_WORKFLOW_2024-12-10.md: Separates API fetch from data processing.
+ *
+ * This handler ONLY:
+ * 1. Hits Dutchie GraphQL API
+ * 2. Saves raw payload to filesystem (gzipped)
+ * 3. Records metadata in raw_crawl_payloads table
+ * 4. Queues a product_refresh task to process the payload
+ *
+ * Benefits of separation:
+ * - Retry-friendly: If normalize fails, re-run refresh without re-crawling
+ * - Faster refreshes: Local file read vs network call
+ * - Replay-able: Run refresh against any historical payload
+ * - Less API pressure: Only this role hits Dutchie
+ */
+
+import { TaskContext, TaskResult } from '../task-worker';
+import {
+  executeGraphQL,
+  startSession,
+  endSession,
+  GRAPHQL_HASHES,
+  DUTCHIE_CONFIG,
+} from '../../platforms/dutchie';
+import { saveRawPayload } from '../../utils/payload-storage';
+import { taskService } from '../task-service';
+
+export async function handlePayloadFetch(ctx: TaskContext): Promise<TaskResult> {
+  const { pool, task } = ctx;
+  const dispensaryId = task.dispensary_id;
+
+  if (!dispensaryId) {
+    return { success: false, error: 'No dispensary_id specified for payload_fetch task' };
+  }
+
+  try {
+    // ============================================================
+    // STEP 1: Load dispensary info
+    // ============================================================
+    const dispResult = await pool.query(`
+      SELECT
+        id, name, platform_dispensary_id, menu_url, menu_type, city, state
+      FROM dispensaries
+      WHERE id = $1 AND crawl_enabled = true
+    `, [dispensaryId]);
+
+    if (dispResult.rows.length === 0) {
+      return { success: false, error: `Dispensary ${dispensaryId} not found or not crawl_enabled` };
+    }
+
+    const dispensary = dispResult.rows[0];
+    const platformId = dispensary.platform_dispensary_id;
+
+    if (!platformId) {
+      return { success: false, error: `Dispensary ${dispensaryId} has no platform_dispensary_id` };
+    }
+
+    // Extract cName from menu_url
+    const cNameMatch = dispensary.menu_url?.match(/\/(?:embedded-menu|dispensary)\/([^/?]+)/);
+    const cName = cNameMatch ? cNameMatch[1] : 'dispensary';
+
+    console.log(`[PayloadFetch] Starting fetch for ${dispensary.name} (ID: ${dispensaryId})`);
+    console.log(`[PayloadFetch] Platform ID: ${platformId}, cName: ${cName}`);
+
+    // ============================================================
+    // STEP 2: Start stealth session
+    // ============================================================
+    const session = startSession();
+    console.log(`[PayloadFetch] Session started: ${session.sessionId}`);
+
+    await ctx.heartbeat();
+
+    // ============================================================
+    // STEP 3: Fetch products via GraphQL (Status: 'All')
+    // ============================================================
+    const allProducts: any[] = [];
+    let page = 0;
+    let totalCount = 0;
+    const perPage = DUTCHIE_CONFIG.perPage;
+    const maxPages = DUTCHIE_CONFIG.maxPages;
+
+    try {
+      while (page < maxPages) {
+        const variables = {
+          includeEnterpriseSpecials: false,
+          productsFilter: {
+            dispensaryId: platformId,
+            pricingType: 'rec',
+            Status: 'All',
+            types: [],
+            useCache: false,
+            isDefaultSort: true,
+            sortBy: 'popularSortIdx',
+            sortDirection: 1,
+            bypassOnlineThresholds: true,
+            isKioskMenu: false,
+            removeProductsBelowOptionThresholds: false,
+          },
+          page,
+          perPage,
+        };
+
+        console.log(`[PayloadFetch] Fetching page ${page + 1}...`);
+
+        const result = await executeGraphQL(
+          'FilteredProducts',
+          variables,
+          GRAPHQL_HASHES.FilteredProducts,
+          { cName, maxRetries: 3 }
+        );
+
+        const data = result?.data?.filteredProducts;
+        if (!data || !data.products) {
+          if (page === 0) {
+            throw new Error('No product data returned from GraphQL');
+          }
+          break;
+        }
+
+        const products = data.products;
+        allProducts.push(...products);
+
+        if (page === 0) {
+          totalCount = data.queryInfo?.totalCount || products.length;
+          console.log(`[PayloadFetch] Total products reported: ${totalCount}`);
+        }
+
+        if (allProducts.length >= totalCount || products.length < perPage) {
+          break;
+        }
+
+        page++;
+
+        if (page < maxPages) {
+          await new Promise(r => setTimeout(r, DUTCHIE_CONFIG.pageDelayMs));
+        }
+
+        if (page % 5 === 0) {
+          await ctx.heartbeat();
+        }
+      }
+
+      console.log(`[PayloadFetch] Fetched ${allProducts.length} products in ${page + 1} pages`);
+
+    } finally {
+      endSession();
+    }
+
+    if (allProducts.length === 0) {
+      return {
+        success: false,
+        error: 'No products returned from GraphQL',
+        productsProcessed: 0,
+      };
+    }
+
+    await ctx.heartbeat();
+
+    // ============================================================
+    // STEP 4: Save raw payload to filesystem
+    // Per TASK_WORKFLOW_2024-12-10.md: Metadata/Payload separation
+    // ============================================================
+    const rawPayload = {
+      dispensaryId,
+      platformId,
+      cName,
+      fetchedAt: new Date().toISOString(),
+      productCount: allProducts.length,
+      products: allProducts,
+    };
+
+    const payloadResult = await saveRawPayload(
+      pool,
+      dispensaryId,
+      rawPayload,
+      null, // crawl_run_id - not using crawl_runs in new system
+      allProducts.length
+    );
+
+    console.log(`[PayloadFetch] Saved payload #${payloadResult.id} (${(payloadResult.sizeBytes / 1024).toFixed(1)}KB)`);
+
+    // ============================================================
+    // STEP 5: Update dispensary last_fetch_at
+    // ============================================================
+    await pool.query(`
+      UPDATE dispensaries
+      SET last_fetch_at = NOW()
+      WHERE id = $1
+    `, [dispensaryId]);
+
+    // ============================================================
+    // STEP 6: Queue product_refresh task to process the payload
+    // Per TASK_WORKFLOW_2024-12-10.md: Task chaining
+    // ============================================================
+    await taskService.createTask({
+      role: 'product_refresh',
+      dispensary_id: dispensaryId,
+      priority: task.priority || 0,
+      payload: { payload_id: payloadResult.id },
+    });
+
+    console.log(`[PayloadFetch] Queued product_refresh task for payload #${payloadResult.id}`);
+
+    return {
+      success: true,
+      payloadId: payloadResult.id,
+      productCount: allProducts.length,
+      sizeBytes: payloadResult.sizeBytes,
+    };
+
+  } catch (error: unknown) {
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+    console.error(`[PayloadFetch] Error for dispensary ${dispensaryId}:`, errorMessage);
+    return {
+      success: false,
+      error: errorMessage,
+    };
+  }
+}

backend/src/tasks/handlers/product-discovery.ts

@@ -1,16 +1,31 @@
 /**
  * Product Discovery Handler
  *
- * Initial product fetch for stores that have 0 products.
+ * Per TASK_WORKFLOW_2024-12-10.md: Initial product fetch for newly discovered stores.
- * Same logic as product_resync, but for initial discovery.
+ *
+ * Flow:
+ * 1. Triggered after store_discovery promotes a new dispensary
+ * 2. Chains to payload_fetch to get initial product data
+ * 3. payload_fetch chains to product_refresh for DB upsert
+ *
+ * Chaining:
+ * store_discovery → (newStoreIds) → product_discovery → payload_fetch → product_refresh
  */
 
 import { TaskContext, TaskResult } from '../task-worker';
-import { handleProductResync } from './product-resync';
+import { handlePayloadFetch } from './payload-fetch';
 
 export async function handleProductDiscovery(ctx: TaskContext): Promise<TaskResult> {
-  // Product discovery is essentially the same as resync for the first time
+  const { task } = ctx;
-  // The main difference is in when this task is triggered (new store vs scheduled)
+  const dispensaryId = task.dispensary_id;
-  console.log(`[ProductDiscovery] Starting initial product fetch for dispensary ${ctx.task.dispensary_id}`);
+
-  return handleProductResync(ctx);
+  if (!dispensaryId) {
+    return { success: false, error: 'No dispensary_id provided' };
+  }
+
+  console.log(`[ProductDiscovery] Starting initial product discovery for dispensary ${dispensaryId}`);
+
+  // Per TASK_WORKFLOW_2024-12-10.md: Chain to payload_fetch for API → disk
+  // payload_fetch will then chain to product_refresh for disk → DB
+  return handlePayloadFetch(ctx);
 }

backend/src/tasks/handlers/product-refresh.ts

@@ -1,42 +1,41 @@
 /**
- * Product Resync Handler
+ * Product Refresh Handler
  *
- * Re-crawls a store to capture price/stock changes using the GraphQL pipeline.
+ * Per TASK_WORKFLOW_2024-12-10.md: Processes a locally-stored payload.
+ *
+ * This handler reads from the filesystem (NOT the Dutchie API).
+ * The payload_fetch handler is responsible for API calls.
  *
  * Flow:
- * 1. Load dispensary info from database
+ * 1. Load payload from filesystem (by payload_id or latest for dispensary)
- * 2. Start stealth session (fingerprint + optional proxy)
+ * 2. Normalize data via DutchieNormalizer
- * 3. Fetch products via GraphQL (Status: 'All')
+ * 3. Upsert to store_products and store_product_snapshots
- * 4. Normalize data via DutchieNormalizer
+ * 4. Track missing products (increment consecutive_misses, mark OOS at 3)
- * 5. Upsert to store_products and store_product_snapshots
+ * 5. Download new product images
- * 6. Track missing products (increment consecutive_misses, mark OOS at 3)
+ *
- * 7. Download new product images
+ * Benefits of separation:
- * 8. End session
+ * - Retry-friendly: If this fails, re-run without re-crawling
+ * - Replay-able: Run against any historical payload
+ * - Faster: Local file read vs network call
  */
 
 import { TaskContext, TaskResult } from '../task-worker';
-import {
-  executeGraphQL,
-  startSession,
-  endSession,
-  GRAPHQL_HASHES,
-  DUTCHIE_CONFIG,
-} from '../../platforms/dutchie';
 import { DutchieNormalizer } from '../../hydration/normalizers/dutchie';
 import {
   upsertStoreProducts,
   createStoreProductSnapshots,
   downloadProductImages,
 } from '../../hydration/canonical-upsert';
+import { loadRawPayloadById, getLatestPayload } from '../../utils/payload-storage';
 
 const normalizer = new DutchieNormalizer();
 
-export async function handleProductResync(ctx: TaskContext): Promise<TaskResult> {
+export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult> {
   const { pool, task } = ctx;
   const dispensaryId = task.dispensary_id;
 
   if (!dispensaryId) {
-    return { success: false, error: 'No dispensary_id specified for product_resync task' };
+    return { success: false, error: 'No dispensary_id specified for product_refresh task' };
   }
 
   try {
@@ -47,129 +46,76 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
       SELECT
         id, name, platform_dispensary_id, menu_url, menu_type, city, state
       FROM dispensaries
-      WHERE id = $1 AND crawl_enabled = true
+      WHERE id = $1
     `, [dispensaryId]);
 
     if (dispResult.rows.length === 0) {
-      return { success: false, error: `Dispensary ${dispensaryId} not found or not crawl_enabled` };
+      return { success: false, error: `Dispensary ${dispensaryId} not found` };
     }
 
     const dispensary = dispResult.rows[0];
-    const platformId = dispensary.platform_dispensary_id;
 
-    if (!platformId) {
+    // Extract cName from menu_url for image storage context
-      return { success: false, error: `Dispensary ${dispensaryId} has no platform_dispensary_id` };
-    }
-
-    // Extract cName from menu_url
     const cNameMatch = dispensary.menu_url?.match(/\/(?:embedded-menu|dispensary)\/([^/?]+)/);
     const cName = cNameMatch ? cNameMatch[1] : 'dispensary';
 
-    console.log(`[ProductResync] Starting crawl for ${dispensary.name} (ID: ${dispensaryId})`);
+    console.log(`[ProductRefresh] Starting refresh for ${dispensary.name} (ID: ${dispensaryId})`);
-    console.log(`[ProductResync] Platform ID: ${platformId}, cName: ${cName}`);
-
-    // ============================================================
-    // STEP 2: Start stealth session
-    // ============================================================
-    const session = startSession(dispensary.state || 'AZ', 'America/Phoenix');
-    console.log(`[ProductResync] Session started: ${session.sessionId}`);
 
     await ctx.heartbeat();
 
     // ============================================================
-    // STEP 3: Fetch products via GraphQL (Status: 'All')
+    // STEP 2: Load payload from filesystem
+    // Per TASK_WORKFLOW_2024-12-10.md: Read local payload, not API
     // ============================================================
-    const allProducts: any[] = [];
+    let payloadData: any;
-    let page = 0;
+    let payloadId: number;
-    let totalCount = 0;
-    const perPage = DUTCHIE_CONFIG.perPage;
-    const maxPages = DUTCHIE_CONFIG.maxPages;
 
-    try {
+    // Check if specific payload_id was provided (from task chaining)
-      while (page < maxPages) {
+    const taskPayload = task.payload as { payload_id?: number } | null;
-        const variables = {
-          includeEnterpriseSpecials: false,
-          productsFilter: {
-            dispensaryId: platformId,
-            pricingType: 'rec',
-            Status: 'All',
-            types: [],
-            useCache: false,
-            isDefaultSort: true,
-            sortBy: 'popularSortIdx',
-            sortDirection: 1,
-            bypassOnlineThresholds: true,
-            isKioskMenu: false,
-            removeProductsBelowOptionThresholds: false,
-          },
-          page,
-          perPage,
-        };
 
-        console.log(`[ProductResync] Fetching page ${page + 1}...`);
+    if (taskPayload?.payload_id) {
-
+      // Load specific payload (from payload_fetch chaining)
-        const result = await executeGraphQL(
+      const result = await loadRawPayloadById(pool, taskPayload.payload_id);
-          'FilteredProducts',
+      if (!result) {
-          variables,
+        return { success: false, error: `Payload ${taskPayload.payload_id} not found` };
-          GRAPHQL_HASHES.FilteredProducts,
-          { cName, maxRetries: 3 }
-        );
-
-        const data = result?.data?.filteredProducts;
-        if (!data || !data.products) {
-          if (page === 0) {
-            throw new Error('No product data returned from GraphQL');
       }
-          break;
+      payloadData = result.payload;
+      payloadId = result.metadata.id;
+      console.log(`[ProductRefresh] Loaded specific payload #${payloadId}`);
+    } else {
+      // Load latest payload for this dispensary
+      const result = await getLatestPayload(pool, dispensaryId);
+      if (!result) {
+        return { success: false, error: `No payload found for dispensary ${dispensaryId}` };
+      }
+      payloadData = result.payload;
+      payloadId = result.metadata.id;
+      console.log(`[ProductRefresh] Loaded latest payload #${payloadId} (${result.metadata.fetchedAt})`);
     }
 
-        const products = data.products;
+    const allProducts = payloadData.products || [];
-        allProducts.push(...products);
-
-        if (page === 0) {
-          totalCount = data.queryInfo?.totalCount || products.length;
-          console.log(`[ProductResync] Total products reported: ${totalCount}`);
-        }
-
-        if (allProducts.length >= totalCount || products.length < perPage) {
-          break;
-        }
-
-        page++;
-
-        if (page < maxPages) {
-          await new Promise(r => setTimeout(r, DUTCHIE_CONFIG.pageDelayMs));
-        }
-
-        if (page % 5 === 0) {
-          await ctx.heartbeat();
-        }
-      }
-
-      console.log(`[ProductResync] Fetched ${allProducts.length} products in ${page + 1} pages`);
-
-    } finally {
-      endSession();
-    }
 
     if (allProducts.length === 0) {
       return {
         success: false,
-        error: 'No products returned from GraphQL',
+        error: 'Payload contains no products',
+        payloadId,
         productsProcessed: 0,
       };
     }
 
+    console.log(`[ProductRefresh] Processing ${allProducts.length} products from payload #${payloadId}`);
+
     await ctx.heartbeat();
 
     // ============================================================
-    // STEP 4: Normalize data
+    // STEP 3: Normalize data
     // ============================================================
-    console.log(`[ProductResync] Normalizing ${allProducts.length} products...`);
+    console.log(`[ProductRefresh] Normalizing ${allProducts.length} products...`);
 
     // Build RawPayload for the normalizer
     const rawPayload = {
-      id: `resync-${dispensaryId}-${Date.now()}`,
+      id: `refresh-${dispensaryId}-${Date.now()}`,
       dispensary_id: dispensaryId,
       crawl_run_id: null,
       platform: 'dutchie',
@@ -189,25 +135,26 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
     const normalizationResult = normalizer.normalize(rawPayload);
 
     if (normalizationResult.errors.length > 0) {
-      console.warn(`[ProductResync] Normalization warnings: ${normalizationResult.errors.map(e => e.message).join(', ')}`);
+      console.warn(`[ProductRefresh] Normalization warnings: ${normalizationResult.errors.map(e => e.message).join(', ')}`);
     }
 
     if (normalizationResult.products.length === 0) {
       return {
         success: false,
         error: 'Normalization produced no products',
+        payloadId,
         productsProcessed: 0,
       };
     }
 
-    console.log(`[ProductResync] Normalized ${normalizationResult.products.length} products`);
+    console.log(`[ProductRefresh] Normalized ${normalizationResult.products.length} products`);
 
     await ctx.heartbeat();
 
     // ============================================================
-    // STEP 5: Upsert to canonical tables
+    // STEP 4: Upsert to canonical tables
     // ============================================================
-    console.log(`[ProductResync] Upserting to store_products...`);
+    console.log(`[ProductRefresh] Upserting to store_products...`);
 
     const upsertResult = await upsertStoreProducts(
       pool,
@@ -216,12 +163,12 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
       normalizationResult.availability
     );
 
-    console.log(`[ProductResync] Upserted: ${upsertResult.upserted} (${upsertResult.new} new, ${upsertResult.updated} updated)`);
+    console.log(`[ProductRefresh] Upserted: ${upsertResult.upserted} (${upsertResult.new} new, ${upsertResult.updated} updated)`);
 
     await ctx.heartbeat();
 
     // Create snapshots
-    console.log(`[ProductResync] Creating snapshots...`);
+    console.log(`[ProductRefresh] Creating snapshots...`);
 
     const snapshotsResult = await createStoreProductSnapshots(
       pool,
@@ -232,12 +179,12 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
       null // No crawl_run_id in new system
     );
 
-    console.log(`[ProductResync] Created ${snapshotsResult.created} snapshots`);
+    console.log(`[ProductRefresh] Created ${snapshotsResult.created} snapshots`);
 
     await ctx.heartbeat();
 
     // ============================================================
-    // STEP 6: Track missing products (consecutive_misses logic)
+    // STEP 5: Track missing products (consecutive_misses logic)
     // - Products in feed: reset consecutive_misses to 0
     // - Products not in feed: increment consecutive_misses
     // - At 3 consecutive misses: mark as OOS
@@ -270,7 +217,7 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
 
     const incrementedCount = incrementResult.rowCount || 0;
     if (incrementedCount > 0) {
-      console.log(`[ProductResync] Incremented consecutive_misses for ${incrementedCount} products`);
+      console.log(`[ProductRefresh] Incremented consecutive_misses for ${incrementedCount} products`);
     }
 
     // Mark as OOS any products that hit 3 consecutive misses
@@ -286,16 +233,16 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
 
     const markedOosCount = oosResult.rowCount || 0;
     if (markedOosCount > 0) {
-      console.log(`[ProductResync] Marked ${markedOosCount} products as OOS (3+ consecutive misses)`);
+      console.log(`[ProductRefresh] Marked ${markedOosCount} products as OOS (3+ consecutive misses)`);
     }
 
     await ctx.heartbeat();
 
     // ============================================================
-    // STEP 7: Download images for new products
+    // STEP 6: Download images for new products
     // ============================================================
     if (upsertResult.productsNeedingImages.length > 0) {
-      console.log(`[ProductResync] Downloading images for ${upsertResult.productsNeedingImages.length} products...`);
+      console.log(`[ProductRefresh] Downloading images for ${upsertResult.productsNeedingImages.length} products...`);
 
       try {
         const dispensaryContext = {
@@ -309,12 +256,12 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
         );
       } catch (imgError: any) {
         // Image download errors shouldn't fail the whole task
-        console.warn(`[ProductResync] Image download error (non-fatal): ${imgError.message}`);
+        console.warn(`[ProductRefresh] Image download error (non-fatal): ${imgError.message}`);
       }
     }
 
     // ============================================================
-    // STEP 8: Update dispensary last_crawl_at
+    // STEP 7: Update dispensary last_crawl_at
     // ============================================================
     await pool.query(`
       UPDATE dispensaries
@@ -322,10 +269,20 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
       WHERE id = $1
     `, [dispensaryId]);
 
-    console.log(`[ProductResync] Completed ${dispensary.name}`);
+    // ============================================================
+    // STEP 8: Mark payload as processed
+    // ============================================================
+    await pool.query(`
+      UPDATE raw_crawl_payloads
+      SET processed_at = NOW()
+      WHERE id = $1
+    `, [payloadId]);
+
+    console.log(`[ProductRefresh] Completed ${dispensary.name}`);
 
     return {
       success: true,
+      payloadId,
       productsProcessed: normalizationResult.products.length,
       snapshotsCreated: snapshotsResult.created,
       newProducts: upsertResult.new,
@@ -335,7 +292,7 @@ export async function handleProductResync(ctx: TaskContext): Promise<TaskResult>
 
   } catch (error: unknown) {
     const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-    console.error(`[ProductResync] Error for dispensary ${dispensaryId}:`, errorMessage);
+    console.error(`[ProductRefresh] Error for dispensary ${dispensaryId}:`, errorMessage);
     return {
       success: false,
       error: errorMessage,

backend/src/tasks/handlers/proxy-test.ts

@@ -0,0 +1,51 @@
+/**
+ * Proxy Test Handler
+ * Tests proxy connectivity by fetching public IP via ipify
+ */
+
+import { TaskContext, TaskResult } from '../task-worker';
+import { execSync } from 'child_process';
+
+export async function handleProxyTest(ctx: TaskContext): Promise<TaskResult> {
+  const { pool } = ctx;
+
+  console.log('[ProxyTest] Testing proxy connection...');
+
+  try {
+    // Get active proxy from DB
+    const proxyResult = await pool.query(`
+      SELECT host, port, username, password
+      FROM proxies
+      WHERE is_active = true
+      LIMIT 1
+    `);
+
+    if (proxyResult.rows.length === 0) {
+      return { success: false, error: 'No active proxy configured' };
+    }
+
+    const p = proxyResult.rows[0];
+    const proxyUrl = p.username
+      ? `http://${p.username}:${p.password}@${p.host}:${p.port}`
+      : `http://${p.host}:${p.port}`;
+
+    console.log(`[ProxyTest] Using proxy: ${p.host}:${p.port}`);
+
+    // Fetch IP via proxy
+    const cmd = `curl -s --proxy '${proxyUrl}' 'https://api.ipify.org?format=json'`;
+    const output = execSync(cmd, { timeout: 30000 }).toString().trim();
+    const data = JSON.parse(output);
+
+    console.log(`[ProxyTest] Proxy IP: ${data.ip}`);
+
+    return {
+      success: true,
+      proxyIp: data.ip,
+      proxyHost: p.host,
+      proxyPort: p.port,
+    };
+  } catch (error: any) {
+    console.error('[ProxyTest] Error:', error.message);
+    return { success: false, error: error.message };
+  }
+}

backend/src/tasks/handlers/store-discovery.ts

@@ -1,8 +1,16 @@
 /**
  * Store Discovery Handler
  *
- * Discovers new stores by crawling location APIs and adding them
+ * Per TASK_WORKFLOW_2024-12-10.md: Discovers new stores and returns their IDs for task chaining.
- * to discovery_locations table.
+ *
+ * Flow:
+ * 1. For each active state, run Dutchie discovery
+ * 2. Discover locations via GraphQL
+ * 3. Auto-promote valid locations to dispensaries table
+ * 4. Return newStoreIds[] for chaining to payload_fetch
+ *
+ * Chaining:
+ * store_discovery → (returns newStoreIds) → payload_fetch → product_refresh
  */
 
 import { TaskContext, TaskResult } from '../task-worker';
@@ -10,23 +18,25 @@ import { discoverState } from '../../discovery';
 
 export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult> {
   const { pool, task } = ctx;
-  const platform = task.platform || 'default';
+  const platform = task.platform || 'dutchie';
 
   console.log(`[StoreDiscovery] Starting discovery for platform: ${platform}`);
 
   try {
     // Get states to discover
     const statesResult = await pool.query(`
-      SELECT code FROM states WHERE active = true ORDER BY code
+      SELECT code FROM states WHERE is_active = true ORDER BY code
     `);
     const stateCodes = statesResult.rows.map(r => r.code);
 
     if (stateCodes.length === 0) {
-      return { success: true, storesDiscovered: 0, message: 'No active states to discover' };
+      return { success: true, storesDiscovered: 0, newStoreIds: [], message: 'No active states to discover' };
     }
 
     let totalDiscovered = 0;
     let totalPromoted = 0;
+    // Per TASK_WORKFLOW_2024-12-10.md: Collect all new store IDs for task chaining
+    const allNewStoreIds: number[] = [];
 
     // Run discovery for each state
     for (const stateCode of stateCodes) {
@@ -39,6 +49,13 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
         const result = await discoverState(pool, stateCode);
         totalDiscovered += result.totalLocationsFound || 0;
         totalPromoted += result.totalLocationsUpserted || 0;
+
+        // Per TASK_WORKFLOW_2024-12-10.md: Collect new IDs for chaining
+        if (result.newDispensaryIds && result.newDispensaryIds.length > 0) {
+          allNewStoreIds.push(...result.newDispensaryIds);
+          console.log(`[StoreDiscovery] ${stateCode}: ${result.newDispensaryIds.length} new stores`);
+        }
+
         console.log(`[StoreDiscovery] ${stateCode}: found ${result.totalLocationsFound}, upserted ${result.totalLocationsUpserted}`);
       } catch (error: unknown) {
         const errorMessage = error instanceof Error ? error.message : 'Unknown error';
@@ -47,13 +64,15 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
       }
     }
 
-    console.log(`[StoreDiscovery] Complete: ${totalDiscovered} discovered, ${totalPromoted} promoted`);
+    console.log(`[StoreDiscovery] Complete: ${totalDiscovered} discovered, ${totalPromoted} promoted, ${allNewStoreIds.length} new stores`);
 
     return {
       success: true,
       storesDiscovered: totalDiscovered,
       storesPromoted: totalPromoted,
       statesProcessed: stateCodes.length,
+      // Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
+      newStoreIds: allNewStoreIds,
     };
   } catch (error: unknown) {
     const errorMessage = error instanceof Error ? error.message : 'Unknown error';
@@ -61,6 +80,7 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
     return {
       success: false,
       error: errorMessage,
+      newStoreIds: [],
     };
   }
 }

backend/src/tasks/index.ts

@@ -17,7 +17,7 @@ export {
 export { TaskWorker, TaskContext, TaskResult } from './task-worker';
 
 export {
-  handleProductResync,
+  handleProductRefresh,
   handleProductDiscovery,
   handleStoreDiscovery,
   handleEntryPointDiscovery,

backend/src/tasks/start-pod.ts

@@ -0,0 +1,93 @@
+#!/usr/bin/env npx tsx
+/**
+ * Start Pod - Simulates a Kubernetes pod locally
+ *
+ * Starts 5 workers with a pod name from the predefined list.
+ *
+ * Usage:
+ *   npx tsx src/tasks/start-pod.ts <pod-index>
+ *   npx tsx src/tasks/start-pod.ts 0   # Starts pod "Aethelgard" with 5 workers
+ *   npx tsx src/tasks/start-pod.ts 1   # Starts pod "Xylos" with 5 workers
+ */
+
+import { spawn } from 'child_process';
+import path from 'path';
+
+const POD_NAMES = [
+  'Aethelgard',
+  'Xylos',
+  'Kryll',
+  'Coriolis',
+  'Dimidium',
+  'Veridia',
+  'Zetani',
+  'Talos IV',
+  'Onyx',
+  'Celestia',
+  'Gormand',
+  'Betha',
+  'Ragnar',
+  'Syphon',
+  'Axiom',
+  'Nadir',
+  'Terra Nova',
+  'Acheron',
+  'Nexus',
+  'Vespera',
+  'Helios Prime',
+  'Oasis',
+  'Mordina',
+  'Cygnus',
+  'Umbra',
+];
+
+const WORKERS_PER_POD = 5;
+
+async function main() {
+  const podIndex = parseInt(process.argv[2] ?? '0', 10);
+
+  if (podIndex < 0 || podIndex >= POD_NAMES.length) {
+    console.error(`Invalid pod index: ${podIndex}. Must be 0-${POD_NAMES.length - 1}`);
+    process.exit(1);
+  }
+
+  const podName = POD_NAMES[podIndex];
+  console.log(`[Pod] Starting pod "${podName}" with ${WORKERS_PER_POD} workers...`);
+
+  const workerScript = path.join(__dirname, 'task-worker.ts');
+  const workers: ReturnType<typeof spawn>[] = [];
+
+  for (let i = 1; i <= WORKERS_PER_POD; i++) {
+    const workerId = `${podName}-worker-${i}`;
+
+    const worker = spawn('npx', ['tsx', workerScript], {
+      env: {
+        ...process.env,
+        WORKER_ID: workerId,
+        POD_NAME: podName,
+      },
+      stdio: 'inherit',
+    });
+
+    workers.push(worker);
+    console.log(`[Pod] Started worker ${i}/${WORKERS_PER_POD}: ${workerId}`);
+  }
+
+  // Handle shutdown
+  const shutdown = () => {
+    console.log(`\n[Pod] Shutting down pod "${podName}"...`);
+    workers.forEach(w => w.kill('SIGTERM'));
+    setTimeout(() => process.exit(0), 2000);
+  };
+
+  process.on('SIGTERM', shutdown);
+  process.on('SIGINT', shutdown);
+
+  // Keep the process alive
+  await new Promise(() => {});
+}
+
+main().catch(err => {
+  console.error('[Pod] Fatal error:', err);
+  process.exit(1);
+});

backend/src/tasks/task-pool-state.ts

@@ -0,0 +1,37 @@
+/**
+ * Task Pool State
+ *
+ * Shared state for task pool pause/resume functionality.
+ * This is kept separate to avoid circular dependencies between
+ * task-service.ts and routes/tasks.ts.
+ *
+ * State is in-memory and resets on server restart.
+ * By default, the pool is PAUSED (closed) - admin must explicitly start it.
+ * This prevents workers from immediately grabbing tasks on deploy before
+ * the system is ready.
+ */
+
+let taskPoolPaused = true;
+
+export function isTaskPoolPaused(): boolean {
+  return taskPoolPaused;
+}
+
+export function pauseTaskPool(): void {
+  taskPoolPaused = true;
+  console.log('[TaskPool] Task pool PAUSED - workers will not pick up new tasks');
+}
+
+export function resumeTaskPool(): void {
+  taskPoolPaused = false;
+  console.log('[TaskPool] Task pool RESUMED - workers can pick up tasks');
+}
+
+export function getTaskPoolStatus(): { paused: boolean; message: string } {
+  return {
+    paused: taskPoolPaused,
+    message: taskPoolPaused
+      ? 'Task pool is paused - workers will not pick up new tasks'
+      : 'Task pool is open - workers are picking up tasks',
+  };
+}

backend/src/tasks/task-service.ts

@@ -9,13 +9,30 @@
  */
 
 import { pool } from '../db/pool';
+import { isTaskPoolPaused } from './task-pool-state';
 
+// Helper to check if a table exists
+async function tableExists(tableName: string): Promise<boolean> {
+  const result = await pool.query(`
+    SELECT EXISTS (
+      SELECT FROM information_schema.tables
+      WHERE table_name = $1
+    ) as exists
+  `, [tableName]);
+  return result.rows[0].exists;
+}
+
+// Per TASK_WORKFLOW_2024-12-10.md: Task roles
+// payload_fetch: Hits Dutchie API, saves raw payload to filesystem
+// product_refresh: Reads local payload, normalizes, upserts to DB
 export type TaskRole =
   | 'store_discovery'
   | 'entry_point_discovery'
   | 'product_discovery'
-  | 'product_resync'
+  | 'payload_fetch'      // NEW: Fetches from API, saves to disk
-  | 'analytics_refresh';
+  | 'product_refresh'    // CHANGED: Now reads from local payload
+  | 'analytics_refresh'
+  | 'proxy_test';        // Tests proxy connectivity via ipify
 
 export type TaskStatus =
   | 'pending'
@@ -29,6 +46,8 @@ export interface WorkerTask {
   id: number;
   role: TaskRole;
   dispensary_id: number | null;
+  dispensary_name?: string;  // JOINed from dispensaries
+  dispensary_slug?: string;  // JOINed from dispensaries
   platform: string | null;
   status: TaskStatus;
   priority: number;
@@ -42,6 +61,7 @@ export interface WorkerTask {
   error_message: string | null;
   retry_count: number;
   max_retries: number;
+  payload: Record<string, unknown> | null;  // Per TASK_WORKFLOW_2024-12-10.md: Task chaining data
   created_at: Date;
   updated_at: Date;
 }
@@ -52,6 +72,7 @@ export interface CreateTaskParams {
   platform?: string;
   priority?: number;
   scheduled_for?: Date;
+  payload?: Record<string, unknown>;  // Per TASK_WORKFLOW_2024-12-10.md: For task chaining data
 }
 
 export interface CapacityMetrics {
@@ -83,8 +104,8 @@ class TaskService {
    */
   async createTask(params: CreateTaskParams): Promise<WorkerTask> {
     const result = await pool.query(
-      `INSERT INTO worker_tasks (role, dispensary_id, platform, priority, scheduled_for)
+      `INSERT INTO worker_tasks (role, dispensary_id, platform, priority, scheduled_for, payload)
-       VALUES ($1, $2, $3, $4, $5)
+       VALUES ($1, $2, $3, $4, $5, $6)
        RETURNING *`,
       [
         params.role,
@@ -92,6 +113,7 @@ class TaskService {
         params.platform ?? null,
         params.priority ?? 0,
         params.scheduled_for ?? null,
+        params.payload ? JSON.stringify(params.payload) : null,
       ]
     );
     return result.rows[0] as WorkerTask;
@@ -128,9 +150,17 @@ class TaskService {
 
   /**
    * Claim a task atomically for a worker
-   * Uses the SQL function for proper locking
+   * If role is null, claims ANY available task (role-agnostic worker)
+   * Returns null if task pool is paused.
    */
-  async claimTask(role: TaskRole, workerId: string): Promise<WorkerTask | null> {
+  async claimTask(role: TaskRole | null, workerId: string): Promise<WorkerTask | null> {
+    // Check if task pool is paused - don't claim any tasks
+    if (isTaskPoolPaused()) {
+      return null;
+    }
+
+    if (role) {
+      // Role-specific claiming - use the SQL function
       const result = await pool.query(
         `SELECT * FROM claim_task($1, $2)`,
         [role, workerId]
@@ -138,6 +168,33 @@ class TaskService {
       return (result.rows[0] as WorkerTask) || null;
     }
 
+    // Role-agnostic claiming - claim ANY pending task
+    const result = await pool.query(`
+      UPDATE worker_tasks
+      SET
+        status = 'claimed',
+        worker_id = $1,
+        claimed_at = NOW()
+      WHERE id = (
+        SELECT id FROM worker_tasks
+        WHERE status = 'pending'
+          AND (scheduled_for IS NULL OR scheduled_for <= NOW())
+          -- Exclude stores that already have an active task
+          AND (dispensary_id IS NULL OR dispensary_id NOT IN (
+            SELECT dispensary_id FROM worker_tasks
+            WHERE status IN ('claimed', 'running')
+            AND dispensary_id IS NOT NULL
+          ))
+        ORDER BY priority DESC, created_at ASC
+        LIMIT 1
+        FOR UPDATE SKIP LOCKED
+      )
+      RETURNING *
+    `, [workerId]);
+
+    return (result.rows[0] as WorkerTask) || null;
+  }
+
   /**
    * Mark a task as running (worker started processing)
    */
@@ -175,15 +232,53 @@ class TaskService {
   }
 
   /**
-   * Mark a task as failed
+   * Mark a task as failed, with auto-retry if under max_retries
+   * Returns true if task was re-queued for retry, false if permanently failed
    */
-  async failTask(taskId: number, errorMessage: string): Promise<void> {
+  async failTask(taskId: number, errorMessage: string): Promise<boolean> {
+    // Get current retry state
+    const result = await pool.query(
+      `SELECT retry_count, max_retries FROM worker_tasks WHERE id = $1`,
+      [taskId]
+    );
+
+    if (result.rows.length === 0) {
+      return false;
+    }
+
+    const { retry_count, max_retries } = result.rows[0];
+    const newRetryCount = (retry_count || 0) + 1;
+
+    if (newRetryCount < (max_retries || 3)) {
+      // Re-queue for retry - reset to pending with incremented retry_count
       await pool.query(
         `UPDATE worker_tasks
-       SET status = 'failed', completed_at = NOW(), error_message = $2
+         SET status = 'pending',
+             worker_id = NULL,
+             claimed_at = NULL,
+             started_at = NULL,
+             retry_count = $2,
+             error_message = $3,
+             updated_at = NOW()
          WHERE id = $1`,
-      [taskId, errorMessage]
+        [taskId, newRetryCount, `Retry ${newRetryCount}: ${errorMessage}`]
       );
+      console.log(`[TaskService] Task ${taskId} queued for retry ${newRetryCount}/${max_retries || 3}`);
+      return true;
+    }
+
+    // Max retries exceeded - mark as permanently failed
+    await pool.query(
+      `UPDATE worker_tasks
+       SET status = 'failed',
+           completed_at = NOW(),
+           retry_count = $2,
+           error_message = $3
+       WHERE id = $1`,
+      [taskId, newRetryCount, `Failed after ${newRetryCount} attempts: ${errorMessage}`]
+    );
+    console.log(`[TaskService] Task ${taskId} permanently failed after ${newRetryCount} attempts`);
+    return false;
   }
 
   /**
@@ -201,32 +296,37 @@ class TaskService {
    * List tasks with filters
    */
   async listTasks(filter: TaskFilter = {}): Promise<WorkerTask[]> {
+    // Return empty list if table doesn't exist
+    if (!await tableExists('worker_tasks')) {
+      return [];
+    }
+
     const conditions: string[] = [];
     const params: (string | number | string[])[] = [];
     let paramIndex = 1;
 
     if (filter.role) {
-      conditions.push(`role = $${paramIndex++}`);
+      conditions.push(`t.role = $${paramIndex++}`);
       params.push(filter.role);
     }
 
     if (filter.status) {
       if (Array.isArray(filter.status)) {
-        conditions.push(`status = ANY($${paramIndex++})`);
+        conditions.push(`t.status = ANY($${paramIndex++})`);
         params.push(filter.status);
       } else {
-        conditions.push(`status = $${paramIndex++}`);
+        conditions.push(`t.status = $${paramIndex++}`);
         params.push(filter.status);
       }
     }
 
     if (filter.dispensary_id) {
-      conditions.push(`dispensary_id = $${paramIndex++}`);
+      conditions.push(`t.dispensary_id = $${paramIndex++}`);
       params.push(filter.dispensary_id);
     }
 
     if (filter.worker_id) {
-      conditions.push(`worker_id = $${paramIndex++}`);
+      conditions.push(`t.worker_id = $${paramIndex++}`);
       params.push(filter.worker_id);
     }
 
@@ -235,9 +335,14 @@ class TaskService {
     const offset = filter.offset ?? 0;
 
     const result = await pool.query(
-      `SELECT * FROM worker_tasks
+      `SELECT
+         t.*,
+         d.name as dispensary_name,
+         d.slug as dispensary_slug
+       FROM worker_tasks t
+       LEFT JOIN dispensaries d ON d.id = t.dispensary_id
        ${whereClause}
-       ORDER BY created_at DESC
+       ORDER BY t.created_at DESC
        LIMIT ${limit} OFFSET ${offset}`,
       params
     );
@@ -249,21 +354,41 @@ class TaskService {
    * Get capacity metrics for all roles
    */
   async getCapacityMetrics(): Promise<CapacityMetrics[]> {
+    // Return empty metrics if worker_tasks table doesn't exist
+    if (!await tableExists('worker_tasks')) {
+      return [];
+    }
+
+    try {
       const result = await pool.query(
         `SELECT * FROM v_worker_capacity`
       );
       return result.rows as CapacityMetrics[];
+    } catch {
+      // View may not exist
+      return [];
+    }
   }
 
   /**
    * Get capacity metrics for a specific role
    */
   async getRoleCapacity(role: TaskRole): Promise<CapacityMetrics | null> {
+    // Return null if worker_tasks table doesn't exist
+    if (!await tableExists('worker_tasks')) {
+      return null;
+    }
+
+    try {
       const result = await pool.query(
         `SELECT * FROM v_worker_capacity WHERE role = $1`,
         [role]
       );
       return (result.rows[0] as CapacityMetrics) || null;
+    } catch {
+      // View may not exist
+      return null;
+    }
   }
 
   /**
@@ -291,6 +416,17 @@ class TaskService {
   /**
    * Chain next task after completion
    * Called automatically when a task completes successfully
+   *
+   * Per TASK_WORKFLOW_2024-12-10.md: Task chaining flow:
+   *
+   * Discovery flow (new stores):
+   *   store_discovery → product_discovery → payload_fetch → product_refresh
+   *
+   * Scheduled flow (existing stores):
+   *   payload_fetch → product_refresh
+   *
+   * Note: entry_point_discovery is deprecated since platform_dispensary_id
+   * is now resolved during store promotion.
    */
   async chainNextTask(completedTask: WorkerTask): Promise<WorkerTask | null> {
     if (completedTask.status !== 'completed') {
@@ -299,12 +435,14 @@ class TaskService {
 
     switch (completedTask.role) {
       case 'store_discovery': {
-        // New stores discovered -> create entry_point_discovery tasks
+        // Per TASK_WORKFLOW_2024-12-10.md: New stores discovered -> create product_discovery tasks
+        // Skip entry_point_discovery since platform_dispensary_id is set during promotion
         const newStoreIds = (completedTask.result as { newStoreIds?: number[] })?.newStoreIds;
         if (newStoreIds && newStoreIds.length > 0) {
+          console.log(`[TaskService] Chaining ${newStoreIds.length} product_discovery tasks for new stores`);
           for (const storeId of newStoreIds) {
             await this.createTask({
-              role: 'entry_point_discovery',
+              role: 'product_discovery',
               dispensary_id: storeId,
               platform: completedTask.platform ?? undefined,
               priority: 10, // High priority for new stores
@@ -315,7 +453,8 @@ class TaskService {
       }
 
       case 'entry_point_discovery': {
-        // Entry point resolved -> create product_discovery task
+        // DEPRECATED: Entry point resolution now happens during store promotion
+        // Kept for backward compatibility with any in-flight tasks
         const success = (completedTask.result as { success?: boolean })?.success;
         if (success && completedTask.dispensary_id) {
           return this.createTask({
@@ -329,8 +468,15 @@ class TaskService {
       }
 
       case 'product_discovery': {
-        // Product discovery done -> store is now ready for regular resync
+        // Per TASK_WORKFLOW_2024-12-10.md: Product discovery chains internally to payload_fetch
-        // No immediate chaining needed; will be picked up by daily batch generation
+        // No external chaining needed - handleProductDiscovery calls handlePayloadFetch directly
+        break;
+      }
+
+      case 'payload_fetch': {
+        // Per TASK_WORKFLOW_2024-12-10.md: payload_fetch chains to product_refresh
+        // This is handled internally by the payload_fetch handler via taskService.createTask
+        // No external chaining needed here
         break;
       }
     }
@@ -389,12 +535,6 @@ class TaskService {
    * Get task counts by status for dashboard
    */
   async getTaskCounts(): Promise<Record<TaskStatus, number>> {
-    const result = await pool.query(
-      `SELECT status, COUNT(*) as count
-       FROM worker_tasks
-       GROUP BY status`
-    );
-
     const counts: Record<TaskStatus, number> = {
       pending: 0,
       claimed: 0,
@@ -404,6 +544,17 @@ class TaskService {
       stale: 0,
     };
 
+    // Return empty counts if table doesn't exist
+    if (!await tableExists('worker_tasks')) {
+      return counts;
+    }
+
+    const result = await pool.query(
+      `SELECT status, COUNT(*) as count
+       FROM worker_tasks
+       GROUP BY status`
+    );
+
     for (const row of result.rows) {
       const typedRow = row as { status: TaskStatus; count: string };
       counts[typedRow.status] = parseInt(typedRow.count, 10);

backend/src/tasks/task-worker.ts

@@ -1,33 +1,110 @@
 /**
  * Task Worker
  *
- * A unified worker that processes tasks from the worker_tasks queue.
+ * A unified worker that pulls tasks from the worker_tasks queue.
- * Replaces the fragmented job systems (job_schedules, dispensary_crawl_jobs, etc.)
+ * Workers register on startup, get a friendly name, and pull tasks.
+ *
+ * Architecture:
+ *   - Tasks are generated on schedule (by scheduler or API)
+ *   - Workers PULL tasks from the pool (not assigned to them)
+ *   - Tasks are claimed in order of priority (DESC) then creation time (ASC)
+ *   - Workers report heartbeats to worker_registry
+ *   - Workers are ROLE-AGNOSTIC by default (can handle any task type)
+ *
+ * Stealth & Anti-Detection:
+ *   PROXIES ARE REQUIRED - workers will fail to start if no proxies available.
+ *
+ *   On startup, workers initialize the CrawlRotator which provides:
+ *   - Proxy rotation: Loads proxies from `proxies` table, ALL requests use proxy
+ *   - User-Agent rotation: Cycles through realistic browser fingerprints
+ *   - Fingerprint rotation: Changes browser profile on blocks
+ *   - Locale/timezone: Matches Accept-Language to target state
+ *
+ *   The CrawlRotator is wired to the Dutchie client via setCrawlRotator().
+ *   Task handlers call startSession() which picks a random fingerprint.
+ *   On 403 errors, the client automatically:
+ *     1. Records failure on current proxy
+ *     2. Rotates to next proxy
+ *     3. Rotates fingerprint
+ *     4. Retries the request
  *
  * Usage:
- *   WORKER_ROLE=product_resync npx tsx src/tasks/task-worker.ts
+ *   npx tsx src/tasks/task-worker.ts                    # Role-agnostic (any task)
+ *   WORKER_ROLE=product_refresh npx tsx src/tasks/task-worker.ts  # Role-specific
  *
  * Environment:
- *   WORKER_ROLE - Which task role to process (required)
+ *   WORKER_ROLE - Which task role to process (optional, null = any task)
- *   WORKER_ID - Optional custom worker ID
+ *   WORKER_ID - Optional custom worker ID (auto-generated if not provided)
+ *   POD_NAME - Kubernetes pod name (optional)
  *   POLL_INTERVAL_MS - How often to check for tasks (default: 5000)
  *   HEARTBEAT_INTERVAL_MS - How often to update heartbeat (default: 30000)
+ *   API_BASE_URL - Backend API URL for registration (default: http://localhost:3010)
  */
 
 import { Pool } from 'pg';
 import { v4 as uuidv4 } from 'uuid';
 import { taskService, TaskRole, WorkerTask } from './task-service';
 import { getPool } from '../db/pool';
+import os from 'os';
+
+// Stealth/rotation support
+import { CrawlRotator } from '../services/crawl-rotator';
+import { setCrawlRotator } from '../platforms/dutchie';
 
 // Task handlers by role
-import { handleProductResync } from './handlers/product-resync';
+// Per TASK_WORKFLOW_2024-12-10.md: payload_fetch and product_refresh are now separate
+import { handlePayloadFetch } from './handlers/payload-fetch';
+import { handleProductRefresh } from './handlers/product-refresh';
 import { handleProductDiscovery } from './handlers/product-discovery';
 import { handleStoreDiscovery } from './handlers/store-discovery';
 import { handleEntryPointDiscovery } from './handlers/entry-point-discovery';
 import { handleAnalyticsRefresh } from './handlers/analytics-refresh';
+import { handleProxyTest } from './handlers/proxy-test';
 
 const POLL_INTERVAL_MS = parseInt(process.env.POLL_INTERVAL_MS || '5000');
 const HEARTBEAT_INTERVAL_MS = parseInt(process.env.HEARTBEAT_INTERVAL_MS || '30000');
+const API_BASE_URL = process.env.API_BASE_URL || 'http://localhost:3010';
+
+// =============================================================================
+// CONCURRENT TASK PROCESSING SETTINGS
+// =============================================================================
+// Workers can process multiple tasks simultaneously using async I/O.
+// This improves throughput for I/O-bound tasks (network calls, DB queries).
+//
+// Resource thresholds trigger "backoff" - the worker stops claiming new tasks
+// but continues processing existing ones until resources return to normal.
+//
+// See: docs/WORKER_TASK_ARCHITECTURE.md#concurrent-task-processing
+// =============================================================================
+
+// Maximum number of tasks this worker will run concurrently
+// Tune based on workload: I/O-bound tasks benefit from higher concurrency
+const MAX_CONCURRENT_TASKS = parseInt(process.env.MAX_CONCURRENT_TASKS || '3');
+
+// When heap memory usage exceeds this threshold (as decimal 0.0-1.0), stop claiming new tasks
+// Default 85% - gives headroom before OOM
+const MEMORY_BACKOFF_THRESHOLD = parseFloat(process.env.MEMORY_BACKOFF_THRESHOLD || '0.85');
+
+// Parse max heap size from NODE_OPTIONS (--max-old-space-size=1500)
+// This is used as the denominator for memory percentage calculation
+// V8's heapTotal is dynamic and stays small when idle, causing false high percentages
+function getMaxHeapSizeMb(): number {
+  const nodeOptions = process.env.NODE_OPTIONS || '';
+  const match = nodeOptions.match(/--max-old-space-size=(\d+)/);
+  if (match) {
+    return parseInt(match[1], 10);
+  }
+  // Fallback: use 512MB if not specified
+  return 512;
+}
+const MAX_HEAP_SIZE_MB = getMaxHeapSizeMb();
+
+// When CPU usage exceeds this threshold (as decimal 0.0-1.0), stop claiming new tasks
+// Default 90% - allows some burst capacity
+const CPU_BACKOFF_THRESHOLD = parseFloat(process.env.CPU_BACKOFF_THRESHOLD || '0.90');
+
+// How long to wait (ms) when in backoff state before rechecking resources
+const BACKOFF_DURATION_MS = parseInt(process.env.BACKOFF_DURATION_MS || '10000');
 
 export interface TaskContext {
   pool: Pool;
@@ -47,26 +124,361 @@ export interface TaskResult {
 
 type TaskHandler = (ctx: TaskContext) => Promise<TaskResult>;
 
+// Per TASK_WORKFLOW_2024-12-10.md: Handler registry
+// payload_fetch: Fetches from Dutchie API, saves to disk, chains to product_refresh
+// product_refresh: Reads local payload, normalizes, upserts to DB
 const TASK_HANDLERS: Record<TaskRole, TaskHandler> = {
-  product_resync: handleProductResync,
+  payload_fetch: handlePayloadFetch,       // NEW: API fetch -> disk
+  product_refresh: handleProductRefresh,   // CHANGED: disk -> DB
   product_discovery: handleProductDiscovery,
   store_discovery: handleStoreDiscovery,
   entry_point_discovery: handleEntryPointDiscovery,
   analytics_refresh: handleAnalyticsRefresh,
+  proxy_test: handleProxyTest,             // Tests proxy via ipify
 };
 
+/**
+ * Resource usage stats reported to the registry and used for backoff decisions.
+ * These values are included in worker heartbeats and displayed in the UI.
+ */
+interface ResourceStats {
+  /** Current heap memory usage as decimal (0.0 to 1.0) */
+  memoryPercent: number;
+  /** Current heap used in MB */
+  memoryMb: number;
+  /** Total heap available in MB */
+  memoryTotalMb: number;
+  /** CPU usage percentage since last check (0 to 100) */
+  cpuPercent: number;
+  /** True if worker is currently in backoff state */
+  isBackingOff: boolean;
+  /** Reason for backoff (e.g., "Memory at 87.3% (threshold: 85%)") */
+  backoffReason: string | null;
+}
+
 export class TaskWorker {
   private pool: Pool;
   private workerId: string;
-  private role: TaskRole;
+  private role: TaskRole | null;  // null = role-agnostic (any task)
+  private friendlyName: string = '';
   private isRunning: boolean = false;
   private heartbeatInterval: NodeJS.Timeout | null = null;
-  private currentTask: WorkerTask | null = null;
+  private registryHeartbeatInterval: NodeJS.Timeout | null = null;
+  private crawlRotator: CrawlRotator;
 
-  constructor(role: TaskRole, workerId?: string) {
+  // ==========================================================================
+  // CONCURRENT TASK TRACKING
+  // ==========================================================================
+  // activeTasks: Map of task ID -> task object for all currently running tasks
+  // taskPromises: Map of task ID -> Promise for cleanup when task completes
+  // maxConcurrentTasks: How many tasks this worker will run in parallel
+  // ==========================================================================
+  private activeTasks: Map<number, WorkerTask> = new Map();
+  private taskPromises: Map<number, Promise<void>> = new Map();
+  private maxConcurrentTasks: number = MAX_CONCURRENT_TASKS;
+
+  // ==========================================================================
+  // RESOURCE MONITORING FOR BACKOFF
+  // ==========================================================================
+  // CPU tracking uses differential measurement - we track last values and
+  // calculate percentage based on elapsed time since last check.
+  // ==========================================================================
+  private lastCpuUsage: { user: number; system: number } = { user: 0, system: 0 };
+  private lastCpuCheck: number = Date.now();
+  private isBackingOff: boolean = false;
+  private backoffReason: string | null = null;
+
+  constructor(role: TaskRole | null = null, workerId?: string) {
     this.pool = getPool();
     this.role = role;
-    this.workerId = workerId || `worker-${role}-${uuidv4().slice(0, 8)}`;
+    this.workerId = workerId || `worker-${uuidv4().slice(0, 8)}`;
+    this.crawlRotator = new CrawlRotator(this.pool);
+
+    // Initialize CPU tracking
+    const cpuUsage = process.cpuUsage();
+    this.lastCpuUsage = { user: cpuUsage.user, system: cpuUsage.system };
+    this.lastCpuCheck = Date.now();
+  }
+
+  /**
+   * Get current resource usage
+   * Memory percentage is calculated against MAX_HEAP_SIZE_MB (from --max-old-space-size)
+   * NOT against V8's dynamic heapTotal which stays small when idle
+   */
+  private getResourceStats(): ResourceStats {
+    const memUsage = process.memoryUsage();
+    const heapUsedMb = memUsage.heapUsed / 1024 / 1024;
+    // Use MAX_HEAP_SIZE_MB as ceiling, not dynamic heapTotal
+    // V8's heapTotal stays small when idle (e.g., 36MB) causing false 95%+ readings
+    // With --max-old-space-size=1500, we should calculate against 1500MB
+    const memoryPercent = heapUsedMb / MAX_HEAP_SIZE_MB;
+
+    // Calculate CPU usage since last check
+    const cpuUsage = process.cpuUsage();
+    const now = Date.now();
+    const elapsed = now - this.lastCpuCheck;
+
+    let cpuPercent = 0;
+    if (elapsed > 0) {
+      const userDiff = (cpuUsage.user - this.lastCpuUsage.user) / 1000; // microseconds to ms
+      const systemDiff = (cpuUsage.system - this.lastCpuUsage.system) / 1000;
+      cpuPercent = ((userDiff + systemDiff) / elapsed) * 100;
+    }
+
+    // Update last values
+    this.lastCpuUsage = { user: cpuUsage.user, system: cpuUsage.system };
+    this.lastCpuCheck = now;
+
+    return {
+      memoryPercent,
+      memoryMb: Math.round(heapUsedMb),
+      memoryTotalMb: MAX_HEAP_SIZE_MB, // Use max-old-space-size, not dynamic heapTotal
+      cpuPercent: Math.min(100, cpuPercent), // Cap at 100%
+      isBackingOff: this.isBackingOff,
+      backoffReason: this.backoffReason,
+    };
+  }
+
+  /**
+   * Check if we should back off from taking new tasks
+   */
+  private shouldBackOff(): { backoff: boolean; reason: string | null } {
+    const stats = this.getResourceStats();
+
+    if (stats.memoryPercent > MEMORY_BACKOFF_THRESHOLD) {
+      return { backoff: true, reason: `Memory at ${(stats.memoryPercent * 100).toFixed(1)}% (threshold: ${MEMORY_BACKOFF_THRESHOLD * 100}%)` };
+    }
+
+    if (stats.cpuPercent > CPU_BACKOFF_THRESHOLD * 100) {
+      return { backoff: true, reason: `CPU at ${stats.cpuPercent.toFixed(1)}% (threshold: ${CPU_BACKOFF_THRESHOLD * 100}%)` };
+    }
+
+    return { backoff: false, reason: null };
+  }
+
+  /**
+   * Get count of currently running tasks
+   */
+  get activeTaskCount(): number {
+    return this.activeTasks.size;
+  }
+
+  /**
+   * Check if we can accept more tasks
+   */
+  private canAcceptMoreTasks(): boolean {
+    return this.activeTasks.size < this.maxConcurrentTasks;
+  }
+
+  /**
+   * Initialize stealth systems (proxy rotation, fingerprints)
+   * Called once on worker startup before processing any tasks.
+   *
+   * IMPORTANT: Proxies are REQUIRED. Workers will wait until proxies are available.
+   * Workers listen for PostgreSQL NOTIFY 'proxy_added' to wake up immediately when proxies are added.
+   */
+  private async initializeStealth(): Promise<void> {
+    const MAX_WAIT_MINUTES = 60;
+    const POLL_INTERVAL_MS = 30000; // 30 seconds fallback polling
+    const maxAttempts = (MAX_WAIT_MINUTES * 60 * 1000) / POLL_INTERVAL_MS;
+    let attempts = 0;
+    let notifyClient: any = null;
+
+    // Set up PostgreSQL LISTEN for proxy notifications
+    try {
+      notifyClient = await this.pool.connect();
+      await notifyClient.query('LISTEN proxy_added');
+      console.log(`[TaskWorker] Listening for proxy_added notifications...`);
+    } catch (err: any) {
+      console.log(`[TaskWorker] Could not set up LISTEN (will poll): ${err.message}`);
+    }
+
+    // Create a promise that resolves when notified
+    let notifyResolve: (() => void) | null = null;
+    if (notifyClient) {
+      notifyClient.on('notification', (msg: any) => {
+        if (msg.channel === 'proxy_added') {
+          console.log(`[TaskWorker] Received proxy_added notification!`);
+          if (notifyResolve) notifyResolve();
+        }
+      });
+    }
+
+    try {
+      while (attempts < maxAttempts) {
+        try {
+          // Load proxies from database
+          await this.crawlRotator.initialize();
+
+          const stats = this.crawlRotator.proxy.getStats();
+          if (stats.activeProxies > 0) {
+            console.log(`[TaskWorker] Loaded ${stats.activeProxies} proxies (${stats.avgSuccessRate.toFixed(1)}% avg success rate)`);
+
+            // Wire rotator to Dutchie client - proxies will be used for ALL requests
+            setCrawlRotator(this.crawlRotator);
+
+            console.log(`[TaskWorker] Stealth initialized: ${this.crawlRotator.userAgent.getCount()} fingerprints, proxy REQUIRED for all requests`);
+            return;
+          }
+
+          attempts++;
+          console.log(`[TaskWorker] No active proxies available (attempt ${attempts}). Waiting for proxies...`);
+
+          // Wait for either notification or timeout
+          await new Promise<void>((resolve) => {
+            notifyResolve = resolve;
+            setTimeout(resolve, POLL_INTERVAL_MS);
+          });
+        } catch (error: any) {
+          attempts++;
+          console.log(`[TaskWorker] Error loading proxies (attempt ${attempts}): ${error.message}. Retrying...`);
+          await this.sleep(POLL_INTERVAL_MS);
+        }
+      }
+
+      throw new Error(`No active proxies available after waiting ${MAX_WAIT_MINUTES} minutes. Add proxies to the database.`);
+    } finally {
+      // Clean up LISTEN connection
+      if (notifyClient) {
+        try {
+          await notifyClient.query('UNLISTEN proxy_added');
+          notifyClient.release();
+        } catch {
+          // Ignore cleanup errors
+        }
+      }
+    }
+  }
+
+  /**
+   * Register worker with the registry (get friendly name)
+   */
+  private async register(): Promise<void> {
+    try {
+      const response = await fetch(`${API_BASE_URL}/api/worker-registry/register`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          role: this.role,
+          worker_id: this.workerId,
+          pod_name: process.env.POD_NAME || process.env.HOSTNAME,
+          hostname: os.hostname(),
+          metadata: {
+            pid: process.pid,
+            node_version: process.version,
+            started_at: new Date().toISOString()
+          }
+        })
+      });
+
+      const data = await response.json();
+      if (data.success) {
+        this.friendlyName = data.friendly_name;
+        console.log(`[TaskWorker] ${data.message}`);
+      } else {
+        console.warn(`[TaskWorker] Registration warning: ${data.error}`);
+        this.friendlyName = this.workerId.slice(0, 12);
+      }
+    } catch (error: any) {
+      // Registration is optional - worker can still function without it
+      console.warn(`[TaskWorker] Could not register with API (will continue): ${error.message}`);
+      this.friendlyName = this.workerId.slice(0, 12);
+    }
+  }
+
+  /**
+   * Deregister worker from the registry
+   */
+  private async deregister(): Promise<void> {
+    try {
+      await fetch(`${API_BASE_URL}/api/worker-registry/deregister`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ worker_id: this.workerId })
+      });
+      console.log(`[TaskWorker] ${this.friendlyName} signed off`);
+    } catch {
+      // Ignore deregistration errors
+    }
+  }
+
+  /**
+   * Send heartbeat to registry with resource usage and proxy location
+   */
+  private async sendRegistryHeartbeat(): Promise<void> {
+    try {
+      const memUsage = process.memoryUsage();
+      const cpuUsage = process.cpuUsage();
+      const proxyLocation = this.crawlRotator.getProxyLocation();
+      const resourceStats = this.getResourceStats();
+
+      // Get array of active task IDs
+      const activeTaskIds = Array.from(this.activeTasks.keys());
+
+      await fetch(`${API_BASE_URL}/api/worker-registry/heartbeat`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          worker_id: this.workerId,
+          current_task_id: activeTaskIds[0] || null, // Primary task for backwards compat
+          current_task_ids: activeTaskIds, // All active tasks
+          active_task_count: this.activeTasks.size,
+          max_concurrent_tasks: this.maxConcurrentTasks,
+          status: this.activeTasks.size > 0 ? 'active' : 'idle',
+          resources: {
+            memory_mb: Math.round(memUsage.heapUsed / 1024 / 1024),
+            memory_total_mb: Math.round(memUsage.heapTotal / 1024 / 1024),
+            memory_rss_mb: Math.round(memUsage.rss / 1024 / 1024),
+            memory_percent: Math.round(resourceStats.memoryPercent * 100),
+            cpu_user_ms: Math.round(cpuUsage.user / 1000),
+            cpu_system_ms: Math.round(cpuUsage.system / 1000),
+            cpu_percent: Math.round(resourceStats.cpuPercent),
+            proxy_location: proxyLocation,
+            is_backing_off: this.isBackingOff,
+            backoff_reason: this.backoffReason,
+          }
+        })
+      });
+    } catch {
+      // Ignore heartbeat errors
+    }
+  }
+
+  /**
+   * Report task completion to registry
+   */
+  private async reportTaskCompletion(success: boolean): Promise<void> {
+    try {
+      await fetch(`${API_BASE_URL}/api/worker-registry/task-completed`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          worker_id: this.workerId,
+          success
+        })
+      });
+    } catch {
+      // Ignore errors
+    }
+  }
+
+  /**
+   * Start registry heartbeat interval
+   */
+  private startRegistryHeartbeat(): void {
+    this.registryHeartbeatInterval = setInterval(async () => {
+      await this.sendRegistryHeartbeat();
+    }, HEARTBEAT_INTERVAL_MS);
+  }
+
+  /**
+   * Stop registry heartbeat interval
+   */
+  private stopRegistryHeartbeat(): void {
+    if (this.registryHeartbeatInterval) {
+      clearInterval(this.registryHeartbeatInterval);
+      this.registryHeartbeatInterval = null;
+    }
   }
 
   /**
@@ -74,47 +486,112 @@ export class TaskWorker {
    */
   async start(): Promise<void> {
     this.isRunning = true;
-    console.log(`[TaskWorker] Starting worker ${this.workerId} for role: ${this.role}`);
+
+    // Initialize stealth systems (proxy rotation, fingerprints)
+    await this.initializeStealth();
+
+    // Register with the API to get a friendly name
+    await this.register();
+
+    // Start registry heartbeat
+    this.startRegistryHeartbeat();
+
+    const roleMsg = this.role ? `for role: ${this.role}` : '(role-agnostic - any task)';
+    console.log(`[TaskWorker] ${this.friendlyName} starting ${roleMsg} (max ${this.maxConcurrentTasks} concurrent tasks)`);
 
     while (this.isRunning) {
       try {
-        await this.processNextTask();
+        await this.mainLoop();
       } catch (error: any) {
         console.error(`[TaskWorker] Loop error:`, error.message);
         await this.sleep(POLL_INTERVAL_MS);
       }
     }
 
+    // Wait for any remaining tasks to complete
+    if (this.taskPromises.size > 0) {
+      console.log(`[TaskWorker] Waiting for ${this.taskPromises.size} active tasks to complete...`);
+      await Promise.allSettled(this.taskPromises.values());
+    }
+
     console.log(`[TaskWorker] Worker ${this.workerId} stopped`);
   }
 
+  /**
+   * Main loop - tries to fill up to maxConcurrentTasks
+   */
+  private async mainLoop(): Promise<void> {
+    // Check resource usage and backoff if needed
+    const { backoff, reason } = this.shouldBackOff();
+    if (backoff) {
+      if (!this.isBackingOff) {
+        console.log(`[TaskWorker] ${this.friendlyName} backing off: ${reason}`);
+      }
+      this.isBackingOff = true;
+      this.backoffReason = reason;
+      await this.sleep(BACKOFF_DURATION_MS);
+      return;
+    }
+
+    // Clear backoff state
+    if (this.isBackingOff) {
+      console.log(`[TaskWorker] ${this.friendlyName} resuming normal operation`);
+      this.isBackingOff = false;
+      this.backoffReason = null;
+    }
+
+    // Check for decommission signal
+    const shouldDecommission = await this.checkDecommission();
+    if (shouldDecommission) {
+      console.log(`[TaskWorker] ${this.friendlyName} received decommission signal - waiting for ${this.activeTasks.size} tasks to complete`);
+      // Stop accepting new tasks, wait for current to finish
+      this.isRunning = false;
+      return;
+    }
+
+    // Try to claim more tasks if we have capacity
+    if (this.canAcceptMoreTasks()) {
+      const task = await taskService.claimTask(this.role, this.workerId);
+
+      if (task) {
+        console.log(`[TaskWorker] ${this.friendlyName} claimed task ${task.id} (${task.role}) [${this.activeTasks.size + 1}/${this.maxConcurrentTasks}]`);
+        this.activeTasks.set(task.id, task);
+
+        // Start task in background (don't await)
+        const taskPromise = this.executeTask(task);
+        this.taskPromises.set(task.id, taskPromise);
+
+        // Clean up when done
+        taskPromise.finally(() => {
+          this.activeTasks.delete(task.id);
+          this.taskPromises.delete(task.id);
+        });
+
+        // Immediately try to claim more tasks (don't wait for poll interval)
+        return;
+      }
+    }
+
+    // No task claimed or at capacity - wait before next poll
+    await this.sleep(POLL_INTERVAL_MS);
+  }
+
   /**
    * Stop the worker
    */
-  stop(): void {
+  async stop(): Promise<void> {
     this.isRunning = false;
     this.stopHeartbeat();
-    console.log(`[TaskWorker] Stopping worker ${this.workerId}...`);
+    this.stopRegistryHeartbeat();
+    await this.deregister();
+    console.log(`[TaskWorker] ${this.friendlyName} stopped`);
   }
 
   /**
-   * Process the next available task
+   * Execute a single task (runs concurrently with other tasks)
    */
-  private async processNextTask(): Promise<void> {
+  private async executeTask(task: WorkerTask): Promise<void> {
-    // Try to claim a task
+    console.log(`[TaskWorker] ${this.friendlyName} starting task ${task.id} (${task.role}) for dispensary ${task.dispensary_id || 'N/A'}`);
-    const task = await taskService.claimTask(this.role, this.workerId);
-
-    if (!task) {
-      // No tasks available, wait and retry
-      await this.sleep(POLL_INTERVAL_MS);
-      return;
-    }
-
-    this.currentTask = task;
-    console.log(`[TaskWorker] Claimed task ${task.id} (${task.role}) for dispensary ${task.dispensary_id || 'N/A'}`);
-
-    // Start heartbeat
-    this.startHeartbeat(task.id);
 
     try {
       // Mark as running
@@ -142,7 +619,8 @@ export class TaskWorker {
       if (result.success) {
         // Mark as completed
         await taskService.completeTask(task.id, result);
-        console.log(`[TaskWorker] Task ${task.id} completed successfully`);
+        await this.reportTaskCompletion(true);
+        console.log(`[TaskWorker] ${this.friendlyName} completed task ${task.id} [${this.activeTasks.size}/${this.maxConcurrentTasks} active]`);
 
         // Chain next task if applicable
         const chainedTask = await taskService.chainNextTask({
@@ -156,15 +634,43 @@ export class TaskWorker {
       } else {
         // Mark as failed
         await taskService.failTask(task.id, result.error || 'Unknown error');
-        console.log(`[TaskWorker] Task ${task.id} failed: ${result.error}`);
+        await this.reportTaskCompletion(false);
+        console.log(`[TaskWorker] ${this.friendlyName} failed task ${task.id}: ${result.error}`);
       }
     } catch (error: any) {
       // Mark as failed
       await taskService.failTask(task.id, error.message);
-      console.error(`[TaskWorker] Task ${task.id} threw error:`, error.message);
+      await this.reportTaskCompletion(false);
-    } finally {
+      console.error(`[TaskWorker] ${this.friendlyName} task ${task.id} error:`, error.message);
-      this.stopHeartbeat();
+    }
-      this.currentTask = null;
+    // Note: cleanup (removing from activeTasks) is handled in mainLoop's finally block
+  }
+
+  /**
+   * Check if this worker has been flagged for decommission
+   * Returns true if worker should stop after current task
+   */
+  private async checkDecommission(): Promise<boolean> {
+    try {
+      // Check worker_registry for decommission flag
+      const result = await this.pool.query(
+        `SELECT decommission_requested, decommission_reason
+         FROM worker_registry
+         WHERE worker_id = $1`,
+        [this.workerId]
+      );
+
+      if (result.rows.length > 0 && result.rows[0].decommission_requested) {
+        const reason = result.rows[0].decommission_reason || 'No reason provided';
+        console.log(`[TaskWorker] Decommission requested: ${reason}`);
+        return true;
+      }
+
+      return false;
+    } catch (error: any) {
+      // If we can't check, continue running
+      console.warn(`[TaskWorker] Could not check decommission status: ${error.message}`);
+      return false;
     }
   }
 
@@ -201,12 +707,25 @@ export class TaskWorker {
   /**
    * Get worker info
    */
-  getInfo(): { workerId: string; role: TaskRole; isRunning: boolean; currentTaskId: number | null } {
+  getInfo(): {
+    workerId: string;
+    role: TaskRole | null;
+    isRunning: boolean;
+    activeTaskIds: number[];
+    activeTaskCount: number;
+    maxConcurrentTasks: number;
+    isBackingOff: boolean;
+    backoffReason: string | null;
+  } {
     return {
       workerId: this.workerId,
       role: this.role,
       isRunning: this.isRunning,
-      currentTaskId: this.currentTask?.id || null,
+      activeTaskIds: Array.from(this.activeTasks.keys()),
+      activeTaskCount: this.activeTasks.size,
+      maxConcurrentTasks: this.maxConcurrentTasks,
+      isBackingOff: this.isBackingOff,
+      backoffReason: this.backoffReason,
     };
   }
 }
@@ -216,30 +735,29 @@ export class TaskWorker {
 // ============================================================
 
 async function main(): Promise<void> {
-  const role = process.env.WORKER_ROLE as TaskRole;
+  const role = process.env.WORKER_ROLE as TaskRole | undefined;
-
-  if (!role) {
-    console.error('Error: WORKER_ROLE environment variable is required');
-    console.error('Valid roles: store_discovery, entry_point_discovery, product_discovery, product_resync, analytics_refresh');
-    process.exit(1);
-  }
 
+  // Per TASK_WORKFLOW_2024-12-10.md: Valid task roles
   const validRoles: TaskRole[] = [
     'store_discovery',
     'entry_point_discovery',
     'product_discovery',
-    'product_resync',
+    'payload_fetch',      // NEW: Fetches from API, saves to disk
+    'product_refresh',    // CHANGED: Reads from disk, processes to DB
     'analytics_refresh',
   ];
 
-  if (!validRoles.includes(role)) {
+  // If role specified, validate it
+  if (role && !validRoles.includes(role)) {
     console.error(`Error: Invalid WORKER_ROLE: ${role}`);
     console.error(`Valid roles: ${validRoles.join(', ')}`);
+    console.error('Or omit WORKER_ROLE for role-agnostic worker (any task)');
     process.exit(1);
   }
 
   const workerId = process.env.WORKER_ID;
-  const worker = new TaskWorker(role, workerId);
+  // Pass null for role-agnostic, or the specific role
+  const worker = new TaskWorker(role || null, workerId);
 
   // Handle graceful shutdown
   process.on('SIGTERM', () => {

backend/src/types/user-agents.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Type declarations for user-agents npm package
+ * Per workflow-12102025.md: Used for realistic UA generation with market-share weighting
+ */
+
+declare module 'user-agents' {
+  interface UserAgentData {
+    userAgent: string;
+    platform: string;
+    screenWidth: number;
+    screenHeight: number;
+    viewportWidth: number;
+    viewportHeight: number;
+    deviceCategory: 'desktop' | 'mobile' | 'tablet';
+    appName: string;
+    connection?: {
+      downlink: number;
+      effectiveType: string;
+      rtt: number;
+    };
+  }
+
+  interface UserAgentOptions {
+    deviceCategory?: 'desktop' | 'mobile' | 'tablet';
+    platform?: RegExp | string;
+    screenWidth?: RegExp | { min?: number; max?: number };
+    screenHeight?: RegExp | { min?: number; max?: number };
+  }
+
+  interface UserAgentInstance {
+    data: UserAgentData;
+    toString(): string;
+    random(): UserAgentInstance;
+  }
+
+  class UserAgent {
+    constructor(options?: UserAgentOptions | UserAgentOptions[]);
+    data: UserAgentData;
+    toString(): string;
+    random(): UserAgentInstance;
+  }
+
+  // Make it callable
+  interface UserAgent {
+    (): UserAgentInstance;
+  }
+
+  export default UserAgent;
+}

backend/src/utils/payload-storage.ts

@@ -0,0 +1,406 @@
+/**
+ * Payload Storage Utility
+ *
+ * Per TASK_WORKFLOW_2024-12-10.md: Store raw GraphQL payloads for historical analysis.
+ *
+ * Design Pattern: Metadata/Payload Separation
+ * - Metadata in PostgreSQL (raw_crawl_payloads table): Small, indexed, queryable
+ * - Payload on filesystem: Gzipped JSON at storage_path
+ *
+ * Storage structure:
+ *   /storage/payloads/{year}/{month}/{day}/store_{dispensary_id}_{timestamp}.json.gz
+ *
+ * Benefits:
+ * - Compare any two crawls to see what changed
+ * - Replay/re-normalize historical data if logic changes
+ * - Debug issues by seeing exactly what the API returned
+ * - DB stays small, backups stay fast
+ * - ~90% compression (1.5MB -> 150KB per crawl)
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as zlib from 'zlib';
+import { promisify } from 'util';
+import { Pool } from 'pg';
+import * as crypto from 'crypto';
+
+const gzip = promisify(zlib.gzip);
+const gunzip = promisify(zlib.gunzip);
+
+// Base path for payload storage (matches image storage pattern)
+const PAYLOAD_BASE_PATH = process.env.PAYLOAD_STORAGE_PATH || './storage/payloads';
+
+/**
+ * Result from saving a payload
+ */
+export interface SavePayloadResult {
+  id: number;
+  storagePath: string;
+  sizeBytes: number;
+  sizeBytesRaw: number;
+  checksum: string;
+}
+
+/**
+ * Result from loading a payload
+ */
+export interface LoadPayloadResult {
+  payload: any;
+  metadata: {
+    id: number;
+    dispensaryId: number;
+    crawlRunId: number | null;
+    productCount: number;
+    fetchedAt: Date;
+    storagePath: string;
+  };
+}
+
+/**
+ * Generate storage path for a payload
+ *
+ * Format: /storage/payloads/{year}/{month}/{day}/store_{dispensary_id}_{timestamp}.json.gz
+ */
+function generateStoragePath(dispensaryId: number, timestamp: Date): string {
+  const year = timestamp.getFullYear();
+  const month = String(timestamp.getMonth() + 1).padStart(2, '0');
+  const day = String(timestamp.getDate()).padStart(2, '0');
+  const ts = timestamp.getTime();
+
+  return path.join(
+    PAYLOAD_BASE_PATH,
+    String(year),
+    month,
+    day,
+    `store_${dispensaryId}_${ts}.json.gz`
+  );
+}
+
+/**
+ * Ensure directory exists for a file path
+ */
+async function ensureDir(filePath: string): Promise<void> {
+  const dir = path.dirname(filePath);
+  await fs.promises.mkdir(dir, { recursive: true });
+}
+
+/**
+ * Calculate SHA256 checksum of data
+ */
+function calculateChecksum(data: Buffer): string {
+  return crypto.createHash('sha256').update(data).digest('hex');
+}
+
+/**
+ * Save a raw crawl payload to filesystem and record metadata in DB
+ *
+ * @param pool - Database connection pool
+ * @param dispensaryId - ID of the dispensary
+ * @param payload - Raw JSON payload from GraphQL
+ * @param crawlRunId - Optional crawl_run ID for linking
+ * @param productCount - Number of products in payload
+ * @returns SavePayloadResult with file info and DB record ID
+ */
+export async function saveRawPayload(
+  pool: Pool,
+  dispensaryId: number,
+  payload: any,
+  crawlRunId: number | null = null,
+  productCount: number = 0
+): Promise<SavePayloadResult> {
+  const timestamp = new Date();
+  const storagePath = generateStoragePath(dispensaryId, timestamp);
+
+  // Serialize and compress
+  const jsonStr = JSON.stringify(payload);
+  const rawSize = Buffer.byteLength(jsonStr, 'utf8');
+  const compressed = await gzip(Buffer.from(jsonStr, 'utf8'));
+  const compressedSize = compressed.length;
+  const checksum = calculateChecksum(compressed);
+
+  // Write to filesystem
+  await ensureDir(storagePath);
+  await fs.promises.writeFile(storagePath, compressed);
+
+  // Record metadata in DB
+  const result = await pool.query(`
+    INSERT INTO raw_crawl_payloads (
+      crawl_run_id,
+      dispensary_id,
+      storage_path,
+      product_count,
+      size_bytes,
+      size_bytes_raw,
+      fetched_at,
+      checksum_sha256
+    ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+    RETURNING id
+  `, [
+    crawlRunId,
+    dispensaryId,
+    storagePath,
+    productCount,
+    compressedSize,
+    rawSize,
+    timestamp,
+    checksum
+  ]);
+
+  console.log(`[PayloadStorage] Saved payload for store ${dispensaryId}: ${storagePath} (${(compressedSize / 1024).toFixed(1)}KB compressed, ${(rawSize / 1024).toFixed(1)}KB raw)`);
+
+  return {
+    id: result.rows[0].id,
+    storagePath,
+    sizeBytes: compressedSize,
+    sizeBytesRaw: rawSize,
+    checksum
+  };
+}
+
+/**
+ * Load a raw payload from filesystem by metadata ID
+ *
+ * @param pool - Database connection pool
+ * @param payloadId - ID from raw_crawl_payloads table
+ * @returns LoadPayloadResult with parsed payload and metadata
+ */
+export async function loadRawPayloadById(
+  pool: Pool,
+  payloadId: number
+): Promise<LoadPayloadResult | null> {
+  const result = await pool.query(`
+    SELECT id, dispensary_id, crawl_run_id, storage_path, product_count, fetched_at
+    FROM raw_crawl_payloads
+    WHERE id = $1
+  `, [payloadId]);
+
+  if (result.rows.length === 0) {
+    return null;
+  }
+
+  const row = result.rows[0];
+  const payload = await loadPayloadFromPath(row.storage_path);
+
+  return {
+    payload,
+    metadata: {
+      id: row.id,
+      dispensaryId: row.dispensary_id,
+      crawlRunId: row.crawl_run_id,
+      productCount: row.product_count,
+      fetchedAt: row.fetched_at,
+      storagePath: row.storage_path
+    }
+  };
+}
+
+/**
+ * Load a raw payload directly from filesystem path
+ *
+ * @param storagePath - Path to gzipped JSON file
+ * @returns Parsed JSON payload
+ */
+export async function loadPayloadFromPath(storagePath: string): Promise<any> {
+  const compressed = await fs.promises.readFile(storagePath);
+  const decompressed = await gunzip(compressed);
+  return JSON.parse(decompressed.toString('utf8'));
+}
+
+/**
+ * Get the latest payload for a dispensary
+ *
+ * @param pool - Database connection pool
+ * @param dispensaryId - ID of the dispensary
+ * @returns LoadPayloadResult or null if none exists
+ */
+export async function getLatestPayload(
+  pool: Pool,
+  dispensaryId: number
+): Promise<LoadPayloadResult | null> {
+  const result = await pool.query(`
+    SELECT id, dispensary_id, crawl_run_id, storage_path, product_count, fetched_at
+    FROM raw_crawl_payloads
+    WHERE dispensary_id = $1
+    ORDER BY fetched_at DESC
+    LIMIT 1
+  `, [dispensaryId]);
+
+  if (result.rows.length === 0) {
+    return null;
+  }
+
+  const row = result.rows[0];
+  const payload = await loadPayloadFromPath(row.storage_path);
+
+  return {
+    payload,
+    metadata: {
+      id: row.id,
+      dispensaryId: row.dispensary_id,
+      crawlRunId: row.crawl_run_id,
+      productCount: row.product_count,
+      fetchedAt: row.fetched_at,
+      storagePath: row.storage_path
+    }
+  };
+}
+
+/**
+ * Get two payloads for comparison (latest and previous, or by IDs)
+ *
+ * @param pool - Database connection pool
+ * @param dispensaryId - ID of the dispensary
+ * @param limit - Number of recent payloads to retrieve (default 2)
+ * @returns Array of LoadPayloadResult, most recent first
+ */
+export async function getRecentPayloads(
+  pool: Pool,
+  dispensaryId: number,
+  limit: number = 2
+): Promise<LoadPayloadResult[]> {
+  const result = await pool.query(`
+    SELECT id, dispensary_id, crawl_run_id, storage_path, product_count, fetched_at
+    FROM raw_crawl_payloads
+    WHERE dispensary_id = $1
+    ORDER BY fetched_at DESC
+    LIMIT $2
+  `, [dispensaryId, limit]);
+
+  const payloads: LoadPayloadResult[] = [];
+
+  for (const row of result.rows) {
+    const payload = await loadPayloadFromPath(row.storage_path);
+    payloads.push({
+      payload,
+      metadata: {
+        id: row.id,
+        dispensaryId: row.dispensary_id,
+        crawlRunId: row.crawl_run_id,
+        productCount: row.product_count,
+        fetchedAt: row.fetched_at,
+        storagePath: row.storage_path
+      }
+    });
+  }
+
+  return payloads;
+}
+
+/**
+ * List payload metadata without loading files (for browsing/pagination)
+ *
+ * @param pool - Database connection pool
+ * @param options - Query options
+ * @returns Array of metadata rows
+ */
+export async function listPayloadMetadata(
+  pool: Pool,
+  options: {
+    dispensaryId?: number;
+    startDate?: Date;
+    endDate?: Date;
+    limit?: number;
+    offset?: number;
+  } = {}
+): Promise<Array<{
+  id: number;
+  dispensaryId: number;
+  crawlRunId: number | null;
+  storagePath: string;
+  productCount: number;
+  sizeBytes: number;
+  sizeBytesRaw: number;
+  fetchedAt: Date;
+}>> {
+  const conditions: string[] = [];
+  const params: any[] = [];
+  let paramIndex = 1;
+
+  if (options.dispensaryId) {
+    conditions.push(`dispensary_id = $${paramIndex++}`);
+    params.push(options.dispensaryId);
+  }
+
+  if (options.startDate) {
+    conditions.push(`fetched_at >= $${paramIndex++}`);
+    params.push(options.startDate);
+  }
+
+  if (options.endDate) {
+    conditions.push(`fetched_at <= $${paramIndex++}`);
+    params.push(options.endDate);
+  }
+
+  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+  const limit = options.limit || 50;
+  const offset = options.offset || 0;
+
+  params.push(limit, offset);
+
+  const result = await pool.query(`
+    SELECT
+      id,
+      dispensary_id,
+      crawl_run_id,
+      storage_path,
+      product_count,
+      size_bytes,
+      size_bytes_raw,
+      fetched_at
+    FROM raw_crawl_payloads
+    ${whereClause}
+    ORDER BY fetched_at DESC
+    LIMIT $${paramIndex++} OFFSET $${paramIndex}
+  `, params);
+
+  return result.rows.map(row => ({
+    id: row.id,
+    dispensaryId: row.dispensary_id,
+    crawlRunId: row.crawl_run_id,
+    storagePath: row.storage_path,
+    productCount: row.product_count,
+    sizeBytes: row.size_bytes,
+    sizeBytesRaw: row.size_bytes_raw,
+    fetchedAt: row.fetched_at
+  }));
+}
+
+/**
+ * Delete old payloads (for retention policy)
+ *
+ * @param pool - Database connection pool
+ * @param olderThan - Delete payloads older than this date
+ * @returns Number of payloads deleted
+ */
+export async function deleteOldPayloads(
+  pool: Pool,
+  olderThan: Date
+): Promise<number> {
+  // Get paths first
+  const result = await pool.query(`
+    SELECT id, storage_path FROM raw_crawl_payloads
+    WHERE fetched_at < $1
+  `, [olderThan]);
+
+  // Delete files
+  for (const row of result.rows) {
+    try {
+      await fs.promises.unlink(row.storage_path);
+    } catch (err: any) {
+      if (err.code !== 'ENOENT') {
+        console.warn(`[PayloadStorage] Failed to delete ${row.storage_path}: ${err.message}`);
+      }
+    }
+  }
+
+  // Delete DB records
+  await pool.query(`
+    DELETE FROM raw_crawl_payloads
+    WHERE fetched_at < $1
+  `, [olderThan]);
+
+  console.log(`[PayloadStorage] Deleted ${result.rows.length} payloads older than ${olderThan.toISOString()}`);
+
+  return result.rows.length;
+}

cannaiq/Dockerfile

@@ -6,8 +6,8 @@ WORKDIR /app
 # Copy package files
 COPY package*.json ./
 
-# Install dependencies
+# Install dependencies (npm install is more forgiving than npm ci)
-RUN npm ci
+RUN npm install
 
 # Copy source files
 COPY . .

cannaiq/dist/index.html

@@ -7,8 +7,8 @@
     <title>CannaIQ - Cannabis Menu Intelligence Platform</title>
     <meta name="description" content="CannaIQ provides real-time cannabis dispensary menu data, product tracking, and analytics for dispensaries across Arizona." />
     <meta name="keywords" content="cannabis, dispensary, menu, products, analytics, Arizona" />
-    <script type="module" crossorigin src="/assets/index-BML8-px1.js"></script>
+    <script type="module" crossorigin src="/assets/index-Dq9S0rVi.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-B2gR-58G.css">
+    <link rel="stylesheet" crossorigin href="/assets/index-DhM09B-d.css">
   </head>
   <body>
     <div id="root"></div>

cannaiq/index.html

@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>CannaIQ - Cannabis Menu Intelligence Platform</title>
     <meta name="description" content="CannaIQ provides real-time cannabis dispensary menu data, product tracking, and analytics for dispensaries across Arizona." />

cannaiq/public/favicon.svg

@@ -0,0 +1,5 @@
+<svg viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg">
+  <rect width="32" height="32" rx="6" fill="#059669"/>
+  <path d="M16 6C12.5 6 9.5 7.5 7.5 10L16 16L24.5 10C22.5 7.5 19.5 6 16 6Z" fill="white"/>
+  <path d="M7.5 10C6 12 5 14.5 5 17C5 22.5 10 26 16 26C22 26 27 22.5 27 17C27 14.5 26 12 24.5 10L16 16L7.5 10Z" fill="white" fill-opacity="0.7"/>
+</svg>

cannaiq/src/App.tsx

@@ -8,6 +8,7 @@ import { ProductDetail } from './pages/ProductDetail';
 import { Stores } from './pages/Stores';
 import { Dispensaries } from './pages/Dispensaries';
 import { DispensaryDetail } from './pages/DispensaryDetail';
+import { DispensarySchedule } from './pages/DispensarySchedule';
 import { StoreDetail } from './pages/StoreDetail';
 import { StoreBrands } from './pages/StoreBrands';
 import { StoreSpecials } from './pages/StoreSpecials';
@@ -46,7 +47,6 @@ import CrossStateCompare from './pages/CrossStateCompare';
 import StateDetail from './pages/StateDetail';
 import { Discovery } from './pages/Discovery';
 import { WorkersDashboard } from './pages/WorkersDashboard';
-import { JobQueue } from './pages/JobQueue';
 import TasksDashboard from './pages/TasksDashboard';
 import { ScraperOverviewDashboard } from './pages/ScraperOverviewDashboard';
 import { SeoOrchestrator } from './pages/admin/seo/SeoOrchestrator';
@@ -66,6 +66,7 @@ export default function App() {
         <Route path="/stores" element={<PrivateRoute><Stores /></PrivateRoute>} />
         <Route path="/dispensaries" element={<PrivateRoute><Dispensaries /></PrivateRoute>} />
         <Route path="/dispensaries/:state/:city/:slug" element={<PrivateRoute><DispensaryDetail /></PrivateRoute>} />
+        <Route path="/dispensaries/:state/:city/:slug/schedule" element={<PrivateRoute><DispensarySchedule /></PrivateRoute>} />
         <Route path="/stores/:state/:storeName/:slug/brands" element={<PrivateRoute><StoreBrands /></PrivateRoute>} />
         <Route path="/stores/:state/:storeName/:slug/specials" element={<PrivateRoute><StoreSpecials /></PrivateRoute>} />
         <Route path="/stores/:state/:storeName/:slug" element={<PrivateRoute><StoreDetail /></PrivateRoute>} />
@@ -123,8 +124,6 @@ export default function App() {
         <Route path="/discovery" element={<PrivateRoute><Discovery /></PrivateRoute>} />
         {/* Workers Dashboard */}
         <Route path="/workers" element={<PrivateRoute><WorkersDashboard /></PrivateRoute>} />
-        {/* Job Queue Management */}
-        <Route path="/job-queue" element={<PrivateRoute><JobQueue /></PrivateRoute>} />
         {/* Task Queue Dashboard */}
         <Route path="/tasks" element={<PrivateRoute><TasksDashboard /></PrivateRoute>} />
         {/* Scraper Overview Dashboard (new primary) */}

cannaiq/src/components/Layout.tsx

@@ -1,8 +1,7 @@
-import { ReactNode, useEffect, useState } from 'react';
+import { ReactNode, useEffect, useState, useRef } from 'react';
-import { useNavigate, useLocation } from 'react-router-dom';
+import { useNavigate, useLocation, Link } from 'react-router-dom';
 import { useAuthStore } from '../store/authStore';
 import { api } from '../lib/api';
-import { StateSelector } from './StateSelector';
 import {
   LayoutDashboard,
   Building2,
@@ -48,8 +47,8 @@ interface NavLinkProps {
 
 function NavLink({ to, icon, label, isActive }: NavLinkProps) {
   return (
-    <a
+    <Link
-      href={to}
+      to={to}
       className={`flex items-center gap-3 px-3 py-2 rounded-lg text-sm font-medium transition-colors ${
         isActive
           ? 'bg-emerald-50 text-emerald-700'
@@ -58,7 +57,7 @@ function NavLink({ to, icon, label, isActive }: NavLinkProps) {
     >
       <span className={`flex-shrink-0 ${isActive ? 'text-emerald-600' : 'text-gray-400'}`}>{icon}</span>
       <span>{label}</span>
-    </a>
+    </Link>
   );
 }
 
@@ -86,6 +85,8 @@ export function Layout({ children }: LayoutProps) {
   const { user, logout } = useAuthStore();
   const [versionInfo, setVersionInfo] = useState<VersionInfo | null>(null);
   const [sidebarOpen, setSidebarOpen] = useState(false);
+  const navRef = useRef<HTMLElement>(null);
+  const scrollPositionRef = useRef<number>(0);
 
   useEffect(() => {
     const fetchVersion = async () => {
@@ -111,16 +112,34 @@ export function Layout({ children }: LayoutProps) {
     return location.pathname.startsWith(path);
   };
 
-  // Close sidebar on route change (mobile)
+  // Save scroll position before route change
+  useEffect(() => {
+    const nav = navRef.current;
+    if (nav) {
+      const handleScroll = () => {
+        scrollPositionRef.current = nav.scrollTop;
+      };
+      nav.addEventListener('scroll', handleScroll);
+      return () => nav.removeEventListener('scroll', handleScroll);
+    }
+  }, []);
+
+  // Restore scroll position after route change and close mobile sidebar
   useEffect(() => {
     setSidebarOpen(false);
+    // Restore scroll position after render
+    requestAnimationFrame(() => {
+      if (navRef.current) {
+        navRef.current.scrollTop = scrollPositionRef.current;
+      }
+    });
   }, [location.pathname]);
 
   const sidebarContent = (
     <>
       {/* Logo/Brand */}
       <div className="px-6 py-5 border-b border-gray-200">
-        <div className="flex items-center gap-3">
+        <Link to="/dashboard" className="flex items-center gap-3 hover:opacity-80 transition-opacity">
           <div className="w-8 h-8 bg-emerald-600 rounded-lg flex items-center justify-center">
             <svg viewBox="0 0 24 24" className="w-5 h-5 text-white" fill="currentColor">
               <path d="M12 2C8.5 2 5.5 3.5 3.5 6L12 12L20.5 6C18.5 3.5 15.5 2 12 2Z" />
@@ -131,21 +150,17 @@ export function Layout({ children }: LayoutProps) {
             <span className="text-lg font-bold text-gray-900">CannaIQ</span>
             {versionInfo && (
               <p className="text-xs text-gray-400">
-                v{versionInfo.version} ({versionInfo.git_sha}) {versionInfo.build_time !== 'unknown' && `- ${new Date(versionInfo.build_time).toLocaleDateString()}`}
+                {versionInfo.git_sha || 'dev'}
               </p>
             )}
           </div>
-        </div>
+        </Link>
         <p className="text-xs text-gray-500 mt-2 truncate">{user?.email}</p>
       </div>
 
-      {/* State Selector */}
-      <div className="px-4 py-3 border-b border-gray-200 bg-gray-50">
-        <StateSelector showLabel={false} />
-      </div>
 
       {/* Navigation */}
-      <nav className="flex-1 px-3 py-4 space-y-6 overflow-y-auto">
+      <nav ref={navRef} className="flex-1 px-3 py-4 space-y-6 overflow-y-auto">
         <NavSection title="Main">
           <NavLink to="/dashboard" icon={<LayoutDashboard className="w-4 h-4" />} label="Dashboard" isActive={isActive('/dashboard', true)} />
           <NavLink to="/dispensaries" icon={<Building2 className="w-4 h-4" />} label="Dispensaries" isActive={isActive('/dispensaries')} />
@@ -164,8 +179,7 @@ export function Layout({ children }: LayoutProps) {
           <NavLink to="/admin/orchestrator" icon={<Activity className="w-4 h-4" />} label="Orchestrator" isActive={isActive('/admin/orchestrator')} />
           <NavLink to="/users" icon={<UserCog className="w-4 h-4" />} label="Users" isActive={isActive('/users')} />
           <NavLink to="/workers" icon={<Users className="w-4 h-4" />} label="Workers" isActive={isActive('/workers')} />
-          <NavLink to="/job-queue" icon={<ListOrdered className="w-4 h-4" />} label="Job Queue" isActive={isActive('/job-queue')} />
+          <NavLink to="/tasks" icon={<ListChecks className="w-4 h-4" />} label="Tasks" isActive={isActive('/tasks')} />
-          <NavLink to="/tasks" icon={<ListChecks className="w-4 h-4" />} label="Task Queue" isActive={isActive('/tasks')} />
           <NavLink to="/admin/seo" icon={<FileText className="w-4 h-4" />} label="SEO Pages" isActive={isActive('/admin/seo')} />
           <NavLink to="/proxies" icon={<Shield className="w-4 h-4" />} label="Proxies" isActive={isActive('/proxies')} />
           <NavLink to="/api-permissions" icon={<Key className="w-4 h-4" />} label="API Keys" isActive={isActive('/api-permissions')} />
@@ -214,7 +228,7 @@ export function Layout({ children }: LayoutProps) {
           <button onClick={() => setSidebarOpen(true)} className="p-2 -ml-2 rounded-lg hover:bg-gray-100">
             <Menu className="w-5 h-5 text-gray-600" />
           </button>
-          <div className="flex items-center gap-2">
+          <Link to="/dashboard" className="flex items-center gap-2 hover:opacity-80 transition-opacity">
             <div className="w-6 h-6 bg-emerald-600 rounded flex items-center justify-center">
               <svg viewBox="0 0 24 24" className="w-4 h-4 text-white" fill="currentColor">
                 <path d="M12 2C8.5 2 5.5 3.5 3.5 6L12 12L20.5 6C18.5 3.5 15.5 2 12 2Z" />
@@ -222,7 +236,7 @@ export function Layout({ children }: LayoutProps) {
               </svg>
             </div>
             <span className="font-semibold text-gray-900">CannaIQ</span>
-          </div>
+          </Link>
         </div>
 
         {/* Page content */}

cannaiq/src/components/PasswordConfirmModal.tsx

@@ -0,0 +1,138 @@
+import { useState, useEffect, useRef } from 'react';
+import { api } from '../lib/api';
+import { Shield, X, Loader2 } from 'lucide-react';
+
+interface PasswordConfirmModalProps {
+  isOpen: boolean;
+  onClose: () => void;
+  onConfirm: () => void;
+  title: string;
+  description: string;
+}
+
+export function PasswordConfirmModal({
+  isOpen,
+  onClose,
+  onConfirm,
+  title,
+  description,
+}: PasswordConfirmModalProps) {
+  const [password, setPassword] = useState('');
+  const [error, setError] = useState('');
+  const [loading, setLoading] = useState(false);
+  const inputRef = useRef<HTMLInputElement>(null);
+
+  useEffect(() => {
+    if (isOpen) {
+      setPassword('');
+      setError('');
+      // Focus the input when modal opens
+      setTimeout(() => inputRef.current?.focus(), 100);
+    }
+  }, [isOpen]);
+
+  const handleSubmit = async (e: React.FormEvent) => {
+    e.preventDefault();
+    if (!password.trim()) {
+      setError('Password is required');
+      return;
+    }
+
+    setLoading(true);
+    setError('');
+
+    try {
+      const result = await api.verifyPassword(password);
+      if (result.verified) {
+        onConfirm();
+        onClose();
+      } else {
+        setError('Invalid password');
+      }
+    } catch (err: any) {
+      setError(err.message || 'Verification failed');
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  if (!isOpen) return null;
+
+  return (
+    <div className="fixed inset-0 z-50 flex items-center justify-center">
+      {/* Backdrop */}
+      <div
+        className="absolute inset-0 bg-black bg-opacity-50"
+        onClick={onClose}
+      />
+
+      {/* Modal */}
+      <div className="relative bg-white rounded-lg shadow-xl max-w-md w-full mx-4">
+        {/* Header */}
+        <div className="flex items-center justify-between px-6 py-4 border-b border-gray-200">
+          <div className="flex items-center gap-3">
+            <div className="p-2 bg-amber-100 rounded-lg">
+              <Shield className="w-5 h-5 text-amber-600" />
+            </div>
+            <h3 className="text-lg font-semibold text-gray-900">{title}</h3>
+          </div>
+          <button
+            onClick={onClose}
+            className="p-1 hover:bg-gray-100 rounded-lg transition-colors"
+          >
+            <X className="w-5 h-5 text-gray-500" />
+          </button>
+        </div>
+
+        {/* Body */}
+        <form onSubmit={handleSubmit}>
+          <div className="px-6 py-4">
+            <p className="text-gray-600 mb-4">{description}</p>
+
+            <div className="space-y-2">
+              <label
+                htmlFor="password"
+                className="block text-sm font-medium text-gray-700"
+              >
+                Enter your password to continue
+              </label>
+              <input
+                ref={inputRef}
+                type="password"
+                id="password"
+                value={password}
+                onChange={(e) => setPassword(e.target.value)}
+                className="w-full px-4 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-emerald-500 focus:border-emerald-500"
+                placeholder="Password"
+                disabled={loading}
+              />
+              {error && (
+                <p className="text-sm text-red-600">{error}</p>
+              )}
+            </div>
+          </div>
+
+          {/* Footer */}
+          <div className="flex justify-end gap-3 px-6 py-4 border-t border-gray-200 bg-gray-50 rounded-b-lg">
+            <button
+              type="button"
+              onClick={onClose}
+              disabled={loading}
+              className="px-4 py-2 text-gray-700 hover:bg-gray-100 rounded-lg transition-colors"
+            >
+              Cancel
+            </button>
+            <button
+              type="submit"
+              disabled={loading}
+              className="px-4 py-2 bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 transition-colors disabled:opacity-50 flex items-center gap-2"
+            >
+              {loading && <Loader2 className="w-4 h-4 animate-spin" />}
+              Confirm
+            </button>
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}

cannaiq/src/lib/api.ts

@@ -69,6 +69,13 @@ class ApiClient {
     return { data };
   }
 
+  async delete<T = any>(endpoint: string): Promise<{ data: T }> {
+    const data = await this.request<T>(endpoint, {
+      method: 'DELETE',
+    });
+    return { data };
+  }
+
   // Auth
   async login(email: string, password: string) {
     return this.request<{ token: string; user: any }>('/api/auth/login', {
@@ -77,6 +84,13 @@ class ApiClient {
     });
   }
 
+  async verifyPassword(password: string) {
+    return this.request<{ verified: boolean; error?: string }>('/api/auth/verify-password', {
+      method: 'POST',
+      body: JSON.stringify({ password }),
+    });
+  }
+
   async getMe() {
     return this.request<{ user: any }>('/api/auth/me');
   }
@@ -113,7 +127,7 @@ class ApiClient {
     });
   }
 
-  async getDispensaries(params?: { limit?: number; offset?: number; search?: string; city?: string; state?: string; crawl_enabled?: string }) {
+  async getDispensaries(params?: { limit?: number; offset?: number; search?: string; city?: string; state?: string; crawl_enabled?: string; status?: string }) {
     const searchParams = new URLSearchParams();
     if (params?.limit) searchParams.append('limit', params.limit.toString());
     if (params?.offset) searchParams.append('offset', params.offset.toString());
@@ -121,10 +135,15 @@ class ApiClient {
     if (params?.city) searchParams.append('city', params.city);
     if (params?.state) searchParams.append('state', params.state);
     if (params?.crawl_enabled) searchParams.append('crawl_enabled', params.crawl_enabled);
+    if (params?.status) searchParams.append('status', params.status);
     const queryString = searchParams.toString() ? `?${searchParams.toString()}` : '';
     return this.request<{ dispensaries: any[]; total: number; limit: number; offset: number; hasMore: boolean }>(`/api/dispensaries${queryString}`);
   }
 
+  async getDroppedStores() {
+    return this.request<{ dropped_count: number; dropped_stores: any[] }>('/api/dispensaries/stats/dropped');
+  }
+
   async getDispensary(slug: string) {
     return this.request<any>(`/api/dispensaries/${slug}`);
   }
@@ -308,7 +327,7 @@ class ApiClient {
   }
 
   async testAllProxies() {
-    return this.request<{ jobId: number; message: string }>('/api/proxies/test-all', {
+    return this.request<{ jobId: number; total: number; message: string }>('/api/proxies/test-all', {
       method: 'POST',
     });
   }
@@ -971,6 +990,47 @@ class ApiClient {
     }>(`/api/markets/stores/${id}/categories`);
   }
 
+  async getStoreCrawlHistory(id: number, limit = 50) {
+    return this.request<{
+      dispensary: {
+        id: number;
+        name: string;
+        dba_name: string | null;
+        slug: string;
+        state: string;
+        city: string;
+        menu_type: string | null;
+        platform_dispensary_id: string | null;
+        last_menu_scrape: string | null;
+      } | null;
+      history: Array<{
+        id: number;
+        runId: string | null;
+        profileKey: string | null;
+        crawlerModule: string | null;
+        stateAtStart: string | null;
+        stateAtEnd: string | null;
+        totalSteps: number;
+        durationMs: number | null;
+        success: boolean;
+        errorMessage: string | null;
+        productsFound: number | null;
+        startedAt: string | null;
+        completedAt: string | null;
+      }>;
+      nextSchedule: {
+        scheduleId: number;
+        jobName: string;
+        enabled: boolean;
+        baseIntervalMinutes: number;
+        jitterMinutes: number;
+        nextRunAt: string | null;
+        lastRunAt: string | null;
+        lastStatus: string | null;
+      } | null;
+    }>(`/api/markets/stores/${id}/crawl-history?limit=${limit}`);
+  }
+
   // Global Brands/Categories (from v_brands/v_categories views)
   async getMarketBrands(params?: { limit?: number; offset?: number }) {
     const searchParams = new URLSearchParams();
@@ -1506,10 +1566,11 @@ class ApiClient {
   }
 
   // Intelligence API
-  async getIntelligenceBrands(params?: { limit?: number; offset?: number }) {
+  async getIntelligenceBrands(params?: { limit?: number; offset?: number; state?: string }) {
     const searchParams = new URLSearchParams();
     if (params?.limit) searchParams.append('limit', params.limit.toString());
     if (params?.offset) searchParams.append('offset', params.offset.toString());
+    if (params?.state) searchParams.append('state', params.state);
     const queryString = searchParams.toString() ? `?${searchParams.toString()}` : '';
     return this.request<{
       brands: Array<{
@@ -1524,7 +1585,10 @@ class ApiClient {
     }>(`/api/admin/intelligence/brands${queryString}`);
   }
 
-  async getIntelligencePricing() {
+  async getIntelligencePricing(params?: { state?: string }) {
+    const searchParams = new URLSearchParams();
+    if (params?.state) searchParams.append('state', params.state);
+    const queryString = searchParams.toString() ? `?${searchParams.toString()}` : '';
     return this.request<{
       byCategory: Array<{
         category: string;
@@ -1540,7 +1604,7 @@ class ApiClient {
         maxPrice: number;
         totalProducts: number;
       };
-    }>('/api/admin/intelligence/pricing');
+    }>(`/api/admin/intelligence/pricing${queryString}`);
   }
 
   async getIntelligenceStoreActivity(params?: { state?: string; chainId?: number; limit?: number }) {
@@ -2872,6 +2936,46 @@ class ApiClient {
       `/api/tasks/store/${dispensaryId}/active`
     );
   }
+
+  // Task Pool Control
+  async getTaskPoolStatus() {
+    return this.request<{ success: boolean; paused: boolean; message: string }>(
+      '/api/tasks/pool/status'
+    );
+  }
+
+  async pauseTaskPool() {
+    return this.request<{ success: boolean; paused: boolean; message: string }>(
+      '/api/tasks/pool/pause',
+      { method: 'POST' }
+    );
+  }
+
+  async resumeTaskPool() {
+    return this.request<{ success: boolean; paused: boolean; message: string }>(
+      '/api/tasks/pool/resume',
+      { method: 'POST' }
+    );
+  }
+
+  // K8s Worker Control
+  async getK8sWorkers() {
+    return this.request<{
+      success: boolean;
+      available: boolean;
+      replicas: number;
+      readyReplicas: number;
+      availableReplicas?: number;
+      error?: string;
+    }>('/api/k8s/workers');
+  }
+
+  async scaleK8sWorkers(replicas: number) {
+    return this.request<{ success: boolean; replicas: number; message?: string; error?: string }>(
+      '/api/k8s/workers/scale',
+      { method: 'POST', body: JSON.stringify({ replicas }) }
+    );
+  }
 }
 
 export const api = new ApiClient(API_URL);

cannaiq/src/pages/ApiPermissions.tsx

@@ -2,7 +2,7 @@ import { useEffect, useState, useRef } from 'react';
 import { Layout } from '../components/Layout';
 import { api } from '../lib/api';
 import { Toast } from '../components/Toast';
-import { Key, Plus, Copy, Check, X, Trash2, Power, PowerOff, Store, Globe, Shield, Clock, Eye, EyeOff, Search, ChevronDown } from 'lucide-react';
+import { Key, Plus, Copy, Check, X, Trash2, Power, PowerOff, Store, Globe, Shield, Clock, Eye, EyeOff, Search, ChevronDown, Pencil } from 'lucide-react';
 
 interface ApiPermission {
   id: number;
@@ -161,6 +161,12 @@ export function ApiPermissions() {
     allowed_ips: '',
     allowed_domains: '',
   });
+  const [editingPermission, setEditingPermission] = useState<ApiPermission | null>(null);
+  const [editForm, setEditForm] = useState({
+    user_name: '',
+    allowed_ips: '',
+    allowed_domains: '',
+  });
   const [notification, setNotification] = useState<{ message: string; type: 'success' | 'error' | 'info' } | null>(null);
 
   useEffect(() => {
@@ -240,6 +246,33 @@ export function ApiPermissions() {
     }
   };
 
+  const handleEdit = (perm: ApiPermission) => {
+    setEditingPermission(perm);
+    setEditForm({
+      user_name: perm.user_name,
+      allowed_ips: perm.allowed_ips || '',
+      allowed_domains: perm.allowed_domains || '',
+    });
+  };
+
+  const handleSaveEdit = async (e: React.FormEvent) => {
+    e.preventDefault();
+    if (!editingPermission) return;
+
+    try {
+      await api.updateApiPermission(editingPermission.id, {
+        user_name: editForm.user_name,
+        allowed_ips: editForm.allowed_ips || undefined,
+        allowed_domains: editForm.allowed_domains || undefined,
+      });
+      setNotification({ message: 'API key updated successfully', type: 'success' });
+      setEditingPermission(null);
+      loadPermissions();
+    } catch (error: any) {
+      setNotification({ message: 'Failed to update permission: ' + error.message, type: 'error' });
+    }
+  };
+
   const copyToClipboard = async (text: string, id: number) => {
     await navigator.clipboard.writeText(text);
     setCopiedId(id);
@@ -494,21 +527,36 @@ export function ApiPermissions() {
                         </button>
                       </div>
 
-                      {/* Restrictions */}
+                      {/* Allowed Domains - Always show */}
-                      {(perm.allowed_ips || perm.allowed_domains) && (
+                      <div className="mt-3 text-xs">
-                        <div className="flex gap-4 mt-3 text-xs text-gray-500">
+                        <span className="text-gray-500 flex items-center gap-1">
-                          {perm.allowed_ips && (
+                          <Globe className="w-3 h-3" />
-                            <span>IPs: {perm.allowed_ips.split('\n').length} allowed</span>
+                          Domains:{' '}
+                          {perm.allowed_domains ? (
+                            <span className="text-gray-700 font-mono">
+                              {perm.allowed_domains.split('\n').filter(d => d.trim()).join(', ')}
+                            </span>
+                          ) : (
+                            <span className="text-amber-600">Any domain (no restriction)</span>
                           )}
-                          {perm.allowed_domains && (
+                        </span>
-                            <span>Domains: {perm.allowed_domains.split('\n').length} allowed</span>
+                        {perm.allowed_ips && (
+                          <span className="text-gray-500 ml-4">
+                            IPs: {perm.allowed_ips.split('\n').filter(ip => ip.trim()).length} allowed
+                          </span>
                         )}
                       </div>
-                      )}
                     </div>
 
                     {/* Actions */}
                     <div className="flex items-center gap-2 ml-4">
+                      <button
+                        onClick={() => handleEdit(perm)}
+                        className="p-2 text-blue-600 hover:bg-blue-50 rounded-lg transition-colors"
+                        title="Edit"
+                      >
+                        <Pencil className="w-5 h-5" />
+                      </button>
                       <button
                         onClick={() => handleToggle(perm.id)}
                         className={`p-2 rounded-lg transition-colors ${
@@ -534,6 +582,86 @@ export function ApiPermissions() {
             </div>
           )}
         </div>
+
+        {/* Edit Modal */}
+        {editingPermission && (
+          <div className="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
+            <div className="bg-white rounded-xl shadow-xl max-w-lg w-full mx-4 max-h-[90vh] overflow-y-auto">
+              <div className="px-6 py-4 border-b border-gray-200">
+                <h2 className="text-lg font-semibold text-gray-900 flex items-center gap-2">
+                  <Pencil className="w-5 h-5 text-blue-600" />
+                  Edit API Key
+                </h2>
+                <p className="text-sm text-gray-500 mt-1">
+                  {editingPermission.store_name}
+                </p>
+              </div>
+
+              <form onSubmit={handleSaveEdit} className="p-6 space-y-5">
+                <div>
+                  <label className="block text-sm font-medium text-gray-700 mb-2">
+                    Label / Website Name
+                  </label>
+                  <input
+                    type="text"
+                    value={editForm.user_name}
+                    onChange={(e) => setEditForm({ ...editForm, user_name: e.target.value })}
+                    className="w-full px-4 py-2.5 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent"
+                    required
+                  />
+                </div>
+
+                <div>
+                  <label className="block text-sm font-medium text-gray-700 mb-2">
+                    <Globe className="w-4 h-4 inline mr-1" />
+                    Allowed Domains
+                  </label>
+                  <textarea
+                    value={editForm.allowed_domains}
+                    onChange={(e) => setEditForm({ ...editForm, allowed_domains: e.target.value })}
+                    rows={4}
+                    className="w-full px-4 py-2.5 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent font-mono text-sm"
+                    placeholder="example.com&#10;*.example.com&#10;subdomain.example.com"
+                  />
+                  <p className="text-xs text-gray-500 mt-1">
+                    One domain per line. Use * for wildcards (e.g., *.example.com). Leave empty to allow any domain.
+                  </p>
+                </div>
+
+                <div>
+                  <label className="block text-sm font-medium text-gray-700 mb-2">
+                    <Shield className="w-4 h-4 inline mr-1" />
+                    Allowed IP Addresses
+                  </label>
+                  <textarea
+                    value={editForm.allowed_ips}
+                    onChange={(e) => setEditForm({ ...editForm, allowed_ips: e.target.value })}
+                    rows={3}
+                    className="w-full px-4 py-2.5 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent font-mono text-sm"
+                    placeholder="192.168.1.1&#10;10.0.0.0/8"
+                  />
+                  <p className="text-xs text-gray-500 mt-1">One per line. CIDR notation supported. Leave empty to allow any IP.</p>
+                </div>
+
+                <div className="flex gap-3 pt-2">
+                  <button
+                    type="submit"
+                    className="flex-1 px-5 py-2.5 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors"
+                  >
+                    Save Changes
+                  </button>
+                  <button
+                    type="button"
+                    onClick={() => setEditingPermission(null)}
+                    className="px-5 py-2.5 bg-gray-100 text-gray-700 rounded-lg hover:bg-gray-200 transition-colors"
+                  >
+                    Cancel
+                  </button>
+                </div>
+              </form>
+            </div>
+          </div>
+        )}
       </div>
     </Layout>
   );

cannaiq/src/pages/Dashboard.tsx

@@ -1,6 +1,5 @@
 import { useEffect, useState } from 'react';
 import { Layout } from '../components/Layout';
-import { HealthPanel } from '../components/HealthPanel';
 import { api } from '../lib/api';
 import { useNavigate } from 'react-router-dom';
 import {
@@ -42,16 +41,36 @@ export function Dashboard() {
   const [activity, setActivity] = useState<any>(null);
   const [nationalStats, setNationalStats] = useState<any>(null);
   const [loading, setLoading] = useState(true);
-  const [refreshing, setRefreshing] = useState(false);
   const [pendingChangesCount, setPendingChangesCount] = useState(0);
   const [showNotification, setShowNotification] = useState(false);
   const [taskCounts, setTaskCounts] = useState<Record<string, number> | null>(null);
+  const [droppedStoresCount, setDroppedStoresCount] = useState(0);
+  const [showDroppedAlert, setShowDroppedAlert] = useState(false);
 
   useEffect(() => {
     loadData();
     checkNotificationStatus();
+    checkDroppedStores();
   }, []);
 
+  const checkDroppedStores = async () => {
+    try {
+      const data = await api.getDroppedStores();
+      setDroppedStoresCount(data.dropped_count);
+      // Check if notification was dismissed for this count
+      const dismissedCount = localStorage.getItem('dismissedDroppedStoresCount');
+      const isDismissed = dismissedCount && parseInt(dismissedCount) >= data.dropped_count;
+      setShowDroppedAlert(data.dropped_count > 0 && !isDismissed);
+    } catch (error) {
+      console.error('Failed to check dropped stores:', error);
+    }
+  };
+
+  const handleDismissDroppedAlert = () => {
+    localStorage.setItem('dismissedDroppedStoresCount', droppedStoresCount.toString());
+    setShowDroppedAlert(false);
+  };
+
   const checkNotificationStatus = async () => {
     try {
       // Fetch real pending changes count from API
@@ -72,10 +91,7 @@ export function Dashboard() {
     }
   };
 
-  const loadData = async (isRefresh = false) => {
+  const loadData = async () => {
-    if (isRefresh) {
-      setRefreshing(true);
-    }
     try {
       // Fetch dashboard data (primary data source)
       const dashboard = await api.getMarketDashboard();
@@ -137,7 +153,6 @@ export function Dashboard() {
       console.error('Failed to load dashboard:', error);
     } finally {
       setLoading(false);
-      setRefreshing(false);
     }
   };
 
@@ -214,25 +229,46 @@ export function Dashboard() {
         </div>
       )}
 
+      {/* Dropped Stores Alert */}
+      {showDroppedAlert && (
+        <div className="mb-6 bg-red-50 border-l-4 border-red-500 rounded-lg p-3 sm:p-4">
+          <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 sm:gap-4">
+            <div className="flex items-start sm:items-center gap-3 flex-1">
+              <Store className="w-5 h-5 text-red-600 flex-shrink-0 mt-0.5 sm:mt-0" />
+              <div className="flex-1">
+                <h3 className="text-sm font-semibold text-red-900">
+                  {droppedStoresCount} dropped store{droppedStoresCount !== 1 ? 's' : ''} need{droppedStoresCount === 1 ? 's' : ''} review
+                </h3>
+                <p className="text-xs sm:text-sm text-red-700 mt-0.5">
+                  These stores were not found in the latest Dutchie discovery and may have stopped using the platform
+                </p>
+              </div>
+            </div>
+            <div className="flex items-center gap-2 pl-8 sm:pl-0">
+              <button
+                onClick={() => navigate('/dispensaries?status=dropped')}
+                className="btn btn-sm bg-red-600 hover:bg-red-700 text-white border-none"
+              >
+                Review
+              </button>
+              <button
+                onClick={handleDismissDroppedAlert}
+                className="btn btn-sm btn-ghost text-red-900 hover:bg-red-100"
+                aria-label="Dismiss notification"
+              >
+                <X className="w-4 h-4" />
+              </button>
+            </div>
+          </div>
+        </div>
+      )}
+
       <div className="space-y-8">
         {/* Header */}
-        <div className="flex flex-col sm:flex-row sm:justify-between sm:items-center gap-4">
         <div>
           <h1 className="text-xl sm:text-2xl font-semibold text-gray-900">Dashboard</h1>
           <p className="text-sm text-gray-500 mt-1">Monitor your dispensary data aggregation</p>
         </div>
-          <button
-            onClick={() => loadData(true)}
-            disabled={refreshing}
-            className="inline-flex items-center justify-center gap-2 px-4 py-2 bg-white border border-gray-200 rounded-lg hover:bg-gray-50 transition-colors text-sm font-medium text-gray-700 self-start sm:self-auto disabled:opacity-50 disabled:cursor-not-allowed"
-          >
-            <RefreshCw className={`w-4 h-4 ${refreshing ? 'animate-spin' : ''}`} />
-            {refreshing ? 'Refreshing...' : 'Refresh'}
-          </button>
-        </div>
-
-        {/* System Health */}
-        <HealthPanel showQueues={false} refreshInterval={60000} />
 
         {/* Stats Grid */}
         <div className="grid grid-cols-2 lg:grid-cols-3 gap-3 sm:gap-6">

cannaiq/src/pages/Dispensaries.tsx

@@ -13,6 +13,7 @@ export function Dispensaries() {
   const [searchTerm, setSearchTerm] = useState('');
   const [debouncedSearch, setDebouncedSearch] = useState('');
   const [filterState, setFilterState] = useState('');
+  const [filterStatus, setFilterStatus] = useState('');
   const [editingDispensary, setEditingDispensary] = useState<any | null>(null);
   const [editForm, setEditForm] = useState<any>({});
   const [total, setTotal] = useState(0);
@@ -51,6 +52,7 @@ export function Dispensaries() {
         offset,
         search: debouncedSearch || undefined,
         state: filterState || undefined,
+        status: filterStatus || undefined,
         crawl_enabled: 'all'
       });
       setDispensaries(data.dispensaries);
@@ -61,7 +63,7 @@ export function Dispensaries() {
     } finally {
       setLoading(false);
     }
-  }, [offset, debouncedSearch, filterState]);
+  }, [offset, debouncedSearch, filterState, filterStatus]);
 
   useEffect(() => {
     loadDispensaries();
@@ -110,6 +112,11 @@ export function Dispensaries() {
     setOffset(0); // Reset to first page
   };
 
+  const handleStatusFilter = (status: string) => {
+    setFilterStatus(status);
+    setOffset(0); // Reset to first page
+  };
+
   return (
     <Layout>
       <div className="space-y-6">
@@ -123,7 +130,7 @@ export function Dispensaries() {
 
         {/* Filters */}
         <div className="bg-white rounded-lg border border-gray-200 p-4">
-          <div className="grid grid-cols-2 gap-4">
+          <div className="grid grid-cols-3 gap-4">
             <div>
               <label className="block text-sm font-medium text-gray-700 mb-2">
                 Search

cannaiq/src/pages/DispensaryDetail.tsx

@@ -204,47 +204,6 @@ export function DispensaryDetail() {
             Back to Dispensaries
           </button>
 
-          {/* Update Dropdown */}
-          <div className="relative">
-            <button
-              onClick={() => setShowUpdateDropdown(!showUpdateDropdown)}
-              disabled={isUpdating}
-              className="flex items-center gap-2 px-4 py-2 text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
-            >
-              <RefreshCw className={`w-4 h-4 ${isUpdating ? 'animate-spin' : ''}`} />
-              {isUpdating ? 'Updating...' : 'Update'}
-              {!isUpdating && <ChevronDown className="w-4 h-4" />}
-            </button>
-
-            {showUpdateDropdown && !isUpdating && (
-              <div className="absolute right-0 mt-2 w-48 bg-white rounded-lg shadow-lg border border-gray-200 z-10">
-                <button
-                  onClick={() => handleUpdate('products')}
-                  className="w-full text-left px-4 py-2 text-sm text-gray-700 hover:bg-gray-100 rounded-t-lg"
-                >
-                  Products
-                </button>
-                <button
-                  onClick={() => handleUpdate('brands')}
-                  className="w-full text-left px-4 py-2 text-sm text-gray-700 hover:bg-gray-100"
-                >
-                  Brands
-                </button>
-                <button
-                  onClick={() => handleUpdate('specials')}
-                  className="w-full text-left px-4 py-2 text-sm text-gray-700 hover:bg-gray-100"
-                >
-                  Specials
-                </button>
-                <button
-                  onClick={() => handleUpdate('all')}
-                  className="w-full text-left px-4 py-2 text-sm text-gray-700 hover:bg-gray-100 rounded-b-lg border-t border-gray-200"
-                >
-                  All
-                </button>
-              </div>
-            )}
-          </div>
         </div>
 
         {/* Dispensary Header */}
@@ -266,7 +225,7 @@ export function DispensaryDetail() {
             <div className="flex items-center gap-2 text-sm text-gray-600 bg-gray-50 px-4 py-2 rounded-lg">
               <Calendar className="w-4 h-4" />
               <div>
-                <span className="font-medium">Last Crawl Date:</span>
+                <span className="font-medium">Last Updated:</span>
                 <span className="ml-2">
                   {dispensary.last_menu_scrape
                     ? new Date(dispensary.last_menu_scrape).toLocaleDateString('en-US', {
@@ -331,7 +290,7 @@ export function DispensaryDetail() {
               </a>
             )}
             <Link
-              to="/schedule"
+              to={`/dispensaries/${state}/${city}/${slug}/schedule`}
               className="flex items-center gap-2 text-sm text-blue-600 hover:text-blue-800"
             >
               <Clock className="w-4 h-4" />
@@ -533,57 +492,31 @@ export function DispensaryDetail() {
                                 `$${product.regular_price}`
                               ) : '-'}
                             </td>
-                            <td className="text-center whitespace-nowrap">
+                            <td className="text-center whitespace-nowrap text-sm text-gray-700">
-                              {product.quantity != null ? (
+                              {product.quantity != null ? product.quantity : '-'}
-                                <span className={`badge badge-sm ${product.quantity > 0 ? 'badge-info' : 'badge-error'}`}>
-                                  {product.quantity}
-                                </span>
-                              ) : '-'}
                             </td>
-                            <td className="text-center whitespace-nowrap">
+                            <td className="text-center whitespace-nowrap text-sm text-gray-700">
-                              {product.thc_percentage ? (
+                              {product.thc_percentage ? `${product.thc_percentage}%` : '-'}
-                                <span className="badge badge-success badge-sm">{product.thc_percentage}%</span>
-                              ) : '-'}
                             </td>
-                            <td className="text-center whitespace-nowrap">
+                            <td className="text-center whitespace-nowrap text-sm text-gray-700">
-                              {product.cbd_percentage ? (
+                              {product.cbd_percentage ? `${product.cbd_percentage}%` : '-'}
-                                <span className="badge badge-info badge-sm">{product.cbd_percentage}%</span>
-                              ) : '-'}
                             </td>
-                            <td className="text-center whitespace-nowrap">
+                            <td className="text-center whitespace-nowrap text-sm text-gray-700">
-                              {product.strain_type ? (
+                              {product.strain_type || '-'}
-                                <span className="badge badge-ghost badge-sm">{product.strain_type}</span>
-                              ) : '-'}
                             </td>
-                            <td className="text-center whitespace-nowrap">
+                            <td className="text-center whitespace-nowrap text-sm text-gray-700">
-                              {product.in_stock ? (
+                              {product.in_stock ? 'Yes' : product.in_stock === false ? 'No' : '-'}
-                                <span className="badge badge-success badge-sm">Yes</span>
-                              ) : product.in_stock === false ? (
-                                <span className="badge badge-error badge-sm">No</span>
-                              ) : '-'}
                             </td>
                             <td className="whitespace-nowrap text-xs text-gray-500">
                               {product.updated_at ? formatDate(product.updated_at) : '-'}
                             </td>
                             <td>
-                              <div className="flex gap-1">
-                                {product.dutchie_url && (
-                                  <a
-                                    href={product.dutchie_url}
-                                    target="_blank"
-                                    rel="noopener noreferrer"
-                                    className="btn btn-xs btn-outline"
-                                  >
-                                    Dutchie
-                                  </a>
-                                )}
                               <button
                                 onClick={() => navigate(`/products/${product.id}`)}
-                                  className="btn btn-xs btn-primary"
+                                className="btn btn-xs btn-ghost text-gray-500 hover:text-gray-700"
                               >
                                 Details
                               </button>
-                              </div>
                             </td>
                           </tr>
                         ))}