fix(ci): Fix buildx cache_from syntax for array format

Plugin was splitting comma-separated values incorrectly.
Use array format with quoted strings instead.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.woodpecker.yml

@@ -86,6 +86,10 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache,mode=max"
       build_args:
         APP_BUILD_VERSION: ${CI_COMMIT_SHA:0:8}
         APP_GIT_SHA: ${CI_COMMIT_SHA}
@@ -94,7 +98,7 @@ steps:
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   docker-cannaiq:
     image: woodpeckerci/plugin-docker-buildx
@@ -112,10 +116,14 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache,mode=max"
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   docker-findadispo:
     image: woodpeckerci/plugin-docker-buildx
@@ -133,10 +141,14 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache,mode=max"
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   docker-findagram:
     image: woodpeckerci/plugin-docker-buildx
@@ -154,38 +166,17 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache"
+      cache_to:
+        - "type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache,mode=max"
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   # ===========================================
-  # STAGE 3: Run Database Migrations (before deploy)
+  # STAGE 3: Deploy and Run Migrations
-  # ===========================================
-  migrate:
-    image: code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8}
-    environment:
-      CANNAIQ_DB_HOST:
-        from_secret: db_host
-      CANNAIQ_DB_PORT:
-        from_secret: db_port
-      CANNAIQ_DB_NAME:
-        from_secret: db_name
-      CANNAIQ_DB_USER:
-        from_secret: db_user
-      CANNAIQ_DB_PASS:
-        from_secret: db_pass
-    commands:
-      - cd /app
-      - node dist/db/migrate.js
-    depends_on:
-      - docker-backend
-    when:
-      branch: master
-      event: [push, manual]
-
-  # ===========================================
-  # STAGE 4: Deploy (after migrations)
   # ===========================================
   deploy:
     image: bitnami/kubectl:latest
@@ -196,18 +187,23 @@ steps:
       - mkdir -p ~/.kube
       - echo "$KUBECONFIG_CONTENT" | tr -d '[:space:]' | base64 -d > ~/.kube/config
       - chmod 600 ~/.kube/config
+      # Deploy backend first
       - kubectl set image deployment/scraper scraper=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
+      # Note: Migrations run automatically at startup via auto-migrate
+      # Deploy remaining services
+      # Resilience: ensure workers are scaled up if at 0
+      - REPLICAS=$(kubectl get deployment scraper-worker -n dispensary-scraper -o jsonpath='{.spec.replicas}'); if [ "$REPLICAS" = "0" ]; then echo "Scaling workers from 0 to 5"; kubectl scale deployment/scraper-worker --replicas=5 -n dispensary-scraper; fi
       - kubectl set image deployment/scraper-worker worker=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/cannaiq-frontend cannaiq-frontend=code.cannabrands.app/creationshop/cannaiq-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/findadispo-frontend findadispo-frontend=code.cannabrands.app/creationshop/findadispo-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/findagram-frontend findagram-frontend=code.cannabrands.app/creationshop/findagram-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
-      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
       - kubectl rollout status deployment/cannaiq-frontend -n dispensary-scraper --timeout=120s
     depends_on:
-      - migrate
+      - docker-backend
       - docker-cannaiq
       - docker-findadispo
       - docker-findagram
     when:
       branch: master
-      event: [push, manual]
+      event: push

.woodpecker/ci.yml

@@ -1,46 +1,38 @@
 steps:
   # ===========================================
-  # PR VALIDATION: Parallel type checks (PRs only)
+  # PR VALIDATION: Only typecheck changed projects
   # ===========================================
   typecheck-backend:
     image: code.cannabrands.app/creationshop/node:20
     commands:
+      - npm config set cache /npm-cache/backend --global
       - cd backend
       - npm ci --prefer-offline
       - npx tsc --noEmit
+    volumes:
+      - npm-cache:/npm-cache
     depends_on: []
     when:
       event: pull_request
+      path:
+        include: ['backend/**']
 
   typecheck-cannaiq:
     image: code.cannabrands.app/creationshop/node:20
     commands:
+      - npm config set cache /npm-cache/cannaiq --global
       - cd cannaiq
       - npm ci --prefer-offline
       - npx tsc --noEmit
+    volumes:
+      - npm-cache:/npm-cache
     depends_on: []
     when:
       event: pull_request
+      path:
+        include: ['cannaiq/**']
 
-  typecheck-findadispo:
+  # findadispo/findagram typechecks skipped - they have || true anyway
-    image: code.cannabrands.app/creationshop/node:20
-    commands:
-      - cd findadispo/frontend
-      - npm ci --prefer-offline
-      - npx tsc --noEmit 2>/dev/null || true
-    depends_on: []
-    when:
-      event: pull_request
-
-  typecheck-findagram:
-    image: code.cannabrands.app/creationshop/node:20
-    commands:
-      - cd findagram/frontend
-      - npm ci --prefer-offline
-      - npx tsc --noEmit 2>/dev/null || true
-    depends_on: []
-    when:
-      event: pull_request
 
   # ===========================================
   # AUTO-MERGE: Merge PR after all checks pass
@@ -62,8 +54,6 @@ steps:
     depends_on:
       - typecheck-backend
       - typecheck-cannaiq
-      - typecheck-findadispo
-      - typecheck-findagram
     when:
       event: pull_request
 
@@ -86,6 +76,8 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/dispensary-scraper:cache,mode=max
       build_args:
         APP_BUILD_VERSION: ${CI_COMMIT_SHA:0:8}
         APP_GIT_SHA: ${CI_COMMIT_SHA}
@@ -94,7 +86,7 @@ steps:
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   docker-cannaiq:
     image: woodpeckerci/plugin-docker-buildx
@@ -112,10 +104,12 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/cannaiq-frontend:cache,mode=max
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   docker-findadispo:
     image: woodpeckerci/plugin-docker-buildx
@@ -133,10 +127,12 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/findadispo-frontend:cache,mode=max
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   docker-findagram:
     image: woodpeckerci/plugin-docker-buildx
@@ -154,38 +150,15 @@ steps:
         from_secret: registry_password
       platforms: linux/amd64
       provenance: false
+      cache_from: type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache
+      cache_to: type=registry,ref=code.cannabrands.app/creationshop/findagram-frontend:cache,mode=max
     depends_on: []
     when:
       branch: master
-      event: [push, manual]
+      event: push
 
   # ===========================================
-  # STAGE 3: Run Database Migrations (before deploy)
+  # STAGE 3: Deploy and Run Migrations
-  # ===========================================
-  migrate:
-    image: code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8}
-    environment:
-      CANNAIQ_DB_HOST:
-        from_secret: db_host
-      CANNAIQ_DB_PORT:
-        from_secret: db_port
-      CANNAIQ_DB_NAME:
-        from_secret: db_name
-      CANNAIQ_DB_USER:
-        from_secret: db_user
-      CANNAIQ_DB_PASS:
-        from_secret: db_pass
-    commands:
-      - cd /app
-      - node dist/db/migrate.js
-    depends_on:
-      - docker-backend
-    when:
-      branch: master
-      event: [push, manual]
-
-  # ===========================================
-  # STAGE 4: Deploy (after migrations)
   # ===========================================
   deploy:
     image: bitnami/kubectl:latest
@@ -196,18 +169,23 @@ steps:
       - mkdir -p ~/.kube
       - echo "$KUBECONFIG_CONTENT" | tr -d '[:space:]' | base64 -d > ~/.kube/config
       - chmod 600 ~/.kube/config
+      # Deploy backend first
       - kubectl set image deployment/scraper scraper=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
+      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
+      # Note: Migrations run automatically at startup via auto-migrate
+      # Deploy remaining services
+      # Resilience: ensure workers are scaled up if at 0
+      - REPLICAS=$(kubectl get deployment scraper-worker -n dispensary-scraper -o jsonpath='{.spec.replicas}'); if [ "$REPLICAS" = "0" ]; then echo "Scaling workers from 0 to 5"; kubectl scale deployment/scraper-worker --replicas=5 -n dispensary-scraper; fi
       - kubectl set image deployment/scraper-worker worker=code.cannabrands.app/creationshop/dispensary-scraper:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/cannaiq-frontend cannaiq-frontend=code.cannabrands.app/creationshop/cannaiq-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/findadispo-frontend findadispo-frontend=code.cannabrands.app/creationshop/findadispo-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
       - kubectl set image deployment/findagram-frontend findagram-frontend=code.cannabrands.app/creationshop/findagram-frontend:${CI_COMMIT_SHA:0:8} -n dispensary-scraper
-      - kubectl rollout status deployment/scraper -n dispensary-scraper --timeout=300s
       - kubectl rollout status deployment/cannaiq-frontend -n dispensary-scraper --timeout=120s
     depends_on:
-      - migrate
+      - docker-backend
       - docker-cannaiq
       - docker-findadispo
       - docker-findagram
     when:
       branch: master
-      event: [push, manual]
+      event: push

CLAUDE.md

@@ -939,7 +939,8 @@ export default defineConfig({
 
 20) **Crawler Architecture**
     - **Scraper pod (1 replica)**: Runs the Express API server + scheduler.
-    - **Scraper-worker pods (5 replicas)**: Each worker runs `dist/dutchie-az/services/worker.js`, polling the job queue.
+    - **Scraper-worker pods (25 replicas)**: Each runs `dist/tasks/task-worker.js`, polling the job queue.
+    - **Worker naming**: Pods use fantasy names (Aethelgard, Xylos, Kryll, Coriolis, etc.) - see `k8s/scraper-worker.yaml` ConfigMap. Worker IDs: `{PodName}-worker-{n}`
     - **Job types**: `menu_detection`, `menu_detection_single`, `dutchie_product_crawl`
     - **Job schedules** (managed in `job_schedules` table):
       - `dutchie_az_menu_detection`: Runs daily with 60-min jitter

backend/migrations/083_discovery_runs.sql

@@ -0,0 +1,88 @@
+-- Migration 083: Discovery Run Tracking
+-- Tracks progress of store discovery runs step-by-step
+
+-- Main discovery runs table
+CREATE TABLE IF NOT EXISTS discovery_runs (
+  id SERIAL PRIMARY KEY,
+  platform VARCHAR(50) NOT NULL DEFAULT 'dutchie',
+  status VARCHAR(20) NOT NULL DEFAULT 'running', -- running, completed, failed
+  started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  finished_at TIMESTAMPTZ,
+  task_id INTEGER REFERENCES worker_task_queue(id),
+
+  -- Totals
+  states_total INTEGER DEFAULT 0,
+  states_completed INTEGER DEFAULT 0,
+  locations_discovered INTEGER DEFAULT 0,
+  locations_promoted INTEGER DEFAULT 0,
+  new_store_ids INTEGER[] DEFAULT '{}',
+
+  -- Error info
+  error_message TEXT,
+
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Per-state progress within a run
+CREATE TABLE IF NOT EXISTS discovery_run_states (
+  id SERIAL PRIMARY KEY,
+  run_id INTEGER NOT NULL REFERENCES discovery_runs(id) ON DELETE CASCADE,
+  state_code VARCHAR(2) NOT NULL,
+  status VARCHAR(20) NOT NULL DEFAULT 'pending', -- pending, running, completed, failed
+  started_at TIMESTAMPTZ,
+  finished_at TIMESTAMPTZ,
+
+  -- Results
+  cities_found INTEGER DEFAULT 0,
+  locations_found INTEGER DEFAULT 0,
+  locations_upserted INTEGER DEFAULT 0,
+  new_dispensary_ids INTEGER[] DEFAULT '{}',
+
+  -- Error info
+  error_message TEXT,
+
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  UNIQUE(run_id, state_code)
+);
+
+-- Step-by-step log for detailed progress tracking
+CREATE TABLE IF NOT EXISTS discovery_run_steps (
+  id SERIAL PRIMARY KEY,
+  run_id INTEGER NOT NULL REFERENCES discovery_runs(id) ON DELETE CASCADE,
+  state_code VARCHAR(2),
+  step_name VARCHAR(100) NOT NULL,
+  status VARCHAR(20) NOT NULL DEFAULT 'started', -- started, completed, failed
+  started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  finished_at TIMESTAMPTZ,
+
+  -- Details (JSON for flexibility)
+  details JSONB DEFAULT '{}',
+
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes for querying
+CREATE INDEX IF NOT EXISTS idx_discovery_runs_status ON discovery_runs(status);
+CREATE INDEX IF NOT EXISTS idx_discovery_runs_platform ON discovery_runs(platform);
+CREATE INDEX IF NOT EXISTS idx_discovery_runs_started_at ON discovery_runs(started_at DESC);
+CREATE INDEX IF NOT EXISTS idx_discovery_run_states_run_id ON discovery_run_states(run_id);
+CREATE INDEX IF NOT EXISTS idx_discovery_run_steps_run_id ON discovery_run_steps(run_id);
+
+-- View for latest run status per platform
+CREATE OR REPLACE VIEW v_latest_discovery_runs AS
+SELECT DISTINCT ON (platform)
+  id,
+  platform,
+  status,
+  started_at,
+  finished_at,
+  states_total,
+  states_completed,
+  locations_discovered,
+  locations_promoted,
+  array_length(new_store_ids, 1) as new_stores_count,
+  error_message,
+  EXTRACT(EPOCH FROM (COALESCE(finished_at, NOW()) - started_at)) as duration_seconds
+FROM discovery_runs
+ORDER BY platform, started_at DESC;

backend/src/routes/worker-registry.ts

@@ -70,21 +70,20 @@ router.post('/register', async (req: Request, res: Response) => {
     );
 
     if (existing.rows.length > 0) {
-      // Re-activate existing worker
+      // Re-activate existing worker - keep existing pod_name (fantasy name), don't overwrite with K8s name
       const { rows } = await pool.query(`
         UPDATE worker_registry
         SET status = 'active',
             role = $1,
-            pod_name = $2,
+            hostname = $2,
-            hostname = $3,
+            ip_address = $3,
-            ip_address = $4,
             last_heartbeat_at = NOW(),
             started_at = NOW(),
-            metadata = $5,
+            metadata = $4,
             updated_at = NOW()
-        WHERE worker_id = $6
+        WHERE worker_id = $5
-        RETURNING id, worker_id, friendly_name, role
+        RETURNING id, worker_id, friendly_name, pod_name, role
-      `, [role, pod_name, finalHostname, clientIp, metadata, finalWorkerId]);
+      `, [role, finalHostname, clientIp, metadata, finalWorkerId]);
 
       const worker = rows[0];
       const roleMsg = role ? `for ${role}` : 'as role-agnostic';
@@ -105,13 +104,13 @@ router.post('/register', async (req: Request, res: Response) => {
     const nameResult = await pool.query('SELECT assign_worker_name($1) as name', [finalWorkerId]);
     const friendlyName = nameResult.rows[0].name;
 
-    // Register the worker
+    // Register the worker - use friendlyName as pod_name (not K8s name)
     const { rows } = await pool.query(`
       INSERT INTO worker_registry (
         worker_id, friendly_name, role, pod_name, hostname, ip_address, status, metadata
       ) VALUES ($1, $2, $3, $4, $5, $6, 'active', $7)
-      RETURNING id, worker_id, friendly_name, role
+      RETURNING id, worker_id, friendly_name, pod_name, role
-    `, [finalWorkerId, friendlyName, role, pod_name, finalHostname, clientIp, metadata]);
+    `, [finalWorkerId, friendlyName, role, friendlyName, finalHostname, clientIp, metadata]);
 
     const worker = rows[0];
     const roleMsg = role ? `for ${role}` : 'as role-agnostic';

backend/src/tasks/handlers/index.ts

@@ -9,3 +9,4 @@ export { handleProductDiscovery } from './product-discovery';
 export { handleStoreDiscovery } from './store-discovery';
 export { handleEntryPointDiscovery } from './entry-point-discovery';
 export { handleAnalyticsRefresh } from './analytics-refresh';
+export { handleProxyTest } from './proxy-test';

backend/src/tasks/handlers/proxy-test.ts

@@ -0,0 +1,51 @@
+/**
+ * Proxy Test Handler
+ * Tests proxy connectivity by fetching public IP via ipify
+ */
+
+import { TaskContext, TaskResult } from '../task-worker';
+import { execSync } from 'child_process';
+
+export async function handleProxyTest(ctx: TaskContext): Promise<TaskResult> {
+  const { pool } = ctx;
+
+  console.log('[ProxyTest] Testing proxy connection...');
+
+  try {
+    // Get active proxy from DB
+    const proxyResult = await pool.query(`
+      SELECT host, port, username, password
+      FROM proxies
+      WHERE is_active = true
+      LIMIT 1
+    `);
+
+    if (proxyResult.rows.length === 0) {
+      return { success: false, error: 'No active proxy configured' };
+    }
+
+    const p = proxyResult.rows[0];
+    const proxyUrl = p.username
+      ? `http://${p.username}:${p.password}@${p.host}:${p.port}`
+      : `http://${p.host}:${p.port}`;
+
+    console.log(`[ProxyTest] Using proxy: ${p.host}:${p.port}`);
+
+    // Fetch IP via proxy
+    const cmd = `curl -s --proxy '${proxyUrl}' 'https://api.ipify.org?format=json'`;
+    const output = execSync(cmd, { timeout: 30000 }).toString().trim();
+    const data = JSON.parse(output);
+
+    console.log(`[ProxyTest] Proxy IP: ${data.ip}`);
+
+    return {
+      success: true,
+      proxyIp: data.ip,
+      proxyHost: p.host,
+      proxyPort: p.port,
+    };
+  } catch (error: any) {
+    console.error('[ProxyTest] Error:', error.message);
+    return { success: false, error: error.message };
+  }
+}

backend/src/tasks/task-service.ts

@@ -31,7 +31,8 @@ export type TaskRole =
   | 'product_discovery'
   | 'payload_fetch'      // NEW: Fetches from API, saves to disk
   | 'product_refresh'    // CHANGED: Now reads from local payload
-  | 'analytics_refresh';
+  | 'analytics_refresh'
+  | 'proxy_test';        // Tests proxy connectivity via ipify
 
 export type TaskStatus =
   | 'pending'

backend/src/tasks/task-worker.ts

@@ -59,6 +59,7 @@ import { handleProductDiscovery } from './handlers/product-discovery';
 import { handleStoreDiscovery } from './handlers/store-discovery';
 import { handleEntryPointDiscovery } from './handlers/entry-point-discovery';
 import { handleAnalyticsRefresh } from './handlers/analytics-refresh';
+import { handleProxyTest } from './handlers/proxy-test';
 
 const POLL_INTERVAL_MS = parseInt(process.env.POLL_INTERVAL_MS || '5000');
 const HEARTBEAT_INTERVAL_MS = parseInt(process.env.HEARTBEAT_INTERVAL_MS || '30000');
@@ -84,6 +85,20 @@ const MAX_CONCURRENT_TASKS = parseInt(process.env.MAX_CONCURRENT_TASKS || '3');
 // Default 85% - gives headroom before OOM
 const MEMORY_BACKOFF_THRESHOLD = parseFloat(process.env.MEMORY_BACKOFF_THRESHOLD || '0.85');
 
+// Parse max heap size from NODE_OPTIONS (--max-old-space-size=1500)
+// This is used as the denominator for memory percentage calculation
+// V8's heapTotal is dynamic and stays small when idle, causing false high percentages
+function getMaxHeapSizeMb(): number {
+  const nodeOptions = process.env.NODE_OPTIONS || '';
+  const match = nodeOptions.match(/--max-old-space-size=(\d+)/);
+  if (match) {
+    return parseInt(match[1], 10);
+  }
+  // Fallback: use 512MB if not specified
+  return 512;
+}
+const MAX_HEAP_SIZE_MB = getMaxHeapSizeMb();
+
 // When CPU usage exceeds this threshold (as decimal 0.0-1.0), stop claiming new tasks
 // Default 90% - allows some burst capacity
 const CPU_BACKOFF_THRESHOLD = parseFloat(process.env.CPU_BACKOFF_THRESHOLD || '0.90');
@@ -119,6 +134,7 @@ const TASK_HANDLERS: Record<TaskRole, TaskHandler> = {
   store_discovery: handleStoreDiscovery,
   entry_point_discovery: handleEntryPointDiscovery,
   analytics_refresh: handleAnalyticsRefresh,
+  proxy_test: handleProxyTest,             // Tests proxy via ipify
 };
 
 /**
@@ -186,12 +202,16 @@ export class TaskWorker {
 
   /**
    * Get current resource usage
+   * Memory percentage is calculated against MAX_HEAP_SIZE_MB (from --max-old-space-size)
+   * NOT against V8's dynamic heapTotal which stays small when idle
    */
   private getResourceStats(): ResourceStats {
     const memUsage = process.memoryUsage();
     const heapUsedMb = memUsage.heapUsed / 1024 / 1024;
-    const heapTotalMb = memUsage.heapTotal / 1024 / 1024;
+    // Use MAX_HEAP_SIZE_MB as ceiling, not dynamic heapTotal
-    const memoryPercent = heapUsedMb / heapTotalMb;
+    // V8's heapTotal stays small when idle (e.g., 36MB) causing false 95%+ readings
+    // With --max-old-space-size=1500, we should calculate against 1500MB
+    const memoryPercent = heapUsedMb / MAX_HEAP_SIZE_MB;
 
     // Calculate CPU usage since last check
     const cpuUsage = process.cpuUsage();
@@ -212,7 +232,7 @@ export class TaskWorker {
     return {
       memoryPercent,
       memoryMb: Math.round(heapUsedMb),
-      memoryTotalMb: Math.round(heapTotalMb),
+      memoryTotalMb: MAX_HEAP_SIZE_MB, // Use max-old-space-size, not dynamic heapTotal
       cpuPercent: Math.min(100, cpuPercent), // Cap at 100%
       isBackingOff: this.isBackingOff,
       backoffReason: this.backoffReason,

cannaiq/index.html

@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>CannaIQ - Cannabis Menu Intelligence Platform</title>
     <meta name="description" content="CannaIQ provides real-time cannabis dispensary menu data, product tracking, and analytics for dispensaries across Arizona." />

cannaiq/public/favicon.svg

@@ -0,0 +1,5 @@
+<svg viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg">
+  <rect width="32" height="32" rx="6" fill="#059669"/>
+  <path d="M16 6C12.5 6 9.5 7.5 7.5 10L16 16L24.5 10C22.5 7.5 19.5 6 16 6Z" fill="white"/>
+  <path d="M7.5 10C6 12 5 14.5 5 17C5 22.5 10 26 16 26C22 26 27 22.5 27 17C27 14.5 26 12 24.5 10L16 16L7.5 10Z" fill="white" fill-opacity="0.7"/>
+</svg>

cannaiq/src/App.tsx

@@ -47,7 +47,6 @@ import CrossStateCompare from './pages/CrossStateCompare';
 import StateDetail from './pages/StateDetail';
 import { Discovery } from './pages/Discovery';
 import { WorkersDashboard } from './pages/WorkersDashboard';
-import { JobQueue } from './pages/JobQueue';
 import TasksDashboard from './pages/TasksDashboard';
 import { ScraperOverviewDashboard } from './pages/ScraperOverviewDashboard';
 import { SeoOrchestrator } from './pages/admin/seo/SeoOrchestrator';
@@ -125,8 +124,6 @@ export default function App() {
         <Route path="/discovery" element={<PrivateRoute><Discovery /></PrivateRoute>} />
         {/* Workers Dashboard */}
         <Route path="/workers" element={<PrivateRoute><WorkersDashboard /></PrivateRoute>} />
-        {/* Job Queue Management */}
-        <Route path="/job-queue" element={<PrivateRoute><JobQueue /></PrivateRoute>} />
         {/* Task Queue Dashboard */}
         <Route path="/tasks" element={<PrivateRoute><TasksDashboard /></PrivateRoute>} />
         {/* Scraper Overview Dashboard (new primary) */}

cannaiq/src/components/Layout.tsx

@@ -2,7 +2,6 @@ import { ReactNode, useEffect, useState, useRef } from 'react';
 import { useNavigate, useLocation, Link } from 'react-router-dom';
 import { useAuthStore } from '../store/authStore';
 import { api } from '../lib/api';
-import { StateSelector } from './StateSelector';
 import {
   LayoutDashboard,
   Building2,
@@ -140,7 +139,7 @@ export function Layout({ children }: LayoutProps) {
     <>
       {/* Logo/Brand */}
       <div className="px-6 py-5 border-b border-gray-200">
-        <div className="flex items-center gap-3">
+        <Link to="/dashboard" className="flex items-center gap-3 hover:opacity-80 transition-opacity">
           <div className="w-8 h-8 bg-emerald-600 rounded-lg flex items-center justify-center">
             <svg viewBox="0 0 24 24" className="w-5 h-5 text-white" fill="currentColor">
               <path d="M12 2C8.5 2 5.5 3.5 3.5 6L12 12L20.5 6C18.5 3.5 15.5 2 12 2Z" />
@@ -155,14 +154,10 @@ export function Layout({ children }: LayoutProps) {
               </p>
             )}
           </div>
-        </div>
+        </Link>
         <p className="text-xs text-gray-500 mt-2 truncate">{user?.email}</p>
       </div>
 
-      {/* State Selector */}
-      <div className="px-4 py-3 border-b border-gray-200 bg-gray-50">
-        <StateSelector showLabel={false} />
-      </div>
 
       {/* Navigation */}
       <nav ref={navRef} className="flex-1 px-3 py-4 space-y-6 overflow-y-auto">
@@ -184,8 +179,7 @@ export function Layout({ children }: LayoutProps) {
           <NavLink to="/admin/orchestrator" icon={<Activity className="w-4 h-4" />} label="Orchestrator" isActive={isActive('/admin/orchestrator')} />
           <NavLink to="/users" icon={<UserCog className="w-4 h-4" />} label="Users" isActive={isActive('/users')} />
           <NavLink to="/workers" icon={<Users className="w-4 h-4" />} label="Workers" isActive={isActive('/workers')} />
-          <NavLink to="/job-queue" icon={<ListOrdered className="w-4 h-4" />} label="Job Queue" isActive={isActive('/job-queue')} />
+          <NavLink to="/tasks" icon={<ListChecks className="w-4 h-4" />} label="Tasks" isActive={isActive('/tasks')} />
-          <NavLink to="/tasks" icon={<ListChecks className="w-4 h-4" />} label="Task Queue" isActive={isActive('/tasks')} />
           <NavLink to="/admin/seo" icon={<FileText className="w-4 h-4" />} label="SEO Pages" isActive={isActive('/admin/seo')} />
           <NavLink to="/proxies" icon={<Shield className="w-4 h-4" />} label="Proxies" isActive={isActive('/proxies')} />
           <NavLink to="/api-permissions" icon={<Key className="w-4 h-4" />} label="API Keys" isActive={isActive('/api-permissions')} />
@@ -234,7 +228,7 @@ export function Layout({ children }: LayoutProps) {
           <button onClick={() => setSidebarOpen(true)} className="p-2 -ml-2 rounded-lg hover:bg-gray-100">
             <Menu className="w-5 h-5 text-gray-600" />
           </button>
-          <div className="flex items-center gap-2">
+          <Link to="/dashboard" className="flex items-center gap-2 hover:opacity-80 transition-opacity">
             <div className="w-6 h-6 bg-emerald-600 rounded flex items-center justify-center">
               <svg viewBox="0 0 24 24" className="w-4 h-4 text-white" fill="currentColor">
                 <path d="M12 2C8.5 2 5.5 3.5 3.5 6L12 12L20.5 6C18.5 3.5 15.5 2 12 2Z" />
@@ -242,7 +236,7 @@ export function Layout({ children }: LayoutProps) {
               </svg>
             </div>
             <span className="font-semibold text-gray-900">CannaIQ</span>
-          </div>
+          </Link>
         </div>
 
         {/* Page content */}

cannaiq/src/pages/JobQueue.tsx

@@ -1,910 +0,0 @@
-import { useState, useEffect, useCallback } from 'react';
-import { Layout } from '../components/Layout';
-import { api } from '../lib/api';
-import {
-  RefreshCw,
-  XCircle,
-  Clock,
-  CheckCircle,
-  Activity,
-  ChevronLeft,
-  ChevronRight,
-  Users,
-  Inbox,
-  Timer,
-  Plus,
-  X,
-  Search,
-  Calendar,
-  Trash2,
-} from 'lucide-react';
-
-// Worker from registry
-interface WorkerResources {
-  memory_mb?: number;
-  memory_total_mb?: number;
-  memory_rss_mb?: number;
-  cpu_user_ms?: number;
-  cpu_system_ms?: number;
-}
-
-interface Worker {
-  id: number;
-  worker_id: string;
-  friendly_name: string;
-  role: string;
-  status: string;
-  pod_name: string | null;
-  hostname: string | null;
-  started_at: string;
-  last_heartbeat_at: string;
-  last_task_at: string | null;
-  tasks_completed: number;
-  tasks_failed: number;
-  current_task_id: number | null;
-  health_status: string;
-  seconds_since_heartbeat: number;
-  metadata?: WorkerResources;
-}
-
-// Task from worker_tasks
-interface Task {
-  id: number;
-  role: string;
-  dispensary_id: number | null;
-  dispensary_name?: string;
-  dispensary_slug?: string;
-  status: string;
-  priority: number;
-  claimed_by: string | null;
-  claimed_at: string | null;
-  started_at: string | null;
-  completed_at: string | null;
-  error: string | null;
-  error_message: string | null;
-  retry_count: number;
-  max_retries: number;
-  result: any;
-  created_at: string;
-  updated_at: string;
-}
-
-interface TaskCounts {
-  pending: number;
-  running: number;
-  completed: number;
-  failed: number;
-  total: number;
-}
-
-interface Store {
-  id: number;
-  name: string;
-  state_code: string;
-  crawl_enabled: boolean;
-}
-
-interface CreateTaskModalProps {
-  isOpen: boolean;
-  onClose: () => void;
-  onTaskCreated: () => void;
-}
-
-const ROLES = [
-  { id: 'product_refresh', name: 'Product Resync', description: 'Re-crawl products for price/stock changes' },
-  { id: 'product_discovery', name: 'Product Discovery', description: 'Initial crawl for new dispensaries' },
-  { id: 'store_discovery', name: 'Store Discovery', description: 'Discover new dispensary locations' },
-  { id: 'entry_point_discovery', name: 'Entry Point Discovery', description: 'Resolve platform IDs from menu URLs' },
-  { id: 'analytics_refresh', name: 'Analytics Refresh', description: 'Refresh materialized views' },
-];
-
-function CreateTaskModal({ isOpen, onClose, onTaskCreated }: CreateTaskModalProps) {
-  const [role, setRole] = useState('product_refresh');
-  const [priority, setPriority] = useState(10);
-  const [scheduleType, setScheduleType] = useState<'now' | 'scheduled'>('now');
-  const [scheduledFor, setScheduledFor] = useState('');
-  const [stores, setStores] = useState<Store[]>([]);
-  const [storeSearch, setStoreSearch] = useState('');
-  const [selectedStores, setSelectedStores] = useState<Store[]>([]);
-  const [loading, setLoading] = useState(false);
-  const [storesLoading, setStoresLoading] = useState(false);
-  const [error, setError] = useState<string | null>(null);
-
-  // Fetch stores when modal opens
-  useEffect(() => {
-    if (isOpen) {
-      fetchStores();
-    }
-  }, [isOpen]);
-
-  const fetchStores = async () => {
-    setStoresLoading(true);
-    try {
-      const res = await api.get('/api/stores?limit=500');
-      setStores(res.data.stores || res.data || []);
-    } catch (err) {
-      console.error('Failed to fetch stores:', err);
-    } finally {
-      setStoresLoading(false);
-    }
-  };
-
-  const filteredStores = stores.filter(s =>
-    s.name.toLowerCase().includes(storeSearch.toLowerCase()) ||
-    s.state_code?.toLowerCase().includes(storeSearch.toLowerCase())
-  );
-
-  const toggleStore = (store: Store) => {
-    if (selectedStores.find(s => s.id === store.id)) {
-      setSelectedStores(selectedStores.filter(s => s.id !== store.id));
-    } else {
-      setSelectedStores([...selectedStores, store]);
-    }
-  };
-
-  const selectAll = () => {
-    setSelectedStores(filteredStores);
-  };
-
-  const clearAll = () => {
-    setSelectedStores([]);
-  };
-
-  const handleSubmit = async () => {
-    setLoading(true);
-    setError(null);
-
-    try {
-      const scheduledDate = scheduleType === 'scheduled' && scheduledFor
-        ? new Date(scheduledFor).toISOString()
-        : undefined;
-
-      // For store_discovery and analytics_refresh, no store is needed
-      if (role === 'store_discovery' || role === 'analytics_refresh') {
-        await api.post('/api/tasks', {
-          role,
-          priority,
-          scheduled_for: scheduledDate,
-          platform: 'dutchie',
-        });
-      } else if (selectedStores.length === 0) {
-        setError('Please select at least one store');
-        setLoading(false);
-        return;
-      } else {
-        // Create tasks for each selected store
-        for (const store of selectedStores) {
-          await api.post('/api/tasks', {
-            role,
-            dispensary_id: store.id,
-            priority,
-            scheduled_for: scheduledDate,
-            platform: 'dutchie',
-          });
-        }
-      }
-
-      onTaskCreated();
-      onClose();
-      // Reset form
-      setSelectedStores([]);
-      setPriority(10);
-      setScheduleType('now');
-      setScheduledFor('');
-    } catch (err: any) {
-      setError(err.response?.data?.error || err.message || 'Failed to create task');
-    } finally {
-      setLoading(false);
-    }
-  };
-
-  if (!isOpen) return null;
-
-  const needsStore = role !== 'store_discovery' && role !== 'analytics_refresh';
-
-  return (
-    <div className="fixed inset-0 z-50 overflow-y-auto">
-      <div className="flex min-h-full items-center justify-center p-4">
-        {/* Backdrop */}
-        <div className="fixed inset-0 bg-black/50" onClick={onClose} />
-
-        {/* Modal */}
-        <div className="relative bg-white rounded-xl shadow-xl max-w-2xl w-full max-h-[90vh] overflow-hidden">
-          {/* Header */}
-          <div className="px-6 py-4 border-b border-gray-200 flex items-center justify-between">
-            <h2 className="text-lg font-semibold text-gray-900">Create New Task</h2>
-            <button onClick={onClose} className="p-1 hover:bg-gray-100 rounded">
-              <X className="w-5 h-5 text-gray-500" />
-            </button>
-          </div>
-
-          {/* Body */}
-          <div className="px-6 py-4 space-y-6 overflow-y-auto max-h-[calc(90vh-140px)]">
-            {error && (
-              <div className="bg-red-50 border border-red-200 rounded-lg p-3 text-red-700 text-sm">
-                {error}
-              </div>
-            )}
-
-            {/* Role Selection */}
-            <div>
-              <label className="block text-sm font-medium text-gray-700 mb-2">Task Role</label>
-              <div className="grid grid-cols-1 gap-2">
-                {ROLES.map(r => (
-                  <button
-                    key={r.id}
-                    onClick={() => setRole(r.id)}
-                    className={`flex items-start gap-3 p-3 rounded-lg border text-left transition-colors ${
-                      role === r.id
-                        ? 'border-emerald-500 bg-emerald-50'
-                        : 'border-gray-200 hover:border-gray-300'
-                    }`}
-                  >
-                    <div className={`w-4 h-4 rounded-full border-2 mt-0.5 flex-shrink-0 ${
-                      role === r.id ? 'border-emerald-500 bg-emerald-500' : 'border-gray-300'
-                    }`}>
-                      {role === r.id && (
-                        <div className="w-full h-full flex items-center justify-center">
-                          <div className="w-1.5 h-1.5 bg-white rounded-full" />
-                        </div>
-                      )}
-                    </div>
-                    <div>
-                      <p className="font-medium text-gray-900">{r.name}</p>
-                      <p className="text-xs text-gray-500">{r.description}</p>
-                    </div>
-                  </button>
-                ))}
-              </div>
-            </div>
-
-            {/* Store Selection (for roles that need it) */}
-            {needsStore && (
-              <div>
-                <label className="block text-sm font-medium text-gray-700 mb-2">
-                  Select Stores ({selectedStores.length} selected)
-                </label>
-                <div className="border border-gray-200 rounded-lg overflow-hidden">
-                  {/* Search */}
-                  <div className="p-2 border-b border-gray-200 bg-gray-50">
-                    <div className="relative">
-                      <Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
-                      <input
-                        type="text"
-                        value={storeSearch}
-                        onChange={(e) => setStoreSearch(e.target.value)}
-                        placeholder="Search stores..."
-                        className="w-full pl-9 pr-3 py-2 text-sm border border-gray-200 rounded"
-                      />
-                    </div>
-                    <div className="flex gap-2 mt-2">
-                      <button
-                        onClick={selectAll}
-                        className="text-xs text-emerald-600 hover:underline"
-                      >
-                        Select all ({filteredStores.length})
-                      </button>
-                      <span className="text-gray-300">|</span>
-                      <button
-                        onClick={clearAll}
-                        className="text-xs text-gray-500 hover:underline"
-                      >
-                        Clear
-                      </button>
-                    </div>
-                  </div>
-
-                  {/* Store List */}
-                  <div className="max-h-48 overflow-y-auto">
-                    {storesLoading ? (
-                      <div className="p-4 text-center text-gray-500">
-                        <RefreshCw className="w-5 h-5 animate-spin mx-auto mb-1" />
-                        Loading stores...
-                      </div>
-                    ) : filteredStores.length === 0 ? (
-                      <div className="p-4 text-center text-gray-500">No stores found</div>
-                    ) : (
-                      filteredStores.map(store => (
-                        <label
-                          key={store.id}
-                          className="flex items-center gap-3 px-3 py-2 hover:bg-gray-50 cursor-pointer"
-                        >
-                          <input
-                            type="checkbox"
-                            checked={!!selectedStores.find(s => s.id === store.id)}
-                            onChange={() => toggleStore(store)}
-                            className="w-4 h-4 text-emerald-600 rounded"
-                          />
-                          <div className="flex-1 min-w-0">
-                            <p className="text-sm text-gray-900 truncate">{store.name}</p>
-                            <p className="text-xs text-gray-500">{store.state_code}</p>
-                          </div>
-                          {!store.crawl_enabled && (
-                            <span className="text-xs text-orange-600 bg-orange-50 px-1.5 py-0.5 rounded">
-                              disabled
-                            </span>
-                          )}
-                        </label>
-                      ))
-                    )}
-                  </div>
-                </div>
-              </div>
-            )}
-
-            {/* Priority */}
-            <div>
-              <label className="block text-sm font-medium text-gray-700 mb-2">
-                Priority: {priority}
-              </label>
-              <input
-                type="range"
-                min="0"
-                max="100"
-                value={priority}
-                onChange={(e) => setPriority(parseInt(e.target.value))}
-                className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer"
-              />
-              <div className="flex justify-between text-xs text-gray-500 mt-1">
-                <span>0 (Low - Batch)</span>
-                <span>10 (Normal)</span>
-                <span>50 (High)</span>
-                <span>100 (Urgent)</span>
-              </div>
-            </div>
-
-            {/* Schedule */}
-            <div>
-              <label className="block text-sm font-medium text-gray-700 mb-2">Schedule</label>
-              <div className="flex gap-4">
-                <label className="flex items-center gap-2 cursor-pointer">
-                  <input
-                    type="radio"
-                    name="schedule"
-                    checked={scheduleType === 'now'}
-                    onChange={() => setScheduleType('now')}
-                    className="w-4 h-4 text-emerald-600"
-                  />
-                  <span className="text-sm text-gray-700">Run immediately</span>
-                </label>
-                <label className="flex items-center gap-2 cursor-pointer">
-                  <input
-                    type="radio"
-                    name="schedule"
-                    checked={scheduleType === 'scheduled'}
-                    onChange={() => setScheduleType('scheduled')}
-                    className="w-4 h-4 text-emerald-600"
-                  />
-                  <span className="text-sm text-gray-700">Schedule for later</span>
-                </label>
-              </div>
-
-              {scheduleType === 'scheduled' && (
-                <div className="mt-3">
-                  <div className="relative">
-                    <Calendar className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
-                    <input
-                      type="datetime-local"
-                      value={scheduledFor}
-                      onChange={(e) => setScheduledFor(e.target.value)}
-                      className="w-full pl-9 pr-3 py-2 text-sm border border-gray-200 rounded"
-                    />
-                  </div>
-                </div>
-              )}
-            </div>
-          </div>
-
-          {/* Footer */}
-          <div className="px-6 py-4 border-t border-gray-200 bg-gray-50 flex items-center justify-between">
-            <div className="text-sm text-gray-500">
-              {needsStore ? (
-                selectedStores.length > 0 ? (
-                  `Will create ${selectedStores.length} task${selectedStores.length > 1 ? 's' : ''}`
-                ) : (
-                  'Select stores to create tasks'
-                )
-              ) : (
-                'Will create 1 task'
-              )}
-            </div>
-            <div className="flex gap-3">
-              <button
-                onClick={onClose}
-                className="px-4 py-2 text-sm text-gray-700 hover:bg-gray-100 rounded-lg"
-              >
-                Cancel
-              </button>
-              <button
-                onClick={handleSubmit}
-                disabled={loading || (needsStore && selectedStores.length === 0)}
-                className="px-4 py-2 text-sm bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2"
-              >
-                {loading && <RefreshCw className="w-4 h-4 animate-spin" />}
-                Create Task{selectedStores.length > 1 ? 's' : ''}
-              </button>
-            </div>
-          </div>
-        </div>
-      </div>
-    </div>
-  );
-}
-
-function formatRelativeTime(dateStr: string | null): string {
-  if (!dateStr) return '-';
-  const date = new Date(dateStr);
-  const now = new Date();
-  const diffMs = now.getTime() - date.getTime();
-  const diffSecs = Math.round(diffMs / 1000);
-  const diffMins = Math.round(diffMs / 60000);
-
-  if (diffSecs < 60) return `${diffSecs}s ago`;
-  if (diffMins < 60) return `${diffMins}m ago`;
-  if (diffMins < 1440) return `${Math.round(diffMins / 60)}h ago`;
-  return `${Math.round(diffMins / 1440)}d ago`;
-}
-
-function formatDuration(startStr: string | null, endStr: string | null): string {
-  if (!startStr) return '-';
-  const start = new Date(startStr);
-  const end = endStr ? new Date(endStr) : new Date();
-  const diffMs = end.getTime() - start.getTime();
-
-  if (diffMs < 1000) return `${diffMs}ms`;
-  if (diffMs < 60000) return `${(diffMs / 1000).toFixed(1)}s`;
-  const mins = Math.floor(diffMs / 60000);
-  const secs = Math.floor((diffMs % 60000) / 1000);
-  if (mins < 60) return `${mins}m ${secs}s`;
-  const hrs = Math.floor(mins / 60);
-  return `${hrs}h ${mins % 60}m`;
-}
-
-// Live timer component for running tasks
-function LiveTimer({ startedAt, isRunning }: { startedAt: string | null; isRunning: boolean }) {
-  const [, setTick] = useState(0);
-
-  useEffect(() => {
-    if (!isRunning || !startedAt) return;
-    const interval = setInterval(() => setTick(t => t + 1), 1000);
-    return () => clearInterval(interval);
-  }, [isRunning, startedAt]);
-
-  if (!startedAt) return <span className="text-gray-400">-</span>;
-
-  const duration = formatDuration(startedAt, null);
-
-  if (isRunning) {
-    return (
-      <span className="inline-flex items-center gap-1 text-blue-600 font-medium">
-        <Timer className="w-3 h-3 animate-pulse" />
-        {duration}
-      </span>
-    );
-  }
-
-  return <span>{duration}</span>;
-}
-
-function WorkerStatusBadge({ status, healthStatus }: { status: string; healthStatus: string }) {
-  const getColors = () => {
-    if (healthStatus === 'offline' || status === 'offline') return 'bg-gray-100 text-gray-600';
-    if (healthStatus === 'stale') return 'bg-yellow-100 text-yellow-700';
-    if (healthStatus === 'busy' || status === 'active') return 'bg-blue-100 text-blue-700';
-    if (healthStatus === 'ready' || status === 'idle') return 'bg-green-100 text-green-700';
-    return 'bg-gray-100 text-gray-600';
-  };
-
-  return (
-    <span className={`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-medium ${getColors()}`}>
-      {healthStatus || status}
-    </span>
-  );
-}
-
-function TaskStatusBadge({ status, error, retryCount }: { status: string; error?: string | null; retryCount?: number }) {
-  const config: Record<string, { bg: string; text: string; icon: any }> = {
-    pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock },
-    running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity },
-    completed: { bg: 'bg-green-100', text: 'text-green-700', icon: CheckCircle },
-    failed: { bg: 'bg-red-100', text: 'text-red-700', icon: XCircle },
-  };
-
-  const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock };
-  const Icon = cfg.icon;
-
-  // Build tooltip text
-  let tooltip = '';
-  if (error) {
-    tooltip = error;
-  }
-  if (retryCount && retryCount > 0) {
-    tooltip = `Attempt ${retryCount + 1}${error ? `: ${error}` : ''}`;
-  }
-
-  return (
-    <span
-      className={`inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium ${cfg.bg} ${cfg.text} ${error ? 'cursor-help' : ''}`}
-      title={tooltip || undefined}
-    >
-      <Icon className="w-3 h-3" />
-      {status}
-      {retryCount && retryCount > 0 && status !== 'failed' && (
-        <span className="text-[10px] opacity-75">({retryCount})</span>
-      )}
-    </span>
-  );
-}
-
-function RoleBadge({ role }: { role: string }) {
-  const colors: Record<string, string> = {
-    product_refresh: 'bg-emerald-100 text-emerald-700',
-    product_discovery: 'bg-blue-100 text-blue-700',
-    store_discovery: 'bg-purple-100 text-purple-700',
-    entry_point_discovery: 'bg-orange-100 text-orange-700',
-    analytics_refresh: 'bg-pink-100 text-pink-700',
-  };
-
-  return (
-    <span className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium ${colors[role] || 'bg-gray-100 text-gray-700'}`}>
-      {role.replace(/_/g, ' ')}
-    </span>
-  );
-}
-
-function PriorityBadge({ priority }: { priority: number }) {
-  let bg = 'bg-gray-100 text-gray-700';
-  if (priority >= 80) bg = 'bg-red-100 text-red-700';
-  else if (priority >= 50) bg = 'bg-orange-100 text-orange-700';
-  else if (priority >= 20) bg = 'bg-yellow-100 text-yellow-700';
-
-  return (
-    <span className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium ${bg}`}>
-      P{priority}
-    </span>
-  );
-}
-
-export function JobQueue() {
-  const [workers, setWorkers] = useState<Worker[]>([]);
-  const [tasks, setTasks] = useState<Task[]>([]);
-  const [counts, setCounts] = useState<TaskCounts | null>(null);
-  const [loading, setLoading] = useState(true);
-  const [error, setError] = useState<string | null>(null);
-  const [showCreateModal, setShowCreateModal] = useState(false);
-
-  // Pagination
-  const [taskPage, setTaskPage] = useState(0);
-  const tasksPerPage = 25;
-
-  // Cleanup stale workers (called once on page load)
-  const cleanupStaleWorkers = useCallback(async () => {
-    try {
-      await api.post('/api/worker-registry/cleanup', { stale_threshold_minutes: 2 });
-    } catch (err: any) {
-      console.error('Failed to cleanup stale workers:', err);
-    }
-  }, []);
-
-  // Fetch workers
-  const fetchWorkers = useCallback(async () => {
-    try {
-      const workersRes = await api.get('/api/worker-registry/workers');
-      setWorkers(workersRes.data.workers || []);
-    } catch (err: any) {
-      console.error('Failed to fetch workers:', err);
-    }
-  }, []);
-
-  // Fetch tasks and counts (auto-refresh every 15s)
-  const fetchTasks = useCallback(async () => {
-    try {
-      const taskUrl = `/api/tasks?limit=${tasksPerPage}&offset=${taskPage * tasksPerPage}`;
-
-      const [tasksRes, countsRes] = await Promise.all([
-        api.get(taskUrl),
-        api.get('/api/tasks/counts'),
-      ]);
-
-      setTasks(tasksRes.data.tasks || []);
-      setCounts(countsRes.data);
-      setError(null);
-    } catch (err: any) {
-      console.error('Fetch error:', err);
-      setError(err.message || 'Failed to fetch data');
-    } finally {
-      setLoading(false);
-    }
-  }, [taskPage]);
-
-  // Initial load - cleanup stale workers first, then fetch
-  useEffect(() => {
-    cleanupStaleWorkers().then(() => {
-      fetchWorkers();
-      fetchTasks();
-    });
-  }, [cleanupStaleWorkers, fetchWorkers, fetchTasks]);
-
-  // Auto-refresh tasks every 15 seconds
-  useEffect(() => {
-    const interval = setInterval(fetchTasks, 15000);
-    return () => clearInterval(interval);
-  }, [fetchTasks]);
-
-  // Refresh workers every 60 seconds
-  useEffect(() => {
-    const interval = setInterval(fetchWorkers, 60000);
-    return () => clearInterval(interval);
-  }, [fetchWorkers]);
-
-  // Delete a task
-  const handleDeleteTask = async (taskId: number) => {
-    if (!confirm('Delete this task?')) return;
-    try {
-      await api.delete(`/api/tasks/${taskId}`);
-      fetchTasks();
-    } catch (err: any) {
-      console.error('Delete error:', err);
-      alert(err.response?.data?.error || 'Failed to delete task');
-    }
-  };
-
-  // Get active workers (for display)
-  const activeWorkers = workers.filter(w => w.status !== 'offline' && w.status !== 'terminated');
-
-  if (loading) {
-    return (
-      <Layout>
-        <div className="flex items-center justify-center h-64">
-          <RefreshCw className="w-8 h-8 text-gray-400 animate-spin" />
-        </div>
-      </Layout>
-    );
-  }
-
-  return (
-    <Layout>
-      <div className="space-y-6">
-        {/* Header */}
-        <div className="flex items-center justify-between">
-          <div>
-            <h1 className="text-2xl font-bold text-gray-900">Task Queue</h1>
-            <p className="text-gray-500 mt-1">
-              Workers pull tasks from the pool by priority (auto-refresh every 15s)
-            </p>
-          </div>
-          <button
-            onClick={() => setShowCreateModal(true)}
-            className="flex items-center gap-2 px-4 py-2 bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 transition-colors"
-          >
-            <Plus className="w-4 h-4" />
-            Create Task
-          </button>
-        </div>
-
-        {/* Create Task Modal */}
-        <CreateTaskModal
-          isOpen={showCreateModal}
-          onClose={() => setShowCreateModal(false)}
-          onTaskCreated={fetchTasks}
-        />
-
-        {error && (
-          <div className="bg-red-50 border border-red-200 rounded-lg p-4">
-            <p className="text-red-700">{error}</p>
-          </div>
-        )}
-
-        {/* Stats Cards */}
-        {counts && (
-          <div className="grid grid-cols-5 gap-4">
-            <div className="bg-white rounded-lg border border-gray-200 p-4">
-              <div className="flex items-center gap-3">
-                <div className="w-10 h-10 bg-emerald-100 rounded-lg flex items-center justify-center">
-                  <Users className="w-5 h-5 text-emerald-600" />
-                </div>
-                <div>
-                  <p className="text-sm text-gray-500">Active Workers</p>
-                  <p className="text-xl font-semibold">{activeWorkers.length}</p>
-                </div>
-              </div>
-            </div>
-            <div className="bg-white rounded-lg border border-gray-200 p-4">
-              <div className="flex items-center gap-3">
-                <div className="w-10 h-10 bg-yellow-100 rounded-lg flex items-center justify-center">
-                  <Inbox className="w-5 h-5 text-yellow-600" />
-                </div>
-                <div>
-                  <p className="text-sm text-gray-500">Pending Tasks</p>
-                  <p className="text-xl font-semibold">{counts.pending}</p>
-                </div>
-              </div>
-            </div>
-            <div className="bg-white rounded-lg border border-gray-200 p-4">
-              <div className="flex items-center gap-3">
-                <div className="w-10 h-10 bg-blue-100 rounded-lg flex items-center justify-center">
-                  <Activity className="w-5 h-5 text-blue-600" />
-                </div>
-                <div>
-                  <p className="text-sm text-gray-500">Running</p>
-                  <p className="text-xl font-semibold">{counts.running}</p>
-                </div>
-              </div>
-            </div>
-            <div className="bg-white rounded-lg border border-gray-200 p-4">
-              <div className="flex items-center gap-3">
-                <div className="w-10 h-10 bg-green-100 rounded-lg flex items-center justify-center">
-                  <CheckCircle className="w-5 h-5 text-green-600" />
-                </div>
-                <div>
-                  <p className="text-sm text-gray-500">Completed</p>
-                  <p className="text-xl font-semibold">{counts.completed}</p>
-                </div>
-              </div>
-            </div>
-            <div className="bg-white rounded-lg border border-gray-200 p-4">
-              <div className="flex items-center gap-3">
-                <div className="w-10 h-10 bg-red-100 rounded-lg flex items-center justify-center">
-                  <XCircle className="w-5 h-5 text-red-600" />
-                </div>
-                <div>
-                  <p className="text-sm text-gray-500">Failed</p>
-                  <p className="text-xl font-semibold">{counts.failed}</p>
-                </div>
-              </div>
-            </div>
-          </div>
-        )}
-
-        {/* Task Pool Section */}
-        <div className="bg-white rounded-lg border border-gray-200 overflow-hidden">
-          <div className="px-4 py-3 border-b border-gray-200 bg-gray-50">
-            <div className="flex items-center justify-between">
-              <div>
-                <h3 className="text-sm font-semibold text-gray-900 flex items-center gap-2">
-                  <Inbox className="w-4 h-4 text-yellow-500" />
-                  Task Pool
-                </h3>
-                <p className="text-xs text-gray-500 mt-0.5">
-                  Tasks waiting to be picked up by workers
-                </p>
-              </div>
-
-            </div>
-          </div>
-
-          <table className="w-full">
-            <thead className="bg-gray-50">
-              <tr>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Priority</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Role</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Dispensary</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Status</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Assigned To</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Created</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Duration</th>
-                <th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase w-16"></th>
-              </tr>
-            </thead>
-            <tbody className="divide-y divide-gray-200">
-              {tasks.length === 0 ? (
-                <tr>
-                  <td colSpan={8} className="px-4 py-8 text-center text-gray-500">
-                    <Inbox className="w-8 h-8 mx-auto mb-2 text-gray-300" />
-                    <p>No tasks found</p>
-                  </td>
-                </tr>
-              ) : (
-                tasks.map((task) => {
-                  // Find worker assigned to this task
-                  const assignedWorker = task.claimed_by
-                    ? workers.find(w => w.worker_id === task.claimed_by)
-                    : null;
-
-                  return (
-                    <tr key={task.id} className="hover:bg-gray-50">
-                      <td className="px-4 py-3">
-                        <PriorityBadge priority={task.priority} />
-                      </td>
-                      <td className="px-4 py-3">
-                        <RoleBadge role={task.role} />
-                      </td>
-                      <td className="px-4 py-3 text-sm">
-                        {task.dispensary_slug ? (
-                          <span
-                            className="font-mono text-gray-700 truncate block max-w-[200px]"
-                            title={task.dispensary_slug}
-                          >
-                            {task.dispensary_slug.length > 25
-                              ? task.dispensary_slug.slice(0, 25) + '…'
-                              : task.dispensary_slug}
-                          </span>
-                        ) : task.dispensary_name ? (
-                          <span title={task.dispensary_name}>
-                            {task.dispensary_name.length > 25
-                              ? task.dispensary_name.slice(0, 25) + '…'
-                              : task.dispensary_name}
-                          </span>
-                        ) : task.dispensary_id ? (
-                          <span className="text-gray-400">ID: {task.dispensary_id}</span>
-                        ) : (
-                          <span className="text-gray-400">-</span>
-                        )}
-                      </td>
-                      <td className="px-4 py-3">
-                        <TaskStatusBadge status={task.status} error={task.error_message || task.error} retryCount={task.retry_count} />
-                      </td>
-                      <td className="px-4 py-3 text-sm">
-                        {assignedWorker ? (
-                          <span className="inline-flex items-center gap-1">
-                            <span className={`w-2 h-2 rounded-full ${
-                              assignedWorker.health_status === 'busy' ? 'bg-blue-500' : 'bg-green-500'
-                            }`} />
-                            {assignedWorker.friendly_name}
-                          </span>
-                        ) : task.claimed_by ? (
-                          <span className="text-gray-400 text-xs font-mono">{task.claimed_by.slice(0, 12)}...</span>
-                        ) : (
-                          <span className="text-gray-400">Unassigned</span>
-                        )}
-                      </td>
-                      <td className="px-4 py-3 text-sm text-gray-500">
-                        {formatRelativeTime(task.created_at)}
-                      </td>
-                      <td className="px-4 py-3 text-sm text-gray-500">
-                        {task.status === 'running' ? (
-                          <LiveTimer startedAt={task.started_at} isRunning={true} />
-                        ) : task.started_at ? (
-                          formatDuration(task.started_at, task.completed_at)
-                        ) : (
-                          '-'
-                        )}
-                      </td>
-                      <td className="px-4 py-3">
-                        {(task.status === 'failed' || task.status === 'completed' || task.status === 'pending') && (
-                          <button
-                            onClick={() => handleDeleteTask(task.id)}
-                            className="p-1 text-gray-400 hover:text-red-500 hover:bg-red-50 rounded transition-colors"
-                            title="Delete task"
-                          >
-                            <Trash2 className="w-4 h-4" />
-                          </button>
-                        )}
-                      </td>
-                    </tr>
-                  );
-                })
-              )}
-            </tbody>
-          </table>
-
-          {/* Pagination */}
-          <div className="px-4 py-3 border-t border-gray-200 bg-gray-50 flex items-center justify-between">
-            <div className="text-sm text-gray-500">
-              Showing {taskPage * tasksPerPage + 1} - {Math.min((taskPage + 1) * tasksPerPage, taskPage * tasksPerPage + tasks.length)} tasks
-            </div>
-            <div className="flex items-center gap-2">
-              <button
-                onClick={() => setTaskPage(p => Math.max(0, p - 1))}
-                disabled={taskPage === 0}
-                className="px-3 py-1 text-sm border border-gray-200 rounded hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed"
-              >
-                <ChevronLeft className="w-4 h-4" />
-              </button>
-              <span className="text-sm text-gray-600">Page {taskPage + 1}</span>
-              <button
-                onClick={() => setTaskPage(p => p + 1)}
-                disabled={tasks.length < tasksPerPage}
-                className="px-3 py-1 text-sm border border-gray-200 rounded hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed"
-              >
-                <ChevronRight className="w-4 h-4" />
-              </button>
-            </div>
-          </div>
-        </div>
-      </div>
-    </Layout>
-  );
-}
-
-export default JobQueue;

cannaiq/src/pages/TasksDashboard.tsx

@@ -12,10 +12,15 @@ import {
   Search,
   ChevronDown,
   ChevronUp,
+  ChevronLeft,
+  ChevronRight,
   Gauge,
   Users,
-  Play,
   Square,
+  Plus,
+  X,
+  Calendar,
+  Trash2,
 } from 'lucide-react';
 
 interface Task {
@@ -65,6 +70,313 @@ interface TaskCounts {
   stale: number;
 }
 
+interface Store {
+  id: number;
+  name: string;
+  state_code: string;
+  crawl_enabled: boolean;
+}
+
+interface CreateTaskModalProps {
+  isOpen: boolean;
+  onClose: () => void;
+  onTaskCreated: () => void;
+}
+
+const TASK_ROLES = [
+  { id: 'product_refresh', name: 'Product Resync', description: 'Re-crawl products for price/stock changes' },
+  { id: 'product_discovery', name: 'Product Discovery', description: 'Initial crawl for new dispensaries' },
+  { id: 'store_discovery', name: 'Store Discovery', description: 'Discover new dispensary locations' },
+  { id: 'entry_point_discovery', name: 'Entry Point Discovery', description: 'Resolve platform IDs from menu URLs' },
+  { id: 'analytics_refresh', name: 'Analytics Refresh', description: 'Refresh materialized views' },
+];
+
+function CreateTaskModal({ isOpen, onClose, onTaskCreated }: CreateTaskModalProps) {
+  const [role, setRole] = useState('product_refresh');
+  const [priority, setPriority] = useState(10);
+  const [scheduleType, setScheduleType] = useState<'now' | 'scheduled'>('now');
+  const [scheduledFor, setScheduledFor] = useState('');
+  const [stores, setStores] = useState<Store[]>([]);
+  const [storeSearch, setStoreSearch] = useState('');
+  const [selectedStores, setSelectedStores] = useState<Store[]>([]);
+  const [loading, setLoading] = useState(false);
+  const [storesLoading, setStoresLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (isOpen) {
+      fetchStores();
+    }
+  }, [isOpen]);
+
+  const fetchStores = async () => {
+    setStoresLoading(true);
+    try {
+      const res = await api.get('/api/stores?limit=500');
+      setStores(res.data.stores || res.data || []);
+    } catch (err) {
+      console.error('Failed to fetch stores:', err);
+    } finally {
+      setStoresLoading(false);
+    }
+  };
+
+  const filteredStores = stores.filter(s =>
+    s.name.toLowerCase().includes(storeSearch.toLowerCase()) ||
+    s.state_code?.toLowerCase().includes(storeSearch.toLowerCase())
+  );
+
+  const toggleStore = (store: Store) => {
+    if (selectedStores.find(s => s.id === store.id)) {
+      setSelectedStores(selectedStores.filter(s => s.id !== store.id));
+    } else {
+      setSelectedStores([...selectedStores, store]);
+    }
+  };
+
+  const selectAll = () => setSelectedStores(filteredStores);
+  const clearAll = () => setSelectedStores([]);
+
+  const handleSubmit = async () => {
+    setLoading(true);
+    setError(null);
+
+    try {
+      const scheduledDate = scheduleType === 'scheduled' && scheduledFor
+        ? new Date(scheduledFor).toISOString()
+        : undefined;
+
+      if (role === 'store_discovery' || role === 'analytics_refresh') {
+        await api.post('/api/tasks', {
+          role,
+          priority,
+          scheduled_for: scheduledDate,
+          platform: 'dutchie',
+        });
+      } else if (selectedStores.length === 0) {
+        setError('Please select at least one store');
+        setLoading(false);
+        return;
+      } else {
+        for (const store of selectedStores) {
+          await api.post('/api/tasks', {
+            role,
+            dispensary_id: store.id,
+            priority,
+            scheduled_for: scheduledDate,
+            platform: 'dutchie',
+          });
+        }
+      }
+
+      onTaskCreated();
+      onClose();
+      setSelectedStores([]);
+      setPriority(10);
+      setScheduleType('now');
+      setScheduledFor('');
+    } catch (err: any) {
+      setError(err.response?.data?.error || err.message || 'Failed to create task');
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  if (!isOpen) return null;
+
+  const needsStore = role !== 'store_discovery' && role !== 'analytics_refresh';
+
+  return (
+    <div className="fixed inset-0 z-50 overflow-y-auto">
+      <div className="flex min-h-full items-center justify-center p-4">
+        <div className="fixed inset-0 bg-black/50" onClick={onClose} />
+        <div className="relative bg-white rounded-xl shadow-xl max-w-2xl w-full max-h-[90vh] overflow-hidden">
+          <div className="px-6 py-4 border-b border-gray-200 flex items-center justify-between">
+            <h2 className="text-lg font-semibold text-gray-900">Create New Task</h2>
+            <button onClick={onClose} className="p-1 hover:bg-gray-100 rounded">
+              <X className="w-5 h-5 text-gray-500" />
+            </button>
+          </div>
+
+          <div className="px-6 py-4 space-y-6 overflow-y-auto max-h-[calc(90vh-140px)]">
+            {error && (
+              <div className="bg-red-50 border border-red-200 rounded-lg p-3 text-red-700 text-sm">
+                {error}
+              </div>
+            )}
+
+            <div>
+              <label className="block text-sm font-medium text-gray-700 mb-2">Task Role</label>
+              <div className="grid grid-cols-1 gap-2">
+                {TASK_ROLES.map(r => (
+                  <button
+                    key={r.id}
+                    onClick={() => setRole(r.id)}
+                    className={`flex items-start gap-3 p-3 rounded-lg border text-left transition-colors ${
+                      role === r.id
+                        ? 'border-emerald-500 bg-emerald-50'
+                        : 'border-gray-200 hover:border-gray-300'
+                    }`}
+                  >
+                    <div className={`w-4 h-4 rounded-full border-2 mt-0.5 flex-shrink-0 ${
+                      role === r.id ? 'border-emerald-500 bg-emerald-500' : 'border-gray-300'
+                    }`}>
+                      {role === r.id && (
+                        <div className="w-full h-full flex items-center justify-center">
+                          <div className="w-1.5 h-1.5 bg-white rounded-full" />
+                        </div>
+                      )}
+                    </div>
+                    <div>
+                      <p className="font-medium text-gray-900">{r.name}</p>
+                      <p className="text-xs text-gray-500">{r.description}</p>
+                    </div>
+                  </button>
+                ))}
+              </div>
+            </div>
+
+            {needsStore && (
+              <div>
+                <label className="block text-sm font-medium text-gray-700 mb-2">
+                  Select Stores ({selectedStores.length} selected)
+                </label>
+                <div className="border border-gray-200 rounded-lg overflow-hidden">
+                  <div className="p-2 border-b border-gray-200 bg-gray-50">
+                    <div className="relative">
+                      <Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
+                      <input
+                        type="text"
+                        value={storeSearch}
+                        onChange={(e) => setStoreSearch(e.target.value)}
+                        placeholder="Search stores..."
+                        className="w-full pl-9 pr-3 py-2 text-sm border border-gray-200 rounded"
+                      />
+                    </div>
+                    <div className="flex gap-2 mt-2">
+                      <button onClick={selectAll} className="text-xs text-emerald-600 hover:underline">
+                        Select all ({filteredStores.length})
+                      </button>
+                      <span className="text-gray-300">|</span>
+                      <button onClick={clearAll} className="text-xs text-gray-500 hover:underline">
+                        Clear
+                      </button>
+                    </div>
+                  </div>
+                  <div className="max-h-48 overflow-y-auto">
+                    {storesLoading ? (
+                      <div className="p-4 text-center text-gray-500">
+                        <RefreshCw className="w-5 h-5 animate-spin mx-auto mb-1" />
+                        Loading stores...
+                      </div>
+                    ) : filteredStores.length === 0 ? (
+                      <div className="p-4 text-center text-gray-500">No stores found</div>
+                    ) : (
+                      filteredStores.map(store => (
+                        <label key={store.id} className="flex items-center gap-3 px-3 py-2 hover:bg-gray-50 cursor-pointer">
+                          <input
+                            type="checkbox"
+                            checked={!!selectedStores.find(s => s.id === store.id)}
+                            onChange={() => toggleStore(store)}
+                            className="w-4 h-4 text-emerald-600 rounded"
+                          />
+                          <div className="flex-1 min-w-0">
+                            <p className="text-sm text-gray-900 truncate">{store.name}</p>
+                            <p className="text-xs text-gray-500">{store.state_code}</p>
+                          </div>
+                          {!store.crawl_enabled && (
+                            <span className="text-xs text-orange-600 bg-orange-50 px-1.5 py-0.5 rounded">disabled</span>
+                          )}
+                        </label>
+                      ))
+                    )}
+                  </div>
+                </div>
+              </div>
+            )}
+
+            <div>
+              <label className="block text-sm font-medium text-gray-700 mb-2">Priority: {priority}</label>
+              <input
+                type="range"
+                min="0"
+                max="100"
+                value={priority}
+                onChange={(e) => setPriority(parseInt(e.target.value))}
+                className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer"
+              />
+              <div className="flex justify-between text-xs text-gray-500 mt-1">
+                <span>0 (Low)</span>
+                <span>10 (Normal)</span>
+                <span>50 (High)</span>
+                <span>100 (Urgent)</span>
+              </div>
+            </div>
+
+            <div>
+              <label className="block text-sm font-medium text-gray-700 mb-2">Schedule</label>
+              <div className="flex gap-4">
+                <label className="flex items-center gap-2 cursor-pointer">
+                  <input
+                    type="radio"
+                    name="schedule"
+                    checked={scheduleType === 'now'}
+                    onChange={() => setScheduleType('now')}
+                    className="w-4 h-4 text-emerald-600"
+                  />
+                  <span className="text-sm text-gray-700">Run immediately</span>
+                </label>
+                <label className="flex items-center gap-2 cursor-pointer">
+                  <input
+                    type="radio"
+                    name="schedule"
+                    checked={scheduleType === 'scheduled'}
+                    onChange={() => setScheduleType('scheduled')}
+                    className="w-4 h-4 text-emerald-600"
+                  />
+                  <span className="text-sm text-gray-700">Schedule for later</span>
+                </label>
+              </div>
+              {scheduleType === 'scheduled' && (
+                <div className="mt-3 relative">
+                  <Calendar className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
+                  <input
+                    type="datetime-local"
+                    value={scheduledFor}
+                    onChange={(e) => setScheduledFor(e.target.value)}
+                    className="w-full pl-9 pr-3 py-2 text-sm border border-gray-200 rounded"
+                  />
+                </div>
+              )}
+            </div>
+          </div>
+
+          <div className="px-6 py-4 border-t border-gray-200 bg-gray-50 flex items-center justify-between">
+            <div className="text-sm text-gray-500">
+              {needsStore ? (
+                selectedStores.length > 0 ? `Will create ${selectedStores.length} task${selectedStores.length > 1 ? 's' : ''}` : 'Select stores to create tasks'
+              ) : 'Will create 1 task'}
+            </div>
+            <div className="flex gap-3">
+              <button onClick={onClose} className="px-4 py-2 text-sm text-gray-700 hover:bg-gray-100 rounded-lg">
+                Cancel
+              </button>
+              <button
+                onClick={handleSubmit}
+                disabled={loading || (needsStore && selectedStores.length === 0)}
+                className="px-4 py-2 text-sm bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2"
+              >
+                {loading && <RefreshCw className="w-4 h-4 animate-spin" />}
+                Create Task{selectedStores.length > 1 ? 's' : ''}
+              </button>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+
 const ROLES = [
   'store_discovery',
   'entry_point_discovery',
@@ -138,7 +450,11 @@ export default function TasksDashboard() {
   const [loading, setLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
   const [poolPaused, setPoolPaused] = useState(false);
-  const [poolLoading, setPoolLoading] = useState(false);
+  const [showCreateModal, setShowCreateModal] = useState(false);
+
+  // Pagination
+  const [page, setPage] = useState(0);
+  const tasksPerPage = 25;
 
   // Filters
   const [roleFilter, setRoleFilter] = useState<string>('');
@@ -172,20 +488,14 @@ export default function TasksDashboard() {
     }
   };
 
-  const togglePool = async () => {
+  const handleDeleteTask = async (taskId: number) => {
-    setPoolLoading(true);
+    if (!confirm('Delete this task?')) return;
     try {
-      if (poolPaused) {
+      await api.delete(`/api/tasks/${taskId}`);
-        await api.resumeTaskPool();
+      fetchData();
-        setPoolPaused(false);
-      } else {
-        await api.pauseTaskPool();
-        setPoolPaused(true);
-      }
     } catch (err: any) {
-      setError(err.message || 'Failed to toggle pool');
+      console.error('Delete error:', err);
-    } finally {
+      alert(err.response?.data?.error || 'Failed to delete task');
-      setPoolLoading(false);
     }
   };
 
@@ -208,6 +518,10 @@ export default function TasksDashboard() {
     return true;
   });
 
+  // Pagination
+  const paginatedTasks = filteredTasks.slice(page * tasksPerPage, (page + 1) * tasksPerPage);
+  const totalPages = Math.ceil(filteredTasks.length / tasksPerPage);
+
   const totalActive = (counts?.claimed || 0) + (counts?.running || 0);
   const totalPending = counts?.pending || 0;
 
@@ -238,28 +552,21 @@ export default function TasksDashboard() {
             </div>
 
             <div className="flex items-center gap-4">
-              {/* Pool Toggle */}
+              {/* Create Task Button */}
               <button
-              onClick={togglePool}
+                onClick={() => setShowCreateModal(true)}
-              disabled={poolLoading}
+                className="flex items-center gap-2 px-4 py-2 bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 transition-colors"
-              className={`flex items-center gap-2 px-4 py-2 rounded-lg font-medium transition-colors ${
-                poolPaused
-                  ? 'bg-emerald-100 text-emerald-700 hover:bg-emerald-200'
-                  : 'bg-red-100 text-red-700 hover:bg-red-200'
-              }`}
               >
-              {poolPaused ? (
+                <Plus className="w-4 h-4" />
-                <>
+                Create Task
-                  <Play className={`w-5 h-5 ${poolLoading ? 'animate-pulse' : ''}`} />
-                  Start Pool
-                </>
-              ) : (
-                <>
-                  <Square className={`w-5 h-5 ${poolLoading ? 'animate-pulse' : ''}`} />
-                  Stop Pool
-                </>
-              )}
               </button>
+              {/* Pool status indicator */}
+              {poolPaused && (
+                <span className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-sm font-medium bg-yellow-100 text-yellow-800">
+                  <Square className="w-4 h-4" />
+                  Pool Paused
+                </span>
+              )}
               <span className="text-sm text-gray-400">Auto-refreshes every 15s</span>
             </div>
           </div>
@@ -269,6 +576,13 @@ export default function TasksDashboard() {
           <div className="p-4 bg-red-50 text-red-700 rounded-lg">{error}</div>
         )}
 
+        {/* Create Task Modal */}
+        <CreateTaskModal
+          isOpen={showCreateModal}
+          onClose={() => setShowCreateModal(false)}
+          onTaskCreated={fetchData}
+        />
+
         {/* Status Summary Cards */}
         <div className="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-4">
           {Object.entries(counts || {}).map(([status, count]) => (
@@ -471,17 +785,19 @@ export default function TasksDashboard() {
                   <th className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">
                     Error
                   </th>
+                  <th className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase w-16">
+                  </th>
                 </tr>
               </thead>
               <tbody className="divide-y divide-gray-200">
-                {filteredTasks.length === 0 ? (
+                {paginatedTasks.length === 0 ? (
                   <tr>
-                    <td colSpan={8} className="px-4 py-8 text-center text-gray-500">
+                    <td colSpan={9} className="px-4 py-8 text-center text-gray-500">
                       No tasks found
                     </td>
                   </tr>
                 ) : (
-                  filteredTasks.map((task) => (
+                  paginatedTasks.map((task) => (
                     <tr key={task.id} className="hover:bg-gray-50">
                       <td className="px-4 py-3 text-sm font-mono text-gray-600">#{task.id}</td>
                       <td className="px-4 py-3 text-sm text-gray-900">
@@ -512,12 +828,47 @@ export default function TasksDashboard() {
                       <td className="px-4 py-3 text-sm text-red-600 max-w-xs truncate">
                         {task.error_message || '-'}
                       </td>
+                      <td className="px-4 py-3">
+                        {(task.status === 'failed' || task.status === 'completed' || task.status === 'pending') && (
+                          <button
+                            onClick={() => handleDeleteTask(task.id)}
+                            className="p-1 text-gray-400 hover:text-red-500 hover:bg-red-50 rounded transition-colors"
+                            title="Delete task"
+                          >
+                            <Trash2 className="w-4 h-4" />
+                          </button>
+                        )}
+                      </td>
                     </tr>
                   ))
                 )}
               </tbody>
             </table>
           </div>
+
+          {/* Pagination */}
+          <div className="px-4 py-3 border-t border-gray-200 bg-gray-50 flex items-center justify-between">
+            <div className="text-sm text-gray-500">
+              Showing {page * tasksPerPage + 1} - {Math.min((page + 1) * tasksPerPage, filteredTasks.length)} of {filteredTasks.length} tasks
+            </div>
+            <div className="flex items-center gap-2">
+              <button
+                onClick={() => setPage(p => Math.max(0, p - 1))}
+                disabled={page === 0}
+                className="px-3 py-1 text-sm border border-gray-200 rounded hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed"
+              >
+                <ChevronLeft className="w-4 h-4" />
+              </button>
+              <span className="text-sm text-gray-600">Page {page + 1} of {totalPages || 1}</span>
+              <button
+                onClick={() => setPage(p => p + 1)}
+                disabled={page >= totalPages - 1}
+                className="px-3 py-1 text-sm border border-gray-200 rounded hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed"
+              >
+                <ChevronRight className="w-4 h-4" />
+              </button>
+            </div>
+          </div>
         </div>
       </div>
     </Layout>

cannaiq/src/pages/WorkersDashboard.tsx

@@ -369,8 +369,10 @@ function PodVisualization({
 
           const isBusy = worker.current_task_id !== null;
           const isDecommissioning = worker.decommission_requested;
-          const workerColor = isDecommissioning ? 'bg-orange-500' : isBusy ? 'bg-blue-500' : 'bg-emerald-500';
+          const isBackingOff = worker.metadata?.is_backing_off;
-          const workerBorder = isDecommissioning ? 'border-orange-300' : isBusy ? 'border-blue-300' : 'border-emerald-300';
+          // Color priority: decommissioning > backing off > busy > idle
+          const workerColor = isDecommissioning ? 'bg-orange-500' : isBackingOff ? 'bg-yellow-500' : isBusy ? 'bg-blue-500' : 'bg-emerald-500';
+          const workerBorder = isDecommissioning ? 'border-orange-300' : isBackingOff ? 'border-yellow-300' : isBusy ? 'border-blue-300' : 'border-emerald-300';
 
           // Line from center to worker
           const lineLength = radius - 10;
@@ -381,7 +383,7 @@ function PodVisualization({
             <div key={worker.id}>
               {/* Connection line */}
               <div
-                className={`absolute w-0.5 ${isDecommissioning ? 'bg-orange-300' : isBusy ? 'bg-blue-300' : 'bg-emerald-300'}`}
+                className={`absolute w-0.5 ${isDecommissioning ? 'bg-orange-300' : isBackingOff ? 'bg-yellow-300' : isBusy ? 'bg-blue-300' : 'bg-emerald-300'}`}
                 style={{
                   height: `${lineLength}px`,
                   left: '50%',
@@ -398,7 +400,7 @@ function PodVisualization({
                   top: '50%',
                   transform: `translate(-50%, -50%) translate(${x}px, ${y}px)`,
                 }}
-                title={`${worker.friendly_name}\nStatus: ${isDecommissioning ? 'Stopping after current task' : isBusy ? `Working on task #${worker.current_task_id}` : 'Idle - waiting for tasks'}\nMemory: ${worker.metadata?.memory_mb || 0} MB\nCPU: ${formatCpuTime(worker.metadata?.cpu_user_ms || 0)} user, ${formatCpuTime(worker.metadata?.cpu_system_ms || 0)} sys\nCompleted: ${worker.tasks_completed} | Failed: ${worker.tasks_failed}\nLast heartbeat: ${new Date(worker.last_heartbeat_at).toLocaleTimeString()}`}
+                title={`${worker.friendly_name}\nStatus: ${isDecommissioning ? 'Stopping after current task' : isBackingOff ? `Backing off: ${worker.metadata?.backoff_reason || 'resource pressure'}` : isBusy ? `Working on task #${worker.current_task_id}` : 'Ready - waiting for tasks'}\nMemory: ${worker.metadata?.memory_mb || 0} MB (${worker.metadata?.memory_percent || 0}%)\nCPU: ${formatCpuTime(worker.metadata?.cpu_user_ms || 0)} user, ${formatCpuTime(worker.metadata?.cpu_system_ms || 0)} sys\nCompleted: ${worker.tasks_completed} | Failed: ${worker.tasks_failed}\nLast heartbeat: ${new Date(worker.last_heartbeat_at).toLocaleTimeString()}`}
               >
                 {index + 1}
               </div>
@@ -700,11 +702,11 @@ export function WorkersDashboard() {
                   Worker Pods ({Array.from(groupWorkersByPod(workers)).length} pods, {activeWorkers.length} workers)
                 </h3>
                 <p className="text-xs text-gray-500 mt-0.5">
-                  <span className="inline-flex items-center gap-1"><span className="w-2 h-2 rounded-full bg-emerald-500"></span> idle</span>
+                  <span className="inline-flex items-center gap-1"><span className="w-2 h-2 rounded-full bg-emerald-500"></span> ready</span>
                   <span className="mx-2">|</span>
                   <span className="inline-flex items-center gap-1"><span className="w-2 h-2 rounded-full bg-blue-500"></span> busy</span>
                   <span className="mx-2">|</span>
-                  <span className="inline-flex items-center gap-1"><span className="w-2 h-2 rounded-full bg-yellow-500"></span> mixed</span>
+                  <span className="inline-flex items-center gap-1"><span className="w-2 h-2 rounded-full bg-yellow-500"></span> backing off</span>
                   <span className="mx-2">|</span>
                   <span className="inline-flex items-center gap-1"><span className="w-2 h-2 rounded-full bg-orange-500"></span> stopping</span>
                 </p>

k8s/scraper-worker.yaml

@@ -40,12 +40,16 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: API_BASE_URL
+              value: "http://scraper"
+            - name: NODE_OPTIONS
+              value: "--max-old-space-size=1500"
           resources:
             requests:
-              memory: "256Mi"
+              memory: "1Gi"
               cpu: "100m"
             limits:
-              memory: "512Mi"
+              memory: "2Gi"
               cpu: "500m"
           livenessProbe:
             exec:

k8s/woodpecker-agent.yaml

@@ -6,6 +6,19 @@ kind: Namespace
 metadata:
   name: woodpecker
 ---
+# PVC for npm cache - shared across CI jobs
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: npm-cache
+  namespace: woodpecker
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi
+---
 apiVersion: v1
 kind: Secret
 metadata:
@@ -52,6 +65,9 @@ spec:
               value: "woodpecker"
             - name: WOODPECKER_BACKEND_K8S_VOLUME_SIZE
               value: "10G"
+            # Allow CI steps to mount the npm-cache PVC
+            - name: WOODPECKER_BACKEND_K8S_VOLUMES
+              value: "npm-cache:/npm-cache"
           resources:
             limits:
               memory: "512Mi"