feat: AZ dispensary harmonization with Dutchie source of truth

Major changes: - Add harmonize-az-dispensaries.ts script to sync dispensaries with Dutchie API - Add migration 057 for crawl_enabled and dutchie_verified fields - Remove legacy dutchie-az module (replaced by platforms/dutchie) - Clean up deprecated crawlers, scrapers, and orchestrator code - Update location-discovery to not fallback to slug when ID is missing - Add crawl-rotator service for proxy rotation - Add types/index.ts for shared type definitions - Add woodpecker-agent k8s manifest Harmonization script: - Queries ConsumerDispensaries API for all 32 AZ cities - Matches dispensaries by platform_dispensary_id (not slug) - Updates existing records with full Dutchie data - Creates new records for unmatched Dutchie dispensaries - Disables dispensaries not found in Dutchie 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-08 10:19:49 -07:00
parent 948a732dd5
commit b7cfec0770
112 changed files with 3163 additions and 34694 deletions
--- a/backend/src/discovery/location-discovery.ts
+++ b/backend/src/discovery/location-discovery.ts
@@ -26,13 +26,377 @@ import {
  mapLocationRowToLocation,
 } from './types';
 import { DiscoveryCity } from './types';
+import {
+  executeGraphQL,
+  fetchPage,
+  extractNextData,
+  GRAPHQL_HASHES,
+  setProxy,
+} from '../platforms/dutchie/client';
+import { getStateProxy, getRandomProxy } from '../utils/proxyManager';

 puppeteer.use(StealthPlugin());

+// ============================================================
+// PROXY INITIALIZATION
+// ============================================================
+// Call initDiscoveryProxy() before any discovery operations to
+// set up proxy if USE_PROXY=true environment variable is set.
+// This is opt-in and does NOT break existing behavior.
+// ============================================================
+
+let proxyInitialized = false;
+
+/**
+ * Initialize proxy for discovery operations
+ * Only runs if USE_PROXY=true is set in environment
+ * Safe to call multiple times - only initializes once
+ *
+ * @param stateCode - Optional state code for state-specific proxy (e.g., 'AZ', 'CA')
+ * @returns true if proxy was set, false if skipped or failed
+ */
+export async function initDiscoveryProxy(stateCode?: string): Promise<boolean> {
+  // Skip if already initialized
+  if (proxyInitialized) {
+    return true;
+  }
+
+  // Skip if USE_PROXY is not enabled
+  if (process.env.USE_PROXY !== 'true') {
+    console.log('[LocationDiscovery] Proxy disabled (USE_PROXY != true)');
+    return false;
+  }
+
+  try {
+    // Get proxy - prefer state-specific if state code provided
+    const proxyConfig = stateCode
+      ? await getStateProxy(stateCode)
+      : await getRandomProxy();
+
+    if (!proxyConfig) {
+      console.warn('[LocationDiscovery] No proxy available, proceeding without proxy');
+      return false;
+    }
+
+    // Build proxy URL with auth if needed
+    let proxyUrl = proxyConfig.server;
+    if (proxyConfig.username && proxyConfig.password) {
+      const url = new URL(proxyConfig.server);
+      url.username = proxyConfig.username;
+      url.password = proxyConfig.password;
+      proxyUrl = url.toString();
+    }
+
+    // Set proxy on the Dutchie client
+    setProxy(proxyUrl);
+    proxyInitialized = true;
+
+    console.log(`[LocationDiscovery] Proxy initialized for ${stateCode || 'general'} discovery`);
+    return true;
+  } catch (error: any) {
+    console.error(`[LocationDiscovery] Failed to initialize proxy: ${error.message}`);
+    return false;
+  }
+}
+
+/**
+ * Reset proxy initialization flag (for testing or re-initialization)
+ */
+export function resetProxyInit(): void {
+  proxyInitialized = false;
+  setProxy(null);
+}
+
 const PLATFORM = 'dutchie';

 // ============================================================
-// GRAPHQL / API FETCHING
+// CITY-BASED DISCOVERY (CANONICAL SOURCE OF TRUTH)
+// ============================================================
+// GraphQL with city+state filter is the SOURCE OF TRUTH for database data.
+//
+// Method:
+// 1. Get city list from statesWithDispensaries (in __NEXT_DATA__)
+// 2. Query stores per city using city + state GraphQL filter
+// 3. This gives us complete, accurate dispensary data
+//
+// Geo-coordinate queries (nearLat/nearLng) are ONLY for showing search
+// results to users (e.g., "stores within 20 miles of me").
+// They are NOT a source of truth for establishing database records.
+// ============================================================
+
+/**
+ * State with dispensary cities from Dutchie's statesWithDispensaries data
+ */
+export interface StateWithCities {
+  name: string;      // State code (e.g., "CA", "AZ")
+  country: string;   // Country code (e.g., "US")
+  cities: string[];  // Array of city names
+}
+
+/**
+ * Fetch all states with their cities from Dutchie's __NEXT_DATA__
+ *
+ * This fetches a city page and extracts the statesWithDispensaries data
+ * which contains all states and their cities where Dutchie has dispensaries.
+ */
+export async function fetchStatesWithDispensaries(
+  options: { verbose?: boolean } = {}
+): Promise<StateWithCities[]> {
+  const { verbose = false } = options;
+
+  // Initialize proxy if USE_PROXY=true
+  await initDiscoveryProxy();
+
+  console.log('[LocationDiscovery] Fetching statesWithDispensaries from Dutchie...');
+
+  // Fetch any city page to get the __NEXT_DATA__ with statesWithDispensaries
+  // Using a known city that's likely to exist
+  const result = await fetchPage('/dispensaries/az/phoenix', { maxRetries: 3 });
+
+  if (!result || result.status !== 200) {
+    console.error('[LocationDiscovery] Failed to fetch city page');
+    return [];
+  }
+
+  const nextData = extractNextData(result.html);
+  if (!nextData) {
+    console.error('[LocationDiscovery] No __NEXT_DATA__ found');
+    return [];
+  }
+
+  // Extract statesWithDispensaries from Apollo state
+  const apolloState = nextData.props?.pageProps?.initialApolloState;
+  if (!apolloState) {
+    console.error('[LocationDiscovery] No initialApolloState found');
+    return [];
+  }
+
+  // Find ROOT_QUERY.statesWithDispensaries
+  const rootQuery = apolloState['ROOT_QUERY'];
+  if (!rootQuery) {
+    console.error('[LocationDiscovery] No ROOT_QUERY found');
+    return [];
+  }
+
+  // The statesWithDispensaries is at ROOT_QUERY.statesWithDispensaries
+  const statesRefs = rootQuery.statesWithDispensaries;
+  if (!Array.isArray(statesRefs)) {
+    console.error('[LocationDiscovery] statesWithDispensaries not found or not an array');
+    return [];
+  }
+
+  // Resolve the references to actual state data
+  const states: StateWithCities[] = [];
+  for (const ref of statesRefs) {
+    // ref might be { __ref: "StateWithDispensaries:0" } or direct object
+    let stateData: any;
+
+    if (ref && ref.__ref) {
+      stateData = apolloState[ref.__ref];
+    } else {
+      stateData = ref;
+    }
+
+    if (stateData && stateData.name) {
+      // Parse cities JSON array if it's a string
+      let cities = stateData.cities;
+      if (typeof cities === 'string') {
+        try {
+          cities = JSON.parse(cities);
+        } catch {
+          cities = [];
+        }
+      }
+
+      states.push({
+        name: stateData.name,
+        country: stateData.country || 'US',
+        cities: Array.isArray(cities) ? cities : [],
+      });
+    }
+  }
+
+  if (verbose) {
+    console.log(`[LocationDiscovery] Found ${states.length} states`);
+    for (const state of states) {
+      console.log(`  ${state.name}: ${state.cities.length} cities`);
+    }
+  }
+
+  console.log(`[LocationDiscovery] Loaded ${states.length} states with cities`);
+  return states;
+}
+
+/**
+ * Get cities for a specific state
+ */
+export async function getCitiesForState(
+  stateCode: string,
+  options: { verbose?: boolean } = {}
+): Promise<string[]> {
+  const states = await fetchStatesWithDispensaries(options);
+  const state = states.find(s => s.name.toUpperCase() === stateCode.toUpperCase());
+
+  if (!state) {
+    console.warn(`[LocationDiscovery] No cities found for state: ${stateCode}`);
+    return [];
+  }
+
+  console.log(`[LocationDiscovery] Found ${state.cities.length} cities for ${stateCode}`);
+  return state.cities;
+}
+
+/**
+ * Fetch dispensaries for a specific city+state using GraphQL
+ *
+ * This is the CORRECT method for establishing database data:
+ * Uses city + state filter, NOT geo-coordinates.
+ */
+export async function fetchDispensariesByCityState(
+  city: string,
+  stateCode: string,
+  options: { verbose?: boolean; perPage?: number; maxPages?: number } = {}
+): Promise<DutchieLocationResponse[]> {
+  const { verbose = false, perPage = 200, maxPages = 10 } = options;
+
+  // Initialize proxy if USE_PROXY=true (state-specific proxy preferred)
+  await initDiscoveryProxy(stateCode);
+
+  console.log(`[LocationDiscovery] Fetching dispensaries for ${city}, ${stateCode}...`);
+
+  const allDispensaries: any[] = [];
+  let page = 0;
+  let hasMore = true;
+
+  while (hasMore && page < maxPages) {
+    const variables = {
+      dispensaryFilter: {
+        activeOnly: true,
+        city: city,
+        state: stateCode,
+      },
+      page,
+      perPage,
+    };
+
+    try {
+      const result = await executeGraphQL(
+        'ConsumerDispensaries',
+        variables,
+        GRAPHQL_HASHES.ConsumerDispensaries,
+        { cName: `${city.toLowerCase().replace(/\s+/g, '-')}-${stateCode.toLowerCase()}`, maxRetries: 2, retryOn403: true }
+      );
+
+      const dispensaries = result?.data?.filteredDispensaries || [];
+
+      if (verbose) {
+        console.log(`[LocationDiscovery] Page ${page}: ${dispensaries.length} dispensaries`);
+      }
+
+      if (dispensaries.length === 0) {
+        hasMore = false;
+      } else {
+        // Filter to ensure we only get dispensaries in the correct state
+        const stateFiltered = dispensaries.filter((d: any) =>
+          d.location?.state?.toUpperCase() === stateCode.toUpperCase()
+        );
+        allDispensaries.push(...stateFiltered);
+
+        if (dispensaries.length < perPage) {
+          hasMore = false;
+        } else {
+          page++;
+        }
+      }
+    } catch (error: any) {
+      console.error(`[LocationDiscovery] Error fetching page ${page}: ${error.message}`);
+      hasMore = false;
+    }
+  }
+
+  // Dedupe by ID
+  const uniqueMap = new Map<string, any>();
+  for (const d of allDispensaries) {
+    const id = d.id || d._id;
+    if (id && !uniqueMap.has(id)) {
+      uniqueMap.set(id, d);
+    }
+  }
+
+  const unique = Array.from(uniqueMap.values());
+  console.log(`[LocationDiscovery] Found ${unique.length} unique dispensaries in ${city}, ${stateCode}`);
+
+  return unique.map(d => normalizeLocationResponse(d));
+}
+
+/**
+ * Fetch ALL dispensaries for a state by querying each city
+ *
+ * This is the canonical method for establishing state data:
+ * 1. Get city list from statesWithDispensaries
+ * 2. Query each city using city+state filter
+ * 3. Dedupe and return all dispensaries
+ */
+export async function fetchAllDispensariesForState(
+  stateCode: string,
+  options: { verbose?: boolean; progressCallback?: (city: string, count: number, total: number) => void } = {}
+): Promise<{ dispensaries: DutchieLocationResponse[]; citiesQueried: number; citiesWithResults: number }> {
+  const { verbose = false, progressCallback } = options;
+
+  console.log(`[LocationDiscovery] Fetching all dispensaries for ${stateCode}...`);
+
+  // Step 1: Get city list
+  const cities = await getCitiesForState(stateCode, { verbose });
+  if (cities.length === 0) {
+    console.warn(`[LocationDiscovery] No cities found for ${stateCode}`);
+    return { dispensaries: [], citiesQueried: 0, citiesWithResults: 0 };
+  }
+
+  console.log(`[LocationDiscovery] Will query ${cities.length} cities for ${stateCode}`);
+
+  // Step 2: Query each city
+  const allDispensaries = new Map<string, DutchieLocationResponse>();
+  let citiesWithResults = 0;
+
+  for (let i = 0; i < cities.length; i++) {
+    const city = cities[i];
+
+    if (progressCallback) {
+      progressCallback(city, i + 1, cities.length);
+    }
+
+    try {
+      const dispensaries = await fetchDispensariesByCityState(city, stateCode, { verbose });
+
+      if (dispensaries.length > 0) {
+        citiesWithResults++;
+        for (const d of dispensaries) {
+          const id = d.id || d.slug;
+          if (id && !allDispensaries.has(id)) {
+            allDispensaries.set(id, d);
+          }
+        }
+      }
+
+      // Small delay between cities to avoid rate limiting
+      await new Promise(r => setTimeout(r, 300));
+    } catch (error: any) {
+      console.error(`[LocationDiscovery] Error querying ${city}: ${error.message}`);
+    }
+  }
+
+  const result = Array.from(allDispensaries.values());
+  console.log(`[LocationDiscovery] Total: ${result.length} unique dispensaries across ${citiesWithResults}/${cities.length} cities`);
+
+  return {
+    dispensaries: result,
+    citiesQueried: cities.length,
+    citiesWithResults,
+  };
+}
+
+// ============================================================
+// GRAPHQL / API FETCHING (LEGACY - PUPPETEER-BASED)
 // ============================================================

 interface SessionCredentials {
@@ -91,57 +455,77 @@ async function closeSession(session: SessionCredentials): Promise<void> {
 }

 /**
- * Fetch locations for a city using Dutchie's internal search API.
+ * Fetch locations for a city.
+ *
+ * PRIMARY METHOD: Uses city+state GraphQL filter (source of truth)
+ * FALLBACK: Legacy Puppeteer-based methods for edge cases
 */
 export async function fetchLocationsForCity(
  city: DiscoveryCity,
  options: {
    session?: SessionCredentials;
    verbose?: boolean;
+    useLegacyMethods?: boolean;
  } = {}
 ): Promise<DutchieLocationResponse[]> {
-  const { verbose = false } = options;
-  let session = options.session;
-  let shouldCloseSession = false;
+  const { verbose = false, useLegacyMethods = false } = options;

-  if (!session) {
-    session = await createSession(city.citySlug);
-    shouldCloseSession = true;
-  }
+  console.log(`[LocationDiscovery] Fetching locations for ${city.cityName}, ${city.stateCode}...`);

-  try {
-    console.log(`[LocationDiscovery] Fetching locations for ${city.cityName}, ${city.stateCode}...`);
-
-    // Try multiple approaches to get location data
-
-    // Approach 1: Extract from page __NEXT_DATA__ or similar
-    const locations = await extractLocationsFromPage(session.page, verbose);
-    if (locations.length > 0) {
-      console.log(`[LocationDiscovery] Found ${locations.length} locations from page data`);
-      return locations;
-    }
-
-    // Approach 2: Try the geo-based GraphQL query
-    const geoLocations = await fetchLocationsViaGraphQL(session, city, verbose);
-    if (geoLocations.length > 0) {
-      console.log(`[LocationDiscovery] Found ${geoLocations.length} locations from GraphQL`);
-      return geoLocations;
-    }
-
-    // Approach 3: Scrape visible location cards
-    const scrapedLocations = await scrapeLocationCards(session.page, verbose);
-    if (scrapedLocations.length > 0) {
-      console.log(`[LocationDiscovery] Found ${scrapedLocations.length} locations from scraping`);
-      return scrapedLocations;
-    }
-
-    console.log(`[LocationDiscovery] No locations found for ${city.cityName}`);
-    return [];
-  } finally {
-    if (shouldCloseSession) {
-      await closeSession(session);
+  // PRIMARY METHOD: City+State GraphQL query (SOURCE OF TRUTH)
+  if (city.cityName && city.stateCode) {
+    try {
+      const locations = await fetchDispensariesByCityState(city.cityName, city.stateCode, { verbose });
+      if (locations.length > 0) {
+        console.log(`[LocationDiscovery] Found ${locations.length} locations via GraphQL city+state`);
+        return locations;
+      }
+    } catch (error: any) {
+      console.warn(`[LocationDiscovery] GraphQL city+state failed: ${error.message}`);
    }
  }
+
+  // FALLBACK: Legacy Puppeteer-based methods (only if explicitly enabled)
+  if (useLegacyMethods) {
+    let session = options.session;
+    let shouldCloseSession = false;
+
+    if (!session) {
+      session = await createSession(city.citySlug);
+      shouldCloseSession = true;
+    }
+
+    try {
+      // Legacy Approach 1: Extract from page __NEXT_DATA__
+      const locations = await extractLocationsFromPage(session.page, verbose);
+      if (locations.length > 0) {
+        console.log(`[LocationDiscovery] Found ${locations.length} locations from page data (legacy)`);
+        return locations;
+      }
+
+      // Legacy Approach 2: Try the geo-based GraphQL query
+      // NOTE: Geo queries are for SEARCH RESULTS only, not source of truth
+      const geoLocations = await fetchLocationsViaGraphQL(session, city, verbose);
+      if (geoLocations.length > 0) {
+        console.log(`[LocationDiscovery] Found ${geoLocations.length} locations from geo GraphQL (legacy)`);
+        return geoLocations;
+      }
+
+      // Legacy Approach 3: Scrape visible location cards
+      const scrapedLocations = await scrapeLocationCards(session.page, verbose);
+      if (scrapedLocations.length > 0) {
+        console.log(`[LocationDiscovery] Found ${scrapedLocations.length} locations from scraping (legacy)`);
+        return scrapedLocations;
+      }
+    } finally {
+      if (shouldCloseSession) {
+        await closeSession(session);
+      }
+    }
+  }
+
+  console.log(`[LocationDiscovery] No locations found for ${city.cityName}`);
+  return [];
 }

 /**
@@ -202,33 +586,52 @@ async function extractLocationsFromPage(

 /**
 * Fetch locations via GraphQL geo-based query.
+ *
+ * Uses ConsumerDispensaries with geo filtering:
+ * - dispensaryFilter.nearLat/nearLng for center point
+ * - dispensaryFilter.distance for radius in miles
+ * - Response at data.filteredDispensaries
 */
 async function fetchLocationsViaGraphQL(
  session: SessionCredentials,
  city: DiscoveryCity,
  verbose: boolean
 ): Promise<DutchieLocationResponse[]> {
-  // Use a known center point for the city or default to a central US location
-  const CITY_COORDS: Record<string, { lat: number; lng: number }> = {
-    'phoenix': { lat: 33.4484, lng: -112.074 },
-    'tucson': { lat: 32.2226, lng: -110.9747 },
-    'scottsdale': { lat: 33.4942, lng: -111.9261 },
-    'mesa': { lat: 33.4152, lng: -111.8315 },
-    'tempe': { lat: 33.4255, lng: -111.94 },
-    'flagstaff': { lat: 35.1983, lng: -111.6513 },
-    // Add more as needed
+  // City center coordinates with appropriate radius
+  const CITY_COORDS: Record<string, { lat: number; lng: number; radius: number }> = {
+    'phoenix': { lat: 33.4484, lng: -112.074, radius: 50 },
+    'tucson': { lat: 32.2226, lng: -110.9747, radius: 50 },
+    'scottsdale': { lat: 33.4942, lng: -111.9261, radius: 30 },
+    'mesa': { lat: 33.4152, lng: -111.8315, radius: 30 },
+    'tempe': { lat: 33.4255, lng: -111.94, radius: 30 },
+    'flagstaff': { lat: 35.1983, lng: -111.6513, radius: 50 },
  };

-  const coords = CITY_COORDS[city.citySlug] || { lat: 33.4484, lng: -112.074 };
+  // State-wide coordinates for full coverage
+  const STATE_COORDS: Record<string, { lat: number; lng: number; radius: number }> = {
+    'AZ': { lat: 33.4484, lng: -112.074, radius: 200 },
+    'CA': { lat: 36.7783, lng: -119.4179, radius: 400 },
+    'CO': { lat: 39.5501, lng: -105.7821, radius: 200 },
+    'FL': { lat: 27.6648, lng: -81.5158, radius: 400 },
+    'MI': { lat: 44.3148, lng: -85.6024, radius: 250 },
+    'NV': { lat: 36.1699, lng: -115.1398, radius: 200 },
+  };

+  // Try city-specific coords first, then state-wide, then default
+  const coords = CITY_COORDS[city.citySlug]
+    || (city.stateCode && STATE_COORDS[city.stateCode])
+    || { lat: 33.4484, lng: -112.074, radius: 200 };
+
+  // Correct GraphQL variables for ConsumerDispensaries
  const variables = {
-    dispensariesFilter: {
-      latitude: coords.lat,
-      longitude: coords.lng,
-      distance: 50, // miles
-      state: city.stateCode,
-      city: city.cityName,
+    dispensaryFilter: {
+      activeOnly: true,
+      nearLat: coords.lat,
+      nearLng: coords.lng,
+      distance: coords.radius,
    },
+    page: 0,
+    perPage: 200,
  };

  const hash = '0a5bfa6ca1d64ae47bcccb7c8077c87147cbc4e6982c17ceec97a2a4948b311b';
@@ -263,8 +666,19 @@ async function fetchLocationsViaGraphQL(
      return [];
    }

-    const dispensaries = response.data?.data?.consumerDispensaries || [];
-    return dispensaries.map((d: any) => normalizeLocationResponse(d));
+    // Response is at data.filteredDispensaries
+    const dispensaries = response.data?.data?.filteredDispensaries || [];
+
+    // Filter to specific state if needed (radius may include neighboring states)
+    const filtered = city.stateCode
+      ? dispensaries.filter((d: any) => d.location?.state === city.stateCode)
+      : dispensaries;
+
+    if (verbose) {
+      console.log(`[LocationDiscovery] GraphQL returned ${dispensaries.length} total, ${filtered.length} in ${city.stateCode || 'all states'}`);
+    }
+
+    return filtered.map((d: any) => normalizeLocationResponse(d));
  } catch (error: any) {
    if (verbose) {
      console.log(`[LocationDiscovery] GraphQL error: ${error.message}`);
@@ -373,13 +787,20 @@ function normalizeLocationResponse(raw: any): DutchieLocationResponse {

 /**
 * Upsert a location into dutchie_discovery_locations.
+ * REQUIRES a valid platform ID (MongoDB ObjectId) - will skip records without one.
 */
 export async function upsertLocation(
  pool: Pool,
  location: DutchieLocationResponse,
  cityId: number | null
-): Promise<{ id: number; isNew: boolean }> {
-  const platformLocationId = location.id || location.slug;
+): Promise<{ id: number; isNew: boolean } | null> {
+  // REQUIRE actual platform ID - NO fallback to slug
+  const platformLocationId = location.id;
+  if (!platformLocationId) {
+    console.warn(`[LocationDiscovery] Skipping location without platform ID: ${location.name} (${location.slug})`);
+    return null;
+  }
+
  const menuUrl = location.menuUrl || `https://dutchie.com/dispensary/${location.slug}`;

  const result = await pool.query(
@@ -642,6 +1063,12 @@ export async function discoverLocationsForCity(

      const result = await upsertLocation(pool, location, city.id);

+      // Skip locations without valid platform ID
+      if (!result) {
+        errors.push(`Location ${location.slug}: No valid platform ID - skipped`);
+        continue;
+      }
+
      if (result.isNew) {
        newCount++;
      } else {