feat: SEO template library, discovery pipeline, and orchestrator enhancements
## SEO Template Library - Add complete template library with 7 page types (state, city, category, brand, product, search, regeneration) - Add Template Library tab in SEO Orchestrator with accordion-based editors - Add template preview, validation, and variable injection engine - Add API endpoints: /api/seo/templates, preview, validate, generate, regenerate ## Discovery Pipeline - Add promotion.ts for discovery location validation and promotion - Add discover-all-states.ts script for multi-state discovery - Add promotion log migration (067) - Enhance discovery routes and types ## Orchestrator & Admin - Add crawl_enabled filter to stores page - Add API permissions page - Add job queue management - Add price analytics routes - Add markets and intelligence routes - Enhance dashboard and worker monitoring ## Infrastructure - Add migrations for worker definitions, SEO settings, field alignment - Add canonical pipeline for scraper v2 - Update hydration and sync orchestrator - Enhance multi-state query service 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,14 +3,23 @@
|
||||
*
|
||||
* Main orchestrator for the Dutchie store discovery pipeline.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Discover cities from Dutchie (or use seeded cities)
|
||||
* 2. For each city, discover store locations
|
||||
* 3. Upsert all data to discovery tables
|
||||
* 4. Admin verifies locations manually
|
||||
* 5. Verified locations are promoted to canonical dispensaries
|
||||
* AUTOMATED FLOW (as of 2025-01):
|
||||
* 1. Fetch cities dynamically from Dutchie GraphQL (getAllCitiesByState)
|
||||
* 2. For each city, discover store locations via ConsumerDispensaries query
|
||||
* 3. Upsert locations to dutchie_discovery_locations (keyed by platform_location_id)
|
||||
* 4. AUTO-VALIDATE: Check required fields (name, city, state, platform_menu_url, platform_location_id)
|
||||
* 5. AUTO-PROMOTE: Valid locations are upserted to dispensaries table with crawl_enabled=true
|
||||
* 6. All actions logged to dutchie_promotion_log for audit
|
||||
*
|
||||
* This module does NOT create canonical dispensaries automatically.
|
||||
* Tables involved:
|
||||
* - dutchie_discovery_cities: Known cities for each state
|
||||
* - dutchie_discovery_locations: Raw discovered store data
|
||||
* - dispensaries: Canonical store records (promoted from discovery)
|
||||
* - dutchie_promotion_log: Audit trail for validation/promotion
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx src/scripts/run-discovery.ts discover:state AZ
|
||||
* npx tsx src/scripts/run-discovery.ts discover:state CA
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
@@ -24,11 +33,12 @@ import {
|
||||
getCitiesToCrawl,
|
||||
getCityBySlug,
|
||||
seedKnownCities,
|
||||
ARIZONA_CITIES,
|
||||
} from './city-discovery';
|
||||
import {
|
||||
discoverLocationsForCity,
|
||||
getCitiesForState,
|
||||
} from './location-discovery';
|
||||
import { promoteDiscoveredLocations } from './promotion';
|
||||
|
||||
// ============================================================
|
||||
// FULL DISCOVERY
|
||||
@@ -162,6 +172,25 @@ export async function runFullDiscovery(
|
||||
console.log(`Errors: ${totalErrors}`);
|
||||
}
|
||||
|
||||
// Step 4: Auto-validate and promote discovered locations
|
||||
if (!dryRun && totalLocationsUpserted > 0) {
|
||||
console.log('\n[Discovery] Step 4: Auto-promoting discovered locations...');
|
||||
const promotionResult = await promoteDiscoveredLocations(stateCode, false);
|
||||
console.log(`[Discovery] Promotion complete:`);
|
||||
console.log(` Created: ${promotionResult.created} new dispensaries`);
|
||||
console.log(` Updated: ${promotionResult.updated} existing dispensaries`);
|
||||
console.log(` Rejected: ${promotionResult.rejected} (validation failed)`);
|
||||
if (promotionResult.rejectedRecords.length > 0) {
|
||||
console.log(` Rejection reasons:`);
|
||||
promotionResult.rejectedRecords.slice(0, 5).forEach(r => {
|
||||
console.log(` - ${r.name}: ${r.errors.join(', ')}`);
|
||||
});
|
||||
if (promotionResult.rejectedRecords.length > 5) {
|
||||
console.log(` ... and ${promotionResult.rejectedRecords.length - 5} more`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
cities: cityResult,
|
||||
locations: locationResults,
|
||||
@@ -235,11 +264,19 @@ export async function discoverState(
|
||||
|
||||
console.log(`[Discovery] Discovering state: ${stateCode}`);
|
||||
|
||||
// Seed known cities for this state
|
||||
if (stateCode === 'AZ') {
|
||||
console.log('[Discovery] Seeding Arizona cities...');
|
||||
const seeded = await seedKnownCities(pool, ARIZONA_CITIES);
|
||||
console.log(`[Discovery] Seeded ${seeded.created} new cities, ${seeded.updated} updated`);
|
||||
// Dynamically fetch and seed cities for this state
|
||||
console.log(`[Discovery] Fetching cities for ${stateCode} from Dutchie...`);
|
||||
const cityNames = await getCitiesForState(stateCode);
|
||||
if (cityNames.length > 0) {
|
||||
const cities = cityNames.map(name => ({
|
||||
name,
|
||||
slug: name.toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, ''),
|
||||
stateCode,
|
||||
}));
|
||||
const seeded = await seedKnownCities(pool, cities);
|
||||
console.log(`[Discovery] Seeded ${seeded.created} new cities, ${seeded.updated} updated for ${stateCode}`);
|
||||
} else {
|
||||
console.log(`[Discovery] No cities found for ${stateCode}`);
|
||||
}
|
||||
|
||||
// Run full discovery for this state
|
||||
|
||||
Reference in New Issue
Block a user