Major changes: - Add harmonize-az-dispensaries.ts script to sync dispensaries with Dutchie API - Add migration 057 for crawl_enabled and dutchie_verified fields - Remove legacy dutchie-az module (replaced by platforms/dutchie) - Clean up deprecated crawlers, scrapers, and orchestrator code - Update location-discovery to not fallback to slug when ID is missing - Add crawl-rotator service for proxy rotation - Add types/index.ts for shared type definitions - Add woodpecker-agent k8s manifest Harmonization script: - Queries ConsumerDispensaries API for all 32 AZ cities - Matches dispensaries by platform_dispensary_id (not slug) - Updates existing records with full Dutchie data - Creates new records for unmatched Dutchie dispensaries - Disables dispensaries not found in Dutchie 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
617 lines
17 KiB
TypeScript
617 lines
17 KiB
TypeScript
import { Router } from 'express';
|
|
import { authMiddleware } from '../auth/middleware';
|
|
import { pool } from '../db/pool';
|
|
|
|
const router = Router();
|
|
router.use(authMiddleware);
|
|
|
|
// Valid menu_type values
|
|
const VALID_MENU_TYPES = ['dutchie', 'treez', 'jane', 'weedmaps', 'leafly', 'meadow', 'blaze', 'flowhub', 'dispense', 'cova', 'other', 'unknown'];
|
|
|
|
// Get all dispensaries
|
|
router.get('/', async (req, res) => {
|
|
try {
|
|
const { menu_type, city, state } = req.query;
|
|
|
|
let query = `
|
|
SELECT
|
|
id,
|
|
name,
|
|
company_name,
|
|
slug,
|
|
address,
|
|
city,
|
|
state,
|
|
zip,
|
|
phone,
|
|
website,
|
|
dba_name,
|
|
latitude,
|
|
longitude,
|
|
menu_url,
|
|
menu_type,
|
|
platform,
|
|
platform_dispensary_id,
|
|
product_count,
|
|
last_crawl_at,
|
|
created_at,
|
|
updated_at
|
|
FROM dispensaries
|
|
`;
|
|
|
|
const params: any[] = [];
|
|
const conditions: string[] = [];
|
|
|
|
// Filter by menu_type if provided
|
|
if (menu_type) {
|
|
conditions.push(`menu_type = $${params.length + 1}`);
|
|
params.push(menu_type);
|
|
}
|
|
|
|
// Filter by city if provided
|
|
if (city) {
|
|
conditions.push(`city ILIKE $${params.length + 1}`);
|
|
params.push(city);
|
|
}
|
|
|
|
// Filter by state if provided
|
|
if (state) {
|
|
conditions.push(`state = $${params.length + 1}`);
|
|
params.push(state);
|
|
}
|
|
|
|
if (conditions.length > 0) {
|
|
query += ` WHERE ${conditions.join(' AND ')}`;
|
|
}
|
|
|
|
query += ` ORDER BY name`;
|
|
|
|
const result = await pool.query(query, params);
|
|
|
|
res.json({ dispensaries: result.rows });
|
|
} catch (error) {
|
|
console.error('Error fetching dispensaries:', error);
|
|
res.status(500).json({ error: 'Failed to fetch dispensaries' });
|
|
}
|
|
});
|
|
|
|
// Get menu type stats
|
|
router.get('/stats/menu-types', async (req, res) => {
|
|
try {
|
|
const result = await pool.query(`
|
|
SELECT menu_type, COUNT(*) as count
|
|
FROM dispensaries
|
|
GROUP BY menu_type
|
|
ORDER BY count DESC
|
|
`);
|
|
res.json({ menu_types: result.rows, valid_types: VALID_MENU_TYPES });
|
|
} catch (error) {
|
|
console.error('Error fetching menu type stats:', error);
|
|
res.status(500).json({ error: 'Failed to fetch menu type stats' });
|
|
}
|
|
});
|
|
|
|
// Get single dispensary by slug or ID
|
|
router.get('/:slugOrId', async (req, res) => {
|
|
try {
|
|
const { slugOrId } = req.params;
|
|
const isNumeric = /^\d+$/.test(slugOrId);
|
|
|
|
const result = await pool.query(`
|
|
SELECT
|
|
id,
|
|
name,
|
|
company_name,
|
|
slug,
|
|
address,
|
|
city,
|
|
state,
|
|
zip,
|
|
phone,
|
|
website,
|
|
dba_name,
|
|
latitude,
|
|
longitude,
|
|
menu_url,
|
|
menu_type,
|
|
platform,
|
|
platform_dispensary_id,
|
|
product_count,
|
|
last_crawl_at,
|
|
raw_metadata,
|
|
created_at,
|
|
updated_at
|
|
FROM dispensaries
|
|
WHERE ${isNumeric ? 'id = $1' : 'slug = $1'}
|
|
`, [isNumeric ? parseInt(slugOrId) : slugOrId]);
|
|
|
|
if (result.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
res.json(result.rows[0]);
|
|
} catch (error) {
|
|
console.error('Error fetching dispensary:', error);
|
|
res.status(500).json({ error: 'Failed to fetch dispensary' });
|
|
}
|
|
});
|
|
|
|
// Update dispensary
|
|
router.put('/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const {
|
|
name,
|
|
dba_name,
|
|
company_name,
|
|
website,
|
|
phone,
|
|
address,
|
|
city,
|
|
state,
|
|
zip,
|
|
latitude,
|
|
longitude,
|
|
menu_url,
|
|
menu_type,
|
|
platform,
|
|
platform_dispensary_id,
|
|
slug,
|
|
} = req.body;
|
|
|
|
// Validate menu_type if provided
|
|
if (menu_type !== undefined && menu_type !== null && menu_type !== '' && !VALID_MENU_TYPES.includes(menu_type)) {
|
|
return res.status(400).json({
|
|
error: `Invalid menu_type. Must be one of: ${VALID_MENU_TYPES.join(', ')}`
|
|
});
|
|
}
|
|
|
|
const result = await pool.query(`
|
|
UPDATE dispensaries
|
|
SET
|
|
name = COALESCE($1, name),
|
|
dba_name = COALESCE($2, dba_name),
|
|
company_name = COALESCE($3, company_name),
|
|
website = COALESCE($4, website),
|
|
phone = COALESCE($5, phone),
|
|
address = COALESCE($6, address),
|
|
city = COALESCE($7, city),
|
|
state = COALESCE($8, state),
|
|
zip = COALESCE($9, zip),
|
|
latitude = COALESCE($10, latitude),
|
|
longitude = COALESCE($11, longitude),
|
|
menu_url = COALESCE($12, menu_url),
|
|
menu_type = COALESCE($13, menu_type),
|
|
platform = COALESCE($14, platform),
|
|
platform_dispensary_id = COALESCE($15, platform_dispensary_id),
|
|
slug = COALESCE($16, slug),
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = $17
|
|
RETURNING *
|
|
`, [
|
|
name,
|
|
dba_name,
|
|
company_name,
|
|
website,
|
|
phone,
|
|
address,
|
|
city,
|
|
state,
|
|
zip,
|
|
latitude,
|
|
longitude,
|
|
menu_url,
|
|
menu_type,
|
|
platform,
|
|
platform_dispensary_id,
|
|
slug,
|
|
id
|
|
]);
|
|
|
|
if (result.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
res.json(result.rows[0]);
|
|
} catch (error) {
|
|
console.error('Error updating dispensary:', error);
|
|
res.status(500).json({ error: 'Failed to update dispensary' });
|
|
}
|
|
});
|
|
|
|
// Get products for a dispensary by slug
|
|
router.get('/:slug/products', async (req, res) => {
|
|
try {
|
|
const { slug } = req.params;
|
|
const { category } = req.query;
|
|
|
|
// First get the dispensary ID from slug
|
|
const dispensaryResult = await pool.query(`
|
|
SELECT id FROM dispensaries WHERE slug = $1
|
|
`, [slug]);
|
|
|
|
if (dispensaryResult.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
const dispensaryId = dispensaryResult.rows[0].id;
|
|
|
|
// Build query for products
|
|
let query = `
|
|
SELECT
|
|
p.id,
|
|
p.name,
|
|
p.brand,
|
|
p.variant,
|
|
p.slug,
|
|
p.description,
|
|
p.regular_price,
|
|
p.sale_price,
|
|
p.thc_percentage,
|
|
p.cbd_percentage,
|
|
p.strain_type,
|
|
p.terpenes,
|
|
p.effects,
|
|
p.flavors,
|
|
p.image_url,
|
|
p.dutchie_url,
|
|
p.in_stock,
|
|
p.created_at,
|
|
p.updated_at
|
|
FROM products p
|
|
WHERE p.dispensary_id = $1
|
|
`;
|
|
|
|
const params: any[] = [dispensaryId];
|
|
|
|
if (category) {
|
|
query += ` AND p.category = $2`;
|
|
params.push(category);
|
|
}
|
|
|
|
query += ` ORDER BY p.created_at DESC`;
|
|
|
|
const result = await pool.query(query, params);
|
|
|
|
res.json({ products: result.rows });
|
|
} catch (error) {
|
|
console.error('Error fetching dispensary products:', error);
|
|
res.status(500).json({ error: 'Failed to fetch products' });
|
|
}
|
|
});
|
|
|
|
// Get unique brands for a dispensary by slug
|
|
router.get('/:slug/brands', async (req, res) => {
|
|
try {
|
|
const { slug } = req.params;
|
|
const { search } = req.query;
|
|
|
|
// First get the dispensary ID from slug
|
|
const dispensaryResult = await pool.query(`
|
|
SELECT id FROM dispensaries WHERE slug = $1
|
|
`, [slug]);
|
|
|
|
if (dispensaryResult.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
const dispensaryId = dispensaryResult.rows[0].id;
|
|
|
|
// Build query with optional search filter
|
|
let query = `
|
|
SELECT DISTINCT
|
|
brand,
|
|
COUNT(*) as product_count
|
|
FROM products
|
|
WHERE dispensary_id = $1 AND brand IS NOT NULL
|
|
`;
|
|
const params: any[] = [dispensaryId];
|
|
|
|
// Add search filter if provided
|
|
if (search) {
|
|
query += ` AND brand ILIKE $2`;
|
|
params.push(`%${search}%`);
|
|
}
|
|
|
|
query += ` GROUP BY brand ORDER BY product_count DESC, brand ASC`;
|
|
|
|
const result = await pool.query(query, params);
|
|
|
|
res.json({ brands: result.rows });
|
|
} catch (error) {
|
|
console.error('Error fetching dispensary brands:', error);
|
|
res.status(500).json({ error: 'Failed to fetch brands' });
|
|
}
|
|
});
|
|
|
|
// Get products with discounts/specials for a dispensary by slug
|
|
router.get('/:slug/specials', async (req, res) => {
|
|
try {
|
|
const { slug } = req.params;
|
|
const { search } = req.query;
|
|
|
|
// First get the dispensary ID from slug
|
|
const dispensaryResult = await pool.query(`
|
|
SELECT id FROM dispensaries WHERE slug = $1
|
|
`, [slug]);
|
|
|
|
if (dispensaryResult.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
const dispensaryId = dispensaryResult.rows[0].id;
|
|
|
|
// Build query to get products with discounts
|
|
let query = `
|
|
SELECT
|
|
p.id,
|
|
p.name,
|
|
p.brand,
|
|
p.variant,
|
|
p.slug,
|
|
p.description,
|
|
p.regular_price,
|
|
p.sale_price,
|
|
p.discount_type,
|
|
p.discount_value,
|
|
p.thc_percentage,
|
|
p.cbd_percentage,
|
|
p.strain_type,
|
|
p.terpenes,
|
|
p.effects,
|
|
p.flavors,
|
|
p.image_url,
|
|
p.dutchie_url,
|
|
p.in_stock,
|
|
p.created_at,
|
|
p.updated_at
|
|
FROM products p
|
|
WHERE p.dispensary_id = $1
|
|
AND p.discount_type IS NOT NULL
|
|
AND p.discount_value IS NOT NULL
|
|
`;
|
|
const params: any[] = [dispensaryId];
|
|
|
|
// Add search filter if provided
|
|
if (search) {
|
|
query += ` AND (p.name ILIKE $2 OR p.brand ILIKE $2 OR p.description ILIKE $2)`;
|
|
params.push(`%${search}%`);
|
|
}
|
|
|
|
query += ` ORDER BY p.created_at DESC`;
|
|
|
|
const result = await pool.query(query, params);
|
|
|
|
res.json({ specials: result.rows });
|
|
} catch (error) {
|
|
console.error('Error fetching dispensary specials:', error);
|
|
res.status(500).json({ error: 'Failed to fetch specials' });
|
|
}
|
|
});
|
|
|
|
// Trigger scraping for a dispensary
|
|
router.post('/:slug/scrape', async (req, res) => {
|
|
try {
|
|
const { slug } = req.params;
|
|
const { type } = req.body; // 'products' | 'brands' | 'specials' | 'all'
|
|
|
|
if (!['products', 'brands', 'specials', 'all'].includes(type)) {
|
|
return res.status(400).json({ error: 'Invalid type. Must be: products, brands, specials, or all' });
|
|
}
|
|
|
|
// Get the dispensary
|
|
const dispensaryResult = await pool.query(`
|
|
SELECT id, name, slug, website, menu_url, scraper_template, scraper_config
|
|
FROM dispensaries
|
|
WHERE slug = $1
|
|
`, [slug]);
|
|
|
|
if (dispensaryResult.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
const dispensary = dispensaryResult.rows[0];
|
|
|
|
if (!dispensary.menu_url && !dispensary.website) {
|
|
return res.status(400).json({ error: 'Dispensary has no menu URL or website configured' });
|
|
}
|
|
|
|
// Update last_menu_scrape time and status
|
|
await pool.query(`
|
|
UPDATE dispensaries
|
|
SET
|
|
last_menu_scrape = CURRENT_TIMESTAMP,
|
|
menu_scrape_status = 'pending',
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = $1
|
|
`, [dispensary.id]);
|
|
|
|
// Log the scrape request
|
|
console.log(`[SCRAPE REQUEST] Dispensary: ${dispensary.name} (${slug}), Type: ${type}`);
|
|
console.log(` Menu URL: ${dispensary.menu_url || dispensary.website}`);
|
|
console.log(` Template: ${dispensary.scraper_template || 'N/A'}`);
|
|
|
|
// TODO: Actually trigger the scraper here
|
|
// For now, this is a placeholder that updates the status
|
|
// You can integrate with your existing scraper infrastructure
|
|
|
|
res.json({
|
|
success: true,
|
|
message: `Scraping queued for ${dispensary.name}`,
|
|
type,
|
|
dispensary: {
|
|
id: dispensary.id,
|
|
name: dispensary.name,
|
|
slug: dispensary.slug
|
|
}
|
|
});
|
|
} catch (error) {
|
|
console.error('Error triggering scrape:', error);
|
|
res.status(500).json({ error: 'Failed to trigger scraping' });
|
|
}
|
|
});
|
|
|
|
// Update menu_type for a dispensary (dedicated endpoint)
|
|
router.patch('/:id/menu-type', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { menu_type } = req.body;
|
|
|
|
// Validate menu_type
|
|
if (menu_type !== null && menu_type !== '' && !VALID_MENU_TYPES.includes(menu_type)) {
|
|
return res.status(400).json({
|
|
error: `Invalid menu_type. Must be one of: ${VALID_MENU_TYPES.join(', ')} (or null to clear)`
|
|
});
|
|
}
|
|
|
|
const result = await pool.query(`
|
|
UPDATE dispensaries
|
|
SET menu_type = $1, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = $2
|
|
RETURNING id, name, slug, menu_type, menu_provider, menu_url
|
|
`, [menu_type || null, id]);
|
|
|
|
if (result.rows.length === 0) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
|
|
res.json({
|
|
success: true,
|
|
dispensary: result.rows[0]
|
|
});
|
|
} catch (error) {
|
|
console.error('Error updating menu_type:', error);
|
|
res.status(500).json({ error: 'Failed to update menu_type' });
|
|
}
|
|
});
|
|
|
|
// Sync dispensary from discovery (upsert by platform_dispensary_id or slug)
|
|
// Used by Alice worker to sync discovered dispensaries to DB
|
|
router.post('/sync', async (req, res) => {
|
|
try {
|
|
const {
|
|
name,
|
|
slug,
|
|
city,
|
|
state,
|
|
address,
|
|
postalCode,
|
|
latitude,
|
|
longitude,
|
|
platformDispensaryId,
|
|
menuType,
|
|
menuUrl,
|
|
platform,
|
|
} = req.body;
|
|
|
|
if (!slug || !platformDispensaryId) {
|
|
return res.status(400).json({ error: 'slug and platformDispensaryId are required' });
|
|
}
|
|
|
|
// Try to find existing by platform_dispensary_id first, then by slug
|
|
const existingResult = await pool.query(`
|
|
SELECT id, name, slug, platform_dispensary_id, menu_type
|
|
FROM dispensaries
|
|
WHERE platform_dispensary_id = $1
|
|
OR (slug = $2 AND platform_dispensary_id IS NULL)
|
|
LIMIT 1
|
|
`, [platformDispensaryId, slug]);
|
|
|
|
if (existingResult.rows.length > 0) {
|
|
// Update existing
|
|
const existing = existingResult.rows[0];
|
|
const result = await pool.query(`
|
|
UPDATE dispensaries
|
|
SET
|
|
name = COALESCE($1, name),
|
|
slug = COALESCE($2, slug),
|
|
city = COALESCE($3, city),
|
|
state = COALESCE($4, state),
|
|
address = COALESCE($5, address),
|
|
zip = COALESCE($6, zip),
|
|
latitude = COALESCE($7, latitude),
|
|
longitude = COALESCE($8, longitude),
|
|
platform_dispensary_id = COALESCE($9, platform_dispensary_id),
|
|
menu_type = COALESCE($10, menu_type),
|
|
menu_url = COALESCE($11, menu_url),
|
|
platform = COALESCE($12, platform),
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = $13
|
|
RETURNING id, name, slug, platform_dispensary_id, menu_type
|
|
`, [
|
|
name, slug, city, state, address, postalCode,
|
|
latitude, longitude, platformDispensaryId, menuType, menuUrl, platform,
|
|
existing.id
|
|
]);
|
|
|
|
const updated = result.rows[0];
|
|
const changed = existing.platform_dispensary_id !== updated.platform_dispensary_id ||
|
|
existing.menu_type !== updated.menu_type ||
|
|
existing.slug !== updated.slug;
|
|
|
|
return res.json({
|
|
action: changed ? 'updated' : 'matched',
|
|
dispensary: updated,
|
|
});
|
|
}
|
|
|
|
// Insert new
|
|
const result = await pool.query(`
|
|
INSERT INTO dispensaries (
|
|
name, slug, city, state, address, zip,
|
|
latitude, longitude, platform_dispensary_id, menu_type, menu_url, platform,
|
|
created_at, updated_at
|
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id, name, slug, platform_dispensary_id, menu_type
|
|
`, [
|
|
name, slug, city, state, address, postalCode,
|
|
latitude, longitude, platformDispensaryId, menuType, menuUrl, platform
|
|
]);
|
|
|
|
return res.json({
|
|
action: 'inserted',
|
|
dispensary: result.rows[0],
|
|
});
|
|
|
|
} catch (error) {
|
|
console.error('Error syncing dispensary:', error);
|
|
res.status(500).json({ error: 'Failed to sync dispensary' });
|
|
}
|
|
});
|
|
|
|
// Bulk update menu_type for multiple dispensaries
|
|
router.post('/bulk/menu-type', async (req, res) => {
|
|
try {
|
|
const { dispensary_ids, menu_type } = req.body;
|
|
|
|
if (!Array.isArray(dispensary_ids) || dispensary_ids.length === 0) {
|
|
return res.status(400).json({ error: 'dispensary_ids must be a non-empty array' });
|
|
}
|
|
|
|
// Validate menu_type
|
|
if (menu_type !== null && menu_type !== '' && !VALID_MENU_TYPES.includes(menu_type)) {
|
|
return res.status(400).json({
|
|
error: `Invalid menu_type. Must be one of: ${VALID_MENU_TYPES.join(', ')} (or null to clear)`
|
|
});
|
|
}
|
|
|
|
const result = await pool.query(`
|
|
UPDATE dispensaries
|
|
SET menu_type = $1, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ANY($2::int[])
|
|
RETURNING id, name, slug, menu_type
|
|
`, [menu_type || null, dispensary_ids]);
|
|
|
|
res.json({
|
|
success: true,
|
|
updated_count: result.rowCount,
|
|
dispensaries: result.rows
|
|
});
|
|
} catch (error) {
|
|
console.error('Error bulk updating menu_type:', error);
|
|
res.status(500).json({ error: 'Failed to bulk update menu_type' });
|
|
}
|
|
});
|
|
|
|
export default router;
|