d91c55a344
- Add backend stale process monitoring API (/api/stale-processes) - Add users management route - Add frontend landing page and stale process monitor UI on /scraper-tools - Move old development scripts to backend/archive/ - Update frontend build with new features 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1730 lines
58 KiB
JavaScript
1730 lines
58 KiB
JavaScript
"use strict";
|
|
/**
|
|
* Dutchie AZ API Routes
|
|
*
|
|
* Express routes for the Dutchie AZ data pipeline.
|
|
* Provides API endpoints for stores, products, categories, and dashboard.
|
|
*/
|
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
}
|
|
Object.defineProperty(o, k2, desc);
|
|
}) : (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
o[k2] = m[k];
|
|
}));
|
|
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
}) : function(o, v) {
|
|
o["default"] = v;
|
|
});
|
|
var __importStar = (this && this.__importStar) || (function () {
|
|
var ownKeys = function(o) {
|
|
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
var ar = [];
|
|
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
return ar;
|
|
};
|
|
return ownKeys(o);
|
|
};
|
|
return function (mod) {
|
|
if (mod && mod.__esModule) return mod;
|
|
var result = {};
|
|
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
__setModuleDefault(result, mod);
|
|
return result;
|
|
};
|
|
})();
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
const express_1 = require("express");
|
|
const connection_1 = require("../db/connection");
|
|
const schema_1 = require("../db/schema");
|
|
const azdhs_import_1 = require("../services/azdhs-import");
|
|
const discovery_1 = require("../services/discovery");
|
|
const product_crawler_1 = require("../services/product-crawler");
|
|
// Explicit column list for dispensaries table (avoids SELECT * issues with schema differences)
|
|
const DISPENSARY_COLUMNS = `
|
|
id, name, dba_name, slug, city, state, zip, address, latitude, longitude,
|
|
menu_type, menu_url, platform_dispensary_id, website,
|
|
provider_detection_data, created_at, updated_at
|
|
`;
|
|
const scheduler_1 = require("../services/scheduler");
|
|
const router = (0, express_1.Router)();
|
|
// ============================================================
|
|
// DASHBOARD
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/dashboard
|
|
* Dashboard stats overview
|
|
*/
|
|
router.get('/dashboard', async (_req, res) => {
|
|
try {
|
|
const { rows } = await (0, connection_1.query)(`SELECT * FROM v_dashboard_stats`);
|
|
const stats = rows[0] || {};
|
|
res.json({
|
|
dispensaryCount: parseInt(stats.dispensary_count || '0', 10),
|
|
productCount: parseInt(stats.product_count || '0', 10),
|
|
snapshotCount24h: parseInt(stats.snapshots_24h || '0', 10),
|
|
lastCrawlTime: stats.last_crawl_time,
|
|
failedJobCount: parseInt(stats.failed_jobs_24h || '0', 10),
|
|
brandCount: parseInt(stats.brand_count || '0', 10),
|
|
categoryCount: parseInt(stats.category_count || '0', 10),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// DISPENSARIES (STORES)
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/stores
|
|
* List all stores with optional filters
|
|
*/
|
|
router.get('/stores', async (req, res) => {
|
|
try {
|
|
const { city, hasPlatformId, limit = '100', offset = '0' } = req.query;
|
|
let whereClause = 'WHERE state = \'AZ\'';
|
|
const params = [];
|
|
let paramIndex = 1;
|
|
if (city) {
|
|
whereClause += ` AND city = $${paramIndex}`;
|
|
params.push(city);
|
|
paramIndex++;
|
|
}
|
|
if (hasPlatformId === 'true') {
|
|
whereClause += ' AND platform_dispensary_id IS NOT NULL';
|
|
}
|
|
else if (hasPlatformId === 'false') {
|
|
whereClause += ' AND platform_dispensary_id IS NULL';
|
|
}
|
|
params.push(parseInt(limit, 10), parseInt(offset, 10));
|
|
const { rows, rowCount } = await (0, connection_1.query)(`
|
|
SELECT ${DISPENSARY_COLUMNS} FROM dispensaries
|
|
${whereClause}
|
|
ORDER BY name
|
|
LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
|
|
`, params);
|
|
// Get total count
|
|
const { rows: countRows } = await (0, connection_1.query)(`SELECT COUNT(*) as total FROM dispensaries ${whereClause}`, params.slice(0, -2));
|
|
res.json({
|
|
stores: rows,
|
|
total: parseInt(countRows[0]?.total || '0', 10),
|
|
limit: parseInt(limit, 10),
|
|
offset: parseInt(offset, 10),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/stores/slug/:slug
|
|
* Resolve a store by slug (case-insensitive) or platform_dispensary_id
|
|
*/
|
|
router.get('/stores/slug/:slug', async (req, res) => {
|
|
try {
|
|
const { slug } = req.params;
|
|
const normalized = slug.toLowerCase();
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT ${DISPENSARY_COLUMNS}
|
|
FROM dispensaries
|
|
WHERE lower(slug) = $1
|
|
OR lower(platform_dispensary_id) = $1
|
|
LIMIT 1
|
|
`, [normalized]);
|
|
if (!rows || rows.length === 0) {
|
|
return res.status(404).json({ error: 'Store not found' });
|
|
}
|
|
res.json(rows[0]);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/stores/:id
|
|
* Get a single store by ID
|
|
*/
|
|
router.get('/stores/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const store = await (0, discovery_1.getDispensaryById)(parseInt(id, 10));
|
|
if (!store) {
|
|
return res.status(404).json({ error: 'Store not found' });
|
|
}
|
|
res.json(store);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/stores/:id/summary
|
|
* Get store summary with product count, categories, and brands
|
|
* This is the main endpoint for the DispensaryDetail panel
|
|
*/
|
|
router.get('/stores/:id/summary', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
// Get dispensary info
|
|
const { rows: dispensaryRows } = await (0, connection_1.query)(`SELECT ${DISPENSARY_COLUMNS} FROM dispensaries WHERE id = $1`, [parseInt(id, 10)]);
|
|
if (dispensaryRows.length === 0) {
|
|
return res.status(404).json({ error: 'Store not found' });
|
|
}
|
|
const dispensary = dispensaryRows[0];
|
|
// Get product counts by stock status
|
|
const { rows: countRows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
COUNT(*) as total_products,
|
|
COUNT(*) FILTER (WHERE stock_status = 'in_stock') as in_stock_count,
|
|
COUNT(*) FILTER (WHERE stock_status = 'out_of_stock') as out_of_stock_count,
|
|
COUNT(*) FILTER (WHERE stock_status = 'unknown') as unknown_count,
|
|
COUNT(*) FILTER (WHERE stock_status = 'missing_from_feed') as missing_count
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1
|
|
`, [id]);
|
|
// Get categories with counts for this store
|
|
const { rows: categories } = await (0, connection_1.query)(`
|
|
SELECT
|
|
type,
|
|
subcategory,
|
|
COUNT(*) as product_count
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1 AND type IS NOT NULL
|
|
GROUP BY type, subcategory
|
|
ORDER BY type, subcategory
|
|
`, [id]);
|
|
// Get brands with counts for this store
|
|
const { rows: brands } = await (0, connection_1.query)(`
|
|
SELECT
|
|
brand_name,
|
|
COUNT(*) as product_count
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1 AND brand_name IS NOT NULL
|
|
GROUP BY brand_name
|
|
ORDER BY product_count DESC
|
|
`, [id]);
|
|
// Get last crawl info
|
|
const { rows: lastCrawl } = await (0, connection_1.query)(`
|
|
SELECT
|
|
id,
|
|
status,
|
|
started_at,
|
|
completed_at,
|
|
products_found,
|
|
products_new,
|
|
products_updated,
|
|
error_message
|
|
FROM dispensary_crawl_jobs
|
|
WHERE dispensary_id = $1
|
|
ORDER BY created_at DESC
|
|
LIMIT 1
|
|
`, [id]);
|
|
const counts = countRows[0] || {};
|
|
res.json({
|
|
dispensary,
|
|
totalProducts: parseInt(counts.total_products || '0', 10),
|
|
inStockCount: parseInt(counts.in_stock_count || '0', 10),
|
|
outOfStockCount: parseInt(counts.out_of_stock_count || '0', 10),
|
|
unknownStockCount: parseInt(counts.unknown_count || '0', 10),
|
|
missingFromFeedCount: parseInt(counts.missing_count || '0', 10),
|
|
categories,
|
|
brands,
|
|
brandCount: brands.length,
|
|
categoryCount: categories.length,
|
|
lastCrawl: lastCrawl[0] || null,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/stores/:id/products
|
|
* Get paginated products for a store with latest snapshot data
|
|
*/
|
|
router.get('/stores/:id/products', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { stockStatus, type, subcategory, brandName, search, limit = '50', offset = '0', } = req.query;
|
|
let whereClause = 'WHERE p.dispensary_id = $1';
|
|
const params = [parseInt(id, 10)];
|
|
let paramIndex = 2;
|
|
if (stockStatus) {
|
|
whereClause += ` AND p.stock_status = $${paramIndex}`;
|
|
params.push(stockStatus);
|
|
paramIndex++;
|
|
}
|
|
if (type) {
|
|
whereClause += ` AND p.type = $${paramIndex}`;
|
|
params.push(type);
|
|
paramIndex++;
|
|
}
|
|
if (subcategory) {
|
|
whereClause += ` AND p.subcategory = $${paramIndex}`;
|
|
params.push(subcategory);
|
|
paramIndex++;
|
|
}
|
|
if (brandName) {
|
|
whereClause += ` AND p.brand_name ILIKE $${paramIndex}`;
|
|
params.push(`%${brandName}%`);
|
|
paramIndex++;
|
|
}
|
|
if (search) {
|
|
whereClause += ` AND (p.name ILIKE $${paramIndex} OR p.brand_name ILIKE $${paramIndex})`;
|
|
params.push(`%${search}%`);
|
|
paramIndex++;
|
|
}
|
|
params.push(parseInt(limit, 10), parseInt(offset, 10));
|
|
// Get products with their latest snapshot data
|
|
const { rows: products } = await (0, connection_1.query)(`
|
|
SELECT
|
|
p.id,
|
|
p.external_product_id,
|
|
p.name,
|
|
p.brand_name,
|
|
p.type,
|
|
p.subcategory,
|
|
p.strain_type,
|
|
p.stock_status,
|
|
p.created_at,
|
|
p.updated_at,
|
|
p.primary_image_url,
|
|
p.thc_content,
|
|
p.cbd_content,
|
|
-- Latest snapshot data (prices in cents)
|
|
s.rec_min_price_cents,
|
|
s.rec_max_price_cents,
|
|
s.med_min_price_cents,
|
|
s.med_max_price_cents,
|
|
s.rec_min_special_price_cents,
|
|
s.med_min_special_price_cents,
|
|
s.total_quantity_available,
|
|
s.options,
|
|
s.stock_status as snapshot_stock_status,
|
|
s.crawled_at as snapshot_at
|
|
FROM dutchie_products p
|
|
LEFT JOIN LATERAL (
|
|
SELECT * FROM dutchie_product_snapshots
|
|
WHERE dutchie_product_id = p.id
|
|
ORDER BY crawled_at DESC
|
|
LIMIT 1
|
|
) s ON true
|
|
${whereClause}
|
|
ORDER BY p.updated_at DESC
|
|
LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
|
|
`, params);
|
|
// Get total count
|
|
const { rows: countRows } = await (0, connection_1.query)(`SELECT COUNT(*) as total FROM dutchie_products p ${whereClause}`, params.slice(0, -2));
|
|
// Transform products for frontend compatibility
|
|
const transformedProducts = products.map((p) => ({
|
|
id: p.id,
|
|
external_id: p.external_product_id,
|
|
name: p.name,
|
|
brand: p.brand_name,
|
|
type: p.type,
|
|
subcategory: p.subcategory,
|
|
strain_type: p.strain_type,
|
|
stock_status: p.snapshot_stock_status || p.stock_status,
|
|
in_stock: (p.snapshot_stock_status || p.stock_status) === 'in_stock',
|
|
// Prices from latest snapshot (convert cents to dollars)
|
|
regular_price: p.rec_min_price_cents ? p.rec_min_price_cents / 100 : null,
|
|
regular_price_max: p.rec_max_price_cents ? p.rec_max_price_cents / 100 : null,
|
|
sale_price: p.rec_min_special_price_cents ? p.rec_min_special_price_cents / 100 : null,
|
|
med_price: p.med_min_price_cents ? p.med_min_price_cents / 100 : null,
|
|
med_price_max: p.med_max_price_cents ? p.med_max_price_cents / 100 : null,
|
|
med_sale_price: p.med_min_special_price_cents ? p.med_min_special_price_cents / 100 : null,
|
|
// Potency from products table
|
|
thc_percentage: p.thc_content,
|
|
cbd_percentage: p.cbd_content,
|
|
// Images from products table
|
|
image_url: p.primary_image_url,
|
|
// Other
|
|
options: p.options,
|
|
total_quantity: p.total_quantity_available,
|
|
// Timestamps
|
|
created_at: p.created_at,
|
|
updated_at: p.updated_at,
|
|
snapshot_at: p.snapshot_at,
|
|
}));
|
|
res.json({
|
|
products: transformedProducts,
|
|
total: parseInt(countRows[0]?.total || '0', 10),
|
|
limit: parseInt(limit, 10),
|
|
offset: parseInt(offset, 10),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/stores/:id/brands
|
|
* Get brands for a specific store
|
|
*/
|
|
router.get('/stores/:id/brands', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { rows: brands } = await (0, connection_1.query)(`
|
|
SELECT
|
|
brand_name as brand,
|
|
COUNT(*) as product_count
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1 AND brand_name IS NOT NULL
|
|
GROUP BY brand_name
|
|
ORDER BY product_count DESC
|
|
`, [parseInt(id, 10)]);
|
|
res.json({ brands });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/stores/:id/categories
|
|
* Get categories for a specific store
|
|
*/
|
|
router.get('/stores/:id/categories', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { rows: categories } = await (0, connection_1.query)(`
|
|
SELECT
|
|
type,
|
|
subcategory,
|
|
COUNT(*) as product_count
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1 AND type IS NOT NULL
|
|
GROUP BY type, subcategory
|
|
ORDER BY type, subcategory
|
|
`, [parseInt(id, 10)]);
|
|
res.json({ categories });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// PRODUCTS
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/products
|
|
* List products with filtering on our own DB
|
|
*/
|
|
router.get('/products', async (req, res) => {
|
|
try {
|
|
const { storeId, stockStatus, type, subcategory, brandName, search, limit = '50', offset = '0', } = req.query;
|
|
let whereClause = 'WHERE 1=1';
|
|
const params = [];
|
|
let paramIndex = 1;
|
|
if (storeId) {
|
|
whereClause += ` AND dispensary_id = $${paramIndex}`;
|
|
params.push(parseInt(storeId, 10));
|
|
paramIndex++;
|
|
}
|
|
if (stockStatus) {
|
|
whereClause += ` AND stock_status = $${paramIndex}`;
|
|
params.push(stockStatus);
|
|
paramIndex++;
|
|
}
|
|
if (type) {
|
|
whereClause += ` AND type = $${paramIndex}`;
|
|
params.push(type);
|
|
paramIndex++;
|
|
}
|
|
if (subcategory) {
|
|
whereClause += ` AND subcategory = $${paramIndex}`;
|
|
params.push(subcategory);
|
|
paramIndex++;
|
|
}
|
|
if (brandName) {
|
|
whereClause += ` AND brand_name ILIKE $${paramIndex}`;
|
|
params.push(`%${brandName}%`);
|
|
paramIndex++;
|
|
}
|
|
if (search) {
|
|
whereClause += ` AND (name ILIKE $${paramIndex} OR brand_name ILIKE $${paramIndex})`;
|
|
params.push(`%${search}%`);
|
|
paramIndex++;
|
|
}
|
|
params.push(parseInt(limit, 10), parseInt(offset, 10));
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
p.*,
|
|
d.name as store_name,
|
|
d.city as store_city
|
|
FROM dutchie_products p
|
|
JOIN dispensaries d ON p.dispensary_id = d.id
|
|
${whereClause}
|
|
ORDER BY p.updated_at DESC
|
|
LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
|
|
`, params);
|
|
// Get total count
|
|
const { rows: countRows } = await (0, connection_1.query)(`SELECT COUNT(*) as total FROM dutchie_products ${whereClause}`, params.slice(0, -2));
|
|
res.json({
|
|
products: rows,
|
|
total: parseInt(countRows[0]?.total || '0', 10),
|
|
limit: parseInt(limit, 10),
|
|
offset: parseInt(offset, 10),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/products/:id
|
|
* Get a single product with its latest snapshot
|
|
*/
|
|
router.get('/products/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { rows: productRows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
p.*,
|
|
d.name as store_name,
|
|
d.city as store_city,
|
|
d.slug as store_slug
|
|
FROM dutchie_products p
|
|
JOIN dispensaries d ON p.dispensary_id = d.id
|
|
WHERE p.id = $1
|
|
`, [id]);
|
|
if (productRows.length === 0) {
|
|
return res.status(404).json({ error: 'Product not found' });
|
|
}
|
|
// Get latest snapshot
|
|
const { rows: snapshotRows } = await (0, connection_1.query)(`
|
|
SELECT * FROM dutchie_product_snapshots
|
|
WHERE dutchie_product_id = $1
|
|
ORDER BY crawled_at DESC
|
|
LIMIT 1
|
|
`, [id]);
|
|
res.json({
|
|
product: productRows[0],
|
|
latestSnapshot: snapshotRows[0] || null,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/products/:id/snapshots
|
|
* Get snapshot history for a product
|
|
*/
|
|
router.get('/products/:id/snapshots', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { limit = '50', offset = '0' } = req.query;
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT * FROM dutchie_product_snapshots
|
|
WHERE dutchie_product_id = $1
|
|
ORDER BY crawled_at DESC
|
|
LIMIT $2 OFFSET $3
|
|
`, [id, parseInt(limit, 10), parseInt(offset, 10)]);
|
|
res.json({ snapshots: rows });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// CATEGORIES
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/categories
|
|
* Get all categories with counts
|
|
*/
|
|
router.get('/categories', async (_req, res) => {
|
|
try {
|
|
const { rows } = await (0, connection_1.query)(`SELECT * FROM v_categories`);
|
|
res.json({ categories: rows });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// BRANDS
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/brands
|
|
* Get all brands with counts
|
|
*/
|
|
router.get('/brands', async (req, res) => {
|
|
try {
|
|
const { limit = '100', offset = '0' } = req.query;
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT * FROM v_brands
|
|
LIMIT $1 OFFSET $2
|
|
`, [parseInt(limit, 10), parseInt(offset, 10)]);
|
|
res.json({ brands: rows });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// ADMIN ACTIONS
|
|
// ============================================================
|
|
/**
|
|
* POST /api/dutchie-az/admin/init-schema
|
|
* Initialize the database schema
|
|
*/
|
|
router.post('/admin/init-schema', async (_req, res) => {
|
|
try {
|
|
await (0, schema_1.ensureSchema)();
|
|
res.json({ success: true, message: 'Schema initialized' });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/import-azdhs
|
|
* Import dispensaries from AZDHS (main database)
|
|
*/
|
|
router.post('/admin/import-azdhs', async (_req, res) => {
|
|
try {
|
|
const result = await (0, azdhs_import_1.importAZDHSDispensaries)();
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/resolve-platform-ids
|
|
* Resolve Dutchie platform IDs for all dispensaries
|
|
*/
|
|
router.post('/admin/resolve-platform-ids', async (_req, res) => {
|
|
try {
|
|
const result = await (0, discovery_1.resolvePlatformDispensaryIds)();
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/crawl-store/:id
|
|
* Crawl a single store
|
|
*/
|
|
router.post('/admin/crawl-store/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { pricingType = 'rec', useBothModes = true } = req.body;
|
|
const dispensary = await (0, discovery_1.getDispensaryById)(parseInt(id, 10));
|
|
if (!dispensary) {
|
|
return res.status(404).json({ error: 'Store not found' });
|
|
}
|
|
const result = await (0, product_crawler_1.crawlDispensaryProducts)(dispensary, pricingType, { useBothModes });
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/stats
|
|
* Get import and crawl statistics
|
|
*/
|
|
router.get('/admin/stats', async (_req, res) => {
|
|
try {
|
|
const importStats = await (0, azdhs_import_1.getImportStats)();
|
|
// Get stock status distribution
|
|
const { rows: stockStats } = await (0, connection_1.query)(`
|
|
SELECT
|
|
stock_status,
|
|
COUNT(*) as count
|
|
FROM dutchie_products
|
|
GROUP BY stock_status
|
|
`);
|
|
// Get recent crawl jobs
|
|
const { rows: recentJobs } = await (0, connection_1.query)(`
|
|
SELECT * FROM dispensary_crawl_jobs
|
|
ORDER BY created_at DESC
|
|
LIMIT 10
|
|
`);
|
|
res.json({
|
|
import: importStats,
|
|
stockDistribution: stockStats,
|
|
recentJobs,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// SCHEDULER ADMIN
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/admin/scheduler/status
|
|
* Get scheduler status
|
|
*/
|
|
router.get('/admin/scheduler/status', async (_req, res) => {
|
|
try {
|
|
const status = (0, scheduler_1.getSchedulerStatus)();
|
|
res.json(status);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/scheduler/start
|
|
* Start the scheduler
|
|
*/
|
|
router.post('/admin/scheduler/start', async (_req, res) => {
|
|
try {
|
|
(0, scheduler_1.startScheduler)();
|
|
res.json({ success: true, message: 'Scheduler started' });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/scheduler/stop
|
|
* Stop the scheduler
|
|
*/
|
|
router.post('/admin/scheduler/stop', async (_req, res) => {
|
|
try {
|
|
(0, scheduler_1.stopScheduler)();
|
|
res.json({ success: true, message: 'Scheduler stopped' });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/scheduler/trigger
|
|
* Trigger an immediate crawl cycle
|
|
*/
|
|
router.post('/admin/scheduler/trigger', async (_req, res) => {
|
|
try {
|
|
const result = await (0, scheduler_1.triggerImmediateCrawl)();
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/crawl/:id
|
|
* Crawl a single dispensary with job tracking
|
|
*/
|
|
router.post('/admin/crawl/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { pricingType = 'rec', useBothModes = true } = req.body;
|
|
// Fetch the dispensary first
|
|
const dispensary = await (0, discovery_1.getDispensaryById)(parseInt(id, 10));
|
|
if (!dispensary) {
|
|
return res.status(404).json({ error: 'Dispensary not found' });
|
|
}
|
|
const result = await (0, scheduler_1.crawlSingleDispensary)(dispensary, pricingType, { useBothModes });
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
const job_queue_1 = require("../services/job-queue");
|
|
/**
|
|
* GET /api/dutchie-az/admin/dutchie-stores
|
|
* Get all Dutchie stores with their crawl status
|
|
*/
|
|
router.get('/admin/dutchie-stores', async (_req, res) => {
|
|
try {
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
d.id,
|
|
d.name,
|
|
d.dba_name,
|
|
d.city,
|
|
d.state,
|
|
d.menu_type,
|
|
d.platform_dispensary_id,
|
|
d.menu_url,
|
|
d.website,
|
|
d.last_crawl_at,
|
|
d.consecutive_failures,
|
|
d.failed_at,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = d.id
|
|
) as product_count,
|
|
(
|
|
SELECT MAX(crawled_at)
|
|
FROM dutchie_product_snapshots s
|
|
JOIN dutchie_products p ON s.dutchie_product_id = p.id
|
|
WHERE p.dispensary_id = d.id
|
|
) as last_snapshot_at
|
|
FROM dispensaries d
|
|
WHERE d.menu_type = 'dutchie'
|
|
AND d.state = 'AZ'
|
|
ORDER BY d.name
|
|
`);
|
|
const ready = rows.filter((r) => r.platform_dispensary_id && !r.failed_at);
|
|
const needsPlatformId = rows.filter((r) => !r.platform_dispensary_id && !r.failed_at);
|
|
const failed = rows.filter((r) => r.failed_at);
|
|
res.json({
|
|
total: rows.length,
|
|
ready: ready.length,
|
|
needsPlatformId: needsPlatformId.length,
|
|
failed: failed.length,
|
|
stores: rows.map((r) => ({
|
|
id: r.id,
|
|
name: r.dba_name || r.name,
|
|
city: r.city,
|
|
state: r.state,
|
|
menuType: r.menu_type,
|
|
platformDispensaryId: r.platform_dispensary_id,
|
|
menuUrl: r.menu_url,
|
|
website: r.website,
|
|
lastCrawlAt: r.last_crawl_at,
|
|
productCount: parseInt(r.product_count || '0', 10),
|
|
lastSnapshotAt: r.last_snapshot_at,
|
|
status: r.failed_at
|
|
? 'failed'
|
|
: r.platform_dispensary_id
|
|
? 'ready'
|
|
: 'needs_platform_id',
|
|
})),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/crawl-all
|
|
* Enqueue crawl jobs for ALL ready Dutchie stores
|
|
* This is a convenience endpoint to queue all stores without triggering the scheduler
|
|
*/
|
|
router.post('/admin/crawl-all', async (req, res) => {
|
|
try {
|
|
const { pricingType = 'rec', useBothModes = true } = req.body;
|
|
// Get all "ready" dispensaries (menu_type='dutchie' AND platform_dispensary_id IS NOT NULL AND not failed)
|
|
const { rows: rawRows } = await (0, connection_1.query)(`
|
|
SELECT id, name, platform_dispensary_id FROM dispensaries
|
|
WHERE state = 'AZ'
|
|
AND menu_type = 'dutchie'
|
|
AND platform_dispensary_id IS NOT NULL
|
|
AND failed_at IS NULL
|
|
ORDER BY last_crawl_at ASC NULLS FIRST
|
|
`);
|
|
if (rawRows.length === 0) {
|
|
return res.json({
|
|
success: true,
|
|
message: 'No ready dispensaries to crawl. Run menu detection first.',
|
|
enqueued: 0,
|
|
skipped: 0,
|
|
dispensaries: [],
|
|
});
|
|
}
|
|
const dispensaryIds = rawRows.map((r) => r.id);
|
|
// Bulk enqueue jobs (skips dispensaries that already have pending/running jobs)
|
|
const { enqueued, skipped } = await (0, job_queue_1.bulkEnqueueJobs)('dutchie_product_crawl', dispensaryIds, {
|
|
priority: 0,
|
|
metadata: { pricingType, useBothModes },
|
|
});
|
|
// Get current queue stats
|
|
const queueStats = await (0, job_queue_1.getQueueStats)();
|
|
res.json({
|
|
success: true,
|
|
message: `Enqueued ${enqueued} crawl jobs for Dutchie stores`,
|
|
totalReady: rawRows.length,
|
|
enqueued,
|
|
skipped,
|
|
queueStats,
|
|
dispensaries: rawRows.map((r) => ({
|
|
id: r.id,
|
|
name: r.name,
|
|
platformDispensaryId: r.platform_dispensary_id,
|
|
})),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/jobs
|
|
* Get crawl job history
|
|
*/
|
|
router.get('/admin/jobs', async (req, res) => {
|
|
try {
|
|
const { status, dispensaryId, limit = '50', offset = '0' } = req.query;
|
|
let whereClause = 'WHERE 1=1';
|
|
const params = [];
|
|
let paramIndex = 1;
|
|
if (status) {
|
|
whereClause += ` AND status = $${paramIndex}`;
|
|
params.push(status);
|
|
paramIndex++;
|
|
}
|
|
if (dispensaryId) {
|
|
whereClause += ` AND dispensary_id = $${paramIndex}`;
|
|
params.push(parseInt(dispensaryId, 10));
|
|
paramIndex++;
|
|
}
|
|
params.push(parseInt(limit, 10), parseInt(offset, 10));
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
cj.*,
|
|
d.name as dispensary_name,
|
|
d.slug as dispensary_slug
|
|
FROM dispensary_crawl_jobs cj
|
|
LEFT JOIN dispensaries d ON cj.dispensary_id = d.id
|
|
${whereClause}
|
|
ORDER BY cj.created_at DESC
|
|
LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
|
|
`, params);
|
|
const { rows: countRows } = await (0, connection_1.query)(`SELECT COUNT(*) as total FROM dispensary_crawl_jobs ${whereClause}`, params.slice(0, -2));
|
|
res.json({
|
|
jobs: rows,
|
|
total: parseInt(countRows[0]?.total || '0', 10),
|
|
limit: parseInt(limit, 10),
|
|
offset: parseInt(offset, 10),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// SCHEDULES (CONFIG CRUD)
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/admin/schedules
|
|
* Get all schedule configurations
|
|
*/
|
|
router.get('/admin/schedules', async (_req, res) => {
|
|
try {
|
|
const schedules = await (0, scheduler_1.getAllSchedules)();
|
|
res.json({ schedules });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/schedules/:id
|
|
* Get a single schedule by ID
|
|
*/
|
|
router.get('/admin/schedules/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const schedule = await (0, scheduler_1.getScheduleById)(parseInt(id, 10));
|
|
if (!schedule) {
|
|
return res.status(404).json({ error: 'Schedule not found' });
|
|
}
|
|
res.json(schedule);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/schedules
|
|
* Create a new schedule
|
|
*/
|
|
router.post('/admin/schedules', async (req, res) => {
|
|
try {
|
|
const { jobName, description, enabled = true, baseIntervalMinutes, jitterMinutes, jobConfig, startImmediately = false, } = req.body;
|
|
if (!jobName || typeof baseIntervalMinutes !== 'number' || typeof jitterMinutes !== 'number') {
|
|
return res.status(400).json({
|
|
error: 'jobName, baseIntervalMinutes, and jitterMinutes are required',
|
|
});
|
|
}
|
|
const schedule = await (0, scheduler_1.createSchedule)({
|
|
jobName,
|
|
description,
|
|
enabled,
|
|
baseIntervalMinutes,
|
|
jitterMinutes,
|
|
jobConfig,
|
|
startImmediately,
|
|
});
|
|
res.status(201).json(schedule);
|
|
}
|
|
catch (error) {
|
|
// Handle unique constraint violation
|
|
if (error.code === '23505') {
|
|
return res.status(409).json({ error: `Schedule "${req.body.jobName}" already exists` });
|
|
}
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* PUT /api/dutchie-az/admin/schedules/:id
|
|
* Update a schedule
|
|
*/
|
|
router.put('/admin/schedules/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { description, enabled, baseIntervalMinutes, jitterMinutes, jobConfig } = req.body;
|
|
const schedule = await (0, scheduler_1.updateSchedule)(parseInt(id, 10), {
|
|
description,
|
|
enabled,
|
|
baseIntervalMinutes,
|
|
jitterMinutes,
|
|
jobConfig,
|
|
});
|
|
if (!schedule) {
|
|
return res.status(404).json({ error: 'Schedule not found' });
|
|
}
|
|
res.json(schedule);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* DELETE /api/dutchie-az/admin/schedules/:id
|
|
* Delete a schedule
|
|
*/
|
|
router.delete('/admin/schedules/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const deleted = await (0, scheduler_1.deleteSchedule)(parseInt(id, 10));
|
|
if (!deleted) {
|
|
return res.status(404).json({ error: 'Schedule not found' });
|
|
}
|
|
res.json({ success: true, message: 'Schedule deleted' });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/schedules/:id/trigger
|
|
* Trigger immediate execution of a schedule
|
|
*/
|
|
router.post('/admin/schedules/:id/trigger', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const result = await (0, scheduler_1.triggerScheduleNow)(parseInt(id, 10));
|
|
if (!result.success) {
|
|
return res.status(400).json({ error: result.message });
|
|
}
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/schedules/init
|
|
* Initialize default schedules if they don't exist
|
|
*/
|
|
router.post('/admin/schedules/init', async (_req, res) => {
|
|
try {
|
|
await (0, scheduler_1.initializeDefaultSchedules)();
|
|
const schedules = await (0, scheduler_1.getAllSchedules)();
|
|
res.json({ success: true, schedules });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/schedules/:id/logs
|
|
* Get run logs for a specific schedule
|
|
*/
|
|
router.get('/admin/schedules/:id/logs', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const { limit = '50', offset = '0' } = req.query;
|
|
const result = await (0, scheduler_1.getRunLogs)({
|
|
scheduleId: parseInt(id, 10),
|
|
limit: parseInt(limit, 10),
|
|
offset: parseInt(offset, 10),
|
|
});
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/run-logs
|
|
* Get all run logs with filtering
|
|
*/
|
|
router.get('/admin/run-logs', async (req, res) => {
|
|
try {
|
|
const { scheduleId, jobName, limit = '50', offset = '0' } = req.query;
|
|
const result = await (0, scheduler_1.getRunLogs)({
|
|
scheduleId: scheduleId ? parseInt(scheduleId, 10) : undefined,
|
|
jobName: jobName,
|
|
limit: parseInt(limit, 10),
|
|
offset: parseInt(offset, 10),
|
|
});
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// DEBUG ROUTES
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/debug/summary
|
|
* Get overall system summary for debugging
|
|
*/
|
|
router.get('/debug/summary', async (_req, res) => {
|
|
try {
|
|
// Get table counts
|
|
const { rows: tableCounts } = await (0, connection_1.query)(`
|
|
SELECT
|
|
(SELECT COUNT(*) FROM dispensaries) as dispensary_count,
|
|
(SELECT COUNT(*) FROM dispensaries WHERE platform_dispensary_id IS NOT NULL) as dispensaries_with_platform_id,
|
|
(SELECT COUNT(*) FROM dutchie_products) as product_count,
|
|
(SELECT COUNT(*) FROM dutchie_product_snapshots) as snapshot_count,
|
|
(SELECT COUNT(*) FROM dispensary_crawl_jobs) as job_count,
|
|
(SELECT COUNT(*) FROM dispensary_crawl_jobs WHERE status = 'completed') as completed_jobs,
|
|
(SELECT COUNT(*) FROM dispensary_crawl_jobs WHERE status = 'failed') as failed_jobs
|
|
`);
|
|
// Get stock status distribution
|
|
const { rows: stockDistribution } = await (0, connection_1.query)(`
|
|
SELECT
|
|
stock_status,
|
|
COUNT(*) as count
|
|
FROM dutchie_products
|
|
GROUP BY stock_status
|
|
ORDER BY count DESC
|
|
`);
|
|
// Get products by dispensary
|
|
const { rows: productsByDispensary } = await (0, connection_1.query)(`
|
|
SELECT
|
|
d.id,
|
|
d.name,
|
|
d.slug,
|
|
d.platform_dispensary_id,
|
|
COUNT(p.id) as product_count,
|
|
MAX(p.updated_at) as last_product_update
|
|
FROM dispensaries d
|
|
LEFT JOIN dutchie_products p ON d.id = p.dispensary_id
|
|
WHERE d.state = 'AZ'
|
|
GROUP BY d.id, d.name, d.slug, d.platform_dispensary_id
|
|
ORDER BY product_count DESC
|
|
LIMIT 20
|
|
`);
|
|
// Get recent snapshots
|
|
const { rows: recentSnapshots } = await (0, connection_1.query)(`
|
|
SELECT
|
|
s.id,
|
|
s.dutchie_product_id,
|
|
p.name as product_name,
|
|
d.name as dispensary_name,
|
|
s.crawled_at
|
|
FROM dutchie_product_snapshots s
|
|
JOIN dutchie_products p ON s.dutchie_product_id = p.id
|
|
JOIN dispensaries d ON p.dispensary_id = d.id
|
|
ORDER BY s.crawled_at DESC
|
|
LIMIT 10
|
|
`);
|
|
res.json({
|
|
tableCounts: tableCounts[0],
|
|
stockDistribution,
|
|
productsByDispensary,
|
|
recentSnapshots,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/debug/store/:id
|
|
* Get detailed debug info for a specific store
|
|
*/
|
|
router.get('/debug/store/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
// Get dispensary info
|
|
const { rows: dispensaryRows } = await (0, connection_1.query)(`SELECT ${DISPENSARY_COLUMNS} FROM dispensaries WHERE id = $1`, [parseInt(id, 10)]);
|
|
if (dispensaryRows.length === 0) {
|
|
return res.status(404).json({ error: 'Store not found' });
|
|
}
|
|
const dispensary = dispensaryRows[0];
|
|
// Get product stats
|
|
const { rows: productStats } = await (0, connection_1.query)(`
|
|
SELECT
|
|
COUNT(*) as total_products,
|
|
COUNT(*) FILTER (WHERE stock_status = 'in_stock') as in_stock,
|
|
COUNT(*) FILTER (WHERE stock_status = 'out_of_stock') as out_of_stock,
|
|
COUNT(*) FILTER (WHERE stock_status = 'unknown') as unknown,
|
|
COUNT(*) FILTER (WHERE stock_status = 'missing_from_feed') as missing_from_feed,
|
|
MIN(first_seen_at) as earliest_product,
|
|
MAX(last_seen_at) as latest_product,
|
|
MAX(updated_at) as last_update
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1
|
|
`, [id]);
|
|
// Get snapshot stats
|
|
const { rows: snapshotStats } = await (0, connection_1.query)(`
|
|
SELECT
|
|
COUNT(*) as total_snapshots,
|
|
MIN(crawled_at) as earliest_snapshot,
|
|
MAX(crawled_at) as latest_snapshot,
|
|
COUNT(DISTINCT dutchie_product_id) as products_with_snapshots
|
|
FROM dutchie_product_snapshots s
|
|
JOIN dutchie_products p ON s.dutchie_product_id = p.id
|
|
WHERE p.dispensary_id = $1
|
|
`, [id]);
|
|
// Get crawl job history
|
|
const { rows: recentJobs } = await (0, connection_1.query)(`
|
|
SELECT
|
|
id,
|
|
status,
|
|
started_at,
|
|
completed_at,
|
|
products_found,
|
|
products_new,
|
|
products_updated,
|
|
error_message,
|
|
created_at
|
|
FROM dispensary_crawl_jobs
|
|
WHERE dispensary_id = $1
|
|
ORDER BY created_at DESC
|
|
LIMIT 10
|
|
`, [id]);
|
|
// Get sample products (5 in-stock, 5 out-of-stock)
|
|
const { rows: sampleInStock } = await (0, connection_1.query)(`
|
|
SELECT
|
|
p.id,
|
|
p.name,
|
|
p.brand_name,
|
|
p.type,
|
|
p.stock_status,
|
|
p.updated_at
|
|
FROM dutchie_products p
|
|
WHERE p.dispensary_id = $1 AND p.stock_status = 'in_stock'
|
|
ORDER BY p.updated_at DESC
|
|
LIMIT 5
|
|
`, [id]);
|
|
const { rows: sampleOutOfStock } = await (0, connection_1.query)(`
|
|
SELECT
|
|
p.id,
|
|
p.name,
|
|
p.brand_name,
|
|
p.type,
|
|
p.stock_status,
|
|
p.updated_at
|
|
FROM dutchie_products p
|
|
WHERE p.dispensary_id = $1 AND p.stock_status = 'out_of_stock'
|
|
ORDER BY p.updated_at DESC
|
|
LIMIT 5
|
|
`, [id]);
|
|
// Get categories breakdown
|
|
const { rows: categories } = await (0, connection_1.query)(`
|
|
SELECT
|
|
type,
|
|
subcategory,
|
|
COUNT(*) as count
|
|
FROM dutchie_products
|
|
WHERE dispensary_id = $1
|
|
GROUP BY type, subcategory
|
|
ORDER BY count DESC
|
|
`, [id]);
|
|
res.json({
|
|
dispensary,
|
|
productStats: productStats[0],
|
|
snapshotStats: snapshotStats[0],
|
|
recentJobs,
|
|
sampleProducts: {
|
|
inStock: sampleInStock,
|
|
outOfStock: sampleOutOfStock,
|
|
},
|
|
categories,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// LIVE CRAWLER STATUS ROUTES
|
|
// ============================================================
|
|
const job_queue_2 = require("../services/job-queue");
|
|
/**
|
|
* GET /api/dutchie-az/monitor/active-jobs
|
|
* Get all currently running jobs with real-time status including worker info
|
|
*/
|
|
router.get('/monitor/active-jobs', async (_req, res) => {
|
|
try {
|
|
// Get running jobs from job_run_logs (scheduled jobs like "enqueue all")
|
|
const { rows: runningScheduledJobs } = await (0, connection_1.query)(`
|
|
SELECT
|
|
jrl.id,
|
|
jrl.schedule_id,
|
|
jrl.job_name,
|
|
jrl.status,
|
|
jrl.started_at,
|
|
jrl.items_processed,
|
|
jrl.items_succeeded,
|
|
jrl.items_failed,
|
|
jrl.metadata,
|
|
js.description as job_description,
|
|
EXTRACT(EPOCH FROM (NOW() - jrl.started_at)) as duration_seconds
|
|
FROM job_run_logs jrl
|
|
LEFT JOIN job_schedules js ON jrl.schedule_id = js.id
|
|
WHERE jrl.status = 'running'
|
|
ORDER BY jrl.started_at DESC
|
|
`);
|
|
// Get running crawl jobs (individual store crawls with worker info)
|
|
const { rows: runningCrawlJobs } = await (0, connection_1.query)(`
|
|
SELECT
|
|
cj.id,
|
|
cj.job_type,
|
|
cj.dispensary_id,
|
|
d.name as dispensary_name,
|
|
d.city,
|
|
d.platform_dispensary_id,
|
|
cj.status,
|
|
cj.started_at,
|
|
cj.claimed_by as worker_id,
|
|
cj.worker_hostname,
|
|
cj.claimed_at,
|
|
cj.products_found,
|
|
cj.products_upserted,
|
|
cj.snapshots_created,
|
|
cj.current_page,
|
|
cj.total_pages,
|
|
cj.last_heartbeat_at,
|
|
cj.retry_count,
|
|
cj.metadata,
|
|
EXTRACT(EPOCH FROM (NOW() - cj.started_at)) as duration_seconds
|
|
FROM dispensary_crawl_jobs cj
|
|
LEFT JOIN dispensaries d ON cj.dispensary_id = d.id
|
|
WHERE cj.status = 'running'
|
|
ORDER BY cj.started_at DESC
|
|
`);
|
|
// Get queue stats
|
|
const queueStats = await (0, job_queue_2.getQueueStats)();
|
|
// Get active workers
|
|
const activeWorkers = await (0, job_queue_2.getActiveWorkers)();
|
|
// Also get in-memory scrapers if any (from the legacy system)
|
|
let inMemoryScrapers = [];
|
|
try {
|
|
const { activeScrapers } = await Promise.resolve().then(() => __importStar(require('../../routes/scraper-monitor')));
|
|
inMemoryScrapers = Array.from(activeScrapers.values()).map(scraper => ({
|
|
...scraper,
|
|
source: 'in_memory',
|
|
duration_seconds: (Date.now() - scraper.startTime.getTime()) / 1000,
|
|
}));
|
|
}
|
|
catch {
|
|
// Legacy scraper monitor not available
|
|
}
|
|
res.json({
|
|
scheduledJobs: runningScheduledJobs,
|
|
crawlJobs: runningCrawlJobs,
|
|
inMemoryScrapers,
|
|
activeWorkers,
|
|
queueStats,
|
|
totalActive: runningScheduledJobs.length + runningCrawlJobs.length + inMemoryScrapers.length,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/monitor/recent-jobs
|
|
* Get recent completed jobs
|
|
*/
|
|
router.get('/monitor/recent-jobs', async (req, res) => {
|
|
try {
|
|
const { limit = '50' } = req.query;
|
|
const limitNum = Math.min(parseInt(limit, 10), 200);
|
|
// Recent job run logs
|
|
const { rows: recentJobLogs } = await (0, connection_1.query)(`
|
|
SELECT
|
|
jrl.id,
|
|
jrl.schedule_id,
|
|
jrl.job_name,
|
|
jrl.status,
|
|
jrl.started_at,
|
|
jrl.completed_at,
|
|
jrl.duration_ms,
|
|
jrl.error_message,
|
|
jrl.items_processed,
|
|
jrl.items_succeeded,
|
|
jrl.items_failed,
|
|
jrl.metadata,
|
|
js.description as job_description
|
|
FROM job_run_logs jrl
|
|
LEFT JOIN job_schedules js ON jrl.schedule_id = js.id
|
|
ORDER BY jrl.created_at DESC
|
|
LIMIT $1
|
|
`, [limitNum]);
|
|
// Recent crawl jobs
|
|
const { rows: recentCrawlJobs } = await (0, connection_1.query)(`
|
|
SELECT
|
|
cj.id,
|
|
cj.job_type,
|
|
cj.dispensary_id,
|
|
d.name as dispensary_name,
|
|
d.city,
|
|
cj.status,
|
|
cj.started_at,
|
|
cj.completed_at,
|
|
cj.error_message,
|
|
cj.products_found,
|
|
cj.snapshots_created,
|
|
cj.metadata,
|
|
EXTRACT(EPOCH FROM (COALESCE(cj.completed_at, NOW()) - cj.started_at)) * 1000 as duration_ms
|
|
FROM dispensary_crawl_jobs cj
|
|
LEFT JOIN dispensaries d ON cj.dispensary_id = d.id
|
|
ORDER BY cj.created_at DESC
|
|
LIMIT $1
|
|
`, [limitNum]);
|
|
res.json({
|
|
jobLogs: recentJobLogs,
|
|
crawlJobs: recentCrawlJobs,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/monitor/errors
|
|
* Get recent job errors
|
|
*/
|
|
router.get('/monitor/errors', async (req, res) => {
|
|
try {
|
|
const { limit = '20', hours = '24' } = req.query;
|
|
const limitNum = Math.min(parseInt(limit, 10), 100);
|
|
const hoursNum = Math.min(parseInt(hours, 10), 168);
|
|
// Errors from job_run_logs
|
|
const { rows: jobErrors } = await (0, connection_1.query)(`
|
|
SELECT
|
|
'job_run_log' as source,
|
|
jrl.id,
|
|
jrl.job_name,
|
|
jrl.status,
|
|
jrl.started_at,
|
|
jrl.completed_at,
|
|
jrl.error_message,
|
|
jrl.items_processed,
|
|
jrl.items_failed,
|
|
jrl.metadata
|
|
FROM job_run_logs jrl
|
|
WHERE jrl.status IN ('error', 'partial')
|
|
AND jrl.created_at > NOW() - INTERVAL '${hoursNum} hours'
|
|
ORDER BY jrl.created_at DESC
|
|
LIMIT $1
|
|
`, [limitNum]);
|
|
// Errors from dispensary_crawl_jobs
|
|
const { rows: crawlErrors } = await (0, connection_1.query)(`
|
|
SELECT
|
|
'crawl_job' as source,
|
|
cj.id,
|
|
cj.job_type as job_name,
|
|
d.name as dispensary_name,
|
|
cj.status,
|
|
cj.started_at,
|
|
cj.completed_at,
|
|
cj.error_message,
|
|
cj.products_found as items_processed,
|
|
cj.metadata
|
|
FROM dispensary_crawl_jobs cj
|
|
LEFT JOIN dispensaries d ON cj.dispensary_id = d.id
|
|
WHERE cj.status = 'failed'
|
|
AND cj.created_at > NOW() - INTERVAL '${hoursNum} hours'
|
|
ORDER BY cj.created_at DESC
|
|
LIMIT $1
|
|
`, [limitNum]);
|
|
res.json({
|
|
errors: [...jobErrors, ...crawlErrors].sort((a, b) => new Date(b.started_at || b.created_at).getTime() -
|
|
new Date(a.started_at || a.created_at).getTime()).slice(0, limitNum),
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/monitor/summary
|
|
* Get overall monitoring summary
|
|
*/
|
|
router.get('/monitor/summary', async (_req, res) => {
|
|
try {
|
|
const { rows: stats } = await (0, connection_1.query)(`
|
|
SELECT
|
|
(SELECT COUNT(*) FROM job_run_logs WHERE status = 'running') as running_scheduled_jobs,
|
|
(SELECT COUNT(*) FROM dispensary_crawl_jobs WHERE status = 'running') as running_dispensary_crawl_jobs,
|
|
(SELECT COUNT(*) FROM job_run_logs WHERE status = 'success' AND created_at > NOW() - INTERVAL '24 hours') as successful_jobs_24h,
|
|
(SELECT COUNT(*) FROM job_run_logs WHERE status IN ('error', 'partial') AND created_at > NOW() - INTERVAL '24 hours') as failed_jobs_24h,
|
|
(SELECT COUNT(*) FROM dispensary_crawl_jobs WHERE status = 'completed' AND created_at > NOW() - INTERVAL '24 hours') as successful_crawls_24h,
|
|
(SELECT COUNT(*) FROM dispensary_crawl_jobs WHERE status = 'failed' AND created_at > NOW() - INTERVAL '24 hours') as failed_crawls_24h,
|
|
(SELECT SUM(products_found) FROM dispensary_crawl_jobs WHERE status = 'completed' AND created_at > NOW() - INTERVAL '24 hours') as products_found_24h,
|
|
(SELECT SUM(snapshots_created) FROM dispensary_crawl_jobs WHERE status = 'completed' AND created_at > NOW() - INTERVAL '24 hours') as snapshots_created_24h,
|
|
(SELECT MAX(started_at) FROM job_run_logs) as last_job_started,
|
|
(SELECT MAX(completed_at) FROM job_run_logs WHERE status = 'success') as last_job_completed
|
|
`);
|
|
// Get next scheduled runs
|
|
const { rows: nextRuns } = await (0, connection_1.query)(`
|
|
SELECT
|
|
id,
|
|
job_name,
|
|
description,
|
|
enabled,
|
|
next_run_at,
|
|
last_status,
|
|
last_run_at
|
|
FROM job_schedules
|
|
WHERE enabled = true AND next_run_at IS NOT NULL
|
|
ORDER BY next_run_at ASC
|
|
LIMIT 5
|
|
`);
|
|
res.json({
|
|
...(stats[0] || {}),
|
|
nextRuns,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// MENU DETECTION ROUTES
|
|
// ============================================================
|
|
const menu_detection_1 = require("../services/menu-detection");
|
|
/**
|
|
* GET /api/dutchie-az/admin/detection/stats
|
|
* Get menu detection statistics
|
|
*/
|
|
router.get('/admin/detection/stats', async (_req, res) => {
|
|
try {
|
|
const stats = await (0, menu_detection_1.getDetectionStats)();
|
|
res.json(stats);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/detection/pending
|
|
* Get dispensaries that need menu detection
|
|
*/
|
|
router.get('/admin/detection/pending', async (req, res) => {
|
|
try {
|
|
const { state = 'AZ', limit = '100' } = req.query;
|
|
const dispensaries = await (0, menu_detection_1.getDispensariesNeedingDetection)({
|
|
state: state,
|
|
limit: parseInt(limit, 10),
|
|
});
|
|
res.json({ dispensaries, total: dispensaries.length });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/detection/detect/:id
|
|
* Detect menu provider and resolve platform ID for a single dispensary
|
|
*/
|
|
router.post('/admin/detection/detect/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const result = await (0, menu_detection_1.detectAndResolveDispensary)(parseInt(id, 10));
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/detection/detect-all
|
|
* Run bulk menu detection on all dispensaries needing it
|
|
*/
|
|
router.post('/admin/detection/detect-all', async (req, res) => {
|
|
try {
|
|
const { state = 'AZ', onlyUnknown = true, onlyMissingPlatformId = false, limit } = req.body;
|
|
const result = await (0, menu_detection_1.runBulkDetection)({
|
|
state,
|
|
onlyUnknown,
|
|
onlyMissingPlatformId,
|
|
limit,
|
|
});
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/detection/trigger
|
|
* Trigger the menu detection scheduled job immediately
|
|
*/
|
|
router.post('/admin/detection/trigger', async (_req, res) => {
|
|
try {
|
|
// Find the menu detection schedule and trigger it
|
|
const schedules = await (0, scheduler_1.getAllSchedules)();
|
|
const menuDetection = schedules.find(s => s.jobName === 'dutchie_az_menu_detection');
|
|
if (!menuDetection) {
|
|
return res.status(404).json({ error: 'Menu detection schedule not found. Run /admin/schedules/init first.' });
|
|
}
|
|
const result = await (0, scheduler_1.triggerScheduleNow)(menuDetection.id);
|
|
res.json(result);
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
// ============================================================
|
|
// FAILED DISPENSARIES ROUTES
|
|
// ============================================================
|
|
/**
|
|
* GET /api/dutchie-az/admin/dispensaries/failed
|
|
* Get all dispensaries flagged as failed (for admin review)
|
|
*/
|
|
router.get('/admin/dispensaries/failed', async (_req, res) => {
|
|
try {
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
id,
|
|
name,
|
|
city,
|
|
state,
|
|
menu_url,
|
|
menu_type,
|
|
platform_dispensary_id,
|
|
consecutive_failures,
|
|
last_failure_at,
|
|
last_failure_reason,
|
|
failed_at,
|
|
failure_notes,
|
|
last_crawl_at,
|
|
updated_at
|
|
FROM dispensaries
|
|
WHERE failed_at IS NOT NULL
|
|
ORDER BY failed_at DESC
|
|
`);
|
|
res.json({
|
|
failed: rows,
|
|
total: rows.length,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/dispensaries/at-risk
|
|
* Get dispensaries with high failure counts (but not yet flagged as failed)
|
|
*/
|
|
router.get('/admin/dispensaries/at-risk', async (_req, res) => {
|
|
try {
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
id,
|
|
name,
|
|
city,
|
|
state,
|
|
menu_url,
|
|
menu_type,
|
|
consecutive_failures,
|
|
last_failure_at,
|
|
last_failure_reason,
|
|
last_crawl_at
|
|
FROM dispensaries
|
|
WHERE consecutive_failures >= 1
|
|
AND failed_at IS NULL
|
|
ORDER BY consecutive_failures DESC, last_failure_at DESC
|
|
`);
|
|
res.json({
|
|
atRisk: rows,
|
|
total: rows.length,
|
|
});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/dispensaries/:id/unfail
|
|
* Restore a failed dispensary - clears failed status and resets for re-detection
|
|
*/
|
|
router.post('/admin/dispensaries/:id/unfail', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
await (0, connection_1.query)(`
|
|
UPDATE dispensaries
|
|
SET failed_at = NULL,
|
|
consecutive_failures = 0,
|
|
last_failure_at = NULL,
|
|
last_failure_reason = NULL,
|
|
failure_notes = NULL,
|
|
menu_type = NULL,
|
|
platform_dispensary_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = $1
|
|
`, [id]);
|
|
res.json({ success: true, message: `Dispensary ${id} restored for re-detection` });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* POST /api/dutchie-az/admin/dispensaries/:id/reset-failures
|
|
* Reset failure counter for a dispensary (without unflagging)
|
|
*/
|
|
router.post('/admin/dispensaries/:id/reset-failures', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
await (0, connection_1.query)(`
|
|
UPDATE dispensaries
|
|
SET consecutive_failures = 0,
|
|
last_failure_at = NULL,
|
|
last_failure_reason = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = $1
|
|
`, [id]);
|
|
res.json({ success: true, message: `Failure counter reset for dispensary ${id}` });
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
/**
|
|
* GET /api/dutchie-az/admin/dispensaries/health-summary
|
|
* Get a summary of dispensary health status
|
|
*/
|
|
router.get('/admin/dispensaries/health-summary', async (_req, res) => {
|
|
try {
|
|
const { rows } = await (0, connection_1.query)(`
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COUNT(*) FILTER (WHERE state = 'AZ') as arizona_total,
|
|
COUNT(*) FILTER (WHERE failed_at IS NOT NULL) as failed,
|
|
COUNT(*) FILTER (WHERE consecutive_failures >= 1 AND failed_at IS NULL) as at_risk,
|
|
COUNT(*) FILTER (WHERE menu_type = 'dutchie' AND platform_dispensary_id IS NOT NULL AND failed_at IS NULL) as ready_to_crawl,
|
|
COUNT(*) FILTER (WHERE menu_type = 'dutchie' AND failed_at IS NULL) as dutchie_detected,
|
|
COUNT(*) FILTER (WHERE (menu_type IS NULL OR menu_type = 'unknown') AND failed_at IS NULL) as needs_detection,
|
|
COUNT(*) FILTER (WHERE menu_type NOT IN ('dutchie', 'unknown') AND menu_type IS NOT NULL AND failed_at IS NULL) as non_dutchie
|
|
FROM dispensaries
|
|
WHERE state = 'AZ'
|
|
`);
|
|
res.json(rows[0] || {});
|
|
}
|
|
catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
exports.default = router;
|