feat: SEO template library, discovery pipeline, and orchestrator enhancements
## SEO Template Library - Add complete template library with 7 page types (state, city, category, brand, product, search, regeneration) - Add Template Library tab in SEO Orchestrator with accordion-based editors - Add template preview, validation, and variable injection engine - Add API endpoints: /api/seo/templates, preview, validate, generate, regenerate ## Discovery Pipeline - Add promotion.ts for discovery location validation and promotion - Add discover-all-states.ts script for multi-state discovery - Add promotion log migration (067) - Enhance discovery routes and types ## Orchestrator & Admin - Add crawl_enabled filter to stores page - Add API permissions page - Add job queue management - Add price analytics routes - Add markets and intelligence routes - Enhance dashboard and worker monitoring ## Infrastructure - Add migrations for worker definitions, SEO settings, field alignment - Add canonical pipeline for scraper v2 - Update hydration and sync orchestrator - Enhance multi-state query service 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
173
backend/src/scripts/estimate-bandwidth.ts
Normal file
173
backend/src/scripts/estimate-bandwidth.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
import axios from 'axios';
|
||||
import { Pool } from 'pg';
|
||||
|
||||
const DUTCHIE_GRAPHQL_URL = 'https://dutchie.com/graphql';
|
||||
|
||||
const MENU_PRODUCTS_QUERY = `
|
||||
query FilteredProducts($productsFilter: ProductFilterInput!) {
|
||||
filteredProducts(productsFilter: $productsFilter) {
|
||||
products {
|
||||
id
|
||||
name
|
||||
brand
|
||||
category
|
||||
subcategory
|
||||
strainType
|
||||
description
|
||||
image
|
||||
images {
|
||||
id
|
||||
url
|
||||
}
|
||||
posId
|
||||
potencyCbd {
|
||||
formatted
|
||||
range
|
||||
unit
|
||||
}
|
||||
potencyThc {
|
||||
formatted
|
||||
range
|
||||
unit
|
||||
}
|
||||
variants {
|
||||
id
|
||||
option
|
||||
price
|
||||
priceMed
|
||||
priceRec
|
||||
quantity
|
||||
specialPrice
|
||||
}
|
||||
status
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
||||
}
|
||||
|
||||
async function measureRequest(dispensaryId: string, mode: 'A' | 'B') {
|
||||
const variables: any = {
|
||||
productsFilter: {
|
||||
dispensaryId,
|
||||
pricingType: 'rec',
|
||||
Status: mode === 'A' ? 'Active' : null,
|
||||
}
|
||||
};
|
||||
|
||||
const requestBody = JSON.stringify({
|
||||
query: MENU_PRODUCTS_QUERY,
|
||||
variables,
|
||||
});
|
||||
|
||||
const requestSize = Buffer.byteLength(requestBody, 'utf8');
|
||||
|
||||
try {
|
||||
const response = await axios.post(DUTCHIE_GRAPHQL_URL, requestBody, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Origin': 'https://dutchie.com',
|
||||
},
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
const responseSize = Buffer.byteLength(JSON.stringify(response.data), 'utf8');
|
||||
const productCount = response.data?.data?.filteredProducts?.products?.length || 0;
|
||||
|
||||
// Debug: show what we got
|
||||
if (productCount === 0) {
|
||||
console.log(` Response preview: ${JSON.stringify(response.data).slice(0, 300)}...`);
|
||||
}
|
||||
|
||||
return { requestSize, responseSize, productCount };
|
||||
} catch (error: any) {
|
||||
console.error(` Error: ${error.message}`);
|
||||
if (error.response) {
|
||||
console.error(` Status: ${error.response.status}`);
|
||||
console.error(` Data: ${JSON.stringify(error.response.data).slice(0, 200)}`);
|
||||
}
|
||||
return { requestSize, responseSize: 0, productCount: 0, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
||||
|
||||
// Get one store with products (use a known good ID)
|
||||
const { rows } = await pool.query(`
|
||||
SELECT d.platform_dispensary_id, d.name, COUNT(sp.id) as product_count
|
||||
FROM dispensaries d
|
||||
LEFT JOIN store_products sp ON d.id = sp.dispensary_id
|
||||
WHERE d.platform_dispensary_id IS NOT NULL
|
||||
GROUP BY d.id
|
||||
ORDER BY product_count DESC
|
||||
LIMIT 1
|
||||
`);
|
||||
|
||||
if (rows.length === 0) {
|
||||
console.log('No crawlable stores found');
|
||||
await pool.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const store = rows[0];
|
||||
console.log('=== Dutchie GraphQL Bandwidth for One Store ===\n');
|
||||
console.log(`Store: ${store.name}`);
|
||||
console.log(`Platform ID: ${store.platform_dispensary_id}`);
|
||||
console.log(`Products in DB: ${store.product_count || 'unknown'}\n`);
|
||||
|
||||
// Mode A (Active products with pricing)
|
||||
console.log('Fetching Mode A (Active products)...');
|
||||
const modeA = await measureRequest(store.platform_dispensary_id, 'A');
|
||||
|
||||
// Mode B (All products)
|
||||
console.log('Fetching Mode B (All products)...');
|
||||
const modeB = await measureRequest(store.platform_dispensary_id, 'B');
|
||||
|
||||
console.log('\n=== Results for ONE STORE ===');
|
||||
console.log('\nMode A (Active products with pricing):');
|
||||
console.log(` Request size: ${formatBytes(modeA.requestSize)}`);
|
||||
console.log(` Response size: ${formatBytes(modeA.responseSize)}`);
|
||||
console.log(` Products: ${modeA.productCount}`);
|
||||
if (modeA.productCount > 0) {
|
||||
console.log(` Per product: ${formatBytes(modeA.responseSize / modeA.productCount)}`);
|
||||
}
|
||||
|
||||
console.log('\nMode B (All products incl. OOS):');
|
||||
console.log(` Request size: ${formatBytes(modeB.requestSize)}`);
|
||||
console.log(` Response size: ${formatBytes(modeB.responseSize)}`);
|
||||
console.log(` Products: ${modeB.productCount}`);
|
||||
if (modeB.productCount > 0) {
|
||||
console.log(` Per product: ${formatBytes(modeB.responseSize / modeB.productCount)}`);
|
||||
}
|
||||
|
||||
console.log('\nDual-Mode Crawl (what we actually do):');
|
||||
const totalRequest = modeA.requestSize + modeB.requestSize;
|
||||
const totalResponse = modeA.responseSize + modeB.responseSize;
|
||||
const totalBandwidth = totalRequest + totalResponse;
|
||||
console.log(` Total request: ${formatBytes(totalRequest)}`);
|
||||
console.log(` Total response: ${formatBytes(totalResponse)}`);
|
||||
console.log(` TOTAL BANDWIDTH: ${formatBytes(totalBandwidth)}`);
|
||||
|
||||
// Per-product average
|
||||
const avgProducts = Math.max(modeA.productCount, modeB.productCount);
|
||||
const bytesPerProduct = avgProducts > 0 ? totalResponse / avgProducts : 0;
|
||||
|
||||
console.log('\n=== Quick Reference ===');
|
||||
console.log(`Average bytes per product: ~${formatBytes(bytesPerProduct)}`);
|
||||
console.log(`\nTypical store sizes:`);
|
||||
console.log(` Small (100 products): ~${formatBytes(bytesPerProduct * 100 + totalRequest)}`);
|
||||
console.log(` Medium (300 products): ~${formatBytes(bytesPerProduct * 300 + totalRequest)}`);
|
||||
console.log(` Large (500 products): ~${formatBytes(bytesPerProduct * 500 + totalRequest)}`);
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
Reference in New Issue
Block a user