#!/usr/bin/env npx tsx /** * Test Image Download - Tests image downloading with a small batch of products * * Usage: * DATABASE_URL="postgresql://dutchie:dutchie_local_pass@localhost:54320/dutchie_menus" \ * STORAGE_DRIVER=local STORAGE_BASE_PATH=./storage \ * npx tsx src/scripts/test-image-download.ts [limit] * * Example: * DATABASE_URL="..." npx tsx src/scripts/test-image-download.ts 112 5 */ import { Pool } from 'pg'; import dotenv from 'dotenv'; import { executeGraphQL, startSession, endSession, GRAPHQL_HASHES, } from '../platforms/dutchie'; import { DutchieNormalizer } from '../hydration/normalizers/dutchie'; import { hydrateToCanonical } from '../hydration/canonical-upsert'; import { initializeImageStorage, getStorageStats } from '../utils/image-storage'; dotenv.config(); // ============================================================ // DATABASE CONNECTION // ============================================================ function getConnectionString(): string { if (process.env.DATABASE_URL) { return process.env.DATABASE_URL; } const host = process.env.CANNAIQ_DB_HOST || 'localhost'; const port = process.env.CANNAIQ_DB_PORT || '54320'; const name = process.env.CANNAIQ_DB_NAME || 'dutchie_menus'; const user = process.env.CANNAIQ_DB_USER || 'dutchie'; const pass = process.env.CANNAIQ_DB_PASS || 'dutchie_local_pass'; return `postgresql://${user}:${pass}@${host}:${port}/${name}`; } const pool = new Pool({ connectionString: getConnectionString() }); // ============================================================ // MAIN // ============================================================ async function main() { const dispensaryId = parseInt(process.argv[2], 10); const limit = parseInt(process.argv[3], 10) || 5; if (!dispensaryId) { console.error('Usage: npx tsx src/scripts/test-image-download.ts [limit]'); console.error('Example: npx tsx src/scripts/test-image-download.ts 112 5'); process.exit(1); } console.log(''); console.log('╔════════════════════════════════════════════════════════════╗'); console.log('║ IMAGE DOWNLOAD TEST ║'); console.log('╚════════════════════════════════════════════════════════════╝'); console.log(''); try { // Initialize image storage console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 1: Initialize Image Storage │'); console.log('└─────────────────────────────────────────────────────────────┘'); await initializeImageStorage(); const statsBefore = await getStorageStats(); console.log(` Base path: ${statsBefore.basePath}`); console.log(` Products before: ${statsBefore.productCount}`); console.log(` Brands before: ${statsBefore.brandCount}`); console.log(''); // Get dispensary info console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 2: Load Dispensary Info │'); console.log('└─────────────────────────────────────────────────────────────┘'); const dispResult = await pool.query(` SELECT id, name, platform_dispensary_id, menu_url, state, slug FROM dispensaries WHERE id = $1 `, [dispensaryId]); if (dispResult.rows.length === 0) { throw new Error(`Dispensary ${dispensaryId} not found`); } const disp = dispResult.rows[0]; console.log(` Dispensary: ${disp.name}`); console.log(` State: ${disp.state}`); console.log(` Slug: ${disp.slug}`); console.log(` Platform ID: ${disp.platform_dispensary_id}`); console.log(''); // Delete some existing store_products to force "new" products console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 3: Clear Store Products (to test new product flow) │'); console.log('└─────────────────────────────────────────────────────────────┘'); const deleteResult = await pool.query(` DELETE FROM store_products WHERE dispensary_id = $1 RETURNING id `, [dispensaryId]); console.log(` Deleted ${deleteResult.rowCount} existing store_products`); console.log(''); // Fetch products from Dutchie console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 4: Fetch Products from Dutchie (limited) │'); console.log('└─────────────────────────────────────────────────────────────┘'); const cNameMatch = disp.menu_url?.match(/\/(?:embedded-menu|dispensary)\/([^/?]+)/); const cName = cNameMatch ? cNameMatch[1] : 'dispensary'; const session = startSession(disp.state || 'AZ', 'America/Phoenix'); console.log(` Session ID: ${session.sessionId}`); console.log(` cName: ${cName}`); console.log(` Limit: ${limit} products`); const variables = { includeEnterpriseSpecials: false, productsFilter: { dispensaryId: disp.platform_dispensary_id, pricingType: 'rec', Status: 'Active', types: [], useCache: true, isDefaultSort: true, sortBy: 'popularSortIdx', sortDirection: 1, bypassOnlineThresholds: true, isKioskMenu: false, removeProductsBelowOptionThresholds: false, }, page: 0, perPage: limit, // Only fetch limited products }; const startTime = Date.now(); const result = await executeGraphQL( 'FilteredProducts', variables, GRAPHQL_HASHES.FilteredProducts, { cName, maxRetries: 3 } ); const elapsed = Date.now() - startTime; endSession(); const products = result?.data?.filteredProducts?.products || []; console.log(` Fetched: ${products.length} products in ${elapsed}ms`); // Show products with images console.log(''); console.log(' Products with images:'); for (let i = 0; i < products.length; i++) { const p = products[i]; const hasImage = !!p.Image; const brandName = p.brand?.name || 'Unknown'; console.log(` ${i + 1}. ${p.name?.slice(0, 40).padEnd(42)} | ${brandName.slice(0, 15).padEnd(17)} | ${hasImage ? '✓ has image' : '✗ no image'}`); } console.log(''); // Normalize and hydrate console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 5: Normalize and Hydrate (with image download) │'); console.log('└─────────────────────────────────────────────────────────────┘'); const normalizer = new DutchieNormalizer(); // Wrap products in expected payload format const payload = { raw_json: products, // DutchieNormalizer.extractProducts handles arrays dispensary_id: dispensaryId, }; const normResult = normalizer.normalize(payload); console.log(` Normalized products: ${normResult.products.length}`); console.log(` Brands found: ${normResult.brands.length}`); const hydrateStart = Date.now(); const hydrateResult = await hydrateToCanonical( pool, dispensaryId, normResult, null, // no crawl run ID for test { dryRun: false, downloadImages: true } ); const hydrateElapsed = Date.now() - hydrateStart; console.log(''); console.log(` Hydration time: ${hydrateElapsed}ms`); console.log(` Products new: ${hydrateResult.productsNew}`); console.log(` Products updated: ${hydrateResult.productsUpdated}`); console.log(` Images downloaded: ${hydrateResult.imagesDownloaded}`); console.log(` Images skipped: ${hydrateResult.imagesSkipped}`); console.log(` Images failed: ${hydrateResult.imagesFailed}`); console.log(` Image bytes: ${(hydrateResult.imagesBytesTotal / 1024).toFixed(1)} KB`); console.log(''); // Check storage stats console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 6: Verify Storage │'); console.log('└─────────────────────────────────────────────────────────────┘'); const statsAfter = await getStorageStats(); console.log(` Products after: ${statsAfter.productCount}`); console.log(` Brands after: ${statsAfter.brandCount}`); console.log(` Total size: ${(statsAfter.totalSizeBytes / 1024).toFixed(1)} KB`); console.log(''); // Check database for local_image_path console.log('┌─────────────────────────────────────────────────────────────┐'); console.log('│ STEP 7: Check Database for Local Image Paths │'); console.log('└─────────────────────────────────────────────────────────────┘'); const dbCheck = await pool.query(` SELECT id, name_raw, local_image_path, images FROM store_products WHERE dispensary_id = $1 LIMIT 10 `, [dispensaryId]); for (const row of dbCheck.rows) { const hasLocal = !!row.local_image_path; const hasImages = !!row.images; console.log(` ${row.id}: ${row.name_raw?.slice(0, 40).padEnd(42)} | local: ${hasLocal ? '✓' : '✗'} | images: ${hasImages ? '✓' : '✗'}`); if (row.local_image_path) { console.log(` → ${row.local_image_path}`); } } console.log(''); // Summary console.log('╔════════════════════════════════════════════════════════════╗'); console.log('║ SUMMARY ║'); console.log('╠════════════════════════════════════════════════════════════╣'); console.log(`║ Dispensary: ${disp.name.slice(0, 37).padEnd(37)} ║`); console.log(`║ Products crawled: ${String(products.length).padEnd(37)} ║`); console.log(`║ Images downloaded: ${String(hydrateResult.imagesDownloaded).padEnd(37)} ║`); console.log(`║ Total image bytes: ${((hydrateResult.imagesBytesTotal / 1024).toFixed(1) + ' KB').padEnd(37)} ║`); console.log(`║ Status: ${'SUCCESS'.padEnd(37)} ║`); console.log('╚════════════════════════════════════════════════════════════╝'); } catch (error: any) { console.error(''); console.error('╔════════════════════════════════════════════════════════════╗'); console.error('║ ERROR ║'); console.error('╚════════════════════════════════════════════════════════════╝'); console.error(` ${error.message}`); if (error.stack) { console.error(''); console.error('Stack trace:'); console.error(error.stack.split('\n').slice(0, 5).join('\n')); } process.exit(1); } finally { await pool.end(); } } main();