feat(jane): Direct Algolia product fetch and multi-platform product-refresh
- Add fetchProductsByStoreIdDirect() for reliable Algolia product fetching - Update product-discovery-jane to use direct Algolia instead of network interception - Fix product-refresh handler to support both Dutchie and Jane payloads - Handle both `products` (Dutchie) and `hits` (Jane) formats - Use platform-appropriate raw_json structure for normalizers - Fix consecutive_misses tracking to use correct provider - Extract product IDs correctly (Dutchie _id vs Jane product_id) - Add store discovery deduplication (prefer REC over MED at same location) - Add storeTypes field to DiscoveredStore interface - Add scripts: run-jane-store-discovery.ts, run-jane-product-discovery.ts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
138
backend/scripts/run-jane-product-discovery.ts
Normal file
138
backend/scripts/run-jane-product-discovery.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Run Jane product discovery for stores in database
|
||||
* Usage: npx ts-node scripts/run-jane-product-discovery.ts [DISPENSARY_ID]
|
||||
* Example: npx ts-node scripts/run-jane-product-discovery.ts 4220
|
||||
* Or run for all Jane stores: npx ts-node scripts/run-jane-product-discovery.ts all
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
import { fetchProductsByStoreIdDirect } from '../src/platforms/jane';
|
||||
import { saveRawPayload } from '../src/utils/payload-storage';
|
||||
|
||||
async function main() {
|
||||
const arg = process.argv[2];
|
||||
|
||||
console.log('='.repeat(60));
|
||||
console.log('Jane Product Discovery');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const pool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
try {
|
||||
// Get dispensaries to process
|
||||
let dispensaries: any[];
|
||||
|
||||
if (arg === 'all') {
|
||||
const result = await pool.query(
|
||||
`SELECT id, name, menu_url, platform_dispensary_id
|
||||
FROM dispensaries
|
||||
WHERE platform = 'jane' AND menu_url IS NOT NULL
|
||||
ORDER BY id`
|
||||
);
|
||||
dispensaries = result.rows;
|
||||
} else if (arg) {
|
||||
const result = await pool.query(
|
||||
`SELECT id, name, menu_url, platform_dispensary_id
|
||||
FROM dispensaries
|
||||
WHERE id = $1`,
|
||||
[parseInt(arg)]
|
||||
);
|
||||
dispensaries = result.rows;
|
||||
} else {
|
||||
// Default: get first Jane store
|
||||
const result = await pool.query(
|
||||
`SELECT id, name, menu_url, platform_dispensary_id
|
||||
FROM dispensaries
|
||||
WHERE platform = 'jane' AND menu_url IS NOT NULL
|
||||
ORDER BY id LIMIT 1`
|
||||
);
|
||||
dispensaries = result.rows;
|
||||
}
|
||||
|
||||
if (dispensaries.length === 0) {
|
||||
console.log('No Jane dispensaries found');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Processing ${dispensaries.length} dispensary(ies)...\n`);
|
||||
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
|
||||
for (const disp of dispensaries) {
|
||||
console.log(`\n${'─'.repeat(60)}`);
|
||||
console.log(`${disp.name} (ID: ${disp.id}, Jane ID: ${disp.platform_dispensary_id})`);
|
||||
console.log('─'.repeat(60));
|
||||
|
||||
try {
|
||||
const result = await fetchProductsByStoreIdDirect(disp.platform_dispensary_id);
|
||||
|
||||
if (result.products.length === 0) {
|
||||
console.log(' ✗ No products captured');
|
||||
failCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(` ✓ Captured ${result.products.length} products`);
|
||||
|
||||
// Build payload
|
||||
const rawPayload = {
|
||||
hits: result.products.map(p => p.raw),
|
||||
store: result.store?.raw || null,
|
||||
capturedAt: new Date().toISOString(),
|
||||
platform: 'jane',
|
||||
dispensaryId: disp.id,
|
||||
storeId: disp.platform_dispensary_id,
|
||||
};
|
||||
|
||||
// Save payload
|
||||
const { id: payloadId, sizeBytes } = await saveRawPayload(
|
||||
pool,
|
||||
disp.id,
|
||||
rawPayload,
|
||||
null,
|
||||
result.products.length,
|
||||
'jane'
|
||||
);
|
||||
|
||||
console.log(` ✓ Saved payload ${payloadId} (${Math.round(sizeBytes / 1024)}KB)`);
|
||||
|
||||
// Update dispensary
|
||||
await pool.query(
|
||||
`UPDATE dispensaries
|
||||
SET stage = 'hydrating',
|
||||
last_fetch_at = NOW(),
|
||||
product_count = $2,
|
||||
consecutive_successes = consecutive_successes + 1,
|
||||
consecutive_failures = 0,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[disp.id, result.products.length]
|
||||
);
|
||||
|
||||
console.log(` ✓ Updated dispensary (product_count: ${result.products.length})`);
|
||||
successCount++;
|
||||
|
||||
} catch (error: any) {
|
||||
console.log(` ✗ Error: ${error.message}`);
|
||||
failCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('RESULTS');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Success: ${successCount}`);
|
||||
console.log(`Failed: ${failCount}`);
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('Error:', error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
137
backend/scripts/run-jane-store-discovery.ts
Normal file
137
backend/scripts/run-jane-store-discovery.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* Run Jane store discovery and insert into database
|
||||
* Usage: npx ts-node scripts/run-jane-store-discovery.ts [STATE_CODE]
|
||||
* Example: npx ts-node scripts/run-jane-store-discovery.ts AZ
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
import { discoverStoresByState } from '../src/platforms/jane';
|
||||
|
||||
/**
|
||||
* Generate slug from store name
|
||||
* e.g., "Hana Meds - Phoenix (REC)" -> "hana-meds-phoenix-rec"
|
||||
*/
|
||||
function generateSlug(name: string): string {
|
||||
return name
|
||||
.toLowerCase()
|
||||
.replace(/[()]/g, '') // Remove parentheses
|
||||
.replace(/[^a-z0-9\s-]/g, '') // Remove special chars
|
||||
.replace(/\s+/g, '-') // Spaces to hyphens
|
||||
.replace(/-+/g, '-') // Collapse multiple hyphens
|
||||
.replace(/^-|-$/g, ''); // Trim hyphens
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const stateCode = process.argv[2] || 'AZ';
|
||||
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Jane Store Discovery - ${stateCode}`);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
// Connect to database
|
||||
const pool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
try {
|
||||
// Test connection
|
||||
const testResult = await pool.query('SELECT COUNT(*) FROM dispensaries WHERE platform = $1', ['jane']);
|
||||
console.log(`Current Jane stores in DB: ${testResult.rows[0].count}`);
|
||||
|
||||
// Discover stores
|
||||
console.log(`\nDiscovering Jane stores in ${stateCode}...`);
|
||||
const stores = await discoverStoresByState(stateCode);
|
||||
|
||||
if (stores.length === 0) {
|
||||
console.log(`No stores found in ${stateCode}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\nFound ${stores.length} stores. Inserting into database...`);
|
||||
|
||||
// Insert stores
|
||||
let inserted = 0;
|
||||
let updated = 0;
|
||||
const newIds: number[] = [];
|
||||
|
||||
for (const store of stores) {
|
||||
const menuUrl = `https://www.iheartjane.com/stores/${store.storeId}/${store.urlSlug || 'menu'}`;
|
||||
const slug = generateSlug(store.name);
|
||||
|
||||
try {
|
||||
const result = await pool.query(
|
||||
`INSERT INTO dispensaries (
|
||||
name, slug, address1, city, state, zipcode,
|
||||
latitude, longitude, menu_url, menu_type, platform,
|
||||
platform_dispensary_id, is_medical, is_recreational,
|
||||
stage, created_at, updated_at
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), NOW())
|
||||
ON CONFLICT (platform_dispensary_id) WHERE platform_dispensary_id IS NOT NULL
|
||||
DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
slug = EXCLUDED.slug,
|
||||
address1 = EXCLUDED.address1,
|
||||
city = EXCLUDED.city,
|
||||
latitude = EXCLUDED.latitude,
|
||||
longitude = EXCLUDED.longitude,
|
||||
menu_url = EXCLUDED.menu_url,
|
||||
is_medical = EXCLUDED.is_medical,
|
||||
is_recreational = EXCLUDED.is_recreational,
|
||||
updated_at = NOW()
|
||||
RETURNING id, (xmax = 0) AS is_new`,
|
||||
[
|
||||
store.name,
|
||||
slug,
|
||||
store.address,
|
||||
store.city,
|
||||
stateCode,
|
||||
store.zip,
|
||||
store.lat,
|
||||
store.long,
|
||||
menuUrl,
|
||||
'embedded', // menu_type: how it's displayed
|
||||
'jane', // platform: who provides the menu
|
||||
store.storeId,
|
||||
store.medical,
|
||||
store.recreational,
|
||||
'discovered',
|
||||
]
|
||||
);
|
||||
|
||||
if (result.rows.length > 0) {
|
||||
const { id, is_new } = result.rows[0];
|
||||
if (is_new) {
|
||||
inserted++;
|
||||
newIds.push(id);
|
||||
console.log(` + Inserted: ${store.name} (DB ID: ${id}, Jane ID: ${store.storeId})`);
|
||||
} else {
|
||||
updated++;
|
||||
console.log(` ~ Updated: ${store.name} (DB ID: ${id})`);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error(` ! Error inserting ${store.name}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('RESULTS');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Stores discovered: ${stores.length}`);
|
||||
console.log(`New stores inserted: ${inserted}`);
|
||||
console.log(`Existing stores updated: ${updated}`);
|
||||
console.log(`New dispensary IDs: ${newIds.join(', ') || '(none)'}`);
|
||||
|
||||
// Show final count
|
||||
const finalResult = await pool.query('SELECT COUNT(*) FROM dispensaries WHERE platform = $1', ['jane']);
|
||||
console.log(`\nTotal Jane stores in DB: ${finalResult.rows[0].count}`);
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('Error:', error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
50
backend/scripts/test-jane-discovery-az.ts
Normal file
50
backend/scripts/test-jane-discovery-az.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Smoke test: Discover Jane stores in Arizona
|
||||
* Usage: npx ts-node scripts/test-jane-discovery-az.ts
|
||||
*/
|
||||
|
||||
import { discoverStoresByState } from '../src/platforms/jane';
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Jane Store Discovery - Arizona Smoke Test');
|
||||
console.log('='.repeat(60));
|
||||
console.log('Using local IP (no proxy)\n');
|
||||
|
||||
try {
|
||||
const stores = await discoverStoresByState('AZ');
|
||||
|
||||
console.log(`\n${'='.repeat(60)}`);
|
||||
console.log(`RESULTS: Found ${stores.length} Jane stores in Arizona`);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
if (stores.length > 0) {
|
||||
console.log('\nSample stores:');
|
||||
for (const store of stores.slice(0, 10)) {
|
||||
console.log(` - ${store.name}`);
|
||||
console.log(` ID: ${store.storeId} | ${store.city}, AZ`);
|
||||
console.log(` Types: ${store.storeTypes?.join(', ') || 'unknown'}`);
|
||||
console.log(` Products: ${store.productCount || 'N/A'}`);
|
||||
console.log('');
|
||||
}
|
||||
|
||||
if (stores.length > 10) {
|
||||
console.log(` ... and ${stores.length - 10} more stores`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('SMOKE TEST PASSED');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
} catch (error: any) {
|
||||
console.error('\n' + '='.repeat(60));
|
||||
console.error('SMOKE TEST FAILED');
|
||||
console.error('='.repeat(60));
|
||||
console.error(`Error: ${error.message}`);
|
||||
console.error(error.stack);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user