feat: Add stale process monitor, users route, landing page, archive old scripts
- Add backend stale process monitoring API (/api/stale-processes) - Add users management route - Add frontend landing page and stale process monitor UI on /scraper-tools - Move old development scripts to backend/archive/ - Update frontend build with new features 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Kelly
177
backend/dist/scripts/jars-az-finder.js
vendored
Normal file
177
backend/dist/scripts/jars-az-finder.js
vendored
Normal file
@@ -0,0 +1,177 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const playwright_1 = require("playwright");
|
||||
async function findJarsAzStores() {
|
||||
const browser = await playwright_1.chromium.launch({ headless: true });
|
||||
const page = await browser.newPage();
|
||||
const capturedRetailerIds = [];
|
||||
const allApiCalls = [];
|
||||
// Intercept ALL requests to find retailer IDs
|
||||
page.on('request', (request) => {
|
||||
const url = request.url();
|
||||
// Log Buddy API calls
|
||||
if (url.includes('buddyapi') || url.includes('dutchie') || url.includes('graphql')) {
|
||||
allApiCalls.push(url);
|
||||
const postData = request.postData();
|
||||
if (postData) {
|
||||
// Look for retailerId in various formats
|
||||
const match = postData.match(/retailerId['":\s]+([a-f0-9-]{36})/i);
|
||||
if (match) {
|
||||
capturedRetailerIds.push({ url, retailerId: match[1] });
|
||||
}
|
||||
}
|
||||
// Also check URL params
|
||||
const urlMatch = url.match(/retailerId=([a-f0-9-]{36})/i);
|
||||
if (urlMatch) {
|
||||
capturedRetailerIds.push({ url, retailerId: urlMatch[1] });
|
||||
}
|
||||
}
|
||||
});
|
||||
try {
|
||||
// First, let's try to find the actual Arizona menu URLs
|
||||
console.log('Loading JARS find-a-dispensary page...');
|
||||
await page.goto('https://jarscannabis.com/find-a-dispensary', {
|
||||
waitUntil: 'networkidle',
|
||||
timeout: 30000
|
||||
});
|
||||
await page.waitForTimeout(3000);
|
||||
// Take screenshot
|
||||
await page.screenshot({ path: '/tmp/jars-find-dispensary.png', fullPage: true });
|
||||
console.log('Screenshot saved to /tmp/jars-find-dispensary.png');
|
||||
// Try to find state selector and click Arizona
|
||||
console.log('\nLooking for state selector...');
|
||||
// Try various ways to select Arizona
|
||||
const stateSelectors = [
|
||||
'select[name*="state"]',
|
||||
'[class*="state"] select',
|
||||
'select option[value="AZ"]',
|
||||
'button:has-text("Arizona")',
|
||||
'a:has-text("Arizona")',
|
||||
'[data-state="AZ"]',
|
||||
'div:has-text("Arizona")',
|
||||
];
|
||||
for (const selector of stateSelectors) {
|
||||
try {
|
||||
const element = page.locator(selector).first();
|
||||
const isVisible = await element.isVisible({ timeout: 1000 });
|
||||
if (isVisible) {
|
||||
console.log(`Found element with selector: ${selector}`);
|
||||
await element.click();
|
||||
await page.waitForTimeout(2000);
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
// Continue to next selector
|
||||
}
|
||||
}
|
||||
// Get all links on the page
|
||||
const links = await page.evaluate(() => {
|
||||
return Array.from(document.querySelectorAll('a')).map(a => ({
|
||||
href: a.href,
|
||||
text: a.textContent?.trim()
|
||||
})).filter(l => l.href.includes('/shop') || l.href.includes('menu') || l.href.includes('arizona') || l.href.includes('-az'));
|
||||
});
|
||||
console.log('\n=== Shop/Menu Links Found ===');
|
||||
links.forEach(l => console.log(`${l.text}: ${l.href}`));
|
||||
// Look for __NEXT_DATA__ which might have location data
|
||||
const nextData = await page.evaluate(() => {
|
||||
const el = document.getElementById('__NEXT_DATA__');
|
||||
return el?.textContent || null;
|
||||
});
|
||||
if (nextData) {
|
||||
console.log('\n=== Analyzing __NEXT_DATA__ ===');
|
||||
const data = JSON.parse(nextData);
|
||||
const dataStr = JSON.stringify(data);
|
||||
// Look for Arizona references
|
||||
if (dataStr.includes('Arizona') || dataStr.includes('AZ')) {
|
||||
console.log('Found Arizona references in __NEXT_DATA__');
|
||||
// Extract all objects that might be Arizona stores
|
||||
const findArizonaStores = (obj, path = '') => {
|
||||
const results = [];
|
||||
if (!obj || typeof obj !== 'object')
|
||||
return results;
|
||||
if (Array.isArray(obj)) {
|
||||
obj.forEach((item, i) => {
|
||||
results.push(...findArizonaStores(item, `${path}[${i}]`));
|
||||
});
|
||||
}
|
||||
else {
|
||||
// Check if this object looks like an AZ store
|
||||
if (obj.state === 'AZ' || obj.state === 'Arizona' ||
|
||||
obj.stateCode === 'AZ' || obj.region === 'Arizona' ||
|
||||
(obj.city && ['Mesa', 'Phoenix', 'Peoria', 'Payson', 'Globe', 'Safford', 'Somerton', 'Prescott Valley'].includes(obj.city))) {
|
||||
results.push({ path, data: obj });
|
||||
}
|
||||
for (const key of Object.keys(obj)) {
|
||||
results.push(...findArizonaStores(obj[key], `${path}.${key}`));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
};
|
||||
const azStores = findArizonaStores(data);
|
||||
console.log(`Found ${azStores.length} Arizona store objects`);
|
||||
azStores.forEach(s => {
|
||||
console.log('\n---');
|
||||
console.log('Path:', s.path);
|
||||
console.log(JSON.stringify(s.data, null, 2));
|
||||
});
|
||||
}
|
||||
// Also look for retailer IDs
|
||||
const retailerMatches = dataStr.match(/"retailerId"\s*:\s*"([a-f0-9-]{36})"/gi);
|
||||
if (retailerMatches) {
|
||||
console.log('\n=== RetailerIds in __NEXT_DATA__ ===');
|
||||
const uniqueIds = [...new Set(retailerMatches.map(m => {
|
||||
const match = m.match(/([a-f0-9-]{36})/i);
|
||||
return match ? match[1] : null;
|
||||
}).filter(Boolean))];
|
||||
uniqueIds.forEach(id => console.log(id));
|
||||
}
|
||||
}
|
||||
// Try loading a known store URL pattern
|
||||
const testUrls = [
|
||||
'https://jarscannabis.com/arizona/',
|
||||
'https://jarscannabis.com/az/',
|
||||
'https://jarscannabis.com/stores/arizona/',
|
||||
'https://jarscannabis.com/locations/arizona/',
|
||||
'https://jarscannabis.com/shop/arizona/',
|
||||
'https://az.jarscannabis.com/',
|
||||
];
|
||||
console.log('\n=== Testing Arizona URLs ===');
|
||||
for (const testUrl of testUrls) {
|
||||
try {
|
||||
const response = await page.goto(testUrl, { waitUntil: 'domcontentloaded', timeout: 10000 });
|
||||
const status = response?.status();
|
||||
console.log(`${testUrl}: ${status}`);
|
||||
if (status === 200) {
|
||||
const title = await page.title();
|
||||
console.log(` Title: ${title}`);
|
||||
// If we found a working page, extract store links
|
||||
const storeLinks = await page.evaluate(() => {
|
||||
return Array.from(document.querySelectorAll('a')).map(a => ({
|
||||
href: a.href,
|
||||
text: a.textContent?.trim()
|
||||
})).filter(l => l.href.includes('shop') || l.href.includes('menu'));
|
||||
});
|
||||
if (storeLinks.length > 0) {
|
||||
console.log(' Store links:');
|
||||
storeLinks.forEach(l => console.log(` ${l.text}: ${l.href}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
console.log(`${testUrl}: Error - ${e.message.substring(0, 50)}`);
|
||||
}
|
||||
}
|
||||
console.log('\n=== Captured Retailer IDs from API calls ===');
|
||||
const uniqueRetailerIds = [...new Map(capturedRetailerIds.map(r => [r.retailerId, r])).values()];
|
||||
uniqueRetailerIds.forEach(r => {
|
||||
console.log(`${r.retailerId} (from: ${r.url.substring(0, 60)}...)`);
|
||||
});
|
||||
console.log('\n=== All API calls ===');
|
||||
allApiCalls.forEach(url => console.log(url.substring(0, 100)));
|
||||
}
|
||||
finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
findJarsAzStores().catch(e => console.error('Error:', e.message));
|
||||
Reference in New Issue
Block a user