- Add backend stale process monitoring API (/api/stale-processes) - Add users management route - Add frontend landing page and stale process monitor UI on /scraper-tools - Move old development scripts to backend/archive/ - Update frontend build with new features 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
152 lines
4.7 KiB
TypeScript
152 lines
4.7 KiB
TypeScript
import { firefox } from 'playwright';
|
||
import { getRandomProxy } from './src/utils/proxyManager.js';
|
||
|
||
async function main() {
|
||
const menuUrl = 'https://dutchie.com/embedded-menu/AZ-Deeply-Rooted';
|
||
const specialsUrl = `${menuUrl}/specials`;
|
||
|
||
console.log('🔍 Investigating Specials Page...\n');
|
||
console.log(`URL: ${specialsUrl}\n`);
|
||
|
||
const proxyConfig = await getRandomProxy();
|
||
if (!proxyConfig) {
|
||
throw new Error('No proxy available');
|
||
}
|
||
console.log(`🔐 Using proxy: ${proxyConfig.server}\n`);
|
||
|
||
const browser = await firefox.launch({
|
||
headless: true,
|
||
proxy: proxyConfig
|
||
});
|
||
|
||
const context = await browser.newContext({
|
||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0'
|
||
});
|
||
|
||
const page = await context.newPage();
|
||
|
||
try {
|
||
console.log('📄 Loading specials page...');
|
||
await page.goto(specialsUrl, {
|
||
waitUntil: 'domcontentloaded',
|
||
timeout: 60000
|
||
});
|
||
|
||
await page.waitForTimeout(3000);
|
||
|
||
// Scroll to load content
|
||
for (let i = 0; i < 10; i++) {
|
||
await page.evaluate(() => window.scrollBy(0, window.innerHeight));
|
||
await page.waitForTimeout(1000);
|
||
}
|
||
|
||
console.log('✅ Page loaded\n');
|
||
|
||
// Check for JSON-LD data
|
||
const jsonLdData = await page.evaluate(() => {
|
||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||
return scripts.map(script => {
|
||
try {
|
||
return JSON.parse(script.textContent || '');
|
||
} catch (e) {
|
||
return null;
|
||
}
|
||
}).filter(Boolean);
|
||
});
|
||
|
||
if (jsonLdData.length > 0) {
|
||
console.log('📋 JSON-LD Data Found:');
|
||
console.log(JSON.stringify(jsonLdData, null, 2));
|
||
console.log('\n');
|
||
}
|
||
|
||
// Look for product cards
|
||
const productCards = await page.evaluate(() => {
|
||
const cards = Array.from(document.querySelectorAll('a[href*="/product/"]'));
|
||
return cards.slice(0, 5).map(card => ({
|
||
href: card.getAttribute('href'),
|
||
text: card.textContent?.trim().substring(0, 100)
|
||
}));
|
||
});
|
||
|
||
console.log('🛍️ Product Cards Found:', productCards.length);
|
||
if (productCards.length > 0) {
|
||
console.log('First 5 products:');
|
||
productCards.forEach((card, idx) => {
|
||
console.log(` ${idx + 1}. ${card.href}`);
|
||
console.log(` Text: ${card.text}\n`);
|
||
});
|
||
}
|
||
|
||
// Look for special indicators
|
||
const specialData = await page.evaluate(() => {
|
||
const pageText = document.body.textContent || '';
|
||
|
||
// Look for common special-related keywords
|
||
const hasDiscount = pageText.toLowerCase().includes('discount');
|
||
const hasSale = pageText.toLowerCase().includes('sale');
|
||
const hasOff = pageText.toLowerCase().includes('off');
|
||
const hasDeal = pageText.toLowerCase().includes('deal');
|
||
const hasPromo = pageText.toLowerCase().includes('promo');
|
||
|
||
// Look for percentage or dollar off indicators
|
||
const percentMatches = pageText.match(/(\d+)%\s*off/gi);
|
||
const dollarMatches = pageText.match(/\$(\d+)\s*off/gi);
|
||
|
||
// Try to find any special tags or badges
|
||
const badges = Array.from(document.querySelectorAll('[class*="badge"], [class*="tag"], [class*="special"], [class*="sale"], [class*="discount"]'));
|
||
const badgeTexts = badges.map(b => b.textContent?.trim()).filter(Boolean).slice(0, 10);
|
||
|
||
return {
|
||
keywords: {
|
||
hasDiscount,
|
||
hasSale,
|
||
hasOff,
|
||
hasDeal,
|
||
hasPromo
|
||
},
|
||
percentMatches: percentMatches || [],
|
||
dollarMatches: dollarMatches || [],
|
||
badgeTexts,
|
||
totalBadges: badges.length
|
||
};
|
||
});
|
||
|
||
console.log('\n🏷️ Special Indicators:');
|
||
console.log(JSON.stringify(specialData, null, 2));
|
||
|
||
// Get page title and any heading text
|
||
const pageInfo = await page.evaluate(() => {
|
||
const title = document.title;
|
||
const h1 = document.querySelector('h1')?.textContent?.trim();
|
||
const h2s = Array.from(document.querySelectorAll('h2')).map(h => h.textContent?.trim()).slice(0, 3);
|
||
|
||
return { title, h1, h2s };
|
||
});
|
||
|
||
console.log('\n📰 Page Info:');
|
||
console.log(JSON.stringify(pageInfo, null, 2));
|
||
|
||
// Check if there are any price elements visible
|
||
const priceInfo = await page.evaluate(() => {
|
||
const pageText = document.body.textContent || '';
|
||
const priceMatches = pageText.match(/\$(\d+\.?\d*)/g);
|
||
|
||
return {
|
||
pricesFound: priceMatches?.length || 0,
|
||
samplePrices: priceMatches?.slice(0, 10) || []
|
||
};
|
||
});
|
||
|
||
console.log('\n💰 Price Info:');
|
||
console.log(JSON.stringify(priceInfo, null, 2));
|
||
|
||
} catch (error: any) {
|
||
console.error('❌ Error:', error.message);
|
||
} finally {
|
||
await browser.close();
|
||
}
|
||
}
|
||
|
||
main().catch(console.error);
|