- Add backend stale process monitoring API (/api/stale-processes) - Add users management route - Add frontend landing page and stale process monitor UI on /scraper-tools - Move old development scripts to backend/archive/ - Update frontend build with new features 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
104 lines
3.6 KiB
TypeScript
104 lines
3.6 KiB
TypeScript
import { chromium } from 'playwright';
|
|
|
|
const GOOGLE_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
|
|
async function main() {
|
|
const browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({
|
|
userAgent: GOOGLE_UA
|
|
});
|
|
|
|
const page = await context.newPage();
|
|
|
|
try {
|
|
console.log('Loading menu page...');
|
|
await page.goto('https://best.treez.io/onlinemenu/?customerType=ADULT', {
|
|
waitUntil: 'networkidle',
|
|
timeout: 30000
|
|
});
|
|
|
|
await page.waitForTimeout(3000);
|
|
|
|
// Check for any category filters or tabs
|
|
const categories = await page.locator('[class*="category"], [class*="filter"], [class*="tab"]').all();
|
|
console.log(`\nFound ${categories.length} potential category/filter elements`);
|
|
|
|
// Check if there's a "Show All" or total count indicator
|
|
const pageText = await page.textContent('body');
|
|
const totalMatch = pageText?.match(/(\d+)\s*(products?|items?)/i);
|
|
if (totalMatch) {
|
|
console.log(`Found total indicator: ${totalMatch[0]}`);
|
|
}
|
|
|
|
// Aggressive scrolling - try 100 times
|
|
console.log('\nStarting aggressive scrolling (100 attempts max)...');
|
|
let previousCount = 0;
|
|
let currentCount = 0;
|
|
let scrollAttempts = 0;
|
|
let unchangedCount = 0;
|
|
const maxScrollAttempts = 100;
|
|
|
|
do {
|
|
previousCount = currentCount;
|
|
|
|
// Scroll to bottom
|
|
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
|
await page.waitForTimeout(1500);
|
|
|
|
// Also try scrolling by a large pixel amount in case body height doesn't update
|
|
await page.evaluate(() => window.scrollBy(0, 5000));
|
|
await page.waitForTimeout(1500);
|
|
|
|
currentCount = await page.locator('.menu-item').count();
|
|
scrollAttempts++;
|
|
|
|
if (currentCount === previousCount) {
|
|
unchangedCount++;
|
|
} else {
|
|
unchangedCount = 0;
|
|
console.log(`Scroll ${scrollAttempts}: ${currentCount} items loaded (+${currentCount - previousCount})`);
|
|
}
|
|
|
|
// Stop if count hasn't changed for 5 consecutive scrolls
|
|
if (unchangedCount >= 5) {
|
|
console.log(`No new items after ${unchangedCount} consecutive scrolls`);
|
|
break;
|
|
}
|
|
|
|
} while (scrollAttempts < maxScrollAttempts);
|
|
|
|
console.log(`\n=== FINAL RESULTS ===`);
|
|
console.log(`Total scroll attempts: ${scrollAttempts}`);
|
|
console.log(`Total items found: ${currentCount}`);
|
|
|
|
// Check if there are any "Load More" buttons
|
|
const loadMoreButtons = await page.locator('button, a').filter({ hasText: /load more|show more|view more/i }).all();
|
|
console.log(`Load More buttons found: ${loadMoreButtons.length}`);
|
|
|
|
// Check for pagination
|
|
const paginationElements = await page.locator('[class*="pagination"], [class*="page"]').all();
|
|
console.log(`Pagination elements found: ${paginationElements.length}`);
|
|
|
|
// Get scroll position info
|
|
const scrollInfo = await page.evaluate(() => ({
|
|
scrollHeight: document.body.scrollHeight,
|
|
scrollTop: window.scrollY,
|
|
clientHeight: window.innerHeight,
|
|
atBottom: (window.innerHeight + window.scrollY) >= document.body.scrollHeight - 100
|
|
}));
|
|
|
|
console.log(`\nScroll position:`);
|
|
console.log(`- Total height: ${scrollInfo.scrollHeight}px`);
|
|
console.log(`- Current scroll: ${scrollInfo.scrollTop}px`);
|
|
console.log(`- Viewport height: ${scrollInfo.clientHeight}px`);
|
|
console.log(`- At bottom: ${scrollInfo.atBottom}`);
|
|
|
|
} catch (error: any) {
|
|
console.error('Error:', error.message);
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
main().catch(console.error);
|