feat: Add stale process monitor, users route, landing page, archive old scripts

- Add backend stale process monitoring API (/api/stale-processes)
- Add users management route
- Add frontend landing page and stale process monitor UI on /scraper-tools
- Move old development scripts to backend/archive/
- Update frontend build with new features

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-05 04:07:31 -07:00
parent d2d44d2aeb
commit d91c55a344
3115 changed files with 5755 additions and 719 deletions

View File

@@ -0,0 +1,57 @@
import { scrapeCategoryPlaywright } from './src/services/scraper-playwright';
import { pool } from './src/db/migrate';
import { getRandomProxy, getStateProxy, getProxyLocationStats } from './src/utils/proxyManager';
async function testStealthScraper() {
console.log('🧪 Testing stealth scraper on Sol Flower Sun City\n');
const categoryUrl = 'https://dutchie.com/dispensary/sol-flower-dispensary/shop/flower';
const categoryName = 'Flower';
const state = 'Arizona';
console.log(`📂 Category: ${categoryName}`);
console.log(`🔗 URL: ${categoryUrl}`);
console.log(`📍 State: ${state}\n`);
try {
// Show proxy location distribution
console.log('📊 Proxy location distribution:');
const stats = await getProxyLocationStats();
console.table(stats.slice(0, 10));
console.log('');
// Get an active residential proxy from database
console.log('🔍 Fetching active residential proxy...');
const proxy = await getRandomProxy();
if (proxy) {
console.log(`✅ Using proxy: ${proxy.server}`);
console.log(` Credentials: ${proxy.username ? 'Yes' : 'No'}\n`);
} else {
console.log('⚠️ No active proxies found, proceeding without proxy\n');
}
const products = await scrapeCategoryPlaywright(categoryUrl, categoryName, state, proxy);
console.log(`\n✅ Scrape completed!`);
console.log(`📦 Found ${products.length} products\n`);
if (products.length > 0) {
console.log('🎉 SUCCESS! Cloudflare bypass worked!\n');
console.log('First 5 products:');
products.slice(0, 5).forEach((product, i) => {
console.log(` ${i + 1}. ${product.name}`);
if (product.brand) console.log(` Brand: ${product.brand}`);
if (product.price) console.log(` Price: $${product.price}`);
});
} else {
console.log('⚠️ No products found - may need more stealth adjustments');
}
} catch (error) {
console.error('❌ Error:', error);
} finally {
await pool.end();
}
}
testStealthScraper();