feat: Add stale process monitor, users route, landing page, archive old scripts
- Add backend stale process monitoring API (/api/stale-processes) - Add users management route - Add frontend landing page and stale process monitor UI on /scraper-tools - Move old development scripts to backend/archive/ - Update frontend build with new features 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
50
backend/archive/launch-25-scrapers.sh
Executable file
50
backend/archive/launch-25-scrapers.sh
Executable file
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Launch 25 parallel brand scrapers for Deeply Rooted (dispensary ID 112)
|
||||
# 90 brands total:
|
||||
# - Workers 1-15: 4 brands each (brands 0-59)
|
||||
# - Workers 16-25: 3 brands each (brands 60-89)
|
||||
|
||||
DISPENSARY_ID=112
|
||||
DB_URL="postgresql://dutchie:dutchie_local_pass@localhost:54320/dutchie_menus"
|
||||
|
||||
echo "🚀 Launching 25 parallel brand scrapers..."
|
||||
echo "============================================================"
|
||||
|
||||
# Workers 1-15: 4 brands each
|
||||
for i in {1..15}; do
|
||||
START=$(( (i-1) * 4 ))
|
||||
END=$(( START + 3 ))
|
||||
echo "Starting Worker $i: brands $START-$END"
|
||||
DATABASE_URL="$DB_URL" npx tsx scrape-parallel-brands.ts $DISPENSARY_ID $START $END "W$i" 2>&1 | tee /tmp/scraper-w$i.log &
|
||||
done
|
||||
|
||||
# Workers 16-25: 3 brands each
|
||||
for i in {16..25}; do
|
||||
START=$(( 60 + (i-16) * 3 ))
|
||||
END=$(( START + 2 ))
|
||||
echo "Starting Worker $i: brands $START-$END"
|
||||
DATABASE_URL="$DB_URL" npx tsx scrape-parallel-brands.ts $DISPENSARY_ID $START $END "W$i" 2>&1 | tee /tmp/scraper-w$i.log &
|
||||
done
|
||||
|
||||
echo "============================================================"
|
||||
echo "✅ All 25 workers launched!"
|
||||
echo ""
|
||||
echo "Monitor progress with:"
|
||||
echo " tail -f /tmp/scraper-w1.log (or w2, w3, etc.)"
|
||||
echo " ps aux | grep scrape-parallel"
|
||||
echo ""
|
||||
echo "View all logs:"
|
||||
echo " tail -f /tmp/scraper-w*.log"
|
||||
echo ""
|
||||
echo "Brand allocation:"
|
||||
echo " Workers 1-15: 4 brands each (brands 0-59)"
|
||||
echo " Workers 16-25: 3 brands each (brands 60-89)"
|
||||
echo " Total: 90 brands across 25 workers"
|
||||
echo "============================================================"
|
||||
|
||||
# Wait for all background jobs to finish
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo "✅ All 25 workers completed!"
|
||||
Reference in New Issue
Block a user