#!/bin/bash # Launch 25 parallel brand scrapers for Deeply Rooted (dispensary ID 112) # 90 brands total: # - Workers 1-15: 4 brands each (brands 0-59) # - Workers 16-25: 3 brands each (brands 60-89) DISPENSARY_ID=112 DB_URL="postgresql://dutchie:dutchie_local_pass@localhost:54320/dutchie_menus" echo "🚀 Launching 25 parallel brand scrapers..." echo "============================================================" # Workers 1-15: 4 brands each for i in {1..15}; do START=$(( (i-1) * 4 )) END=$(( START + 3 )) echo "Starting Worker $i: brands $START-$END" DATABASE_URL="$DB_URL" npx tsx scrape-parallel-brands.ts $DISPENSARY_ID $START $END "W$i" 2>&1 | tee /tmp/scraper-w$i.log & done # Workers 16-25: 3 brands each for i in {16..25}; do START=$(( 60 + (i-16) * 3 )) END=$(( START + 2 )) echo "Starting Worker $i: brands $START-$END" DATABASE_URL="$DB_URL" npx tsx scrape-parallel-brands.ts $DISPENSARY_ID $START $END "W$i" 2>&1 | tee /tmp/scraper-w$i.log & done echo "============================================================" echo "✅ All 25 workers launched!" echo "" echo "Monitor progress with:" echo " tail -f /tmp/scraper-w1.log (or w2, w3, etc.)" echo " ps aux | grep scrape-parallel" echo "" echo "View all logs:" echo " tail -f /tmp/scraper-w*.log" echo "" echo "Brand allocation:" echo " Workers 1-15: 4 brands each (brands 0-59)" echo " Workers 16-25: 3 brands each (brands 60-89)" echo " Total: 90 brands across 25 workers" echo "============================================================" # Wait for all background jobs to finish wait echo "" echo "✅ All 25 workers completed!"