fix(k8s): Update worker deployment to use v2 hydration worker

The old dutchie-az/services/worker.js no longer exists. Workers now use
the hydration pipeline at dist/scripts/run-hydration.js with --loop mode.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-09 15:01:18 -07:00
parent bbe039c868
commit 7f9cf559cf

View File

@@ -1,12 +1,12 @@
# Dutchie AZ Worker Deployment
# These workers poll the job queue and process crawl jobs.
# Scale this deployment to increase crawl throughput.
# Hydration Worker Deployment
# These workers process raw_payloads → canonical tables.
# Scale this deployment to increase hydration throughput.
#
# Architecture:
# - The main 'scraper' deployment runs the API server + scheduler (1 replica)
# - This 'scraper-worker' deployment runs workers that poll and claim jobs (5 replicas)
# - Workers use DB-level locking (FOR UPDATE SKIP LOCKED) to prevent double-crawls
# - Each worker sends heartbeats; stale jobs are recovered automatically
# - This 'scraper-worker' deployment runs hydration workers (5 replicas)
# - Workers use DB-level locking to prevent double-processing
# - Each worker processes payloads in batches with configurable limits
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -27,9 +27,9 @@ spec:
containers:
- name: worker
image: code.cannabrands.app/creationshop/dispensary-scraper:latest
# Run the worker process instead of the main server
# Run the hydration worker in loop mode
command: ["node"]
args: ["dist/dutchie-az/services/worker.js"]
args: ["dist/scripts/run-hydration.js", "--mode=payload", "--loop"]
envFrom:
- configMapRef:
name: scraper-config
@@ -57,9 +57,9 @@ spec:
command:
- /bin/sh
- -c
- "pgrep -f 'worker.js' > /dev/null"
- "pgrep -f 'run-hydration' > /dev/null"
initialDelaySeconds: 10
periodSeconds: 30
failureThreshold: 3
# Graceful shutdown - give workers time to complete current job
# Graceful shutdown - give workers time to complete current batch
terminationGracePeriodSeconds: 60