fix(k8s): Update worker deployment to use v2 hydration worker

The old dutchie-az/services/worker.js no longer exists. Workers now use
the hydration pipeline at dist/scripts/run-hydration.js with --loop mode.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-09 15:01:18 -07:00
parent bbe039c868
commit 7f9cf559cf

View File

@@ -1,12 +1,12 @@
# Dutchie AZ Worker Deployment # Hydration Worker Deployment
# These workers poll the job queue and process crawl jobs. # These workers process raw_payloads → canonical tables.
# Scale this deployment to increase crawl throughput. # Scale this deployment to increase hydration throughput.
# #
# Architecture: # Architecture:
# - The main 'scraper' deployment runs the API server + scheduler (1 replica) # - The main 'scraper' deployment runs the API server + scheduler (1 replica)
# - This 'scraper-worker' deployment runs workers that poll and claim jobs (5 replicas) # - This 'scraper-worker' deployment runs hydration workers (5 replicas)
# - Workers use DB-level locking (FOR UPDATE SKIP LOCKED) to prevent double-crawls # - Workers use DB-level locking to prevent double-processing
# - Each worker sends heartbeats; stale jobs are recovered automatically # - Each worker processes payloads in batches with configurable limits
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
@@ -27,9 +27,9 @@ spec:
containers: containers:
- name: worker - name: worker
image: code.cannabrands.app/creationshop/dispensary-scraper:latest image: code.cannabrands.app/creationshop/dispensary-scraper:latest
# Run the worker process instead of the main server # Run the hydration worker in loop mode
command: ["node"] command: ["node"]
args: ["dist/dutchie-az/services/worker.js"] args: ["dist/scripts/run-hydration.js", "--mode=payload", "--loop"]
envFrom: envFrom:
- configMapRef: - configMapRef:
name: scraper-config name: scraper-config
@@ -57,9 +57,9 @@ spec:
command: command:
- /bin/sh - /bin/sh
- -c - -c
- "pgrep -f 'worker.js' > /dev/null" - "pgrep -f 'run-hydration' > /dev/null"
initialDelaySeconds: 10 initialDelaySeconds: 10
periodSeconds: 30 periodSeconds: 30
failureThreshold: 3 failureThreshold: 3
# Graceful shutdown - give workers time to complete current job # Graceful shutdown - give workers time to complete current batch
terminationGracePeriodSeconds: 60 terminationGracePeriodSeconds: 60