fix(worker): Wait for proxies instead of crashing on startup
- Task worker now waits up to 60 minutes for active proxies - Retries every 30 seconds with clear logging - Updated K8s scraper-worker.yaml with Deployment definition - Deployment uses task-worker.js entrypoint with correct liveness probe 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,67 @@
|
||||
# Task Worker Pods
|
||||
# Task Worker Deployment
|
||||
#
|
||||
# Simple Deployment that runs task-worker.js to process tasks from worker_tasks queue.
|
||||
# Workers pull tasks using DB-level locking (FOR UPDATE SKIP LOCKED).
|
||||
#
|
||||
# The worker will wait up to 60 minutes for active proxies to be added before failing.
|
||||
# This allows deployment to succeed even if proxies aren't configured yet.
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: scraper-worker
|
||||
namespace: dispensary-scraper
|
||||
spec:
|
||||
replicas: 5
|
||||
selector:
|
||||
matchLabels:
|
||||
app: scraper-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: scraper-worker
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: regcred
|
||||
containers:
|
||||
- name: worker
|
||||
image: code.cannabrands.app/creationshop/dispensary-scraper:latest
|
||||
command: ["node"]
|
||||
args: ["dist/tasks/task-worker.js"]
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: scraper-config
|
||||
- secretRef:
|
||||
name: scraper-secrets
|
||||
env:
|
||||
- name: WORKER_MODE
|
||||
value: "true"
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- "pgrep -f 'task-worker' > /dev/null"
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
terminationGracePeriodSeconds: 60
|
||||
---
|
||||
# =============================================================================
|
||||
# ALTERNATIVE: StatefulSet with multiple workers per pod (not currently used)
|
||||
# =============================================================================
|
||||
# Task Worker Pods (StatefulSet)
|
||||
# Each pod runs 5 role-agnostic workers that pull tasks from worker_tasks queue.
|
||||
#
|
||||
# Architecture:
|
||||
|
||||
Reference in New Issue
Block a user