From 4e84f30f8b51b9221633314aeec59275d17159c0 Mon Sep 17 00:00:00 2001 From: Kelly Date: Wed, 10 Dec 2025 08:41:14 -0700 Subject: [PATCH] feat: Auto-retry tasks, 403 proxy rotation, task deletion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix 403 handler to rotate BOTH proxy and fingerprint (was only fingerprint) - Add auto-retry logic to task service (retry up to max_retries before failing) - Add error tooltip on task status badge showing retry count and error message - Add DELETE /api/tasks/:id endpoint (only for non-running tasks) - Add delete button to JobQueue task table 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- backend/src/platforms/dutchie/client.ts | 6 ++- backend/src/routes/tasks.ts | 30 +++++++++++++++ backend/src/tasks/task-service.ts | 46 ++++++++++++++++++++-- cannaiq/src/pages/JobQueue.tsx | 51 +++++++++++++++++++++++-- 4 files changed, 123 insertions(+), 10 deletions(-) diff --git a/backend/src/platforms/dutchie/client.ts b/backend/src/platforms/dutchie/client.ts index 4817e476..817b0fd4 100644 --- a/backend/src/platforms/dutchie/client.ts +++ b/backend/src/platforms/dutchie/client.ts @@ -534,7 +534,8 @@ export async function executeGraphQL( } if (response.status === 403 && retryOn403) { - console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`); + console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`); + await rotateProxyOn403('403 Forbidden on GraphQL'); rotateFingerprint(); attempt++; await sleep(1000 * attempt); @@ -617,7 +618,8 @@ export async function fetchPage( } if (response.status === 403 && retryOn403) { - console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`); + console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`); + await rotateProxyOn403('403 Forbidden on page fetch'); rotateFingerprint(); attempt++; await sleep(1000 * attempt); diff --git a/backend/src/routes/tasks.ts b/backend/src/routes/tasks.ts index 6c68e3cc..83c97ed3 100644 --- a/backend/src/routes/tasks.ts +++ b/backend/src/routes/tasks.ts @@ -145,6 +145,36 @@ router.get('/:id', async (req: Request, res: Response) => { } }); +/** + * DELETE /api/tasks/:id + * Delete a specific task by ID + * Only allows deletion of failed, completed, or pending tasks (not running) + */ +router.delete('/:id', async (req: Request, res: Response) => { + try { + const taskId = parseInt(req.params.id, 10); + + // First check if task exists and its status + const task = await taskService.getTask(taskId); + if (!task) { + return res.status(404).json({ error: 'Task not found' }); + } + + // Don't allow deleting running tasks + if (task.status === 'running' || task.status === 'claimed') { + return res.status(400).json({ error: 'Cannot delete a running or claimed task' }); + } + + // Delete the task + await pool.query('DELETE FROM worker_tasks WHERE id = $1', [taskId]); + + res.json({ success: true, message: `Task ${taskId} deleted` }); + } catch (error: unknown) { + console.error('Error deleting task:', error); + res.status(500).json({ error: 'Failed to delete task' }); + } +}); + /** * POST /api/tasks * Create a new task diff --git a/backend/src/tasks/task-service.ts b/backend/src/tasks/task-service.ts index 979e3401..c28c6a76 100644 --- a/backend/src/tasks/task-service.ts +++ b/backend/src/tasks/task-service.ts @@ -206,15 +206,53 @@ class TaskService { } /** - * Mark a task as failed + * Mark a task as failed, with auto-retry if under max_retries + * Returns true if task was re-queued for retry, false if permanently failed */ - async failTask(taskId: number, errorMessage: string): Promise { + async failTask(taskId: number, errorMessage: string): Promise { + // Get current retry state + const result = await pool.query( + `SELECT retry_count, max_retries FROM worker_tasks WHERE id = $1`, + [taskId] + ); + + if (result.rows.length === 0) { + return false; + } + + const { retry_count, max_retries } = result.rows[0]; + const newRetryCount = (retry_count || 0) + 1; + + if (newRetryCount < (max_retries || 3)) { + // Re-queue for retry - reset to pending with incremented retry_count + await pool.query( + `UPDATE worker_tasks + SET status = 'pending', + worker_id = NULL, + claimed_at = NULL, + started_at = NULL, + retry_count = $2, + error_message = $3, + updated_at = NOW() + WHERE id = $1`, + [taskId, newRetryCount, `Retry ${newRetryCount}: ${errorMessage}`] + ); + console.log(`[TaskService] Task ${taskId} queued for retry ${newRetryCount}/${max_retries || 3}`); + return true; + } + + // Max retries exceeded - mark as permanently failed await pool.query( `UPDATE worker_tasks - SET status = 'failed', completed_at = NOW(), error_message = $2 + SET status = 'failed', + completed_at = NOW(), + retry_count = $2, + error_message = $3 WHERE id = $1`, - [taskId, errorMessage] + [taskId, newRetryCount, `Failed after ${newRetryCount} attempts: ${errorMessage}`] ); + console.log(`[TaskService] Task ${taskId} permanently failed after ${newRetryCount} attempts`); + return false; } /** diff --git a/cannaiq/src/pages/JobQueue.tsx b/cannaiq/src/pages/JobQueue.tsx index 3dbdace9..37b6cf3e 100644 --- a/cannaiq/src/pages/JobQueue.tsx +++ b/cannaiq/src/pages/JobQueue.tsx @@ -17,6 +17,7 @@ import { X, Search, Calendar, + Trash2, } from 'lucide-react'; // Worker from registry @@ -61,6 +62,9 @@ interface Task { started_at: string | null; completed_at: string | null; error: string | null; + error_message: string | null; + retry_count: number; + max_retries: number; result: any; created_at: string; updated_at: string; @@ -499,7 +503,7 @@ function WorkerStatusBadge({ status, healthStatus }: { status: string; healthSta ); } -function TaskStatusBadge({ status }: { status: string }) { +function TaskStatusBadge({ status, error, retryCount }: { status: string; error?: string | null; retryCount?: number }) { const config: Record = { pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock }, running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity }, @@ -510,10 +514,25 @@ function TaskStatusBadge({ status }: { status: string }) { const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock }; const Icon = cfg.icon; + // Build tooltip text + let tooltip = ''; + if (error) { + tooltip = error; + } + if (retryCount && retryCount > 0) { + tooltip = `Attempt ${retryCount + 1}${error ? `: ${error}` : ''}`; + } + return ( - + {status} + {retryCount && retryCount > 0 && status !== 'failed' && ( + ({retryCount}) + )} ); } @@ -735,6 +754,18 @@ export function JobQueue() { return () => clearInterval(interval); }, [fetchWorkers]); + // Delete a task + const handleDeleteTask = async (taskId: number) => { + if (!confirm('Delete this task?')) return; + try { + await api.delete(`/api/tasks/${taskId}`); + fetchTasks(); + } catch (err: any) { + console.error('Delete error:', err); + alert(err.response?.data?.error || 'Failed to delete task'); + } + }; + // Get active workers (for display) const activeWorkers = workers.filter(w => w.status !== 'offline' && w.status !== 'terminated'); const busyWorkers = workers.filter(w => w.current_task_id !== null); @@ -910,12 +941,13 @@ export function JobQueue() { Assigned To Created Duration + {tasks.length === 0 ? ( - +

No tasks found

@@ -958,7 +990,7 @@ export function JobQueue() { )} - + {assignedWorker ? ( @@ -986,6 +1018,17 @@ export function JobQueue() { '-' )} + + {(task.status === 'failed' || task.status === 'completed' || task.status === 'pending') && ( + + )} + ); })