feat: Auto-retry tasks, 403 proxy rotation, task deletion

- Fix 403 handler to rotate BOTH proxy and fingerprint (was only fingerprint)
- Add auto-retry logic to task service (retry up to max_retries before failing)
- Add error tooltip on task status badge showing retry count and error message
- Add DELETE /api/tasks/:id endpoint (only for non-running tasks)
- Add delete button to JobQueue task table

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-10 08:41:14 -07:00
parent b20a0a4fa5
commit 4e84f30f8b
4 changed files with 123 additions and 10 deletions

View File

@@ -534,7 +534,8 @@ export async function executeGraphQL(
}
if (response.status === 403 && retryOn403) {
console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`);
console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`);
await rotateProxyOn403('403 Forbidden on GraphQL');
rotateFingerprint();
attempt++;
await sleep(1000 * attempt);
@@ -617,7 +618,8 @@ export async function fetchPage(
}
if (response.status === 403 && retryOn403) {
console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`);
console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`);
await rotateProxyOn403('403 Forbidden on page fetch');
rotateFingerprint();
attempt++;
await sleep(1000 * attempt);

View File

@@ -145,6 +145,36 @@ router.get('/:id', async (req: Request, res: Response) => {
}
});
/**
* DELETE /api/tasks/:id
* Delete a specific task by ID
* Only allows deletion of failed, completed, or pending tasks (not running)
*/
router.delete('/:id', async (req: Request, res: Response) => {
try {
const taskId = parseInt(req.params.id, 10);
// First check if task exists and its status
const task = await taskService.getTask(taskId);
if (!task) {
return res.status(404).json({ error: 'Task not found' });
}
// Don't allow deleting running tasks
if (task.status === 'running' || task.status === 'claimed') {
return res.status(400).json({ error: 'Cannot delete a running or claimed task' });
}
// Delete the task
await pool.query('DELETE FROM worker_tasks WHERE id = $1', [taskId]);
res.json({ success: true, message: `Task ${taskId} deleted` });
} catch (error: unknown) {
console.error('Error deleting task:', error);
res.status(500).json({ error: 'Failed to delete task' });
}
});
/**
* POST /api/tasks
* Create a new task

View File

@@ -206,15 +206,53 @@ class TaskService {
}
/**
* Mark a task as failed
* Mark a task as failed, with auto-retry if under max_retries
* Returns true if task was re-queued for retry, false if permanently failed
*/
async failTask(taskId: number, errorMessage: string): Promise<void> {
async failTask(taskId: number, errorMessage: string): Promise<boolean> {
// Get current retry state
const result = await pool.query(
`SELECT retry_count, max_retries FROM worker_tasks WHERE id = $1`,
[taskId]
);
if (result.rows.length === 0) {
return false;
}
const { retry_count, max_retries } = result.rows[0];
const newRetryCount = (retry_count || 0) + 1;
if (newRetryCount < (max_retries || 3)) {
// Re-queue for retry - reset to pending with incremented retry_count
await pool.query(
`UPDATE worker_tasks
SET status = 'pending',
worker_id = NULL,
claimed_at = NULL,
started_at = NULL,
retry_count = $2,
error_message = $3,
updated_at = NOW()
WHERE id = $1`,
[taskId, newRetryCount, `Retry ${newRetryCount}: ${errorMessage}`]
);
console.log(`[TaskService] Task ${taskId} queued for retry ${newRetryCount}/${max_retries || 3}`);
return true;
}
// Max retries exceeded - mark as permanently failed
await pool.query(
`UPDATE worker_tasks
SET status = 'failed', completed_at = NOW(), error_message = $2
SET status = 'failed',
completed_at = NOW(),
retry_count = $2,
error_message = $3
WHERE id = $1`,
[taskId, errorMessage]
[taskId, newRetryCount, `Failed after ${newRetryCount} attempts: ${errorMessage}`]
);
console.log(`[TaskService] Task ${taskId} permanently failed after ${newRetryCount} attempts`);
return false;
}
/**