Compare commits
5 Commits
feat/steal
...
feat/auto-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aea93bc96b | ||
|
|
4e84f30f8b | ||
|
|
b20a0a4fa5 | ||
|
|
6eb1babc86 | ||
|
|
9a9c2f76a2 |
@@ -129,7 +129,6 @@ import { createStatesRouter } from './routes/states';
|
|||||||
import { createAnalyticsV2Router } from './routes/analytics-v2';
|
import { createAnalyticsV2Router } from './routes/analytics-v2';
|
||||||
import { createDiscoveryRoutes } from './discovery';
|
import { createDiscoveryRoutes } from './discovery';
|
||||||
import pipelineRoutes from './routes/pipeline';
|
import pipelineRoutes from './routes/pipeline';
|
||||||
import { getPool } from './db/pool';
|
|
||||||
|
|
||||||
// Consumer API routes (findadispo.com, findagram.co)
|
// Consumer API routes (findadispo.com, findagram.co)
|
||||||
import consumerAuthRoutes from './routes/consumer-auth';
|
import consumerAuthRoutes from './routes/consumer-auth';
|
||||||
|
|||||||
@@ -534,7 +534,8 @@ export async function executeGraphQL(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (response.status === 403 && retryOn403) {
|
if (response.status === 403 && retryOn403) {
|
||||||
console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`);
|
console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`);
|
||||||
|
await rotateProxyOn403('403 Forbidden on GraphQL');
|
||||||
rotateFingerprint();
|
rotateFingerprint();
|
||||||
attempt++;
|
attempt++;
|
||||||
await sleep(1000 * attempt);
|
await sleep(1000 * attempt);
|
||||||
@@ -617,7 +618,8 @@ export async function fetchPage(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (response.status === 403 && retryOn403) {
|
if (response.status === 403 && retryOn403) {
|
||||||
console.warn(`[Dutchie Client] 403 blocked - rotating fingerprint...`);
|
console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`);
|
||||||
|
await rotateProxyOn403('403 Forbidden on page fetch');
|
||||||
rotateFingerprint();
|
rotateFingerprint();
|
||||||
attempt++;
|
attempt++;
|
||||||
await sleep(1000 * attempt);
|
await sleep(1000 * attempt);
|
||||||
|
|||||||
@@ -145,6 +145,36 @@ router.get('/:id', async (req: Request, res: Response) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DELETE /api/tasks/:id
|
||||||
|
* Delete a specific task by ID
|
||||||
|
* Only allows deletion of failed, completed, or pending tasks (not running)
|
||||||
|
*/
|
||||||
|
router.delete('/:id', async (req: Request, res: Response) => {
|
||||||
|
try {
|
||||||
|
const taskId = parseInt(req.params.id, 10);
|
||||||
|
|
||||||
|
// First check if task exists and its status
|
||||||
|
const task = await taskService.getTask(taskId);
|
||||||
|
if (!task) {
|
||||||
|
return res.status(404).json({ error: 'Task not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Don't allow deleting running tasks
|
||||||
|
if (task.status === 'running' || task.status === 'claimed') {
|
||||||
|
return res.status(400).json({ error: 'Cannot delete a running or claimed task' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete the task
|
||||||
|
await pool.query('DELETE FROM worker_tasks WHERE id = $1', [taskId]);
|
||||||
|
|
||||||
|
res.json({ success: true, message: `Task ${taskId} deleted` });
|
||||||
|
} catch (error: unknown) {
|
||||||
|
console.error('Error deleting task:', error);
|
||||||
|
res.status(500).json({ error: 'Failed to delete task' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* POST /api/tasks
|
* POST /api/tasks
|
||||||
* Create a new task
|
* Create a new task
|
||||||
|
|||||||
@@ -206,15 +206,53 @@ class TaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mark a task as failed
|
* Mark a task as failed, with auto-retry if under max_retries
|
||||||
|
* Returns true if task was re-queued for retry, false if permanently failed
|
||||||
*/
|
*/
|
||||||
async failTask(taskId: number, errorMessage: string): Promise<void> {
|
async failTask(taskId: number, errorMessage: string): Promise<boolean> {
|
||||||
|
// Get current retry state
|
||||||
|
const result = await pool.query(
|
||||||
|
`SELECT retry_count, max_retries FROM worker_tasks WHERE id = $1`,
|
||||||
|
[taskId]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.rows.length === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { retry_count, max_retries } = result.rows[0];
|
||||||
|
const newRetryCount = (retry_count || 0) + 1;
|
||||||
|
|
||||||
|
if (newRetryCount < (max_retries || 3)) {
|
||||||
|
// Re-queue for retry - reset to pending with incremented retry_count
|
||||||
|
await pool.query(
|
||||||
|
`UPDATE worker_tasks
|
||||||
|
SET status = 'pending',
|
||||||
|
worker_id = NULL,
|
||||||
|
claimed_at = NULL,
|
||||||
|
started_at = NULL,
|
||||||
|
retry_count = $2,
|
||||||
|
error_message = $3,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = $1`,
|
||||||
|
[taskId, newRetryCount, `Retry ${newRetryCount}: ${errorMessage}`]
|
||||||
|
);
|
||||||
|
console.log(`[TaskService] Task ${taskId} queued for retry ${newRetryCount}/${max_retries || 3}`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Max retries exceeded - mark as permanently failed
|
||||||
await pool.query(
|
await pool.query(
|
||||||
`UPDATE worker_tasks
|
`UPDATE worker_tasks
|
||||||
SET status = 'failed', completed_at = NOW(), error_message = $2
|
SET status = 'failed',
|
||||||
|
completed_at = NOW(),
|
||||||
|
retry_count = $2,
|
||||||
|
error_message = $3
|
||||||
WHERE id = $1`,
|
WHERE id = $1`,
|
||||||
[taskId, errorMessage]
|
[taskId, newRetryCount, `Failed after ${newRetryCount} attempts: ${errorMessage}`]
|
||||||
);
|
);
|
||||||
|
console.log(`[TaskService] Task ${taskId} permanently failed after ${newRetryCount} attempts`);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -69,6 +69,13 @@ class ApiClient {
|
|||||||
return { data };
|
return { data };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async delete<T = any>(endpoint: string): Promise<{ data: T }> {
|
||||||
|
const data = await this.request<T>(endpoint, {
|
||||||
|
method: 'DELETE',
|
||||||
|
});
|
||||||
|
return { data };
|
||||||
|
}
|
||||||
|
|
||||||
// Auth
|
// Auth
|
||||||
async login(email: string, password: string) {
|
async login(email: string, password: string) {
|
||||||
return this.request<{ token: string; user: any }>('/api/auth/login', {
|
return this.request<{ token: string; user: any }>('/api/auth/login', {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
X,
|
X,
|
||||||
Search,
|
Search,
|
||||||
Calendar,
|
Calendar,
|
||||||
|
Trash2,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
|
|
||||||
// Worker from registry
|
// Worker from registry
|
||||||
@@ -61,6 +62,9 @@ interface Task {
|
|||||||
started_at: string | null;
|
started_at: string | null;
|
||||||
completed_at: string | null;
|
completed_at: string | null;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
|
error_message: string | null;
|
||||||
|
retry_count: number;
|
||||||
|
max_retries: number;
|
||||||
result: any;
|
result: any;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
@@ -499,7 +503,7 @@ function WorkerStatusBadge({ status, healthStatus }: { status: string; healthSta
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function TaskStatusBadge({ status }: { status: string }) {
|
function TaskStatusBadge({ status, error, retryCount }: { status: string; error?: string | null; retryCount?: number }) {
|
||||||
const config: Record<string, { bg: string; text: string; icon: any }> = {
|
const config: Record<string, { bg: string; text: string; icon: any }> = {
|
||||||
pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock },
|
pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock },
|
||||||
running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity },
|
running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity },
|
||||||
@@ -510,10 +514,25 @@ function TaskStatusBadge({ status }: { status: string }) {
|
|||||||
const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock };
|
const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock };
|
||||||
const Icon = cfg.icon;
|
const Icon = cfg.icon;
|
||||||
|
|
||||||
|
// Build tooltip text
|
||||||
|
let tooltip = '';
|
||||||
|
if (error) {
|
||||||
|
tooltip = error;
|
||||||
|
}
|
||||||
|
if (retryCount && retryCount > 0) {
|
||||||
|
tooltip = `Attempt ${retryCount + 1}${error ? `: ${error}` : ''}`;
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<span className={`inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium ${cfg.bg} ${cfg.text}`}>
|
<span
|
||||||
|
className={`inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium ${cfg.bg} ${cfg.text} ${error ? 'cursor-help' : ''}`}
|
||||||
|
title={tooltip || undefined}
|
||||||
|
>
|
||||||
<Icon className="w-3 h-3" />
|
<Icon className="w-3 h-3" />
|
||||||
{status}
|
{status}
|
||||||
|
{retryCount && retryCount > 0 && status !== 'failed' && (
|
||||||
|
<span className="text-[10px] opacity-75">({retryCount})</span>
|
||||||
|
)}
|
||||||
</span>
|
</span>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -735,6 +754,18 @@ export function JobQueue() {
|
|||||||
return () => clearInterval(interval);
|
return () => clearInterval(interval);
|
||||||
}, [fetchWorkers]);
|
}, [fetchWorkers]);
|
||||||
|
|
||||||
|
// Delete a task
|
||||||
|
const handleDeleteTask = async (taskId: number) => {
|
||||||
|
if (!confirm('Delete this task?')) return;
|
||||||
|
try {
|
||||||
|
await api.delete(`/api/tasks/${taskId}`);
|
||||||
|
fetchTasks();
|
||||||
|
} catch (err: any) {
|
||||||
|
console.error('Delete error:', err);
|
||||||
|
alert(err.response?.data?.error || 'Failed to delete task');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Get active workers (for display)
|
// Get active workers (for display)
|
||||||
const activeWorkers = workers.filter(w => w.status !== 'offline' && w.status !== 'terminated');
|
const activeWorkers = workers.filter(w => w.status !== 'offline' && w.status !== 'terminated');
|
||||||
const busyWorkers = workers.filter(w => w.current_task_id !== null);
|
const busyWorkers = workers.filter(w => w.current_task_id !== null);
|
||||||
@@ -910,12 +941,13 @@ export function JobQueue() {
|
|||||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Assigned To</th>
|
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Assigned To</th>
|
||||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Created</th>
|
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Created</th>
|
||||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Duration</th>
|
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Duration</th>
|
||||||
|
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase w-16"></th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody className="divide-y divide-gray-200">
|
<tbody className="divide-y divide-gray-200">
|
||||||
{tasks.length === 0 ? (
|
{tasks.length === 0 ? (
|
||||||
<tr>
|
<tr>
|
||||||
<td colSpan={7} className="px-4 py-8 text-center text-gray-500">
|
<td colSpan={8} className="px-4 py-8 text-center text-gray-500">
|
||||||
<Inbox className="w-8 h-8 mx-auto mb-2 text-gray-300" />
|
<Inbox className="w-8 h-8 mx-auto mb-2 text-gray-300" />
|
||||||
<p>No tasks found</p>
|
<p>No tasks found</p>
|
||||||
</td>
|
</td>
|
||||||
@@ -958,7 +990,7 @@ export function JobQueue() {
|
|||||||
)}
|
)}
|
||||||
</td>
|
</td>
|
||||||
<td className="px-4 py-3">
|
<td className="px-4 py-3">
|
||||||
<TaskStatusBadge status={task.status} />
|
<TaskStatusBadge status={task.status} error={task.error_message || task.error} retryCount={task.retry_count} />
|
||||||
</td>
|
</td>
|
||||||
<td className="px-4 py-3 text-sm">
|
<td className="px-4 py-3 text-sm">
|
||||||
{assignedWorker ? (
|
{assignedWorker ? (
|
||||||
@@ -986,6 +1018,17 @@ export function JobQueue() {
|
|||||||
'-'
|
'-'
|
||||||
)}
|
)}
|
||||||
</td>
|
</td>
|
||||||
|
<td className="px-4 py-3">
|
||||||
|
{(task.status === 'failed' || task.status === 'completed' || task.status === 'pending') && (
|
||||||
|
<button
|
||||||
|
onClick={() => handleDeleteTask(task.id)}
|
||||||
|
className="p-1 text-gray-400 hover:text-red-500 hover:bg-red-50 rounded transition-colors"
|
||||||
|
title="Delete task"
|
||||||
|
>
|
||||||
|
<Trash2 className="w-4 h-4" />
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
);
|
);
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user