Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Remove Run Now, add source tracking, optimize dashboard

Browse Source 
- Remove /run-now endpoint (use task priority instead)
- Add source tracking to worker_tasks (source, source_schedule_id, source_metadata)
- Parallelize dashboard API calls (Promise.all)
- Add 1-5 min caching to /markets/dashboard and /national/summary
- Add performance indexes for dashboard queries

Migrations:
- 104: Task source tracking columns
- 105: Dashboard performance indexes

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-13 13:23:35 -07:00
parent a8fec97bcb
commit 8b3ae40089
13 changed files with 271 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
backend/src/routes/tasks.ts
View File

@@ -16,9 +16,11 @@
* PUT /api/tasks/schedules/:id - Update schedule
* DELETE /api/tasks/schedules/:id - Delete schedule
* DELETE /api/tasks/schedules - Bulk delete schedules
* POST /api/tasks/schedules/:id/run-now - Trigger schedule immediately
* POST /api/tasks/schedules/:id/toggle - Toggle schedule enabled/disabled
*
* Note: "Run Now" was removed - use task priority instead.
* Higher priority tasks get picked up first (ORDER BY priority DESC).
*
* Note: Schedule routes are defined BEFORE /:id to avoid route conflicts
* (Express matches routes in order, and "schedules" would match /:id otherwise)
*/
@@ -29,7 +31,21 @@ import {
TaskRole,
TaskStatus,
TaskFilter,
TaskSource,
} from '../tasks/task-service';
/**
* Extract request metadata for source tracking
*/
function getRequestMetadata(req: Request): Record<string, unknown> {
return {
ip: req.ip || req.socket?.remoteAddress || 'unknown',
userAgent: req.get('user-agent') || 'unknown',
endpoint: req.originalUrl,
method: req.method,
timestamp: new Date().toISOString(),
};
}
import { pool } from '../db/pool';
import {
isTaskPoolPaused,
@@ -524,146 +540,6 @@ router.delete('/schedules/:id', async (req: Request, res: Response) => {
}
});
/**
* POST /api/tasks/schedules/:id/run-now
* Manually trigger a scheduled task to run immediately
*
* For product_discovery schedules with state_code, this creates individual
* tasks for each store in that state (fans out properly).
*/
router.post('/schedules/:id/run-now', async (req: Request, res: Response) => {
try {
const scheduleId = parseInt(req.params.id, 10);
// Get the full schedule
const scheduleResult = await pool.query(`
SELECT id, name, role, state_code, dispensary_id, platform, priority, interval_hours, method
FROM task_schedules WHERE id = $1
`, [scheduleId]);
if (scheduleResult.rows.length === 0) {
return res.status(404).json({ error: 'Schedule not found' });
}
const schedule = scheduleResult.rows[0];
let tasksCreated = 0;
const isCrawlRole = ['product_discovery', 'product_refresh', 'payload_fetch'].includes(schedule.role);
// Single-dispensary schedule (e.g., "Deeply Rooted Hourly")
if (isCrawlRole && schedule.dispensary_id) {
// Check if this specific store can be refreshed (no pending task)
const storeResult = await pool.query(`
SELECT d.id, d.name
FROM dispensaries d
WHERE d.id = $1
AND d.crawl_enabled = true
AND d.platform_dispensary_id IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM worker_tasks t
WHERE t.dispensary_id = d.id
AND t.role IN ('product_discovery', 'product_refresh', 'payload_fetch')
AND t.status IN ('pending', 'claimed', 'running')
)
`, [schedule.dispensary_id]);
if (storeResult.rows.length > 0) {
await taskService.createTask({
role: 'product_discovery',
dispensary_id: schedule.dispensary_id,
platform: schedule.platform || 'dutchie',
priority: schedule.priority + 10,
method: schedule.method || 'http',
});
tasksCreated = 1;
} else {
return res.json({
success: true,
message: `Store ${schedule.dispensary_id} has a pending task or is disabled`,
tasksCreated: 0,
dispensaryId: schedule.dispensary_id,
});
}
}
// Per-state schedule (e.g., "AZ Product Refresh")
else if (isCrawlRole && schedule.state_code) {
// Find stores in this state needing refresh
const storeResult = await pool.query(`
SELECT d.id
FROM dispensaries d
JOIN states s ON d.state_id = s.id
WHERE d.crawl_enabled = true
AND d.platform_dispensary_id IS NOT NULL
AND s.code = $1
-- No pending/running crawl task already
AND NOT EXISTS (
SELECT 1 FROM worker_tasks t
WHERE t.dispensary_id = d.id
AND t.role IN ('product_discovery', 'product_refresh', 'payload_fetch')
AND t.status IN ('pending', 'claimed', 'running')
)
ORDER BY d.last_fetch_at NULLS FIRST, d.id
`, [schedule.state_code]);
const dispensaryIds = storeResult.rows.map((r: { id: number }) => r.id);
if (dispensaryIds.length > 0) {
// Create staggered tasks for all stores (always use product_discovery role)
const result = await taskService.createStaggeredTasks(
dispensaryIds,
'product_discovery', // Normalize to product_discovery
15, // 15 seconds stagger
schedule.platform || 'dutchie',
schedule.method || 'http'
);
tasksCreated = result.created;
} else {
// No stores need refresh - return early with message
return res.json({
success: true,
message: `No ${schedule.state_code} stores need refresh at this time`,
tasksCreated: 0,
stateCode: schedule.state_code,
});
}
} else if (!isCrawlRole) {
// For other schedules (store_discovery, analytics_refresh), create a single task
await taskService.createTask({
role: schedule.role,
platform: schedule.platform,
priority: schedule.priority + 10,
method: schedule.method,
});
tasksCreated = 1;
} else {
// Crawl role without dispensary_id or state_code - reject
return res.status(400).json({
error: `${schedule.role} schedules require a dispensary_id or state_code`,
});
}
// Update last_run_at on the schedule
await pool.query(`
UPDATE task_schedules
SET last_run_at = NOW(),
next_run_at = NOW() + (interval_hours || ' hours')::interval,
last_task_count = $2,
updated_at = NOW()
WHERE id = $1
`, [scheduleId, tasksCreated]);
res.json({
success: true,
message: `Schedule "${schedule.name}" triggered`,
tasksCreated,
stateCode: schedule.state_code,
});
} catch (error: unknown) {
console.error('Error running schedule:', error);
res.status(500).json({ error: 'Failed to run schedule' });
}
});
/**
* POST /api/tasks/schedules/:id/toggle
* Toggle a schedule's enabled status
@@ -1275,10 +1151,15 @@ router.post('/migration/full-migrate', async (req: Request, res: Response) => {
*/
router.post('/batch/staggered', async (req: Request, res: Response) => {
try {
const requestMetadata = getRequestMetadata(req);
// Log the request for tracking phantom tasks
console.log(`[TaskAPI] POST /batch/staggered from ${requestMetadata.ip} (${requestMetadata.userAgent})`);
const {
dispensary_ids,
role,
stagger_seconds = 15,
stagger_seconds = 0, // Default to 0 (no stagger) - worker controls pacing
platform = 'dutchie',
method = null,
} = req.body;
@@ -1291,12 +1172,18 @@ router.post('/batch/staggered', async (req: Request, res: Response) => {
return res.status(400).json({ error: 'role is required' });
}
console.log(`[TaskAPI] Creating ${dispensary_ids.length} ${role} tasks for dispensaries: ${dispensary_ids.slice(0, 5).join(',')}...`);
const result = await taskService.createStaggeredTasks(
dispensary_ids,
role as TaskRole,
stagger_seconds,
platform,
method
method,
{
source: 'api_batch_staggered',
source_metadata: requestMetadata,
}
);
const totalDuration = (result.created - 1) * stagger_seconds;
@@ -1320,49 +1207,6 @@ router.post('/batch/staggered', async (req: Request, res: Response) => {
}
});
/**
* POST /api/tasks/batch/az-stores
* Convenience endpoint to create staggered tasks for Arizona stores
*
* Body:
* - total_tasks: number (default: 24) - Total tasks to create
* - stagger_seconds: number (default: 15) - Seconds between each task
* - split_roles: boolean (default: true) - Split between product_refresh and product_discovery
*/
router.post('/batch/az-stores', async (req: Request, res: Response) => {
try {
const {
total_tasks = 24,
stagger_seconds = 15,
split_roles = true,
} = req.body;
const result = await taskService.createAZStoreTasks(
total_tasks,
stagger_seconds,
split_roles
);
const totalDuration = (result.total - 1) * stagger_seconds;
const estimatedEndTime = new Date(Date.now() + totalDuration * 1000);
res.status(201).json({
success: true,
total: result.total,
product_refresh: result.product_refresh,
product_discovery: result.product_discovery,
task_ids: result.taskIds,
stagger_seconds,
total_duration_seconds: totalDuration,
estimated_completion: estimatedEndTime.toISOString(),
message: `Created ${result.total} staggered tasks for AZ stores (${result.product_refresh} refresh, ${result.product_discovery} discovery)`,
});
} catch (error: unknown) {
console.error('Error creating AZ store tasks:', error);
res.status(500).json({ error: 'Failed to create AZ store tasks' });
}
});
/**
* POST /api/tasks/batch/entry-point-discovery
* Create entry_point_discovery tasks for stores missing platform_dispensary_id
@@ -1556,13 +1400,13 @@ router.post('/batch/store-discovery', async (req: Request, res: Response) => {
* Create product_discovery tasks for all stores in a state
*
* This is the primary endpoint for triggering crawls by state.
* Creates staggered tasks for all crawl-enabled stores in the specified state.
* Creates tasks for all crawl-enabled stores in the specified state.
*
* Params:
* - stateCode: State code (e.g., 'AZ', 'CA', 'CO')
*
* Body (optional):
* - stagger_seconds: number (default: 15) - Seconds between each task
* - stagger_seconds: number (default: 0) - Seconds between each task (0 = worker controls pacing)
* - priority: number (default: 10) - Task priority
* - method: 'curl' | 'http' | null (default: 'http')
*
@@ -1574,8 +1418,13 @@ router.post('/batch/store-discovery', async (req: Request, res: Response) => {
router.post('/crawl-state/:stateCode', async (req: Request, res: Response) => {
try {
const stateCode = req.params.stateCode.toUpperCase();
const requestMetadata = getRequestMetadata(req);
// Log the request for tracking phantom tasks
console.log(`[TaskAPI] POST /crawl-state/${stateCode} from ${requestMetadata.ip} (${requestMetadata.userAgent})`);
const {
stagger_seconds = 15,
stagger_seconds = 0, // Default to 0 (no stagger) - worker controls pacing
priority = 10,
method = 'http',
} = req.body;
@@ -1617,13 +1466,19 @@ router.post('/crawl-state/:stateCode', async (req: Request, res: Response) => {
const dispensaryIds = dispensariesResult.rows.map((d: { id: number }) => d.id);
// Create staggered tasks
console.log(`[TaskAPI] Creating ${dispensaryIds.length} product_discovery tasks for ${stateCode}`);
// Create tasks with source tracking
const result = await taskService.createStaggeredTasks(
dispensaryIds,
'product_discovery',
stagger_seconds,
'dutchie',
method
method,
{
source: 'api_crawl_state',
source_metadata: { ...requestMetadata, stateCode },
}
);
const totalDuration = (result.created - 1) * stagger_seconds;