"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.activeScrapers = void 0; exports.registerScraper = registerScraper; exports.updateScraperStats = updateScraperStats; exports.completeScraper = completeScraper; const express_1 = require("express"); const middleware_1 = require("../auth/middleware"); const migrate_1 = require("../db/migrate"); const router = (0, express_1.Router)(); router.use(middleware_1.authMiddleware); exports.activeScrapers = new Map(); // Get all active scrapers router.get('/active', async (req, res) => { try { const scrapers = Array.from(exports.activeScrapers.values()).map(scraper => ({ ...scraper, duration: Date.now() - scraper.startTime.getTime(), isStale: Date.now() - scraper.lastUpdate.getTime() > 60000 // 1 minute })); res.json({ scrapers }); } catch (error) { console.error('Error fetching active scrapers:', error); res.status(500).json({ error: 'Failed to fetch active scrapers' }); } }); // Get scraper by ID router.get('/active/:id', async (req, res) => { try { const { id } = req.params; const scraper = exports.activeScrapers.get(id); if (!scraper) { return res.status(404).json({ error: 'Scraper not found' }); } res.json({ scraper: { ...scraper, duration: Date.now() - scraper.startTime.getTime(), isStale: Date.now() - scraper.lastUpdate.getTime() > 60000 } }); } catch (error) { console.error('Error fetching scraper:', error); res.status(500).json({ error: 'Failed to fetch scraper' }); } }); // Get scraper history (last 50 completed scrapes) router.get('/history', async (req, res) => { try { const { limit = 50, store_id } = req.query; let query = ` SELECT s.id as store_id, s.name as store_name, c.id as category_id, c.name as category_name, c.last_scraped_at, ( SELECT COUNT(*) FROM products p WHERE p.store_id = s.id AND p.category_id = c.id ) as product_count FROM stores s LEFT JOIN categories c ON c.store_id = s.id WHERE c.last_scraped_at IS NOT NULL `; const params = []; let paramCount = 1; if (store_id) { query += ` AND s.id = $${paramCount}`; params.push(store_id); paramCount++; } query += ` ORDER BY c.last_scraped_at DESC LIMIT $${paramCount}`; params.push(limit); const result = await migrate_1.pool.query(query, params); res.json({ history: result.rows }); } catch (error) { console.error('Error fetching scraper history:', error); res.status(500).json({ error: 'Failed to fetch scraper history' }); } }); // Helper function to register a scraper function registerScraper(id, storeId, storeName, categoryId, categoryName) { exports.activeScrapers.set(id, { id, storeId, storeName, categoryId, categoryName, startTime: new Date(), lastUpdate: new Date(), status: 'running', stats: { requestsTotal: 0, requestsSuccess: 0, itemsSaved: 0, itemsDropped: 0, errorsCount: 0 } }); } // Helper function to update scraper stats function updateScraperStats(id, stats, currentActivity) { const scraper = exports.activeScrapers.get(id); if (scraper) { scraper.stats = { ...scraper.stats, ...stats }; scraper.lastUpdate = new Date(); if (currentActivity) { scraper.currentActivity = currentActivity; } } } // Helper function to mark scraper as completed function completeScraper(id, error) { const scraper = exports.activeScrapers.get(id); if (scraper) { scraper.status = error ? 'error' : 'completed'; scraper.lastUpdate = new Date(); // Remove after 5 minutes setTimeout(() => { exports.activeScrapers.delete(id); }, 5 * 60 * 1000); } } exports.default = router;