131 lines
4.2 KiB
JavaScript
131 lines
4.2 KiB
JavaScript
"use strict";
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.activeScrapers = void 0;
|
|
exports.registerScraper = registerScraper;
|
|
exports.updateScraperStats = updateScraperStats;
|
|
exports.completeScraper = completeScraper;
|
|
const express_1 = require("express");
|
|
const middleware_1 = require("../auth/middleware");
|
|
const migrate_1 = require("../db/migrate");
|
|
const router = (0, express_1.Router)();
|
|
router.use(middleware_1.authMiddleware);
|
|
exports.activeScrapers = new Map();
|
|
// Get all active scrapers
|
|
router.get('/active', async (req, res) => {
|
|
try {
|
|
const scrapers = Array.from(exports.activeScrapers.values()).map(scraper => ({
|
|
...scraper,
|
|
duration: Date.now() - scraper.startTime.getTime(),
|
|
isStale: Date.now() - scraper.lastUpdate.getTime() > 60000 // 1 minute
|
|
}));
|
|
res.json({ scrapers });
|
|
}
|
|
catch (error) {
|
|
console.error('Error fetching active scrapers:', error);
|
|
res.status(500).json({ error: 'Failed to fetch active scrapers' });
|
|
}
|
|
});
|
|
// Get scraper by ID
|
|
router.get('/active/:id', async (req, res) => {
|
|
try {
|
|
const { id } = req.params;
|
|
const scraper = exports.activeScrapers.get(id);
|
|
if (!scraper) {
|
|
return res.status(404).json({ error: 'Scraper not found' });
|
|
}
|
|
res.json({
|
|
scraper: {
|
|
...scraper,
|
|
duration: Date.now() - scraper.startTime.getTime(),
|
|
isStale: Date.now() - scraper.lastUpdate.getTime() > 60000
|
|
}
|
|
});
|
|
}
|
|
catch (error) {
|
|
console.error('Error fetching scraper:', error);
|
|
res.status(500).json({ error: 'Failed to fetch scraper' });
|
|
}
|
|
});
|
|
// Get scraper history (last 50 completed scrapes)
|
|
router.get('/history', async (req, res) => {
|
|
try {
|
|
const { limit = 50, store_id } = req.query;
|
|
let query = `
|
|
SELECT
|
|
s.id as store_id,
|
|
s.name as store_name,
|
|
c.id as category_id,
|
|
c.name as category_name,
|
|
c.last_scraped_at,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM products p
|
|
WHERE p.store_id = s.id
|
|
AND p.category_id = c.id
|
|
) as product_count
|
|
FROM stores s
|
|
LEFT JOIN categories c ON c.store_id = s.id
|
|
WHERE c.last_scraped_at IS NOT NULL
|
|
`;
|
|
const params = [];
|
|
let paramCount = 1;
|
|
if (store_id) {
|
|
query += ` AND s.id = $${paramCount}`;
|
|
params.push(store_id);
|
|
paramCount++;
|
|
}
|
|
query += ` ORDER BY c.last_scraped_at DESC LIMIT $${paramCount}`;
|
|
params.push(limit);
|
|
const result = await migrate_1.pool.query(query, params);
|
|
res.json({ history: result.rows });
|
|
}
|
|
catch (error) {
|
|
console.error('Error fetching scraper history:', error);
|
|
res.status(500).json({ error: 'Failed to fetch scraper history' });
|
|
}
|
|
});
|
|
// Helper function to register a scraper
|
|
function registerScraper(id, storeId, storeName, categoryId, categoryName) {
|
|
exports.activeScrapers.set(id, {
|
|
id,
|
|
storeId,
|
|
storeName,
|
|
categoryId,
|
|
categoryName,
|
|
startTime: new Date(),
|
|
lastUpdate: new Date(),
|
|
status: 'running',
|
|
stats: {
|
|
requestsTotal: 0,
|
|
requestsSuccess: 0,
|
|
itemsSaved: 0,
|
|
itemsDropped: 0,
|
|
errorsCount: 0
|
|
}
|
|
});
|
|
}
|
|
// Helper function to update scraper stats
|
|
function updateScraperStats(id, stats, currentActivity) {
|
|
const scraper = exports.activeScrapers.get(id);
|
|
if (scraper) {
|
|
scraper.stats = { ...scraper.stats, ...stats };
|
|
scraper.lastUpdate = new Date();
|
|
if (currentActivity) {
|
|
scraper.currentActivity = currentActivity;
|
|
}
|
|
}
|
|
}
|
|
// Helper function to mark scraper as completed
|
|
function completeScraper(id, error) {
|
|
const scraper = exports.activeScrapers.get(id);
|
|
if (scraper) {
|
|
scraper.status = error ? 'error' : 'completed';
|
|
scraper.lastUpdate = new Date();
|
|
// Remove after 5 minutes
|
|
setTimeout(() => {
|
|
exports.activeScrapers.delete(id);
|
|
}, 5 * 60 * 1000);
|
|
}
|
|
}
|
|
exports.default = router;
|