Improve ScraperMonitor tab loading efficiency and add New/Updated columns
- Tab-specific data loading: Only fetch APIs needed for the active tab - AZ Live tab fetches only AZ monitor APIs - Dispensary Jobs tab fetches only legacy job APIs - Crawl History tab fetches only scraper history APIs - Auto-refresh now respects active tab - Added New and Updated columns to Crawl History table with color coding 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Kelly
@@ -21,45 +21,42 @@ export function ScraperMonitor() {
|
||||
const [azRecentJobs, setAzRecentJobs] = useState<any>({ jobLogs: [], crawlJobs: [] });
|
||||
const [azErrors, setAzErrors] = useState<any[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
loadData();
|
||||
|
||||
if (autoRefresh) {
|
||||
const interval = setInterval(loadData, 3000); // Refresh every 3 seconds
|
||||
return () => clearInterval(interval);
|
||||
}
|
||||
}, [autoRefresh]);
|
||||
|
||||
const loadData = async () => {
|
||||
// Load data based on active tab for efficiency
|
||||
const loadTabData = async (tab: typeof activeTab) => {
|
||||
try {
|
||||
const [activeData, historyData, statsData, jobsData, workersData, recentJobsData] = await Promise.all([
|
||||
api.getActiveScrapers(),
|
||||
api.getScraperHistory(),
|
||||
api.getJobStats(),
|
||||
api.getActiveJobs(),
|
||||
api.getWorkerStats(),
|
||||
api.getRecentJobs({ limit: 50 })
|
||||
]);
|
||||
|
||||
setActiveScrapers(activeData.scrapers || []);
|
||||
setHistory(historyData.history || []);
|
||||
setJobStats(statsData);
|
||||
setActiveJobs(jobsData.jobs || []);
|
||||
setWorkers(workersData.workers || []);
|
||||
setRecentJobs(recentJobsData.jobs || []);
|
||||
|
||||
// Load AZ monitor data
|
||||
const [azSummaryData, azActiveData, azRecentData, azErrorsData] = await Promise.all([
|
||||
api.getAZMonitorSummary().catch(() => null),
|
||||
api.getAZMonitorActiveJobs().catch(() => ({ scheduledJobs: [], crawlJobs: [], inMemoryScrapers: [], totalActive: 0 })),
|
||||
api.getAZMonitorRecentJobs(30).catch(() => ({ jobLogs: [], crawlJobs: [] })),
|
||||
api.getAZMonitorErrors({ limit: 10, hours: 24 }).catch(() => ({ errors: [] })),
|
||||
]);
|
||||
|
||||
setAzSummary(azSummaryData);
|
||||
setAzActiveJobs(azActiveData);
|
||||
setAzRecentJobs(azRecentData);
|
||||
setAzErrors(azErrorsData?.errors || []);
|
||||
if (tab === 'az-live') {
|
||||
// Only load AZ data for AZ Live tab
|
||||
const [azSummaryData, azActiveData, azRecentData, azErrorsData] = await Promise.all([
|
||||
api.getAZMonitorSummary().catch(() => null),
|
||||
api.getAZMonitorActiveJobs().catch(() => ({ scheduledJobs: [], crawlJobs: [], inMemoryScrapers: [], totalActive: 0 })),
|
||||
api.getAZMonitorRecentJobs(30).catch(() => ({ jobLogs: [], crawlJobs: [] })),
|
||||
api.getAZMonitorErrors({ limit: 10, hours: 24 }).catch(() => ({ errors: [] })),
|
||||
]);
|
||||
setAzSummary(azSummaryData);
|
||||
setAzActiveJobs(azActiveData);
|
||||
setAzRecentJobs(azRecentData);
|
||||
setAzErrors(azErrorsData?.errors || []);
|
||||
} else if (tab === 'jobs') {
|
||||
// Only load legacy job data for Dispensary Jobs tab
|
||||
const [statsData, jobsData, workersData, recentJobsData] = await Promise.all([
|
||||
api.getJobStats(),
|
||||
api.getActiveJobs(),
|
||||
api.getWorkerStats(),
|
||||
api.getRecentJobs({ limit: 50 })
|
||||
]);
|
||||
setJobStats(statsData);
|
||||
setActiveJobs(jobsData.jobs || []);
|
||||
setWorkers(workersData.workers || []);
|
||||
setRecentJobs(recentJobsData.jobs || []);
|
||||
} else if (tab === 'scrapers') {
|
||||
// Only load scraper data for Crawl History tab
|
||||
const [activeData, historyData] = await Promise.all([
|
||||
api.getActiveScrapers(),
|
||||
api.getScraperHistory()
|
||||
]);
|
||||
setActiveScrapers(activeData.scrapers || []);
|
||||
setHistory(historyData.history || []);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load scraper data:', error);
|
||||
} finally {
|
||||
@@ -67,6 +64,19 @@ export function ScraperMonitor() {
|
||||
}
|
||||
};
|
||||
|
||||
// Initial load and tab change handler
|
||||
useEffect(() => {
|
||||
loadTabData(activeTab);
|
||||
}, [activeTab]);
|
||||
|
||||
// Auto-refresh based on active tab
|
||||
useEffect(() => {
|
||||
if (autoRefresh) {
|
||||
const interval = setInterval(() => loadTabData(activeTab), 3000);
|
||||
return () => clearInterval(interval);
|
||||
}
|
||||
}, [autoRefresh, activeTab]);
|
||||
|
||||
const formatDuration = (ms: number) => {
|
||||
const seconds = Math.floor(ms / 1000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
@@ -264,53 +274,77 @@ export function ScraperMonitor() {
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Individual Crawl Jobs */}
|
||||
{azActiveJobs.crawlJobs.map((job: any) => (
|
||||
<div key={`crawl-${job.id}`} style={{
|
||||
{/* Individual Crawl Jobs - Table View */}
|
||||
{azActiveJobs.crawlJobs.length > 0 && (
|
||||
<div style={{
|
||||
background: 'white',
|
||||
padding: '20px',
|
||||
borderRadius: '8px',
|
||||
boxShadow: '0 2px 8px rgba(0,0,0,0.1)',
|
||||
borderLeft: '4px solid #3b82f6'
|
||||
overflow: 'hidden',
|
||||
marginTop: '15px'
|
||||
}}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'start' }}>
|
||||
<div style={{ flex: 1 }}>
|
||||
<div style={{ fontSize: '18px', fontWeight: '600', marginBottom: '8px' }}>
|
||||
{job.dispensary_name || 'Unknown Store'}
|
||||
</div>
|
||||
<div style={{ fontSize: '14px', color: '#666', marginBottom: '12px' }}>
|
||||
{job.city} | {job.job_type || 'crawl'}
|
||||
</div>
|
||||
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(120px, 1fr))', gap: '12px' }}>
|
||||
<div>
|
||||
<div style={{ fontSize: '12px', color: '#999', marginBottom: '4px' }}>Products Found</div>
|
||||
<div style={{ fontSize: '16px', fontWeight: '600', color: '#8b5cf6' }}>{job.products_found || 0}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style={{ fontSize: '12px', color: '#999', marginBottom: '4px' }}>Snapshots</div>
|
||||
<div style={{ fontSize: '16px', fontWeight: '600', color: '#06b6d4' }}>{job.snapshots_created || 0}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style={{ fontSize: '12px', color: '#999', marginBottom: '4px' }}>Duration</div>
|
||||
<div style={{ fontSize: '16px', fontWeight: '600' }}>
|
||||
{Math.floor((job.duration_seconds || 0) / 60)}m {Math.floor((job.duration_seconds || 0) % 60)}s
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{
|
||||
padding: '6px 12px',
|
||||
borderRadius: '4px',
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
background: '#dbeafe',
|
||||
color: '#1e40af'
|
||||
}}>
|
||||
CRAWLING
|
||||
</div>
|
||||
<div style={{ padding: '15px 20px', borderBottom: '2px solid #eee', background: '#f8f8f8' }}>
|
||||
<h3 style={{ margin: 0, fontSize: '16px', fontWeight: '600' }}>Active Crawler Sessions ({azActiveJobs.crawlJobs.length})</h3>
|
||||
</div>
|
||||
<table style={{ width: '100%', borderCollapse: 'collapse' }}>
|
||||
<thead>
|
||||
<tr style={{ borderBottom: '1px solid #eee' }}>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'left', fontWeight: '600', fontSize: '13px', color: '#666' }}>Store</th>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'left', fontWeight: '600', fontSize: '13px', color: '#666' }}>Worker</th>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'center', fontWeight: '600', fontSize: '13px', color: '#666' }}>Page</th>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', fontSize: '13px', color: '#666' }}>Products</th>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', fontSize: '13px', color: '#666' }}>Snapshots</th>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', fontSize: '13px', color: '#666' }}>Duration</th>
|
||||
<th style={{ padding: '12px 15px', textAlign: 'center', fontWeight: '600', fontSize: '13px', color: '#666' }}>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{azActiveJobs.crawlJobs.map((job: any) => (
|
||||
<tr key={`crawl-${job.id}`} style={{ borderBottom: '1px solid #eee' }}>
|
||||
<td style={{ padding: '12px 15px' }}>
|
||||
<div style={{ fontWeight: '600', marginBottom: '2px' }}>{job.dispensary_name || 'Unknown'}</div>
|
||||
<div style={{ fontSize: '12px', color: '#999' }}>{job.city} | ID: {job.dispensary_id}</div>
|
||||
</td>
|
||||
<td style={{ padding: '12px 15px', fontSize: '13px' }}>
|
||||
<div style={{ fontFamily: 'monospace', fontSize: '11px', color: '#666' }}>
|
||||
{job.worker_id ? job.worker_id.substring(0, 8) : '-'}
|
||||
</div>
|
||||
{job.worker_hostname && (
|
||||
<div style={{ fontSize: '11px', color: '#999' }}>{job.worker_hostname}</div>
|
||||
)}
|
||||
</td>
|
||||
<td style={{ padding: '12px 15px', textAlign: 'center', fontSize: '13px' }}>
|
||||
{job.current_page && job.total_pages ? (
|
||||
<span>{job.current_page}/{job.total_pages}</span>
|
||||
) : '-'}
|
||||
</td>
|
||||
<td style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', color: '#8b5cf6' }}>
|
||||
{job.products_found || 0}
|
||||
</td>
|
||||
<td style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', color: '#06b6d4' }}>
|
||||
{job.snapshots_created || 0}
|
||||
</td>
|
||||
<td style={{ padding: '12px 15px', textAlign: 'right', fontSize: '13px' }}>
|
||||
{Math.floor((job.duration_seconds || 0) / 60)}m {Math.floor((job.duration_seconds || 0) % 60)}s
|
||||
</td>
|
||||
<td style={{ padding: '12px 15px', textAlign: 'center' }}>
|
||||
<span style={{
|
||||
padding: '4px 10px',
|
||||
borderRadius: '12px',
|
||||
fontSize: '11px',
|
||||
fontWeight: '600',
|
||||
background: job.last_heartbeat_at && (Date.now() - new Date(job.last_heartbeat_at).getTime() > 60000) ? '#fef3c7' : '#dbeafe',
|
||||
color: job.last_heartbeat_at && (Date.now() - new Date(job.last_heartbeat_at).getTime() > 60000) ? '#92400e' : '#1e40af'
|
||||
}}>
|
||||
{job.last_heartbeat_at && (Date.now() - new Date(job.last_heartbeat_at).getTime() > 60000) ? 'STALE' : 'CRAWLING'}
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
))}
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -843,6 +877,8 @@ export function ScraperMonitor() {
|
||||
<th style={{ padding: '15px', textAlign: 'left', fontWeight: '600' }}>Dispensary</th>
|
||||
<th style={{ padding: '15px', textAlign: 'left', fontWeight: '600' }}>Status</th>
|
||||
<th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>Found</th>
|
||||
<th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>New</th>
|
||||
<th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>Updated</th>
|
||||
<th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>Products</th>
|
||||
<th style={{ padding: '15px', textAlign: 'left', fontWeight: '600' }}>Last Crawled</th>
|
||||
</tr>
|
||||
@@ -866,6 +902,12 @@ export function ScraperMonitor() {
|
||||
<td style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>
|
||||
{item.products_found || '-'}
|
||||
</td>
|
||||
<td style={{ padding: '15px', textAlign: 'right', fontWeight: '600', color: '#059669' }}>
|
||||
{item.products_new || 0}
|
||||
</td>
|
||||
<td style={{ padding: '15px', textAlign: 'right', fontWeight: '600', color: '#2563eb' }}>
|
||||
{item.products_updated || 0}
|
||||
</td>
|
||||
<td style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>
|
||||
{item.product_count}
|
||||
</td>
|
||||
|
||||
Reference in New Issue
Block a user