From d2d44d2aebb165c21c4949c2cc9b1d46c0832aa3 Mon Sep 17 00:00:00 2001
From: Kelly <kelly@cannabrands.com>
Date: Thu, 4 Dec 2025 18:26:56 -0700
Subject: [PATCH] Improve ScraperMonitor tab loading efficiency and add
 New/Updated columns
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Tab-specific data loading: Only fetch APIs needed for the active tab
- AZ Live tab fetches only AZ monitor APIs
- Dispensary Jobs tab fetches only legacy job APIs
- Crawl History tab fetches only scraper history APIs
- Auto-refresh now respects active tab
- Added New and Updated columns to Crawl History table with color coding

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 frontend/src/pages/ScraperMonitor.tsx | 200 ++++++++++++++++----------
 1 file changed, 121 insertions(+), 79 deletions(-)

diff --git a/frontend/src/pages/ScraperMonitor.tsx b/frontend/src/pages/ScraperMonitor.tsx
index df685cb3..e9f4967f 100644
--- a/frontend/src/pages/ScraperMonitor.tsx
+++ b/frontend/src/pages/ScraperMonitor.tsx
@@ -21,45 +21,42 @@ export function ScraperMonitor() {
   const [azRecentJobs, setAzRecentJobs] = useState<any>({ jobLogs: [], crawlJobs: [] });
   const [azErrors, setAzErrors] = useState<any[]>([]);
 
-  useEffect(() => {
-    loadData();
-
-    if (autoRefresh) {
-      const interval = setInterval(loadData, 3000); // Refresh every 3 seconds
-      return () => clearInterval(interval);
-    }
-  }, [autoRefresh]);
-
-  const loadData = async () => {
+  // Load data based on active tab for efficiency
+  const loadTabData = async (tab: typeof activeTab) => {
     try {
-      const [activeData, historyData, statsData, jobsData, workersData, recentJobsData] = await Promise.all([
-        api.getActiveScrapers(),
-        api.getScraperHistory(),
-        api.getJobStats(),
-        api.getActiveJobs(),
-        api.getWorkerStats(),
-        api.getRecentJobs({ limit: 50 })
-      ]);
-
-      setActiveScrapers(activeData.scrapers || []);
-      setHistory(historyData.history || []);
-      setJobStats(statsData);
-      setActiveJobs(jobsData.jobs || []);
-      setWorkers(workersData.workers || []);
-      setRecentJobs(recentJobsData.jobs || []);
-
-      // Load AZ monitor data
-      const [azSummaryData, azActiveData, azRecentData, azErrorsData] = await Promise.all([
-        api.getAZMonitorSummary().catch(() => null),
-        api.getAZMonitorActiveJobs().catch(() => ({ scheduledJobs: [], crawlJobs: [], inMemoryScrapers: [], totalActive: 0 })),
-        api.getAZMonitorRecentJobs(30).catch(() => ({ jobLogs: [], crawlJobs: [] })),
-        api.getAZMonitorErrors({ limit: 10, hours: 24 }).catch(() => ({ errors: [] })),
-      ]);
-
-      setAzSummary(azSummaryData);
-      setAzActiveJobs(azActiveData);
-      setAzRecentJobs(azRecentData);
-      setAzErrors(azErrorsData?.errors || []);
+      if (tab === 'az-live') {
+        // Only load AZ data for AZ Live tab
+        const [azSummaryData, azActiveData, azRecentData, azErrorsData] = await Promise.all([
+          api.getAZMonitorSummary().catch(() => null),
+          api.getAZMonitorActiveJobs().catch(() => ({ scheduledJobs: [], crawlJobs: [], inMemoryScrapers: [], totalActive: 0 })),
+          api.getAZMonitorRecentJobs(30).catch(() => ({ jobLogs: [], crawlJobs: [] })),
+          api.getAZMonitorErrors({ limit: 10, hours: 24 }).catch(() => ({ errors: [] })),
+        ]);
+        setAzSummary(azSummaryData);
+        setAzActiveJobs(azActiveData);
+        setAzRecentJobs(azRecentData);
+        setAzErrors(azErrorsData?.errors || []);
+      } else if (tab === 'jobs') {
+        // Only load legacy job data for Dispensary Jobs tab
+        const [statsData, jobsData, workersData, recentJobsData] = await Promise.all([
+          api.getJobStats(),
+          api.getActiveJobs(),
+          api.getWorkerStats(),
+          api.getRecentJobs({ limit: 50 })
+        ]);
+        setJobStats(statsData);
+        setActiveJobs(jobsData.jobs || []);
+        setWorkers(workersData.workers || []);
+        setRecentJobs(recentJobsData.jobs || []);
+      } else if (tab === 'scrapers') {
+        // Only load scraper data for Crawl History tab
+        const [activeData, historyData] = await Promise.all([
+          api.getActiveScrapers(),
+          api.getScraperHistory()
+        ]);
+        setActiveScrapers(activeData.scrapers || []);
+        setHistory(historyData.history || []);
+      }
     } catch (error) {
       console.error('Failed to load scraper data:', error);
     } finally {
@@ -67,6 +64,19 @@ export function ScraperMonitor() {
     }
   };
 
+  // Initial load and tab change handler
+  useEffect(() => {
+    loadTabData(activeTab);
+  }, [activeTab]);
+
+  // Auto-refresh based on active tab
+  useEffect(() => {
+    if (autoRefresh) {
+      const interval = setInterval(() => loadTabData(activeTab), 3000);
+      return () => clearInterval(interval);
+    }
+  }, [autoRefresh, activeTab]);
+
   const formatDuration = (ms: number) => {
     const seconds = Math.floor(ms / 1000);
     const minutes = Math.floor(seconds / 60);
@@ -264,53 +274,77 @@ export function ScraperMonitor() {
                     </div>
                   ))}
 
-                  {/* Individual Crawl Jobs */}
-                  {azActiveJobs.crawlJobs.map((job: any) => (
-                    <div key={`crawl-${job.id}`} style={{
+                  {/* Individual Crawl Jobs - Table View */}
+                  {azActiveJobs.crawlJobs.length > 0 && (
+                    <div style={{
                       background: 'white',
-                      padding: '20px',
                       borderRadius: '8px',
                       boxShadow: '0 2px 8px rgba(0,0,0,0.1)',
-                      borderLeft: '4px solid #3b82f6'
+                      overflow: 'hidden',
+                      marginTop: '15px'
                     }}>
-                      <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'start' }}>
-                        <div style={{ flex: 1 }}>
-                          <div style={{ fontSize: '18px', fontWeight: '600', marginBottom: '8px' }}>
-                            {job.dispensary_name || 'Unknown Store'}
-                          </div>
-                          <div style={{ fontSize: '14px', color: '#666', marginBottom: '12px' }}>
-                            {job.city} | {job.job_type || 'crawl'}
-                          </div>
-                          <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(120px, 1fr))', gap: '12px' }}>
-                            <div>
-                              <div style={{ fontSize: '12px', color: '#999', marginBottom: '4px' }}>Products Found</div>
-                              <div style={{ fontSize: '16px', fontWeight: '600', color: '#8b5cf6' }}>{job.products_found || 0}</div>
-                            </div>
-                            <div>
-                              <div style={{ fontSize: '12px', color: '#999', marginBottom: '4px' }}>Snapshots</div>
-                              <div style={{ fontSize: '16px', fontWeight: '600', color: '#06b6d4' }}>{job.snapshots_created || 0}</div>
-                            </div>
-                            <div>
-                              <div style={{ fontSize: '12px', color: '#999', marginBottom: '4px' }}>Duration</div>
-                              <div style={{ fontSize: '16px', fontWeight: '600' }}>
-                                {Math.floor((job.duration_seconds || 0) / 60)}m {Math.floor((job.duration_seconds || 0) % 60)}s
-                              </div>
-                            </div>
-                          </div>
-                        </div>
-                        <div style={{
-                          padding: '6px 12px',
-                          borderRadius: '4px',
-                          fontSize: '13px',
-                          fontWeight: '600',
-                          background: '#dbeafe',
-                          color: '#1e40af'
-                        }}>
-                          CRAWLING
-                        </div>
+                      <div style={{ padding: '15px 20px', borderBottom: '2px solid #eee', background: '#f8f8f8' }}>
+                        <h3 style={{ margin: 0, fontSize: '16px', fontWeight: '600' }}>Active Crawler Sessions ({azActiveJobs.crawlJobs.length})</h3>
                       </div>
+                      <table style={{ width: '100%', borderCollapse: 'collapse' }}>
+                        <thead>
+                          <tr style={{ borderBottom: '1px solid #eee' }}>
+                            <th style={{ padding: '12px 15px', textAlign: 'left', fontWeight: '600', fontSize: '13px', color: '#666' }}>Store</th>
+                            <th style={{ padding: '12px 15px', textAlign: 'left', fontWeight: '600', fontSize: '13px', color: '#666' }}>Worker</th>
+                            <th style={{ padding: '12px 15px', textAlign: 'center', fontWeight: '600', fontSize: '13px', color: '#666' }}>Page</th>
+                            <th style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', fontSize: '13px', color: '#666' }}>Products</th>
+                            <th style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', fontSize: '13px', color: '#666' }}>Snapshots</th>
+                            <th style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', fontSize: '13px', color: '#666' }}>Duration</th>
+                            <th style={{ padding: '12px 15px', textAlign: 'center', fontWeight: '600', fontSize: '13px', color: '#666' }}>Status</th>
+                          </tr>
+                        </thead>
+                        <tbody>
+                          {azActiveJobs.crawlJobs.map((job: any) => (
+                            <tr key={`crawl-${job.id}`} style={{ borderBottom: '1px solid #eee' }}>
+                              <td style={{ padding: '12px 15px' }}>
+                                <div style={{ fontWeight: '600', marginBottom: '2px' }}>{job.dispensary_name || 'Unknown'}</div>
+                                <div style={{ fontSize: '12px', color: '#999' }}>{job.city} | ID: {job.dispensary_id}</div>
+                              </td>
+                              <td style={{ padding: '12px 15px', fontSize: '13px' }}>
+                                <div style={{ fontFamily: 'monospace', fontSize: '11px', color: '#666' }}>
+                                  {job.worker_id ? job.worker_id.substring(0, 8) : '-'}
+                                </div>
+                                {job.worker_hostname && (
+                                  <div style={{ fontSize: '11px', color: '#999' }}>{job.worker_hostname}</div>
+                                )}
+                              </td>
+                              <td style={{ padding: '12px 15px', textAlign: 'center', fontSize: '13px' }}>
+                                {job.current_page && job.total_pages ? (
+                                  <span>{job.current_page}/{job.total_pages}</span>
+                                ) : '-'}
+                              </td>
+                              <td style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', color: '#8b5cf6' }}>
+                                {job.products_found || 0}
+                              </td>
+                              <td style={{ padding: '12px 15px', textAlign: 'right', fontWeight: '600', color: '#06b6d4' }}>
+                                {job.snapshots_created || 0}
+                              </td>
+                              <td style={{ padding: '12px 15px', textAlign: 'right', fontSize: '13px' }}>
+                                {Math.floor((job.duration_seconds || 0) / 60)}m {Math.floor((job.duration_seconds || 0) % 60)}s
+                              </td>
+                              <td style={{ padding: '12px 15px', textAlign: 'center' }}>
+                                <span style={{
+                                  padding: '4px 10px',
+                                  borderRadius: '12px',
+                                  fontSize: '11px',
+                                  fontWeight: '600',
+                                  background: job.last_heartbeat_at && (Date.now() - new Date(job.last_heartbeat_at).getTime() > 60000) ? '#fef3c7' : '#dbeafe',
+                                  color: job.last_heartbeat_at && (Date.now() - new Date(job.last_heartbeat_at).getTime() > 60000) ? '#92400e' : '#1e40af'
+                                }}>
+                                  {job.last_heartbeat_at && (Date.now() - new Date(job.last_heartbeat_at).getTime() > 60000) ? 'STALE' : 'CRAWLING'}
+                                </span>
+                              </td>
+                            </tr>
+                          ))}
+                        </tbody>
+                      </table>
                     </div>
-                  ))}
+                  )}
                 </div>
               )}
             </div>
@@ -843,6 +877,8 @@ export function ScraperMonitor() {
                       <th style={{ padding: '15px', textAlign: 'left', fontWeight: '600' }}>Dispensary</th>
                       <th style={{ padding: '15px', textAlign: 'left', fontWeight: '600' }}>Status</th>
                       <th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>Found</th>
+                      <th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>New</th>
+                      <th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>Updated</th>
                       <th style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>Products</th>
                       <th style={{ padding: '15px', textAlign: 'left', fontWeight: '600' }}>Last Crawled</th>
                     </tr>
@@ -866,6 +902,12 @@ export function ScraperMonitor() {
                         <td style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>
                           {item.products_found || '-'}
                         </td>
+                        <td style={{ padding: '15px', textAlign: 'right', fontWeight: '600', color: '#059669' }}>
+                          {item.products_new || 0}
+                        </td>
+                        <td style={{ padding: '15px', textAlign: 'right', fontWeight: '600', color: '#2563eb' }}>
+                          {item.products_updated || 0}
+                        </td>
                         <td style={{ padding: '15px', textAlign: 'right', fontWeight: '600' }}>
                           {item.product_count}
                         </td>