From e518bb8169cac71be954c64e4228e419f58c7e83 Mon Sep 17 00:00:00 2001
From: Kelly <kelly@cannabrands.com>
Date: Mon, 1 Dec 2025 09:33:32 -0700
Subject: [PATCH] Fix Dutchie scraper to wait for React content to load
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Change waitUntil from 'domcontentloaded' to 'networkidle2' for SPAs
- Add waitForSelector to wait for product elements before parsing
- WordPress plugin: update API endpoints to use hardcoded URL

The scraper was returning 0 products because it wasn't waiting for
React to render the product list. Now it properly waits for either
the product list items or an empty state indicator.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 backend/src/scraper-v2/downloader.ts | 19 +++++++++---
 wordpress-plugin/dutchie-menus.php   | 45 ++++++++--------------------
 2 files changed, 28 insertions(+), 36 deletions(-)

diff --git a/backend/src/scraper-v2/downloader.ts b/backend/src/scraper-v2/downloader.ts
index 4614b408..455d0c21 100644
--- a/backend/src/scraper-v2/downloader.ts
+++ b/backend/src/scraper-v2/downloader.ts
@@ -378,9 +378,9 @@ export class Downloader {
         await page.setUserAgent(request.metadata.userAgent);
       }
 
-      // Navigate to page
+      // Navigate to page - use networkidle2 for SPAs like Dutchie
       const navigationPromise = page.goto(request.url, {
-        waitUntil: 'domcontentloaded',
+        waitUntil: 'networkidle2',
         timeout: 60000
       });
 
@@ -390,8 +390,19 @@ export class Downloader {
         throw new Error('Navigation failed - no response');
       }
 
-      // Wait for initial render
-      await page.waitForTimeout(3000);
+      // Wait for React to render product content
+      // Try to wait for products, but don't fail if they don't appear (empty category)
+      try {
+        await page.waitForSelector('[data-testid="product-list-item"], [data-testid="empty-state"]', {
+          timeout: 10000
+        });
+      } catch {
+        // Products might not exist in this category - continue anyway
+        logger.debug('scraper', 'No products found within timeout - continuing');
+      }
+
+      // Additional wait for any lazy-loaded content
+      await page.waitForTimeout(2000);
 
       // Check for lazy-loaded content
       await this.autoScroll(page);
diff --git a/wordpress-plugin/dutchie-menus.php b/wordpress-plugin/dutchie-menus.php
index 74fc276b..30a774ea 100644
--- a/wordpress-plugin/dutchie-menus.php
+++ b/wordpress-plugin/dutchie-menus.php
@@ -1,11 +1,11 @@
 <?php
 /**
  * Plugin Name: Dutchie Menus
- * Plugin URI: https://github.com/yourusername/dutchie-menus
+ * Plugin URI: https://creationshop.io
  * Description: Display cannabis product menus from your Dutchie scraper with Elementor integration
- * Version: 1.0.0
- * Author: Your Name
- * Author URI: https://yoursite.com
+ * Version: 1.1.0
+ * Author: Creationshop
+ * Author URI: https://creationshop.io
  * License: GPL v2 or later
  * Text Domain: dutchie-menus
  * Requires PHP: 7.4
@@ -15,7 +15,8 @@ if (!defined('ABSPATH')) {
     exit; // Exit if accessed directly
 }
 
-define('DUTCHIE_MENUS_VERSION', '1.0.0');
+define('DUTCHIE_MENUS_VERSION', '1.1.0');
+define('DUTCHIE_MENUS_API_URL', 'https://dispos.crawlsy.com/api');
 define('DUTCHIE_MENUS_PLUGIN_DIR', plugin_dir_path(__FILE__));
 define('DUTCHIE_MENUS_PLUGIN_URL', plugin_dir_url(__FILE__));
 
@@ -100,9 +101,7 @@ class Dutchie_Menus_Plugin {
      * Register Plugin Settings
      */
     public function register_settings() {
-        register_setting('dutchie_menus_settings', 'dutchie_api_url');
         register_setting('dutchie_menus_settings', 'dutchie_api_token');
-        register_setting('dutchie_menus_settings', 'dutchie_default_store_id');
     }
 
     /**
@@ -112,36 +111,20 @@ class Dutchie_Menus_Plugin {
         ?>
         <div class="wrap">
             <h1>Dutchie Menus Settings</h1>
+            <p>Version <?php echo DUTCHIE_MENUS_VERSION; ?> by <a href="https://creationshop.io" target="_blank">Creationshop</a></p>
+
             <form method="post" action="options.php">
                 <?php settings_fields('dutchie_menus_settings'); ?>
                 <?php do_settings_sections('dutchie_menus_settings'); ?>
 
                 <table class="form-table">
-                    <tr>
-                        <th scope="row"><label for="dutchie_api_url">API URL</label></th>
-                        <td>
-                            <input type="url" id="dutchie_api_url" name="dutchie_api_url"
-                                   value="<?php echo esc_attr(get_option('dutchie_api_url', 'http://localhost:3010')); ?>"
-                                   class="regular-text" />
-                            <p class="description">Your Dutchie Menus API endpoint (e.g., http://localhost:3010)</p>
-                        </td>
-                    </tr>
                     <tr>
                         <th scope="row"><label for="dutchie_api_token">API Token</label></th>
                         <td>
                             <input type="password" id="dutchie_api_token" name="dutchie_api_token"
                                    value="<?php echo esc_attr(get_option('dutchie_api_token')); ?>"
                                    class="regular-text" />
-                            <p class="description">Your authentication token from the admin dashboard</p>
-                        </td>
-                    </tr>
-                    <tr>
-                        <th scope="row"><label for="dutchie_default_store_id">Default Store ID</label></th>
-                        <td>
-                            <input type="number" id="dutchie_default_store_id" name="dutchie_default_store_id"
-                                   value="<?php echo esc_attr(get_option('dutchie_default_store_id', '1')); ?>"
-                                   class="small-text" />
-                            <p class="description">Default store ID to use</p>
+                            <p class="description">Your authentication token from the admin dashboard. The token includes your store configuration.</p>
                         </td>
                     </tr>
                 </table>
@@ -158,13 +141,13 @@ class Dutchie_Menus_Plugin {
             <script>
             jQuery(document).ready(function($) {
                 $('#test-api-connection').on('click', function() {
-                    var apiUrl = $('#dutchie_api_url').val();
+                    var apiUrl = '<?php echo DUTCHIE_MENUS_API_URL; ?>';
                     var apiToken = $('#dutchie_api_token').val();
 
                     $('#api-test-result').html('<p>Testing connection...</p>');
 
                     $.ajax({
-                        url: apiUrl + '/api/auth/me',
+                        url: apiUrl + '/auth/me',
                         method: 'GET',
                         headers: {
                             'Authorization': 'Bearer ' + apiToken
@@ -239,7 +222,6 @@ class Dutchie_Menus_Plugin {
      * Fetch Products from API
      */
     public function fetch_products($args = []) {
-        $api_url = get_option('dutchie_api_url', 'http://localhost:3010');
         $api_token = get_option('dutchie_api_token');
 
         if (!$api_token) {
@@ -247,7 +229,7 @@ class Dutchie_Menus_Plugin {
         }
 
         $query_args = http_build_query($args);
-        $url = $api_url . '/api/products?' . $query_args;
+        $url = DUTCHIE_MENUS_API_URL . '/products?' . $query_args;
 
         $response = wp_remote_get($url, [
             'headers' => [
@@ -270,14 +252,13 @@ class Dutchie_Menus_Plugin {
      * Fetch Single Product from API
      */
     public function fetch_product($id) {
-        $api_url = get_option('dutchie_api_url', 'http://localhost:3010');
         $api_token = get_option('dutchie_api_token');
 
         if (!$api_token) {
             return false;
         }
 
-        $url = $api_url . '/api/products/' . intval($id);
+        $url = DUTCHIE_MENUS_API_URL . '/products/' . intval($id);
 
         $response = wp_remote_get($url, [
             'headers' => [