diff --git a/backend/Dockerfile b/backend/Dockerfile
index 84592ca0..8e9a8d3e 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -24,7 +24,6 @@ RUN apt-get update && apt-get install -y \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
-# Tell Puppeteer to use system Chromium
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
@@ -35,6 +34,9 @@ RUN npm ci --only=production
COPY --from=builder /app/dist ./dist
+# Create local images directory for when MinIO is not configured
+RUN mkdir -p /app/public/images/products
+
EXPOSE 3010
CMD ["node", "dist/index.js"]
diff --git a/backend/package-lock.json b/backend/package-lock.json
index 18139db6..7a703c09 100644
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -26,7 +26,7 @@
"puppeteer": "^21.0.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
- "sharp": "^0.34.5",
+ "sharp": "^0.32.0",
"socks-proxy-agent": "^8.0.2",
"uuid": "^9.0.1",
"zod": "^3.22.4"
@@ -38,6 +38,7 @@
"@types/jsonwebtoken": "^9.0.5",
"@types/node": "^20.10.5",
"@types/node-cron": "^3.0.11",
+ "@types/pg": "^8.15.6",
"@types/uuid": "^9.0.7",
"tsx": "^4.7.0",
"typescript": "^5.3.3"
@@ -64,15 +65,6 @@
"node": ">=6.9.0"
}
},
- "node_modules/@emnapi/runtime": {
- "version": "1.7.1",
- "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz",
- "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==",
- "optional": true,
- "dependencies": {
- "tslib": "^2.4.0"
- }
- },
"node_modules/@esbuild/aix-ppc64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
@@ -489,446 +481,6 @@
"node": ">=18"
}
},
- "node_modules/@img/colour": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz",
- "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==",
- "engines": {
- "node": ">=18"
- }
- },
- "node_modules/@img/sharp-darwin-arm64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz",
- "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "darwin"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-darwin-arm64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-darwin-x64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz",
- "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "darwin"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-darwin-x64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-libvips-darwin-arm64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz",
- "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "darwin"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-darwin-x64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz",
- "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "darwin"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linux-arm": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz",
- "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==",
- "cpu": [
- "arm"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linux-arm64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz",
- "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linux-ppc64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz",
- "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==",
- "cpu": [
- "ppc64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linux-riscv64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz",
- "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==",
- "cpu": [
- "riscv64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linux-s390x": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz",
- "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==",
- "cpu": [
- "s390x"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linux-x64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz",
- "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linuxmusl-arm64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz",
- "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-libvips-linuxmusl-x64": {
- "version": "1.2.4",
- "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz",
- "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-linux-arm": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz",
- "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==",
- "cpu": [
- "arm"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linux-arm": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linux-arm64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz",
- "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linux-arm64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linux-ppc64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz",
- "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==",
- "cpu": [
- "ppc64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linux-ppc64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linux-riscv64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz",
- "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==",
- "cpu": [
- "riscv64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linux-riscv64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linux-s390x": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz",
- "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==",
- "cpu": [
- "s390x"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linux-s390x": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linux-x64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz",
- "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linux-x64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linuxmusl-arm64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz",
- "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linuxmusl-arm64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-linuxmusl-x64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz",
- "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "linux"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-libvips-linuxmusl-x64": "1.2.4"
- }
- },
- "node_modules/@img/sharp-wasm32": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz",
- "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==",
- "cpu": [
- "wasm32"
- ],
- "optional": true,
- "dependencies": {
- "@emnapi/runtime": "^1.7.0"
- },
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-win32-arm64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz",
- "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==",
- "cpu": [
- "arm64"
- ],
- "optional": true,
- "os": [
- "win32"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-win32-ia32": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz",
- "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==",
- "cpu": [
- "ia32"
- ],
- "optional": true,
- "os": [
- "win32"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
- "node_modules/@img/sharp-win32-x64": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz",
- "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==",
- "cpu": [
- "x64"
- ],
- "optional": true,
- "os": [
- "win32"
- ],
- "engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
- },
- "funding": {
- "url": "https://opencollective.com/libvips"
- }
- },
"node_modules/@mapbox/node-pre-gyp": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
@@ -1149,6 +701,17 @@
"integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==",
"dev": true
},
+ "node_modules/@types/pg": {
+ "version": "8.15.6",
+ "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz",
+ "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==",
+ "dev": true,
+ "dependencies": {
+ "@types/node": "*",
+ "pg-protocol": "*",
+ "pg-types": "^2.2.0"
+ }
+ },
"node_modules/@types/qs": {
"version": "6.14.0",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz",
@@ -1411,6 +974,16 @@
"node": ">= 10.0.0"
}
},
+ "node_modules/bl": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+ "dependencies": {
+ "buffer": "^5.5.0",
+ "inherits": "^2.0.4",
+ "readable-stream": "^3.4.0"
+ }
+ },
"node_modules/block-stream2": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz",
@@ -1600,6 +1173,18 @@
"node": ">=0.10.0"
}
},
+ "node_modules/color": {
+ "version": "4.2.3",
+ "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
+ "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
+ "dependencies": {
+ "color-convert": "^2.0.1",
+ "color-string": "^1.9.0"
+ },
+ "engines": {
+ "node": ">=12.5.0"
+ }
+ },
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -1616,6 +1201,15 @@
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
+ "node_modules/color-string": {
+ "version": "1.9.1",
+ "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
+ "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
+ "dependencies": {
+ "color-name": "^1.0.0",
+ "simple-swizzle": "^0.2.2"
+ }
+ },
"node_modules/color-support": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
@@ -1746,6 +1340,28 @@
"node": ">=0.10"
}
},
+ "node_modules/decompress-response": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+ "dependencies": {
+ "mimic-response": "^3.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/deep-extend": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+ "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
@@ -2059,6 +1675,14 @@
"bare-events": "^2.7.0"
}
},
+ "node_modules/expand-template": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+ "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/express": {
"version": "4.21.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
@@ -2296,6 +1920,11 @@
"node": ">= 0.6"
}
},
+ "node_modules/fs-constants": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+ },
"node_modules/fs-extra": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
@@ -2489,6 +2118,11 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
+ "node_modules/github-from-package": {
+ "version": "0.0.0",
+ "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
+ },
"node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@@ -2726,6 +2360,11 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
+ "node_modules/ini": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
+ },
"node_modules/ip-address": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz",
@@ -3113,6 +2752,17 @@
"node": ">= 0.6"
}
},
+ "node_modules/mimic-response": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -3124,6 +2774,14 @@
"node": "*"
}
},
+ "node_modules/minimist": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/minio": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz",
@@ -3225,6 +2883,11 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
+ "node_modules/napi-build-utils": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
+ "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA=="
+ },
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3241,6 +2904,17 @@
"node": ">= 0.4.0"
}
},
+ "node_modules/node-abi": {
+ "version": "3.85.0",
+ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.85.0.tgz",
+ "integrity": "sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg==",
+ "dependencies": {
+ "semver": "^7.3.5"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
"node_modules/node-addon-api": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
@@ -3667,6 +3341,62 @@
"node": ">=0.10.0"
}
},
+ "node_modules/prebuild-install": {
+ "version": "7.1.3",
+ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
+ "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",
+ "dependencies": {
+ "detect-libc": "^2.0.0",
+ "expand-template": "^2.0.3",
+ "github-from-package": "0.0.0",
+ "minimist": "^1.2.3",
+ "mkdirp-classic": "^0.5.3",
+ "napi-build-utils": "^2.0.0",
+ "node-abi": "^3.3.0",
+ "pump": "^3.0.0",
+ "rc": "^1.2.7",
+ "simple-get": "^4.0.0",
+ "tar-fs": "^2.0.0",
+ "tunnel-agent": "^0.6.0"
+ },
+ "bin": {
+ "prebuild-install": "bin.js"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
+ "node_modules/prebuild-install/node_modules/chownr": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+ },
+ "node_modules/prebuild-install/node_modules/tar-fs": {
+ "version": "2.1.4",
+ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+ "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
+ "dependencies": {
+ "chownr": "^1.1.1",
+ "mkdirp-classic": "^0.5.2",
+ "pump": "^3.0.0",
+ "tar-stream": "^2.1.4"
+ }
+ },
+ "node_modules/prebuild-install/node_modules/tar-stream": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+ "dependencies": {
+ "bl": "^4.0.3",
+ "end-of-stream": "^1.4.1",
+ "fs-constants": "^1.0.0",
+ "inherits": "^2.0.3",
+ "readable-stream": "^3.1.1"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/progress": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
@@ -4092,6 +3822,20 @@
"node": ">= 0.8"
}
},
+ "node_modules/rc": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+ "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+ "dependencies": {
+ "deep-extend": "^0.6.0",
+ "ini": "~1.3.0",
+ "minimist": "^1.2.0",
+ "strip-json-comments": "~2.0.1"
+ },
+ "bin": {
+ "rc": "cli.js"
+ }
+ },
"node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -4311,49 +4055,32 @@
}
},
"node_modules/sharp": {
- "version": "0.34.5",
- "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz",
- "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==",
+ "version": "0.32.6",
+ "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz",
+ "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==",
"hasInstallScript": true,
- "license": "Apache-2.0",
"dependencies": {
- "@img/colour": "^1.0.0",
- "detect-libc": "^2.1.2",
- "semver": "^7.7.3"
+ "color": "^4.2.3",
+ "detect-libc": "^2.0.2",
+ "node-addon-api": "^6.1.0",
+ "prebuild-install": "^7.1.1",
+ "semver": "^7.5.4",
+ "simple-get": "^4.0.1",
+ "tar-fs": "^3.0.4",
+ "tunnel-agent": "^0.6.0"
},
"engines": {
- "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+ "node": ">=14.15.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
- },
- "optionalDependencies": {
- "@img/sharp-darwin-arm64": "0.34.5",
- "@img/sharp-darwin-x64": "0.34.5",
- "@img/sharp-libvips-darwin-arm64": "1.2.4",
- "@img/sharp-libvips-darwin-x64": "1.2.4",
- "@img/sharp-libvips-linux-arm": "1.2.4",
- "@img/sharp-libvips-linux-arm64": "1.2.4",
- "@img/sharp-libvips-linux-ppc64": "1.2.4",
- "@img/sharp-libvips-linux-riscv64": "1.2.4",
- "@img/sharp-libvips-linux-s390x": "1.2.4",
- "@img/sharp-libvips-linux-x64": "1.2.4",
- "@img/sharp-libvips-linuxmusl-arm64": "1.2.4",
- "@img/sharp-libvips-linuxmusl-x64": "1.2.4",
- "@img/sharp-linux-arm": "0.34.5",
- "@img/sharp-linux-arm64": "0.34.5",
- "@img/sharp-linux-ppc64": "0.34.5",
- "@img/sharp-linux-riscv64": "0.34.5",
- "@img/sharp-linux-s390x": "0.34.5",
- "@img/sharp-linux-x64": "0.34.5",
- "@img/sharp-linuxmusl-arm64": "0.34.5",
- "@img/sharp-linuxmusl-x64": "0.34.5",
- "@img/sharp-wasm32": "0.34.5",
- "@img/sharp-win32-arm64": "0.34.5",
- "@img/sharp-win32-ia32": "0.34.5",
- "@img/sharp-win32-x64": "0.34.5"
}
},
+ "node_modules/sharp/node_modules/node-addon-api": {
+ "version": "6.1.0",
+ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
+ "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
+ },
"node_modules/side-channel": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
@@ -4427,6 +4154,62 @@
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
},
+ "node_modules/simple-concat": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
+ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
+ "node_modules/simple-get": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
+ "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "dependencies": {
+ "decompress-response": "^6.0.0",
+ "once": "^1.3.1",
+ "simple-concat": "^1.0.0"
+ }
+ },
+ "node_modules/simple-swizzle": {
+ "version": "0.2.4",
+ "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz",
+ "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==",
+ "dependencies": {
+ "is-arrayish": "^0.3.1"
+ }
+ },
+ "node_modules/simple-swizzle/node_modules/is-arrayish": {
+ "version": "0.3.4",
+ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz",
+ "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA=="
+ },
"node_modules/smart-buffer": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
@@ -4566,6 +4349,14 @@
"node": ">=8"
}
},
+ "node_modules/strip-json-comments": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+ "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/strnum": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz",
@@ -4671,6 +4462,17 @@
"fsevents": "~2.3.3"
}
},
+ "node_modules/tunnel-agent": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+ "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+ "dependencies": {
+ "safe-buffer": "^5.0.1"
+ },
+ "engines": {
+ "node": "*"
+ }
+ },
"node_modules/type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
diff --git a/backend/package.json b/backend/package.json
index 032ce072..9706a1d0 100755
--- a/backend/package.json
+++ b/backend/package.json
@@ -29,7 +29,7 @@
"puppeteer": "^21.0.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
- "sharp": "^0.34.5",
+ "sharp": "^0.32.0",
"socks-proxy-agent": "^8.0.2",
"uuid": "^9.0.1",
"zod": "^3.22.4"
@@ -41,6 +41,7 @@
"@types/jsonwebtoken": "^9.0.5",
"@types/node": "^20.10.5",
"@types/node-cron": "^3.0.11",
+ "@types/pg": "^8.15.6",
"@types/uuid": "^9.0.7",
"tsx": "^4.7.0",
"typescript": "^5.3.3"
diff --git a/backend/src/index.ts b/backend/src/index.ts
index 56aa55d1..3a2f55e7 100755
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -1,7 +1,8 @@
import express from 'express';
import cors from 'cors';
+import path from 'path';
import dotenv from 'dotenv';
-import { initializeMinio } from './utils/minio';
+import { initializeMinio, isMinioEnabled } from './utils/minio';
import { logger } from './services/logger';
import { cleanupOrphanedJobs } from './services/proxyTestQueue';
@@ -13,10 +14,25 @@ const PORT = process.env.PORT || 3010;
app.use(cors());
app.use(express.json());
+// Serve static images when MinIO is not configured
+const LOCAL_IMAGES_PATH = process.env.LOCAL_IMAGES_PATH || '/app/public/images';
+app.use('/images', express.static(LOCAL_IMAGES_PATH));
+
app.get('/health', (req, res) => {
res.json({ status: 'ok', timestamp: new Date().toISOString() });
});
+// Endpoint to check server's outbound IP (for proxy whitelist setup)
+app.get('/outbound-ip', async (req, res) => {
+ try {
+ const axios = require('axios');
+ const response = await axios.get('https://api.ipify.org?format=json', { timeout: 10000 });
+ res.json({ outbound_ip: response.data.ip });
+ } catch (error: any) {
+ res.status(500).json({ error: error.message });
+ }
+});
+
import authRoutes from './routes/auth';
import dashboardRoutes from './routes/dashboard';
import storesRoutes from './routes/stores';
@@ -32,6 +48,7 @@ import logsRoutes from './routes/logs';
import scraperMonitorRoutes from './routes/scraper-monitor';
import apiTokensRoutes from './routes/api-tokens';
import apiPermissionsRoutes from './routes/api-permissions';
+import parallelScrapeRoutes from './routes/parallel-scrape';
import { trackApiUsage, checkRateLimit } from './middleware/apiTokenTracker';
import { validateWordPressPermissions } from './middleware/wordpressPermissions';
@@ -57,13 +74,14 @@ app.use('/api/logs', logsRoutes);
app.use('/api/scraper-monitor', scraperMonitorRoutes);
app.use('/api/api-tokens', apiTokensRoutes);
app.use('/api/api-permissions', apiPermissionsRoutes);
+app.use('/api/parallel-scrape', parallelScrapeRoutes);
async function startServer() {
try {
logger.info('system', 'Starting server...');
await initializeMinio();
- logger.info('system', 'Minio initialized');
+ logger.info('system', isMinioEnabled() ? 'MinIO storage initialized' : 'Local filesystem storage initialized');
// Clean up any orphaned proxy test jobs from previous server runs
await cleanupOrphanedJobs();
diff --git a/backend/src/middleware/wordpressPermissions.ts b/backend/src/middleware/wordpressPermissions.ts
index b356612e..66db2f89 100644
--- a/backend/src/middleware/wordpressPermissions.ts
+++ b/backend/src/middleware/wordpressPermissions.ts
@@ -161,7 +161,7 @@ export async function validateWordPressPermissions(
UPDATE wp_dutchie_api_permissions
SET last_used_at = CURRENT_TIMESTAMP
WHERE id = $1
- `, [permission.id]).catch(err => {
+ `, [permission.id]).catch((err: Error) => {
console.error('Error updating last_used_at:', err);
});
diff --git a/backend/src/routes/categories.ts b/backend/src/routes/categories.ts
index 21908b0d..7d61e918 100644
--- a/backend/src/routes/categories.ts
+++ b/backend/src/routes/categories.ts
@@ -67,12 +67,12 @@ router.get('/tree', async (req, res) => {
const tree: any[] = [];
// First pass: create map
- categories.forEach(cat => {
+ categories.forEach((cat: { id: number; parent_id?: number }) => {
categoryMap.set(cat.id, { ...cat, children: [] });
});
-
+
// Second pass: build tree
- categories.forEach(cat => {
+ categories.forEach((cat: { id: number; parent_id?: number }) => {
const node = categoryMap.get(cat.id);
if (cat.parent_id) {
const parent = categoryMap.get(cat.parent_id);
diff --git a/backend/src/routes/parallel-scrape.ts b/backend/src/routes/parallel-scrape.ts
new file mode 100644
index 00000000..aa057a81
--- /dev/null
+++ b/backend/src/routes/parallel-scrape.ts
@@ -0,0 +1,252 @@
+import { Router } from 'express';
+import { pool } from '../db/migrate';
+import { getActiveProxy, putProxyInTimeout, isBotDetectionError } from '../services/proxy';
+import { authMiddleware } from '../auth/middleware';
+
+const router = Router();
+router.use(authMiddleware);
+
+const FIREFOX_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0';
+
+interface ScrapeJob {
+ id: string;
+ storeName: string;
+ status: 'running' | 'completed' | 'failed';
+ workers: number;
+ startedAt: Date;
+ completedAt?: Date;
+ results: {
+ category: string;
+ success: boolean;
+ products: number;
+ error?: string;
+ }[];
+}
+
+// In-memory job tracking
+const activeJobs = new Map {error} {product.brand} {product.description} {metadata.lineage}Product not found
+
+ ) : (
+
{product.name}
+
+ {product.brand && (
+ Cannabinoid Content
+ Description
+ Terpenes
+ Effects
+ Flavors
+ Lineage
+