chore: bump task worker version comment
Force new git SHA to avoid CI scientific notation bug. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
179
backend/scripts/test-all-brands-scroll.ts
Normal file
179
backend/scripts/test-all-brands-scroll.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Loading ALL brands from https://shop.bestdispensary.com/brands');
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Age gate detected, bypassing...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
console.log('Current URL:', page.url());
|
||||
|
||||
// Get initial brand count
|
||||
let brandCount = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href) seen.add(href);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
console.log(`Initial brand count: ${brandCount}`);
|
||||
|
||||
// Aggressive scrolling
|
||||
console.log('\nScrolling to load ALL brands...');
|
||||
let previousCount = 0;
|
||||
let sameCount = 0;
|
||||
|
||||
for (let i = 0; i < 50; i++) {
|
||||
// Scroll to bottom
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1000);
|
||||
|
||||
brandCount = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href) seen.add(href);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
|
||||
if (brandCount === previousCount) {
|
||||
sameCount++;
|
||||
if (sameCount >= 5) {
|
||||
console.log(` Scroll ${i+1}: ${brandCount} brands (stopping - no change)`);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
sameCount = 0;
|
||||
console.log(` Scroll ${i+1}: ${brandCount} brands`);
|
||||
}
|
||||
previousCount = brandCount;
|
||||
}
|
||||
|
||||
// Get all unique brands
|
||||
const brands = await page.evaluate(() => {
|
||||
const results: { name: string; href: string }[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const normalizedHref = href.toLowerCase();
|
||||
if (seen.has(normalizedHref)) return;
|
||||
seen.add(normalizedHref);
|
||||
|
||||
// Get brand name
|
||||
let name = '';
|
||||
const heading = a.querySelector('h3, h4, h5, [class*="name"]');
|
||||
if (heading) {
|
||||
name = heading.textContent?.trim() || '';
|
||||
}
|
||||
if (!name) {
|
||||
name = a.textContent?.trim().split('\n')[0] || '';
|
||||
}
|
||||
if (!name) {
|
||||
name = href.split('/brand/')[1]?.replace(/-/g, ' ') || '';
|
||||
}
|
||||
|
||||
results.push({ name: name.slice(0, 50), href });
|
||||
});
|
||||
|
||||
return results.sort((a, b) => a.name.localeCompare(b.name));
|
||||
});
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('TOTAL BRANDS FOUND: ' + brands.length);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
brands.forEach((b, i) => {
|
||||
const num = (i + 1).toString().padStart(3, ' ');
|
||||
console.log(`${num}. ${b.name} (${b.href})`);
|
||||
});
|
||||
|
||||
// Now visit each brand page and count products
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('PRODUCTS PER BRAND');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const brandProducts: { brand: string; products: number }[] = [];
|
||||
|
||||
for (let i = 0; i < brands.length; i++) {
|
||||
const brand = brands[i];
|
||||
try {
|
||||
const brandUrl = brand.href.startsWith('http')
|
||||
? brand.href
|
||||
: `https://shop.bestdispensary.com${brand.href}`;
|
||||
|
||||
await page.goto(brandUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(1500);
|
||||
|
||||
// Scroll to load products
|
||||
for (let j = 0; j < 10; j++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(800);
|
||||
}
|
||||
|
||||
const productCount = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((a: Element) => {
|
||||
const img = a.querySelector('img');
|
||||
const name = img?.getAttribute('alt') || a.textContent?.trim() || '';
|
||||
if (name) seen.add(name);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
|
||||
brandProducts.push({ brand: brand.name, products: productCount });
|
||||
console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ${productCount} products`);
|
||||
|
||||
} catch (err: any) {
|
||||
console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ERROR - ${err.message?.slice(0, 30)}`);
|
||||
brandProducts.push({ brand: brand.name, products: 0 });
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
const totalProducts = brandProducts.reduce((sum, b) => sum + b.products, 0);
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('SUMMARY');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Total brands: ${brands.length}`);
|
||||
console.log(`Total products: ${totalProducts}`);
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
92
backend/scripts/test-bestdispensary-brands.ts
Normal file
92
backend/scripts/test-bestdispensary-brands.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Navigating to https://shop.bestdispensary.com/brands');
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
// Go directly to the brands page
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate if present
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Age gate detected, bypassing...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
console.log('Current URL:', page.url());
|
||||
|
||||
// Scroll to load all content
|
||||
console.log('\nScrolling to load all brands...');
|
||||
let previousHeight = 0;
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1500);
|
||||
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (currentHeight === previousHeight) {
|
||||
console.log(` Scroll ${i+1}: No new content`);
|
||||
break;
|
||||
}
|
||||
previousHeight = currentHeight;
|
||||
|
||||
const brandCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('a[href*="/brand/"]').length
|
||||
);
|
||||
console.log(` Scroll ${i+1}: height=${currentHeight}, brand links=${brandCount}`);
|
||||
}
|
||||
|
||||
// Get all brand links
|
||||
const brands = await page.evaluate(() => {
|
||||
const results: { name: string; href: string }[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (seen.has(href)) return;
|
||||
seen.add(href);
|
||||
|
||||
const name = a.textContent?.trim() || href.split('/brand/')[1] || '';
|
||||
results.push({ name, href });
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log(`\nFound ${brands.length} brands:`);
|
||||
brands.forEach(b => console.log(` - ${b.name} (${b.href})`));
|
||||
|
||||
// Take screenshot
|
||||
await page.screenshot({ path: '/tmp/bestdispensary-brands.png', fullPage: true });
|
||||
console.log('\nScreenshot saved to /tmp/bestdispensary-brands.png');
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
108
backend/scripts/test-brands-debug.ts
Normal file
108
backend/scripts/test-brands-debug.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Check Load More button
|
||||
const btnInfo = await page.evaluate(() => {
|
||||
const btn = document.querySelector('button.collection__load-more');
|
||||
if (!btn) return { found: false };
|
||||
|
||||
const rect = btn.getBoundingClientRect();
|
||||
return {
|
||||
found: true,
|
||||
text: btn.textContent?.trim(),
|
||||
visible: rect.width > 0 && rect.height > 0,
|
||||
top: rect.top,
|
||||
disabled: (btn as HTMLButtonElement).disabled,
|
||||
class: btn.className,
|
||||
};
|
||||
});
|
||||
|
||||
console.log('Load More button:', btnInfo);
|
||||
|
||||
// Scroll to button and click
|
||||
console.log('\nScrolling to button and clicking...');
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (!btn) {
|
||||
console.log('Button not found');
|
||||
break;
|
||||
}
|
||||
|
||||
// Scroll button into view
|
||||
await page.evaluate((b) => b.scrollIntoView({ behavior: 'smooth', block: 'center' }), btn);
|
||||
await sleep(500);
|
||||
|
||||
// Check if button is still there and clickable
|
||||
const stillThere = await page.evaluate(() => {
|
||||
const b = document.querySelector('button.collection__load-more');
|
||||
return b ? b.textContent?.trim() : null;
|
||||
});
|
||||
|
||||
if (!stillThere) {
|
||||
console.log('Button disappeared - all loaded');
|
||||
break;
|
||||
}
|
||||
|
||||
// Click it
|
||||
await btn.click();
|
||||
console.log(`Click ${i+1}...`);
|
||||
await sleep(2000);
|
||||
|
||||
const count = await page.evaluate(() =>
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').length
|
||||
);
|
||||
console.log(` Brands: ${count}`);
|
||||
}
|
||||
|
||||
// Final count
|
||||
const brands = await page.evaluate(() => {
|
||||
const list: string[] = [];
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').forEach((a: Element) => {
|
||||
list.push(a.textContent?.trim() || '');
|
||||
});
|
||||
return list;
|
||||
});
|
||||
|
||||
console.log(`\nTotal brands: ${brands.length}`);
|
||||
console.log(brands.join(', '));
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
157
backend/scripts/test-brands-load-all.ts
Normal file
157
backend/scripts/test-brands-load-all.ts
Normal file
@@ -0,0 +1,157 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Click "LOAD MORE" until all brands are loaded
|
||||
console.log('Loading all brands...\n');
|
||||
|
||||
let loadMoreClicks = 0;
|
||||
while (true) {
|
||||
const loadMoreBtn = await page.$('button.collection__load-more');
|
||||
if (!loadMoreBtn) {
|
||||
console.log('No more "Load More" button - all brands loaded!');
|
||||
break;
|
||||
}
|
||||
|
||||
const isVisible = await page.evaluate((btn) => {
|
||||
const rect = btn.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, loadMoreBtn);
|
||||
|
||||
if (!isVisible) {
|
||||
console.log('Load More button not visible - all brands loaded!');
|
||||
break;
|
||||
}
|
||||
|
||||
await loadMoreBtn.click();
|
||||
loadMoreClicks++;
|
||||
await sleep(1500);
|
||||
|
||||
const brandCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').length
|
||||
);
|
||||
console.log(` Click ${loadMoreClicks}: ${brandCount} brands loaded`);
|
||||
|
||||
if (loadMoreClicks > 20) break; // Safety limit
|
||||
}
|
||||
|
||||
// Get all brands
|
||||
const brands = await page.evaluate(() => {
|
||||
const results: { name: string; href: string }[] = [];
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const name = a.textContent?.trim() || '';
|
||||
if (name && href) {
|
||||
results.push({ name, href });
|
||||
}
|
||||
});
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log(`TOTAL BRANDS: ${brands.length}`);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
// Visit each brand and count products
|
||||
console.log('\nCounting products per brand...\n');
|
||||
|
||||
const results: { brand: string; products: number }[] = [];
|
||||
|
||||
for (let i = 0; i < brands.length; i++) {
|
||||
const brand = brands[i];
|
||||
const brandUrl = `https://shop.bestdispensary.com${brand.href}`;
|
||||
|
||||
try {
|
||||
await page.goto(brandUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(1000);
|
||||
|
||||
// Click load more on brand page too
|
||||
for (let j = 0; j < 10; j++) {
|
||||
const loadMore = await page.$('button.collection__load-more');
|
||||
if (!loadMore) break;
|
||||
|
||||
const isVisible = await page.evaluate((btn) => {
|
||||
const rect = btn.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, loadMore);
|
||||
|
||||
if (!isVisible) break;
|
||||
await loadMore.click();
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
const productCount = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href) seen.add(href);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
|
||||
results.push({ brand: brand.name, products: productCount });
|
||||
console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ${productCount} products`);
|
||||
|
||||
} catch (err: any) {
|
||||
console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ERROR`);
|
||||
results.push({ brand: brand.name, products: 0 });
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
const totalProducts = results.reduce((sum, r) => sum + r.products, 0);
|
||||
const brandsWithProducts = results.filter(r => r.products > 0).length;
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('SUMMARY');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Total brands: ${brands.length}`);
|
||||
console.log(`Brands with products: ${brandsWithProducts}`);
|
||||
console.log(`Total products: ${totalProducts}`);
|
||||
|
||||
// Top brands by product count
|
||||
console.log('\nTop 20 brands by product count:');
|
||||
results
|
||||
.sort((a, b) => b.products - a.products)
|
||||
.slice(0, 20)
|
||||
.forEach((r, i) => console.log(` ${i+1}. ${r.brand}: ${r.products}`));
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
108
backend/scripts/test-brands-products.ts
Normal file
108
backend/scripts/test-brands-products.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Try clicking Load More multiple times with JS
|
||||
console.log('Loading all brands...');
|
||||
for (let i = 0; i < 15; i++) {
|
||||
const clicked = await page.evaluate(() => {
|
||||
const btn = document.querySelector('button.collection__load-more') as HTMLButtonElement;
|
||||
if (btn) { btn.click(); return true; }
|
||||
return false;
|
||||
});
|
||||
if (!clicked) break;
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Get all brands
|
||||
const brands = await page.evaluate(() => {
|
||||
const list: { name: string; href: string }[] = [];
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').forEach((a: Element) => {
|
||||
list.push({
|
||||
name: a.textContent?.trim() || '',
|
||||
href: a.getAttribute('href') || '',
|
||||
});
|
||||
});
|
||||
return list;
|
||||
});
|
||||
|
||||
console.log('Total brands found: ' + brands.length + '\n');
|
||||
console.log('PRODUCTS PER BRAND');
|
||||
console.log('==================\n');
|
||||
|
||||
const results: { brand: string; products: number }[] = [];
|
||||
|
||||
for (let i = 0; i < brands.length; i++) {
|
||||
const brand = brands[i];
|
||||
const url = 'https://shop.bestdispensary.com' + brand.href;
|
||||
|
||||
try {
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(1000);
|
||||
|
||||
// Click load more on brand page
|
||||
for (let j = 0; j < 20; j++) {
|
||||
const clicked = await page.evaluate(() => {
|
||||
const btn = document.querySelector('button.collection__load-more') as HTMLButtonElement;
|
||||
if (btn) { btn.click(); return true; }
|
||||
return false;
|
||||
});
|
||||
if (!clicked) break;
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
const productCount = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href) seen.add(href);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
|
||||
results.push({ brand: brand.name, products: productCount });
|
||||
const num = (i + 1).toString().padStart(2, ' ');
|
||||
console.log(num + '. ' + brand.name + ': ' + productCount);
|
||||
|
||||
} catch (err) {
|
||||
results.push({ brand: brand.name, products: 0 });
|
||||
const num = (i + 1).toString().padStart(2, ' ');
|
||||
console.log(num + '. ' + brand.name + ': ERROR');
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
const total = results.reduce((s, r) => s + r.products, 0);
|
||||
console.log('\n==================');
|
||||
console.log('TOTAL: ' + brands.length + ' brands, ' + total + ' products');
|
||||
console.log('==================');
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
130
backend/scripts/test-brands-selector.ts
Normal file
130
backend/scripts/test-brands-selector.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Use the selector hint: /html/body/main/section
|
||||
console.log('Looking at main > section structure...\n');
|
||||
|
||||
const sectionInfo = await page.evaluate(() => {
|
||||
const main = document.querySelector('main');
|
||||
if (!main) return { error: 'No main element' };
|
||||
|
||||
const sections = main.querySelectorAll('section');
|
||||
const results: any[] = [];
|
||||
|
||||
sections.forEach((section, i) => {
|
||||
const children = section.children;
|
||||
const childInfo: string[] = [];
|
||||
|
||||
for (let j = 0; j < Math.min(children.length, 10); j++) {
|
||||
const child = children[j];
|
||||
childInfo.push(child.tagName + '.' + (child.className?.slice(0, 30) || ''));
|
||||
}
|
||||
|
||||
results.push({
|
||||
index: i,
|
||||
class: section.className?.slice(0, 50),
|
||||
childCount: children.length,
|
||||
sampleChildren: childInfo,
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Sections in main:');
|
||||
console.log(JSON.stringify(sectionInfo, null, 2));
|
||||
|
||||
// Look for brand cards within the section
|
||||
console.log('\nLooking for brand cards in main > section...');
|
||||
|
||||
const brandCards = await page.evaluate(() => {
|
||||
const section = document.querySelector('main > section');
|
||||
if (!section) return [];
|
||||
|
||||
// Get all child elements that might be brand cards
|
||||
const cards: { tag: string; text: string; href: string }[] = [];
|
||||
|
||||
section.querySelectorAll('a').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const text = a.textContent?.trim().slice(0, 50) || '';
|
||||
cards.push({ tag: 'a', text, href });
|
||||
});
|
||||
|
||||
return cards;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandCards.length} links in section:`);
|
||||
brandCards.slice(0, 30).forEach(c => console.log(` ${c.text} -> ${c.href}`));
|
||||
|
||||
// Get the grid of brand cards
|
||||
console.log('\nLooking for grid container...');
|
||||
|
||||
const gridCards = await page.evaluate(() => {
|
||||
// Look for grid-like containers
|
||||
const grids = document.querySelectorAll('[class*="grid"], [class*="Grid"], main section > div');
|
||||
const results: any[] = [];
|
||||
|
||||
grids.forEach((grid, i) => {
|
||||
const links = grid.querySelectorAll('a[href*="/brand/"]');
|
||||
if (links.length > 5) {
|
||||
const brands: string[] = [];
|
||||
links.forEach((a: Element) => {
|
||||
const text = a.textContent?.trim().split('\n')[0] || '';
|
||||
if (text && !brands.includes(text)) brands.push(text);
|
||||
});
|
||||
results.push({
|
||||
class: grid.className?.slice(0, 40),
|
||||
brandCount: brands.length,
|
||||
brands: brands.slice(0, 50),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Grid containers with brands:');
|
||||
gridCards.forEach(g => {
|
||||
console.log(`\n[${g.brandCount} brands] class="${g.class}"`);
|
||||
g.brands.forEach((b: string, i: number) => console.log(` ${i+1}. ${b}`));
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
138
backend/scripts/test-treez-all-endpoints.ts
Normal file
138
backend/scripts/test-treez-all-endpoints.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Capture ALL requests to treez.io
|
||||
const treezRequests: any[] = [];
|
||||
|
||||
page.on('request', (req) => {
|
||||
const url = req.url();
|
||||
if (url.includes('treez.io') && !url.includes('.js') && !url.includes('.css')) {
|
||||
treezRequests.push({
|
||||
url: url,
|
||||
method: req.method(),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Also intercept and capture ES API responses
|
||||
page.on('response', async (res) => {
|
||||
const url = res.url();
|
||||
if (url.includes('gapcommerceapi.com') && res.status() === 200) {
|
||||
try {
|
||||
const json = await res.json();
|
||||
const total = json.hits?.total?.value;
|
||||
const count = json.hits?.hits?.length;
|
||||
if (total || count) {
|
||||
console.log('\nES Response: total=' + total + ', returned=' + count);
|
||||
if (json.hits?.hits?.[0]?._source) {
|
||||
const src = json.hits.hits[0]._source;
|
||||
console.log('First product fields: ' + Object.keys(src).slice(0, 20).join(', '));
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading /shop page...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Click load more several times
|
||||
console.log('\nClicking Load More...');
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (!btn) break;
|
||||
await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
console.log('\n=== TREEZ API ENDPOINTS CALLED ===\n');
|
||||
const uniqueUrls = [...new Set(treezRequests.map(r => r.url.split('?')[0]))];
|
||||
uniqueUrls.forEach(url => console.log(url));
|
||||
|
||||
// Now intercept the ES response data by making a request from browser context
|
||||
console.log('\n=== FETCHING ALL PRODUCTS VIA BROWSER ===\n');
|
||||
|
||||
const allProducts = await page.evaluate(async () => {
|
||||
const apiKey = 'V3jHL9dFzi3Gj4UISM4lr38Nm0GSxcps5OBz1PbS';
|
||||
const url = 'https://search-kyrok9udlk.gapcommerceapi.com/product/search';
|
||||
|
||||
const query = {
|
||||
from: 0,
|
||||
size: 1000,
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ bool: { filter: { range: { customMinPrice: { gte: 0.01, lte: 500000 }}}}},
|
||||
{ bool: { should: [{ match: { isAboveThreshold: true }}]}},
|
||||
{ bool: { should: [{ match: { isHideFromMenu: false }}]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
},
|
||||
body: JSON.stringify(query),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
return {
|
||||
total: data.hits?.total?.value,
|
||||
count: data.hits?.hits?.length,
|
||||
sample: data.hits?.hits?.[0]?._source,
|
||||
allProducts: data.hits?.hits?.map((h: any) => h._source),
|
||||
};
|
||||
} catch (err: any) {
|
||||
return { error: err.message };
|
||||
}
|
||||
});
|
||||
|
||||
if (allProducts.error) {
|
||||
console.log('Error: ' + allProducts.error);
|
||||
} else {
|
||||
console.log('Total products: ' + allProducts.total);
|
||||
console.log('Returned: ' + allProducts.count);
|
||||
|
||||
if (allProducts.sample) {
|
||||
console.log('\n=== PRODUCT FIELDS ===\n');
|
||||
console.log(Object.keys(allProducts.sample).sort().join('\n'));
|
||||
|
||||
console.log('\n=== SAMPLE PRODUCT ===\n');
|
||||
console.log(JSON.stringify(allProducts.sample, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
203
backend/scripts/test-treez-all-products.ts
Normal file
203
backend/scripts/test-treez-all-products.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Extract ALL product elements and find unique products
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Extracting ALL product elements');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
const url = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Get ALL elements with product_product__ class
|
||||
console.log('\n[1] Counting all product_product__ elements...');
|
||||
|
||||
const elementAnalysis = await page.evaluate(() => {
|
||||
const all = document.querySelectorAll('[class*="product_product__"]');
|
||||
const byTag: Record<string, number> = {};
|
||||
const anchorHrefs: string[] = [];
|
||||
const imgAlts: string[] = [];
|
||||
|
||||
all.forEach(el => {
|
||||
const tag = el.tagName;
|
||||
byTag[tag] = (byTag[tag] || 0) + 1;
|
||||
|
||||
if (tag === 'A') {
|
||||
const href = el.getAttribute('href');
|
||||
if (href && href.includes('/product/')) {
|
||||
anchorHrefs.push(href);
|
||||
}
|
||||
}
|
||||
|
||||
if (tag === 'IMG') {
|
||||
const alt = el.getAttribute('alt');
|
||||
if (alt) imgAlts.push(alt);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
total: all.length,
|
||||
byTag,
|
||||
anchorHrefs: anchorHrefs.slice(0, 20),
|
||||
uniqueAnchors: new Set(anchorHrefs).size,
|
||||
imgAlts: imgAlts.slice(0, 20),
|
||||
uniqueImgAlts: new Set(imgAlts).size,
|
||||
};
|
||||
});
|
||||
|
||||
console.log(`Total elements: ${elementAnalysis.total}`);
|
||||
console.log(`By tag:`, elementAnalysis.byTag);
|
||||
console.log(`Unique anchor hrefs: ${elementAnalysis.uniqueAnchors}`);
|
||||
console.log(`Unique image alts: ${elementAnalysis.uniqueImgAlts}`);
|
||||
console.log(`\nSample anchor hrefs:`, elementAnalysis.anchorHrefs.slice(0, 5));
|
||||
console.log(`Sample image alts:`, elementAnalysis.imgAlts.slice(0, 5));
|
||||
|
||||
// Try to extract using different approaches
|
||||
console.log('\n[2] Testing extraction approaches...');
|
||||
|
||||
const approaches = await page.evaluate(() => {
|
||||
const results: Record<string, { count: number; unique: number; sample: string[] }> = {};
|
||||
|
||||
// Approach 1: Anchor elements with product links
|
||||
const anchors = document.querySelectorAll('a[href*="/product/"]');
|
||||
const anchorNames = new Set<string>();
|
||||
anchors.forEach(a => {
|
||||
const img = a.querySelector('img');
|
||||
const name = img?.getAttribute('alt') || a.textContent?.trim().split('\n')[0] || '';
|
||||
if (name) anchorNames.add(name);
|
||||
});
|
||||
results['a[href*="/product/"]'] = {
|
||||
count: anchors.length,
|
||||
unique: anchorNames.size,
|
||||
sample: Array.from(anchorNames).slice(0, 5),
|
||||
};
|
||||
|
||||
// Approach 2: Images with alt text inside product areas
|
||||
const productImgs = document.querySelectorAll('[class*="product_product__"] img[alt]');
|
||||
const imgNames = new Set<string>();
|
||||
productImgs.forEach(img => {
|
||||
const alt = img.getAttribute('alt');
|
||||
if (alt && alt.length > 2) imgNames.add(alt);
|
||||
});
|
||||
results['[class*="product_product__"] img[alt]'] = {
|
||||
count: productImgs.length,
|
||||
unique: imgNames.size,
|
||||
sample: Array.from(imgNames).slice(0, 5),
|
||||
};
|
||||
|
||||
// Approach 3: H5 elements (product names)
|
||||
const h5s = document.querySelectorAll('h5.product_product__name__JcEk0, h5[class*="product__name"]');
|
||||
const h5Names = new Set<string>();
|
||||
h5s.forEach(h5 => {
|
||||
const text = h5.textContent?.trim();
|
||||
if (text) h5Names.add(text);
|
||||
});
|
||||
results['h5[class*="product__name"]'] = {
|
||||
count: h5s.length,
|
||||
unique: h5Names.size,
|
||||
sample: Array.from(h5Names).slice(0, 5),
|
||||
};
|
||||
|
||||
// Approach 4: Link class with product_product__
|
||||
const links = document.querySelectorAll('a.product_product__ERWtJ, a[class*="product_product__"][class*="link"]');
|
||||
const linkNames = new Set<string>();
|
||||
links.forEach(link => {
|
||||
const h5 = link.querySelector('h5');
|
||||
const img = link.querySelector('img');
|
||||
const name = h5?.textContent?.trim() || img?.getAttribute('alt') || '';
|
||||
if (name) linkNames.add(name);
|
||||
});
|
||||
results['a.product_product__ERWtJ'] = {
|
||||
count: links.length,
|
||||
unique: linkNames.size,
|
||||
sample: Array.from(linkNames).slice(0, 5),
|
||||
};
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
Object.entries(approaches).forEach(([sel, data]) => {
|
||||
console.log(`\n${sel}:`);
|
||||
console.log(` Count: ${data.count}, Unique: ${data.unique}`);
|
||||
console.log(` Sample: ${data.sample.join(', ')}`);
|
||||
});
|
||||
|
||||
// The best approach: use images with alt as the source of truth
|
||||
console.log('\n[3] Full product extraction using img[alt] approach...');
|
||||
|
||||
const products = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
const products: { name: string; href: string; price: string }[] = [];
|
||||
|
||||
// Get all product links
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach(a => {
|
||||
const img = a.querySelector('img');
|
||||
const name = img?.getAttribute('alt') || '';
|
||||
|
||||
if (!name || seen.has(name)) return;
|
||||
seen.add(name);
|
||||
|
||||
const href = a.getAttribute('href') || '';
|
||||
|
||||
// Get price from within the link or parent
|
||||
let price = '';
|
||||
const priceEl = a.querySelector('[class*="price"]');
|
||||
if (priceEl) {
|
||||
const priceMatch = priceEl.textContent?.match(/\$(\d+(?:\.\d{2})?)/);
|
||||
price = priceMatch ? priceMatch[1] : '';
|
||||
}
|
||||
|
||||
products.push({ name, href, price });
|
||||
});
|
||||
|
||||
return products;
|
||||
});
|
||||
|
||||
console.log(`Extracted ${products.length} unique products`);
|
||||
console.log('\nSample products:');
|
||||
products.slice(0, 10).forEach(p => {
|
||||
console.log(` - ${p.name} | ${p.price ? '$' + p.price : 'N/A'} | ${p.href.slice(0, 40)}...`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
52
backend/scripts/test-treez-api.ts
Normal file
52
backend/scripts/test-treez-api.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
import axios from 'axios';
|
||||
|
||||
async function main() {
|
||||
const url = 'https://search-kyrok9udlk.gapcommerceapi.com/product/search';
|
||||
|
||||
const query = {
|
||||
from: 0,
|
||||
size: 500,
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ bool: { filter: { range: { customMinPrice: { gte: 0.01, lte: 500000 }}}}},
|
||||
{ bool: { should: [{ match: { isAboveThreshold: true }}]}},
|
||||
{ bool: { should: [{ match: { isHideFromMenu: false }}]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
console.log('Querying Treez Elasticsearch API...\n');
|
||||
|
||||
try {
|
||||
const response = await axios.post(url, query, {
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
|
||||
const data = response.data;
|
||||
const total = data.hits?.total?.value || data.hits?.total;
|
||||
const products = data.hits?.hits || [];
|
||||
|
||||
console.log('Total products: ' + total);
|
||||
console.log('Products returned: ' + products.length + '\n');
|
||||
|
||||
if (products.length > 0) {
|
||||
const first = products[0]._source;
|
||||
console.log('=== PRODUCT FIELDS AVAILABLE ===\n');
|
||||
console.log(Object.keys(first).sort().join('\n'));
|
||||
|
||||
console.log('\n=== SAMPLE PRODUCT ===\n');
|
||||
console.log(JSON.stringify(first, null, 2));
|
||||
}
|
||||
|
||||
} catch (err: any) {
|
||||
console.log('Error: ' + err.message);
|
||||
if (err.response) {
|
||||
console.log('Status: ' + err.response.status);
|
||||
console.log('Data: ' + JSON.stringify(err.response.data));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
97
backend/scripts/test-treez-auth-api.ts
Normal file
97
backend/scripts/test-treez-auth-api.ts
Normal file
@@ -0,0 +1,97 @@
|
||||
import axios from 'axios';
|
||||
|
||||
async function main() {
|
||||
// Test Elasticsearch API with API key
|
||||
console.log('=== ELASTICSEARCH API ===\n');
|
||||
|
||||
const esUrl = 'https://search-kyrok9udlk.gapcommerceapi.com/product/search';
|
||||
const apiKey = 'V3jHL9dFzi3Gj4UISM4lr38Nm0GSxcps5OBz1PbS';
|
||||
|
||||
const query = {
|
||||
from: 0,
|
||||
size: 1000,
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ bool: { filter: { range: { customMinPrice: { gte: 0.01, lte: 500000 }}}}},
|
||||
{ bool: { should: [{ match: { isAboveThreshold: true }}]}},
|
||||
{ bool: { should: [{ match: { isHideFromMenu: false }}]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await axios.post(esUrl, query, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
'Origin': 'https://shop.bestdispensary.com',
|
||||
'Referer': 'https://shop.bestdispensary.com/',
|
||||
},
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
const data = response.data;
|
||||
const total = data.hits?.total?.value || data.hits?.total;
|
||||
const products = data.hits?.hits || [];
|
||||
|
||||
console.log('Total products: ' + total);
|
||||
console.log('Products returned: ' + products.length);
|
||||
|
||||
if (products.length > 0) {
|
||||
const first = products[0]._source;
|
||||
console.log('\n=== PRODUCT FIELDS ===\n');
|
||||
console.log(Object.keys(first).sort().join('\n'));
|
||||
|
||||
console.log('\n=== SAMPLE PRODUCT ===\n');
|
||||
console.log(JSON.stringify(first, null, 2));
|
||||
}
|
||||
|
||||
} catch (err: any) {
|
||||
console.log('Elasticsearch Error: ' + err.message);
|
||||
if (err.response) {
|
||||
console.log('Status: ' + err.response.status);
|
||||
}
|
||||
}
|
||||
|
||||
// Test Treez Headless API
|
||||
console.log('\n\n=== TREEZ HEADLESS API ===\n');
|
||||
|
||||
const treezUrl = 'https://headless.treez.io/v2.0/dispensary/best/ecommerce/discounts?excludeInactive=true&hideUnset=true&includeProdInfo=true';
|
||||
|
||||
try {
|
||||
const response = await axios.get(treezUrl, {
|
||||
headers: {
|
||||
'client_id': '29dce682258145c6b1cf71027282d083',
|
||||
'client_secret': 'A57bB49AfD7F4233B1750a0B501B4E16',
|
||||
'cache-control': 'max-age=0, no-cache, must-revalidate, proxy-revalidate',
|
||||
'Origin': 'https://shop.bestdispensary.com',
|
||||
'Referer': 'https://shop.bestdispensary.com/',
|
||||
},
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
const data = response.data;
|
||||
console.log('Response type: ' + typeof data);
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
console.log('Array length: ' + data.length);
|
||||
if (data.length > 0) {
|
||||
console.log('First item: ' + JSON.stringify(data[0], null, 2).slice(0, 1000));
|
||||
}
|
||||
} else {
|
||||
console.log('Keys: ' + Object.keys(data).join(', '));
|
||||
console.log('Data: ' + JSON.stringify(data, null, 2).slice(0, 2000));
|
||||
}
|
||||
|
||||
} catch (err: any) {
|
||||
console.log('Treez Error: ' + err.message);
|
||||
if (err.response) {
|
||||
console.log('Status: ' + err.response.status);
|
||||
console.log('Data: ' + JSON.stringify(err.response.data).slice(0, 500));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
243
backend/scripts/test-treez-brand-products.ts
Normal file
243
backend/scripts/test-treez-brand-products.ts
Normal file
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Visit each brand page and extract products
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollToLoadAll(page: Page): Promise<void> {
|
||||
let previousHeight = 0;
|
||||
let sameCount = 0;
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (currentHeight === previousHeight) {
|
||||
sameCount++;
|
||||
if (sameCount >= 3) break;
|
||||
} else {
|
||||
sameCount = 0;
|
||||
}
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1000);
|
||||
previousHeight = currentHeight;
|
||||
}
|
||||
}
|
||||
|
||||
async function extractProducts(page: Page): Promise<{ name: string; price: string; href: string }[]> {
|
||||
return page.evaluate(() => {
|
||||
const products: { name: string; price: string; href: string }[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const img = a.querySelector('img');
|
||||
const h5 = a.querySelector('h5');
|
||||
const name = img?.getAttribute('alt') || h5?.textContent?.trim() || '';
|
||||
|
||||
if (!name || seen.has(name)) return;
|
||||
seen.add(name);
|
||||
|
||||
const priceEl = a.querySelector('[class*="price"]');
|
||||
const priceMatch = priceEl?.textContent?.match(/\$(\d+(?:\.\d{2})?)/);
|
||||
const price = priceMatch ? priceMatch[1] : '';
|
||||
|
||||
products.push({ name, price, href });
|
||||
});
|
||||
|
||||
return products;
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Extracting Products from All Brands');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
// Go to brands page and get all brand links
|
||||
const brandsUrl = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
console.log(`\n[1] Getting brand list from ${brandsUrl}`);
|
||||
|
||||
await page.goto(brandsUrl, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// The 142 items on /brands ARE brands (shown as product cards with brand info)
|
||||
// Get the brand names from the product hrefs (they contain brand name in URL)
|
||||
const brandInfo = await page.evaluate(() => {
|
||||
const brands: { name: string; slug: string }[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Extract brand info from product URLs
|
||||
// URL pattern: /product/{brand}-{product}-{details}
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
// Try to extract brand from URL - first segment before product name
|
||||
const match = href.match(/\/product\/([^-]+(?:-[^-]+)?)-/);
|
||||
if (match) {
|
||||
const slug = match[1];
|
||||
if (!seen.has(slug)) {
|
||||
seen.add(slug);
|
||||
// Also look for brand text in the card
|
||||
const brandEl = a.querySelector('[class*="brand"], [class*="Brand"]');
|
||||
const name = brandEl?.textContent?.trim() || slug;
|
||||
brands.push({ name, slug });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return brands;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandInfo.length} potential brands from product URLs`);
|
||||
console.log('Sample:', brandInfo.slice(0, 5));
|
||||
|
||||
// Actually, let's look for brand page links directly
|
||||
console.log('\n[2] Looking for brand page links...');
|
||||
|
||||
const brandLinks = await page.evaluate(() => {
|
||||
const links: { name: string; href: string }[] = [];
|
||||
|
||||
// Look for links to /brand/ pages
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const text = a.textContent?.trim() || '';
|
||||
if (href && !links.some(l => l.href === href)) {
|
||||
links.push({ name: text, href });
|
||||
}
|
||||
});
|
||||
|
||||
return links;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandLinks.length} brand page links`);
|
||||
if (brandLinks.length > 0) {
|
||||
console.log('Sample:', brandLinks.slice(0, 10));
|
||||
}
|
||||
|
||||
// If no brand links, try to find them in section headers
|
||||
console.log('\n[3] Looking for brand sections...');
|
||||
|
||||
const brandSections = await page.evaluate(() => {
|
||||
const sections: { brandName: string; sampleProduct: string }[] = [];
|
||||
|
||||
document.querySelectorAll('[class*="products_product__section"]').forEach(section => {
|
||||
const header = section.querySelector('h2, h3, [class*="heading"]');
|
||||
const brandName = header?.textContent?.trim() || '';
|
||||
const firstProduct = section.querySelector('a[href*="/product/"]');
|
||||
const productName = firstProduct?.querySelector('h5')?.textContent?.trim() ||
|
||||
firstProduct?.querySelector('img')?.getAttribute('alt') || '';
|
||||
|
||||
if (brandName) {
|
||||
sections.push({ brandName, sampleProduct: productName });
|
||||
}
|
||||
});
|
||||
|
||||
return sections;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandSections.length} brand sections`);
|
||||
brandSections.slice(0, 10).forEach(s => {
|
||||
console.log(` - Brand: "${s.brandName}" | Sample: "${s.sampleProduct}"`);
|
||||
});
|
||||
|
||||
// Try visiting a brand page directly using the section name
|
||||
if (brandSections.length > 0) {
|
||||
console.log('\n[4] Testing brand page URLs...');
|
||||
|
||||
// Try different URL patterns for first brand
|
||||
const testBrand = brandSections[0].brandName;
|
||||
const testSlug = testBrand.toLowerCase().replace(/[^a-z0-9]+/g, '-');
|
||||
|
||||
const urlPatterns = [
|
||||
`/onlinemenu/brand/${encodeURIComponent(testBrand)}`,
|
||||
`/onlinemenu/brand/${testSlug}`,
|
||||
`/brand/${encodeURIComponent(testBrand)}`,
|
||||
`/brand/${testSlug}`,
|
||||
];
|
||||
|
||||
for (const path of urlPatterns) {
|
||||
const testUrl = `https://${STORE_ID}.treez.io${path}?customerType=ADULT`;
|
||||
try {
|
||||
console.log(` Trying: ${testUrl}`);
|
||||
await page.goto(testUrl, { waitUntil: 'networkidle2', timeout: 15000 });
|
||||
await sleep(2000);
|
||||
|
||||
const products = await extractProducts(page);
|
||||
console.log(` Products found: ${products.length}`);
|
||||
|
||||
if (products.length > 0) {
|
||||
console.log(` ✓ Working URL pattern: ${path}`);
|
||||
break;
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.log(` Error: ${e.message.slice(0, 50)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if clicking on a brand section leads to a brand page
|
||||
console.log('\n[5] Checking if brand sections have clickable headers...');
|
||||
|
||||
await page.goto(brandsUrl, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
|
||||
const clickableHeaders = await page.evaluate(() => {
|
||||
const results: { text: string; tag: string; href: string; clickable: boolean }[] = [];
|
||||
|
||||
document.querySelectorAll('[class*="products_product__section"] h2, [class*="products_product__section"] h3').forEach(header => {
|
||||
const link = header.closest('a') || header.querySelector('a');
|
||||
const text = header.textContent?.trim() || '';
|
||||
const href = link?.getAttribute('href') || '';
|
||||
|
||||
results.push({
|
||||
text,
|
||||
tag: header.tagName,
|
||||
href,
|
||||
clickable: !!link,
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Section headers:');
|
||||
clickableHeaders.slice(0, 10).forEach(h => {
|
||||
console.log(` [${h.tag}] "${h.text}" - ${h.clickable ? `Link: ${h.href}` : 'Not clickable'}`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
183
backend/scripts/test-treez-brands-detailed.ts
Normal file
183
backend/scripts/test-treez-brands-detailed.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* Detailed brand section analysis
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log(' Age gate detected, bypassing...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Detailed Brand Section Analysis');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
const url = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
console.log(`\nNavigating to ${url}`);
|
||||
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Scroll multiple times to load all content
|
||||
console.log('\n[1] Scrolling to load all content...');
|
||||
let previousHeight = 0;
|
||||
let scrollCount = 0;
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1500);
|
||||
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
const productCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('a[href*="/product/"]').length
|
||||
);
|
||||
|
||||
console.log(` Scroll ${i + 1}: height=${currentHeight}, products=${productCount}`);
|
||||
|
||||
if (currentHeight === previousHeight) {
|
||||
scrollCount++;
|
||||
if (scrollCount >= 3) break;
|
||||
} else {
|
||||
scrollCount = 0;
|
||||
}
|
||||
previousHeight = currentHeight;
|
||||
}
|
||||
|
||||
// Look at ALL h2/h3 headers on page
|
||||
console.log('\n[2] Finding ALL h2/h3 headers on page...');
|
||||
|
||||
const headers = await page.evaluate(() => {
|
||||
const results: { tag: string; text: string; parentClass: string }[] = [];
|
||||
|
||||
document.querySelectorAll('h2, h3').forEach((el: Element) => {
|
||||
results.push({
|
||||
tag: el.tagName,
|
||||
text: el.textContent?.trim().slice(0, 80) || '',
|
||||
parentClass: el.parentElement?.className?.slice(0, 50) || '',
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log(`Found ${headers.length} headers:`);
|
||||
headers.forEach((h: { tag: string; text: string }) =>
|
||||
console.log(` [${h.tag}] "${h.text}"`)
|
||||
);
|
||||
|
||||
// Get products grouped by their section heading
|
||||
console.log('\n[3] Getting products per section...');
|
||||
|
||||
const sectionProducts = await page.evaluate(() => {
|
||||
const results: { heading: string; products: number }[] = [];
|
||||
|
||||
// Find all sections that contain products
|
||||
document.querySelectorAll('[class*="products_product__section"]').forEach((section: Element) => {
|
||||
const heading = section.querySelector('h2, h3');
|
||||
const headingText = heading?.textContent?.trim() || 'Unknown';
|
||||
const products = section.querySelectorAll('a[href*="/product/"]');
|
||||
|
||||
results.push({
|
||||
heading: headingText,
|
||||
products: products.length,
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log(`Found ${sectionProducts.length} brand sections:`);
|
||||
let totalProducts = 0;
|
||||
sectionProducts.forEach((s: { heading: string; products: number }) => {
|
||||
console.log(` ${s.heading}: ${s.products} products`);
|
||||
totalProducts += s.products;
|
||||
});
|
||||
console.log(`\nTotal products across all sections: ${totalProducts}`);
|
||||
|
||||
// Also extract brand from each product's URL/card
|
||||
console.log('\n[4] Extracting brand from product URLs/cards...');
|
||||
|
||||
const brandCounts = await page.evaluate(() => {
|
||||
const byBrand: Record<string, number> = {};
|
||||
const seen = new Set<string>();
|
||||
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const img = a.querySelector('img');
|
||||
const name = img?.getAttribute('alt') || '';
|
||||
|
||||
if (!name || seen.has(name)) return;
|
||||
seen.add(name);
|
||||
|
||||
// Try to find brand from the card
|
||||
const brandEl = a.querySelector('[class*="brand"], [class*="Brand"], span, p');
|
||||
let brand = '';
|
||||
|
||||
// Try various methods to find brand
|
||||
const allSpans = a.querySelectorAll('span, p');
|
||||
allSpans.forEach((span: Element) => {
|
||||
const text = span.textContent?.trim() || '';
|
||||
if (text && text.length < 50 && text !== name && !text.includes('$')) {
|
||||
if (!brand) brand = text;
|
||||
}
|
||||
});
|
||||
|
||||
// Fallback: get brand from parent section heading
|
||||
if (!brand) {
|
||||
const section = a.closest('[class*="products_product__section"]');
|
||||
const heading = section?.querySelector('h2, h3');
|
||||
brand = heading?.textContent?.trim() || 'Unknown';
|
||||
}
|
||||
|
||||
byBrand[brand] = (byBrand[brand] || 0) + 1;
|
||||
});
|
||||
|
||||
return byBrand;
|
||||
});
|
||||
|
||||
console.log('Products by brand:');
|
||||
Object.entries(brandCounts)
|
||||
.sort((a, b) => (b[1] as number) - (a[1] as number))
|
||||
.forEach(([brand, count]) => {
|
||||
console.log(` ${brand}: ${count}`);
|
||||
});
|
||||
|
||||
const uniqueTotal = Object.values(brandCounts).reduce((sum: number, c) => sum + (c as number), 0);
|
||||
console.log(`\nTotal unique products: ${uniqueTotal}`);
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
113
backend/scripts/test-treez-capture-auth.ts
Normal file
113
backend/scripts/test-treez-capture-auth.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Capture request headers for API calls
|
||||
const apiRequests: any[] = [];
|
||||
|
||||
page.on('request', (req) => {
|
||||
const url = req.url();
|
||||
if (url.includes('treez.io') || url.includes('gapcommerce')) {
|
||||
apiRequests.push({
|
||||
url: url,
|
||||
method: req.method(),
|
||||
headers: req.headers(),
|
||||
postData: req.postData(),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading page to capture API auth headers...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
console.log('=== API REQUESTS WITH HEADERS ===\n');
|
||||
|
||||
apiRequests.forEach((req, i) => {
|
||||
console.log((i+1) + '. ' + req.method + ' ' + req.url.slice(0, 100));
|
||||
console.log(' Headers:');
|
||||
Object.entries(req.headers).forEach(([k, v]) => {
|
||||
if (k.toLowerCase().includes('auth') ||
|
||||
k.toLowerCase().includes('token') ||
|
||||
k.toLowerCase().includes('key') ||
|
||||
k.toLowerCase().includes('api') ||
|
||||
k.toLowerCase() === 'authorization' ||
|
||||
k.toLowerCase() === 'x-api-key') {
|
||||
console.log(' >>> ' + k + ': ' + v);
|
||||
}
|
||||
});
|
||||
// Show all headers for treez.io requests
|
||||
if (req.url.includes('headless.treez.io')) {
|
||||
console.log(' ALL HEADERS:');
|
||||
Object.entries(req.headers).forEach(([k, v]) => {
|
||||
console.log(' ' + k + ': ' + String(v).slice(0, 80));
|
||||
});
|
||||
}
|
||||
console.log('');
|
||||
});
|
||||
|
||||
// Also check for API keys in page scripts
|
||||
console.log('=== CHECKING FOR API KEYS IN PAGE ===\n');
|
||||
|
||||
const pageData = await page.evaluate(() => {
|
||||
const data: any = {};
|
||||
|
||||
// Check window object for API keys
|
||||
const win = window as any;
|
||||
if (win.__NEXT_DATA__) {
|
||||
data.nextData = win.__NEXT_DATA__;
|
||||
}
|
||||
|
||||
// Check for any global config
|
||||
if (win.config || win.CONFIG) {
|
||||
data.config = win.config || win.CONFIG;
|
||||
}
|
||||
|
||||
// Look for treez-related globals
|
||||
Object.keys(win).forEach(key => {
|
||||
if (key.toLowerCase().includes('treez') ||
|
||||
key.toLowerCase().includes('api') ||
|
||||
key.toLowerCase().includes('config')) {
|
||||
try {
|
||||
data[key] = JSON.stringify(win[key]).slice(0, 500);
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
return data;
|
||||
});
|
||||
|
||||
if (pageData.nextData?.props?.pageProps) {
|
||||
console.log('Next.js pageProps keys: ' + Object.keys(pageData.nextData.props.pageProps).join(', '));
|
||||
}
|
||||
if (pageData.nextData?.runtimeConfig) {
|
||||
console.log('Runtime config: ' + JSON.stringify(pageData.nextData.runtimeConfig).slice(0, 500));
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
100
backend/scripts/test-treez-capture-response.ts
Normal file
100
backend/scripts/test-treez-capture-response.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
import fs from 'fs';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Capture ES API responses
|
||||
let allProductData: any[] = [];
|
||||
|
||||
page.on('response', async (res) => {
|
||||
const url = res.url();
|
||||
if (url.includes('gapcommerceapi.com/product/search') && res.status() === 200) {
|
||||
try {
|
||||
const json = await res.json();
|
||||
const products = json.hits?.hits?.map((h: any) => h._source) || [];
|
||||
allProductData = allProductData.concat(products);
|
||||
console.log('Captured ' + products.length + ' products (total: ' + allProductData.length + ')');
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading /shop page to capture product data...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Click load more many times to get all products
|
||||
console.log('\nClicking Load More to capture all products...');
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (!btn) {
|
||||
console.log('No more Load More button');
|
||||
break;
|
||||
}
|
||||
|
||||
const isVisible = await page.evaluate((b) => {
|
||||
const rect = b.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, btn);
|
||||
|
||||
if (!isVisible) {
|
||||
console.log('Load More not visible');
|
||||
break;
|
||||
}
|
||||
|
||||
await btn.click();
|
||||
await sleep(1500);
|
||||
console.log('Click ' + (i+1) + ': ' + allProductData.length + ' total products');
|
||||
}
|
||||
|
||||
console.log('\n=== RESULTS ===\n');
|
||||
console.log('Total products captured: ' + allProductData.length);
|
||||
|
||||
if (allProductData.length > 0) {
|
||||
// Dedupe by some ID
|
||||
const seen = new Set();
|
||||
const unique = allProductData.filter(p => {
|
||||
const id = p.id || p.productId || p.name;
|
||||
if (seen.has(id)) return false;
|
||||
seen.add(id);
|
||||
return true;
|
||||
});
|
||||
|
||||
console.log('Unique products: ' + unique.length);
|
||||
console.log('\n=== PRODUCT FIELDS ===\n');
|
||||
console.log(Object.keys(unique[0]).sort().join('\n'));
|
||||
|
||||
console.log('\n=== SAMPLE PRODUCT ===\n');
|
||||
console.log(JSON.stringify(unique[0], null, 2));
|
||||
|
||||
// Save to file
|
||||
fs.writeFileSync('/tmp/treez-products.json', JSON.stringify(unique, null, 2));
|
||||
console.log('\nSaved to /tmp/treez-products.json');
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
88
backend/scripts/test-treez-capture-text.ts
Normal file
88
backend/scripts/test-treez-capture-text.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
import fs from 'fs';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Capture ES API responses as text
|
||||
let allProducts: any[] = [];
|
||||
|
||||
page.on('response', async (res) => {
|
||||
const url = res.url();
|
||||
if (url.includes('gapcommerceapi.com/product/search')) {
|
||||
console.log('ES Response: status=' + res.status());
|
||||
if (res.status() === 200) {
|
||||
try {
|
||||
const text = await res.text();
|
||||
console.log('Response length: ' + text.length);
|
||||
const json = JSON.parse(text);
|
||||
const products = json.hits?.hits?.map((h: any) => h._source) || [];
|
||||
allProducts = allProducts.concat(products);
|
||||
console.log('Got ' + products.length + ' products (total: ' + allProducts.length + ')');
|
||||
} catch (err: any) {
|
||||
console.log('Parse error: ' + err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading page...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(5000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
// Wait for initial products to load
|
||||
await sleep(3000);
|
||||
|
||||
console.log('\nInitial products captured: ' + allProducts.length);
|
||||
|
||||
// Try scrolling to trigger more loads
|
||||
console.log('\nScrolling...');
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1500);
|
||||
|
||||
// Also click load more if present
|
||||
try {
|
||||
await page.click('button.collection__load-more');
|
||||
console.log('Clicked load more');
|
||||
} catch {}
|
||||
}
|
||||
|
||||
console.log('\n=== FINAL RESULTS ===\n');
|
||||
console.log('Total products: ' + allProducts.length);
|
||||
|
||||
if (allProducts.length > 0) {
|
||||
console.log('\nFields: ' + Object.keys(allProducts[0]).sort().join(', '));
|
||||
console.log('\nSample:\n' + JSON.stringify(allProducts[0], null, 2));
|
||||
|
||||
fs.writeFileSync('/tmp/treez-products.json', JSON.stringify(allProducts, null, 2));
|
||||
console.log('\nSaved to /tmp/treez-products.json');
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
192
backend/scripts/test-treez-categories.ts
Normal file
192
backend/scripts/test-treez-categories.ts
Normal file
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Navigate to each category page and count products
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollToLoadAll(page: Page): Promise<void> {
|
||||
let previousHeight = 0;
|
||||
let scrollCount = 0;
|
||||
let sameCount = 0;
|
||||
|
||||
while (scrollCount < 50) {
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (currentHeight === previousHeight) {
|
||||
sameCount++;
|
||||
if (sameCount >= 3) break;
|
||||
} else {
|
||||
sameCount = 0;
|
||||
}
|
||||
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1500);
|
||||
previousHeight = currentHeight;
|
||||
scrollCount++;
|
||||
}
|
||||
}
|
||||
|
||||
async function countProducts(page: Page): Promise<number> {
|
||||
return page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach(a => {
|
||||
const img = a.querySelector('img');
|
||||
const name = img?.getAttribute('alt') || a.querySelector('h5')?.textContent?.trim() || '';
|
||||
if (name) seen.add(name);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Testing Treez Category Pages');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
// Categories from the nav menu
|
||||
const categories = [
|
||||
'cartridges',
|
||||
'flower',
|
||||
'pre-rolls',
|
||||
'edibles',
|
||||
'extracts',
|
||||
'tinctures',
|
||||
'capsules',
|
||||
'topicals',
|
||||
'accessories',
|
||||
'drink',
|
||||
];
|
||||
|
||||
const results: { category: string; products: number }[] = [];
|
||||
let ageGateBypassed = false;
|
||||
|
||||
for (const category of categories) {
|
||||
// Try different URL patterns
|
||||
const urls = [
|
||||
`https://${STORE_ID}.treez.io/onlinemenu/${category}?customerType=ADULT`,
|
||||
`https://${STORE_ID}.treez.io/onlinemenu/category/${category}?customerType=ADULT`,
|
||||
`https://${STORE_ID}.treez.io/${category}?customerType=ADULT`,
|
||||
];
|
||||
|
||||
for (const url of urls) {
|
||||
try {
|
||||
console.log(`\nTrying: ${url}`);
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(2000);
|
||||
|
||||
if (!ageGateBypassed) {
|
||||
await bypassAgeGate(page);
|
||||
ageGateBypassed = true;
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
const initialCount = await countProducts(page);
|
||||
if (initialCount > 0) {
|
||||
console.log(` Initial: ${initialCount} products`);
|
||||
|
||||
await scrollToLoadAll(page);
|
||||
const finalCount = await countProducts(page);
|
||||
console.log(` After scroll: ${finalCount} products`);
|
||||
|
||||
results.push({ category, products: finalCount });
|
||||
break; // Found working URL, move to next category
|
||||
} else {
|
||||
console.log(` No products found`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.log(` Error: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also try the main shop page
|
||||
console.log('\nTrying main shop page...');
|
||||
try {
|
||||
const shopUrl = `https://${STORE_ID}.treez.io/onlinemenu/shop?customerType=ADULT`;
|
||||
await page.goto(shopUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(2000);
|
||||
|
||||
const initialCount = await countProducts(page);
|
||||
console.log(`Shop page initial: ${initialCount} products`);
|
||||
|
||||
if (initialCount > 0) {
|
||||
await scrollToLoadAll(page);
|
||||
const finalCount = await countProducts(page);
|
||||
console.log(`Shop page after scroll: ${finalCount} products`);
|
||||
results.push({ category: 'shop', products: finalCount });
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.log(`Shop page error: ${error.message}`);
|
||||
}
|
||||
|
||||
// Try to find and click on category links from the nav
|
||||
console.log('\n[Alternative] Trying to find nav category links...');
|
||||
|
||||
const homeUrl = `https://${STORE_ID}.treez.io/onlinemenu/?customerType=ADULT`;
|
||||
await page.goto(homeUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(1000);
|
||||
|
||||
const navLinks = await page.evaluate(() => {
|
||||
const links: { text: string; href: string }[] = [];
|
||||
document.querySelectorAll('nav a, [class*="nav"] a').forEach(a => {
|
||||
const text = a.textContent?.trim() || '';
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (href && text && !links.some(l => l.href === href)) {
|
||||
links.push({ text, href });
|
||||
}
|
||||
});
|
||||
return links;
|
||||
});
|
||||
|
||||
console.log('Nav links found:');
|
||||
navLinks.forEach(l => console.log(` - "${l.text}" → ${l.href}`));
|
||||
|
||||
// Summary
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('Summary');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
let total = 0;
|
||||
results.forEach(r => {
|
||||
console.log(`${r.category}: ${r.products} products`);
|
||||
total += r.products;
|
||||
});
|
||||
console.log(`\nTotal across categories: ${total} products`);
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
160
backend/scripts/test-treez-containers.ts
Normal file
160
backend/scripts/test-treez-containers.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* Find the correct product card container selector
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Finding Treez product card containers...\n');
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
const url = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Find product card containers by looking for elements that contain both name AND price
|
||||
const analysis = await page.evaluate(() => {
|
||||
// Strategy: find all H5 elements (which contain names), then get their parent containers
|
||||
const nameElements = document.querySelectorAll('h5.product_product__name__JcEk0');
|
||||
const containers: Map<string, { count: number; sample: string }> = new Map();
|
||||
|
||||
nameElements.forEach(nameEl => {
|
||||
// Walk up to find the product card container
|
||||
let current = nameEl.parentElement;
|
||||
let depth = 0;
|
||||
|
||||
while (current && depth < 10) {
|
||||
const className = current.className?.toString?.() || '';
|
||||
|
||||
// Look for ProductCard in the class name
|
||||
if (className.includes('ProductCard')) {
|
||||
const key = className.slice(0, 100);
|
||||
const existing = containers.get(key) || { count: 0, sample: '' };
|
||||
existing.count++;
|
||||
if (!existing.sample) {
|
||||
existing.sample = current.outerHTML.slice(0, 300);
|
||||
}
|
||||
containers.set(key, existing);
|
||||
break;
|
||||
}
|
||||
|
||||
current = current.parentElement;
|
||||
depth++;
|
||||
}
|
||||
});
|
||||
|
||||
return Array.from(containers.entries()).map(([cls, data]) => ({
|
||||
class: cls,
|
||||
count: data.count,
|
||||
sample: data.sample,
|
||||
}));
|
||||
});
|
||||
|
||||
console.log('Product card containers found:');
|
||||
analysis.forEach(({ class: cls, count, sample }) => {
|
||||
console.log(`\n[${count}x] ${cls}`);
|
||||
console.log(`Sample: ${sample.slice(0, 200)}...`);
|
||||
});
|
||||
|
||||
// Now test various container selectors
|
||||
console.log('\n\n--- Testing container selectors ---');
|
||||
|
||||
const selectorTests = await page.evaluate(() => {
|
||||
const tests: Record<string, { total: number; withName: number; withPrice: number }> = {};
|
||||
|
||||
const selectors = [
|
||||
'[class*="ProductCardWithBtn"]',
|
||||
'[class*="ProductCard_product"]',
|
||||
'[class*="ProductCard__"]',
|
||||
'article[class*="product"]',
|
||||
'div[class*="ProductCard"]',
|
||||
'a[class*="ProductCard"]',
|
||||
'[class*="product_product__"][class*="link"]',
|
||||
'article',
|
||||
];
|
||||
|
||||
selectors.forEach(sel => {
|
||||
const elements = document.querySelectorAll(sel);
|
||||
let withName = 0;
|
||||
let withPrice = 0;
|
||||
|
||||
elements.forEach(el => {
|
||||
if (el.querySelector('h5, [class*="product__name"]')) withName++;
|
||||
if (el.querySelector('[class*="price"]')) withPrice++;
|
||||
});
|
||||
|
||||
tests[sel] = { total: elements.length, withName, withPrice };
|
||||
});
|
||||
|
||||
return tests;
|
||||
});
|
||||
|
||||
Object.entries(selectorTests).forEach(([sel, { total, withName, withPrice }]) => {
|
||||
console.log(`${sel}: ${total} total, ${withName} with name, ${withPrice} with price`);
|
||||
});
|
||||
|
||||
// Get the actual product card class pattern
|
||||
console.log('\n\n--- Finding exact product card class ---');
|
||||
|
||||
const exactClasses = await page.evaluate(() => {
|
||||
// Find elements that have both h5 name AND price child
|
||||
const allElements = document.querySelectorAll('*');
|
||||
const matches: { tag: string; class: string }[] = [];
|
||||
|
||||
allElements.forEach(el => {
|
||||
const hasName = el.querySelector('h5.product_product__name__JcEk0');
|
||||
const hasPrice = el.querySelector('[class*="price__ins"], [class*="price__"]');
|
||||
|
||||
if (hasName && hasPrice) {
|
||||
const className = el.className?.toString?.() || '';
|
||||
if (className && !matches.some(m => m.class === className)) {
|
||||
matches.push({ tag: el.tagName, class: className.slice(0, 150) });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return matches;
|
||||
});
|
||||
|
||||
console.log('Elements containing both name and price:');
|
||||
exactClasses.forEach(({ tag, class: cls }) => {
|
||||
console.log(` [${tag}] ${cls}`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
191
backend/scripts/test-treez-find-brands.ts
Normal file
191
backend/scripts/test-treez-find-brands.ts
Normal file
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* Find actual brand elements on /brands page
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log(' Age gate detected, bypassing...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Finding Brand Elements on /brands page');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
const url = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
console.log(`\nNavigating to ${url}`);
|
||||
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Check current URL
|
||||
const currentUrl = page.url();
|
||||
console.log(`\nCurrent URL: ${currentUrl}`);
|
||||
|
||||
// Look for ANY links on the page that might be brand links
|
||||
console.log('\n[1] Looking for all anchor links with "brand" in href or class...');
|
||||
|
||||
const brandLinks = await page.evaluate(() => {
|
||||
const links: { href: string; text: string }[] = [];
|
||||
|
||||
document.querySelectorAll('a').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const text = a.textContent?.trim().slice(0, 50) || '';
|
||||
const className = a.className || '';
|
||||
|
||||
if (href.includes('brand') || href.includes('Brand') ||
|
||||
className.includes('brand') || className.includes('Brand')) {
|
||||
links.push({ href, text });
|
||||
}
|
||||
});
|
||||
|
||||
return links;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandLinks.length} brand-related links:`);
|
||||
brandLinks.slice(0, 30).forEach(l => console.log(` "${l.text}" → ${l.href}`));
|
||||
|
||||
// Look for the navigation/dropdown
|
||||
console.log('\n[2] Looking at navigation structure...');
|
||||
|
||||
const navItems = await page.evaluate(() => {
|
||||
const items: string[] = [];
|
||||
|
||||
document.querySelectorAll('nav a, [class*="nav"] a, header a').forEach((a: Element) => {
|
||||
const text = a.textContent?.trim();
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (text && text.length < 30) {
|
||||
items.push(`${text} (${href})`);
|
||||
}
|
||||
});
|
||||
|
||||
return [...new Set(items)];
|
||||
});
|
||||
|
||||
console.log('Navigation items:');
|
||||
navItems.forEach(item => console.log(` - ${item}`));
|
||||
|
||||
// Look for grid containers that might hold brand cards
|
||||
console.log('\n[3] Looking for brand card containers...');
|
||||
|
||||
const containers = await page.evaluate(() => {
|
||||
const results: { selector: string; count: number; sample: string }[] = [];
|
||||
|
||||
// Try various selectors for brand cards
|
||||
const selectors = [
|
||||
'[class*="brand_brand"]',
|
||||
'[class*="brands_brand"]',
|
||||
'[class*="brand-card"]',
|
||||
'[class*="brandCard"]',
|
||||
'[class*="BrandCard"]',
|
||||
'a[href*="/brand/"]',
|
||||
'[data-testid*="brand"]',
|
||||
];
|
||||
|
||||
selectors.forEach(sel => {
|
||||
const els = document.querySelectorAll(sel);
|
||||
if (els.length > 0) {
|
||||
const first = els[0];
|
||||
results.push({
|
||||
selector: sel,
|
||||
count: els.length,
|
||||
sample: first.textContent?.trim().slice(0, 50) || '',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Brand containers found:');
|
||||
containers.forEach(c => console.log(` ${c.selector}: ${c.count} elements, sample: "${c.sample}"`));
|
||||
|
||||
// Get ALL unique hrefs that contain /brand/
|
||||
console.log('\n[4] All links containing "/brand/" in href...');
|
||||
|
||||
const brandHrefs = await page.evaluate(() => {
|
||||
const hrefs: string[] = [];
|
||||
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href && !hrefs.includes(href)) {
|
||||
hrefs.push(href);
|
||||
}
|
||||
});
|
||||
|
||||
return hrefs;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandHrefs.length} unique brand hrefs:`);
|
||||
brandHrefs.forEach(href => console.log(` ${href}`));
|
||||
|
||||
// Take screenshot
|
||||
await page.screenshot({ path: '/tmp/treez-brands-page.png', fullPage: false });
|
||||
console.log('\n[5] Screenshot saved to /tmp/treez-brands-page.png');
|
||||
|
||||
// Scroll and see if more brands load
|
||||
console.log('\n[6] Scrolling to load more brands...');
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1500);
|
||||
|
||||
const brandCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('a[href*="/brand/"]').length
|
||||
);
|
||||
const productCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('a[href*="/product/"]').length
|
||||
);
|
||||
|
||||
console.log(` Scroll ${i + 1}: brand links=${brandCount}, product links=${productCount}`);
|
||||
}
|
||||
|
||||
// Final brand href list
|
||||
const finalBrandHrefs = await page.evaluate(() => {
|
||||
const hrefs: string[] = [];
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href && !hrefs.includes(href)) hrefs.push(href);
|
||||
});
|
||||
return hrefs;
|
||||
});
|
||||
|
||||
console.log(`\n[7] Final brand href list (${finalBrandHrefs.length} brands):`);
|
||||
finalBrandHrefs.forEach(href => console.log(` ${href}`));
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
221
backend/scripts/test-treez-full-crawl.ts
Normal file
221
backend/scripts/test-treez-full-crawl.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Full crawl: Visit each brand page and aggregate all products
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollToLoadAll(page: Page): Promise<void> {
|
||||
let previousHeight = 0;
|
||||
let sameCount = 0;
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (currentHeight === previousHeight) {
|
||||
sameCount++;
|
||||
if (sameCount >= 3) break;
|
||||
} else {
|
||||
sameCount = 0;
|
||||
}
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1000);
|
||||
previousHeight = currentHeight;
|
||||
}
|
||||
}
|
||||
|
||||
async function extractProducts(page: Page): Promise<{ name: string; brand: string; price: string; href: string }[]> {
|
||||
return page.evaluate(() => {
|
||||
const products: { name: string; brand: string; price: string; href: string }[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const img = a.querySelector('img');
|
||||
const h5 = a.querySelector('h5');
|
||||
const name = img?.getAttribute('alt') || h5?.textContent?.trim() || '';
|
||||
|
||||
if (!name || seen.has(href)) return;
|
||||
seen.add(href);
|
||||
|
||||
// Extract brand from href pattern: /product/{brand}-{product}
|
||||
const brandMatch = href.match(/\/product\/([^\/]+)/);
|
||||
const productSlug = brandMatch ? brandMatch[1] : '';
|
||||
|
||||
const priceEl = a.querySelector('[class*="price"]');
|
||||
const priceMatch = priceEl?.textContent?.match(/\$(\d+(?:\.\d{2})?)/);
|
||||
const price = priceMatch ? priceMatch[1] : '';
|
||||
|
||||
products.push({ name, brand: productSlug.split('-')[0] || '', price, href });
|
||||
});
|
||||
|
||||
return products;
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Full Treez Crawl - All Brands');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
// Step 1: Go to brands page and extract all brand links
|
||||
const brandsUrl = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
console.log(`\n[1] Getting brand list...`);
|
||||
|
||||
await page.goto(brandsUrl, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Get all brand links from the page
|
||||
const brandLinks = await page.evaluate(() => {
|
||||
const links: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Get all /brand/ links
|
||||
document.querySelectorAll('a[href*="/brand/"]').forEach(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (href && !seen.has(href)) {
|
||||
seen.add(href);
|
||||
links.push(href);
|
||||
}
|
||||
});
|
||||
|
||||
return links;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandLinks.length} brand links: ${brandLinks.join(', ')}`);
|
||||
|
||||
// Step 2: Also extract unique brands from product URLs
|
||||
const productBrands = await page.evaluate(() => {
|
||||
const brands = new Set<string>();
|
||||
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach(a => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
// Pattern: /product/{brand}-{product}-...
|
||||
// Extract first part before first hyphen that looks like brand
|
||||
const match = href.match(/\/product\/([a-z0-9]+(?:-[a-z0-9]+)?)-/i);
|
||||
if (match) {
|
||||
brands.add(match[1].toLowerCase());
|
||||
}
|
||||
});
|
||||
|
||||
return Array.from(brands);
|
||||
});
|
||||
|
||||
console.log(`Found ${productBrands.length} brands from product URLs`);
|
||||
|
||||
// Step 3: Build full brand URL list
|
||||
const allBrandUrls = new Set<string>();
|
||||
|
||||
// Add direct brand links
|
||||
brandLinks.forEach(link => {
|
||||
if (link.startsWith('/')) {
|
||||
allBrandUrls.add(`https://${STORE_ID}.treez.io${link}`);
|
||||
} else {
|
||||
allBrandUrls.add(link);
|
||||
}
|
||||
});
|
||||
|
||||
// Add brand URLs from product slugs
|
||||
productBrands.forEach(brand => {
|
||||
allBrandUrls.add(`https://${STORE_ID}.treez.io/brand/${encodeURIComponent(brand)}`);
|
||||
});
|
||||
|
||||
console.log(`Total brand URLs to visit: ${allBrandUrls.size}`);
|
||||
|
||||
// Step 4: Visit each brand page and collect products
|
||||
const allProducts = new Map<string, { name: string; brand: string; price: string; href: string }>();
|
||||
let visitedBrands = 0;
|
||||
|
||||
for (const brandUrl of allBrandUrls) {
|
||||
try {
|
||||
const fullUrl = brandUrl.includes('customerType') ? brandUrl : `${brandUrl}?customerType=ADULT`;
|
||||
console.log(`\n[${++visitedBrands}/${allBrandUrls.size}] Visiting: ${fullUrl}`);
|
||||
|
||||
await page.goto(fullUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(1500);
|
||||
|
||||
// Scroll to load all
|
||||
await scrollToLoadAll(page);
|
||||
|
||||
const products = await extractProducts(page);
|
||||
console.log(` Found ${products.length} products`);
|
||||
|
||||
products.forEach(p => {
|
||||
if (!allProducts.has(p.href)) {
|
||||
allProducts.set(p.href, p);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(` Total unique so far: ${allProducts.size}`);
|
||||
|
||||
} catch (error: any) {
|
||||
console.log(` Error: ${error.message.slice(0, 50)}`);
|
||||
}
|
||||
|
||||
// Small delay between requests
|
||||
await sleep(500);
|
||||
}
|
||||
|
||||
// Summary
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('SUMMARY');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Brands visited: ${visitedBrands}`);
|
||||
console.log(`Total unique products: ${allProducts.size}`);
|
||||
|
||||
// Count by brand
|
||||
const brandCounts: Record<string, number> = {};
|
||||
allProducts.forEach(p => {
|
||||
brandCounts[p.brand] = (brandCounts[p.brand] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log('\nProducts by brand:');
|
||||
Object.entries(brandCounts)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 20)
|
||||
.forEach(([brand, count]) => {
|
||||
console.log(` ${brand}: ${count}`);
|
||||
});
|
||||
|
||||
// Sample products
|
||||
console.log('\nSample products:');
|
||||
Array.from(allProducts.values()).slice(0, 10).forEach(p => {
|
||||
console.log(` - ${p.name} | ${p.brand} | $${p.price || 'N/A'}`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
57
backend/scripts/test-treez-headless-api.ts
Normal file
57
backend/scripts/test-treez-headless-api.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
import axios from 'axios';
|
||||
|
||||
async function main() {
|
||||
const storeId = 'best';
|
||||
const baseUrl = `https://headless.treez.io/v2.0/dispensary/${storeId}`;
|
||||
|
||||
// Try various endpoints
|
||||
const endpoints = [
|
||||
'/ecommerce/discounts?excludeInactive=true&hideUnset=true&includeProdInfo=true',
|
||||
'/ecommerce/products',
|
||||
'/products',
|
||||
'/menu',
|
||||
'/inventory',
|
||||
'/catalog',
|
||||
];
|
||||
|
||||
console.log('Testing Treez Headless API endpoints...\n');
|
||||
|
||||
for (const endpoint of endpoints) {
|
||||
const url = baseUrl + endpoint;
|
||||
console.log('GET ' + url);
|
||||
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
console.log(' Status: ' + response.status);
|
||||
const data = response.data;
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
console.log(' Array length: ' + data.length);
|
||||
if (data.length > 0) {
|
||||
console.log(' First item keys: ' + Object.keys(data[0]).join(', '));
|
||||
console.log(' Sample: ' + JSON.stringify(data[0]).slice(0, 300));
|
||||
}
|
||||
} else if (typeof data === 'object') {
|
||||
console.log(' Keys: ' + Object.keys(data).join(', '));
|
||||
console.log(' Sample: ' + JSON.stringify(data).slice(0, 500));
|
||||
}
|
||||
console.log('');
|
||||
|
||||
} catch (err: any) {
|
||||
console.log(' Error: ' + (err.response?.status || err.message));
|
||||
if (err.response?.data) {
|
||||
console.log(' Data: ' + JSON.stringify(err.response.data).slice(0, 200));
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
166
backend/scripts/test-treez-inventory.ts
Normal file
166
backend/scripts/test-treez-inventory.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Go to a product detail page
|
||||
await page.goto('https://shop.bestdispensary.com/brand/dime', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Get first product URL
|
||||
const productUrl = await page.evaluate(() => {
|
||||
const a = document.querySelector('a[href*="/product/"]');
|
||||
return a ? 'https://shop.bestdispensary.com' + a.getAttribute('href') : null;
|
||||
});
|
||||
|
||||
if (!productUrl) {
|
||||
console.log('No product found');
|
||||
await browser.close();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Checking product: ' + productUrl + '\n');
|
||||
await page.goto(productUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(2000);
|
||||
|
||||
// Look for inventory/stock info
|
||||
const inventoryData = await page.evaluate(() => {
|
||||
const data: any = {};
|
||||
|
||||
// Check for stock/inventory elements
|
||||
const stockSelectors = [
|
||||
'[class*="stock"]',
|
||||
'[class*="Stock"]',
|
||||
'[class*="inventory"]',
|
||||
'[class*="Inventory"]',
|
||||
'[class*="quantity"]',
|
||||
'[class*="Quantity"]',
|
||||
'[class*="available"]',
|
||||
'[class*="Available"]',
|
||||
'[class*="in-stock"]',
|
||||
'[class*="out-of-stock"]',
|
||||
'[data-stock]',
|
||||
'[data-quantity]',
|
||||
'[data-inventory]',
|
||||
];
|
||||
|
||||
data.stockElements = [];
|
||||
stockSelectors.forEach(sel => {
|
||||
document.querySelectorAll(sel).forEach(el => {
|
||||
data.stockElements.push({
|
||||
selector: sel,
|
||||
text: el.textContent?.trim().slice(0, 100),
|
||||
dataAttrs: Object.keys((el as HTMLElement).dataset || {}),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Check for "Add to cart" button state (disabled = out of stock)
|
||||
const addToCartBtn = document.querySelector('button[class*="add"], button[class*="cart"]');
|
||||
data.addToCartBtn = {
|
||||
found: !!addToCartBtn,
|
||||
disabled: (addToCartBtn as HTMLButtonElement)?.disabled,
|
||||
text: addToCartBtn?.textContent?.trim(),
|
||||
};
|
||||
|
||||
// Check page source for inventory keywords
|
||||
const bodyText = document.body.innerText;
|
||||
data.hasStockText = bodyText.includes('stock') || bodyText.includes('Stock');
|
||||
data.hasInventoryText = bodyText.includes('inventory') || bodyText.includes('Inventory');
|
||||
data.hasQuantityText = bodyText.includes('quantity') || bodyText.includes('Quantity');
|
||||
data.hasAvailableText = bodyText.includes('available') || bodyText.includes('Available');
|
||||
|
||||
// Get all data attributes on the page
|
||||
data.allDataAttrs = [];
|
||||
document.querySelectorAll('[data-product-id], [data-sku], [data-variant]').forEach(el => {
|
||||
const attrs: any = {};
|
||||
Object.entries((el as HTMLElement).dataset).forEach(([k, v]) => {
|
||||
attrs[k] = v;
|
||||
});
|
||||
if (Object.keys(attrs).length > 0) {
|
||||
data.allDataAttrs.push(attrs);
|
||||
}
|
||||
});
|
||||
|
||||
// Check for JSON-LD or schema data
|
||||
const scripts = document.querySelectorAll('script[type="application/ld+json"]');
|
||||
data.jsonLd = [];
|
||||
scripts.forEach(s => {
|
||||
try {
|
||||
const json = JSON.parse(s.textContent || '');
|
||||
data.jsonLd.push(json);
|
||||
} catch {}
|
||||
});
|
||||
|
||||
// Check Next.js data
|
||||
const nextData = document.getElementById('__NEXT_DATA__');
|
||||
if (nextData) {
|
||||
try {
|
||||
const json = JSON.parse(nextData.textContent || '');
|
||||
data.hasNextData = true;
|
||||
data.nextDataKeys = Object.keys(json);
|
||||
// Look for product data in props
|
||||
if (json.props?.pageProps?.product) {
|
||||
data.productFromNext = json.props.pageProps.product;
|
||||
}
|
||||
if (json.props?.pageProps) {
|
||||
data.pagePropsKeys = Object.keys(json.props.pageProps);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
return data;
|
||||
});
|
||||
|
||||
console.log('Inventory Analysis:\n');
|
||||
console.log('Stock elements found: ' + inventoryData.stockElements.length);
|
||||
inventoryData.stockElements.forEach((s: any) => {
|
||||
console.log(' - ' + s.selector + ': "' + s.text + '"');
|
||||
});
|
||||
|
||||
console.log('\nAdd to Cart button: ' + JSON.stringify(inventoryData.addToCartBtn));
|
||||
|
||||
console.log('\nText checks:');
|
||||
console.log(' Has "stock": ' + inventoryData.hasStockText);
|
||||
console.log(' Has "inventory": ' + inventoryData.hasInventoryText);
|
||||
console.log(' Has "quantity": ' + inventoryData.hasQuantityText);
|
||||
console.log(' Has "available": ' + inventoryData.hasAvailableText);
|
||||
|
||||
console.log('\nData attributes: ' + JSON.stringify(inventoryData.allDataAttrs));
|
||||
console.log('\nJSON-LD: ' + JSON.stringify(inventoryData.jsonLd, null, 2));
|
||||
|
||||
if (inventoryData.hasNextData) {
|
||||
console.log('\nNext.js data found!');
|
||||
console.log(' Keys: ' + inventoryData.nextDataKeys);
|
||||
console.log(' Page props keys: ' + inventoryData.pagePropsKeys);
|
||||
if (inventoryData.productFromNext) {
|
||||
console.log('\n Product data from Next.js:');
|
||||
console.log(JSON.stringify(inventoryData.productFromNext, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
211
backend/scripts/test-treez-load-all.ts
Normal file
211
backend/scripts/test-treez-load-all.ts
Normal file
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* Find and interact with "load more brands" selector
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log(' Age gate detected, bypassing...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('='.repeat(60));
|
||||
console.log('Finding "Load More Brands" control');
|
||||
console.log('='.repeat(60));
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Don't block stylesheets - might affect layout
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
const url = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
console.log(`\nNavigating to ${url}`);
|
||||
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Find all selects and dropdowns
|
||||
console.log('\n[1] Looking for select elements...');
|
||||
|
||||
const selectInfo = await page.evaluate(() => {
|
||||
const results: any[] = [];
|
||||
|
||||
// Native select elements
|
||||
document.querySelectorAll('select').forEach((sel, i) => {
|
||||
const options = Array.from(sel.options).map(o => ({ value: o.value, text: o.text }));
|
||||
results.push({
|
||||
type: 'select',
|
||||
id: sel.id || `select-${i}`,
|
||||
class: sel.className,
|
||||
options: options.slice(0, 10),
|
||||
totalOptions: sel.options.length,
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Native selects found:', JSON.stringify(selectInfo, null, 2));
|
||||
|
||||
// Look for custom dropdown buttons
|
||||
console.log('\n[2] Looking for dropdown/button elements...');
|
||||
|
||||
const dropdownInfo = await page.evaluate(() => {
|
||||
const results: any[] = [];
|
||||
|
||||
// Look for common dropdown patterns
|
||||
const selectors = [
|
||||
'[class*="dropdown"]',
|
||||
'[class*="Dropdown"]',
|
||||
'[class*="select"]',
|
||||
'[class*="Select"]',
|
||||
'[class*="picker"]',
|
||||
'[class*="Picker"]',
|
||||
'[role="listbox"]',
|
||||
'[role="combobox"]',
|
||||
'button[aria-haspopup]',
|
||||
'[class*="brand"] button',
|
||||
'[class*="Brand"] button',
|
||||
'[class*="filter"]',
|
||||
'[class*="Filter"]',
|
||||
];
|
||||
|
||||
selectors.forEach(sel => {
|
||||
document.querySelectorAll(sel).forEach((el, i) => {
|
||||
const text = el.textContent?.trim().slice(0, 100) || '';
|
||||
const className = el.className?.toString?.().slice(0, 100) || '';
|
||||
if (text.toLowerCase().includes('brand') || text.toLowerCase().includes('more') || text.toLowerCase().includes('all')) {
|
||||
results.push({
|
||||
selector: sel,
|
||||
tag: el.tagName,
|
||||
class: className,
|
||||
text: text.slice(0, 50),
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Dropdown-like elements:', JSON.stringify(dropdownInfo.slice(0, 10), null, 2));
|
||||
|
||||
// Look for any element containing "brand" text
|
||||
console.log('\n[3] Looking for elements with "brand" or "more" text...');
|
||||
|
||||
const brandTextElements = await page.evaluate(() => {
|
||||
const results: any[] = [];
|
||||
const textContent = ['brand', 'more', 'load', 'view all', 'show all'];
|
||||
|
||||
document.querySelectorAll('button, a, [role="button"], select, [class*="select"]').forEach(el => {
|
||||
const text = el.textContent?.toLowerCase() || '';
|
||||
if (textContent.some(t => text.includes(t))) {
|
||||
results.push({
|
||||
tag: el.tagName,
|
||||
class: el.className?.toString?.().slice(0, 80) || '',
|
||||
text: el.textContent?.trim().slice(0, 100) || '',
|
||||
href: el.getAttribute('href') || '',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Elements with brand/more text:', JSON.stringify(brandTextElements.slice(0, 15), null, 2));
|
||||
|
||||
// Count current brand sections
|
||||
console.log('\n[4] Counting brand sections...');
|
||||
|
||||
const brandSections = await page.evaluate(() => {
|
||||
// Look for brand section headers or containers
|
||||
const sections: { title: string; productCount: number }[] = [];
|
||||
|
||||
document.querySelectorAll('[class*="products_product__section"]').forEach(section => {
|
||||
const header = section.querySelector('h2, h3, [class*="heading"]');
|
||||
const title = header?.textContent?.trim() || 'Unknown';
|
||||
const products = section.querySelectorAll('a[class*="product_product__"]');
|
||||
sections.push({ title, productCount: products.length });
|
||||
});
|
||||
|
||||
return sections;
|
||||
});
|
||||
|
||||
console.log(`Found ${brandSections.length} brand sections:`);
|
||||
brandSections.slice(0, 20).forEach(s => console.log(` - ${s.title}: ${s.productCount} products`));
|
||||
|
||||
// Take a screenshot
|
||||
await page.screenshot({ path: '/tmp/treez-brands-full.png', fullPage: true });
|
||||
console.log('\n[5] Full page screenshot saved to /tmp/treez-brands-full.png');
|
||||
|
||||
// Try scrolling to bottom to trigger any lazy loading
|
||||
console.log('\n[6] Scrolling to load more content...');
|
||||
|
||||
let previousHeight = 0;
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1500);
|
||||
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
const sectionCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('[class*="products_product__section"]').length
|
||||
);
|
||||
|
||||
console.log(` Scroll ${i + 1}: height=${currentHeight}, sections=${sectionCount}`);
|
||||
|
||||
if (currentHeight === previousHeight) {
|
||||
console.log(' No new content, stopping');
|
||||
break;
|
||||
}
|
||||
previousHeight = currentHeight;
|
||||
}
|
||||
|
||||
// Final count
|
||||
const finalSections = await page.evaluate(() => {
|
||||
const sections: { title: string; productCount: number }[] = [];
|
||||
document.querySelectorAll('[class*="products_product__section"]').forEach(section => {
|
||||
const header = section.querySelector('h2, h3, [class*="heading"]');
|
||||
const title = header?.textContent?.trim() || 'Unknown';
|
||||
const products = section.querySelectorAll('a[class*="product_product__"]');
|
||||
sections.push({ title, productCount: products.length });
|
||||
});
|
||||
return sections;
|
||||
});
|
||||
|
||||
console.log(`\n[7] After scrolling: ${finalSections.length} brand sections`);
|
||||
finalSections.forEach(s => console.log(` - ${s.title}: ${s.productCount} products`));
|
||||
|
||||
const totalProducts = finalSections.reduce((sum, s) => sum + s.productCount, 0);
|
||||
console.log(`\nTotal products across all sections: ${totalProducts}`);
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
104
backend/scripts/test-treez-network.ts
Normal file
104
backend/scripts/test-treez-network.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Capture all network requests
|
||||
const requests: any[] = [];
|
||||
const responses: any[] = [];
|
||||
|
||||
page.on('request', (req) => {
|
||||
const url = req.url();
|
||||
if (url.includes('api') || url.includes('graphql') ||
|
||||
url.includes('product') || url.includes('menu') ||
|
||||
url.includes('treez') || url.includes('inventory')) {
|
||||
requests.push({
|
||||
url: url.slice(0, 150),
|
||||
method: req.method(),
|
||||
headers: req.headers(),
|
||||
postData: req.postData()?.slice(0, 500),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
page.on('response', async (res) => {
|
||||
const url = res.url();
|
||||
if (url.includes('api') || url.includes('graphql') ||
|
||||
url.includes('product') || url.includes('menu') ||
|
||||
url.includes('inventory')) {
|
||||
try {
|
||||
const contentType = res.headers()['content-type'] || '';
|
||||
if (contentType.includes('json')) {
|
||||
const body = await res.text();
|
||||
responses.push({
|
||||
url: url.slice(0, 150),
|
||||
status: res.status(),
|
||||
bodyPreview: body.slice(0, 1000),
|
||||
});
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading page and capturing network requests...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Click load more to trigger more API calls
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (btn) {
|
||||
await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
// Also visit a product page
|
||||
console.log('\nVisiting a product page...\n');
|
||||
await page.goto('https://shop.bestdispensary.com/product/dime-sour-grapes-2g-disposable-cartridge-2-grams', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 30000
|
||||
});
|
||||
await sleep(2000);
|
||||
|
||||
console.log('=== API REQUESTS FOUND ===\n');
|
||||
requests.forEach((r, i) => {
|
||||
console.log((i+1) + '. ' + r.method + ' ' + r.url);
|
||||
if (r.postData) {
|
||||
console.log(' POST data: ' + r.postData);
|
||||
}
|
||||
});
|
||||
|
||||
console.log('\n=== JSON RESPONSES ===\n');
|
||||
responses.forEach((r, i) => {
|
||||
console.log((i+1) + '. ' + r.url);
|
||||
console.log(' Status: ' + r.status);
|
||||
console.log(' Body: ' + r.bodyPreview.slice(0, 300) + '...\n');
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
110
backend/scripts/test-treez-page-fetch.ts
Normal file
110
backend/scripts/test-treez-page-fetch.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
import fs from 'fs';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
console.log('Loading page first to establish session...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
// Wait for page to fully load
|
||||
await sleep(2000);
|
||||
|
||||
console.log('\nMaking fetch request from page context...\n');
|
||||
|
||||
// Try to make the ES request from within page context
|
||||
const result = await page.evaluate(async () => {
|
||||
const url = 'https://search-kyrok9udlk.gapcommerceapi.com/product/search';
|
||||
const apiKey = 'V3jHL9dFzi3Gj4UISM4lr38Nm0GSxcps5OBz1PbS';
|
||||
|
||||
const query = {
|
||||
from: 0,
|
||||
size: 1000,
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ bool: { filter: { range: { customMinPrice: { gte: 0.01, lte: 500000 }}}}},
|
||||
{ bool: { should: [{ match: { isAboveThreshold: true }}]}},
|
||||
{ bool: { should: [{ match: { isHideFromMenu: false }}]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': apiKey,
|
||||
},
|
||||
body: JSON.stringify(query),
|
||||
credentials: 'include',
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return { error: 'HTTP ' + response.status, statusText: response.statusText };
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return {
|
||||
total: data.hits?.total?.value,
|
||||
count: data.hits?.hits?.length,
|
||||
firstProduct: data.hits?.hits?.[0]?._source,
|
||||
products: data.hits?.hits?.map((h: any) => h._source),
|
||||
};
|
||||
} catch (err: any) {
|
||||
return { error: err.message };
|
||||
}
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
console.log('Error: ' + result.error);
|
||||
if (result.statusText) console.log('Status: ' + result.statusText);
|
||||
} else {
|
||||
console.log('Total products in ES: ' + result.total);
|
||||
console.log('Products returned: ' + result.count);
|
||||
|
||||
if (result.firstProduct) {
|
||||
console.log('\n=== PRODUCT FIELDS ===\n');
|
||||
console.log(Object.keys(result.firstProduct).sort().join('\n'));
|
||||
|
||||
console.log('\n=== SAMPLE PRODUCT ===\n');
|
||||
console.log(JSON.stringify(result.firstProduct, null, 2));
|
||||
|
||||
// Save all products
|
||||
if (result.products) {
|
||||
fs.writeFileSync('/tmp/treez-all-products.json', JSON.stringify(result.products, null, 2));
|
||||
console.log('\nSaved ' + result.products.length + ' products to /tmp/treez-all-products.json');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
171
backend/scripts/test-treez-page-state.ts
Normal file
171
backend/scripts/test-treez-page-state.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
import fs from 'fs';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
console.log('Loading page...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
// Extract __NEXT_DATA__
|
||||
console.log('\n=== NEXT.JS DATA ===\n');
|
||||
|
||||
const nextData = await page.evaluate(() => {
|
||||
const script = document.getElementById('__NEXT_DATA__');
|
||||
if (script) {
|
||||
try {
|
||||
return JSON.parse(script.textContent || '');
|
||||
} catch { return null; }
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
if (nextData) {
|
||||
console.log('Top keys: ' + Object.keys(nextData).join(', '));
|
||||
if (nextData.props?.pageProps) {
|
||||
console.log('pageProps keys: ' + Object.keys(nextData.props.pageProps).join(', '));
|
||||
|
||||
// Look for products
|
||||
const pp = nextData.props.pageProps;
|
||||
if (pp.products) {
|
||||
console.log('\nFound products: ' + pp.products.length);
|
||||
if (pp.products[0]) {
|
||||
console.log('Product fields: ' + Object.keys(pp.products[0]).join(', '));
|
||||
console.log('\nSample:\n' + JSON.stringify(pp.products[0], null, 2));
|
||||
}
|
||||
}
|
||||
if (pp.initialProducts) {
|
||||
console.log('\nFound initialProducts: ' + pp.initialProducts.length);
|
||||
}
|
||||
if (pp.data) {
|
||||
console.log('\nFound data: ' + (Array.isArray(pp.data) ? pp.data.length + ' items' : typeof pp.data));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check window object
|
||||
console.log('\n=== WINDOW GLOBALS ===\n');
|
||||
|
||||
const windowData = await page.evaluate(() => {
|
||||
const win = window as any;
|
||||
const result: any = {};
|
||||
|
||||
// Common patterns for storing product data
|
||||
const patterns = ['products', 'items', 'data', 'state', 'store', 'redux', 'apollo'];
|
||||
|
||||
Object.keys(win).forEach(key => {
|
||||
const lowerKey = key.toLowerCase();
|
||||
if (patterns.some(p => lowerKey.includes(p))) {
|
||||
try {
|
||||
const val = win[key];
|
||||
if (typeof val === 'object' && val !== null) {
|
||||
result[key] = {
|
||||
type: Array.isArray(val) ? 'array' : 'object',
|
||||
keys: Object.keys(val).slice(0, 10),
|
||||
length: Array.isArray(val) ? val.length : undefined,
|
||||
};
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
});
|
||||
|
||||
console.log('Window globals with data-like names:');
|
||||
Object.entries(windowData).forEach(([k, v]: [string, any]) => {
|
||||
console.log(' ' + k + ': ' + v.type + (v.length ? ' (' + v.length + ')' : '') + ' - keys: ' + v.keys?.join(', '));
|
||||
});
|
||||
|
||||
// Try to find React state
|
||||
console.log('\n=== EXTRACTING FROM DOM ===\n');
|
||||
|
||||
const domProducts = await page.evaluate(() => {
|
||||
const products: any[] = [];
|
||||
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((card: Element) => {
|
||||
const product: any = {};
|
||||
|
||||
product.href = card.getAttribute('href');
|
||||
product.name = card.querySelector('h3, h4, h5')?.textContent?.trim();
|
||||
|
||||
// Get all text
|
||||
const allText = card.textContent || '';
|
||||
|
||||
// Extract THC %
|
||||
const thcMatch = allText.match(/(\d+(?:\.\d+)?)\s*%/);
|
||||
if (thcMatch) product.thc = thcMatch[1];
|
||||
|
||||
// Extract price
|
||||
const priceMatch = allText.match(/\$(\d+(?:\.\d+)?)/);
|
||||
if (priceMatch) product.price = priceMatch[1];
|
||||
|
||||
// Extract weight
|
||||
const weightMatch = allText.match(/(\d+(?:\.\d+)?)\s*[gG]/);
|
||||
if (weightMatch) product.weight = weightMatch[1] + 'g';
|
||||
|
||||
// Get brand from card
|
||||
const brandEl = card.querySelector('[class*="brand"]');
|
||||
product.brand = brandEl?.textContent?.trim();
|
||||
|
||||
// Get strain type
|
||||
const strainTypes = ['Indica', 'Sativa', 'Hybrid', 'I/S', 'S/I', 'CBD'];
|
||||
strainTypes.forEach(st => {
|
||||
if (allText.includes(st)) product.strainType = st;
|
||||
});
|
||||
|
||||
// Get image
|
||||
const img = card.querySelector('img');
|
||||
product.image = img?.getAttribute('src');
|
||||
|
||||
products.push(product);
|
||||
});
|
||||
|
||||
return products;
|
||||
});
|
||||
|
||||
console.log('Products from DOM: ' + domProducts.length);
|
||||
if (domProducts.length > 0) {
|
||||
console.log('\nSample:\n' + JSON.stringify(domProducts[0], null, 2));
|
||||
|
||||
// Show variety
|
||||
console.log('\n=== DATA QUALITY ===');
|
||||
const withThc = domProducts.filter(p => p.thc).length;
|
||||
const withPrice = domProducts.filter(p => p.price).length;
|
||||
const withBrand = domProducts.filter(p => p.brand).length;
|
||||
const withStrain = domProducts.filter(p => p.strainType).length;
|
||||
|
||||
console.log('With THC%: ' + withThc + '/' + domProducts.length);
|
||||
console.log('With Price: ' + withPrice + '/' + domProducts.length);
|
||||
console.log('With Brand: ' + withBrand + '/' + domProducts.length);
|
||||
console.log('With Strain: ' + withStrain + '/' + domProducts.length);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
89
backend/scripts/test-treez-product-api.ts
Normal file
89
backend/scripts/test-treez-product-api.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import axios from 'axios';
|
||||
|
||||
async function main() {
|
||||
const clientId = '29dce682258145c6b1cf71027282d083';
|
||||
const clientSecret = 'A57bB49AfD7F4233B1750a0B501B4E16';
|
||||
const storeId = 'best';
|
||||
|
||||
// Try various Treez API endpoints for products
|
||||
const endpoints = [
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/ecommerce/products',
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/ecommerce/menu',
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/ecommerce/inventory',
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/ecommerce/catalog',
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/menu',
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/products',
|
||||
'https://api.treez.io/v2.0/dispensary/' + storeId + '/ecommerce/products',
|
||||
'https://api.treez.io/v2.0/dispensary/' + storeId + '/products',
|
||||
'https://selltreez.com/api/dispensary/' + storeId + '/products',
|
||||
];
|
||||
|
||||
console.log('Testing Treez product endpoints...\n');
|
||||
|
||||
for (const url of endpoints) {
|
||||
console.log('GET ' + url);
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: {
|
||||
'client_id': clientId,
|
||||
'client_secret': clientSecret,
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
console.log(' Status: ' + response.status);
|
||||
const data = response.data;
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
console.log(' Array: ' + data.length + ' items');
|
||||
if (data[0]) console.log(' Keys: ' + Object.keys(data[0]).slice(0, 10).join(', '));
|
||||
} else if (data?.data && Array.isArray(data.data)) {
|
||||
console.log(' data[]: ' + data.data.length + ' items');
|
||||
if (data.data[0]) console.log(' Keys: ' + Object.keys(data.data[0]).slice(0, 10).join(', '));
|
||||
} else {
|
||||
console.log(' Type: ' + typeof data);
|
||||
console.log(' Keys: ' + (typeof data === 'object' ? Object.keys(data).join(', ') : 'N/A'));
|
||||
}
|
||||
console.log('');
|
||||
|
||||
} catch (err: any) {
|
||||
const status = err.response?.status || 'network error';
|
||||
console.log(' Error: ' + status + '\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Also check the working discounts endpoint for clues
|
||||
console.log('\n=== CHECKING DISCOUNTS FOR PRODUCT REFERENCES ===\n');
|
||||
|
||||
try {
|
||||
const response = await axios.get(
|
||||
'https://headless.treez.io/v2.0/dispensary/' + storeId + '/ecommerce/discounts?excludeInactive=true&hideUnset=true&includeProdInfo=true',
|
||||
{
|
||||
headers: {
|
||||
'client_id': clientId,
|
||||
'client_secret': clientSecret,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
const data = response.data?.data || response.data;
|
||||
if (Array.isArray(data) && data.length > 0) {
|
||||
console.log('Discounts: ' + data.length);
|
||||
console.log('First discount keys: ' + Object.keys(data[0]).join(', '));
|
||||
|
||||
// Check if it has product info
|
||||
if (data[0].products) {
|
||||
console.log('\nHas products array: ' + data[0].products.length);
|
||||
if (data[0].products[0]) {
|
||||
console.log('Product keys: ' + Object.keys(data[0].products[0]).join(', '));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.log('Error: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
174
backend/scripts/test-treez-product-data.ts
Normal file
174
backend/scripts/test-treez-product-data.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
// Go to a brand page with products
|
||||
await page.goto('https://shop.bestdispensary.com/brand/best', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Get detailed product card structure
|
||||
console.log('Analyzing product card structure...\n');
|
||||
|
||||
const productData = await page.evaluate(() => {
|
||||
const products: any[] = [];
|
||||
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((card: Element) => {
|
||||
const product: any = {};
|
||||
|
||||
// URL/slug
|
||||
product.href = card.getAttribute('href');
|
||||
product.slug = product.href?.split('/product/')[1];
|
||||
|
||||
// Image
|
||||
const img = card.querySelector('img');
|
||||
product.imageUrl = img?.getAttribute('src');
|
||||
product.imageAlt = img?.getAttribute('alt');
|
||||
|
||||
// Name (usually in h3 or similar)
|
||||
const nameEl = card.querySelector('h3, h4, h5, [class*="name"], [class*="title"]');
|
||||
product.name = nameEl?.textContent?.trim();
|
||||
|
||||
// Brand
|
||||
const brandEl = card.querySelector('[class*="brand"], [class*="Brand"]');
|
||||
product.brand = brandEl?.textContent?.trim();
|
||||
|
||||
// Price
|
||||
const priceEl = card.querySelector('[class*="price"], [class*="Price"]');
|
||||
product.priceText = priceEl?.textContent?.trim();
|
||||
|
||||
// Category/Type badges
|
||||
const badges: string[] = [];
|
||||
card.querySelectorAll('[class*="badge"], [class*="tag"], [class*="label"]').forEach((b: Element) => {
|
||||
const text = b.textContent?.trim();
|
||||
if (text) badges.push(text);
|
||||
});
|
||||
product.badges = badges;
|
||||
|
||||
// THC/CBD info
|
||||
const thcEl = card.querySelector('[class*="thc"], [class*="THC"]');
|
||||
const cbdEl = card.querySelector('[class*="cbd"], [class*="CBD"]');
|
||||
product.thc = thcEl?.textContent?.trim();
|
||||
product.cbd = cbdEl?.textContent?.trim();
|
||||
|
||||
// Weight/size
|
||||
const weightEl = card.querySelector('[class*="weight"], [class*="size"], [class*="gram"]');
|
||||
product.weight = weightEl?.textContent?.trim();
|
||||
|
||||
// Get all text content for analysis
|
||||
product.allText = card.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
|
||||
|
||||
// Get all classes on the card
|
||||
product.cardClasses = card.className;
|
||||
|
||||
products.push(product);
|
||||
});
|
||||
|
||||
return products;
|
||||
});
|
||||
|
||||
console.log('Found ' + productData.length + ' products\n');
|
||||
console.log('Sample product data:\n');
|
||||
|
||||
// Show first 3 products in detail
|
||||
productData.slice(0, 3).forEach((p: any, i: number) => {
|
||||
console.log('Product ' + (i+1) + ':');
|
||||
console.log(' Name: ' + p.name);
|
||||
console.log(' Brand: ' + p.brand);
|
||||
console.log(' Slug: ' + p.slug);
|
||||
console.log(' Price: ' + p.priceText);
|
||||
console.log(' THC: ' + p.thc);
|
||||
console.log(' CBD: ' + p.cbd);
|
||||
console.log(' Weight: ' + p.weight);
|
||||
console.log(' Badges: ' + JSON.stringify(p.badges));
|
||||
console.log(' Image: ' + (p.imageUrl ? p.imageUrl.slice(0, 60) + '...' : 'none'));
|
||||
console.log(' All Text: ' + p.allText);
|
||||
console.log('');
|
||||
});
|
||||
|
||||
// Now visit a product detail page
|
||||
if (productData.length > 0) {
|
||||
const productUrl = 'https://shop.bestdispensary.com' + productData[0].href;
|
||||
console.log('\n=== PRODUCT DETAIL PAGE ===');
|
||||
console.log('Visiting: ' + productUrl + '\n');
|
||||
|
||||
await page.goto(productUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(2000);
|
||||
|
||||
const detailData = await page.evaluate(() => {
|
||||
const data: any = {};
|
||||
|
||||
// Get all text elements
|
||||
data.h1 = document.querySelector('h1')?.textContent?.trim();
|
||||
data.h2s = Array.from(document.querySelectorAll('h2')).map(h => h.textContent?.trim());
|
||||
|
||||
// Price
|
||||
const priceEls = document.querySelectorAll('[class*="price"], [class*="Price"]');
|
||||
data.prices = Array.from(priceEls).map(p => p.textContent?.trim());
|
||||
|
||||
// Description
|
||||
const descEl = document.querySelector('[class*="description"], [class*="Description"], p');
|
||||
data.description = descEl?.textContent?.trim().slice(0, 300);
|
||||
|
||||
// THC/CBD
|
||||
data.cannabinoids = [];
|
||||
document.querySelectorAll('[class*="thc"], [class*="THC"], [class*="cbd"], [class*="CBD"], [class*="cannabinoid"]').forEach(el => {
|
||||
data.cannabinoids.push(el.textContent?.trim());
|
||||
});
|
||||
|
||||
// Category/strain type
|
||||
const typeEls = document.querySelectorAll('[class*="strain"], [class*="type"], [class*="category"]');
|
||||
data.types = Array.from(typeEls).map(t => t.textContent?.trim());
|
||||
|
||||
// Weight options
|
||||
const weightEls = document.querySelectorAll('[class*="weight"], [class*="size"], [class*="option"]');
|
||||
data.weights = Array.from(weightEls).map(w => w.textContent?.trim()).filter(w => w && w.length < 30);
|
||||
|
||||
// Images
|
||||
const imgs = document.querySelectorAll('img[src*="product"], img[src*="menu"]');
|
||||
data.images = Array.from(imgs).map(img => img.getAttribute('src')).slice(0, 3);
|
||||
|
||||
// Get body text for analysis
|
||||
const main = document.querySelector('main');
|
||||
data.mainText = main?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 500);
|
||||
|
||||
return data;
|
||||
});
|
||||
|
||||
console.log('Product Detail:');
|
||||
console.log(' H1: ' + detailData.h1);
|
||||
console.log(' H2s: ' + JSON.stringify(detailData.h2s));
|
||||
console.log(' Prices: ' + JSON.stringify(detailData.prices));
|
||||
console.log(' Description: ' + (detailData.description || 'none'));
|
||||
console.log(' Cannabinoids: ' + JSON.stringify(detailData.cannabinoids));
|
||||
console.log(' Types: ' + JSON.stringify(detailData.types));
|
||||
console.log(' Weights: ' + JSON.stringify(detailData.weights));
|
||||
console.log(' Images: ' + JSON.stringify(detailData.images));
|
||||
console.log('\n Main text sample: ' + detailData.mainText);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
143
backend/scripts/test-treez-scroll.ts
Normal file
143
backend/scripts/test-treez-scroll.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Test aggressive scrolling to load all products
|
||||
*/
|
||||
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
const capturedProducts: any[] = [];
|
||||
|
||||
// CDP interception
|
||||
const client = await page.target().createCDPSession();
|
||||
await client.send('Network.enable');
|
||||
|
||||
client.on('Network.responseReceived', async (event: any) => {
|
||||
if (event.response.url.includes('gapcommerceapi.com/product/search') && event.response.status === 200) {
|
||||
try {
|
||||
const response = await client.send('Network.getResponseBody', { requestId: event.requestId });
|
||||
const body = response.base64Encoded ? Buffer.from(response.body, 'base64').toString('utf8') : response.body;
|
||||
const json = JSON.parse(body);
|
||||
const products = json.hits?.hits?.map((h: any) => h._source) || [];
|
||||
capturedProducts.push(...products);
|
||||
console.log('Captured ' + products.length + ' (total: ' + capturedProducts.length + ')');
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
// Try direct treez.io URL - may have more products
|
||||
const url = process.argv[2] || 'https://best.treez.io/onlinemenu/';
|
||||
console.log('Loading ' + url);
|
||||
|
||||
try {
|
||||
await page.goto(url, { waitUntil: 'networkidle0', timeout: 60000 });
|
||||
} catch (e: any) {
|
||||
console.log('Navigation warning: ' + e.message);
|
||||
}
|
||||
|
||||
await sleep(5000);
|
||||
console.log('Current URL: ' + page.url());
|
||||
|
||||
// Age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
console.log('After initial load: ' + capturedProducts.length + ' products');
|
||||
|
||||
// Aggressive scrolling and clicking
|
||||
let lastCount = 0;
|
||||
let staleCount = 0;
|
||||
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(800);
|
||||
|
||||
try {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (btn) {
|
||||
const visible = await page.evaluate((b: Element) => {
|
||||
const rect = b.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, btn);
|
||||
|
||||
if (visible) {
|
||||
await page.evaluate((b: Element) => b.scrollIntoView({ block: 'center' }), btn);
|
||||
await sleep(300);
|
||||
await btn.click();
|
||||
await sleep(2000);
|
||||
console.log('Clicked Load More at scroll ' + (i+1) + ' - ' + capturedProducts.length + ' products');
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Check for stale data
|
||||
if (capturedProducts.length === lastCount) {
|
||||
staleCount++;
|
||||
if (staleCount >= 5) {
|
||||
console.log('No new products for 5 iterations, stopping');
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
staleCount = 0;
|
||||
}
|
||||
lastCount = capturedProducts.length;
|
||||
}
|
||||
|
||||
console.log('\nFinal count: ' + capturedProducts.length + ' products');
|
||||
|
||||
// Dedupe
|
||||
const seen = new Set<string>();
|
||||
const unique = capturedProducts.filter(p => {
|
||||
if (!p.id || seen.has(p.id)) return false;
|
||||
seen.add(p.id);
|
||||
return true;
|
||||
});
|
||||
|
||||
console.log('Unique: ' + unique.length);
|
||||
|
||||
// Categories
|
||||
const cats: Record<string, number> = {};
|
||||
unique.forEach(p => {
|
||||
cats[p.category] = (cats[p.category] || 0) + 1;
|
||||
});
|
||||
console.log('\nCategories:');
|
||||
Object.entries(cats).sort((a, b) => b[1] - a[1]).forEach(([c, n]) => console.log(' ' + c + ': ' + n));
|
||||
|
||||
// Sample cannabis product
|
||||
const cannabis = unique.find(p => p.category === 'FLOWER' || p.category === 'VAPE');
|
||||
if (cannabis) {
|
||||
console.log('\nSample cannabis product:');
|
||||
console.log(JSON.stringify({
|
||||
id: cannabis.id,
|
||||
name: cannabis.name,
|
||||
brand: cannabis.brand,
|
||||
category: cannabis.category,
|
||||
subtype: cannabis.subtype,
|
||||
availableUnits: cannabis.availableUnits,
|
||||
customMinPrice: cannabis.customMinPrice,
|
||||
}, null, 2));
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
178
backend/scripts/test-treez-selectors.ts
Normal file
178
backend/scripts/test-treez-selectors.ts
Normal file
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Analyze all product element structures to find all selector patterns
|
||||
*/
|
||||
|
||||
import puppeteer, { Page } from 'puppeteer';
|
||||
|
||||
const STORE_ID = 'best';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function bypassAgeGate(page: Page): Promise<void> {
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Analyzing Treez product selectors...\n');
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
const url = `https://${STORE_ID}.treez.io/onlinemenu/brands?customerType=ADULT`;
|
||||
await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await sleep(3000);
|
||||
await bypassAgeGate(page);
|
||||
await sleep(2000);
|
||||
|
||||
// Analyze product elements
|
||||
const analysis = await page.evaluate(() => {
|
||||
const products = document.querySelectorAll('[class*="product_product__"]');
|
||||
const results: {
|
||||
hasName: number;
|
||||
hasPrice: number;
|
||||
noName: number;
|
||||
noPrice: number;
|
||||
nameClasses: Record<string, number>;
|
||||
priceClasses: Record<string, number>;
|
||||
sampleNoNameHTML: string[];
|
||||
sampleWithNameHTML: string[];
|
||||
h5Count: number;
|
||||
h4Count: number;
|
||||
h3Count: number;
|
||||
allHeadingsWithName: number;
|
||||
} = {
|
||||
hasName: 0,
|
||||
hasPrice: 0,
|
||||
noName: 0,
|
||||
noPrice: 0,
|
||||
nameClasses: {},
|
||||
priceClasses: {},
|
||||
sampleNoNameHTML: [],
|
||||
sampleWithNameHTML: [],
|
||||
h5Count: 0,
|
||||
h4Count: 0,
|
||||
h3Count: 0,
|
||||
allHeadingsWithName: 0,
|
||||
};
|
||||
|
||||
products.forEach((el, i) => {
|
||||
// Check current selectors
|
||||
const nameEl = el.querySelector('[class*="product__name"], [class*="name__"]');
|
||||
const priceEl = el.querySelector('[class*="price"]');
|
||||
|
||||
if (nameEl) {
|
||||
results.hasName++;
|
||||
const cls = nameEl.className?.toString?.() || '';
|
||||
results.nameClasses[cls] = (results.nameClasses[cls] || 0) + 1;
|
||||
} else {
|
||||
results.noName++;
|
||||
if (results.sampleNoNameHTML.length < 3) {
|
||||
results.sampleNoNameHTML.push(el.innerHTML.slice(0, 500));
|
||||
}
|
||||
}
|
||||
|
||||
if (priceEl) {
|
||||
results.hasPrice++;
|
||||
} else {
|
||||
results.noPrice++;
|
||||
}
|
||||
|
||||
// Check for headings that might contain names
|
||||
const h5 = el.querySelector('h5');
|
||||
const h4 = el.querySelector('h4');
|
||||
const h3 = el.querySelector('h3');
|
||||
if (h5) results.h5Count++;
|
||||
if (h4) results.h4Count++;
|
||||
if (h3) results.h3Count++;
|
||||
|
||||
// Any heading with text
|
||||
const anyHeading = el.querySelector('h1, h2, h3, h4, h5, h6');
|
||||
if (anyHeading?.textContent?.trim()) {
|
||||
results.allHeadingsWithName++;
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('Product Analysis:');
|
||||
console.log(`Total products: ${analysis.hasName + analysis.noName}`);
|
||||
console.log(`With name (current selector): ${analysis.hasName}`);
|
||||
console.log(`Without name (current selector): ${analysis.noName}`);
|
||||
console.log(`With price: ${analysis.hasPrice}`);
|
||||
console.log(`\nHeading counts:`);
|
||||
console.log(` H5: ${analysis.h5Count}`);
|
||||
console.log(` H4: ${analysis.h4Count}`);
|
||||
console.log(` H3: ${analysis.h3Count}`);
|
||||
console.log(` Any heading with text: ${analysis.allHeadingsWithName}`);
|
||||
|
||||
console.log('\nName classes found:');
|
||||
Object.entries(analysis.nameClasses).forEach(([cls, count]) => {
|
||||
console.log(` (${count}x) ${cls.slice(0, 80)}`);
|
||||
});
|
||||
|
||||
console.log('\n--- Sample products WITHOUT name selector ---');
|
||||
analysis.sampleNoNameHTML.forEach((html, i) => {
|
||||
console.log(`\n[Sample ${i + 1}]:`);
|
||||
console.log(html);
|
||||
});
|
||||
|
||||
// Try different selectors
|
||||
console.log('\n\n--- Testing alternative selectors ---');
|
||||
|
||||
const altResults = await page.evaluate(() => {
|
||||
const products = document.querySelectorAll('[class*="product_product__"]');
|
||||
const tests: Record<string, number> = {};
|
||||
|
||||
const selectors = [
|
||||
'h5',
|
||||
'h4',
|
||||
'h3',
|
||||
'[class*="heading"]',
|
||||
'[class*="title"]',
|
||||
'[class*="name"]',
|
||||
'a[href*="/product/"]',
|
||||
'.product_product__name__JcEk0',
|
||||
'[class*="ProductCard"]',
|
||||
];
|
||||
|
||||
selectors.forEach(sel => {
|
||||
let count = 0;
|
||||
products.forEach(el => {
|
||||
if (el.querySelector(sel)) count++;
|
||||
});
|
||||
tests[sel] = count;
|
||||
});
|
||||
|
||||
return tests;
|
||||
});
|
||||
|
||||
Object.entries(altResults).forEach(([sel, count]) => {
|
||||
console.log(` ${sel}: ${count} products`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
116
backend/scripts/test-treez-stealth.ts
Normal file
116
backend/scripts/test-treez-stealth.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
import fs from 'fs';
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
const capturedProducts: any[] = [];
|
||||
|
||||
// Use CDP to intercept responses
|
||||
const client = await page.target().createCDPSession();
|
||||
await client.send('Network.enable');
|
||||
|
||||
client.on('Network.responseReceived', async (event) => {
|
||||
const url = event.response.url;
|
||||
if (url.includes('gapcommerceapi.com/product/search') && event.response.status === 200) {
|
||||
try {
|
||||
const response = await client.send('Network.getResponseBody', {
|
||||
requestId: event.requestId,
|
||||
});
|
||||
|
||||
const body = response.base64Encoded
|
||||
? Buffer.from(response.body, 'base64').toString('utf8')
|
||||
: response.body;
|
||||
|
||||
const json = JSON.parse(body);
|
||||
const products = json.hits?.hits?.map((h: any) => h._source) || [];
|
||||
capturedProducts.push(...products);
|
||||
console.log('Captured ' + products.length + ' products (total: ' + capturedProducts.length + ')');
|
||||
|
||||
} catch (err: any) {
|
||||
// Ignore errors
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading page with Stealth plugin...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
// Wait for API calls
|
||||
await sleep(5000);
|
||||
|
||||
console.log('Initial capture: ' + capturedProducts.length + ' products');
|
||||
|
||||
// Scroll and click load more
|
||||
console.log('\nScrolling and clicking Load More...');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||
await sleep(1000);
|
||||
|
||||
try {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (btn) {
|
||||
await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
} catch {}
|
||||
|
||||
if (i % 5 === 0) {
|
||||
console.log('Progress: ' + capturedProducts.length + ' products');
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n=== RESULTS ===\n');
|
||||
|
||||
if (capturedProducts.length > 0) {
|
||||
const seen = new Set();
|
||||
const unique = capturedProducts.filter(p => {
|
||||
const id = p.id || p.productId;
|
||||
if (!id || seen.has(id)) return false;
|
||||
seen.add(id);
|
||||
return true;
|
||||
});
|
||||
|
||||
console.log('Total captured: ' + capturedProducts.length);
|
||||
console.log('Unique products: ' + unique.length);
|
||||
console.log('\nFields available:');
|
||||
console.log(Object.keys(unique[0]).sort().join('\n'));
|
||||
console.log('\nSample product:\n' + JSON.stringify(unique[0], null, 2));
|
||||
|
||||
fs.writeFileSync('/tmp/treez-products.json', JSON.stringify(unique, null, 2));
|
||||
console.log('\nSaved to /tmp/treez-products.json');
|
||||
} else {
|
||||
console.log('No products captured - API still blocking');
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
117
backend/scripts/test-treez-xhr-intercept.ts
Normal file
117
backend/scripts/test-treez-xhr-intercept.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
import fs from 'fs';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
// Enable request interception but don't modify requests
|
||||
await page.setRequestInterception(true);
|
||||
|
||||
const capturedProducts: any[] = [];
|
||||
|
||||
page.on('request', (request) => {
|
||||
request.continue();
|
||||
});
|
||||
|
||||
// Use CDP to intercept responses
|
||||
const client = await page.target().createCDPSession();
|
||||
await client.send('Network.enable');
|
||||
|
||||
client.on('Network.responseReceived', async (event) => {
|
||||
const url = event.response.url;
|
||||
if (url.includes('gapcommerceapi.com/product/search')) {
|
||||
console.log('ES Response detected: ' + event.response.status);
|
||||
|
||||
try {
|
||||
const response = await client.send('Network.getResponseBody', {
|
||||
requestId: event.requestId,
|
||||
});
|
||||
|
||||
const body = response.base64Encoded
|
||||
? Buffer.from(response.body, 'base64').toString('utf8')
|
||||
: response.body;
|
||||
|
||||
const json = JSON.parse(body);
|
||||
const products = json.hits?.hits?.map((h: any) => h._source) || [];
|
||||
capturedProducts.push(...products);
|
||||
console.log('Captured ' + products.length + ' products (total: ' + capturedProducts.length + ')');
|
||||
|
||||
} catch (err: any) {
|
||||
console.log('Could not get response body: ' + err.message);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading page with CDP interception...\n');
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/shop', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(3000);
|
||||
}
|
||||
|
||||
// Click load more many times
|
||||
console.log('\nClicking Load More...');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
try {
|
||||
const btn = await page.$('button.collection__load-more');
|
||||
if (!btn) break;
|
||||
|
||||
const visible = await page.evaluate((b) => {
|
||||
const rect = b.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, btn);
|
||||
|
||||
if (!visible) break;
|
||||
|
||||
await btn.click();
|
||||
await sleep(1500);
|
||||
console.log('Click ' + (i+1) + ': ' + capturedProducts.length + ' products');
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n=== RESULTS ===\n');
|
||||
console.log('Total captured: ' + capturedProducts.length);
|
||||
|
||||
if (capturedProducts.length > 0) {
|
||||
// Dedupe
|
||||
const seen = new Set();
|
||||
const unique = capturedProducts.filter(p => {
|
||||
const id = p.id || p.productId;
|
||||
if (!id || seen.has(id)) return false;
|
||||
seen.add(id);
|
||||
return true;
|
||||
});
|
||||
|
||||
console.log('Unique products: ' + unique.length);
|
||||
console.log('\nFields: ' + Object.keys(unique[0]).sort().join('\n'));
|
||||
console.log('\nSample:\n' + JSON.stringify(unique[0], null, 2));
|
||||
|
||||
fs.writeFileSync('/tmp/treez-products.json', JSON.stringify(unique, null, 2));
|
||||
console.log('\nSaved to /tmp/treez-products.json');
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -92,6 +92,10 @@ router.get('/', async (req: Request, res: Response) => {
|
||||
filter.worker_id = req.query.worker_id as string;
|
||||
}
|
||||
|
||||
if (req.query.pool_id) {
|
||||
filter.pool_id = parseInt(req.query.pool_id as string, 10);
|
||||
}
|
||||
|
||||
if (req.query.limit) {
|
||||
filter.limit = parseInt(req.query.limit as string, 10);
|
||||
}
|
||||
@@ -122,6 +126,31 @@ router.get('/counts', async (_req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/tasks/counts/by-state
|
||||
* Get pending task counts grouped by state
|
||||
*/
|
||||
router.get('/counts/by-state', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT
|
||||
d.state as state_code,
|
||||
COUNT(*) FILTER (WHERE t.status = 'pending') as pending,
|
||||
COUNT(*) FILTER (WHERE t.status IN ('claimed', 'running')) as active,
|
||||
COUNT(*) as total
|
||||
FROM worker_tasks t
|
||||
JOIN dispensaries d ON t.dispensary_id = d.id
|
||||
WHERE t.status IN ('pending', 'claimed', 'running')
|
||||
GROUP BY d.state
|
||||
ORDER BY COUNT(*) DESC
|
||||
`);
|
||||
res.json({ states: result.rows });
|
||||
} catch (error: unknown) {
|
||||
console.error('Error getting task counts by state:', error);
|
||||
res.status(500).json({ error: 'Failed to get task counts by state' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/tasks/capacity
|
||||
* Get capacity metrics for all roles
|
||||
@@ -1638,4 +1667,145 @@ router.post('/pool/resume', async (_req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// GEO TASK POOLS - View pools and their contents
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* GET /api/tasks/pools/summary
|
||||
* Quick summary of all pools for dashboard
|
||||
* NOTE: Must be defined BEFORE /pools/:id to avoid route conflict
|
||||
*/
|
||||
router.get('/pools/summary', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const { rows } = await pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT tp.id) as total_pools,
|
||||
COUNT(DISTINCT tp.id) FILTER (WHERE tp.is_active) as active_pools,
|
||||
COUNT(DISTINCT d.id) as total_stores,
|
||||
COUNT(DISTINCT d.id) FILTER (WHERE d.pool_id IS NOT NULL) as assigned_stores,
|
||||
COUNT(DISTINCT t.id) FILTER (WHERE t.status = 'pending') as pending_tasks,
|
||||
COUNT(DISTINCT t.id) FILTER (WHERE t.status = 'running') as running_tasks
|
||||
FROM task_pools tp
|
||||
LEFT JOIN dispensaries d ON d.pool_id = tp.id
|
||||
LEFT JOIN worker_tasks t ON t.dispensary_id = d.id
|
||||
`);
|
||||
|
||||
const poolStatus = await getTaskPoolStatus();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
...rows[0],
|
||||
pool_open: poolStatus.open,
|
||||
});
|
||||
} catch (err: any) {
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/tasks/pools
|
||||
* List all geo task pools with their stats
|
||||
*/
|
||||
router.get('/pools', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const { rows } = await pool.query(`
|
||||
SELECT
|
||||
tp.id,
|
||||
tp.name,
|
||||
tp.display_name,
|
||||
tp.state_code,
|
||||
tp.city,
|
||||
tp.timezone,
|
||||
tp.radius_miles,
|
||||
tp.is_active,
|
||||
COUNT(DISTINCT d.id) as store_count,
|
||||
COUNT(DISTINCT t.id) FILTER (WHERE t.status = 'pending') as pending_tasks,
|
||||
COUNT(DISTINCT t.id) FILTER (WHERE t.status = 'running') as running_tasks,
|
||||
COUNT(DISTINCT t.id) FILTER (WHERE t.status = 'completed') as completed_tasks,
|
||||
COUNT(DISTINCT wr.worker_id) FILTER (WHERE wr.current_pool_id = tp.id) as active_workers
|
||||
FROM task_pools tp
|
||||
LEFT JOIN dispensaries d ON d.pool_id = tp.id
|
||||
LEFT JOIN worker_tasks t ON t.dispensary_id = d.id
|
||||
LEFT JOIN worker_registry wr ON wr.current_pool_id = tp.id
|
||||
GROUP BY tp.id
|
||||
ORDER BY COUNT(DISTINCT t.id) FILTER (WHERE t.status = 'pending') DESC, tp.display_name
|
||||
`);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
pools: rows,
|
||||
total: rows.length,
|
||||
});
|
||||
} catch (err: any) {
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/tasks/pools/:id
|
||||
* Get a single pool with its stores and tasks
|
||||
*/
|
||||
router.get('/pools/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const poolId = parseInt(req.params.id);
|
||||
|
||||
// Get pool info
|
||||
const { rows: poolRows } = await pool.query(`
|
||||
SELECT * FROM task_pools WHERE id = $1
|
||||
`, [poolId]);
|
||||
|
||||
if (poolRows.length === 0) {
|
||||
return res.status(404).json({ success: false, error: 'Pool not found' });
|
||||
}
|
||||
|
||||
// Get stores in this pool
|
||||
const { rows: stores } = await pool.query(`
|
||||
SELECT
|
||||
d.id,
|
||||
d.name,
|
||||
d.city,
|
||||
d.state,
|
||||
d.latitude,
|
||||
d.longitude,
|
||||
COUNT(t.id) FILTER (WHERE t.status = 'pending') as pending_tasks,
|
||||
COUNT(t.id) FILTER (WHERE t.status = 'running') as running_tasks
|
||||
FROM dispensaries d
|
||||
LEFT JOIN worker_tasks t ON t.dispensary_id = d.id
|
||||
WHERE d.pool_id = $1
|
||||
GROUP BY d.id
|
||||
ORDER BY COUNT(t.id) FILTER (WHERE t.status = 'pending') DESC, d.name
|
||||
`, [poolId]);
|
||||
|
||||
// Get active workers for this pool
|
||||
const { rows: workers } = await pool.query(`
|
||||
SELECT
|
||||
worker_id,
|
||||
friendly_name,
|
||||
current_state,
|
||||
current_city,
|
||||
http_ip as proxy_ip,
|
||||
pool_stores_visited,
|
||||
pool_max_stores
|
||||
FROM worker_registry
|
||||
WHERE current_pool_id = $1
|
||||
`, [poolId]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
pool: poolRows[0],
|
||||
stores,
|
||||
workers,
|
||||
stats: {
|
||||
store_count: stores.length,
|
||||
worker_count: workers.length,
|
||||
pending_tasks: stores.reduce((sum, s) => sum + parseInt(s.pending_tasks || '0'), 0),
|
||||
running_tasks: stores.reduce((sum, s) => sum + parseInt(s.running_tasks || '0'), 0),
|
||||
},
|
||||
});
|
||||
} catch (err: any) {
|
||||
res.status(500).json({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
import { pool } from '../db/pool';
|
||||
import { buildEvomiProxyUrl, getEvomiConfig } from './crawl-rotator';
|
||||
import { isTaskPoolOpen } from '../tasks/task-pool-state';
|
||||
|
||||
export interface TaskPool {
|
||||
pool_id: number;
|
||||
@@ -46,12 +47,19 @@ export interface ClaimedPoolResult {
|
||||
|
||||
/**
|
||||
* Get all pools that have pending tasks
|
||||
* Returns empty if global pool is closed
|
||||
*/
|
||||
export async function getPoolsWithPendingTasks(): Promise<PoolWithPendingTasks[]> {
|
||||
const { rows } = await pool.query<PoolWithPendingTasks>(
|
||||
// Check global pool toggle first
|
||||
const poolOpen = await isTaskPoolOpen();
|
||||
if (!poolOpen) {
|
||||
return []; // Global pool is closed, no pools available
|
||||
}
|
||||
|
||||
const { rows } = await pool.query(
|
||||
`SELECT * FROM get_pools_with_pending_tasks()`
|
||||
);
|
||||
return rows;
|
||||
return rows as PoolWithPendingTasks[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -63,7 +71,7 @@ export async function claimPool(
|
||||
poolId?: number
|
||||
): Promise<ClaimedPoolResult | null> {
|
||||
// Claim pool in database
|
||||
const { rows } = await pool.query<TaskPool>(
|
||||
const { rows } = await pool.query(
|
||||
`SELECT * FROM worker_claim_pool($1, $2)`,
|
||||
[workerId, poolId || null]
|
||||
);
|
||||
@@ -73,7 +81,7 @@ export async function claimPool(
|
||||
return null;
|
||||
}
|
||||
|
||||
const claimedPool = rows[0];
|
||||
const claimedPool = rows[0] as TaskPool;
|
||||
|
||||
// Build Evomi proxy URL for this pool's geo
|
||||
const evomiConfig = getEvomiConfig();
|
||||
@@ -107,17 +115,18 @@ export async function pullTasksFromPool(
|
||||
workerId: string,
|
||||
maxStores: number = 6
|
||||
): Promise<PoolTask[]> {
|
||||
const { rows } = await pool.query<PoolTask>(
|
||||
const { rows } = await pool.query(
|
||||
`SELECT * FROM pull_tasks_from_pool($1, $2)`,
|
||||
[workerId, maxStores]
|
||||
);
|
||||
|
||||
if (rows.length > 0) {
|
||||
const storeIds = [...new Set(rows.map(t => t.dispensary_id))];
|
||||
console.log(`[TaskPool] ${workerId} pulled ${rows.length} tasks for ${storeIds.length} stores`);
|
||||
const tasks = rows as PoolTask[];
|
||||
if (tasks.length > 0) {
|
||||
const storeIds = [...new Set(tasks.map(t => t.dispensary_id))];
|
||||
console.log(`[TaskPool] ${workerId} pulled ${tasks.length} tasks for ${storeIds.length} stores`);
|
||||
}
|
||||
|
||||
return rows;
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -148,7 +157,7 @@ export async function isWorkerExhausted(workerId: string): Promise<boolean> {
|
||||
* Get worker's current pool info
|
||||
*/
|
||||
export async function getWorkerPool(workerId: string): Promise<TaskPool | null> {
|
||||
const { rows } = await pool.query<TaskPool>(
|
||||
const { rows } = await pool.query(
|
||||
`SELECT tp.id as pool_id, tp.name as pool_name, tp.display_name,
|
||||
tp.state_code, tp.city, tp.latitude, tp.longitude, tp.timezone
|
||||
FROM worker_registry wr
|
||||
@@ -156,7 +165,7 @@ export async function getWorkerPool(workerId: string): Promise<TaskPool | null>
|
||||
WHERE wr.worker_id = $1`,
|
||||
[workerId]
|
||||
);
|
||||
return rows[0] || null;
|
||||
return (rows[0] as TaskPool) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -111,6 +111,7 @@ export interface TaskFilter {
|
||||
status?: TaskStatus | TaskStatus[];
|
||||
dispensary_id?: number;
|
||||
worker_id?: string;
|
||||
pool_id?: number;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
@@ -431,6 +432,11 @@ class TaskService {
|
||||
params.push(filter.worker_id);
|
||||
}
|
||||
|
||||
if (filter.pool_id) {
|
||||
conditions.push(`d.pool_id = $${paramIndex++}`);
|
||||
params.push(filter.pool_id);
|
||||
}
|
||||
|
||||
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
const limit = filter.limit ?? 100;
|
||||
const offset = filter.offset ?? 0;
|
||||
@@ -440,9 +446,12 @@ class TaskService {
|
||||
t.*,
|
||||
d.name as dispensary_name,
|
||||
d.slug as dispensary_slug,
|
||||
d.pool_id as pool_id,
|
||||
tp.display_name as pool_name,
|
||||
w.friendly_name as worker_name
|
||||
FROM worker_tasks t
|
||||
LEFT JOIN dispensaries d ON d.id = t.dispensary_id
|
||||
LEFT JOIN task_pools tp ON tp.id = d.pool_id
|
||||
LEFT JOIN worker_registry w ON w.worker_id = t.worker_id
|
||||
${whereClause}
|
||||
ORDER BY t.created_at DESC
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* Task Worker
|
||||
* Task Worker (v2.1)
|
||||
*
|
||||
* A unified worker that pulls tasks from the worker_tasks queue.
|
||||
* Workers register on startup, get a friendly name, and pull tasks.
|
||||
@@ -2043,7 +2043,7 @@ export class TaskWorker {
|
||||
*/
|
||||
private async initStealthOnly(): Promise<boolean> {
|
||||
try {
|
||||
await this.ensureStealthReady();
|
||||
await this.initializeStealth();
|
||||
this.stealthInitialized = true;
|
||||
return true;
|
||||
} catch (err: any) {
|
||||
|
||||
@@ -51,8 +51,8 @@ const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY;
|
||||
const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY;
|
||||
const MINIO_BUCKET = process.env.MINIO_BUCKET || 'cannaiq';
|
||||
|
||||
// Check if MinIO is configured
|
||||
const useMinIO = !!(MINIO_ENDPOINT && MINIO_ACCESS_KEY && MINIO_SECRET_KEY);
|
||||
// Check if MinIO is configured (endpoint required, keys optional for IP-based auth)
|
||||
const useMinIO = !!MINIO_ENDPOINT;
|
||||
|
||||
let minioClient: Minio.Client | null = null;
|
||||
|
||||
@@ -62,8 +62,8 @@ function getMinioClient(): Minio.Client {
|
||||
endPoint: MINIO_ENDPOINT!,
|
||||
port: MINIO_PORT,
|
||||
useSSL: MINIO_USE_SSL,
|
||||
accessKey: MINIO_ACCESS_KEY!,
|
||||
secretKey: MINIO_SECRET_KEY!,
|
||||
accessKey: MINIO_ACCESS_KEY || '', // Empty for IP-based auth
|
||||
secretKey: MINIO_SECRET_KEY || '', // Empty for IP-based auth
|
||||
});
|
||||
}
|
||||
return minioClient!;
|
||||
|
||||
98
backend/treez-intercept.js
Normal file
98
backend/treez-intercept.js
Normal file
@@ -0,0 +1,98 @@
|
||||
const puppeteer = require('puppeteer');
|
||||
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox']
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
// Collect all API requests
|
||||
const apiRequests = [];
|
||||
|
||||
page.on('request', (request) => {
|
||||
const url = request.url();
|
||||
// Focus on API/data requests
|
||||
if (url.includes('api') || url.includes('graphql') ||
|
||||
url.includes('.json') || request.resourceType() === 'xhr' ||
|
||||
request.resourceType() === 'fetch') {
|
||||
apiRequests.push({
|
||||
url: url,
|
||||
method: request.method(),
|
||||
headers: request.headers(),
|
||||
type: request.resourceType()
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
page.on('response', async (response) => {
|
||||
const url = response.url();
|
||||
const status = response.status();
|
||||
|
||||
// Log API responses with content type
|
||||
if (url.includes('api') || url.includes('graphql') ||
|
||||
url.includes('.json') || url.includes('product')) {
|
||||
const contentType = response.headers()['content-type'] || '';
|
||||
console.log('[' + status + '] ' + url.substring(0, 120));
|
||||
|
||||
// Try to get JSON responses
|
||||
if (contentType.includes('json') && status === 200) {
|
||||
try {
|
||||
const text = await response.text();
|
||||
const preview = text.substring(0, 500);
|
||||
console.log(' Preview: ' + preview);
|
||||
} catch (e) {}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Loading https://best.treez.io/onlinemenu/...');
|
||||
|
||||
await page.goto('https://best.treez.io/onlinemenu/', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
|
||||
// Wait for any lazy-loaded content
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
// Try scrolling to trigger more loads
|
||||
await page.evaluate(() => {
|
||||
window.scrollTo(0, document.body.scrollHeight / 2);
|
||||
});
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('\n=== ALL API REQUESTS ===');
|
||||
apiRequests.forEach(req => {
|
||||
console.log(req.method + ' ' + req.url);
|
||||
if (req.headers['authorization']) {
|
||||
console.log(' Auth: ' + req.headers['authorization'].substring(0, 50) + '...');
|
||||
}
|
||||
});
|
||||
|
||||
// Check page content
|
||||
const content = await page.content();
|
||||
console.log('\n=== PAGE TITLE ===');
|
||||
console.log(await page.title());
|
||||
|
||||
// Look for product data in page
|
||||
const productData = await page.evaluate(() => {
|
||||
// Check for React state or window variables
|
||||
const windowKeys = Object.keys(window).filter(k =>
|
||||
k.includes('store') || k.includes('product') || k.includes('__')
|
||||
);
|
||||
return {
|
||||
windowKeys: windowKeys.slice(0, 20),
|
||||
bodyText: document.body.innerText.substring(0, 1000)
|
||||
};
|
||||
});
|
||||
|
||||
console.log('\n=== WINDOW KEYS ===');
|
||||
console.log(productData.windowKeys);
|
||||
|
||||
console.log('\n=== PAGE TEXT PREVIEW ===');
|
||||
console.log(productData.bodyText);
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
4
cannaiq/dist/index.html
vendored
4
cannaiq/dist/index.html
vendored
@@ -7,8 +7,8 @@
|
||||
<title>CannaIQ - Cannabis Menu Intelligence Platform</title>
|
||||
<meta name="description" content="CannaIQ provides real-time cannabis dispensary menu data, product tracking, and analytics for dispensaries across Arizona." />
|
||||
<meta name="keywords" content="cannabis, dispensary, menu, products, analytics, Arizona" />
|
||||
<script type="module" crossorigin src="/assets/index-Cgew9i_-.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-yJj_6wf9.css">
|
||||
<script type="module" crossorigin src="/assets/index-CvnXE72u.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-B3fDWC0G.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
4115
cannaiq/node_modules/.package-lock.json
generated
vendored
4115
cannaiq/node_modules/.package-lock.json
generated
vendored
File diff suppressed because it is too large
Load Diff
4414
cannaiq/package-lock.json
generated
4414
cannaiq/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -25,10 +25,10 @@
|
||||
"autoprefixer": "^10.4.16",
|
||||
"daisyui": "^4.4.19",
|
||||
"postcss": "^8.4.32",
|
||||
"sharp": "^0.33.5",
|
||||
"tailwindcss": "^3.4.1",
|
||||
"typescript": "^5.3.3",
|
||||
"vite": "^5.0.8",
|
||||
"vite-plugin-pwa": "^0.21.1",
|
||||
"sharp": "^0.33.5"
|
||||
"vite-plugin-pwa": "^0.21.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2902,6 +2902,7 @@ class ApiClient {
|
||||
role?: string;
|
||||
status?: string;
|
||||
dispensary_id?: number;
|
||||
pool_id?: number;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}) {
|
||||
@@ -2909,6 +2910,7 @@ class ApiClient {
|
||||
if (params?.role) query.set('role', params.role);
|
||||
if (params?.status) query.set('status', params.status);
|
||||
if (params?.dispensary_id) query.set('dispensary_id', String(params.dispensary_id));
|
||||
if (params?.pool_id) query.set('pool_id', String(params.pool_id));
|
||||
if (params?.limit) query.set('limit', String(params.limit));
|
||||
if (params?.offset) query.set('offset', String(params.offset));
|
||||
const qs = query.toString();
|
||||
@@ -2930,6 +2932,17 @@ class ApiClient {
|
||||
}>('/api/tasks/counts');
|
||||
}
|
||||
|
||||
async getTaskCountsByState() {
|
||||
return this.request<{
|
||||
states: Array<{
|
||||
state_code: string;
|
||||
pending: string;
|
||||
active: string;
|
||||
total: string;
|
||||
}>;
|
||||
}>('/api/tasks/counts/by-state');
|
||||
}
|
||||
|
||||
async getTaskCapacity() {
|
||||
return this.request<{ metrics: any[] }>('/api/tasks/capacity');
|
||||
}
|
||||
|
||||
@@ -47,6 +47,16 @@ interface Task {
|
||||
retry_count: number;
|
||||
created_at: string;
|
||||
duration_sec?: number;
|
||||
pool_id?: number | null;
|
||||
pool_name?: string | null;
|
||||
}
|
||||
|
||||
interface TaskPool {
|
||||
id: number;
|
||||
name: string;
|
||||
display_name: string;
|
||||
state_code: string;
|
||||
city: string;
|
||||
}
|
||||
|
||||
interface CapacityMetric {
|
||||
@@ -892,9 +902,17 @@ interface Worker {
|
||||
friendly_name: string;
|
||||
}
|
||||
|
||||
interface StateCounts {
|
||||
state_code: string;
|
||||
pending: string;
|
||||
active: string;
|
||||
total: string;
|
||||
}
|
||||
|
||||
export default function TasksDashboard() {
|
||||
const [tasks, setTasks] = useState<Task[]>([]);
|
||||
const [counts, setCounts] = useState<TaskCounts | null>(null);
|
||||
const [stateCounts, setStateCounts] = useState<StateCounts[]>([]);
|
||||
const [capacity, setCapacity] = useState<CapacityMetric[]>([]);
|
||||
const [workers, setWorkers] = useState<Worker[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -917,31 +935,40 @@ export default function TasksDashboard() {
|
||||
// Filters
|
||||
const [roleFilter, setRoleFilter] = useState<string>('');
|
||||
const [statusFilter, setStatusFilter] = useState<string>('');
|
||||
const [poolFilter, setPoolFilter] = useState<string>('');
|
||||
const [searchQuery, setSearchQuery] = useState('');
|
||||
const [showCapacity, setShowCapacity] = useState(true);
|
||||
|
||||
// Pools for filter dropdown
|
||||
const [pools, setPools] = useState<TaskPool[]>([]);
|
||||
|
||||
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
const [tasksRes, countsRes, capacityRes, poolStatus, schedulesRes, workersRes] = await Promise.all([
|
||||
const [tasksRes, countsRes, stateCountsRes, capacityRes, poolStatus, schedulesRes, workersRes, poolsRes] = await Promise.all([
|
||||
api.getTasks({
|
||||
role: roleFilter || undefined,
|
||||
status: statusFilter || undefined,
|
||||
pool_id: poolFilter ? parseInt(poolFilter) : undefined,
|
||||
limit: 100,
|
||||
}),
|
||||
api.getTaskCounts(),
|
||||
api.getTaskCountsByState(),
|
||||
api.getTaskCapacity(),
|
||||
api.getTaskPoolStatus(),
|
||||
api.getTaskSchedules(),
|
||||
api.getWorkerRegistry().catch(() => ({ workers: [] })),
|
||||
api.get('/api/tasks/pools').catch(() => ({ data: { pools: [] } })),
|
||||
]);
|
||||
|
||||
setTasks(tasksRes.tasks || []);
|
||||
setCounts(countsRes);
|
||||
setStateCounts(stateCountsRes.states || []);
|
||||
setCapacity(capacityRes.metrics || []);
|
||||
setPoolOpen(poolStatus.open ?? !poolStatus.paused);
|
||||
setSchedules(schedulesRes.schedules || []);
|
||||
setWorkers(workersRes.workers || []);
|
||||
setPools(poolsRes.data?.pools || []);
|
||||
setError(null);
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Failed to load tasks');
|
||||
@@ -1052,7 +1079,7 @@ export default function TasksDashboard() {
|
||||
fetchData();
|
||||
const interval = setInterval(fetchData, 15000); // Auto-refresh every 15 seconds
|
||||
return () => clearInterval(interval);
|
||||
}, [roleFilter, statusFilter]);
|
||||
}, [roleFilter, statusFilter, poolFilter]);
|
||||
|
||||
// Create worker name lookup map (fallback for tasks without joined worker_name)
|
||||
const workerNameMap = new Map(workers.map(w => [w.worker_id, w.friendly_name]));
|
||||
@@ -1096,6 +1123,7 @@ export default function TasksDashboard() {
|
||||
task.dispensary_name?.toLowerCase().includes(query) ||
|
||||
task.worker_id?.toLowerCase().includes(query) ||
|
||||
workerName.toLowerCase().includes(query) ||
|
||||
task.pool_name?.toLowerCase().includes(query) ||
|
||||
String(task.id).includes(query)
|
||||
);
|
||||
}
|
||||
@@ -1136,25 +1164,6 @@ export default function TasksDashboard() {
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-4">
|
||||
{/* Pool Toggle */}
|
||||
<button
|
||||
onClick={handleTogglePool}
|
||||
disabled={poolToggling}
|
||||
className={`flex items-center gap-2 px-4 py-2 rounded-lg font-medium transition-colors ${
|
||||
poolOpen
|
||||
? 'bg-emerald-100 text-emerald-800 hover:bg-emerald-200'
|
||||
: 'bg-red-100 text-red-800 hover:bg-red-200'
|
||||
} disabled:opacity-50 disabled:cursor-not-allowed`}
|
||||
>
|
||||
{poolToggling ? (
|
||||
<RefreshCw className="w-4 h-4 animate-spin" />
|
||||
) : poolOpen ? (
|
||||
<PlayCircle className="w-4 h-4" />
|
||||
) : (
|
||||
<Square className="w-4 h-4" />
|
||||
)}
|
||||
{poolOpen ? 'Pool is Open' : 'Pool is Closed'}
|
||||
</button>
|
||||
{/* Create Task Button */}
|
||||
<button
|
||||
onClick={() => setShowCreateModal(true)}
|
||||
@@ -1191,7 +1200,32 @@ export default function TasksDashboard() {
|
||||
/>
|
||||
|
||||
{/* Status Summary Cards */}
|
||||
<div className="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-4">
|
||||
<div className="grid grid-cols-2 sm:grid-cols-4 lg:grid-cols-8 gap-4">
|
||||
{/* Pool Toggle Tile */}
|
||||
<div
|
||||
className={`p-4 rounded-lg border cursor-pointer hover:shadow-md transition-all ${
|
||||
poolOpen ? 'bg-emerald-50 border-emerald-300' : 'bg-red-50 border-red-300'
|
||||
}`}
|
||||
onClick={handleTogglePool}
|
||||
>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className={`p-1.5 rounded ${poolOpen ? 'bg-emerald-100' : 'bg-red-100'}`}>
|
||||
{poolToggling ? (
|
||||
<RefreshCw className={`w-4 h-4 animate-spin ${poolOpen ? 'text-emerald-600' : 'text-red-600'}`} />
|
||||
) : poolOpen ? (
|
||||
<PlayCircle className="w-4 h-4 text-emerald-600" />
|
||||
) : (
|
||||
<Square className="w-4 h-4 text-red-600" />
|
||||
)}
|
||||
</span>
|
||||
<span className={`text-sm font-medium ${poolOpen ? 'text-emerald-700' : 'text-red-700'}`}>Pool</span>
|
||||
</div>
|
||||
<div className={`text-2xl font-bold ${poolOpen ? 'text-emerald-700' : 'text-red-700'}`}>
|
||||
{poolOpen ? 'On' : 'Off'}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Status Cards */}
|
||||
{Object.entries(counts || {}).map(([status, count]) => (
|
||||
<div
|
||||
key={status}
|
||||
@@ -1209,6 +1243,33 @@ export default function TasksDashboard() {
|
||||
<div className="text-2xl font-bold text-gray-900">{count}</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* States Tile */}
|
||||
<div className="p-4 rounded-lg border bg-white hover:shadow-md transition-shadow">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className="p-1.5 rounded bg-blue-100">
|
||||
<Globe className="w-4 h-4 text-blue-600" />
|
||||
</span>
|
||||
<span className="text-sm font-medium text-gray-600">States</span>
|
||||
</div>
|
||||
<div className="max-h-16 overflow-y-auto">
|
||||
{stateCounts.length === 0 ? (
|
||||
<span className="text-gray-400 text-sm">No active tasks</span>
|
||||
) : (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{stateCounts.map((s) => (
|
||||
<span
|
||||
key={s.state_code}
|
||||
className="inline-flex items-center px-1.5 py-0.5 rounded text-xs font-medium bg-blue-50 text-blue-700"
|
||||
title={`${s.state_code}: ${s.pending} pending, ${s.active} active`}
|
||||
>
|
||||
{s.state_code}: {parseInt(s.total)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Capacity Planning Section */}
|
||||
@@ -1597,6 +1658,19 @@ export default function TasksDashboard() {
|
||||
<option value="failed">Failed</option>
|
||||
<option value="stale">Stale</option>
|
||||
</select>
|
||||
|
||||
<select
|
||||
value={poolFilter}
|
||||
onChange={(e) => setPoolFilter(e.target.value)}
|
||||
className="px-4 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-emerald-500"
|
||||
>
|
||||
<option value="">All Pools</option>
|
||||
{pools.map((pool) => (
|
||||
<option key={pool.id} value={pool.id}>
|
||||
{pool.display_name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{/* Tasks Table */}
|
||||
@@ -1614,6 +1688,9 @@ export default function TasksDashboard() {
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Store
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Pool
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Status
|
||||
</th>
|
||||
@@ -1636,7 +1713,7 @@ export default function TasksDashboard() {
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{paginatedTasks.length === 0 ? (
|
||||
<tr>
|
||||
<td colSpan={9} className="px-4 py-8 text-center text-gray-500">
|
||||
<td colSpan={10} className="px-4 py-8 text-center text-gray-500">
|
||||
No tasks found
|
||||
</td>
|
||||
</tr>
|
||||
@@ -1650,6 +1727,16 @@ export default function TasksDashboard() {
|
||||
<td className="px-4 py-3 text-sm text-gray-600">
|
||||
{task.dispensary_name || task.dispensary_id || '-'}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-sm text-gray-600">
|
||||
{task.pool_name ? (
|
||||
<span className="inline-flex items-center gap-1 px-2 py-0.5 bg-blue-50 text-blue-700 rounded text-xs font-medium">
|
||||
<Globe className="w-3 h-3" />
|
||||
{task.pool_name}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">-</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<span
|
||||
className={`inline-flex items-center gap-1 px-2 py-1 rounded-full text-xs font-medium ${
|
||||
|
||||
@@ -35,6 +35,7 @@ interface OriginFormData {
|
||||
origin_type: 'domain' | 'ip' | 'pattern';
|
||||
origin_value: string;
|
||||
description: string;
|
||||
active?: boolean;
|
||||
}
|
||||
|
||||
export function Users() {
|
||||
@@ -60,7 +61,7 @@ export function Users() {
|
||||
// Add dropdown state
|
||||
const [showAddDropdown, setShowAddDropdown] = useState(false);
|
||||
const [editingOrigin, setEditingOrigin] = useState<TrustedOrigin | null>(null);
|
||||
const [originFormData, setOriginFormData] = useState<OriginFormData>({ name: '', origin_type: 'domain', origin_value: '', description: '' });
|
||||
const [originFormData, setOriginFormData] = useState<OriginFormData>({ name: '', origin_type: 'domain', origin_value: '', description: '', active: true });
|
||||
const [originFormError, setOriginFormError] = useState<string | null>(null);
|
||||
const [originSaving, setOriginSaving] = useState(false);
|
||||
|
||||
@@ -175,7 +176,7 @@ export function Users() {
|
||||
};
|
||||
|
||||
// Auto-detect origin type from input value
|
||||
const detectOriginType = (value: string): { type: 'domain' | 'ip' | 'regex'; value: string } => {
|
||||
const detectOriginType = (value: string): { type: 'domain' | 'ip' | 'pattern'; value: string } => {
|
||||
const trimmed = value.trim();
|
||||
|
||||
// IP address pattern (simple check for IPv4)
|
||||
@@ -183,15 +184,15 @@ export function Users() {
|
||||
return { type: 'ip', value: trimmed };
|
||||
}
|
||||
|
||||
// Wildcard pattern like *.example.com -> convert to regex
|
||||
// Wildcard pattern like *.example.com -> convert to regex pattern
|
||||
if (trimmed.startsWith('*.')) {
|
||||
const domain = trimmed.slice(2).replace(/\./g, '\\.');
|
||||
return { type: 'regex', value: `^https?://.*\\.${domain}$` };
|
||||
return { type: 'pattern', value: `^https?://.*\\.${domain}$` };
|
||||
}
|
||||
|
||||
// Already a regex (starts with ^ or contains regex characters)
|
||||
// Already a regex pattern (starts with ^ or contains regex characters)
|
||||
if (trimmed.startsWith('^') || /[\\()\[\]{}|+?]/.test(trimmed)) {
|
||||
return { type: 'regex', value: trimmed };
|
||||
return { type: 'pattern', value: trimmed };
|
||||
}
|
||||
|
||||
// Strip protocol if provided for domain
|
||||
@@ -224,7 +225,7 @@ export function Users() {
|
||||
|
||||
await api.createTrustedOrigin(payload);
|
||||
setShowOriginModal(false);
|
||||
setOriginFormData({ name: '', origin_type: 'domain', origin_value: '', description: '' });
|
||||
setOriginFormData({ name: '', origin_type: 'domain', origin_value: '', description: '', active: true });
|
||||
fetchOrigins();
|
||||
} catch (err: any) {
|
||||
setOriginFormError(err.message || 'Failed to create origin');
|
||||
@@ -249,7 +250,7 @@ export function Users() {
|
||||
|
||||
await api.updateTrustedOrigin(editingOrigin.id, payload);
|
||||
setEditingOrigin(null);
|
||||
setOriginFormData({ name: '', origin_type: 'domain', origin_value: '', description: '' });
|
||||
setOriginFormData({ name: '', origin_type: 'domain', origin_value: '', description: '', active: true });
|
||||
fetchOrigins();
|
||||
} catch (err: any) {
|
||||
setOriginFormError(err.message || 'Failed to update origin');
|
||||
@@ -283,7 +284,8 @@ export function Users() {
|
||||
name: origin.name,
|
||||
origin_type: origin.origin_type,
|
||||
origin_value: origin.origin_value,
|
||||
description: origin.description || ''
|
||||
description: origin.description || '',
|
||||
active: origin.active
|
||||
});
|
||||
setOriginFormError(null);
|
||||
};
|
||||
@@ -291,7 +293,7 @@ export function Users() {
|
||||
const closeOriginModal = () => {
|
||||
setShowOriginModal(false);
|
||||
setEditingOrigin(null);
|
||||
setOriginFormData({ name: '', origin_type: 'domain', origin_value: '', description: '' });
|
||||
setOriginFormData({ name: '', origin_type: 'domain', origin_value: '', description: '', active: true });
|
||||
setOriginFormError(null);
|
||||
};
|
||||
|
||||
@@ -819,6 +821,29 @@ export function Users() {
|
||||
placeholder="Brief description of this origin"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Status toggle - only show when editing */}
|
||||
{editingOrigin && (
|
||||
<div className="flex items-center justify-between">
|
||||
<label className="text-sm font-medium text-gray-700">Status</label>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOriginFormData({ ...originFormData, active: !originFormData.active })}
|
||||
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors ${
|
||||
originFormData.active ? 'bg-emerald-600' : 'bg-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span
|
||||
className={`inline-block h-4 w-4 transform rounded-full bg-white transition-transform ${
|
||||
originFormData.active ? 'translate-x-6' : 'translate-x-1'
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
<span className={`ml-2 text-sm ${originFormData.active ? 'text-emerald-600' : 'text-gray-500'}`}>
|
||||
{originFormData.active ? 'Active' : 'Inactive'}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-end gap-3 px-6 py-4 border-t border-gray-200 bg-gray-50">
|
||||
|
||||
@@ -357,7 +357,7 @@ function ResourceBadge({ worker }: { worker: Worker }) {
|
||||
}
|
||||
|
||||
// Preflight Summary - shows IP, fingerprint, antidetect status, and qualification
|
||||
function PreflightSummary({ worker }: { worker: Worker }) {
|
||||
function PreflightSummary({ worker, poolOpen = true }: { worker: Worker; poolOpen?: boolean }) {
|
||||
const httpStatus = worker.preflight_http_status || 'pending';
|
||||
const httpIp = worker.http_ip;
|
||||
const fingerprint = worker.fingerprint_data;
|
||||
@@ -369,6 +369,8 @@ function PreflightSummary({ worker }: { worker: Worker }) {
|
||||
// Worker is ONLY qualified if http preflight passed AND has geo assigned
|
||||
const hasGeo = Boolean(geoState);
|
||||
const isQualified = (worker.is_qualified || httpStatus === 'passed') && hasGeo;
|
||||
// Worker is actively working if they have tasks
|
||||
const isActive = worker.current_task_id !== null || (worker.active_task_count || 0) > 0;
|
||||
|
||||
// Build detailed tooltip
|
||||
const tooltipLines: string[] = [];
|
||||
@@ -388,6 +390,65 @@ function PreflightSummary({ worker }: { worker: Worker }) {
|
||||
}
|
||||
if (httpError) tooltipLines.push(`Error: ${httpError}`);
|
||||
|
||||
// Get preflight step from metadata
|
||||
const preflightStep = worker.metadata?.current_step;
|
||||
const preflightDetail = worker.metadata?.current_step_detail;
|
||||
|
||||
// Preflight step labels for display
|
||||
const PREFLIGHT_STEP_LABELS: Record<string, { label: string; color: string }> = {
|
||||
init: { label: 'Initializing', color: 'text-blue-600' },
|
||||
claiming: { label: 'Claiming Tasks', color: 'text-blue-600' },
|
||||
proxy: { label: 'Setting Proxy', color: 'text-purple-600' },
|
||||
preflight: { label: 'Preflight', color: 'text-amber-600' },
|
||||
preflight_ip: { label: 'Detecting IP', color: 'text-amber-600' },
|
||||
antidetect: { label: 'Antidetect', color: 'text-emerald-600' },
|
||||
ready: { label: 'Ready', color: 'text-emerald-600' },
|
||||
waiting: { label: 'Waiting', color: 'text-gray-500' },
|
||||
failed: { label: 'Failed', color: 'text-red-600' },
|
||||
error: { label: 'Error', color: 'text-red-600' },
|
||||
};
|
||||
|
||||
// When pool is closed and worker is not active, show "Waiting" status
|
||||
if (!poolOpen && !isActive && !isQualified) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1" title="Pool is closed - waiting for tasks">
|
||||
<div className="inline-flex items-center gap-1.5 px-2 py-1 rounded-lg bg-gray-100 border border-gray-300">
|
||||
<Shield className="w-4 h-4 text-gray-500" />
|
||||
<span className="text-xs font-bold text-gray-600">WAITING</span>
|
||||
</div>
|
||||
<div className="text-xs text-gray-500">
|
||||
Pool closed
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Show preflight steps when worker is actively preflighting (not idle, not qualified yet)
|
||||
if (preflightStep && !isQualified && preflightStep !== 'idle' && preflightStep !== 'waiting') {
|
||||
const stepInfo = PREFLIGHT_STEP_LABELS[preflightStep] || { label: preflightStep, color: 'text-gray-600' };
|
||||
const isError = preflightStep === 'failed' || preflightStep === 'error';
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1" title={tooltipLines.join('\n')}>
|
||||
<div className={`inline-flex items-center gap-1.5 px-2 py-1 rounded-lg ${
|
||||
isError ? 'bg-red-100 border border-red-300' : 'bg-blue-100 border border-blue-300'
|
||||
}`}>
|
||||
{isError ? (
|
||||
<ShieldX className="w-4 h-4 text-red-600" />
|
||||
) : (
|
||||
<Shield className="w-4 h-4 text-blue-600 animate-pulse" />
|
||||
)}
|
||||
<span className={`text-xs font-bold ${stepInfo.color}`}>{stepInfo.label.toUpperCase()}</span>
|
||||
</div>
|
||||
{preflightDetail && (
|
||||
<div className={`text-xs ${isError ? 'text-red-600' : 'text-gray-600'} max-w-[150px] truncate`} title={preflightDetail}>
|
||||
{preflightDetail}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Qualification styling - gold shield + geo for qualified (requires geo)
|
||||
if (isQualified) {
|
||||
return (
|
||||
@@ -547,7 +608,7 @@ function TaskCountBadge({ worker, tasks }: { worker: Worker; tasks: Task[] }) {
|
||||
if (activeCount === 0 && activeTasks.length === 0) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
<span className="text-gray-400 text-sm">Idle</span>
|
||||
<span className="text-gray-400 text-sm">Waiting</span>
|
||||
{/* Show empty slots */}
|
||||
<div className="flex gap-1">
|
||||
{Array.from({ length: maxCount }).map((_, i) => (
|
||||
@@ -947,6 +1008,7 @@ export function WorkersDashboard() {
|
||||
const [recentTasks, setRecentTasks] = useState<Task[]>([]); // Recent completed/failed
|
||||
const [pendingTaskCount, setPendingTaskCount] = useState<number>(0);
|
||||
const [taskCounts, setTaskCounts] = useState<{ completed: number; failed: number }>({ completed: 0, failed: 0 });
|
||||
const [poolOpen, setPoolOpen] = useState(true);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
@@ -959,17 +1021,19 @@ export function WorkersDashboard() {
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
try {
|
||||
// Fetch workers from registry, running tasks, recent tasks, and task counts
|
||||
const [workersRes, tasksRes, recentCompletedRes, recentFailedRes, countsRes] = await Promise.all([
|
||||
// Fetch workers from registry, running tasks, recent tasks, task counts, and pool status
|
||||
const [workersRes, tasksRes, recentCompletedRes, recentFailedRes, countsRes, poolRes] = await Promise.all([
|
||||
api.get('/api/worker-registry/workers'),
|
||||
api.get('/api/tasks?status=running&limit=100'),
|
||||
api.get('/api/tasks?status=completed&limit=10'),
|
||||
api.get('/api/tasks?status=failed&limit=5'),
|
||||
api.get('/api/tasks/counts'),
|
||||
api.getTaskPoolStatus(),
|
||||
]);
|
||||
|
||||
setWorkers(workersRes.data.workers || []);
|
||||
setTasks(tasksRes.data.tasks || []);
|
||||
setPoolOpen(poolRes.open ?? !poolRes.paused);
|
||||
|
||||
// Combine recent completed and failed, sort by completion time
|
||||
const recentCompleted = recentCompletedRes.data.tasks || [];
|
||||
@@ -1132,7 +1196,7 @@ export function WorkersDashboard() {
|
||||
<Cpu className="w-5 h-5 text-gray-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">Idle</p>
|
||||
<p className="text-sm text-gray-500">Waiting</p>
|
||||
<p className="text-xl font-semibold">{idleWorkers.length}</p>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1310,7 +1374,7 @@ export function WorkersDashboard() {
|
||||
) : isBusy ? (
|
||||
<span className="text-blue-600">Working on task #{worker.current_task_id}</span>
|
||||
) : (
|
||||
<span className="text-emerald-600">Idle - ready for tasks</span>
|
||||
<span className="text-gray-500">Waiting</span>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
@@ -1481,7 +1545,7 @@ export function WorkersDashboard() {
|
||||
<HealthBadge status={worker.status} healthStatus={worker.health_status} />
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<PreflightSummary worker={worker} />
|
||||
<PreflightSummary worker={worker} poolOpen={poolOpen} />
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<ResourceBadge worker={worker} />
|
||||
|
||||
Reference in New Issue
Block a user