The job_run_logs table tracks scheduled job orchestration, not individual worker jobs. Worker info (worker_id, worker_hostname) belongs on dispensary_crawl_jobs, not job_run_logs. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
297 lines
10 KiB
JavaScript
297 lines
10 KiB
JavaScript
"use strict";
|
|
/**
|
|
* Local Image Storage Utility
|
|
*
|
|
* Downloads and stores product images to local filesystem.
|
|
* Replaces MinIO-based storage with simple local file storage.
|
|
*
|
|
* Directory structure:
|
|
* /images/products/<dispensary_id>/<product_id>.webp
|
|
* /images/products/<dispensary_id>/<product_id>-thumb.webp
|
|
* /images/products/<dispensary_id>/<product_id>-medium.webp
|
|
* /images/brands/<brand_slug>.webp
|
|
*/
|
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
}
|
|
Object.defineProperty(o, k2, desc);
|
|
}) : (function(o, m, k, k2) {
|
|
if (k2 === undefined) k2 = k;
|
|
o[k2] = m[k];
|
|
}));
|
|
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
}) : function(o, v) {
|
|
o["default"] = v;
|
|
});
|
|
var __importStar = (this && this.__importStar) || (function () {
|
|
var ownKeys = function(o) {
|
|
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
var ar = [];
|
|
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
return ar;
|
|
};
|
|
return ownKeys(o);
|
|
};
|
|
return function (mod) {
|
|
if (mod && mod.__esModule) return mod;
|
|
var result = {};
|
|
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
__setModuleDefault(result, mod);
|
|
return result;
|
|
};
|
|
})();
|
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
};
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.downloadProductImage = downloadProductImage;
|
|
exports.downloadBrandLogo = downloadBrandLogo;
|
|
exports.imageExists = imageExists;
|
|
exports.deleteProductImages = deleteProductImages;
|
|
exports.initializeImageStorage = initializeImageStorage;
|
|
exports.getStorageStats = getStorageStats;
|
|
const axios_1 = __importDefault(require("axios"));
|
|
const sharp_1 = __importDefault(require("sharp"));
|
|
const fs = __importStar(require("fs/promises"));
|
|
const path = __importStar(require("path"));
|
|
const crypto_1 = require("crypto");
|
|
// Base path for image storage - configurable via env
|
|
const IMAGES_BASE_PATH = process.env.IMAGES_PATH || '/app/public/images';
|
|
// Public URL base for serving images
|
|
const IMAGES_PUBLIC_URL = process.env.IMAGES_PUBLIC_URL || '/images';
|
|
/**
|
|
* Ensure a directory exists
|
|
*/
|
|
async function ensureDir(dirPath) {
|
|
try {
|
|
await fs.mkdir(dirPath, { recursive: true });
|
|
}
|
|
catch (error) {
|
|
if (error.code !== 'EEXIST')
|
|
throw error;
|
|
}
|
|
}
|
|
/**
|
|
* Generate a short hash from a URL for deduplication
|
|
*/
|
|
function hashUrl(url) {
|
|
return (0, crypto_1.createHash)('md5').update(url).digest('hex').substring(0, 8);
|
|
}
|
|
/**
|
|
* Download an image from a URL and return the buffer
|
|
*/
|
|
async function downloadImage(imageUrl) {
|
|
const response = await axios_1.default.get(imageUrl, {
|
|
responseType: 'arraybuffer',
|
|
timeout: 30000,
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
|
|
},
|
|
});
|
|
return Buffer.from(response.data);
|
|
}
|
|
/**
|
|
* Process and save image in multiple sizes
|
|
* Returns the file paths relative to IMAGES_BASE_PATH
|
|
*/
|
|
async function processAndSaveImage(buffer, outputDir, baseFilename) {
|
|
await ensureDir(outputDir);
|
|
const fullPath = path.join(outputDir, `${baseFilename}.webp`);
|
|
const mediumPath = path.join(outputDir, `${baseFilename}-medium.webp`);
|
|
const thumbPath = path.join(outputDir, `${baseFilename}-thumb.webp`);
|
|
// Process images in parallel
|
|
const [fullBuffer, mediumBuffer, thumbBuffer] = await Promise.all([
|
|
// Full: max 1200x1200, high quality
|
|
(0, sharp_1.default)(buffer)
|
|
.resize(1200, 1200, { fit: 'inside', withoutEnlargement: true })
|
|
.webp({ quality: 85 })
|
|
.toBuffer(),
|
|
// Medium: 600x600
|
|
(0, sharp_1.default)(buffer)
|
|
.resize(600, 600, { fit: 'inside', withoutEnlargement: true })
|
|
.webp({ quality: 80 })
|
|
.toBuffer(),
|
|
// Thumb: 200x200
|
|
(0, sharp_1.default)(buffer)
|
|
.resize(200, 200, { fit: 'inside', withoutEnlargement: true })
|
|
.webp({ quality: 75 })
|
|
.toBuffer(),
|
|
]);
|
|
// Save all sizes
|
|
await Promise.all([
|
|
fs.writeFile(fullPath, fullBuffer),
|
|
fs.writeFile(mediumPath, mediumBuffer),
|
|
fs.writeFile(thumbPath, thumbBuffer),
|
|
]);
|
|
const totalBytes = fullBuffer.length + mediumBuffer.length + thumbBuffer.length;
|
|
return {
|
|
full: fullPath,
|
|
medium: mediumPath,
|
|
thumb: thumbPath,
|
|
totalBytes,
|
|
};
|
|
}
|
|
/**
|
|
* Convert a file path to a public URL
|
|
*/
|
|
function pathToUrl(filePath) {
|
|
const relativePath = filePath.replace(IMAGES_BASE_PATH, '');
|
|
return `${IMAGES_PUBLIC_URL}${relativePath}`;
|
|
}
|
|
/**
|
|
* Download and store a product image locally
|
|
*
|
|
* @param imageUrl - The third-party image URL to download
|
|
* @param dispensaryId - The dispensary ID (for directory organization)
|
|
* @param productId - The product ID or external ID (for filename)
|
|
* @returns Download result with local URLs
|
|
*/
|
|
async function downloadProductImage(imageUrl, dispensaryId, productId) {
|
|
try {
|
|
if (!imageUrl) {
|
|
return { success: false, error: 'No image URL provided' };
|
|
}
|
|
// Download the image
|
|
const buffer = await downloadImage(imageUrl);
|
|
// Organize by dispensary ID
|
|
const outputDir = path.join(IMAGES_BASE_PATH, 'products', String(dispensaryId));
|
|
// Use product ID + URL hash for uniqueness
|
|
const urlHash = hashUrl(imageUrl);
|
|
const baseFilename = `${productId}-${urlHash}`;
|
|
// Process and save
|
|
const result = await processAndSaveImage(buffer, outputDir, baseFilename);
|
|
return {
|
|
success: true,
|
|
urls: {
|
|
full: pathToUrl(result.full),
|
|
medium: pathToUrl(result.medium),
|
|
thumb: pathToUrl(result.thumb),
|
|
},
|
|
bytesDownloaded: result.totalBytes,
|
|
};
|
|
}
|
|
catch (error) {
|
|
return {
|
|
success: false,
|
|
error: error.message || 'Failed to download image',
|
|
};
|
|
}
|
|
}
|
|
/**
|
|
* Download and store a brand logo locally
|
|
*
|
|
* @param logoUrl - The brand logo URL
|
|
* @param brandId - The brand ID or slug
|
|
* @returns Download result with local URL
|
|
*/
|
|
async function downloadBrandLogo(logoUrl, brandId) {
|
|
try {
|
|
if (!logoUrl) {
|
|
return { success: false, error: 'No logo URL provided' };
|
|
}
|
|
// Download the image
|
|
const buffer = await downloadImage(logoUrl);
|
|
// Brand logos go in /images/brands/
|
|
const outputDir = path.join(IMAGES_BASE_PATH, 'brands');
|
|
// Sanitize brand ID for filename
|
|
const safeBrandId = brandId.replace(/[^a-zA-Z0-9-_]/g, '_');
|
|
const urlHash = hashUrl(logoUrl);
|
|
const baseFilename = `${safeBrandId}-${urlHash}`;
|
|
// Process and save (single size for logos)
|
|
await ensureDir(outputDir);
|
|
const logoPath = path.join(outputDir, `${baseFilename}.webp`);
|
|
const logoBuffer = await (0, sharp_1.default)(buffer)
|
|
.resize(400, 400, { fit: 'inside', withoutEnlargement: true })
|
|
.webp({ quality: 85 })
|
|
.toBuffer();
|
|
await fs.writeFile(logoPath, logoBuffer);
|
|
return {
|
|
success: true,
|
|
urls: {
|
|
full: pathToUrl(logoPath),
|
|
medium: pathToUrl(logoPath),
|
|
thumb: pathToUrl(logoPath),
|
|
},
|
|
bytesDownloaded: logoBuffer.length,
|
|
};
|
|
}
|
|
catch (error) {
|
|
return {
|
|
success: false,
|
|
error: error.message || 'Failed to download brand logo',
|
|
};
|
|
}
|
|
}
|
|
/**
|
|
* Check if a local image already exists
|
|
*/
|
|
async function imageExists(dispensaryId, productId, imageUrl) {
|
|
const urlHash = hashUrl(imageUrl);
|
|
const imagePath = path.join(IMAGES_BASE_PATH, 'products', String(dispensaryId), `${productId}-${urlHash}.webp`);
|
|
try {
|
|
await fs.access(imagePath);
|
|
return true;
|
|
}
|
|
catch {
|
|
return false;
|
|
}
|
|
}
|
|
/**
|
|
* Delete a product's local images
|
|
*/
|
|
async function deleteProductImages(dispensaryId, productId, imageUrl) {
|
|
const productDir = path.join(IMAGES_BASE_PATH, 'products', String(dispensaryId));
|
|
const prefix = imageUrl
|
|
? `${productId}-${hashUrl(imageUrl)}`
|
|
: String(productId);
|
|
try {
|
|
const files = await fs.readdir(productDir);
|
|
const toDelete = files.filter(f => f.startsWith(prefix));
|
|
await Promise.all(toDelete.map(f => fs.unlink(path.join(productDir, f))));
|
|
}
|
|
catch {
|
|
// Directory might not exist, that's fine
|
|
}
|
|
}
|
|
/**
|
|
* Initialize the image storage directories
|
|
*/
|
|
async function initializeImageStorage() {
|
|
await ensureDir(path.join(IMAGES_BASE_PATH, 'products'));
|
|
await ensureDir(path.join(IMAGES_BASE_PATH, 'brands'));
|
|
console.log(`✅ Image storage initialized at ${IMAGES_BASE_PATH}`);
|
|
}
|
|
/**
|
|
* Get storage stats
|
|
*/
|
|
async function getStorageStats() {
|
|
const productsDir = path.join(IMAGES_BASE_PATH, 'products');
|
|
const brandsDir = path.join(IMAGES_BASE_PATH, 'brands');
|
|
let productCount = 0;
|
|
let brandCount = 0;
|
|
try {
|
|
const productDirs = await fs.readdir(productsDir);
|
|
for (const dir of productDirs) {
|
|
const files = await fs.readdir(path.join(productsDir, dir));
|
|
productCount += files.filter(f => f.endsWith('.webp') && !f.includes('-')).length;
|
|
}
|
|
}
|
|
catch { /* ignore */ }
|
|
try {
|
|
const brandFiles = await fs.readdir(brandsDir);
|
|
brandCount = brandFiles.filter(f => f.endsWith('.webp')).length;
|
|
}
|
|
catch { /* ignore */ }
|
|
return {
|
|
productsDir,
|
|
brandsDir,
|
|
productCount,
|
|
brandCount,
|
|
};
|
|
}
|