The job_run_logs table tracks scheduled job orchestration, not individual worker jobs. Worker info (worker_id, worker_hostname) belongs on dispensary_crawl_jobs, not job_run_logs. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
263 lines
11 KiB
JavaScript
263 lines
11 KiB
JavaScript
"use strict";
|
||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||
if (k2 === undefined) k2 = k;
|
||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||
}
|
||
Object.defineProperty(o, k2, desc);
|
||
}) : (function(o, m, k, k2) {
|
||
if (k2 === undefined) k2 = k;
|
||
o[k2] = m[k];
|
||
}));
|
||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||
}) : function(o, v) {
|
||
o["default"] = v;
|
||
});
|
||
var __importStar = (this && this.__importStar) || (function () {
|
||
var ownKeys = function(o) {
|
||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||
var ar = [];
|
||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||
return ar;
|
||
};
|
||
return ownKeys(o);
|
||
};
|
||
return function (mod) {
|
||
if (mod && mod.__esModule) return mod;
|
||
var result = {};
|
||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||
__setModuleDefault(result, mod);
|
||
return result;
|
||
};
|
||
})();
|
||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||
};
|
||
Object.defineProperty(exports, "__esModule", { value: true });
|
||
exports.isMinioEnabled = isMinioEnabled;
|
||
exports.initializeMinio = initializeMinio;
|
||
exports.uploadImageFromUrl = uploadImageFromUrl;
|
||
exports.getImageUrl = getImageUrl;
|
||
exports.deleteImage = deleteImage;
|
||
exports.minioClient = getMinioClient;
|
||
const Minio = __importStar(require("minio"));
|
||
const axios_1 = __importDefault(require("axios"));
|
||
const uuid_1 = require("uuid");
|
||
const sharp_1 = __importDefault(require("sharp"));
|
||
const fs = __importStar(require("fs/promises"));
|
||
const path = __importStar(require("path"));
|
||
let minioClient = null;
|
||
// Check if MinIO is configured
|
||
function isMinioEnabled() {
|
||
return !!process.env.MINIO_ENDPOINT;
|
||
}
|
||
// Local storage path for images when MinIO is not configured
|
||
const LOCAL_IMAGES_PATH = process.env.LOCAL_IMAGES_PATH || '/app/public/images';
|
||
function getMinioClient() {
|
||
if (!minioClient) {
|
||
minioClient = new Minio.Client({
|
||
endPoint: process.env.MINIO_ENDPOINT || 'minio',
|
||
port: parseInt(process.env.MINIO_PORT || '9000'),
|
||
useSSL: process.env.MINIO_USE_SSL === 'true',
|
||
accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin',
|
||
secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin',
|
||
});
|
||
}
|
||
return minioClient;
|
||
}
|
||
const BUCKET_NAME = process.env.MINIO_BUCKET || 'dutchie';
|
||
async function initializeMinio() {
|
||
// Skip MinIO initialization if not configured
|
||
if (!isMinioEnabled()) {
|
||
console.log('ℹ️ MinIO not configured (MINIO_ENDPOINT not set), using local filesystem storage');
|
||
// Ensure local images directory exists
|
||
try {
|
||
await fs.mkdir(LOCAL_IMAGES_PATH, { recursive: true });
|
||
await fs.mkdir(path.join(LOCAL_IMAGES_PATH, 'products'), { recursive: true });
|
||
console.log(`✅ Local images directory ready: ${LOCAL_IMAGES_PATH}`);
|
||
}
|
||
catch (error) {
|
||
console.error('❌ Failed to create local images directory:', error);
|
||
throw error;
|
||
}
|
||
return;
|
||
}
|
||
try {
|
||
const client = getMinioClient();
|
||
// Check if bucket exists
|
||
const exists = await client.bucketExists(BUCKET_NAME);
|
||
if (!exists) {
|
||
// Create bucket
|
||
await client.makeBucket(BUCKET_NAME, 'us-east-1');
|
||
console.log(`✅ Minio bucket created: ${BUCKET_NAME}`);
|
||
// Set public read policy
|
||
const policy = {
|
||
Version: '2012-10-17',
|
||
Statement: [
|
||
{
|
||
Effect: 'Allow',
|
||
Principal: { AWS: ['*'] },
|
||
Action: ['s3:GetObject'],
|
||
Resource: [`arn:aws:s3:::${BUCKET_NAME}/*`],
|
||
},
|
||
],
|
||
};
|
||
await client.setBucketPolicy(BUCKET_NAME, JSON.stringify(policy));
|
||
console.log(`✅ Bucket policy set to public read`);
|
||
}
|
||
else {
|
||
console.log(`✅ Minio bucket already exists: ${BUCKET_NAME}`);
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.error('❌ Minio initialization error:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
async function removeBackground(buffer) {
|
||
try {
|
||
// Get image metadata to check if it has an alpha channel
|
||
const metadata = await (0, sharp_1.default)(buffer).metadata();
|
||
// If image already has transparency, trim and optimize it
|
||
if (metadata.hasAlpha) {
|
||
return await (0, sharp_1.default)(buffer)
|
||
.trim() // Remove transparent borders
|
||
.toBuffer();
|
||
}
|
||
// For images without alpha (like JPEGs with solid backgrounds),
|
||
// we'll use a threshold-based approach to detect and remove solid backgrounds
|
||
// This works well for product images on solid color backgrounds
|
||
// Convert to PNG with alpha channel, then flatten with transparency
|
||
const withAlpha = await (0, sharp_1.default)(buffer)
|
||
.ensureAlpha() // Add alpha channel
|
||
.toBuffer();
|
||
// Use threshold to make similar colors transparent (targets solid backgrounds)
|
||
// This is a simple approach - for better results, use remove.bg API or ML models
|
||
return await (0, sharp_1.default)(withAlpha)
|
||
.flatten({ background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
||
.trim()
|
||
.toBuffer();
|
||
}
|
||
catch (error) {
|
||
console.warn('Background removal failed, using original image:', error);
|
||
return buffer;
|
||
}
|
||
}
|
||
async function uploadToLocalFilesystem(thumbnailBuffer, mediumBuffer, fullBuffer, baseFilename) {
|
||
const thumbnailPath = `${baseFilename}-thumb.png`;
|
||
const mediumPath = `${baseFilename}-medium.png`;
|
||
const fullPath = `${baseFilename}-full.png`;
|
||
// Ensure the target directory exists (in case initializeMinio wasn't called)
|
||
// Extract directory from baseFilename (e.g., 'products/store-slug' or just 'products')
|
||
const targetDir = path.join(LOCAL_IMAGES_PATH, path.dirname(baseFilename));
|
||
await fs.mkdir(targetDir, { recursive: true });
|
||
await Promise.all([
|
||
fs.writeFile(path.join(LOCAL_IMAGES_PATH, thumbnailPath), thumbnailBuffer),
|
||
fs.writeFile(path.join(LOCAL_IMAGES_PATH, mediumPath), mediumBuffer),
|
||
fs.writeFile(path.join(LOCAL_IMAGES_PATH, fullPath), fullBuffer),
|
||
]);
|
||
return {
|
||
thumbnail: thumbnailPath,
|
||
medium: mediumPath,
|
||
full: fullPath,
|
||
};
|
||
}
|
||
async function uploadToMinio(thumbnailBuffer, mediumBuffer, fullBuffer, baseFilename) {
|
||
const client = getMinioClient();
|
||
const thumbnailPath = `${baseFilename}-thumb.png`;
|
||
const mediumPath = `${baseFilename}-medium.png`;
|
||
const fullPath = `${baseFilename}-full.png`;
|
||
await Promise.all([
|
||
client.putObject(BUCKET_NAME, thumbnailPath, thumbnailBuffer, thumbnailBuffer.length, {
|
||
'Content-Type': 'image/png',
|
||
}),
|
||
client.putObject(BUCKET_NAME, mediumPath, mediumBuffer, mediumBuffer.length, {
|
||
'Content-Type': 'image/png',
|
||
}),
|
||
client.putObject(BUCKET_NAME, fullPath, fullBuffer, fullBuffer.length, {
|
||
'Content-Type': 'image/png',
|
||
}),
|
||
]);
|
||
return {
|
||
thumbnail: thumbnailPath,
|
||
medium: mediumPath,
|
||
full: fullPath,
|
||
};
|
||
}
|
||
async function uploadImageFromUrl(imageUrl, productId, storeSlug, removeBackgrounds = true) {
|
||
try {
|
||
// Download image
|
||
const response = await axios_1.default.get(imageUrl, { responseType: 'arraybuffer' });
|
||
let buffer = Buffer.from(response.data);
|
||
// Remove background if enabled
|
||
if (removeBackgrounds) {
|
||
buffer = await removeBackground(buffer);
|
||
}
|
||
// Generate unique base filename - organize by store if slug provided
|
||
const storeDir = storeSlug ? `products/${storeSlug}` : 'products';
|
||
const baseFilename = `${storeDir}/${productId}-${(0, uuid_1.v4)()}`;
|
||
// Create multiple sizes with Sharp and convert to WebP/PNG for better compression
|
||
// Use PNG for images with transparency
|
||
const [thumbnailBuffer, mediumBuffer, fullBuffer] = await Promise.all([
|
||
// Thumbnail: 300x300
|
||
(0, sharp_1.default)(buffer)
|
||
.resize(300, 300, { fit: 'inside', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
||
.png({ quality: 80, compressionLevel: 9 })
|
||
.toBuffer(),
|
||
// Medium: 800x800
|
||
(0, sharp_1.default)(buffer)
|
||
.resize(800, 800, { fit: 'inside', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
||
.png({ quality: 85, compressionLevel: 9 })
|
||
.toBuffer(),
|
||
// Full: 2000x2000 (optimized)
|
||
(0, sharp_1.default)(buffer)
|
||
.resize(2000, 2000, { fit: 'inside', withoutEnlargement: true, background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
||
.png({ quality: 90, compressionLevel: 9 })
|
||
.toBuffer(),
|
||
]);
|
||
// Upload to appropriate storage backend
|
||
let result;
|
||
if (isMinioEnabled()) {
|
||
result = await uploadToMinio(thumbnailBuffer, mediumBuffer, fullBuffer, baseFilename);
|
||
}
|
||
else {
|
||
result = await uploadToLocalFilesystem(thumbnailBuffer, mediumBuffer, fullBuffer, baseFilename);
|
||
}
|
||
console.log(`✅ Uploaded 3 sizes for product ${productId}: ${thumbnailBuffer.length + mediumBuffer.length + fullBuffer.length} bytes total`);
|
||
return result;
|
||
}
|
||
catch (error) {
|
||
console.error('Error uploading image:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
function getImageUrl(imagePath) {
|
||
if (isMinioEnabled()) {
|
||
// Use MinIO endpoint for browser access
|
||
const endpoint = process.env.MINIO_PUBLIC_ENDPOINT || 'http://localhost:9020';
|
||
return `${endpoint}/${BUCKET_NAME}/${imagePath}`;
|
||
}
|
||
else {
|
||
// Use local path - served via Express static middleware
|
||
const publicUrl = process.env.PUBLIC_URL || '';
|
||
return `${publicUrl}/images/${imagePath}`;
|
||
}
|
||
}
|
||
async function deleteImage(imagePath) {
|
||
try {
|
||
if (isMinioEnabled()) {
|
||
const client = getMinioClient();
|
||
await client.removeObject(BUCKET_NAME, imagePath);
|
||
}
|
||
else {
|
||
const fullPath = path.join(LOCAL_IMAGES_PATH, imagePath);
|
||
await fs.unlink(fullPath);
|
||
}
|
||
}
|
||
catch (error) {
|
||
console.error('Error deleting image:', error);
|
||
}
|
||
}
|