Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Use MinIO for permanent product image storage

Browse Source 
- Rewrite image-storage.ts to use MinIO instead of ephemeral local filesystem
- Images downloaded ONCE from Dutchie CDN, stored permanently in MinIO
- Check MinIO before downloading (skipIfExists) to avoid re-downloads
- Convert images to webp before storage
- Storage path: images/products/<state>/<store>/<brand>/<product>/image-<hash>.webp
- Public URL: https://cdn.cannabrands.app/cannaiq/images/...

This fixes the 2.4GB bandwidth issue from repeatedly downloading images
that were lost when K8s pods restarted.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-13 03:24:22 -07:00
parent 5c08135007
commit 268429b86c
1 changed files with 175 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

421
backend/src/utils/image-storage.ts
View File

@@ -1,45 +1,54 @@
/**
* Local Image Storage Utility
* Image Storage Utility
*
* Downloads and stores product images to local filesystem with proper hierarchy.
* Downloads and stores product images to MinIO/S3 with proper hierarchy.
* Images are downloaded ONCE on first discovery/refresh, then served from our CDN.
*
* Directory structure:
* /images/products/<state>/<store_slug>/<brand_slug>/<product_id>/image.webp
* /images/products/<state>/<store_slug>/<brand_slug>/<product_id>/image-medium.webp
* /images/products/<state>/<store_slug>/<brand_slug>/<product_id>/image-thumb.webp
* /images/brands/<brand_slug>/logo.webp
* Storage structure in MinIO:
* cannaiq/images/products/<state>/<store_slug>/<brand_slug>/<product_id>/image-<hash>.webp
* cannaiq/images/brands/<brand_slug>/logo.webp
*
* This structure allows:
* - Easy migration to MinIO/S3 (bucket per state)
* - Browsing by state/store/brand
* - Multiple images per product (future: gallery)
* Public URL format:
* https://cdn.cannabrands.app/cannaiq/images/products/az/store/brand/123/image-abc123.webp
*/
import axios from 'axios';
// @ts-ignore - sharp module typing quirk
const sharp = require('sharp');
import * as fs from 'fs/promises';
import * as path from 'path';
import { createHash } from 'crypto';
import * as Minio from 'minio';
// Base path for image storage - configurable via env
function getImagesBasePath(): string {
if (process.env.IMAGES_PATH) {
return process.env.IMAGES_PATH;
// MinIO configuration
const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || 'cdn.cannabrands.app';
const MINIO_PORT = parseInt(process.env.MINIO_PORT || '443');
const MINIO_USE_SSL = process.env.MINIO_USE_SSL !== 'false';
const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY;
const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY;
const MINIO_BUCKET = process.env.MINIO_BUCKET || 'cannaiq';
const MINIO_PUBLIC_URL = process.env.MINIO_PUBLIC_ENDPOINT || 'https://cdn.cannabrands.app';
// Check if MinIO is configured
const useMinIO = !!(MINIO_ENDPOINT && MINIO_ACCESS_KEY && MINIO_SECRET_KEY);
let minioClient: Minio.Client | null = null;
function getMinioClient(): Minio.Client | null {
if (!useMinIO) return null;
if (!minioClient) {
minioClient = new Minio.Client({
endPoint: MINIO_ENDPOINT,
port: MINIO_PORT,
useSSL: MINIO_USE_SSL,
accessKey: MINIO_ACCESS_KEY!,
secretKey: MINIO_SECRET_KEY!,
});
}
if (process.env.STORAGE_BASE_PATH) {
return path.join(process.env.STORAGE_BASE_PATH, 'images');
}
return './storage/images';
return minioClient;
}
const IMAGES_BASE_PATH = getImagesBasePath();
// Public URL base for serving images
const IMAGES_PUBLIC_URL = process.env.IMAGES_PUBLIC_URL || '/images';
export interface LocalImageSizes {
original: string; // URL path to original image
// Legacy compatibility - all point to original until we add image proxy
full: string;
medium: string;
thumb: string;
@@ -62,17 +71,6 @@ export interface ProductImageContext {
dispensaryId?: number; // For backwards compat
}
/**
* Ensure a directory exists
*/
async function ensureDir(dirPath: string): Promise<void> {
try {
await fs.mkdir(dirPath, { recursive: true });
} catch (error: any) {
if (error.code !== 'EEXIST') throw error;
}
}
/**
* Sanitize a string for use in file paths
*/
@@ -91,6 +89,26 @@ function hashUrl(url: string): string {
return createHash('md5').update(url).digest('hex').substring(0, 8);
}
/**
* Build the MinIO object key for a product image
* Structure: images/products/<state>/<store>/<brand>/<product>/image-<hash>.webp
*/
function buildImageKey(ctx: ProductImageContext, urlHash: string): string {
const state = slugify(ctx.stateCode || 'unknown');
const store = slugify(ctx.storeSlug || 'unknown');
const brand = slugify(ctx.brandSlug || 'unknown');
const product = slugify(ctx.productId || 'unknown');
return `images/products/${state}/${store}/${brand}/${product}/image-${urlHash}.webp`;
}
/**
* Convert MinIO object key to public URL
*/
function keyToUrl(key: string): string {
return `${MINIO_PUBLIC_URL}/${MINIO_BUCKET}/${key}`;
}
/**
* Download an image from a URL and return the buffer
*/
@@ -107,73 +125,43 @@ async function downloadImage(imageUrl: string): Promise<Buffer> {
}
/**
* Process and save original image (convert to webp for consistency)
*
* We store only the original - resizing will be done on-demand via
* an image proxy service (imgproxy, thumbor, or similar) in the future.
* Check if an image already exists in MinIO
*/
async function processAndSaveImage(
buffer: Buffer,
outputDir: string,
baseFilename: string
): Promise<{ original: string; totalBytes: number }> {
await ensureDir(outputDir);
async function imageExistsInMinIO(key: string): Promise<boolean> {
const client = getMinioClient();
if (!client) return false;
const originalPath = path.join(outputDir, `${baseFilename}.webp`);
// Convert to webp, preserve original dimensions, high quality
const originalBuffer = await sharp(buffer)
.webp({ quality: 90 })
.toBuffer();
await fs.writeFile(originalPath, originalBuffer);
return {
original: originalPath,
totalBytes: originalBuffer.length,
};
try {
await client.statObject(MINIO_BUCKET, key);
return true;
} catch (error: any) {
if (error.code === 'NotFound') return false;
// For other errors, assume it doesn't exist
return false;
}
}
/**
* Convert a file path to a public URL
* Upload image buffer to MinIO
*/
function pathToUrl(filePath: string): string {
// Find /products/ or /brands/ in the path and extract from there
const productsMatch = filePath.match(/(\/products\/.*)/);
const brandsMatch = filePath.match(/(\/brands\/.*)/);
if (productsMatch) {
return `${IMAGES_PUBLIC_URL}${productsMatch[1]}`;
}
if (brandsMatch) {
return `${IMAGES_PUBLIC_URL}${brandsMatch[1]}`;
async function uploadToMinIO(buffer: Buffer, key: string): Promise<void> {
const client = getMinioClient();
if (!client) {
throw new Error('MinIO not configured');
}
// Fallback: try to replace base path (works if paths match exactly)
const relativePath = filePath.replace(IMAGES_BASE_PATH, '');
return `${IMAGES_PUBLIC_URL}${relativePath}`;
await client.putObject(MINIO_BUCKET, key, buffer, buffer.length, {
'Content-Type': 'image/webp',
});
}
/**
* Build the directory path for a product image
* Structure: /images/products/<state>/<store>/<brand>/<product>/
*/
function buildProductImagePath(ctx: ProductImageContext): string {
const state = slugify(ctx.stateCode || 'unknown');
const store = slugify(ctx.storeSlug || 'unknown');
const brand = slugify(ctx.brandSlug || 'unknown');
const product = slugify(ctx.productId || 'unknown');
return path.join(IMAGES_BASE_PATH, 'products', state, store, brand, product);
}
/**
* Download and store a product image with proper hierarchy
* Download and store a product image
*
* @param imageUrl - The third-party image URL to download
* @param imageUrl - The source image URL (e.g., from Dutchie CDN)
* @param ctx - Product context (state, store, brand, product)
* @param options - Download options
* @returns Download result with local URLs
* @returns Download result with our CDN URLs
*/
export async function downloadProductImage(
imageUrl: string,
@@ -187,17 +175,18 @@ export async function downloadProductImage(
return { success: false, error: 'No image URL provided' };
}
const outputDir = buildProductImagePath(ctx);
const urlHash = hashUrl(imageUrl);
const baseFilename = `image-${urlHash}`;
if (!useMinIO) {
return { success: false, error: 'MinIO not configured - images require MinIO storage' };
}
// Check if image already exists
const urlHash = hashUrl(imageUrl);
const key = buildImageKey(ctx, urlHash);
// Check if image already exists in MinIO
if (skipIfExists) {
const existingPath = path.join(outputDir, `${baseFilename}.webp`);
try {
await fs.access(existingPath);
// Image exists, return existing URL
const url = pathToUrl(existingPath);
const exists = await imageExistsInMinIO(key);
if (exists) {
const url = keyToUrl(key);
return {
success: true,
skipped: true,
@@ -208,23 +197,27 @@ export async function downloadProductImage(
thumb: url,
},
localPaths: {
original: existingPath,
full: existingPath,
medium: existingPath,
thumb: existingPath,
original: key,
full: key,
medium: key,
thumb: key,
},
};
} catch {
// Image doesn't exist, continue to download
}
}
// Download the image
// Download the image from source (e.g., Dutchie CDN)
const buffer = await downloadImage(imageUrl);
// Process and save (original only)
const result = await processAndSaveImage(buffer, outputDir, baseFilename);
const url = pathToUrl(result.original);
// Convert to webp for consistency and compression
const webpBuffer = await sharp(buffer)
.webp({ quality: 90 })
.toBuffer();
// Upload to MinIO
await uploadToMinIO(webpBuffer, key);
const url = keyToUrl(key);
return {
success: true,
@@ -235,111 +228,86 @@ export async function downloadProductImage(
thumb: url,
},
localPaths: {
original: result.original,
full: result.original,
medium: result.original,
thumb: result.original,
original: key,
full: key,
medium: key,
thumb: key,
},
bytesDownloaded: result.totalBytes,
bytesDownloaded: webpBuffer.length,
};
} catch (error: any) {
return {
success: false,
error: error.message || 'Failed to download image',
error: error.message || 'Unknown error',
};
}
}
/**
* Legacy function - backwards compatible with old signature
* Maps to new hierarchy using dispensary_id as store identifier
*/
export async function downloadProductImageLegacy(
imageUrl: string,
dispensaryId: number,
productId: string | number
): Promise<DownloadResult> {
return downloadProductImage(imageUrl, {
stateCode: 'unknown',
storeSlug: `store-${dispensaryId}`,
brandSlug: 'unknown',
productId: String(productId),
dispensaryId,
});
}
/**
* Download and store a brand logo
*
* @param logoUrl - The brand logo URL
* @param brandSlug - The brand slug/ID
* @returns Download result with local URL
* Download a brand logo
*/
export async function downloadBrandLogo(
logoUrl: string,
imageUrl: string,
brandSlug: string,
options: { skipIfExists?: boolean } = {}
): Promise<DownloadResult> {
const { skipIfExists = true } = options;
try {
if (!logoUrl) {
return { success: false, error: 'No logo URL provided' };
if (!imageUrl) {
return { success: false, error: 'No image URL provided' };
}
const safeBrandSlug = slugify(brandSlug);
const outputDir = path.join(IMAGES_BASE_PATH, 'brands', safeBrandSlug);
const urlHash = hashUrl(logoUrl);
const baseFilename = `logo-${urlHash}`;
if (!useMinIO) {
return { success: false, error: 'MinIO not configured - images require MinIO storage' };
}
const slug = slugify(brandSlug);
const key = `images/brands/${slug}/logo.webp`;
// Check if logo already exists
if (skipIfExists) {
const existingPath = path.join(outputDir, `${baseFilename}.webp`);
try {
await fs.access(existingPath);
const exists = await imageExistsInMinIO(key);
if (exists) {
const url = keyToUrl(key);
return {
success: true,
skipped: true,
urls: {
original: pathToUrl(existingPath),
full: pathToUrl(existingPath),
medium: pathToUrl(existingPath),
thumb: pathToUrl(existingPath),
original: url,
full: url,
medium: url,
thumb: url,
},
};
} catch {
// Logo doesn't exist, continue
}
}
// Download the image
const buffer = await downloadImage(logoUrl);
// Brand logos in their own directory
await ensureDir(outputDir);
const logoPath = path.join(outputDir, `${baseFilename}.webp`);
const logoBuffer = await sharp(buffer)
.resize(400, 400, { fit: 'inside', withoutEnlargement: true })
.webp({ quality: 85 })
// Download and process
const buffer = await downloadImage(imageUrl);
const webpBuffer = await sharp(buffer)
.webp({ quality: 90 })
.toBuffer();
await fs.writeFile(logoPath, logoBuffer);
// Upload to MinIO
await uploadToMinIO(webpBuffer, key);
const url = keyToUrl(key);
return {
success: true,
urls: {
original: pathToUrl(logoPath),
full: pathToUrl(logoPath),
medium: pathToUrl(logoPath),
thumb: pathToUrl(logoPath),
original: url,
full: url,
medium: url,
thumb: url,
},
bytesDownloaded: logoBuffer.length,
bytesDownloaded: webpBuffer.length,
};
} catch (error: any) {
return {
success: false,
error: error.message || 'Failed to download brand logo',
error: error.message || 'Unknown error',
};
}
}
@@ -348,44 +316,14 @@ export async function downloadBrandLogo(
* Check if a product image already exists
*/
export async function productImageExists(
ctx: ProductImageContext,
imageUrl: string
imageUrl: string,
ctx: ProductImageContext
): Promise<boolean> {
const outputDir = buildProductImagePath(ctx);
if (!useMinIO) return false;
const urlHash = hashUrl(imageUrl);
const imagePath = path.join(outputDir, `image-${urlHash}.webp`);
try {
await fs.access(imagePath);
return true;
} catch {
return false;
}
}
/**
* Get the local image URL for a product (if exists)
*/
export async function getProductImageUrl(
ctx: ProductImageContext,
imageUrl: string
): Promise<LocalImageSizes | null> {
const outputDir = buildProductImagePath(ctx);
const urlHash = hashUrl(imageUrl);
const imagePath = path.join(outputDir, `image-${urlHash}.webp`);
try {
await fs.access(imagePath);
const url = pathToUrl(imagePath);
return {
original: url,
full: url,
medium: url,
thumb: url,
};
} catch {
return null;
}
const key = buildImageKey(ctx, urlHash);
return imageExistsInMinIO(key);
}
// Track whether image storage is available
@@ -396,23 +334,42 @@ export function isImageStorageReady(): boolean {
}
/**
* Initialize the image storage directories
* Initialize image storage
*/
export async function initializeImageStorage(): Promise<void> {
if (!useMinIO) {
console.warn('[ImageStorage] MinIO not configured (MINIO_ENDPOINT/ACCESS_KEY/SECRET_KEY required)');
console.warn('[ImageStorage] Image downloads will be disabled');
imageStorageReady = false;
return;
}
try {
await ensureDir(path.join(IMAGES_BASE_PATH, 'products'));
await ensureDir(path.join(IMAGES_BASE_PATH, 'brands'));
console.log(`[ImageStorage] Initialized at ${IMAGES_BASE_PATH}`);
const client = getMinioClient();
if (!client) {
throw new Error('Failed to create MinIO client');
}
// Verify bucket exists and is accessible
const exists = await client.bucketExists(MINIO_BUCKET);
if (!exists) {
console.warn(`[ImageStorage] Bucket ${MINIO_BUCKET} does not exist`);
imageStorageReady = false;
return;
}
console.log(`[ImageStorage] Initialized with MinIO: ${MINIO_ENDPOINT}/${MINIO_BUCKET}`);
console.log(`[ImageStorage] Public URL: ${MINIO_PUBLIC_URL}/${MINIO_BUCKET}/images/...`);
imageStorageReady = true;
} catch (error: any) {
console.warn(`[ImageStorage] WARNING: Could not initialize at ${IMAGES_BASE_PATH}: ${error.message}`);
console.warn(' Image features disabled. Server will continue without image downloads.');
console.warn(`[ImageStorage] WARNING: Could not initialize MinIO: ${error.message}`);
console.warn('[ImageStorage] Image downloads will be disabled');
imageStorageReady = false;
}
}
/**
* Get storage stats
* Get storage stats (placeholder - would need to list objects)
*/
export async function getStorageStats(): Promise<{
basePath: string;
@@ -420,38 +377,10 @@ export async function getStorageStats(): Promise<{
brandCount: number;
totalSizeBytes: number;
}> {
let productCount = 0;
let brandCount = 0;
let totalSizeBytes = 0;
async function countDir(dirPath: string): Promise<{ count: number; size: number }> {
let count = 0;
let size = 0;
try {
const entries = await fs.readdir(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
if (entry.isDirectory()) {
const sub = await countDir(fullPath);
count += sub.count;
size += sub.size;
} else if (entry.name.endsWith('.webp') && !entry.name.includes('-')) {
count++;
const stat = await fs.stat(fullPath);
size += stat.size;
}
}
} catch { /* ignore */ }
return { count, size };
}
const products = await countDir(path.join(IMAGES_BASE_PATH, 'products'));
const brands = await countDir(path.join(IMAGES_BASE_PATH, 'brands'));
return {
basePath: IMAGES_BASE_PATH,
productCount: products.count,
brandCount: brands.count,
totalSizeBytes: products.size + brands.size,
basePath: `${MINIO_ENDPOINT}/${MINIO_BUCKET}/images`,
productCount: 0, // Would need to list objects
brandCount: 0,
totalSizeBytes: 0,
};
}