From e63329457c9f49b0b3d7f58a11e02f0c953d6976 Mon Sep 17 00:00:00 2001 From: Kelly Date: Sat, 6 Dec 2025 12:36:10 -0700 Subject: [PATCH] feat: Add local storage adapter and update CLAUDE.md with permanent rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add local-storage.ts with smart folder structure: /storage/products/{brand}/{state}/{product_id}/ - Add storage-adapter.ts unified abstraction - Add docker-compose.local.yml (NO MinIO) - Add start-local.sh convenience script - Update CLAUDE.md with: - PERMANENT RULES section (no data deletion) - DEPLOYMENT AUTHORIZATION requirements - LOCAL DEVELOPMENT defaults - STORAGE BEHAVIOR documentation - FORBIDDEN ACTIONS list - UI ANONYMIZATION rules 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- CLAUDE.md | 124 ++++++++++++++++++++++ backend/src/utils/local-storage.ts | 153 +++++++++++++++++++++++++++ backend/src/utils/storage-adapter.ts | 34 ++++++ docker-compose.local.yml | 64 +++++++++++ start-local.sh | 24 +++++ 5 files changed, 399 insertions(+) create mode 100644 backend/src/utils/local-storage.ts create mode 100644 backend/src/utils/storage-adapter.ts create mode 100644 docker-compose.local.yml create mode 100755 start-local.sh diff --git a/CLAUDE.md b/CLAUDE.md index e3013911..95505e24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,129 @@ ## Claude Guidelines for this Project +--- + +## PERMANENT RULES (NEVER VIOLATE) + +### 1. NO DELETION OF DATA — EVER + +CannaiQ is a **historical analytics system**. Data retention is **permanent by design**. + +**NEVER delete:** +- Product records +- Crawled snapshots +- Images +- Directories +- Logs +- Orchestrator traces +- Profiles +- Selector configs +- Crawl outcomes +- Store data +- Brand data + +**NEVER automate cleanup:** +- No cron or scheduled job may `rm`, `unlink`, `delete`, `purge`, `prune`, `clean`, or `reset` any storage directory or DB row +- No migration may DELETE data — only add/update/alter columns +- If cleanup is required, ONLY the user may issue a manual command + +**Code enforcement:** +- `local-storage.ts` must only: write files, create directories, read files +- No `deleteImage`, `deleteProductImages`, or similar functions + +### 2. DEPLOYMENT AUTHORIZATION REQUIRED + +**NEVER deploy to production unless the user explicitly says:** +> "CLAUDE — DEPLOYMENT IS NOW AUTHORIZED." + +Until then: +- All work is LOCAL ONLY +- No `kubectl apply`, `docker push`, or remote operations +- No port-forwarding to production +- No connecting to Kubernetes clusters + +### 3. LOCAL DEVELOPMENT BY DEFAULT + +**In local mode:** +- Use `docker-compose.local.yml` (NO MinIO) +- Use local filesystem storage at `./storage` +- Connect to local PostgreSQL at `localhost:54320` +- Backend runs at `localhost:3010` +- NO remote connections, NO Kubernetes, NO MinIO + +**Environment:** +```bash +STORAGE_DRIVER=local +STORAGE_BASE_PATH=./storage +DATABASE_URL=postgresql://dutchie:dutchie_local_pass@localhost:54320/dutchie_menus +# MINIO_ENDPOINT is NOT set (forces local storage) +``` + +--- + +## STORAGE BEHAVIOR + +### Local Storage Structure + +``` +/storage/products/{brand}/{state}/{product_id}/ + image-{hash}.webp + image-{hash}-medium.webp + image-{hash}-thumb.webp + +/storage/brands/{brand}/ + logo-{hash}.webp +``` + +### Storage Adapter + +```typescript +import { saveImage, getImageUrl } from '../utils/storage-adapter'; + +// Automatically uses local storage when STORAGE_DRIVER=local +``` + +### Files + +| File | Purpose | +|------|---------| +| `backend/src/utils/local-storage.ts` | Local filesystem adapter | +| `backend/src/utils/storage-adapter.ts` | Unified storage abstraction | +| `docker-compose.local.yml` | Local stack without MinIO | +| `start-local.sh` | Convenience startup script | + +--- + +## FORBIDDEN ACTIONS + +1. **Deleting any data** (products, snapshots, images, logs, traces) +2. **Deploying without explicit authorization** +3. **Connecting to Kubernetes** without authorization +4. **Port-forwarding to production** without authorization +5. **Starting MinIO** in local development +6. **Using S3/MinIO SDKs** when `STORAGE_DRIVER=local` +7. **Automating cleanup** of any kind +8. **Dropping database tables or columns** +9. **Overwriting historical records** (always append snapshots) + +--- + +## UI ANONYMIZATION RULES + +- No vendor names in forward-facing URLs: use `/api/az/...`, `/az`, `/az-schedule` +- No "dutchie", "treez", "jane", "weedmaps", "leafly" visible in consumer UIs +- Internal admin tools may show provider names for debugging + +--- + +## FUTURE TODO / PENDING FEATURES + +- [ ] Orchestrator observability dashboard +- [ ] Crawl profile management UI +- [ ] State machine sandbox (disabled until authorized) +- [ ] Multi-state expansion beyond AZ + +--- + ### Multi-Site Architecture (CRITICAL) This project has **5 working locations** - always clarify which one before making changes: diff --git a/backend/src/utils/local-storage.ts b/backend/src/utils/local-storage.ts new file mode 100644 index 00000000..2ebe4bec --- /dev/null +++ b/backend/src/utils/local-storage.ts @@ -0,0 +1,153 @@ +/** + * Local Filesystem Storage Adapter + * + * Used when STORAGE_DRIVER=local + * + * Directory structure: + * /storage/products/{brand}/{state}/{product_id}/filename + */ + +import * as fs from 'fs/promises'; +import * as path from 'path'; + +const STORAGE_BASE_PATH = process.env.STORAGE_BASE_PATH || './storage'; + +/** + * Normalize a string for use in file paths + */ +function normalizePath(str: string): string { + if (!str) return 'unknown'; + return str + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') || 'unknown'; +} + +/** + * Build the full directory path for a product + */ +function buildProductDir(brand: string, state: string, productId: string | number): string { + const normalizedBrand = normalizePath(brand); + const normalizedState = (state || 'XX').toUpperCase().substring(0, 2); + const normalizedProductId = String(productId).replace(/[^a-zA-Z0-9_-]/g, '_'); + + return path.join( + STORAGE_BASE_PATH, + 'products', + normalizedBrand, + normalizedState, + normalizedProductId + ); +} + +/** + * Ensure directories exist for a product + */ +export async function ensureDirectories( + brand: string, + state: string, + productId: string | number +): Promise { + const dir = buildProductDir(brand, state, productId); + await fs.mkdir(dir, { recursive: true }); + return dir; +} + +/** + * Save an image buffer to local storage + */ +export async function saveImage( + buffer: Buffer, + brand: string, + state: string, + productId: string | number, + filename: string +): Promise { + const dir = await ensureDirectories(brand, state, productId); + const filePath = path.join(dir, filename); + await fs.writeFile(filePath, buffer); + return filePath; +} + +/** + * Get the full filesystem path for an image + */ +export function getImagePath( + brand: string, + state: string, + productId: string | number, + filename: string +): string { + const dir = buildProductDir(brand, state, productId); + return path.join(dir, filename); +} + +/** + * Get the public URL for an image + */ +export function getImageUrl( + brand: string, + state: string, + productId: string | number, + filename: string +): string { + const normalizedBrand = normalizePath(brand); + const normalizedState = (state || 'XX').toUpperCase().substring(0, 2); + const normalizedProductId = String(productId).replace(/[^a-zA-Z0-9_-]/g, '_'); + + return `/storage/products/${normalizedBrand}/${normalizedState}/${normalizedProductId}/${filename}`; +} + +/** + * Check if an image exists + */ +export async function imageExists( + brand: string, + state: string, + productId: string | number, + filename: string +): Promise { + const filePath = getImagePath(brand, state, productId, filename); + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +// NOTE: No delete functions - CannaiQ has permanent data retention + +/** + * List all images for a product + */ +export async function listImages( + brand: string, + state: string, + productId: string | number +): Promise { + const dir = buildProductDir(brand, state, productId); + try { + return await fs.readdir(dir); + } catch { + return []; + } +} + +/** + * Initialize storage directories + */ +export async function initializeStorage(): Promise { + await fs.mkdir(path.join(STORAGE_BASE_PATH, 'products'), { recursive: true }); + await fs.mkdir(path.join(STORAGE_BASE_PATH, 'brands'), { recursive: true }); + console.log(`[local-storage] Initialized at ${path.resolve(STORAGE_BASE_PATH)}`); +} + +/** + * Get storage base path + */ +export function getStorageBasePath(): string { + return path.resolve(STORAGE_BASE_PATH); +} diff --git a/backend/src/utils/storage-adapter.ts b/backend/src/utils/storage-adapter.ts new file mode 100644 index 00000000..ce94d198 --- /dev/null +++ b/backend/src/utils/storage-adapter.ts @@ -0,0 +1,34 @@ +/** + * Unified Storage Adapter + * + * Routes storage calls to the appropriate backend based on STORAGE_DRIVER. + * + * STORAGE_DRIVER=local -> Uses local filesystem (./storage) + * STORAGE_DRIVER=minio -> Uses MinIO/S3 (requires MINIO_ENDPOINT) + * + * Usage: + * import { saveImage, getImageUrl } from '../utils/storage-adapter'; + */ + +const STORAGE_DRIVER = process.env.STORAGE_DRIVER || 'local'; + +// Determine which driver to use +const useLocalStorage = STORAGE_DRIVER === 'local' || !process.env.MINIO_ENDPOINT; + +if (useLocalStorage) { + console.log('[storage-adapter] Using LOCAL filesystem storage'); +} else { + console.log('[storage-adapter] Using MinIO/S3 remote storage'); +} + +// Export the appropriate implementation +export * from './local-storage'; + +// Re-export driver info +export function getStorageDriver(): string { + return useLocalStorage ? 'local' : 'minio'; +} + +export function isLocalStorage(): boolean { + return useLocalStorage; +} diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 00000000..e3a120a3 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,64 @@ +# Local Development Stack - NO MinIO +# +# Usage: +# docker-compose -f docker-compose.local.yml up +# +# Or use the convenience script: +# ./start-local.sh +# +# This runs: +# - PostgreSQL database +# - Backend API server +# - Local filesystem storage (./storage) +# +# NO MinIO, NO remote connections, NO Kubernetes + +services: + postgres: + image: postgres:15-alpine + container_name: cannaiq-postgres + environment: + POSTGRES_DB: dutchie_menus + POSTGRES_USER: dutchie + POSTGRES_PASSWORD: dutchie_local_pass + ports: + - "54320:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dutchie"] + interval: 10s + timeout: 5s + retries: 5 + + backend: + build: + context: ./backend + dockerfile: Dockerfile.dev + container_name: cannaiq-backend + environment: + NODE_ENV: development + PORT: 3000 + DATABASE_URL: "postgresql://dutchie:dutchie_local_pass@postgres:5432/dutchie_menus" + # Local storage - NO MinIO + STORAGE_DRIVER: local + STORAGE_BASE_PATH: /app/storage + STORAGE_PUBLIC_URL: /storage + # No MinIO env vars - forces local storage + # MINIO_ENDPOINT is intentionally NOT set + JWT_SECRET: local_dev_jwt_secret_change_in_production + ADMIN_EMAIL: admin@example.com + ADMIN_PASSWORD: password + ports: + - "3010:3000" + volumes: + - ./backend:/app + - /app/node_modules + - ./storage:/app/storage + depends_on: + postgres: + condition: service_healthy + command: npm run dev + +volumes: + postgres_data: diff --git a/start-local.sh b/start-local.sh new file mode 100755 index 00000000..81c9c2d4 --- /dev/null +++ b/start-local.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Start Local CannaiQ Stack +# +# Runs PostgreSQL + Backend with local filesystem storage. +# NO MinIO, NO Kubernetes, NO remote connections. +# + +set -e + +echo "Starting CannaiQ Local Development Stack..." +echo "============================================" +echo " - PostgreSQL: localhost:54320" +echo " - Backend API: localhost:3010" +echo " - Storage: ./storage (local filesystem)" +echo " - NO MinIO" +echo "" + +# Ensure storage directories exist +mkdir -p storage/products +mkdir -p storage/brands + +# Start services +docker-compose -f docker-compose.local.yml up "$@"