Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Auto-migrations on startup, worker exit location, proxy improvements

Browse Source 
- Add auto-migration system that runs SQL files from migrations/ on server startup
- Track applied migrations in schema_migrations table
- Show proxy exit location in Workers dashboard
- Add "Cleanup Stale" button to remove old workers
- Add remove button for individual workers
- Include proxy location (city, state, country) in worker heartbeats
- Update Proxy interface with location fields
- Re-enable bulk proxy import without ON CONFLICT

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-10 01:42:00 -07:00
parent 9a9c2f76a2
commit 6eb1babc86
9 changed files with 598 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
backend/src/services/proxyTestQueue.ts
View File

@@ -8,8 +8,12 @@ interface ProxyTestJob {
tested_proxies: number;
passed_proxies: number;
failed_proxies: number;
mode?: string; // 'all' | 'failed' | 'inactive'
}
// Concurrency settings
const DEFAULT_CONCURRENCY = 10; // Test 10 proxies at a time
// Simple in-memory queue - could be replaced with Bull/Bee-Queue for production
const activeJobs = new Map<number, { cancelled: boolean }>();
@@ -33,18 +37,35 @@ export async function cleanupOrphanedJobs(): Promise<void> {
}
}
export async function createProxyTestJob(): Promise<number> {
export type ProxyTestMode = 'all' | 'failed' | 'inactive';
export async function createProxyTestJob(mode: ProxyTestMode = 'all', concurrency: number = DEFAULT_CONCURRENCY): Promise<number> {
// Check for existing running jobs first
const existingJob = await getActiveProxyTestJob();
if (existingJob) {
throw new Error('A proxy test job is already running. Please cancel it first.');
}
const result = await pool.query(`
SELECT COUNT(*) as count FROM proxies
`);
// Get count based on mode
let countQuery: string;
switch (mode) {
case 'failed':
countQuery = `SELECT COUNT(*) as count FROM proxies WHERE test_result = 'failed' OR active = false`;
break;
case 'inactive':
countQuery = `SELECT COUNT(*) as count FROM proxies WHERE active = false`;
break;
default:
countQuery = `SELECT COUNT(*) as count FROM proxies`;
}
const result = await pool.query(countQuery);
const totalProxies = parseInt(result.rows[0].count);
if (totalProxies === 0) {
throw new Error(`No proxies to test with mode '${mode}'`);
}
const jobResult = await pool.query(`
INSERT INTO proxy_test_jobs (status, total_proxies)
VALUES ('pending', $1)
@@ -53,8 +74,8 @@ export async function createProxyTestJob(): Promise<number> {
const jobId = jobResult.rows[0].id;
// Start job in background
runProxyTestJob(jobId).catch(err => {
// Start job in background with mode and concurrency
runProxyTestJob(jobId, mode, concurrency).catch(err => {
console.error(`❌ Proxy test job ${jobId} failed:`, err);
});
@@ -111,7 +132,7 @@ export async function cancelProxyTestJob(jobId: number): Promise<boolean> {
return result.rows.length > 0;
}
async function runProxyTestJob(jobId: number): Promise<void> {
async function runProxyTestJob(jobId: number, mode: ProxyTestMode = 'all', concurrency: number = DEFAULT_CONCURRENCY): Promise<void> {
// Register job as active
activeJobs.set(jobId, { cancelled: false });
@@ -125,20 +146,30 @@ async function runProxyTestJob(jobId: number): Promise<void> {
WHERE id = $1
`, [jobId]);
console.log(`🔍 Starting proxy test job ${jobId}...`);
console.log(`🔍 Starting proxy test job ${jobId} (mode: ${mode}, concurrency: ${concurrency})...`);
// Get all proxies
const result = await pool.query(`
SELECT id, host, port, protocol, username, password
FROM proxies
ORDER BY id
`);
// Get proxies based on mode
let query: string;
switch (mode) {
case 'failed':
query = `SELECT id, host, port, protocol, username, password FROM proxies WHERE test_result = 'failed' OR active = false ORDER BY id`;
break;
case 'inactive':
query = `SELECT id, host, port, protocol, username, password FROM proxies WHERE active = false ORDER BY id`;
break;
default:
query = `SELECT id, host, port, protocol, username, password FROM proxies ORDER BY id`;
}
const result = await pool.query(query);
const proxies = result.rows;
let tested = 0;
let passed = 0;
let failed = 0;
for (const proxy of result.rows) {
// Process proxies in batches for parallel testing
for (let i = 0; i < proxies.length; i += concurrency) {
// Check if job was cancelled
const jobControl = activeJobs.get(jobId);
if (jobControl?.cancelled) {
@@ -146,23 +177,34 @@ async function runProxyTestJob(jobId: number): Promise<void> {
break;
}
// Test the proxy
const testResult = await testProxy(
proxy.host,
proxy.port,
proxy.protocol,
proxy.username,
proxy.password
const batch = proxies.slice(i, i + concurrency);
// Test batch in parallel
const batchResults = await Promise.all(
batch.map(async (proxy) => {
const testResult = await testProxy(
proxy.host,
proxy.port,
proxy.protocol,
proxy.username,
proxy.password
);
// Save result
await saveProxyTestResult(proxy.id, testResult);
return testResult.success;
})
);
// Save result
await saveProxyTestResult(proxy.id, testResult);
tested++;
if (testResult.success) {
passed++;
} else {
failed++;
// Count results
for (const success of batchResults) {
tested++;
if (success) {
passed++;
} else {
failed++;
}
}
// Update job progress
@@ -175,10 +217,8 @@ async function runProxyTestJob(jobId: number): Promise<void> {
WHERE id = $4
`, [tested, passed, failed, jobId]);
// Log progress every 10 proxies
if (tested % 10 === 0) {
console.log(`📊 Job ${jobId}: ${tested}/${result.rows.length} proxies tested (${passed} passed, ${failed} failed)`);
}
// Log progress
console.log(`📊 Job ${jobId}: ${tested}/${proxies.length} proxies tested (${passed} passed, ${failed} failed)`);
}
// Mark job as completed