cannaiq/backend/dist/services/proxy.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.isBotDetectionError = isBotDetectionError;
exports.putProxyInTimeout = putProxyInTimeout;
exports.isProxyInTimeout = isProxyInTimeout;
exports.getActiveProxy = getActiveProxy;
exports.testProxy = testProxy;
exports.saveProxyTestResult = saveProxyTestResult;
exports.testAllProxies = testAllProxies;
exports.addProxy = addProxy;
exports.addProxiesFromList = addProxiesFromList;
exports.moveProxyToFailed = moveProxyToFailed;
exports.incrementProxyFailure = incrementProxyFailure;
const axios_1 = __importDefault(require("axios"));
const socks_proxy_agent_1 = require("socks-proxy-agent");
const https_proxy_agent_1 = require("https-proxy-agent");
const migrate_1 = require("../db/migrate");
// In-memory proxy timeout tracking
// Maps proxy ID to timestamp when timeout expires
const proxyTimeouts = new Map();
const PROXY_TIMEOUT_MS = 35000; // 35 seconds timeout for bot-detected proxies
// Check if error message indicates bot detection
function isBotDetectionError(errorMsg) {
    const botPatterns = [
        /bot detection/i,
        /captcha/i,
        /challenge/i,
        /cloudflare/i,
        /access denied/i,
        /rate limit/i,
        /too many requests/i,
        /temporarily blocked/i,
        /suspicious activity/i,
    ];
    return botPatterns.some(pattern => pattern.test(errorMsg));
}
// Put proxy in timeout (bot detection cooldown)
function putProxyInTimeout(proxyId, reason) {
    const timeoutUntil = Date.now() + PROXY_TIMEOUT_MS;
    proxyTimeouts.set(proxyId, timeoutUntil);
    console.log(`🚫 Proxy ${proxyId} in timeout for ${PROXY_TIMEOUT_MS / 1000}s: ${reason}`);
}
// Check if proxy is currently in timeout
function isProxyInTimeout(proxyId) {
    const timeoutUntil = proxyTimeouts.get(proxyId);
    if (!timeoutUntil)
        return false;
    if (Date.now() >= timeoutUntil) {
        // Timeout expired, remove it
        proxyTimeouts.delete(proxyId);
        console.log(`✅ Proxy ${proxyId} timeout expired, back in rotation`);
        return false;
    }
    return true;
}
// Get active proxy that's not in timeout
async function getActiveProxy() {
    const result = await migrate_1.pool.query(`
    SELECT id, host, port, protocol, username, password
    FROM proxies
    WHERE active = true
    ORDER BY RANDOM()
  `);
    // Filter out proxies in timeout
    for (const proxy of result.rows) {
        if (!isProxyInTimeout(proxy.id)) {
            return proxy;
        }
    }
    // All proxies are in timeout, wait for first one to expire
    if (proxyTimeouts.size > 0) {
        const nextAvailable = Math.min(...Array.from(proxyTimeouts.values()));
        const waitTime = Math.max(0, nextAvailable - Date.now());
        console.log(`⏳ All proxies in timeout, waiting ${Math.ceil(waitTime / 1000)}s for next available...`);
        await new Promise(resolve => setTimeout(resolve, waitTime));
        // Try again after waiting
        return getActiveProxy();
    }
    console.log('⚠️  No active proxies available');
    return null;
}
async function getSettings() {
    const result = await migrate_1.pool.query(`
    SELECT key, value FROM settings
    WHERE key IN ('proxy_timeout_ms', 'proxy_test_url')
  `);
    const settings = {};
    result.rows.forEach((row) => {
        settings[row.key] = row.value;
    });
    return {
        timeout: parseInt(settings.proxy_timeout_ms || '3000'),
        testUrl: settings.proxy_test_url || 'https://httpbin.org/ip'
    };
}
async function testProxy(host, port, protocol, username, password) {
    try {
        const { timeout, testUrl } = await getSettings();
        const startTime = Date.now();
        // Construct proxy URL
        let proxyUrl;
        if (username && password) {
            proxyUrl = `${protocol}://${username}:${password}@${host}:${port}`;
        }
        else {
            proxyUrl = `${protocol}://${host}:${port}`;
        }
        // Create appropriate agent based on protocol
        let agent;
        if (protocol === 'socks5' || protocol === 'socks') {
            agent = new socks_proxy_agent_1.SocksProxyAgent(proxyUrl);
        }
        else if (protocol === 'http' || protocol === 'https') {
            agent = new https_proxy_agent_1.HttpsProxyAgent(proxyUrl);
        }
        else {
            return {
                success: false,
                error: `Unsupported protocol: ${protocol}`
            };
        }
        // Make test request
        const response = await axios_1.default.get(testUrl, {
            httpAgent: agent,
            httpsAgent: agent,
            timeout,
        });
        const responseTimeMs = Date.now() - startTime;
        // Check anonymity - the test URL should return our IP
        // If it returns the proxy's IP, we're anonymous
        let isAnonymous = false;
        if (response.data && response.data.origin) {
            // If the returned IP is different from our actual IP, the proxy is working
            // For simplicity, we'll consider it anonymous if we get a response
            isAnonymous = true;
        }
        return {
            success: true,
            responseTimeMs,
            isAnonymous
        };
    }
    catch (error) {
        return {
            success: false,
            error: error.message || 'Unknown error'
        };
    }
}
async function saveProxyTestResult(proxyId, result) {
    await migrate_1.pool.query(`
    UPDATE proxies
    SET last_tested_at = CURRENT_TIMESTAMP,
        test_result = $1,
        response_time_ms = $2,
        is_anonymous = $3,
        active = $4,
        updated_at = CURRENT_TIMESTAMP
    WHERE id = $5
  `, [
        result.success ? 'success' : 'failed',
        result.responseTimeMs || null,
        result.isAnonymous || false,
        result.success,
        proxyId
    ]);
}
async function testAllProxies() {
    console.log('🔍 Testing all proxies...');
    const result = await migrate_1.pool.query(`
    SELECT id, host, port, protocol, username, password
    FROM proxies
  `);
    for (const proxy of result.rows) {
        console.log(`Testing proxy: ${proxy.protocol}://${proxy.host}:${proxy.port}`);
        const testResult = await testProxy(proxy.host, proxy.port, proxy.protocol, proxy.username, proxy.password);
        await saveProxyTestResult(proxy.id, testResult);
        if (testResult.success) {
            console.log(`✅ Proxy OK (${testResult.responseTimeMs}ms, anonymous: ${testResult.isAnonymous})`);
        }
        else {
            console.log(`❌ Proxy failed: ${testResult.error}`);
        }
        // Small delay between tests
        await new Promise(resolve => setTimeout(resolve, 500));
    }
    console.log('✅ Proxy testing complete');
}
async function addProxy(host, port, protocol, username, password) {
    // Test the proxy first
    const testResult = await testProxy(host, port, protocol, username, password);
    if (!testResult.success) {
        throw new Error(`Proxy test failed: ${testResult.error}`);
    }
    // Insert into database
    const result = await migrate_1.pool.query(`
    INSERT INTO proxies (host, port, protocol, username, password, active, is_anonymous, test_result, response_time_ms, last_tested_at)
    VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CURRENT_TIMESTAMP)
    RETURNING id
  `, [
        host,
        port,
        protocol,
        username,
        password,
        testResult.success,
        testResult.isAnonymous,
        'success',
        testResult.responseTimeMs
    ]);
    return result.rows[0].id;
}
async function addProxiesFromList(proxies) {
    let added = 0;
    let failed = 0;
    let duplicates = 0;
    const errors = [];
    console.log(`📥 Importing ${proxies.length} proxies without testing...`);
    for (const proxy of proxies) {
        try {
            // Insert without testing first
            await migrate_1.pool.query(`
        INSERT INTO proxies (host, port, protocol, username, password, active)
        VALUES ($1, $2, $3, $4, $5, false)
        ON CONFLICT (host, port, protocol) DO NOTHING
      `, [
                proxy.host,
                proxy.port,
                proxy.protocol,
                proxy.username,
                proxy.password
            ]);
            // Check if it was actually inserted
            const result = await migrate_1.pool.query(`
        SELECT id FROM proxies
        WHERE host = $1 AND port = $2 AND protocol = $3
      `, [proxy.host, proxy.port, proxy.protocol]);
            if (result.rows.length > 0) {
                // Check if it was just inserted (no last_tested_at means new)
                const checkResult = await migrate_1.pool.query(`
          SELECT last_tested_at FROM proxies
          WHERE host = $1 AND port = $2 AND protocol = $3
        `, [proxy.host, proxy.port, proxy.protocol]);
                if (checkResult.rows[0].last_tested_at === null) {
                    added++;
                    if (added % 100 === 0) {
                        console.log(`📥 Imported ${added} proxies...`);
                    }
                }
                else {
                    duplicates++;
                }
            }
        }
        catch (error) {
            failed++;
            const errorMsg = `${proxy.host}:${proxy.port} - ${error.message}`;
            errors.push(errorMsg);
            console.log(`❌ Failed to add proxy: ${errorMsg}`);
        }
    }
    console.log(`✅ Import complete: ${added} added, ${duplicates} duplicates, ${failed} failed`);
    return { added, failed, duplicates, errors };
}
async function moveProxyToFailed(proxyId, errorMsg) {
    // Get proxy details
    const proxyResult = await migrate_1.pool.query(`
    SELECT host, port, protocol, username, password, failure_count
    FROM proxies
    WHERE id = $1
  `, [proxyId]);
    if (proxyResult.rows.length === 0) {
        return;
    }
    const proxy = proxyResult.rows[0];
    // Insert into failed_proxies table
    await migrate_1.pool.query(`
    INSERT INTO failed_proxies (host, port, protocol, username, password, failure_count, last_error)
    VALUES ($1, $2, $3, $4, $5, $6, $7)
    ON CONFLICT (host, port, protocol)
    DO UPDATE SET
      failure_count = $6,
      last_error = $7,
      failed_at = CURRENT_TIMESTAMP
  `, [
        proxy.host,
        proxy.port,
        proxy.protocol,
        proxy.username,
        proxy.password,
        proxy.failure_count,
        errorMsg
    ]);
    // Delete from active proxies
    await migrate_1.pool.query(`DELETE FROM proxies WHERE id = $1`, [proxyId]);
    console.log(`🔴 Moved proxy to failed: ${proxy.protocol}://${proxy.host}:${proxy.port} (${proxy.failure_count} failures)`);
}
async function incrementProxyFailure(proxyId, errorMsg) {
    // Increment failure count
    const result = await migrate_1.pool.query(`
    UPDATE proxies
    SET failure_count = failure_count + 1,
        active = false,
        updated_at = CURRENT_TIMESTAMP
    WHERE id = $1
    RETURNING failure_count, host, port, protocol
  `, [proxyId]);
    if (result.rows.length === 0) {
        return false;
    }
    const proxy = result.rows[0];
    const failureCount = proxy.failure_count;
    console.log(`⚠️  Proxy failure #${failureCount}: ${proxy.protocol}://${proxy.host}:${proxy.port}`);
    // If failed 3 times, move to failed table
    if (failureCount >= 3) {
        await moveProxyToFailed(proxyId, errorMsg);
        return true; // Moved to failed
    }
    return false; // Still in active proxies
}