feat: Auto-retry failed proxies after cooldown period
- Add last_failed_at column to track failure time - Failed proxies auto-retry after 4 hours (configurable) - Proxies permanently failed after 10 failures - Add /retry-stats and /reenable-failed API endpoints - markProxySuccess() re-enables recovered proxies 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
81
backend/migrations/095_proxy_auto_retry.sql
Normal file
81
backend/migrations/095_proxy_auto_retry.sql
Normal file
@@ -0,0 +1,81 @@
|
||||
-- Migration: Auto-retry failed proxies after cooldown period
|
||||
-- Proxies that fail will be retried after a configurable interval
|
||||
|
||||
-- Add last_failed_at column to track when proxy last failed
|
||||
ALTER TABLE proxies ADD COLUMN IF NOT EXISTS last_failed_at TIMESTAMP;
|
||||
|
||||
-- Add retry settings
|
||||
INSERT INTO settings (key, value, description)
|
||||
VALUES
|
||||
('proxy_retry_interval_hours', '4', 'Hours to wait before retrying a failed proxy'),
|
||||
('proxy_max_failures_before_permanent', '10', 'Max failures before proxy is permanently disabled')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- Create function to get eligible proxies (active OR failed but past retry interval)
|
||||
CREATE OR REPLACE FUNCTION get_eligible_proxy_ids()
|
||||
RETURNS TABLE(proxy_id INT) AS $$
|
||||
DECLARE
|
||||
retry_hours INT;
|
||||
BEGIN
|
||||
-- Get retry interval from settings (default 4 hours)
|
||||
SELECT COALESCE(value::int, 4) INTO retry_hours
|
||||
FROM settings WHERE key = 'proxy_retry_interval_hours';
|
||||
|
||||
RETURN QUERY
|
||||
SELECT p.id
|
||||
FROM proxies p
|
||||
WHERE p.active = true
|
||||
OR (
|
||||
p.active = false
|
||||
AND p.last_failed_at IS NOT NULL
|
||||
AND p.last_failed_at < NOW() - (retry_hours || ' hours')::interval
|
||||
AND p.failure_count < 10 -- Don't retry if too many failures
|
||||
)
|
||||
ORDER BY
|
||||
p.active DESC, -- Prefer active proxies
|
||||
p.failure_count ASC, -- Then prefer proxies with fewer failures
|
||||
RANDOM();
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Create scheduled job to periodically re-enable proxies past their retry window
|
||||
-- This runs every hour and marks proxies as active if they're past retry interval
|
||||
CREATE OR REPLACE FUNCTION auto_reenable_proxies()
|
||||
RETURNS INT AS $$
|
||||
DECLARE
|
||||
retry_hours INT;
|
||||
max_failures INT;
|
||||
reenabled_count INT;
|
||||
BEGIN
|
||||
-- Get settings
|
||||
SELECT COALESCE(value::int, 4) INTO retry_hours
|
||||
FROM settings WHERE key = 'proxy_retry_interval_hours';
|
||||
|
||||
SELECT COALESCE(value::int, 10) INTO max_failures
|
||||
FROM settings WHERE key = 'proxy_max_failures_before_permanent';
|
||||
|
||||
-- Re-enable proxies that have cooled down
|
||||
UPDATE proxies
|
||||
SET active = true,
|
||||
updated_at = NOW()
|
||||
WHERE active = false
|
||||
AND last_failed_at IS NOT NULL
|
||||
AND last_failed_at < NOW() - (retry_hours || ' hours')::interval
|
||||
AND failure_count < max_failures;
|
||||
|
||||
GET DIAGNOSTICS reenabled_count = ROW_COUNT;
|
||||
|
||||
IF reenabled_count > 0 THEN
|
||||
RAISE NOTICE 'Auto-reenabled % proxies after % hour cooldown', reenabled_count, retry_hours;
|
||||
END IF;
|
||||
|
||||
RETURN reenabled_count;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Add index for efficient querying
|
||||
CREATE INDEX IF NOT EXISTS idx_proxies_retry
|
||||
ON proxies(active, last_failed_at, failure_count);
|
||||
|
||||
COMMENT ON COLUMN proxies.last_failed_at IS 'Timestamp of last failure - used for auto-retry logic';
|
||||
COMMENT ON FUNCTION auto_reenable_proxies() IS 'Call periodically to re-enable failed proxies that have cooled down';
|
||||
Reference in New Issue
Block a user