<?php
/**
 * ============================================
 * FLOWBOT DCI - CRAWLER ULTIMATE v11.0 VIRAL+
 * ============================================
 * Professional crawler with VIRAL MODE:
 * - Multi-search engines (Bing, DuckDuckGo, Yandex, Baidu, Yahoo)
 * - Deep crawl with depth control
 * - Include/exclude filters
 * - robots.txt respect
 * - cURL multi for performance
 * - Direct import to pinfeeds table
 * - Preview before import
 * - ♾️ INFINITE MODE: Crawl + Import simultaneously
 * - 🦠 VIRAL MODE: Parallel spreading mesh
 *   - cURL multi (10 parallel per wave)
 *   - Exponential link pool growth
 *   - Micro-batch imports (5 per insert)
 *   - Network mesh spreading pattern
 *   - Real-time stats + activity log
 *   - RE-SEEDING: Auto-refills pool from engines when < 100
 *   - Smart URL filtering (skip 404-prone patterns)
 *   - Skipped vs Errors separation (4xx = skip, 5xx = error)
 * ============================================
 */

// ============================================
// CONFIGURATION
// ============================================
ini_set('display_errors', 1);
error_reporting(E_ALL);
set_time_limit(0);
ini_set('memory_limit', '2048M');

$DB_HOST = 'localhost';
$DB_NAME = 'digupdog_FEED';
$DB_USER = 'digupdog_FEEDadmin';
$DB_PASS = 'Raimundinho1';

try {
    $pdo = new PDO(
        "mysql:host=$DB_HOST;dbname=$DB_NAME;charset=utf8mb4",
        $DB_USER,
        $DB_PASS,
        [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]
    );
} catch (PDOException $e) {
    die("Database connection failed: " . $e->getMessage());
}

// S2: CSRF Token generation
session_start();
if (empty($_SESSION['csrf_token'])) {
    $_SESSION['csrf_token'] = bin2hex(random_bytes(32));
}
$csrfToken = $_SESSION['csrf_token'];
session_write_close();

// R6: Structured error logging function
function logCrawlerError($type, $data) {
    $logEntry = json_encode(array_merge([
        'time' => date('c'),
        'type' => $type,
    ], $data)) . "\n";
    @file_put_contents('/tmp/crawler_errors.log', $logEntry, FILE_APPEND | LOCK_EX);
}

// ============================================
// MULTI-PROCESS SYSTEM (1000+ concurrent processes)
// ============================================
define('MP_MAX_PROCESSES', 1000);
define('MP_HEARTBEAT_SEC', 30);
define('MP_DEAD_TIMEOUT', 120);
define('MP_CLAIM_BATCH', 50);
define('MP_STATS_INTERVAL', 5);

// --- PROCESS LIFECYCLE ---
function mpCreateProcess($pdo, $mode, $keywords, $config) {
    $id = bin2hex(random_bytes(16));
    $stmt = $pdo->prepare("INSERT INTO crawler_processes (id, mode, status, keywords, config) VALUES (?, ?, 'pending', ?, ?)");
    $stmt->execute([$id, $mode, json_encode($keywords), json_encode($config)]);
    return $id;
}

function mpRegisterProcess($pdo, $processId) {
    $stmt = $pdo->prepare("UPDATE crawler_processes SET status='running', started_at=NOW(), last_heartbeat=NOW(), pid=? WHERE id=? AND status='pending'");
    $stmt->execute([getmypid(), $processId]);
    return $stmt->rowCount() > 0;
}

function mpHeartbeat($pdo, $processId, $stats) {
    $stmt = $pdo->prepare("UPDATE crawler_processes SET last_heartbeat=NOW(), discovered=?, processed=?, imported=?, duplicates=?, errors=?, skipped=?, pool_size=?, waves=?, rate=? WHERE id=?");
    $stmt->execute([
        $stats['discovered'] ?? 0, $stats['processed'] ?? 0, $stats['imported'] ?? 0,
        $stats['duplicates'] ?? 0, $stats['errors'] ?? 0, $stats['skipped'] ?? 0,
        $stats['pool_size'] ?? 0, $stats['waves'] ?? 0, $stats['rate'] ?? 0, $processId
    ]);
}

function mpCheckControl($pdo, $processId) {
    $stmt = $pdo->prepare("SELECT status FROM crawler_processes WHERE id=?");
    $stmt->execute([$processId]);
    return $stmt->fetchColumn() ?: 'stopped';
}

function mpControlProcess($pdo, $processId, $action) {
    $map = ['stop' => 'stopped', 'pause' => 'paused', 'resume' => 'running'];
    $newStatus = $map[$action] ?? null;
    if (!$newStatus) return false;
    $extra = ($action === 'stop') ? ", stopped_at=NOW()" : "";
    $stmt = $pdo->prepare("UPDATE crawler_processes SET status=?$extra WHERE id=?");
    return $stmt->execute([$newStatus, $processId]);
}

function mpCleanupDead($pdo) {
    // Mark dead processes (no heartbeat for 2 min)
    $pdo->exec("UPDATE crawler_processes SET status='dead', stopped_at=NOW() WHERE status IN ('running','paused') AND last_heartbeat < DATE_SUB(NOW(), INTERVAL " . MP_DEAD_TIMEOUT . " SECOND)");
    // Release their claimed URLs
    $pdo->exec("UPDATE crawler_pool SET status='pending', process_id=NULL, claimed_at=NULL WHERE status='claimed' AND process_id IN (SELECT id FROM crawler_processes WHERE status='dead')");
}

function mpCanStart($pdo) {
    $count = $pdo->query("SELECT COUNT(*) FROM crawler_processes WHERE status IN ('running','paused')")->fetchColumn();
    return $count < MP_MAX_PROCESSES;
}

// --- URL POOL ---
function mpAddToPool($pdo, $urls, $sourceProcess, $priority = 50) {
    if (empty($urls)) return 0;
    $added = 0;
    foreach (array_chunk($urls, 500) as $chunk) {
        $ph = []; $vals = [];
        foreach ($chunk as $u) {
            $ph[] = '(?, ?, ?, ?, ?)';
            $vals[] = $u;
            $vals[] = md5($u);
            $vals[] = $priority;
            $vals[] = $sourceProcess;
            $vals[] = 'pending';
        }
        $stmt = $pdo->prepare("INSERT IGNORE INTO crawler_pool (url, url_hash, priority, source_process, status) VALUES " . implode(',', $ph));
        $stmt->execute($vals);
        $added += $stmt->rowCount();
    }
    return $added;
}

function mpClaimUrls($pdo, $processId, $batchSize = 50) {
    // Atomic claim: UPDATE LIMIT prevents 2 processes from claiming same URLs
    $stmt = $pdo->prepare("UPDATE crawler_pool SET process_id=?, status='claimed', claimed_at=NOW(), attempts=attempts+1 WHERE status='pending' AND attempts<3 ORDER BY priority DESC, id ASC LIMIT ?");
    $stmt->execute([$processId, $batchSize]);
    if ($stmt->rowCount() === 0) return [];
    // Fetch what we just claimed
    $stmt = $pdo->prepare("SELECT id, url FROM crawler_pool WHERE process_id=? AND status='claimed' ORDER BY priority DESC, id ASC LIMIT ?");
    $stmt->execute([$processId, $batchSize]);
    return $stmt->fetchAll(PDO::FETCH_ASSOC);
}

function mpMarkDone($pdo, $poolIds, $status = 'done') {
    if (empty($poolIds)) return;
    $ph = implode(',', array_fill(0, count($poolIds), '?'));
    $pdo->prepare("UPDATE crawler_pool SET status=? WHERE id IN ($ph)")->execute(array_merge([$status], $poolIds));
}

function mpUrlPriority($url) {
    if (preg_match('/\/post[s]?\/|\/article|\/blog\/|\/news\/|\/story\/|\d{4}[\/-]\d{2}/i', $url)) return 80;
    $path = parse_url($url, PHP_URL_PATH) ?? '/';
    if ($path === '/' || $path === '') return 20;
    return 50;
}

// --- SHARED DEDUP ---
function mpMarkSeen($pdo, $url, $processId, $result, $canonicalUrl = null) {
    $stmt = $pdo->prepare("INSERT IGNORE INTO crawler_seen (url_hash, canonical_hash, process_id, result) VALUES (?, ?, ?, ?)");
    $stmt->execute([md5($url), $canonicalUrl ? md5($canonicalUrl) : null, $processId, $result]);
}

function mpIsSeenBatch($pdo, $urls) {
    if (empty($urls)) return [];
    $hashes = array_map('md5', $urls);
    $ph = implode(',', array_fill(0, count($hashes), '?'));
    $stmt = $pdo->prepare("SELECT url_hash FROM crawler_seen WHERE url_hash IN ($ph)");
    $stmt->execute($hashes);
    $seen = [];
    while ($row = $stmt->fetch(PDO::FETCH_COLUMN)) $seen[$row] = true;
    $result = [];
    foreach ($urls as $u) $result[$u] = isset($seen[md5($u)]);
    return $result;
}

// --- SAFE CREATE (race-condition-free INSERT IGNORE + SELECT) ---
function mpGetOrCreateUser($pdo, $author) {
    static $cache = [];
    if (isset($cache[$author])) return $cache[$author];

    $email = strtolower(preg_replace('/[^a-zA-Z0-9]/', '', $author)) . rand(1,999) . '@digupdog.com';
    $parts = explode(' ', $author);
    $firstName = ucfirst($parts[0] ?? 'User');
    $lastName = ucfirst($parts[1] ?? 'Web');
    $password = password_hash('Raimundinho1', PASSWORD_DEFAULT);
    $birthdate = date('Y-m-d', strtotime('-' . rand(18, 50) . ' years'));

    try {
        $pdo->prepare("INSERT IGNORE INTO user_myhashtag (username, email, senha, first_name, last_name, address, phone_number, created_at, birthdate, profile_picture, bio, gender, status, user_role, privacy_settings, social_links, preferences) VALUES (?, ?, ?, ?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?)")
            ->execute([$author, $email, $password, $firstName, $lastName, '123 Web St', '+1'.rand(1000000000,9999999999), $birthdate, '', 'Content creator.', rand(0,1)?'male':'female', 'active', 'user', '{}', '{}', '{}']);
    } catch (Exception $e) {}

    $stmt = $pdo->prepare("SELECT ID FROM user_myhashtag WHERE username=? LIMIT 1");
    $stmt->execute([$author]);
    $id = (int)($stmt->fetchColumn() ?: 0);
    if ($id && count($cache) < 500) $cache[$author] = $id;
    return $id;
}

function mpGetOrCreateDomain($pdo, $url) {
    static $cache = [];
    $host = parse_url($url, PHP_URL_HOST);
    if (!$host) return ['domain_id' => 0, 'category_id' => 0];
    if (isset($cache[$host])) return $cache[$host];

    $baseUrl = 'https://' . $host;
    try {
        $pdo->prepare("INSERT IGNORE INTO feed_data (website_base, website_feed, main_category_id) VALUES (?, ?, 0)")
            ->execute([$baseUrl, $url]);
    } catch (Exception $e) {}

    $stmt = $pdo->prepare("SELECT id, main_category_id FROM feed_data WHERE website_base=? LIMIT 1");
    $stmt->execute([$baseUrl]);
    $row = $stmt->fetch(PDO::FETCH_ASSOC);
    $result = ['domain_id' => (int)($row['id'] ?? 0), 'category_id' => (int)($row['main_category_id'] ?? 0)];
    if (count($cache) < 500) $cache[$host] = $result;
    return $result;
}

// --- LOG ---
function mpWriteLog($pdo, $processId, $logs) {
    if (empty($logs)) return;
    $batch = array_slice($logs, -10); // Only last 10
    $ph = []; $vals = [];
    foreach ($batch as $log) {
        $ph[] = '(?, ?, ?, ?, ?)';
        $vals[] = $processId;
        $vals[] = mb_substr($log['url'] ?? '', 0, 2048);
        $vals[] = $log['status'] ?? 'info';
        $vals[] = mb_substr($log['title'] ?? '', 0, 500);
        $vals[] = mb_substr($log['domain'] ?? '', 0, 255);
    }
    try {
        $pdo->prepare("INSERT INTO crawler_log (process_id, url, status, title, domain) VALUES " . implode(',', $ph))->execute($vals);
    } catch (Exception $e) {}
}

// --- CLEANUP ---
function mpCleanup($pdo) {
    $pdo->exec("DELETE FROM crawler_pool WHERE status IN ('done','failed') AND created_at < DATE_SUB(NOW(), INTERVAL 24 HOUR)");
    $pdo->exec("DELETE FROM crawler_seen WHERE created_at < DATE_SUB(NOW(), INTERVAL 7 DAY)");
    $pdo->exec("DELETE FROM crawler_log WHERE created_at < DATE_SUB(NOW(), INTERVAL 2 HOUR)");
    $pdo->exec("DELETE FROM crawler_edges WHERE created_at < DATE_SUB(NOW(), INTERVAL 24 HOUR)");
    // Release stuck claims (claimed > 10 min ago)
    $pdo->exec("UPDATE crawler_pool SET status='pending', process_id=NULL WHERE status='claimed' AND claimed_at < DATE_SUB(NOW(), INTERVAL 10 MINUTE)");
}

// --- ULTIMATE HELPERS: Edge Storage + Domain Quality ---
function mpStoreEdges($pdo, $edges, $processId) {
    if (empty($edges)) return 0;
    $added = 0;
    foreach (array_chunk($edges, 100) as $chunk) {
        $ph = []; $vals = [];
        foreach ($chunk as $edge) {
            $src = $edge[0] ?? '';
            $tgt = $edge[1] ?? '';
            if (empty($src) || empty($tgt) || $src === $tgt) continue;
            $ph[] = '(?, ?, ?, ?)';
            $vals[] = mb_substr($src, 0, 255);
            $vals[] = mb_substr($tgt, 0, 255);
            $vals[] = md5($src . '>' . $tgt);
            $vals[] = $processId;
        }
        if (!empty($ph)) {
            try {
                $stmt = $pdo->prepare("INSERT IGNORE INTO crawler_edges (source_domain, target_domain, edge_hash, process_id) VALUES " . implode(',', $ph));
                $stmt->execute($vals);
                $added += $stmt->rowCount();
            } catch (Exception $e) {}
        }
    }
    return $added;
}

function mpUpdateDomainQuality($pdo, $domain, $result, $linksFound = 0) {
    if (empty($domain)) return;
    $domain = mb_substr($domain, 0, 255);
    $imported = ($result === 'imported') ? 1 : 0;
    $errors = ($result === 'error') ? 1 : 0;
    $qualityDelta = $imported ? 2 : ($errors ? -3 : -1);
    $status = $errors ? 'error' : ($imported ? 'imported' : 'active');
    try {
        $stmt = $pdo->prepare("INSERT INTO crawler_domains (domain, total_crawled, total_imported, urls_errors, quality_score, links_found, last_status, last_crawled)
            VALUES (?, 1, ?, ?, GREATEST(0, LEAST(100, 50 + ?)), ?, ?, NOW())
            ON DUPLICATE KEY UPDATE
                total_crawled = total_crawled + 1,
                total_imported = total_imported + ?,
                urls_errors = urls_errors + ?,
                quality_score = GREATEST(0, LEAST(100, quality_score + ?)),
                links_found = links_found + ?,
                last_status = ?,
                last_crawled = NOW()");
        $stmt->execute([$domain, $imported, $errors, $qualityDelta, $linksFound, $status,
                        $imported, $errors, $qualityDelta, $linksFound, $status]);
    } catch (Exception $e) {}
}

function mpGetDomainQuality($pdo, $domain) {
    static $cache = [];
    if (isset($cache[$domain])) return $cache[$domain];
    try {
        $stmt = $pdo->prepare("SELECT quality_score, is_blocked FROM crawler_domains WHERE domain=? LIMIT 1");
        $stmt->execute([$domain]);
        $row = $stmt->fetch(PDO::FETCH_ASSOC);
        $result = $row ? ['quality' => (int)$row['quality_score'], 'blocked' => (bool)$row['is_blocked']] : ['quality' => 50, 'blocked' => false];
        if (count($cache) < 1000) $cache[$domain] = $result;
        return $result;
    } catch (Exception $e) { return ['quality' => 50, 'blocked' => false]; }
}

// --- MULTI-PROCESS IMPORT ---
function mpImportBatch($pdo, $records, $processId) {
    if (empty($records)) return 0;
    $columns = 'title, description, thumbnail, pubDate, link, updated, source_website, author, favicon, tags, embed_code, source_domain, user_id, source_domain_id, main_category_id, title_cat_id, description_cat_id, tag_cat_id';
    $inserted = 0;

    foreach (array_chunk($records, 25) as $chunk) {
        $placeholders = []; $values = [];
        foreach ($chunk as $data) {
            $link = $data['link'] ?? '';
            if (!filter_var($link, FILTER_VALIDATE_URL)) continue;
            $host = parse_url($link, PHP_URL_HOST) ?? '';
            $author = $data['author'] ?? '';
            if (empty($author) || $author === 'Anonymous' || mb_strlen($author) < 3) {
                $author = generateRandomAuthor();
            }
            $userId = mpGetOrCreateUser($pdo, $author);
            $domainData = mpGetOrCreateDomain($pdo, $link);
            $embedCode = extractEmbedCode($link);
            $tags = $data['tags'] ?? '';
            if (empty($tags) && !empty($data['title'])) $tags = extractTagsFromTitle($data['title']);

            $placeholders[] = '(?, ?, ?, NOW(), ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0)';
            $values[] = mb_substr($data['title'] ?? 'No title', 0, 255);
            $values[] = mb_substr($data['description'] ?? '', 0, 1000);
            $values[] = mb_substr($data['thumbnail'] ?? '', 0, 500);
            $values[] = $link;
            $values[] = mb_substr($host, 0, 255);
            $values[] = mb_substr($author, 0, 255);
            $values[] = mb_substr($data['favicon'] ?? '', 0, 255);
            $values[] = mb_substr($tags, 0, 500);
            $values[] = mb_substr($embedCode, 0, 2000);
            $values[] = mb_substr($host, 0, 255);
            $values[] = $userId;
            $values[] = $domainData['domain_id'];
            $values[] = $domainData['category_id'];
        }
        if (!empty($placeholders)) {
            try {
                $sql = "INSERT IGNORE INTO pinfeeds ($columns) VALUES " . implode(', ', $placeholders);
                $stmt = $pdo->prepare($sql);
                $stmt->execute($values);
                $inserted += $stmt->rowCount();
            } catch (Exception $e) {
                error_log("mpImportBatch error: " . $e->getMessage());
            }
        }
    }
    return $inserted;
}

// --- VIRAL WORKER (DB-based, no SSE, runs in background) ---
function runViralWorker($processId, $keywords, $config, $pdo) {
    ignore_user_abort(true);
    set_time_limit(0);
    ini_set('memory_limit', '512M');

    if (!mpRegisterProcess($pdo, $processId)) return;

    $waveSize = $config['wave_size'] ?? 25;
    $maxImports = $config['max_imports'] ?? 10000000;
    $linksPerPage = min($config['links_per_page'] ?? 150, 250);
    $includeTerms = $config['include_terms'] ?? [];
    $excludeTerms = $config['exclude_terms'] ?? [];
    $forcedDomains = $config['forced_domains'] ?? [];
    $commonTerm = $config['common_term'] ?? '';
    $MIN_RELEVANCE = 2;

    // Apply common term to keywords
    if (!empty($commonTerm)) {
        $keywords = array_map(function($kw) use ($commonTerm) {
            if (preg_match('/^https?:\/\//i', $kw)) return $kw;
            return $kw . ' ' . $commonTerm;
        }, $keywords);
    }

    $stats = ['discovered'=>0,'processed'=>0,'imported'=>0,'duplicates'=>0,'errors'=>0,'skipped'=>0,'pool_size'=>0,'waves'=>0,'rate'=>0,'status'=>'running'];
    $importQueue = [];
    $domainFailCount = [];
    $domainStats = [];
    $domainQuality = [];
    $domainBackoffUntil = [];
    $lastHeartbeat = time();
    $lastCleanup = time();
    $MAX_DOMAIN_FAILS = 4;
    $localSeenCache = [];

    // SEED PHASE: Search engines -> add URLs to shared pool
    $seedCount = 0;
    foreach ($keywords as $kw) {
        if (preg_match('/^https?:\/\//i', trim($kw))) {
            mpAddToPool($pdo, [trim($kw)], $processId, 90);
            $seedCount++;
            continue;
        }
        $engines = [
            'searxng' => function($q) { return getSearXNGLinks($q, 20); },
            'google' => function($q) { return getGoogleSearchLinks($q, 20); },
            'bing' => function($q) { return getBingSearchLinks($q, 15); },
            'duckduckgo' => function($q) { return getDuckDuckGoLinks($q, 15); },
            'yahoo' => function($q) { return getYahooSearchLinks($q, 10); },
            'brave' => function($q) { return getBraveSearchLinks($q, 15); },
        ];
        foreach ($engines as $eName => $eFn) {
            $links = $eFn($kw);
            $validLinks = [];
            foreach ($links as $link) {
                $host = parse_url($link, PHP_URL_HOST) ?? '';
                if (!isDomainSkipped($host)) $validLinks[] = $link;
            }
            if (!empty($validLinks)) {
                $seedCount += mpAddToPool($pdo, $validLinks, $processId, mpUrlPriority($validLinks[0]));
            }
            if ($seedCount >= 20) break;
        }
        if ($seedCount >= 50) break;
    }

    $stats['discovered'] = $seedCount;
    mpHeartbeat($pdo, $processId, $stats);
    mpWriteLog($pdo, $processId, [['url'=>'','status'=>'imported','title'=>"Seeded: $seedCount URLs",'domain'=>'SYSTEM']]);

    // MAIN VIRAL LOOP
    $reseedIndex = 0;
    while ($stats['imported'] < $maxImports) {
        // 1. CHECK CONTROL
        $status = mpCheckControl($pdo, $processId);
        if ($status === 'stopped' || $status === 'dead') break;
        while ($status === 'paused') {
            sleep(2);
            $status = mpCheckControl($pdo, $processId);
            if ($status === 'stopped') break 2;
        }

        // 2. HEARTBEAT (every 30s)
        if (time() - $lastHeartbeat >= MP_HEARTBEAT_SEC) {
            $elapsed = max(1, time() - ($config['_start_time'] ?? time()));
            $stats['rate'] = round($stats['imported'] / $elapsed, 2);
            mpHeartbeat($pdo, $processId, $stats);
            $lastHeartbeat = time();
        }

        // 3. CLEANUP (every 5 min)
        if (time() - $lastCleanup >= 300) {
            mpCleanupDead($pdo);
            $lastCleanup = time();
        }

        // 4. CLAIM URLs from shared pool
        $claimed = mpClaimUrls($pdo, $processId, $waveSize);
        if (empty($claimed) && empty($importQueue)) {
            // RE-SEED if pool is empty
            $reseedEngines = ['searxng','google','bing','duckduckgo','brave','yahoo'];
            $engine = $reseedEngines[$reseedIndex % count($reseedEngines)];
            $reseedIndex++;
            $kw = $keywords[array_rand($keywords)];
            $variations = ['', ' news', ' blog', ' article', ' site:.br'];
            $variedKw = trim($kw . ($variations[$reseedIndex % count($variations)] ?? ''));
            $newSeeds = [];
            switch ($engine) {
                case 'searxng': $newSeeds = getSearXNGLinks($variedKw, 20); break;
                case 'google': $newSeeds = getGoogleSearchLinks($variedKw, 20); break;
                case 'bing': $newSeeds = getBingSearchLinks($variedKw, 15); break;
                case 'duckduckgo': $newSeeds = getDuckDuckGoLinks($variedKw, 15); break;
                case 'brave': $newSeeds = getBraveSearchLinks($variedKw, 15); break;
                case 'yahoo': $newSeeds = getYahooSearchLinks($variedKw, 10); break;
            }
            $validSeeds = array_filter($newSeeds, function($l) { return !isDomainSkipped(parse_url($l, PHP_URL_HOST) ?? ''); });
            if (!empty($validSeeds)) {
                mpAddToPool($pdo, $validSeeds, $processId, 50);
                $stats['discovered'] += count($validSeeds);
            }
            usleep(500000);
            continue;
        }

        if (empty($claimed)) { usleep(200000); continue; }

        // 5. FETCH WAVE
        $wave = array_column($claimed, 'url');
        $waveIds = array_column($claimed, 'id');
        $stats['waves']++;

        $results = fetchBatchWithCurl($wave, 4, count($wave));
        $logEntries = [];

        foreach ($results as $url => $result) {
            $fetchDomain = parse_url($url, PHP_URL_HOST) ?? '';

            if ($result['http_code'] < 200 || $result['http_code'] >= 400 || empty($result['html'])) {
                $domainFailCount[$fetchDomain] = ($domainFailCount[$fetchDomain] ?? 0) + 1;
                if ($result['http_code'] >= 400 && $result['http_code'] < 500) {
                    $stats['skipped']++;
                } else {
                    $stats['errors']++;
                }
                mpMarkSeen($pdo, $url, $processId, 'error');
                continue;
            }

            // Reset fail count on success
            $domainFailCount[$fetchDomain] = max(0, ($domainFailCount[$fetchDomain] ?? 0) - 1);
            $html = $result['html'];
            $isSerpPage = preg_match('/google\.|bing\.|yahoo\.|yandex\.|duckduckgo\.|baidu\.|ecosia\.|brave\.|searx/i', $fetchDomain);

            $meta = extractMetadata($url, $html);

            // INCLUDE TERMS CHECK
            $passesInclude = true;
            if (!empty($includeTerms) && !$isSerpPage) {
                $passesInclude = false;
                $cleanBody = preg_replace('/<(script|style|noscript|nav|footer|header)[^>]*>.*?<\/\1>/si', '', $html);
                $bodyText = mb_substr(trim(preg_replace('/\s+/', ' ', strip_tags($cleanBody))), 0, 3000);
                $searchable = strtolower($url . ' ' . ($meta['title'] ?? '') . ' ' . ($meta['description'] ?? '') . ' ' . $bodyText);
                foreach ($includeTerms as $term) {
                    if (!empty($term) && stripos($searchable, $term) !== false) { $passesInclude = true; break; }
                }
            }

            // SPREAD LINKS (add to shared pool)
            if (($passesInclude || $isSerpPage) && !hasNoindexNofollow($html)['nofollow']) {
                if ($isSerpPage) {
                    $pageLinks = extractSearchResultsFromUrl($url, $html);
                    if (empty($pageLinks)) $pageLinks = extractAllLinks($html, $url, $linksPerPage, $includeTerms, $excludeTerms, $config);
                } else {
                    $pageLinks = extractAllLinks($html, $url, $linksPerPage, $includeTerms, $excludeTerms, $config);
                }
                // Filter and add to shared pool
                $newLinks = [];
                foreach ($pageLinks as $pl) {
                    $plHost = parse_url($pl, PHP_URL_HOST) ?? '';
                    if (isDomainSkipped($plHost)) continue;
                    if (($domainFailCount[$plHost] ?? 0) >= $MAX_DOMAIN_FAILS) continue;
                    if (preg_match('/\/wp-content\/|\/wp-includes\/|\/wp-admin\/|\/cdn-cgi\/|\/static\/css\/|\/static\/js\/|\/feed\/?$|\/xmlrpc|\/wp-json|^javascript:|^mailto:/i', $pl)) continue;
                    $newLinks[] = $pl;
                }
                if (!empty($newLinks)) {
                    // Check against shared seen table (batch)
                    $seenCheck = mpIsSeenBatch($pdo, array_slice($newLinks, 0, 200));
                    $unseenLinks = array_filter($newLinks, function($l) use ($seenCheck) { return !($seenCheck[$l] ?? false); });
                    if (!empty($unseenLinks)) {
                        mpAddToPool($pdo, array_slice($unseenLinks, 0, 200), $processId, mpUrlPriority($unseenLinks[0]));
                        $stats['discovered'] += count($unseenLinks);
                    }
                }
            }

            // IMPORT DECISION
            if ($passesInclude && !$isSerpPage) {
                // Noindex check
                if (hasNoindexNofollow($html)['noindex'] && !isForcedDomain($fetchDomain, $forcedDomains)) {
                    $stats['skipped']++;
                    mpMarkSeen($pdo, $url, $processId, 'noindex');
                    continue;
                }
                // Canonical dedup
                $canonicalUrl = getCanonicalUrl($html, $url);
                $canonicalHash = md5($canonicalUrl);
                if (isset($localSeenCache[$canonicalHash]) && $canonicalUrl !== $url) {
                    $stats['duplicates']++;
                    mpMarkSeen($pdo, $url, $processId, 'duplicate', $canonicalUrl);
                    continue;
                }
                $localSeenCache[$canonicalHash] = true;
                if (count($localSeenCache) > 20000) $localSeenCache = array_slice($localSeenCache, -10000, null, true);

                $quality = scoreContentQuality($url, $html, $meta);
                $relevance = calculateRelevanceScore($html, $keywords, $url);
                $isForced = isForcedDomain($fetchDomain, $forcedDomains);

                if ($isForced || ($quality >= 20 && ($relevance >= $MIN_RELEVANCE || $quality >= 60))) {
                    $importQueue[] = array_merge(['link' => $canonicalUrl], $meta);
                    $domainQuality[$fetchDomain] = min(100, ($domainQuality[$fetchDomain] ?? 50) + 2);
                    mpMarkSeen($pdo, $url, $processId, 'imported', $canonicalUrl);
                    $logEntries[] = ['url'=>$url,'status'=>'imported','title'=>$meta['title']??'','domain'=>$fetchDomain];
                } else {
                    $stats['skipped']++;
                    mpMarkSeen($pdo, $url, $processId, 'skipped', $canonicalUrl);
                    $domainQuality[$fetchDomain] = max(0, ($domainQuality[$fetchDomain] ?? 50) - 2);
                }
            } else {
                $stats['skipped']++;
                mpMarkSeen($pdo, $url, $processId, 'skipped');
            }
        }

        // Mark wave URLs as done
        mpMarkDone($pdo, $waveIds);
        $stats['processed'] += count($wave);

        // 6. IMPORT QUEUE (micro-batch)
        if (count($importQueue) >= 10) {
            $imported = mpImportBatch($pdo, $importQueue, $processId);
            $stats['imported'] += $imported;
            $stats['duplicates'] += (count($importQueue) - $imported);
            $importQueue = [];
        }

        // 7. WRITE LOG
        if (!empty($logEntries)) {
            mpWriteLog($pdo, $processId, $logEntries);
        }

        // Update pool size estimate
        $poolCount = $pdo->query("SELECT COUNT(*) FROM crawler_pool WHERE status='pending'")->fetchColumn();
        $stats['pool_size'] = (int)$poolCount;
    }

    // Final: flush remaining imports
    if (!empty($importQueue)) {
        $stats['imported'] += mpImportBatch($pdo, $importQueue, $processId);
    }

    // Mark process as completed
    $elapsed = max(1, time() - ($config['_start_time'] ?? time()));
    $stats['rate'] = round($stats['imported'] / $elapsed, 2);
    mpHeartbeat($pdo, $processId, $stats);
    $pdo->prepare("UPDATE crawler_processes SET status='completed', stopped_at=NOW() WHERE id=?")->execute([$processId]);
    mpWriteLog($pdo, $processId, [['url'=>'','status'=>'completed','title'=>'Process completed. Imported: '.$stats['imported'],'domain'=>'SYSTEM']]);
}

// --- ULTIMATE WORKER (Enhanced: All Engines + Quality + Edges + Pagination) ---
function runUltimateWorker($processId, $keywords, $config, $pdo) {
    ignore_user_abort(true);
    set_time_limit(0);
    ini_set('memory_limit', '512M');

    if (!mpRegisterProcess($pdo, $processId)) return;

    $waveSize = min(100, max(1, $config['wave_size'] ?? 25));
    $maxImports = $config['max_imports'] ?? 10000000;
    $linksPerPage = min($config['links_per_page'] ?? 150, 250);
    $includeTerms = $config['include_terms'] ?? [];
    $excludeTerms = $config['exclude_terms'] ?? [];
    $forcedDomains = $config['forced_domains'] ?? [];
    $commonTerm = $config['common_term'] ?? '';
    $qualityThreshold = max(0, min(100, $config['quality_threshold'] ?? 20));
    $relevanceThreshold = max(0, $config['relevance_threshold'] ?? 2);
    $maxDepth = max(1, min(10, $config['max_depth'] ?? 5));
    $enablePagination = $config['enable_pagination'] ?? true;
    $enabledEngines = $config['engines'] ?? ['searxng','google','bing','bing_regional','duckduckgo','yahoo','yandex','brave','direct_sites','wikipedia','baidu'];
    $MAX_DOMAIN_FAILS = 4;
    $JUNK_THRESHOLD = 12;
    // v2.0 enhancements
    $urlPatternFilter = trim($config['url_pattern_filter'] ?? '');
    $httpAuth = '';
    if (!empty($config['http_auth_user'])) {
        $httpAuth = $config['http_auth_user'] . ':' . ($config['http_auth_pass'] ?? '');
    }
    $fetchDelayMs = max(0, min(5000, (int)($config['fetch_delay_ms'] ?? 0)));
    $totalFetchTime = 0;
    $fetchCount = 0;

    if (!empty($commonTerm)) {
        $keywords = array_map(function($kw) use ($commonTerm) {
            if (preg_match('/^https?:\/\//i', $kw)) return $kw;
            return $kw . ' ' . $commonTerm;
        }, $keywords);
    }

    $stats = ['discovered'=>0,'processed'=>0,'imported'=>0,'duplicates'=>0,'errors'=>0,'skipped'=>0,'pool_size'=>0,'waves'=>0,'rate'=>0,'status'=>'running'];
    $importQueue = [];
    $edgeQueue = [];
    $domainFailCount = [];
    $domainCrawlCount = [];
    $localSeenCache = [];
    $lastHeartbeat = time();
    $lastCleanup = time();
    $lastEdgeFlush = time();
    $startTime = time();

    // ALL 11 ENGINES for seeding
    $allEngines = [
        'searxng' => function($q) { return getSearXNGLinks($q, 20); },
        'google' => function($q) { return getGoogleSearchLinks($q, 20); },
        'bing' => function($q) { return getBingSearchLinks($q, 15); },
        'bing_regional' => function($q) { return getBingRegionalLinks($q, 20); },
        'duckduckgo' => function($q) { return getDuckDuckGoLinks($q, 15); },
        'yahoo' => function($q) { return getYahooSearchLinks($q, 15); },
        'yandex' => function($q) { return getYandexSearchLinks($q, 15); },
        'brave' => function($q) { return getBraveSearchLinks($q, 15); },
        'direct_sites' => function($q) { return getDirectSearchLinks($q, 15); },
        'wikipedia' => function($q) { return getWikipediaLinks($q, 15); },
        'baidu' => function($q) { return getBaiduSearchLinks($q, 15); },
    ];
    $activeEngines = array_intersect_key($allEngines, array_flip($enabledEngines));

    // SEED PHASE: All engines + seed URLs
    $seedCount = 0;
    foreach ($keywords as $kw) {
        if (preg_match('/^https?:\/\//i', trim($kw))) {
            mpAddToPool($pdo, [trim($kw)], $processId, 90);
            $seedCount++;
            continue;
        }
        foreach ($activeEngines as $eName => $eFn) {
            try {
                $links = $eFn($kw);
                $validLinks = array_filter($links, function($l) { return !isDomainSkipped(parse_url($l, PHP_URL_HOST) ?? ''); });
                if (!empty($validLinks)) {
                    $seedCount += mpAddToPool($pdo, array_values($validLinks), $processId, mpUrlPriority($validLinks[array_key_first($validLinks)]));
                }
            } catch (Exception $e) {}
            if ($seedCount >= 30) break;
        }
        if ($seedCount >= 80) break;
    }

    // Pagination seed: if pattern provided
    if ($enablePagination && !empty($config['pagination_pattern'])) {
        $pagStart = max(1, $config['start_page'] ?? 1);
        $pagEnd = min(200, $config['end_page'] ?? 50);
        $pagUrls = [];
        for ($i = $pagStart; $i <= $pagEnd; $i++) {
            $pagUrls[] = preg_replace('/\d+/', $i, $config['pagination_pattern'], 1);
        }
        if (!empty($pagUrls)) {
            $seedCount += mpAddToPool($pdo, $pagUrls, $processId, 70);
        }
    }

    $stats['discovered'] = $seedCount;
    $config['_start_time'] = $startTime;
    mpHeartbeat($pdo, $processId, $stats);
    mpWriteLog($pdo, $processId, [['url'=>'','status'=>'imported','title'=>"ULTIMATE Seeded: $seedCount URLs via ".count($activeEngines)." engines",'domain'=>'SYSTEM']]);

    // MAIN ULTIMATE LOOP
    $reseedIndex = 0;
    $engineKeys = array_keys($activeEngines);
    while ($stats['imported'] < $maxImports) {
        // 1. CONTROL CHECK
        $status = mpCheckControl($pdo, $processId);
        if ($status === 'stopped' || $status === 'dead') break;
        while ($status === 'paused') {
            sleep(2);
            $status = mpCheckControl($pdo, $processId);
            if ($status === 'stopped') break 2;
        }

        // 2. HEARTBEAT (every 30s)
        if (time() - $lastHeartbeat >= MP_HEARTBEAT_SEC) {
            $elapsed = max(1, time() - $startTime);
            $stats['rate'] = round($stats['imported'] / $elapsed, 2);
            $stats['avg_fetch_ms'] = $fetchCount > 0 ? round(($totalFetchTime / $fetchCount) * 1000) : 0;
            mpHeartbeat($pdo, $processId, $stats);
            $lastHeartbeat = time();
        }

        // 3. CLEANUP (every 5 min)
        if (time() - $lastCleanup >= 300) {
            mpCleanupDead($pdo);
            $lastCleanup = time();
        }

        // 4. FLUSH EDGES (every 30s)
        if (!empty($edgeQueue) && time() - $lastEdgeFlush >= 30) {
            mpStoreEdges($pdo, $edgeQueue, $processId);
            $edgeQueue = [];
            $lastEdgeFlush = time();
        }

        // 5. CLAIM URLs from shared pool
        $claimed = mpClaimUrls($pdo, $processId, $waveSize);
        if (empty($claimed) && empty($importQueue)) {
            // RE-SEED with cycling through ALL engines + variations
            $engine = $engineKeys[$reseedIndex % count($engineKeys)];
            $reseedIndex++;
            $kw = $keywords[array_rand($keywords)];
            $variations = ['', ' news', ' blog', ' article', ' tutorial', ' site:.br', ' site:.com', ' latest', ' 2025', ' review'];
            $variedKw = trim(preg_replace('/\s+' . preg_quote($commonTerm, '/') . '\s*$/i', '', $kw) . ' ' . ($variations[$reseedIndex % count($variations)] ?? ''));
            if (!empty($commonTerm)) $variedKw .= ' ' . $commonTerm;
            $newSeeds = [];
            if (isset($activeEngines[$engine])) {
                try { $newSeeds = $activeEngines[$engine]($variedKw); } catch (Exception $e) {}
            }
            $validSeeds = array_filter($newSeeds, function($l) { return !isDomainSkipped(parse_url($l, PHP_URL_HOST) ?? ''); });
            if (!empty($validSeeds)) {
                mpAddToPool($pdo, array_values($validSeeds), $processId, 50);
                $stats['discovered'] += count($validSeeds);
            }
            usleep(500000);
            continue;
        }

        if (empty($claimed)) { usleep(200000); continue; }

        // 6. FETCH WAVE
        $wave = array_column($claimed, 'url');
        $waveIds = array_column($claimed, 'id');
        $stats['waves']++;

        // Pre-filter: skip domains with low quality or too many fails
        $filteredWave = [];
        $filteredIds = [];
        foreach ($wave as $idx => $wUrl) {
            // URL Pattern Filter: skip URLs not matching user regex
            if (!empty($urlPatternFilter)) {
                $patternToUse = $urlPatternFilter;
                if (@preg_match($patternToUse, '') === false) {
                    $patternToUse = '/' . preg_quote($urlPatternFilter, '/') . '/i';
                }
                if (!@preg_match($patternToUse, $wUrl)) {
                    $stats['skipped']++;
                    continue;
                }
            }
            $wHost = parse_url($wUrl, PHP_URL_HOST) ?? '';
            if (($domainFailCount[$wHost] ?? 0) >= $MAX_DOMAIN_FAILS) {
                $stats['skipped']++;
                continue;
            }
            $dq = mpGetDomainQuality($pdo, $wHost);
            if ($dq['blocked'] || $dq['quality'] < 10) {
                $stats['skipped']++;
                continue;
            }
            $filteredWave[] = $wUrl;
            $filteredIds[] = $waveIds[$idx];
        }

        if (empty($filteredWave)) {
            mpMarkDone($pdo, $waveIds, 'done');
            continue;
        }

        $fetchStart = microtime(true);
        $results = fetchBatchWithCurl($filteredWave, 4, count($filteredWave), $httpAuth);
        $fetchElapsed = microtime(true) - $fetchStart;
        $totalFetchTime += $fetchElapsed;
        $fetchCount++;
        if ($fetchDelayMs > 0) usleep($fetchDelayMs * 1000);
        $logEntries = [];

        foreach ($results as $url => $result) {
            $fetchDomain = parse_url($url, PHP_URL_HOST) ?? '';

            if ($result['http_code'] < 200 || $result['http_code'] >= 400 || empty($result['html'])) {
                $domainFailCount[$fetchDomain] = ($domainFailCount[$fetchDomain] ?? 0) + 1;
                if ($result['http_code'] == 429) {
                    $domainFailCount[$fetchDomain] = $MAX_DOMAIN_FAILS; // Instant blacklist on 429
                }
                $stats[$result['http_code'] >= 400 && $result['http_code'] < 500 ? 'skipped' : 'errors']++;
                mpMarkSeen($pdo, $url, $processId, 'error');
                mpUpdateDomainQuality($pdo, $fetchDomain, 'error', 0);
                continue;
            }

            // Success: reduce fail count
            $domainFailCount[$fetchDomain] = max(0, ($domainFailCount[$fetchDomain] ?? 0) - 1);
            $domainCrawlCount[$fetchDomain] = ($domainCrawlCount[$fetchDomain] ?? 0) + 1;
            $html = $result['html'];
            $isSerpPage = preg_match('/google\.|bing\.|yahoo\.|yandex\.|duckduckgo\.|baidu\.|ecosia\.|brave\.|searx/i', $fetchDomain);

            $meta = extractMetadata($url, $html);

            // INCLUDE TERMS CHECK
            $passesInclude = true;
            if (!empty($includeTerms) && !$isSerpPage) {
                $passesInclude = false;
                $cleanBody = preg_replace('/<(script|style|noscript|nav|footer|header)[^>]*>.*?<\/\1>/si', '', $html);
                $bodyText = mb_substr(trim(preg_replace('/\s+/', ' ', strip_tags($cleanBody))), 0, 3000);
                $searchable = strtolower($url . ' ' . ($meta['title'] ?? '') . ' ' . ($meta['description'] ?? '') . ' ' . $bodyText);
                foreach ($includeTerms as $term) {
                    if (!empty($term) && stripos($searchable, $term) !== false) { $passesInclude = true; break; }
                }
            }

            // LINK SPREADING + EDGE COLLECTION
            $nofollow = hasNoindexNofollow($html)['nofollow'];
            if (($passesInclude || $isSerpPage) && !$nofollow) {
                if ($isSerpPage) {
                    $pageLinks = extractSearchResultsFromUrl($url, $html);
                    if (empty($pageLinks)) $pageLinks = extractAllLinks($html, $url, $linksPerPage, $includeTerms, $excludeTerms, $config);
                } else {
                    $pageLinks = extractAllLinks($html, $url, $linksPerPage, $includeTerms, $excludeTerms, $config);
                }

                $newLinks = [];
                foreach ($pageLinks as $pl) {
                    $plHost = parse_url($pl, PHP_URL_HOST) ?? '';
                    if (isDomainSkipped($plHost)) continue;
                    if (($domainFailCount[$plHost] ?? 0) >= $MAX_DOMAIN_FAILS) continue;
                    if (preg_match('/\/wp-content\/|\/wp-includes\/|\/wp-admin\/|\/cdn-cgi\/|\/static\/css\/|\/static\/js\/|\/feed\/?$|\/xmlrpc|\/wp-json|^javascript:|^mailto:/i', $pl)) continue;
                    $newLinks[] = $pl;
                    // Collect edges
                    if ($plHost !== $fetchDomain && !$isSerpPage) {
                        $edgeQueue[] = [$fetchDomain, $plHost];
                    }
                }

                // PAGINATION DISCOVERY: detect paginated URLs
                if ($enablePagination && !$isSerpPage) {
                    foreach ($pageLinks as $pl) {
                        if (preg_match('/[?&]page=(\d+)|\/page\/(\d+)|\/p\/(\d+)|[?&]p=(\d+)/i', $pl, $pm)) {
                            $pageNum = (int)($pm[1] ?: ($pm[2] ?: ($pm[3] ?: $pm[4])));
                            if ($pageNum > 1 && $pageNum <= 50) {
                                // Generate adjacent pages
                                for ($pg = max(1, $pageNum - 2); $pg <= min($pageNum + 5, 50); $pg++) {
                                    $pagUrl = preg_replace('/([?&]page=|\/page\/|\/p\/|[?&]p=)\d+/i', '${1}' . $pg, $pl, 1);
                                    if ($pagUrl !== $pl) $newLinks[] = $pagUrl;
                                }
                                break; // Only discover pagination once per page
                            }
                        }
                    }
                }

                if (!empty($newLinks)) {
                    $seenCheck = mpIsSeenBatch($pdo, array_slice($newLinks, 0, 200));
                    $unseenLinks = array_filter($newLinks, function($l) use ($seenCheck) { return !($seenCheck[$l] ?? false); });
                    if (!empty($unseenLinks)) {
                        mpAddToPool($pdo, array_slice(array_values($unseenLinks), 0, 200), $processId, mpUrlPriority($unseenLinks[array_key_first($unseenLinks)]));
                        $stats['discovered'] += count($unseenLinks);
                    }
                }

                mpUpdateDomainQuality($pdo, $fetchDomain, 'active', count($pageLinks));
            }

            // IMPORT DECISION
            if ($passesInclude && !$isSerpPage) {
                $noindex = hasNoindexNofollow($html)['noindex'];
                if ($noindex && !isForcedDomain($fetchDomain, $forcedDomains)) {
                    $stats['skipped']++;
                    mpMarkSeen($pdo, $url, $processId, 'noindex');
                    mpUpdateDomainQuality($pdo, $fetchDomain, 'skipped', 0);
                    continue;
                }

                $canonicalUrl = getCanonicalUrl($html, $url);
                $canonicalHash = md5($canonicalUrl);
                if (isset($localSeenCache[$canonicalHash]) && $canonicalUrl !== $url) {
                    $stats['duplicates']++;
                    mpMarkSeen($pdo, $url, $processId, 'duplicate', $canonicalUrl);
                    continue;
                }
                $localSeenCache[$canonicalHash] = true;
                if (count($localSeenCache) > 20000) $localSeenCache = array_slice($localSeenCache, -10000, null, true);

                $quality = scoreContentQuality($url, $html, $meta);
                $relevance = calculateRelevanceScore($html, $keywords, $url);
                $isForced = isForcedDomain($fetchDomain, $forcedDomains);

                // JUNK detection: domain crawled many times with 0 imports
                $domainImports = $domainCrawlCount[$fetchDomain] ?? 0;
                $isJunk = (!$isForced && $domainImports >= $JUNK_THRESHOLD && ($stats['imported'] == 0 || $domainCrawlCount[$fetchDomain] > $JUNK_THRESHOLD));

                if (!$isJunk && ($isForced || ($quality >= $qualityThreshold && ($relevance >= $relevanceThreshold || $quality >= 60)))) {
                    $importQueue[] = array_merge(['link' => $canonicalUrl], $meta);
                    mpMarkSeen($pdo, $url, $processId, 'imported', $canonicalUrl);
                    mpUpdateDomainQuality($pdo, $fetchDomain, 'imported', 0);
                    $logEntries[] = ['url'=>$url,'status'=>'imported','title'=>$meta['title']??'','domain'=>$fetchDomain];
                } else {
                    $stats['skipped']++;
                    mpMarkSeen($pdo, $url, $processId, 'skipped', $canonicalUrl);
                    mpUpdateDomainQuality($pdo, $fetchDomain, 'skipped', 0);
                }
            } else {
                $stats['skipped']++;
                mpMarkSeen($pdo, $url, $processId, 'skipped');
            }
        }

        // Mark wave as done
        mpMarkDone($pdo, $waveIds);
        $stats['processed'] += count($filteredWave);

        // 7. IMPORT QUEUE (micro-batch)
        if (count($importQueue) >= 10) {
            $imported = mpImportBatch($pdo, $importQueue, $processId);
            $stats['imported'] += $imported;
            $stats['duplicates'] += (count($importQueue) - $imported);
            $importQueue = [];
        }

        // 8. WRITE LOG
        if (!empty($logEntries)) {
            mpWriteLog($pdo, $processId, $logEntries);
        }

        // Update pool size
        $poolCount = $pdo->query("SELECT COUNT(*) FROM crawler_pool WHERE status='pending'")->fetchColumn();
        $stats['pool_size'] = (int)$poolCount;
    }

    // Final flush
    if (!empty($importQueue)) {
        $stats['imported'] += mpImportBatch($pdo, $importQueue, $processId);
    }
    if (!empty($edgeQueue)) {
        mpStoreEdges($pdo, $edgeQueue, $processId);
    }

    $elapsed = max(1, time() - $startTime);
    $stats['rate'] = round($stats['imported'] / $elapsed, 2);
    mpHeartbeat($pdo, $processId, $stats);
    $pdo->prepare("UPDATE crawler_processes SET status='completed', stopped_at=NOW() WHERE id=?")->execute([$processId]);
    mpWriteLog($pdo, $processId, [['url'=>'','status'=>'completed','title'=>'ULTIMATE completed. Imported: '.$stats['imported'].' in '.$elapsed.'s','domain'=>'SYSTEM']]);
}

// --- SSE MULTI-PROCESS AGGREGATOR ---
function runMultiProcessSSE($pdo) {
    ignore_user_abort(true);
    header('Content-Type: text/event-stream');
    header('Cache-Control: no-cache');
    header('Connection: keep-alive');
    header('X-Accel-Buffering: no');
    ob_implicit_flush(true);
    if (ob_get_level()) ob_end_clean();

    while (!connection_aborted()) {
        // Get all active processes
        $procs = $pdo->query("SELECT id, mode, status, keywords, discovered, processed, imported, duplicates, errors, skipped, pool_size, waves, rate, started_at, last_heartbeat, created_at FROM crawler_processes WHERE status IN ('running','paused','pending') ORDER BY created_at DESC LIMIT 200")->fetchAll(PDO::FETCH_ASSOC);

        // Also get recently completed/stopped (last 5 min)
        $recent = $pdo->query("SELECT id, mode, status, discovered, processed, imported, duplicates, errors, skipped, pool_size, waves, rate, started_at, stopped_at, created_at FROM crawler_processes WHERE status IN ('completed','stopped','dead') AND stopped_at > DATE_SUB(NOW(), INTERVAL 5 MINUTE) ORDER BY stopped_at DESC LIMIT 50")->fetchAll(PDO::FETCH_ASSOC);

        $agg = ['total_running'=>0, 'total_imported'=>0, 'total_discovered'=>0, 'total_processed'=>0, 'total_duplicates'=>0, 'total_errors'=>0, 'total_rate'=>0, 'processes'=>[], 'recent'=>$recent];
        foreach ($procs as $p) {
            if ($p['status'] === 'running') $agg['total_running']++;
            $agg['total_imported'] += $p['imported'];
            $agg['total_discovered'] += $p['discovered'];
            $agg['total_processed'] += $p['processed'];
            $agg['total_duplicates'] += $p['duplicates'];
            $agg['total_errors'] += $p['errors'];
            $agg['total_rate'] += (float)$p['rate'];

            // Get last 5 log entries for this process
            $logStmt = $pdo->prepare("SELECT url, status, title, domain FROM crawler_log WHERE process_id=? ORDER BY created_at DESC LIMIT 5");
            $logStmt->execute([$p['id']]);
            $p['recent_log'] = $logStmt->fetchAll(PDO::FETCH_ASSOC);
            $p['elapsed'] = $p['started_at'] ? max(1, time() - strtotime($p['started_at'])) : 0;
            $agg['processes'][] = $p;
        }

        // Pool + seen stats
        $agg['pool'] = [
            'pending' => (int)$pdo->query("SELECT COUNT(*) FROM crawler_pool WHERE status='pending'")->fetchColumn(),
            'claimed' => (int)$pdo->query("SELECT COUNT(*) FROM crawler_pool WHERE status='claimed'")->fetchColumn()
        ];
        $agg['seen_total'] = (int)$pdo->query("SELECT COUNT(*) FROM crawler_seen")->fetchColumn();

        echo "data: " . json_encode($agg) . "\n\n";
        if (ob_get_level()) ob_flush();
        flush();

        sleep(2);
    }
}

// --- SSE ULTIMATE AGGREGATOR (Enhanced: Network + Domain Quality + Leaderboard) ---
function runUltimateSSE($pdo) {
    ignore_user_abort(true);
    @ini_set('zlib.output_compression', 0);
    @ini_set('output_buffering', 0);
    header('Content-Type: text/event-stream');
    header('Cache-Control: no-cache');
    header('Connection: keep-alive');
    header('X-Accel-Buffering: no');
    header('Content-Encoding: none');
    while (ob_get_level()) ob_end_clean();
    ob_implicit_flush(true);

    $lastEdgeId = 0;
    $padding = "\n: " . str_repeat(' ', 8192) . "\n"; // flush padding for proxy buffers

    while (!connection_aborted()) {
        // Get all active processes
        $procs = $pdo->query("SELECT id, mode, status, keywords, discovered, processed, imported, duplicates, errors, skipped, pool_size, waves, rate, started_at, last_heartbeat, created_at FROM crawler_processes WHERE status IN ('running','paused','pending') AND mode='ultimate' ORDER BY created_at DESC LIMIT 200")->fetchAll(PDO::FETCH_ASSOC);

        // Recently completed
        $recent = $pdo->query("SELECT id, mode, status, discovered, processed, imported, duplicates, errors, skipped, pool_size, waves, rate, started_at, stopped_at FROM crawler_processes WHERE status IN ('completed','stopped','dead') AND mode='ultimate' AND stopped_at > DATE_SUB(NOW(), INTERVAL 5 MINUTE) ORDER BY stopped_at DESC LIMIT 50")->fetchAll(PDO::FETCH_ASSOC);

        $agg = ['total_running'=>0, 'total_imported'=>0, 'total_discovered'=>0, 'total_processed'=>0, 'total_duplicates'=>0, 'total_errors'=>0, 'total_skipped'=>0, 'total_rate'=>0, 'processes'=>[], 'recent'=>$recent];
        foreach ($procs as $p) {
            if ($p['status'] === 'running') $agg['total_running']++;
            $agg['total_imported'] += (int)$p['imported'];
            $agg['total_discovered'] += (int)$p['discovered'];
            $agg['total_processed'] += (int)$p['processed'];
            $agg['total_duplicates'] += (int)$p['duplicates'];
            $agg['total_errors'] += (int)$p['errors'];
            $agg['total_skipped'] += (int)$p['skipped'];
            $agg['total_rate'] += (float)$p['rate'];

            $logStmt = $pdo->prepare("SELECT url, status, title, domain FROM crawler_log WHERE process_id=? ORDER BY created_at DESC LIMIT 5");
            $logStmt->execute([$p['id']]);
            $p['recent_log'] = $logStmt->fetchAll(PDO::FETCH_ASSOC);
            $p['elapsed'] = $p['started_at'] ? max(1, time() - strtotime($p['started_at'])) : 0;
            $agg['processes'][] = $p;
        }

        // Pool + seen stats
        $agg['pool'] = [
            'pending' => (int)$pdo->query("SELECT COUNT(*) FROM crawler_pool WHERE status='pending'")->fetchColumn(),
            'claimed' => (int)$pdo->query("SELECT COUNT(*) FROM crawler_pool WHERE status='claimed'")->fetchColumn()
        ];
        $agg['seen_total'] = (int)$pdo->query("SELECT COUNT(*) FROM crawler_seen")->fetchColumn();

        // NETWORK EDGES (incremental: only new since last query)
        $edgeStmt = $pdo->prepare("SELECT id, source_domain, target_domain FROM crawler_edges WHERE id > ? ORDER BY id ASC LIMIT 100");
        $edgeStmt->execute([$lastEdgeId]);
        $newEdges = $edgeStmt->fetchAll(PDO::FETCH_ASSOC);
        $agg['network'] = [];
        foreach ($newEdges as $e) {
            $agg['network'][] = [$e['source_domain'], $e['target_domain']];
            $lastEdgeId = max($lastEdgeId, (int)$e['id']);
        }
        $agg['total_edges'] = (int)$pdo->query("SELECT COUNT(*) FROM crawler_edges")->fetchColumn();
        $agg['total_nodes'] = (int)$pdo->query("SELECT COUNT(DISTINCT domain) FROM crawler_domains WHERE total_crawled > 0")->fetchColumn();

        // DOMAIN LEADERBOARD (top 50 by imports)
        $domStmt = $pdo->query("SELECT domain, total_crawled as crawled, total_imported as imported, urls_errors as errors, quality_score as quality, links_found, last_status as status FROM crawler_domains WHERE total_crawled > 0 ORDER BY total_imported DESC LIMIT 50");
        $agg['domain_stats'] = $domStmt->fetchAll(PDO::FETCH_ASSOC);

        echo "data: " . json_encode($agg) . "\n\n" . $padding;
        if (ob_get_level()) ob_flush();
        flush();

        sleep(2);
    }
}

// ============================================
// ROTATING USER-AGENTS (15+ diverse agents)
// ============================================
function getRotatingUserAgents() {
    static $agents = null;
    if ($agents === null) {
        $agents = [
            // Chrome Windows
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
            // Chrome Mac
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
            // Chrome Linux
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            // Firefox Windows
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
            // Firefox Mac
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
            // Safari Mac
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
            // Edge Windows
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0',
            // Mobile (iPhone)
            'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1',
            // Mobile (Android)
            'Mozilla/5.0 (Linux; Android 14; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.210 Mobile Safari/537.36',
            'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36',
        ];
    }
    return $agents;
}

// ============================================
// CANONICAL URL DETECTION
// ============================================
function getCanonicalUrl($html, $currentUrl) {
    if (empty($html)) return $currentUrl;
    $head = substr($html, 0, 50000);
    // <link rel="canonical" href="...">
    if (preg_match('/<link[^>]+rel=["\']canonical["\'][^>]*href=["\']([^"\']+)/i', $head, $m)) {
        $canonical = trim($m[1]);
        if (filter_var($canonical, FILTER_VALIDATE_URL)) {
            return $canonical;
        }
    }
    // Also check: href before rel
    if (preg_match('/<link[^>]+href=["\']([^"\']+)["\'][^>]*rel=["\']canonical/i', $head, $m)) {
        $canonical = trim($m[1]);
        if (filter_var($canonical, FILTER_VALIDATE_URL)) {
            return $canonical;
        }
    }
    // og:url as fallback canonical
    if (preg_match('/property=["\']og:url["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $canonical = trim($m[1]);
        if (filter_var($canonical, FILTER_VALIDATE_URL)) {
            return $canonical;
        }
    }
    return $currentUrl;
}

// ============================================
// META NOINDEX/NOFOLLOW DETECTION
// ============================================
function hasNoindexNofollow($html) {
    if (empty($html)) return ['noindex' => false, 'nofollow' => false];
    $head = substr($html, 0, 30000);
    $noindex = false;
    $nofollow = false;
    // Check <meta name="robots" content="noindex, nofollow">
    if (preg_match('/<meta[^>]+name=["\']robots["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $content = strtolower($m[1]);
        $noindex = (strpos($content, 'noindex') !== false);
        $nofollow = (strpos($content, 'nofollow') !== false);
    }
    // Also check content before name
    if (!$noindex && preg_match('/<meta[^>]+content=["\']([^"\']+)["\'][^>]*name=["\']robots/i', $head, $m)) {
        $content = strtolower($m[1]);
        $noindex = (strpos($content, 'noindex') !== false);
        $nofollow = (strpos($content, 'nofollow') !== false);
    }
    // Check googlebot-specific (more restrictive)
    if (preg_match('/<meta[^>]+name=["\']googlebot["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $content = strtolower($m[1]);
        if (strpos($content, 'noindex') !== false) $noindex = true;
        if (strpos($content, 'nofollow') !== false) $nofollow = true;
    }
    return ['noindex' => $noindex, 'nofollow' => $nofollow];
}

// ============================================
// WEIGHTED RELEVANCE SCORING (term-based)
// ============================================
function calculateRelevanceScore($html, $searchTerms, $url = '') {
    if (empty($html) || empty($searchTerms)) return 0;

    $score = 0;
    $head = substr($html, 0, 50000);

    // Extract sections with different weights
    $title = '';
    if (preg_match('/<title[^>]*>([^<]+)<\/title>/si', $head, $m)) {
        $title = strtolower(trim(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8')));
    }

    $description = '';
    if (preg_match('/name=["\']description["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $description = strtolower(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8'));
    }

    $h1Text = '';
    if (preg_match_all('/<h1[^>]*>(.*?)<\/h1>/si', $html, $m)) {
        $h1Text = strtolower(implode(' ', array_map('strip_tags', $m[1])));
    }

    $h2Text = '';
    if (preg_match_all('/<h2[^>]*>(.*?)<\/h2>/si', substr($html, 0, 100000), $m)) {
        $h2Text = strtolower(implode(' ', array_map('strip_tags', $m[1])));
    }

    // Body text (cleaned, first 5000 chars)
    $cleanBody = preg_replace('/<(script|style|noscript|nav|footer|header|aside)[^>]*>.*?<\/\1>/si', '', substr($html, 0, 80000));
    $bodyText = strtolower(mb_substr(trim(preg_replace('/\s+/', ' ', strip_tags($cleanBody))), 0, 5000));

    $urlLower = strtolower($url);

    // Calculate score for each search term
    foreach ($searchTerms as $term) {
        $term = strtolower(trim($term));
        if (empty($term) || preg_match('/^https?:\/\//i', $term)) continue;

        // Remove common_term suffix for matching (match individual words)
        $termWords = array_filter(explode(' ', $term));

        foreach ($termWords as $word) {
            if (mb_strlen($word) < 3) continue; // skip short words

            // WEIGHTED SCORING:
            // Title: 4x weight
            if (strpos($title, $word) !== false) $score += 4;
            // Description: 3x weight
            if (strpos($description, $word) !== false) $score += 3;
            // H1: 2x weight
            if (strpos($h1Text, $word) !== false) $score += 2;
            // H2: 1.5x weight (rounded to 2 for int)
            if (strpos($h2Text, $word) !== false) $score += 2;
            // URL: 2x weight
            if (strpos($urlLower, $word) !== false) $score += 2;
            // Body: 1x weight
            if (strpos($bodyText, $word) !== false) $score += 1;
        }
    }

    return $score;
}

// ============================================
// FORCED DOMAINS CHECK
// ============================================
function isForcedDomain($host, $forcedDomains) {
    if (empty($forcedDomains)) return false;
    $host = strtolower($host);
    foreach ($forcedDomains as $fd) {
        $fd = strtolower(trim($fd));
        if (empty($fd)) continue;
        if ($host === $fd || substr($host, -(strlen($fd) + 1)) === '.' . $fd) {
            return true;
        }
    }
    return false;
}

// ============================================
// SEARCH ENGINE FUNCTIONS
// ============================================
function getBingSearchLinks($query, $numResults = 50) {
    $links = [];
    $pages = ceil($numResults / 10);

    for ($page = 0; $page < $pages; $page++) {
        $offset = $page * 10;
        $url = "https://www.bing.com/search?q=" . urlencode($query) . "&first=" . ($offset + 1);

        $html = fetchWithCurl($url);
        if (!$html) continue;

        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches);
        foreach ($matches[1] as $link) {
            if (!preg_match('/bing\.com|microsoft\.com|msn\.com/i', $link)) {
                $links[] = $link;
            }
        }
        usleep(500000);
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getDuckDuckGoLinks($query, $numResults = 50) {
    $links = [];
    $url = "https://html.duckduckgo.com/html/?q=" . urlencode($query);

    $html = fetchWithCurl($url, 15, true); // Use cookie jar
    if (!$html) return [];

    // Method 1: Extract from DDG redirect URLs (uddg parameter)
    preg_match_all('/href="[^"]*(?:\/\/duckduckgo\.com\/l\/\?[^"]*uddg=([^&"]+))/i', $html, $redirectMatches);
    foreach ($redirectMatches[1] as $encoded) {
        $decoded = urldecode($encoded);
        if (filter_var($decoded, FILTER_VALIDATE_URL)) {
            $links[] = $decoded;
        }
    }

    // Method 2: result__a class (primary result links)
    preg_match_all('/<a[^>]+class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"/i', $html, $matches2);
    foreach ($matches2[1] as $link) {
        if (strpos($link, 'duckduckgo.com') !== false) {
            preg_match('/uddg=([^&]+)/', $link, $urlMatch);
            if (!empty($urlMatch[1])) $links[] = urldecode($urlMatch[1]);
        } elseif (filter_var($link, FILTER_VALIDATE_URL)) {
            $links[] = $link;
        }
    }

    // Method 3: result__url class (URL display links)
    preg_match_all('/<a[^>]+class="[^"]*result__url[^"]*"[^>]*href="([^"]+)"/i', $html, $matches3);
    foreach ($matches3[1] as $link) {
        if (strpos($link, 'duckduckgo.com') !== false) {
            preg_match('/uddg=([^&]+)/', $link, $urlMatch);
            if (!empty($urlMatch[1])) $links[] = urldecode($urlMatch[1]);
        } elseif (filter_var($link, FILTER_VALIDATE_URL)) {
            $links[] = $link;
        }
    }

    // Method 4: Generic fallback - all external links (not DDG)
    if (empty($links)) {
        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $allMatches);
        foreach ($allMatches[1] as $link) {
            if (!preg_match('/duckduckgo\.com|duck\.co|spreadprivacy\.com/i', $link)) {
                $links[] = $link;
            }
        }
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getYandexSearchLinks($query, $numResults = 30) {
    $links = [];
    $url = "https://yandex.com/search/?text=" . urlencode($query);

    $html = fetchWithCurl($url);
    if ($html) {
        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches);
        foreach ($matches[1] as $link) {
            if (!preg_match('/yandex\.|yastatic\./i', $link)) {
                $links[] = $link;
            }
        }
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getBaiduSearchLinks($query, $numResults = 30) {
    $links = [];
    $url = "https://www.baidu.com/s?wd=" . urlencode($query);

    $html = fetchWithCurl($url);
    if ($html) {
        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches);
        foreach ($matches[1] as $link) {
            if (!preg_match('/baidu\.com|bdstatic\.com/i', $link)) {
                $links[] = $link;
            }
        }
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getYahooSearchLinks($query, $numResults = 30) {
    $links = [];
    $url = "https://search.yahoo.com/search?p=" . urlencode($query);

    $html = fetchWithCurl($url);
    if ($html) {
        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches);
        foreach ($matches[1] as $link) {
            if (!preg_match('/yahoo\.com|yimg\.com/i', $link)) {
                $links[] = $link;
            }
        }
    }

    return array_unique(array_slice($links, 0, $numResults));
}

// ============================================
// BOT-FRIENDLY SEARCH SOURCES
// ============================================
function getSearXNGLinks($query, $numResults = 30) {
    $links = [];

    // Public SearXNG instances with JSON API support
    $instances = [
        'https://search.sapti.me',
        'https://searx.be',
        'https://paulgo.io',
        'https://search.bus-hit.me',
        'https://searx.tiekoetter.com',
        'https://search.ononoki.org',
        'https://searx.thegpm.org',
    ];
    shuffle($instances);

    foreach ($instances as $instance) {
        $url = $instance . '/search?q=' . urlencode($query) . '&format=json&categories=general&language=all&pageno=1';

        $response = fetchWithCurl($url, 15);
        if (!$response) continue;

        $data = json_decode($response, true);
        if (!$data || empty($data['results'])) continue;

        foreach ($data['results'] as $result) {
            if (!empty($result['url']) && filter_var($result['url'], FILTER_VALIDATE_URL)) {
                $links[] = $result['url'];
            }
        }

        if (count($links) >= $numResults) break;
        usleep(300000); // 300ms between instances
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getWikipediaLinks($query, $numResults = 30) {
    $links = [];

    // Step 1: Search Wikipedia for articles matching the query
    $searchUrl = 'https://en.wikipedia.org/w/api.php?action=opensearch&search='
               . urlencode($query) . '&limit=5&format=json';

    $response = fetchWithCurl($searchUrl, 10);
    if ($response) {
        $data = json_decode($response, true);
        if ($data && !empty($data[3])) {
            foreach ($data[3] as $wikiUrl) {
                $links[] = $wikiUrl;

                // Fetch each Wikipedia article and extract external reference links
                $articleHtml = fetchWithCurl($wikiUrl, 10);
                if ($articleHtml) {
                    preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*class="external/i', $articleHtml, $extMatches);
                    if (empty($extMatches[1])) {
                        // Fallback: extract any external links from references section
                        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $articleHtml, $allMatches);
                        foreach ($allMatches[1] as $link) {
                            if (!preg_match('/wikipedia\.org|wikimedia\.org|wikidata\.org|mediawiki\.org|w\.wiki/i', $link)) {
                                $links[] = $link;
                            }
                        }
                    } else {
                        foreach ($extMatches[1] as $extLink) {
                            if (!preg_match('/wikipedia\.org|wikimedia\.org|wikidata\.org/i', $extLink)) {
                                $links[] = $extLink;
                            }
                        }
                    }
                }

                if (count($links) >= $numResults) break;
                usleep(200000); // 200ms between wiki pages
            }
        }
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getDirectSearchLinks($query, $numResults = 30) {
    $links = [];

    // News/content sites with accessible search endpoints
    $searchEndpoints = [
        'https://www.reuters.com/site-search/?query=' . urlencode($query),
        'https://www.bbc.co.uk/search?q=' . urlencode($query),
        'https://www.aljazeera.com/search/' . urlencode($query),
        'https://www.theguardian.com/search?q=' . urlencode($query),
        'https://www.npr.org/search?query=' . urlencode($query),
        'https://news.un.org/en/search/' . urlencode($query),
    ];

    // Also add known topic pages directly
    $topicPages = [
        'https://en.wikipedia.org/wiki/' . urlencode(ucfirst($query)),
        'https://www.britannica.com/search?query=' . urlencode($query),
    ];
    foreach ($topicPages as $tp) {
        $links[] = $tp;
    }

    // Crawl each search page for result links
    foreach ($searchEndpoints as $searchUrl) {
        $html = fetchWithCurl($searchUrl, 12, true);
        if (!$html) continue;

        $searchDomain = parse_url($searchUrl, PHP_URL_HOST) ?? '';
        $baseDomain = str_replace('www.', '', $searchDomain);

        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches);
        foreach ($matches[1] as $link) {
            $linkDomain = parse_url($link, PHP_URL_HOST) ?? '';
            // Keep links from the same domain (actual article results)
            if (strpos($linkDomain, $baseDomain) !== false && strlen($link) > 30) {
                $links[] = $link;
            }
        }

        if (count($links) >= $numResults) break;
        usleep(500000); // 500ms politeness
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getBraveSearchLinks($query, $numResults = 20) {
    $links = [];
    $url = "https://search.brave.com/search?q=" . urlencode($query) . "&source=web";

    $html = fetchWithCurl($url, 12, true);
    if (!$html) return [];

    // Method 1: Brave result links with specific attributes
    preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*(?:data-type="web"|class="[^"]*result[^"]*")[^>]*>/i', $html, $matches1);
    $links = array_merge($links, $matches1[1] ?? []);

    // Method 2: General external links (filter out Brave internal)
    if (empty($links)) {
        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches2);
        foreach ($matches2[1] as $link) {
            if (!preg_match('/brave\.com|bravesoftware\.com|brave\.page/i', $link)) {
                $links[] = $link;
            }
        }
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getGoogleSearchLinks($query, $numResults = 30) {
    $links = [];

    // Regional Google domains (shuffle for variety)
    $domains = [
        'www.google.com',
        'www.google.com.br',
        'www.google.co.uk',
        'www.google.ru',
        'www.google.de',
        'www.google.fr',
        'www.google.es',
        'www.google.it',
        'www.google.co.jp',
        'www.google.co.in',
        'www.google.com.au',
        'www.google.ca',
    ];
    shuffle($domains);

    foreach ($domains as $domain) {
        $url = "https://{$domain}/search?q=" . urlencode($query) . "&num=20";
        $html = fetchWithCurl($url, 12, true);
        if (!$html) continue;

        // Detect CAPTCHA/block
        if (stripos($html, 'captcha') !== false || stripos($html, 'unusual traffic') !== false
            || stripos($html, 'before you continue') !== false) {
            continue;
        }

        // Method 1: Google redirect URLs (/url?q=...)
        preg_match_all('/\/url\?q=(https?[^&"]+)/i', $html, $matches);
        foreach ($matches[1] as $link) {
            $decoded = urldecode($link);
            if (!preg_match('/google\.|gstatic\.|googleapis\.|youtube\.com/i', $decoded)) {
                $links[] = $decoded;
            }
        }

        // Method 2: Direct href links (newer Google format)
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $all);
            foreach ($all[1] as $l) {
                if (!preg_match('/google\.|gstatic\.|googleapis\.|youtube\.com|schema\.org/i', $l)) {
                    $links[] = $l;
                }
            }
        }

        if (count($links) >= $numResults) break;
        usleep(1000000); // 1s between Google domains
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function getBingRegionalLinks($query, $numResults = 30) {
    $links = [];
    $markets = ['pt-BR', 'en-US', 'en-GB', 'ru-RU', 'de-DE', 'fr-FR', 'es-ES', 'ja-JP', 'it-IT', 'zh-CN', 'ar-SA', 'ko-KR'];
    shuffle($markets);

    foreach (array_slice($markets, 0, 3) as $mkt) {
        $url = "https://www.bing.com/search?q=" . urlencode($query) . "&mkt=$mkt&count=20";
        $html = fetchWithCurl($url, 10, true);
        if (!$html) continue;

        preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $matches);
        foreach ($matches[1] as $link) {
            if (!preg_match('/bing\.com|microsoft\.com|msn\.com|live\.com/i', $link)) {
                $links[] = $link;
            }
        }
        usleep(500000);
    }

    return array_unique(array_slice($links, 0, $numResults));
}

function extractSearchResultsFromUrl($url, $html) {
    $links = [];
    $host = parse_url($url, PHP_URL_HOST) ?? '';
    $path = parse_url($url, PHP_URL_PATH) ?? '';
    $query = parse_url($url, PHP_URL_QUERY) ?? '';

    // ===== GOOGLE (all types: web, images, videos, news, scholar) =====
    if (strpos($host, 'google') !== false) {
        $isImages = (strpos($query, 'udm=2') !== false || strpos($query, 'tbm=isch') !== false);
        $isVideos = (strpos($query, 'tbm=vid') !== false);
        $isNews = (strpos($query, 'tbm=nws') !== false);
        $isScholar = (strpos($host, 'scholar.google') !== false);

        if ($isImages) {
            // Google Images: extract image source URLs and page URLs
            // Pattern 1: imgres redirect URLs
            preg_match_all('/imgres\?imgurl=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) $links[] = urldecode($l);
            // Pattern 2: data-src image URLs
            preg_match_all('/data-src="(https?:\/\/[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/gstatic\.|google\./i', $l)) $links[] = $l;
            }
            // Pattern 3: JSON embedded image URLs
            preg_match_all('/"(https?:\/\/[^"]+\.(?:jpg|jpeg|png|gif|webp|svg)(?:\?[^"]*)?)"/', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/gstatic\.|google\.|encrypted-/i', $l)) $links[] = $l;
            }
            // Pattern 4: Source page URLs from imgres
            preg_match_all('/imgrefurl=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) $links[] = urldecode($l);
            // Pattern 5: JSON "ou" (original URL) in AF_initDataCallback
            preg_match_all('/"ou":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = str_replace(['\\u003d','\\u0026','\\/','\\"'], ['=','&','/',''], $l);
                if (!preg_match('/gstatic\.|google\.|encrypted-/i', $decoded)) $links[] = $decoded;
            }
            // Pattern 6: JSON "ru" (referring/source page URL)
            preg_match_all('/"ru":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = str_replace(['\\u003d','\\u0026','\\/','\\"'], ['=','&','/',''], $l);
                if (!preg_match('/gstatic\.|google\./i', $decoded)) $links[] = $decoded;
            }
            // Pattern 7: data-lpage attribute (landing page)
            preg_match_all('/data-lpage="(https?:\/\/[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = $l;
            // Pattern 8: data-action-url attribute
            preg_match_all('/data-action-url="(https?:\/\/[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/google\./i', $l)) $links[] = $l;
            }
            // Pattern 9: Thumbnail source with target page in data attributes
            preg_match_all('/data-(?:thumbnail-url|deferred-src)="(https?:\/\/[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/gstatic\.|google\.|encrypted-tbn/i', $l)) $links[] = $l;
            }
        }
        elseif ($isVideos) {
            // Google Videos: extract video page URLs (YouTube, Vimeo, etc.)
            preg_match_all('/\/url\?q=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = urldecode($l);
                if (!preg_match('/google\./i', $decoded)) $links[] = $decoded;
            }
            // Direct video platform links
            preg_match_all('/href="(https?:\/\/(?:www\.)?(?:youtube\.com|youtu\.be|vimeo\.com|dailymotion\.com|twitch\.tv)[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = $l;
        }
        elseif ($isNews) {
            // Google News: article URLs
            preg_match_all('/\/url\?q=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = urldecode($l);
                if (!preg_match('/google\.|gstatic\./i', $decoded)) $links[] = $decoded;
            }
            // Direct article links
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/google\.|gstatic\.|googleapis\./i', $l)) $links[] = $l;
            }
        }
        elseif ($isScholar) {
            // Google Scholar: paper/article links
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/google\.|gstatic\.|googleapis\.|scholar\./i', $l)) $links[] = $l;
            }
            // PDF links
            preg_match_all('/href="(https?:\/\/[^"]+\.pdf[^"]*)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = $l;
        }
        else {
            // Google Web Search (default)
            preg_match_all('/\/url\?q=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = urldecode($l);
                if (!preg_match('/google\.|gstatic\.|googleapis\./i', $decoded)) $links[] = $decoded;
            }
        }

        // Universal Google fallback
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m2);
            foreach ($m2[1] as $l) {
                if (!preg_match('/google\.|gstatic\.|googleapis\.|schema\.org/i', $l)) $links[] = $l;
            }
        }
    }

    // ===== BING (all types: web, images, videos, news) =====
    elseif (strpos($host, 'bing') !== false) {
        $isImages = (strpos($path, '/images') !== false);
        $isVideos = (strpos($path, '/videos') !== false);
        $isNews = (strpos($path, '/news') !== false);

        if ($isImages) {
            // Bing Images: extract image source URLs and page URLs
            // Pattern 1: mediaurl parameter
            preg_match_all('/mediaurl=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) $links[] = urldecode($l);
            // Pattern 2: purl (page URL)
            preg_match_all('/purl=(https?[^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) $links[] = urldecode($l);
            // Pattern 3: data-src image URLs
            preg_match_all('/(?:data-src|src)="(https?:\/\/[^"]+\.(?:jpg|jpeg|png|gif|webp)[^"]*)"/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/bing\.com|bing\.net|microsoft\.com/i', $l)) $links[] = $l;
            }
            // Pattern 4: JSON murl (media URL)
            preg_match_all('/"murl":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', $l);
            // Pattern 5: JSON purl (page URL)
            preg_match_all('/"purl":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', $l);
        }
        elseif ($isVideos) {
            // Bing Videos: extract video page URLs
            // Pattern 1: JSON video URLs
            preg_match_all('/"(?:videoUrl|contentUrl|hostPageUrl)"\s*:\s*"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', $l);
            // Pattern 2: data-href on video cards
            preg_match_all('/data-href="([^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                if (preg_match('/https?:\/\//i', $l) && !preg_match('/bing\.com/i', $l)) $links[] = $l;
            }
            // Pattern 3: Direct video platform links
            preg_match_all('/href="(https?:\/\/(?:www\.)?(?:youtube\.com|youtu\.be|vimeo\.com|dailymotion\.com|tiktok\.com|twitch\.tv)[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = $l;
            // Pattern 4: vredirect URLs
            preg_match_all('/vredirect[^"]*url=([^&"]+)/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = urldecode($l);
                if (filter_var($decoded, FILTER_VALIDATE_URL)) $links[] = $decoded;
            }
        }
        elseif ($isNews) {
            // Bing News: article links
            preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*class="[^"]*title[^"]*"/is', $html, $m);
            $links = $m[1] ?? [];
            if (empty($links)) {
                preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $all);
                foreach ($all[1] as $l) {
                    if (!preg_match('/bing\.com|microsoft\.com|msn\.com|bing\.net/i', $l)) $links[] = $l;
                }
            }
        }
        else {
            // Bing Web Search (default)
            preg_match_all('/<h2[^>]*>.*?<a[^>]+href="(https?:\/\/[^"]+)"/is', $html, $m);
            $links = $m[1] ?? [];
        }

        // Universal Bing fallback
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $all);
            foreach ($all[1] as $l) {
                if (!preg_match('/bing\.com|microsoft\.com|msn\.com|bing\.net/i', $l)) $links[] = $l;
            }
        }
    }

    // ===== YAHOO (web, images, videos, news) =====
    elseif (strpos($host, 'yahoo') !== false) {
        $isImages = (strpos($path, '/image') !== false || strpos($host, 'images.search') !== false);
        $isVideos = (strpos($path, '/video') !== false || strpos($host, 'video.search') !== false);
        $isNews = (strpos($path, '/news') !== false || strpos($host, 'news.search') !== false);

        // Yahoo uses RU redirects
        preg_match_all('/RU=(https?[^\/&"]+)/i', $html, $m);
        foreach ($m[1] as $l) $links[] = urldecode($l);

        // Direct external links
        preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
        foreach ($m[1] as $l) {
            if (!preg_match('/yahoo\.com|yimg\.com|oath\.com|aol\.com/i', $l)) $links[] = $l;
        }
    }

    // ===== YANDEX (web, images, videos) =====
    elseif (strpos($host, 'yandex') !== false) {
        $isImages = (strpos($path, '/images') !== false);
        $isVideos = (strpos($path, '/video') !== false);

        if ($isImages) {
            // Yandex Images: JSON img_url
            preg_match_all('/"img_url":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', urldecode($l));
            // Source page URLs
            preg_match_all('/"source_url":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', urldecode($l));
        }
        elseif ($isVideos) {
            // Yandex Videos: player URLs and source pages
            preg_match_all('/"url":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = str_replace('\\/', '/', $l);
                if (!preg_match('/yandex\.|yastatic\./i', $decoded)) $links[] = $decoded;
            }
        }

        // Yandex general external links
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/yandex\.|yastatic\.|yandex-team\./i', $l)) $links[] = $l;
            }
        }
    }

    // ===== DUCKDUCKGO (web, images, videos) =====
    elseif (strpos($host, 'duckduckgo') !== false) {
        $isImages = (strpos($query, 'iax=images') !== false || strpos($query, 'ia=images') !== false);
        $isVideos = (strpos($query, 'iax=videos') !== false || strpos($query, 'ia=videos') !== false);

        if ($isImages) {
            // DDG Images: image URLs in JSON/vqd data
            preg_match_all('/"image":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', $l);
            preg_match_all('/"url":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) {
                $decoded = str_replace('\\/', '/', $l);
                if (!preg_match('/duckduckgo\.com/i', $decoded)) $links[] = $decoded;
            }
        }
        elseif ($isVideos) {
            // DDG Videos: video platform links
            preg_match_all('/"content":"(https?:[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = str_replace('\\/', '/', $l);
            // Direct video links
            preg_match_all('/href="(https?:\/\/(?:www\.)?(?:youtube\.com|youtu\.be|vimeo\.com|dailymotion\.com)[^"]+)"/i', $html, $m);
            foreach ($m[1] as $l) $links[] = $l;
        }

        // DDG uddg= redirect pattern (web search)
        preg_match_all('/href="[^"]*uddg=([^&"]+)/i', $html, $m);
        foreach ($m[1] as $encoded) {
            $decoded = urldecode($encoded);
            if (filter_var($decoded, FILTER_VALIDATE_URL)) $links[] = $decoded;
        }

        // DDG fallback
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m2);
            foreach ($m2[1] as $l) {
                if (!preg_match('/duckduckgo\.com/i', $l)) $links[] = $l;
            }
        }
    }

    // ===== BAIDU =====
    elseif (strpos($host, 'baidu') !== false) {
        // Baidu uses redirect URLs
        preg_match_all('/href="(https?:\/\/www\.baidu\.com\/link\?url=[^"]+)"/i', $html, $m);
        foreach ($m[1] as $l) $links[] = $l;
        // Direct external links
        preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
        foreach ($m[1] as $l) {
            if (!preg_match('/baidu\.com|bdstatic\.com|baiducontent\./i', $l)) $links[] = $l;
        }
    }

    // ===== ECOSIA =====
    elseif (strpos($host, 'ecosia') !== false) {
        preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
        foreach ($m[1] as $l) {
            if (!preg_match('/ecosia\.org/i', $l)) $links[] = $l;
        }
    }

    // ===== STARTPAGE =====
    elseif (strpos($host, 'startpage') !== false) {
        preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*class="[^"]*result[^"]*"/i', $html, $m);
        $links = $m[1] ?? [];
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/startpage\.com|ixquick\./i', $l)) $links[] = $l;
            }
        }
    }

    // ===== BRAVE SEARCH =====
    elseif (strpos($host, 'search.brave') !== false) {
        preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
        foreach ($m[1] as $l) {
            if (!preg_match('/brave\.com|bravesoftware\./i', $l)) $links[] = $l;
        }
    }

    // ===== SEARXNG / SEARX =====
    elseif (preg_match('/searx/i', $host)) {
        preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*class="[^"]*url[^"]*"/i', $html, $m);
        $links = $m[1] ?? [];
        if (empty($links)) {
            preg_match_all('/href="(https?:\/\/[^"]+)"[^>]*>/i', $html, $m);
            foreach ($m[1] as $l) {
                if (!preg_match('/searx|searxng/i', $l)) $links[] = $l;
            }
        }
    }

    return array_unique(array_slice($links, 0, 200));
}

// ============================================
// CURL FUNCTIONS
// ============================================
function fetchWithCurl($url, $timeout = 10, $useCookieJar = false) {
    static $cookieJarFile = null;
    if ($useCookieJar && $cookieJarFile === null) {
        $cookieJarFile = sys_get_temp_dir() . '/crawler_cookies_' . getmypid() . '.txt';
    }

    $userAgents = getRotatingUserAgents();

    $ch = curl_init();
    $options = [
        CURLOPT_URL => $url,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_MAXREDIRS => 5,
        CURLOPT_TIMEOUT => $timeout,
        CURLOPT_CONNECTTIMEOUT => 5,
        CURLOPT_SSL_VERIFYPEER => false,
        CURLOPT_ENCODING => 'gzip, deflate',
        CURLOPT_USERAGENT => $userAgents[array_rand($userAgents)],
        CURLOPT_HTTPHEADER => [
            'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
            'Accept-Language: en-US,en;q=0.9',
            'Connection: keep-alive',
            'Upgrade-Insecure-Requests: 1',
            'Sec-Fetch-Dest: document',
            'Sec-Fetch-Mode: navigate',
            'Sec-Fetch-Site: none',
            'Sec-Fetch-User: ?1',
            'Cache-Control: max-age=0',
        ],
    ];

    if ($useCookieJar && $cookieJarFile) {
        $options[CURLOPT_COOKIEJAR] = $cookieJarFile;
        $options[CURLOPT_COOKIEFILE] = $cookieJarFile;
    }

    curl_setopt_array($ch, $options);
    $response = curl_exec($ch);
    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
    curl_close($ch);

    return ($httpCode >= 200 && $httpCode < 400) ? $response : false;
}

function fetchBatchWithCurl($urls, $timeout = 10, $concurrency = 10, $authCredentials = '') {
    static $htmlCache = []; // HTML Response Cache: avoid re-fetching same URLs
    $userAgents = getRotatingUserAgents();

    // Check cache first, separate cached vs uncached URLs
    $cachedResults = [];
    $uncachedUrls = [];
    foreach ($urls as $u) {
        if (isset($htmlCache[$u])) {
            $cachedResults[$u] = $htmlCache[$u];
        } else {
            $uncachedUrls[] = $u;
        }
    }
    if (empty($uncachedUrls)) return $cachedResults;
    $urls = $uncachedUrls;

    $results = [];
    $chunks = array_chunk($urls, $concurrency);

    foreach ($chunks as $chunk) {
        $multiHandle = curl_multi_init();
        $handles = [];

        foreach ($chunk as $url) {
            $ch = curl_init();
            $urlHost = parse_url($url, PHP_URL_HOST) ?? '';
            $headers = [
                'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language: en-US,en;q=0.9,pt-BR;q=0.8,pt;q=0.7',
                'Sec-Fetch-Dest: document',
                'Sec-Fetch-Mode: navigate',
            ];
            // Google consent bypass cookie
            if (strpos($urlHost, 'google') !== false) {
                $headers[] = 'Cookie: SOCS=CAISHAgCEhJnd3NfMjAyNDA4MDctMF9SQzIaAmVuIAEaBgiA_LW2Bg; CONSENT=PENDING+987';
            }
            $curlOpts = [
                CURLOPT_URL => $url,
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_FOLLOWLOCATION => true,
                CURLOPT_MAXREDIRS => 5,
                CURLOPT_TIMEOUT => $timeout,
                CURLOPT_CONNECTTIMEOUT => 5,
                CURLOPT_SSL_VERIFYPEER => false,
                CURLOPT_ENCODING => 'gzip, deflate, br',
                CURLOPT_USERAGENT => $userAgents[array_rand($userAgents)],
                CURLOPT_HTTPHEADER => $headers,
                CURLOPT_TCP_KEEPALIVE => 1,
                CURLOPT_TCP_KEEPIDLE => 60,
            ];
            if (!empty($authCredentials)) {
                $curlOpts[CURLOPT_USERPWD] = $authCredentials;
                $curlOpts[CURLOPT_HTTPAUTH] = CURLAUTH_BASIC;
            }
            curl_setopt_array($ch, $curlOpts);

            curl_multi_add_handle($multiHandle, $ch);
            $handles[$url] = $ch;
        }

        $running = null;
        do {
            curl_multi_exec($multiHandle, $running);
            curl_multi_select($multiHandle);
        } while ($running > 0);

        foreach ($handles as $url => $ch) {
            $results[$url] = [
                'html' => curl_multi_getcontent($ch),
                'http_code' => curl_getinfo($ch, CURLINFO_HTTP_CODE),
                'error' => curl_error($ch),
                'size_download' => curl_getinfo($ch, CURLINFO_SIZE_DOWNLOAD),
                'total_time' => curl_getinfo($ch, CURLINFO_TOTAL_TIME),
            ];
            curl_multi_remove_handle($multiHandle, $ch);
            curl_close($ch);
        }

        curl_multi_close($multiHandle);
    }

    // Store successful fetches in cache (limit to 2000 entries to avoid memory issues)
    foreach ($results as $cacheUrl => $cacheResult) {
        if ($cacheResult['http_code'] >= 200 && $cacheResult['http_code'] < 400 && !empty($cacheResult['html'])) {
            if (count($htmlCache) < 2000) {
                $htmlCache[$cacheUrl] = $cacheResult;
            }
        }
    }

    // Merge cached + fresh results
    return array_merge($cachedResults, $results);
}

// ============================================
// DEEP CRAWL FUNCTIONS
// ============================================
function deepCrawl($seedUrl, $maxDepth = 3, $maxPages = 100, $linkType = 'both', $includeTerms = [], $excludeTerms = [], $respectRobots = true) {
    $visited = [];
    $queue = [['url' => $seedUrl, 'depth' => 0]];
    $foundLinks = [];
    $parsedSeed = parse_url($seedUrl);
    $seedHost = $parsedSeed['host'] ?? '';

    while (!empty($queue) && count($foundLinks) < $maxPages) {
        $current = array_shift($queue);
        $url = $current['url'];
        $depth = $current['depth'];

        if (in_array($url, $visited) || $depth > $maxDepth) {
            continue;
        }

        $visited[] = $url;

        // Check robots.txt
        if ($respectRobots && !isAllowedByRobots($url)) {
            continue;
        }

        $html = fetchWithCurl($url);
        if (!$html) continue;

        // Extract links
        preg_match_all('/<a[^>]+href="([^"]+)"/i', $html, $matches);

        foreach ($matches[1] as $link) {
            $link = normalizeUrl($link, $url);
            if (!$link || in_array($link, $visited)) continue;

            $linkHost = parse_url($link, PHP_URL_HOST);

            // Filter by link type
            if ($linkType === 'origin' && $linkHost !== $seedHost) continue;
            if ($linkType === 'external' && $linkHost === $seedHost) continue;

            // Apply include/exclude filters
            if (!passesFilters($link, $includeTerms, $excludeTerms)) continue;

            $foundLinks[] = $link;

            if ($depth < $maxDepth) {
                $queue[] = ['url' => $link, 'depth' => $depth + 1];
            }
        }
    }

    return array_unique($foundLinks);
}

function normalizeUrl($link, $baseUrl) {
    if (empty($link) || $link[0] === '#' || strpos($link, 'javascript:') === 0) {
        return null;
    }

    if (strpos($link, '//') === 0) {
        $link = 'https:' . $link;
    } elseif (strpos($link, '/') === 0) {
        $parsed = parse_url($baseUrl);
        $link = $parsed['scheme'] . '://' . $parsed['host'] . $link;
    } elseif (strpos($link, 'http') !== 0) {
        $link = rtrim(dirname($baseUrl), '/') . '/' . $link;
    }

    return filter_var($link, FILTER_VALIDATE_URL) ? $link : null;
}

function passesFilters($link, $includeTerms, $excludeTerms) {
    // Check exclude terms first
    foreach ($excludeTerms as $term) {
        if (!empty($term) && stripos($link, $term) !== false) {
            return false;
        }
    }

    // If no include terms, accept all
    if (empty($includeTerms) || (count($includeTerms) === 1 && empty($includeTerms[0]))) {
        return true;
    }

    // Check if any include term matches
    foreach ($includeTerms as $term) {
        if (!empty($term) && stripos($link, $term) !== false) {
            return true;
        }
    }

    return false;
}

function isAllowedByRobots($url) {
    static $robotsCache = [];

    $parsed = parse_url($url);
    $host = $parsed['host'] ?? '';

    if (!isset($robotsCache[$host])) {
        $robotsUrl = $parsed['scheme'] . '://' . $host . '/robots.txt';
        $robotsTxt = @file_get_contents($robotsUrl);
        $robotsCache[$host] = $robotsTxt !== false ? $robotsTxt : '';
    }

    if (empty($robotsCache[$host])) {
        return true;
    }

    $path = $parsed['path'] ?? '/';
    $lines = explode("\n", $robotsCache[$host]);
    $userAgentMatch = false;

    foreach ($lines as $line) {
        $line = trim($line);
        if (stripos($line, 'User-agent:') === 0) {
            $agent = trim(substr($line, 11));
            $userAgentMatch = ($agent === '*' || stripos($agent, 'bot') !== false);
        }

        if ($userAgentMatch && stripos($line, 'Disallow:') === 0) {
            $disallowed = trim(substr($line, 9));
            if (!empty($disallowed) && strpos($path, $disallowed) === 0) {
                return false;
            }
        }
    }

    return true;
}

// ============================================
// PAGINATION FUNCTIONS
// ============================================
function generatePaginationUrls($pattern, $start, $end) {
    $urls = [];
    for ($i = $start; $i <= $end; $i++) {
        $urls[] = preg_replace('/\d+/', $i, $pattern, 1);
    }
    return $urls;
}

// ============================================
// METADATA EXTRACTION
// ============================================
function extractMetadata($url, $html = null) {
    if (!$html) {
        $html = fetchWithCurl($url);
    }

    $empty = ['title' => basename($url), 'description' => '', 'thumbnail' => '', 'favicon' => '', 'author' => 'Anonymous', 'tags' => ''];
    if (!$html) return $empty;

    // FAST REGEX extraction (10x faster than DOMDocument)
    // Only parse the first 50KB of HTML (head section is always there)
    $head = substr($html, 0, 50000);

    // Title: <title> tag or og:title
    $title = '';
    if (preg_match('/<title[^>]*>([^<]+)<\/title>/si', $head, $m)) {
        $title = trim(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8'));
    }
    if (empty($title) && preg_match('/property=["\']og:title["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $title = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
    }
    if (empty($title) && preg_match('/content=["\']([^"\']+)["\'][^>]*property=["\']og:title/i', $head, $m)) {
        $title = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
    }

    // Description: meta description or og:description
    $description = '';
    if (preg_match('/name=["\']description["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $description = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
    } elseif (preg_match('/content=["\']([^"\']+)["\'][^>]*name=["\']description/i', $head, $m)) {
        $description = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
    }
    if (empty($description) && preg_match('/property=["\']og:description["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $description = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
    }

    // Thumbnail: comprehensive hierarchy (14+ sources)
    $thumbnail = '';
    $thumbPatterns = [
        '/property=["\']og:image:secure_url["\'][^>]*content=["\']([^"\']+)/i',
        '/content=["\']([^"\']+)["\'][^>]*property=["\']og:image:secure_url/i',
        '/property=["\']og:image:url["\'][^>]*content=["\']([^"\']+)/i',
        '/content=["\']([^"\']+)["\'][^>]*property=["\']og:image:url/i',
        '/property=["\']og:image["\'][^>]*content=["\']([^"\']+)/i',
        '/content=["\']([^"\']+)["\'][^>]*property=["\']og:image/i',
        '/name=["\']twitter:image:src["\'][^>]*content=["\']([^"\']+)/i',
        '/name=["\']twitter:image["\'][^>]*content=["\']([^"\']+)/i',
        '/content=["\']([^"\']+)["\'][^>]*name=["\']twitter:image/i',
        '/<link[^>]+rel=["\']apple-touch-icon["\'][^>]*href=["\']([^"\']+)/i',
        '/<link[^>]+rel=["\']apple-touch-icon-precomposed["\'][^>]*href=["\']([^"\']+)/i',
    ];
    foreach ($thumbPatterns as $pat) {
        if (preg_match($pat, $head, $m)) {
            $thumbnail = $m[1];
            break;
        }
    }
    // Body fallbacks: WordPress, article images, video posters
    if (empty($thumbnail)) {
        $body = substr($html, 0, 120000);
        $bodyPatterns = [
            '/<img[^>]+class=["\'][^"\']*wp-post-image[^"\']*["\'][^>]*src=["\']([^"\']+)/i',
            '/<img[^>]+class=["\'][^"\']*post-thumbnail[^"\']*["\'][^>]*src=["\']([^"\']+)/i',
            '/<figure[^>]*>\s*<img[^>]+src=["\']([^"\']+)/i',
            '/<div[^>]+class=["\'][^"\']*post-thumbnail[^"\']*["\'][^>]*>\s*<img[^>]+src=["\']([^"\']+)/i',
            '/<video[^>]+poster=["\']([^"\']+)/i',
            '/<div[^>]+class=["\'][^"\']*video-thumbnail[^"\']*["\'][^>]*>\s*<img[^>]+src=["\']([^"\']+)/i',
            '/<img[^>]+src=["\']([^"\']+\.(jpg|jpeg|png|webp))["\'][^>]*width=["\']([2-9]\d{2}|\d{4,})/i',
            '/<img[^>]+width=["\']([2-9]\d{2}|\d{4,})["\'][^>]*src=["\']([^"\']+\.(jpg|jpeg|png|webp))/i',
        ];
        foreach ($bodyPatterns as $pat) {
            if (preg_match($pat, $body, $m)) {
                $thumbnail = $m[1];
                // For the width-first pattern, img URL is in group 2
                if (isset($m[2]) && filter_var($m[2], FILTER_VALIDATE_URL)) $thumbnail = $m[2];
                break;
            }
        }
    }
    // CSS background-image fallback
    if (empty($thumbnail)) {
        if (preg_match('/background-image:\s*url\(["\']?([^"\')\s]+\.(jpg|jpeg|png|webp))["\']?\)/i', substr($html, 0, 80000), $m)) {
            $thumbnail = $m[1];
        }
    }
    // Normalize relative thumbnail URLs to absolute
    if (!empty($thumbnail) && !preg_match('#^https?://#i', $thumbnail)) {
        $thumbnail = normalizeUrl($thumbnail, $url) ?: $thumbnail;
    }

    // Favicon
    $parsed = parse_url($url);
    $favicon = ($parsed['scheme'] ?? 'https') . '://' . ($parsed['host'] ?? '') . '/favicon.ico';
    if (preg_match('/<link[^>]+rel=["\'](?:shortcut )?icon["\'][^>]*href=["\']([^"\']+)/i', $head, $m)) {
        $favicon = normalizeUrl($m[1], $url) ?: $favicon;
    } elseif (preg_match('/<link[^>]+href=["\']([^"\']+)["\'][^>]*rel=["\'](?:shortcut )?icon/i', $head, $m)) {
        $favicon = normalizeUrl($m[1], $url) ?: $favicon;
    }

    // Author (meta tag -> og:article:author -> random fallback)
    $author = '';
    if (preg_match('/name=["\']author["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $author = trim(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8'));
    } elseif (preg_match('/property=["\']article:author["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $author = trim(html_entity_decode($m[1], ENT_QUOTES, 'UTF-8'));
    } elseif (preg_match('/name=["\']twitter:creator["\'][^>]*content=["\']@?([^"\']+)/i', $head, $m)) {
        $author = trim($m[1]);
    }
    // Fallback to random author if empty or generic
    if (empty($author) || $author === 'Anonymous' || $author === 'admin' || mb_strlen($author) < 3) {
        $author = generateRandomAuthor();
    }

    // Tags/Keywords (meta keywords -> extract from title)
    $tags = '';
    if (preg_match('/name=["\']keywords["\'][^>]*content=["\']([^"\']+)/i', $head, $m)) {
        $tags = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
    }
    // Fallback: extract tags from title if no meta keywords
    if (empty($tags) && !empty($title)) {
        $tags = extractTagsFromTitle($title);
    }

    return [
        'title' => mb_substr($title ?: basename($url), 0, 255),
        'description' => mb_substr($description, 0, 1000),
        'thumbnail' => $thumbnail,
        'favicon' => $favicon,
        'author' => $author ?: 'Anonymous',
        'tags' => mb_substr($tags, 0, 500),
    ];
}

// QUALITY SCORING: Skip junk pages that would pollute pinfeeds
function scoreContentQuality($url, $html, $meta) {
    $score = 50; // Start neutral

    // Title quality (0-25 points)
    $title = $meta['title'] ?? '';
    if (empty($title) || $title === basename($url)) $score -= 20;
    elseif (mb_strlen($title) > 15 && mb_strlen($title) < 200) $score += 15;
    elseif (mb_strlen($title) >= 10) $score += 8;

    // Description presence (+10)
    if (!empty($meta['description']) && mb_strlen($meta['description']) > 30) $score += 10;

    // Body text quality (0-30 points)
    $cleanBody = preg_replace('/<(script|style|noscript|nav|footer|header|aside|menu)[^>]*>.*?<\/\1>/si', '', substr($html, 0, 80000));
    $bodyText = trim(preg_replace('/\s+/', ' ', strip_tags($cleanBody)));
    $bodyLen = mb_strlen($bodyText);

    if ($bodyLen < 100) $score -= 25;        // Almost no content
    elseif ($bodyLen < 300) $score -= 10;    // Very thin content
    elseif ($bodyLen > 800) $score += 15;    // Good content length
    elseif ($bodyLen > 2000) $score += 25;   // Rich content

    // Junk URL patterns (-30)
    if (preg_match('/\/(login|signup|register|cart|checkout|account|admin|wp-login|privacy|cookie|terms|disclaimer|404|error|search\?|tag\/|category\/|page\/\d+|author\/|attachment\/)/i', $url)) {
        $score -= 30;
    }

    // Article/post URL patterns (+15)
    if (preg_match('/\/post[s]?\/|\/article[s]?\/|\/blog\/|\/news\/|\/story\/|\d{4}[\/\-]\d{2}[\/\-]|[a-z]+-[a-z]+-[a-z]+-[a-z]+/i', $url)) {
        $score += 15;
    }

    // Thumbnail present (+5)
    if (!empty($meta['thumbnail'])) $score += 5;

    // Tags/keywords present (+5)
    if (!empty($meta['tags'])) $score += 5;

    // Junk title patterns (-20)
    if (preg_match('/^(404|error|not found|page not|untitled|home|index|redirect|loading|please wait|access denied|forbidden)/i', $title)) {
        $score -= 20;
    }

    // High link-to-text ratio = likely navigation/index page (-15)
    $linkCount = substr_count(substr($html, 0, 80000), '<a ');
    if ($bodyLen > 0 && $linkCount > 0) {
        $linkRatio = $linkCount / max(1, ($bodyLen / 100));
        if ($linkRatio > 5) $score -= 15; // More links than content
    }

    // Meta noindex penalty (-30): page explicitly says don't index
    $robots = hasNoindexNofollow($html);
    if ($robots['noindex']) $score -= 30;

    return max(0, min(100, $score));
}

function extractBatchMetadata($urls, $concurrency = 10) {
    $results = fetchBatchWithCurl($urls, 10, $concurrency);
    $metadata = [];

    foreach ($results as $url => $result) {
        if ($result['http_code'] >= 200 && $result['http_code'] < 400) {
            $metadata[$url] = extractMetadata($url, $result['html']);
        } else {
            $metadata[$url] = [
                'title' => basename($url),
                'description' => '',
                'thumbnail' => '',
                'favicon' => '',
                'author' => 'Anonymous',
                'tags' => '',
                'error' => $result['error'] ?: 'HTTP ' . $result['http_code'],
            ];
        }
    }

    return $metadata;
}

// ============================================
// AUTHOR + USER + EMBED + TAGS FUNCTIONS
// ============================================
function generateRandomAuthor() {
    $authors = [
        'James Wilson', 'Sarah Johnson', 'Michael Brown', 'Emily Davis', 'Robert Miller',
        'Jessica Taylor', 'David Anderson', 'Ashley Thomas', 'William Jackson', 'Amanda White',
        'Christopher Harris', 'Megan Martin', 'Matthew Thompson', 'Brittany Garcia', 'Andrew Martinez',
        'Jennifer Robinson', 'Daniel Clark', 'Stephanie Rodriguez', 'Joshua Lewis', 'Nicole Lee',
        'Ryan Walker', 'Lauren Hall', 'Brandon Allen', 'Kayla Young', 'Justin Hernandez',
        'Rachel King', 'Kevin Wright', 'Samantha Lopez', 'Jason Hill', 'Amber Scott',
        'Aaron Green', 'Heather Adams', 'Nathan Baker', 'Danielle Gonzalez', 'Adam Nelson',
        'Victoria Carter', 'Brian Mitchell', 'Christina Perez', 'Timothy Roberts', 'Rebecca Turner',
        'Patrick Phillips', 'Katherine Campbell', 'Sean Parker', 'Laura Evans', 'Jeremy Edwards',
        'Michelle Collins', 'Jonathan Stewart', 'Maria Sanchez', 'Mark Morris', 'Elizabeth Rogers',
        'Alexander Reed', 'Melissa Cook', 'Tyler Morgan', 'Angela Bell', 'Stephen Murphy',
        'Kimberly Bailey', 'Eric Rivera', 'Lisa Cooper', 'Benjamin Richardson', 'Amy Cox',
        'Jacob Howard', 'Julia Ward', 'Samuel Torres', 'Kelly Peterson', 'Gregory Gray',
        'Andrea Ramirez', 'George James', 'Christine Watson', 'Kenneth Brooks', 'Tracy Sanders',
        'Edward Price', 'Tiffany Bennett', 'Nicholas Wood', 'Diane Barnes', 'Charles Ross',
        'Cynthia Henderson', 'Peter Coleman', 'Donna Jenkins', 'Frank Perry', 'Karen Powell',
        'Douglas Long', 'Sharon Patterson', 'Steven Hughes', 'Debra Flores', 'Raymond Washington',
        'Carol Butler', 'Anthony Simmons', 'Sandra Foster', 'Paul Bryant', 'Linda Russell',
        'Donald Griffin', 'Susan Diaz', 'Larry Hayes', 'Janet Myers', 'Jeffrey Ford',
        'Catherine Hamilton', 'Scott Graham', 'Margaret Sullivan', 'Gary Wallace', 'Ruth West',
        'Oliver Stone', 'Sophie Turner', 'Lucas Martin', 'Emma Watson', 'Leo Valdez',
        'Luna Garcia', 'Felix Chen', 'Aria Park', 'Oscar Lopez', 'Maya Singh',
        'Hugo Rivera', 'Zoe Williams', 'Ethan Moore', 'Chloe Taylor', 'Liam Johnson',
        'Ava Brown', 'Noah Smith', 'Sophia Davis', 'Mason Wilson', 'Isabella Martinez',
        'Logan Anderson', 'Mia Thomas', 'Elijah Jackson', 'Charlotte White', 'Aiden Harris',
        'Amelia Robinson', 'Sebastian Clark', 'Harper Lewis', 'Jack Walker', 'Evelyn Hall',
        'Owen Allen', 'Abigail Young', 'Henry King', 'Emily Wright', 'Caleb Scott',
        'Grace Green', 'Wyatt Baker', 'Victoria Adams', 'Luke Nelson', 'Scarlett Hill',
        'Gabriel Carter', 'Lily Mitchell', 'Carter Perez', 'Penelope Roberts', 'Julian Turner',
        'Layla Phillips', 'Isaac Campbell', 'Riley Parker', 'Jayden Evans', 'Zoey Edwards',
        'Marcus Reed', 'Nora Collins', 'Leo Stewart', 'Hannah Sanchez', 'Theo Morris',
        'Stella Rogers', 'Adrian Cook', 'Aurora Morgan', 'Roman Bell', 'Savannah Murphy',
        'Jace Bailey', 'Audrey Rivera', 'Thomas Cooper', 'Brooklyn Richardson', 'Finn Howard',
        'Claire Ward', 'Miles Torres', 'Skylar Peterson', 'Axel Gray', 'Paisley James',
        'Mateo Watson', 'Bella Brooks', 'Max Sanders', 'Violet Bennett', 'Cole Wood',
        'Hazel Barnes', 'Kai Ross', 'Ivy Henderson', 'Asher Coleman', 'Ruby Jenkins',
        'Elias Perry', 'Willow Powell', 'Grayson Long', 'Autumn Patterson', 'Ezra Hughes',
        'Jade Flores', 'Carson Washington', 'Daisy Butler', 'Xavier Simmons', 'Ellie Foster',
        'Landon Bryant', 'Alice Russell', 'Cooper Griffin', 'Sadie Diaz', 'River Hayes',
        'Ariana Myers', 'Jaxon Ford', 'Naomi Hamilton', 'Emery Graham', 'Eva Sullivan',
        'Ryder Wallace', 'Athena West', 'Declan Stone', 'Valentina Cruz', 'Brooks Reyes',
        'Aaliyah Kim', 'Phoenix Nguyen', 'Iris Patel', 'Atlas Sharma', 'Freya Costa',
        'Dante Morales', 'Sienna Ortiz', 'Zander Gutierrez', 'Mila Ramos', 'Bodhi Mendez',
        'Reagan Dominguez', 'Rowan Vargas', 'Ember Castillo', 'Knox Vasquez', 'Sage Delgado',
        'Beckett Guerrero', 'Nova Medina', 'Jasper Aguilar', 'Juniper Estrada', 'Maddox Reyes',
        'Wren Alvarez', 'Rhett Herrera', 'Maeve Santiago', 'Cyrus Cardenas', 'Ivy Cervantes',
        'Rafael Soto', 'Camila Duran', 'Dante Figueroa', 'Lucia Espinoza', 'Nico Paredes',
        'Elena Ochoa', 'Marco Vega', 'Diana Salazar', 'Sergio Mendoza', 'Rosa Padilla',
        'Antonio Fuentes', 'Carmen Cortez', 'Francisco Luna', 'Gloria Campos', 'Alejandro Rios',
        'Valentina Nunez', 'Santiago Rojas', 'Gabriela Contreras', 'Matias Cabrera', 'Isabella Ruiz',
        'Andres Sandoval', 'Paula Ibarra', 'Felipe Navarro', 'Martina Molina', 'Emiliano Torres',
        'Catalina Acosta', 'Joaquin Valenzuela', 'Renata Herrera', 'Thiago Guerrero', 'Daniela Cruz',
        'Leonardo Ramos', 'Fernanda Solis', 'Bruno Montes', 'Ana Reyes', 'Tomas Rivera',
        'Sofia Delgado', 'Diego Moreno', 'Valentina Jimenez', 'Pablo Romero', 'Natalia Vargas',
        'Hiroshi Tanaka', 'Yuki Sato', 'Kenji Watanabe', 'Sakura Suzuki', 'Takeshi Honda',
        'Akiko Yamada', 'Ryu Nakamura', 'Hana Ito', 'Sora Kobayashi', 'Aoi Yoshida',
        'Wei Zhang', 'Li Wang', 'Jing Liu', 'Chen Zhao', 'Ming Huang',
        'Mei Lin', 'Yong Wu', 'Xin Yang', 'Hong Chen', 'Fei Zhou',
        'Ji-Hoon Kim', 'Min-Jee Park', 'Seung Lee', 'Eun-Ji Choi', 'Hyun Woo',
        'In-Soo Jung', 'Soo-Min Kang', 'Dong-Hyun Cho', 'Yeon-Hee Yoon', 'Tae-Young Jang',
        'Arjun Patel', 'Priya Sharma', 'Raj Kumar', 'Ananya Singh', 'Vikram Gupta',
        'Deepa Verma', 'Rohan Malhotra', 'Pooja Reddy', 'Aditya Joshi', 'Kavya Nair',
        'Mohammed Ali', 'Fatima Hassan', 'Omar Khan', 'Aisha Ahmed', 'Youssef Ibrahim',
        'Layla Mohamed', 'Hassan Mahmoud', 'Noor Abdel', 'Karim Mustafa', 'Amira Khalil',
        'Ivan Petrov', 'Natasha Ivanova', 'Dmitri Volkov', 'Olga Smirnova', 'Alexei Kuznetsov',
        'Marina Popova', 'Sergei Vasiliev', 'Ekaterina Novikova', 'Andrei Morozov', 'Tatiana Fedorova',
        'Pierre Dubois', 'Marie Laurent', 'Jean Martin', 'Sophie Bernard', 'Louis Moreau',
        'Camille Petit', 'Antoine Leroy', 'Chloe Rousseau', 'Emile Fontaine', 'Amelie Dupont',
        'Hans Mueller', 'Greta Fischer', 'Klaus Schmidt', 'Heidi Weber', 'Fritz Wagner',
        'Ingrid Becker', 'Wolfgang Schulz', 'Lena Hoffman', 'Dieter Richter', 'Ursula Koch',
        'Luca Rossi', 'Giulia Romano', 'Marco Colombo', 'Chiara Ricci', 'Alessandro Marino',
        'Valentina Greco', 'Giovanni Ferrari', 'Francesca Conti', 'Matteo Esposito', 'Sara Bianchi',
        'Joao Silva', 'Ana Santos', 'Pedro Oliveira', 'Maria Souza', 'Carlos Lima',
        'Juliana Costa', 'Lucas Ferreira', 'Camila Pereira', 'Bruno Almeida', 'Beatriz Carvalho',
        'Patrick Murphy', 'Siobhan Kelly', 'Conor Ryan', 'Aoife Walsh', 'Declan Byrne',
        'Niamh McCarthy', 'Cian Sullivan', 'Roisin Doyle', 'Oisin Murray', 'Fiona Lynch',
        'SilverFox', 'NightOwl', 'StormRider', 'CyberNinja', 'PixelHeart',
        'DarkPhoenix', 'CosmicRay', 'QuantumLeap', 'NeonDream', 'SolarFlare',
        'MoonShadow', 'FireStorm', 'AquaMarine', 'ThunderBolt', 'CrystalWave',
        'VelvetSky', 'GoldenEagle', 'MidnightSun', 'DiamondDust', 'RubyStone',
        'EchoValley', 'FrostBite', 'BlazeStar', 'OceanBreeze', 'WildHeart',
        'SkyWalker', 'IronWolf', 'GhostRider', 'ShadowHawk', 'DragonFly',
        'StarLight', 'TechWiz', 'CodeMaster', 'DataPunk', 'ByteRunner',
        'NetSurfer', 'WebWizard', 'CloudNine', 'PixelPioneer', 'CyberPulse',
        'DigitalSage', 'VirtualVoyager', 'TechOracle', 'BinaryBlaze', 'LogicLord',
        'AlphaWave', 'BetaCoder', 'GammaRay', 'DeltaForce', 'OmegaPoint',
        'ZenMaster', 'KarmaKing', 'LotusFlower', 'PeaceMaker', 'DreamCatcher',
        'SoulSearcher', 'MindBender', 'HeartBreaker', 'PathFinder', 'TruthSeeker',
        'John Smith', 'Maria Garcia', 'David Lee', 'Sarah Kim', 'Michael Johnson',
        'Jennifer Brown', 'Robert Taylor', 'Lisa Anderson', 'William Thomas', 'Nancy Jackson',
        'Richard White', 'Betty Harris', 'Joseph Martin', 'Dorothy Thompson', 'Thomas Garcia',
        'Margaret Martinez', 'Charles Robinson', 'Helen Clark', 'Daniel Rodriguez', 'Carol Lewis',
        'Alex Rivers', 'Jordan Taylor', 'Casey Morgan', 'Riley Brooks', 'Quinn Harper',
        'Avery Stone', 'Parker Lane', 'Blake Foster', 'Drew Mitchell', 'Charlie West',
        'Sage Cooper', 'Phoenix Reed', 'Emerson Gray', 'Hayden Fox', 'Dakota Pierce',
        'Skyler Quinn', 'Finley Rose', 'Lennox Cole', 'Marlowe Hunt', 'Harlow Dean',
        'Joaquin Vasquez', 'Esperanza Flores', 'Roberto Mendez', 'Luz Ramirez', 'Fernando Castillo',
        'Veronica Salazar', 'Miguel Herrera', 'Patricia Cruz', 'Ricardo Dominguez', 'Alejandra Ortiz',
        'Vladimir Kozlov', 'Anastasia Romanova', 'Boris Sokolov', 'Svetlana Petrova', 'Igor Volkov',
        'Yelena Kuznetsova', 'Maxim Orlov', 'Daria Novak', 'Pavel Morozov', 'Irina Popova',
        'Tariq Al-Rashid', 'Leila Nazari', 'Hamid Hosseini', 'Zahra Ahmadi', 'Reza Mohammadi',
        'Soraya Farahani', 'Kian Tehrani', 'Parisa Shirazi', 'Behnam Kazemi', 'Maryam Javadi',
        'Kwame Mensah', 'Adama Diallo', 'Chidi Okafor', 'Amara Toure', 'Jabari Osei',
        'Fatou Diop', 'Tendai Moyo', 'Aisha Kamara', 'Emeka Uzoma', 'Zainab Bello',
        'Olaf Johansen', 'Astrid Eriksen', 'Bjorn Larsen', 'Freya Andersen', 'Magnus Nilsen',
        'Ingrid Olsen', 'Lars Kristiansen', 'Sigrid Pedersen', 'Erik Haugen', 'Solveig Berg',
        'Yuki Miyamoto', 'Haruto Shimizu', 'Akira Fujimoto', 'Rin Hayashi', 'Sota Mori',
        'Mio Ogawa', 'Kaito Hashimoto', 'Mei Ishikawa', 'Ren Okada', 'Hinata Maeda',
        'Carlos Alberto', 'Fernanda Lima', 'Paulo Ricardo', 'Amanda Santos', 'Rafael Oliveira',
        'Gabriela Ribeiro', 'Thiago Nascimento', 'Leticia Barbosa', 'Gustavo Araujo', 'Larissa Moreira',
        'Nikolai Orlov', 'Misha Volkov', 'Sasha Petrov', 'Katya Ivanova', 'Yuri Sokolov',
        'Harper James', 'Jasmine Cloud', 'Felix Storm', 'Ruby Ocean', 'Zephyr Moon',
        'Sage Autumn', 'River Dawn', 'Phoenix Ember', 'Aurora Night', 'Orion Sky',
        'Atlas World', 'Luna Silver', 'Nova Bright', 'Stella Frost', 'Cassius Blaze'
    ];
    return $authors[array_rand($authors)];
}

function getOrCreateUserId($pdo, $author) {
    // Check if user already exists
    $stmt = $pdo->prepare("SELECT ID FROM user_myhashtag WHERE username = ? LIMIT 1");
    $stmt->execute([$author]);
    $existingId = $stmt->fetchColumn();
    if ($existingId) return (int)$existingId;

    // Create new user
    $parts = explode(' ', $author);
    $firstName = ucfirst($parts[0]);
    $lastName = ucfirst($parts[1] ?? 'User');
    $email = strtolower(preg_replace('/[^a-zA-Z0-9]/', '', $author)) . '@digupdog.com';
    $password = password_hash('Raimundinho1', PASSWORD_DEFAULT);
    $birthdate = date('Y-m-d', strtotime('-' . rand(18, 50) . ' years'));
    $bio = "Content creator and web explorer. Sharing discoveries from around the internet.";
    $gender = rand(0, 1) ? 'male' : 'female';
    $privacy = json_encode(['visibility' => 'public']);
    $social = json_encode(['facebook' => '', 'twitter' => '']);
    $prefs = json_encode(['theme' => 'dark', 'notifications' => true]);

    try {
        $stmt = $pdo->prepare("INSERT INTO user_myhashtag (username, email, senha, first_name, last_name, address, phone_number, created_at, birthdate, profile_picture, bio, gender, status, user_role, privacy_settings, social_links, preferences) VALUES (?, ?, ?, ?, ?, ?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?)");
        $stmt->execute([
            $author, $email, $password, $firstName, $lastName,
            '123 Web Street, Internet City', '+1' . rand(1000000000, 9999999999),
            $birthdate, 'https://example.com/default_profile.jpg', $bio,
            $gender, 'active', 'user', $privacy, $social, $prefs
        ]);
        return (int)$pdo->lastInsertId();
    } catch (Exception $e) {
        return 0; // Fallback to 0 if user creation fails
    }
}

function getOrCreateDomainId($pdo, $url) {
    $host = parse_url($url, PHP_URL_HOST);
    if (!$host) return ['domain_id' => 0, 'category_id' => 0];

    $baseUrl = 'https://' . $host;

    // Check if domain exists in feed_data
    $stmt = $pdo->prepare("SELECT id, main_category_id FROM feed_data WHERE website_base LIKE ? LIMIT 1");
    $stmt->execute(['%' . $host . '%']);
    $row = $stmt->fetch(PDO::FETCH_ASSOC);

    if ($row) {
        return ['domain_id' => (int)$row['id'], 'category_id' => (int)$row['main_category_id']];
    }

    // Create new entry in feed_data
    try {
        $stmt = $pdo->prepare("INSERT INTO feed_data (website_base, website_feed, main_category_id, subcategory_id, subcategory_name, subsubcategory_id, subsubcategory_name) VALUES (?, ?, 0, NULL, NULL, NULL, NULL)");
        $stmt->execute([$baseUrl, $url]);
        return ['domain_id' => (int)$pdo->lastInsertId(), 'category_id' => 0];
    } catch (Exception $e) {
        return ['domain_id' => 0, 'category_id' => 0];
    }
}

function extractEmbedCode($url) {
    $host = parse_url($url, PHP_URL_HOST);
    if (!$host) return '';
    $host = strtolower(preg_replace('/^www\./', '', $host));

    // YouTube
    if (in_array($host, ['youtube.com', 'm.youtube.com', 'youtu.be'])) {
        $videoId = null;
        if ($host === 'youtu.be') {
            $videoId = ltrim(parse_url($url, PHP_URL_PATH), '/');
        } else {
            parse_str(parse_url($url, PHP_URL_QUERY) ?? '', $q);
            $videoId = $q['v'] ?? null;
            if (!$videoId && preg_match('#/(shorts|embed|v)/([a-zA-Z0-9_-]+)#', $url, $m)) {
                $videoId = $m[2];
            }
        }
        if ($videoId) {
            return '<iframe src="https://www.youtube.com/embed/' . htmlspecialchars($videoId) . '" width="560" height="315" frameborder="0" allowfullscreen></iframe>';
        }
    }

    // Vimeo
    if (in_array($host, ['vimeo.com', 'player.vimeo.com'])) {
        if (preg_match('#/(\d+)#', parse_url($url, PHP_URL_PATH), $m)) {
            return '<iframe src="https://player.vimeo.com/video/' . $m[1] . '" width="560" height="315" frameborder="0" allowfullscreen></iframe>';
        }
    }

    // TikTok
    if (strpos($host, 'tiktok.com') !== false) {
        if (preg_match('#/video/(\d+)#', $url, $m)) {
            return '<iframe src="https://www.tiktok.com/embed/' . $m[1] . '" width="560" height="315" frameborder="0" allowfullscreen></iframe>';
        }
    }

    // Dailymotion
    if (in_array($host, ['dailymotion.com', 'dai.ly'])) {
        if (preg_match('#/(video|embed/video)/([a-zA-Z0-9]+)#', $url, $m)) {
            return '<iframe src="https://www.dailymotion.com/embed/video/' . $m[2] . '" width="560" height="315" frameborder="0" allowfullscreen></iframe>';
        }
    }

    // Twitch
    if ($host === 'twitch.tv' || $host === 'clips.twitch.tv') {
        if (preg_match('#twitch\.tv/([a-zA-Z0-9_]+)#', $url, $m)) {
            return '<iframe src="https://player.twitch.tv/?channel=' . $m[1] . '&parent=digupdog.net" width="560" height="315" frameborder="0" allowfullscreen></iframe>';
        }
    }

    // SoundCloud
    if ($host === 'soundcloud.com') {
        return '<iframe width="100%" height="166" scrolling="no" frameborder="no" src="https://w.soundcloud.com/player/?url=' . urlencode($url) . '"></iframe>';
    }

    // Spotify
    if (strpos($host, 'spotify.com') !== false) {
        if (preg_match('#/(track|album|playlist|episode)/([a-zA-Z0-9]+)#', $url, $m)) {
            return '<iframe src="https://open.spotify.com/embed/' . $m[1] . '/' . $m[2] . '" width="300" height="380" frameborder="0" allowfullscreen></iframe>';
        }
    }

    return ''; // No embed for non-video/audio URLs
}

function extractTagsFromTitle($title) {
    if (empty($title)) return '';
    $title = preg_replace("/[.,\/#!$%\^&\*;:{}=\-_`~()\[\]\"']/", " ", $title);
    $title = mb_strtolower($title);
    $words = preg_split('/\s+/', trim($title));

    $commonWords = ['a','an','the','and','or','but','in','at','on','with','to','for','is','of',
        'that','it','by','from','as','are','was','be','has','have','will','this','which','its',
        'about','up','more','who','also','they','out','he','she','you','their','we','her','his',
        'them','been','these','would','some','can','like','there','if','all','my','what','so',
        'then','into','just','over','do','than','when','other','how','our','any','new','me',
        'after','most','made','only','time','where','year','years','make','does','could','were',
        'your','good','well','not','no','very','much','get','got','one','two','still','now',
        'first','last','next','back','being','here','those','such','us','own','way','each'];

    $keywords = array_filter($words, function($w) use ($commonWords) {
        return mb_strlen($w) > 2 && !in_array($w, $commonWords) && !is_numeric($w);
    });

    return mb_substr(implode(', ', array_values(array_unique($keywords))), 0, 500);
}

// ============================================
// IMPORT FUNCTIONS
// ============================================
function checkDuplicates($pdo, $links) {
    if (empty($links)) return [];

    $placeholders = implode(',', array_fill(0, count($links), '?'));
    $stmt = $pdo->prepare("SELECT link FROM pinfeeds WHERE link IN ($placeholders)");
    $stmt->execute($links);

    $existing = [];
    while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
        $existing[$row['link']] = true;
    }

    return $existing;
}

function importToPinfeeds($pdo, $records) {
    if (empty($records)) return 0;

    static $domainCache = [];
    static $userCache = [];

    $columns = 'title, description, thumbnail, pubDate, link, updated, source_website, author, favicon, tags, embed_code, source_domain, user_id, source_domain_id, main_category_id, title_cat_id, description_cat_id, tag_cat_id';

    $pdo->beginTransaction();
    $inserted = 0;

    try {
        foreach (array_chunk($records, 25) as $chunk) {
            $placeholders = [];
            $values = [];

            foreach ($chunk as $data) {
                $link = $data['link'] ?? '';
                if (!filter_var($link, FILTER_VALIDATE_URL)) continue;

                $host = parse_url($link, PHP_URL_HOST) ?? '';
                $author = $data['author'] ?? '';
                if (empty($author) || $author === 'Anonymous' || mb_strlen($author) < 3) {
                    $author = generateRandomAuthor();
                }

                // User ID
                if (!isset($userCache[$author])) {
                    $userCache[$author] = getOrCreateUserId($pdo, $author);
                }
                $userId = $userCache[$author];

                // Domain ID
                if (!isset($domainCache[$host])) {
                    $domainCache[$host] = getOrCreateDomainId($pdo, $link);
                }
                $domainId = $domainCache[$host]['domain_id'];
                $categoryId = $domainCache[$host]['category_id'];

                $embedCode = extractEmbedCode($link);
                $tags = $data['tags'] ?? '';
                if (empty($tags) && !empty($data['title'])) {
                    $tags = extractTagsFromTitle($data['title']);
                }

                $placeholders[] = '(?, ?, ?, NOW(), ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0)';
                $values[] = mb_substr($data['title'] ?? 'No title', 0, 255);
                $values[] = mb_substr($data['description'] ?? '', 0, 1000);
                $values[] = mb_substr($data['thumbnail'] ?? '', 0, 500);
                $values[] = $link;
                $values[] = mb_substr($host, 0, 255);
                $values[] = mb_substr($author, 0, 255);
                $values[] = mb_substr($data['favicon'] ?? '', 0, 255);
                $values[] = mb_substr($tags, 0, 500);
                $values[] = mb_substr($embedCode, 0, 2000);
                $values[] = mb_substr($host, 0, 255);
                $values[] = $userId;
                $values[] = $domainId;
                $values[] = $categoryId;
            }

            if (!empty($placeholders)) {
                $sql = "INSERT INTO pinfeeds ($columns) VALUES " . implode(', ', $placeholders);
                $stmt = $pdo->prepare($sql);
                $stmt->execute($values);
                $inserted += $stmt->rowCount();
            }
        }

        $pdo->commit();
    } catch (Exception $e) {
        $pdo->rollBack();
        throw $e;
    }

    return $inserted;
}

// ============================================
// INFINITE MODE FUNCTIONS
// ============================================
function runInfiniteMode($keywords, $config, $pdo) {
    // Persist even if browser disconnects (critical for long crawls)
    ignore_user_abort(true);

    // Set up streaming output
    header('Content-Type: text/event-stream');
    header('Cache-Control: no-cache');
    header('Connection: keep-alive');
    header('X-Accel-Buffering: no');

    ob_implicit_flush(true);
    if (ob_get_level()) ob_end_clean();

    $stats = [
        'discovered' => 0,
        'processed' => 0,
        'imported' => 0,
        'duplicates' => 0,
        'errors' => 0,
        'queue_size' => 0,
        'start_time' => time(),
        'status' => 'running',
        'current_engine' => '',
        'last_url' => '',
        'log' => []
    ];

    $queue = [];
    $visited = [];
    $engines = ['searxng', 'wikipedia', 'google', 'bing_regional', 'direct_sites', 'brave', 'duckduckgo', 'bing', 'yandex', 'yahoo'];
    $engineIndex = 0;
    $pageOffsets = ['bing' => 1, 'duckduckgo' => 0, 'yandex' => 0, 'yahoo' => 0, 'searxng' => 0, 'wikipedia' => 0, 'direct_sites' => 0, 'brave' => 0, 'google' => 0, 'bing_regional' => 0];

    // Session-based stop flag
    session_start();
    $_SESSION['infinite_running'] = true;
    session_write_close();

    outputStats($stats);

    $maxImports = $config['max_imports'] ?? 10000000;
    $batchSize = $config['batch_size'] ?? 50;

    while ($stats['imported'] < $maxImports) {
        // Check stop flag
        session_start();
        if (empty($_SESSION['infinite_running'])) {
            $stats['status'] = 'stopped';
            outputStats($stats);
            break;
        }
        session_write_close();

        // 1. DISCOVER - Add more links to queue if needed
        if (count($queue) < 1000) {
            $engine = $engines[$engineIndex % count($engines)];
            $stats['current_engine'] = $engine;

            $newLinks = discoverLinksForInfinite($keywords, $engine, $pageOffsets[$engine]);
            $pageOffsets[$engine] += 10;
            $engineIndex++;

            // Filter out already visited
            $newLinks = array_diff($newLinks, $visited);
            $queue = array_merge($queue, $newLinks);
            $queue = array_unique($queue);

            $stats['discovered'] += count($newLinks);
            $stats['queue_size'] = count($queue);

            // Deep crawl some links for more URLs
            if ($config['deep_crawl'] && count($newLinks) > 0) {
                $sampleLinks = array_slice($newLinks, 0, 3);
                foreach ($sampleLinks as $link) {
                    $deepLinks = extractLinksFromPage($link);
                    $deepLinks = array_diff($deepLinks, $visited);
                    $queue = array_merge($queue, array_slice($deepLinks, 0, 50));
                    $stats['discovered'] += min(count($deepLinks), 50);
                }
            }

            outputStats($stats);
        }

        // 2. PROCESS BATCH
        if (empty($queue)) {
            usleep(500000); // Wait 0.5s if queue empty
            continue;
        }

        $batch = array_splice($queue, 0, $batchSize);
        $stats['queue_size'] = count($queue);

        // 3. CHECK DUPLICATES
        $existingLinks = checkDuplicatesArray($pdo, $batch);
        $newLinks = array_filter($batch, function($link) use ($existingLinks) {
            return !isset($existingLinks[$link]);
        });

        $dupCount = count($batch) - count($newLinks);
        $stats['duplicates'] += $dupCount;

        // Log duplicates
        $stats['log'] = [];
        foreach ($batch as $link) {
            if (isset($existingLinks[$link])) {
                $stats['log'][] = ['url' => $link, 'status' => 'duplicate', 'title' => ''];
            }
        }

        if (empty($newLinks)) {
            $stats['processed'] += count($batch);
            outputStats($stats);
            continue;
        }

        // 4. EXTRACT METADATA
        $metadata = extractBatchMetadata($newLinks, 10);

        // 5. IMPORT DIRECTLY
        $records = [];
        foreach ($newLinks as $link) {
            $meta = $metadata[$link] ?? [];
            if (isset($meta['error'])) {
                $stats['errors']++;
                $stats['log'][] = ['url' => $link, 'status' => 'error', 'title' => $meta['error'] ?? 'Fetch failed'];
                continue;
            }
            $records[] = array_merge(['link' => $link], $meta);
            $visited[$link] = true;
        }

        if (!empty($records)) {
            try {
                $imported = importToPinfeedsDirect($pdo, $records);
                $stats['imported'] += $imported;
                $stats['last_url'] = end($records)['link'] ?? '';

                // Log imported records
                foreach ($records as $rec) {
                    $stats['log'][] = [
                        'url' => $rec['link'],
                        'status' => 'imported',
                        'title' => mb_substr($rec['title'] ?? '', 0, 60),
                        'domain' => parse_url($rec['link'], PHP_URL_HOST) ?? ''
                    ];
                }
            } catch (Exception $e) {
                $stats['errors'] += count($records);
                foreach ($records as $rec) {
                    $stats['log'][] = ['url' => $rec['link'], 'status' => 'error', 'title' => 'DB Error'];
                }
            }
        }

        $stats['processed'] += count($batch);
        outputStats($stats);

        // Small delay to prevent overload
        usleep(100000); // 0.1s
    }

    $stats['status'] = 'completed';
    outputStats($stats);
    exit;
}

function discoverLinksForInfinite($keywords, $engine, $offset) {
    $links = [];
    $keyword = $keywords[array_rand($keywords)];

    switch ($engine) {
        case 'searxng':
            $links = getSearXNGLinks($keyword, 20);
            break;
        case 'wikipedia':
            $links = getWikipediaLinks($keyword, 15);
            break;
        case 'google':
            $links = getGoogleSearchLinks($keyword, 20);
            break;
        case 'bing_regional':
            $links = getBingRegionalLinks($keyword, 20);
            break;
        case 'direct_sites':
            $links = getDirectSearchLinks($keyword, 15);
            break;
        case 'brave':
            $links = getBraveSearchLinks($keyword, 15);
            break;
        case 'duckduckgo':
            $links = getDuckDuckGoLinks($keyword, 20);
            break;
        case 'bing':
            $links = getBingSearchLinks($keyword, 20);
            break;
        case 'yandex':
            $links = getYandexSearchLinks($keyword, 15);
            break;
        case 'yahoo':
            $links = getYahooSearchLinks($keyword, 15);
            break;
    }

    return $links;
}

function extractLinksFromPage($url) {
    $html = fetchWithCurl($url, 5);
    if (!$html) return [];

    $links = [];
    preg_match_all('/<a[^>]+href="(https?:\/\/[^"]+)"/i', $html, $matches);

    foreach ($matches[1] as $link) {
        // Skip social, search engines, etc
        if (preg_match('/facebook|twitter|instagram|linkedin|google|bing|yahoo|yandex/i', $link)) {
            continue;
        }
        $links[] = $link;
    }

    return array_unique(array_slice($links, 0, 100));
}

function checkDuplicatesArray($pdo, $links) {
    if (empty($links)) return [];

    $placeholders = implode(',', array_fill(0, count($links), '?'));
    $stmt = $pdo->prepare("SELECT link FROM pinfeeds WHERE link IN ($placeholders)");
    $stmt->execute(array_values($links));

    $existing = [];
    while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
        $existing[$row['link']] = true;
    }

    return $existing;
}

function importToPinfeedsDirect($pdo, $records) {
    if (empty($records)) return 0;

    static $domainCache = [];
    static $userCache = [];

    $columns = 'title, description, thumbnail, pubDate, link, updated, source_website, author, favicon, tags, embed_code, source_domain, user_id, source_domain_id, main_category_id, title_cat_id, description_cat_id, tag_cat_id';
    $inserted = 0;

    foreach (array_chunk($records, 25) as $chunk) {
        try {
            $placeholders = [];
            $values = [];

            foreach ($chunk as $data) {
                $link = $data['link'] ?? '';
                if (!filter_var($link, FILTER_VALIDATE_URL)) continue;

                $host = parse_url($link, PHP_URL_HOST) ?? '';
                $author = $data['author'] ?? '';
                if (empty($author) || $author === 'Anonymous' || mb_strlen($author) < 3) {
                    $author = generateRandomAuthor();
                }

                if (!isset($userCache[$author])) {
                    $userCache[$author] = getOrCreateUserId($pdo, $author);
                }
                $userId = $userCache[$author];

                if (!isset($domainCache[$host])) {
                    $domainCache[$host] = getOrCreateDomainId($pdo, $link);
                }
                $domainId = $domainCache[$host]['domain_id'];
                $categoryId = $domainCache[$host]['category_id'];

                $embedCode = extractEmbedCode($link);
                $tags = $data['tags'] ?? '';
                if (empty($tags) && !empty($data['title'])) {
                    $tags = extractTagsFromTitle($data['title']);
                }

                $placeholders[] = '(?, ?, ?, NOW(), ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0)';
                $values[] = mb_substr($data['title'] ?? 'No title', 0, 255);
                $values[] = mb_substr($data['description'] ?? '', 0, 1000);
                $values[] = mb_substr($data['thumbnail'] ?? '', 0, 500);
                $values[] = $link;
                $values[] = mb_substr($host, 0, 255);
                $values[] = mb_substr($author, 0, 255);
                $values[] = mb_substr($data['favicon'] ?? '', 0, 255);
                $values[] = mb_substr($tags, 0, 500);
                $values[] = mb_substr($embedCode, 0, 2000);
                $values[] = mb_substr($host, 0, 255);
                $values[] = $userId;
                $values[] = $domainId;
                $values[] = $categoryId;
            }

            if (!empty($placeholders)) {
                $sql = "INSERT IGNORE INTO pinfeeds ($columns) VALUES " . implode(', ', $placeholders);
                $stmt = $pdo->prepare($sql);
                $stmt->execute($values);
                $inserted += $stmt->rowCount();
            }
        } catch (Exception $e) {
            error_log("Import error: " . $e->getMessage());
        }
    }

    return $inserted;
}

function outputStats($stats) {
    $elapsed = max(1, time() - $stats['start_time']);
    $stats['rate'] = round($stats['imported'] / $elapsed, 1);
    $stats['elapsed'] = $elapsed;

    echo "data: " . json_encode($stats) . "\n\n";

    if (ob_get_level()) {
        ob_flush();
    }
    flush();
}

// ============================================
// UTILITY FUNCTIONS (used by viral + infinite modes)
// ============================================
function isDomainSkipped($host) {
    static $hashSet = null;
    if ($hashSet === null) {
        $domains = [
            // Social media
            'facebook.com', 'twitter.com', 'x.com', 'instagram.com', 'linkedin.com',
            'pinterest.com', 'tiktok.com', 'reddit.com', 'tumblr.com', 'snapchat.com',
            'threads.net', 'mastodon.social', 'bsky.app',
            // Messaging
            'whatsapp.com', 'wa.me', 'telegram.org', 't.me', 'signal.org',
            'discord.com', 'discord.gg', 'slack.com',
            // Search engines
            'google.com', 'bing.com', 'yahoo.com', 'yandex.ru', 'yandex.com',
            'youtube.com', 'duckduckgo.com', 'baidu.com',
            // URL shorteners
            't.co', 'bit.ly', 'goo.gl', 'tinyurl.com', 'ow.ly', 'is.gd', 'buff.ly',
            'rebrand.ly', 'cutt.ly', 'shorturl.at',
            // E-commerce (no content value)
            'amazon.com', 'ebay.com', 'apple.com', 'aliexpress.com', 'shopify.com',
            'mercadolivre.com.br', 'magazineluiza.com.br', 'americanas.com.br',
            // CMS/Platform internals
            'wordpress.org', 'developer.wordpress.org', 'developer.mozilla.org',
            'w3.org', 'schema.org', 'github.com', 'gitlab.com', 'bitbucket.org',
            'stackoverflow.com', 'stackexchange.com',
            // Wiki/Educational
            'wikipedia.org', 'wikimedia.org', 'wikimediafoundation.org',
            'wikidata.org', 'wikisource.org', 'wiktionary.org',
            // International orgs
            'un.org', 'unesco.org', 'who.int', 'worldbank.org', 'imf.org', 'europa.eu',
            // CDN/Assets/Infrastructure
            'cloudflare.com', 'jsdelivr.net', 'cdnjs.cloudflare.com',
            'unpkg.com', 'fonts.googleapis.com', 'fonts.gstatic.com',
            'googletagmanager.com', 'googlesyndication.com', 'doubleclick.net',
            'google-analytics.com', 'googleadservices.com', 'fbcdn.net',
            'akamaihd.net', 'cloudfront.net', 'amazonaws.com',
            // Auth/OAuth
            'accounts.google.com', 'login.microsoftonline.com',
            // App stores
            'play.google.com', 'apps.apple.com', 'chrome.google.com',
            // Archive/cache
            'web.archive.org', 'archive.org', 'webcache.googleusercontent.com',
            // Ad/Tracking networks
            'doubleclick.net', 'adsense.google.com', 'ads.google.com',
            'facebook.net', 'connect.facebook.net', 'pixel.facebook.com',
            // Document/File hosting (no crawlable content)
            'docs.google.com', 'drive.google.com', 'dropbox.com',
            'onedrive.live.com', 'icloud.com',
            // Government (usually not content-relevant for pinfeeds)
            'gov.br', 'gov.uk', 'gov.us',
        ];
        // O(1) lookup using hash set
        $hashSet = array_flip($domains);
    }
    $host = strtolower($host);
    // Direct match O(1)
    if (isset($hashSet[$host])) return true;
    // Subdomain match: strip until found or empty
    $parts = explode('.', $host);
    for ($i = 1; $i < count($parts) - 1; $i++) {
        $parent = implode('.', array_slice($parts, $i));
        if (isset($hashSet[$parent])) return true;
    }
    return false;
}

function getBaseDomain($host) {
    $host = strtolower($host);
    $host = preg_replace('/^www\d*\./', '', $host);
    // Brazilian multi-part TLDs
    if (preg_match('/([^.]+\.(?:com|org|net|gov|edu|mil|leg|jus|blog|wiki|tur|eti)\.br)$/i', $host, $m)) {
        return $m[1];
    }
    // Other multi-part TLDs (co.uk, com.au, etc.)
    if (preg_match('/([^.]+\.(?:co|com|org|net|gov|edu|ac)\.[a-z]{2})$/i', $host, $m)) {
        return $m[1];
    }
    // Generic: last 2 parts
    $parts = explode('.', $host);
    return count($parts) >= 2 ? implode('.', array_slice($parts, -2)) : $host;
}

// ============================================
// VIRAL MODE FUNCTIONS
// ============================================
function runViralMode($keywords, $config, $pdo) {
    // Persist even if browser disconnects (critical for long crawls)
    ignore_user_abort(true);

    header('Content-Type: text/event-stream');
    header('Cache-Control: no-cache');
    header('Connection: keep-alive');
    header('X-Accel-Buffering: no');

    ob_implicit_flush(true);
    if (ob_get_level()) ob_end_clean();

    $pool = [];
    $visited = [];
    $importQueue = [];
    $networkEdgesSeen = [];
    $stats = [
        'discovered' => 0,
        'processed' => 0,
        'imported' => 0,
        'duplicates' => 0,
        'errors' => 0,
        'skipped' => 0,
        'pool_size' => 0,
        'waves' => 0,
        'start_time' => time(),
        'status' => 'running',
        'last_url' => '',
        'log' => [],
        'network' => []
    ];

    // Session-based stop flag
    session_start();
    $_SESSION['viral_running'] = true;
    session_write_close();

    $waveSize = $config['wave_size'] ?? 25;
    $initialWaveSize = $waveSize;
    $microBatchSize = $config['micro_batch'] ?? 10;
    $maxPool = $config['max_pool'] ?? 2000000;    // 2M pool for massive crawling
    $maxImports = $config['max_imports'] ?? 10000000;
    $domainLastFetch = []; // Rate limiting: domain => microtime
    $bandwidthBytes = 0;   // Total bytes downloaded
    $recentImports = [];   // In-memory dedup cache (last 100K)
    $linksPerPage = min($config['links_per_page'] ?? 150, 250);
    $waveDelayMs = 0;      // Zero delay for maximum throughput
    $includeTerms = $config['include_terms'] ?? [];
    $excludeTerms = $config['exclude_terms'] ?? [];
    $forcedDomains = $config['forced_domains'] ?? [];
    $canonicalSeen = []; // Canonical URL dedup: canonical => true
    $MIN_RELEVANCE = 2; // Minimum weighted relevance score to import

    // Domain-level tracking for quality control
    $domainFailCount = [];      // domain => consecutive fail count
    $domainUrlCount = [];       // domain => URLs added to pool (diversity limit)
    $domainStats = [];          // domain => {crawled, imported, errors, status, links}
    $domainBackoffUntil = [];   // R2: domain => wave number to resume (exponential backoff)
    $domainQuality = [];        // Smart quality: domain => score (0-100)
    $MAX_DOMAIN_FAILS = 4;     // Blacklist domain after 4 failures
    $maxPerDomainConfig = $config['max_per_domain'] ?? 0;
    $MAX_URLS_PER_DOMAIN = $maxPerDomainConfig > 0 ? $maxPerDomainConfig : PHP_INT_MAX; // 0 = Ilimitado
    $JUNK_THRESHOLD = 12;      // Auto-blacklist after 12 crawls with 0 imports

    // ENGINE CIRCUIT BREAKER: Track engine failures to skip broken engines
    $engineFailures = [];       // engine => consecutive failure count
    $engineCooldown = [];       // engine => timestamp when cooldown ends
    $ENGINE_FAIL_THRESHOLD = 3; // Skip engine after 3 consecutive failures
    $ENGINE_COOLDOWN_SECS = 120; // Cooldown period in seconds (2 minutes)

    // DOMAIN SEEDING CACHE: Avoid re-fetching same domains during seeding
    $seededDomains = [];        // domain => true (already seen in seeding)
    $seededDomainsLimit = 3;    // Max URLs per domain during initial seeding (diversity)

    // INFINITE MODE: Query rotation variables
    $QUERY_ROTATION_INTERVAL = 500;  // URLs processed before rotating query
    $queryVariations = ['{kw}', '{kw} site:.br', '{kw} artigo', '{kw} blog', '{kw} noticias', '{kw} 2024', '{kw} 2025', '{kw} video', '{kw} forum'];
    $currentQueryVariation = 0;

    // MEMORY MANAGEMENT
    $MEMORY_LIMIT_MB = 512;
    $GC_INTERVAL = 50;
    $consecutiveErrors = 0;
    $ERROR_THRESHOLD = 10;
    $PAUSE_ON_ERRORS = 60;

    // HELPER: Count URLs available for processing (not in cooldown)
    $countAvailableUrls = function($pool, $domainBackoffUntil, $domainLastFetch, $currentWave) {
        $available = 0;
        $now = microtime(true);
        foreach ($pool as $url) {
            $domain = parse_url($url, PHP_URL_HOST) ?? '';
            // Pular se em backoff
            if (isset($domainBackoffUntil[$domain]) && $currentWave < $domainBackoffUntil[$domain]) {
                continue;
            }
            // Pular se rate-limited (0.8s entre requests do mesmo dominio)
            if (isset($domainLastFetch[$domain]) && ($now - $domainLastFetch[$domain]) < 0.8) {
                continue;
            }
            $available++;
            if ($available >= 50) return $available; // Basta saber que tem suficiente
        }
        return $available;
    };

    // SEED: Get initial links - try ALL engines, accept URLs directly
    $stats['status'] = 'seeding';
    $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => 'Seeding: searching for initial links...', 'domain' => 'SYSTEM']];
    outputViralStats($stats);

    $poolSet = []; // Fast lookup: url => true

    foreach ($keywords as $kw) {
        // If keyword is a URL, add directly to pool with PRIORITY (front)
        if (preg_match('/^https?:\/\//i', trim($kw))) {
            $url = trim($kw);
            if (!isset($poolSet[$url])) {
                array_unshift($pool, $url); // Front of pool = crawled first
                $poolSet[$url] = true;
            }
            continue;
        }

        // Try search engines (bot-friendly first, then traditional)
        $engines = [
            'searxng' => function($q) { return getSearXNGLinks($q, 20); },
            'google' => function($q) { return getGoogleSearchLinks($q, 20); },
            'bing_regional' => function($q) { return getBingRegionalLinks($q, 20); },
            'direct_sites' => function($q) { return getDirectSearchLinks($q, 15); },
            'brave' => function($q) { return getBraveSearchLinks($q, 15); },
            'duckduckgo' => function($q) { return getDuckDuckGoLinks($q, 15); },
            'bing' => function($q) { return getBingSearchLinks($q, 15); },
            'yandex' => function($q) { return getYandexSearchLinks($q, 10); },
            'yahoo' => function($q) { return getYahooSearchLinks($q, 10); },
        ];

        foreach ($engines as $engineName => $engineFn) {
            // CIRCUIT BREAKER: Skip engines that have failed too many times
            $currentTime = time();
            if (isset($engineCooldown[$engineName]) && $currentTime < $engineCooldown[$engineName]) {
                $remaining = $engineCooldown[$engineName] - $currentTime;
                $stats['log'] = [['url' => '', 'status' => 'skipped', 'title' => "$engineName: COOLDOWN ({$remaining}s remaining)", 'domain' => 'CIRCUIT']];
                outputViralStats($stats);
                continue; // Skip this engine, it's in cooldown
            }

            $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => "Seeding: trying $engineName for '$kw'...", 'domain' => 'SEED']];
            outputViralStats($stats);

            $links = $engineFn($kw);
            $addedFromEngine = 0;

            foreach ($links as $link) {
                $seedHost = parse_url($link, PHP_URL_HOST) ?? '';
                if (isDomainSkipped($seedHost)) continue;

                // DOMAIN DIVERSITY: Limit URLs per domain during seeding
                $seedBaseDomain = getBaseDomain($seedHost);
                if (($seededDomains[$seedBaseDomain] ?? 0) >= $seededDomainsLimit) continue;

                if (!isset($poolSet[$link])) {
                    $pool[] = $link;
                    $poolSet[$link] = true;
                    $seededDomains[$seedBaseDomain] = ($seededDomains[$seedBaseDomain] ?? 0) + 1;
                    $addedFromEngine++;
                }
            }

            if ($addedFromEngine > 0) {
                // SUCCESS: Reset failure count for this engine
                $engineFailures[$engineName] = 0;
                unset($engineCooldown[$engineName]);

                $stats['discovered'] = count($pool);
                $stats['pool_size'] = count($pool);
                $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => "$engineName: +$addedFromEngine links for '$kw' (diversity filtered)", 'domain' => 'SEED']];
                outputViralStats($stats);
            } else {
                // FAILURE: Increment failure count
                $engineFailures[$engineName] = ($engineFailures[$engineName] ?? 0) + 1;

                if ($engineFailures[$engineName] >= $ENGINE_FAIL_THRESHOLD) {
                    // Put engine in cooldown
                    $engineCooldown[$engineName] = $currentTime + $ENGINE_COOLDOWN_SECS;
                    $stats['log'] = [['url' => '', 'status' => 'error', 'title' => "$engineName: BLOCKED ({$engineFailures[$engineName]}x fails) - cooldown {$ENGINE_COOLDOWN_SECS}s", 'domain' => 'CIRCUIT']];
                } else {
                    $stats['log'] = [['url' => '', 'status' => 'skipped', 'title' => "$engineName: 0 results (fail #{$engineFailures[$engineName]})", 'domain' => 'SEED']];
                }
                outputViralStats($stats);
            }

            // Once we have enough diverse seeds for THIS keyword, try next keyword
            if (count($pool) >= 10) {
                break;
            }
        }

        // CONTINUE TO NEXT KEYWORD even with seeds (collect more diverse seeds)
        // Only stop seeding when we have plenty from multiple keywords
        if (count($pool) >= 50) {
            break;
        }
    }

    // LAST RESORT: If still empty, try Wikipedia API directly (always accessible)
    if (empty($pool)) {
        $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => 'All engines failed. Trying Wikipedia API...', 'domain' => 'FALLBACK']];
        outputViralStats($stats);

        foreach ($keywords as $kw) {
            $wikiLinks = getWikipediaLinks(trim($kw), 30);
            foreach ($wikiLinks as $link) {
                $wikiHost = parse_url($link, PHP_URL_HOST) ?? '';
                if (isDomainSkipped($wikiHost)) continue;
                if (!isset($poolSet[$link])) {
                    $pool[] = $link;
                    $poolSet[$link] = true;
                }
            }
            if (count($pool) >= 5) break;
        }
    }

    $stats['discovered'] = count($pool);
    $stats['pool_size'] = count($pool);

    if (empty($pool)) {
        $stats['status'] = 'completed';
        $stats['log'] = [['url' => '', 'status' => 'error', 'title' => 'No seeds found. Try using direct URLs as keywords (https://...)', 'domain' => 'ERROR']];
        outputViralStats($stats);
        exit;
    }

    $stats['status'] = 'running';
    $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => 'Pool seeded with ' . count($pool) . ' links. Spreading...', 'domain' => 'SYSTEM']];
    outputViralStats($stats);

    // RE-SEED engine rotation (bot-friendly first)
    $reseedEngines = ['searxng', 'google', 'bing_regional', 'direct_sites', 'brave', 'duckduckgo', 'bing', 'yandex', 'yahoo'];
    $reseedIndex = 0;
    $engineStats = []; // I1: Track seeds per engine

    // VIRAL LOOP
    while ((count($pool) > 0 || !empty($importQueue)) && $stats['imported'] < $maxImports) {
        // Check stop/pause flags
        session_start();
        if (empty($_SESSION['viral_running'])) {
            session_write_close();
            $stats['status'] = 'stopped';
            // Graceful shutdown: process remaining importQueue
            if (!empty($importQueue)) {
                $links = array_column($importQueue, 'link');
                $existing = checkDuplicatesArray($pdo, $links);
                $toImport = array_filter($importQueue, fn($r) => !isset($existing[$r['link']]));
                if (!empty($toImport)) {
                    try { $stats['imported'] += importMicroBatch($pdo, array_values($toImport)); } catch (Exception $e) {}
                }
                $importQueue = [];
            }
            $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => 'Stopped by user. Final: ' . $stats['imported'] . ' imports.', 'domain' => 'SYSTEM']];
            outputViralStats($stats);
            exit;
        }
        // Pause support
        while (!empty($_SESSION['viral_paused']) && !empty($_SESSION['viral_running'])) {
            session_write_close();
            $stats['status'] = 'paused';
            $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => 'Paused. Waiting...', 'domain' => 'SYSTEM']];
            outputViralStats($stats);
            sleep(2);
            session_start();
        }
        session_write_close();

        // DB health check every 100 waves
        if ($stats['waves'] > 0 && $stats['waves'] % 100 === 0) {
            try { $pdo->query('SELECT 1'); } catch (Exception $e) {
                try {
                    $pdo = new PDO("mysql:host=$DB_HOST;dbname=$DB_NAME;charset=utf8mb4", $DB_USER, $DB_PASS, [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]);
                } catch (PDOException $pe) {
                    $stats['log'] = [['url' => '', 'status' => 'error', 'title' => 'DB connection lost!', 'domain' => 'SYSTEM']];
                    outputViralStats($stats);
                    sleep(5);
                }
            }
        }

        // RE-SEED: When AVAILABLE URLs run low (not just total pool)
        // ANTES: count($pool) < 100 - PROBLEMA: pool pode ter 874 URLs mas todas em cooldown
        // AGORA: Verifica URLs realmente disponiveis
        $availableForReseed = $countAvailableUrls($pool, $domainBackoffUntil, $domainLastFetch, $stats['waves']);
        if (($availableForReseed < 30 || count($pool) < 200) && !empty($keywords)) {
            // CIRCUIT BREAKER: Find an engine that's not in cooldown
            $currentTime = time();
            $engine = null;
            $engineAttempts = 0;
            $maxEngineAttempts = count($reseedEngines);

            while ($engineAttempts < $maxEngineAttempts) {
                $candidateEngine = $reseedEngines[$reseedIndex % count($reseedEngines)];
                $reseedIndex++;
                $engineAttempts++;

                // Check if engine is in cooldown
                if (isset($engineCooldown[$candidateEngine]) && $currentTime < $engineCooldown[$candidateEngine]) {
                    continue; // Try next engine
                }
                $engine = $candidateEngine;
                break;
            }

            if ($engine === null) {
                // All engines in cooldown - wait and skip this re-seed
                $stats['log'] = [['url' => '', 'status' => 'skipped', 'title' => "Re-seed: ALL ENGINES IN COOLDOWN - waiting...", 'domain' => 'CIRCUIT']];
                outputViralStats($stats);
            } else {
                $keyword = $keywords[array_rand($keywords)];
                // Vary query to get different results each time
                $queryVariations = ['', ' noticias', ' blog', ' artigo', ' site:.br', ' 2024', ' 2025'];
                $variedKeyword = trim($keyword . ($queryVariations[$reseedIndex % count($queryVariations)] ?? ''));

                $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => "Re-seeding: $engine for '$variedKeyword'...", 'domain' => 'RESEED']];
                outputViralStats($stats);

                $newSeeds = [];
                switch ($engine) {
                    case 'searxng': $newSeeds = getSearXNGLinks($variedKeyword, 20); break;
                    case 'google': $newSeeds = getGoogleSearchLinks($variedKeyword, 20); break;
                    case 'bing_regional': $newSeeds = getBingRegionalLinks($variedKeyword, 20); break;
                    case 'direct_sites': $newSeeds = getDirectSearchLinks($variedKeyword, 15); break;
                    case 'brave': $newSeeds = getBraveSearchLinks($variedKeyword, 15); break;
                    case 'duckduckgo': $newSeeds = getDuckDuckGoLinks($variedKeyword, 15); break;
                    case 'bing': $newSeeds = getBingSearchLinks($variedKeyword, 15); break;
                    case 'yandex': $newSeeds = getYandexSearchLinks($variedKeyword, 10); break;
                    case 'yahoo': $newSeeds = getYahooSearchLinks($variedKeyword, 10); break;
                }

                $seedsAdded = 0;
                foreach ($newSeeds as $seed) {
                    if (!isset($poolSet[$seed]) && !isset($visited[$seed])) {
                        $seedDomain = parse_url($seed, PHP_URL_HOST) ?? '';
                        if (isDomainSkipped($seedDomain)) continue;
                        if (($domainFailCount[$seedDomain] ?? 0) >= $MAX_DOMAIN_FAILS) continue;
                        $seedBaseDomain = getBaseDomain($seedDomain);
                        if (($domainUrlCount[$seedBaseDomain] ?? 0) >= $MAX_URLS_PER_DOMAIN) continue;
                        $domainUrlCount[$seedBaseDomain] = ($domainUrlCount[$seedBaseDomain] ?? 0) + 1;
                        $pool[] = $seed;
                        $poolSet[$seed] = true;
                        $seedsAdded++;
                    }
                }

                if ($seedsAdded > 0) {
                    // SUCCESS: Reset engine failure count
                    $engineFailures[$engine] = 0;
                    unset($engineCooldown[$engine]);

                    $stats['discovered'] += $seedsAdded;
                    $stats['pool_size'] = count($pool);
                    // I1: Track engine stats
                    if (!isset($engineStats[$engine])) $engineStats[$engine] = ['seeds' => 0, 'reseeds' => 0, 'failures' => 0];
                    $engineStats[$engine]['seeds'] += $seedsAdded;
                    $engineStats[$engine]['reseeds']++;
                    $stats['engine_stats'] = $engineStats;
                    $stats['log'] = [['url' => '', 'status' => 'imported', 'title' => "Re-seed: +$seedsAdded links from $engine", 'domain' => 'RESEED']];
                    outputViralStats($stats);
                } else {
                    // FAILURE: Increment engine failure count
                    $engineFailures[$engine] = ($engineFailures[$engine] ?? 0) + 1;
                    if (!isset($engineStats[$engine])) $engineStats[$engine] = ['seeds' => 0, 'reseeds' => 0, 'failures' => 0];
                    $engineStats[$engine]['failures'] = ($engineStats[$engine]['failures'] ?? 0) + 1;

                    if ($engineFailures[$engine] >= $ENGINE_FAIL_THRESHOLD) {
                        // Put engine in cooldown
                        $engineCooldown[$engine] = $currentTime + $ENGINE_COOLDOWN_SECS;
                        $stats['log'] = [['url' => '', 'status' => 'error', 'title' => "Re-seed: $engine BLOCKED ({$engineFailures[$engine]}x fails) - cooldown {$ENGINE_COOLDOWN_SECS}s", 'domain' => 'CIRCUIT']];
                    } else {
                        $stats['log'] = [['url' => '', 'status' => 'skipped', 'title' => "Re-seed: $engine 0 results (fail #{$engineFailures[$engine]})", 'domain' => 'RESEED']];
                    }
                    $stats['engine_stats'] = $engineStats;
                    outputViralStats($stats);
                }
            } // end if ($engine !== null)
        } // end if (availableForReseed < 30 || count($pool) < 200)

        // URL PRIORITY: Every 5 waves, sort pool to prefer productive domains
        if ($stats['waves'] > 0 && $stats['waves'] % 5 === 0 && count($pool) > 50) {
            usort($pool, function($a, $b) use ($domainQuality, $domainStats) {
                $hostA = parse_url($a, PHP_URL_HOST) ?? '';
                $hostB = parse_url($b, PHP_URL_HOST) ?? '';
                $qA = $domainQuality[$hostA] ?? 50;
                $qB = $domainQuality[$hostB] ?? 50;
                // Bonus for domains that already produced imports
                if (isset($domainStats[$hostA]) && ($domainStats[$hostA]['imported'] ?? 0) > 0) $qA += 20;
                if (isset($domainStats[$hostB]) && ($domainStats[$hostB]['imported'] ?? 0) > 0) $qB += 20;
                // Bonus for article-like URLs
                if (preg_match('/\/post|\/article|\/blog|\/news|\/story|\d{4}[\/-]\d{2}/i', $a)) $qA += 10;
                if (preg_match('/\/post|\/article|\/blog|\/news|\/story|\d{4}[\/-]\d{2}/i', $b)) $qB += 10;
                return $qB - $qA; // Higher quality first
            });
        }

        // WAVE: Take up to waveSize URLs from pool (R2: exponential backoff + F5: rate limiting)
        $wave = [];
        $skipIndices = [];
        $now = microtime(true);
        foreach ($pool as $i => $poolUrl) {
            if (count($wave) >= $waveSize) break;
            $pd = parse_url($poolUrl, PHP_URL_HOST) ?? '';
            // R2: Exponential backoff - skip domain for 2^fails waves (max 32 waves delay)
            $fails = $domainFailCount[$pd] ?? 0;
            if ($fails >= $MAX_DOMAIN_FAILS) {
                $skipIndices[] = $i; // permanently blacklisted
                continue;
            }
            if ($fails > 0) {
                $backoffWaves = min(pow(2, $fails), 32);
                $domainLastFailWave = $domainBackoffUntil[$pd] ?? 0;
                if ($stats['waves'] < $domainLastFailWave) {
                    continue; // skip but keep in pool for later
                }
            }
            // F5: Rate limiting per domain (0.8s between fetches - fast but polite)
            if (isset($domainLastFetch[$pd]) && ($now - $domainLastFetch[$pd]) < 0.8) {
                continue; // skip but keep in pool for later
            }
            $wave[] = $poolUrl;
            $skipIndices[] = $i;
            $domainLastFetch[$pd] = $now;
        }
        // Remove selected + skipped from pool
        foreach (array_reverse($skipIndices) as $idx) {
            unset($pool[$idx]);
        }
        $pool = array_values($pool);

        // INFINITE MODE: Nao para se tem URLs esperando cooldown
        if (empty($wave) && empty($importQueue)) {
            $availableNow = $countAvailableUrls($pool, $domainBackoffUntil, $domainLastFetch, $stats['waves']);

            if (count($pool) > 0 && $availableNow === 0) {
                // Todas URLs em cooldown - ESPERAR ao inves de PARAR
                $stats['status'] = 'cooldown';
                $stats['log'] = [['url' => '', 'status' => 'imported',
                    'title' => 'All ' . count($pool) . ' URLs in domain cooldown. Waiting 1s...',
                    'domain' => 'COOLDOWN']];
                outputViralStats($stats);
                sleep(1);
                $stats['status'] = 'running';
                continue; // Tentar novamente - NAO PARA!
            }

            if (count($pool) > 0 && $availableNow > 0) {
                // Tem URLs disponiveis mas wave ficou vazio por algum motivo - continuar
                continue;
            }

            // Pool REALMENTE vazio - so entao para
            break;
        }

        if (!empty($wave)) {
            $stats['waves']++;

            // Fetch all pages in parallel with cURL multi
            $results = fetchBatchWithCurl($wave, 4, $waveSize);
            $stats['log'] = [];

            foreach ($results as $url => $result) {
                $visited[$url] = true;
                $poolSet[$url] = true;
                $fetchDomain = parse_url($url, PHP_URL_HOST) ?? '';

                if ($result['http_code'] < 200 || $result['http_code'] >= 400 || empty($result['html'])) {
                    // Track domain failure + R2: set exponential backoff
                    $domainFailCount[$fetchDomain] = ($domainFailCount[$fetchDomain] ?? 0) + 1;
                    $fails = $domainFailCount[$fetchDomain];
                    $domainBackoffUntil[$fetchDomain] = $stats['waves'] + min(pow(2, $fails), 32);
                    if ($result['http_code'] === 429) {
                        $domainFailCount[$fetchDomain] = $MAX_DOMAIN_FAILS;
                        $domainBackoffUntil[$fetchDomain] = $stats['waves'] + 64; // longer backoff for rate limit
                    }
                    // Domain stats: track error
                    if (!isset($domainStats[$fetchDomain])) $domainStats[$fetchDomain] = ['crawled'=>0,'imported'=>0,'errors'=>0,'status'=>0,'links'=>0];
                    $domainStats[$fetchDomain]['errors']++;
                    $domainStats[$fetchDomain]['status'] = $result['http_code'];

                    if ($result['http_code'] >= 400 && $result['http_code'] < 500) {
                        $stats['skipped']++;
                        $stats['log'][] = ['url' => $url, 'status' => 'skipped', 'title' => 'HTTP ' . $result['http_code'], 'domain' => $fetchDomain];
                    } else {
                        $stats['errors']++;
                        $stats['log'][] = ['url' => $url, 'status' => 'error', 'title' => 'HTTP ' . $result['http_code'], 'domain' => $fetchDomain];
                        // R6: Log fetch errors (5xx and connection errors)
                        logCrawlerError('fetch_error', ['url' => $url, 'http_code' => $result['http_code'], 'error' => $result['error'] ?? '', 'domain' => $fetchDomain, 'wave' => $stats['waves']]);
                        $consecutiveErrors++; // Track for error recovery
                    }
                    continue;
                }

                // Reset consecutive errors on success
                $consecutiveErrors = 0;

                // Reset fail count on success + clear backoff
                $domainFailCount[$fetchDomain] = max(0, ($domainFailCount[$fetchDomain] ?? 0) - 1);
                unset($domainBackoffUntil[$fetchDomain]);
                // Domain stats: track crawl
                if (!isset($domainStats[$fetchDomain])) $domainStats[$fetchDomain] = ['crawled'=>0,'imported'=>0,'errors'=>0,'status'=>0,'links'=>0];
                $domainStats[$fetchDomain]['crawled']++;
                $domainStats[$fetchDomain]['status'] = $result['http_code'];

                $html = $result['html'];
                $sourceDomain = parse_url($url, PHP_URL_HOST) ?? '';
                $isSerpPage = preg_match('/google\.|bing\.|yahoo\.|yandex\.|duckduckgo\.|baidu\.|ecosia\.|startpage\.|brave\.|searx|ask\.com|aol\.|dogpile\.|metacrawler\./i', $sourceDomain);

                // CHECK RELEVANCE FIRST (body text + metadata)
                $meta = extractMetadata($url, $html);
                $passesInclude = true;
                $hasIncludeFilter = !empty($includeTerms) && !(count($includeTerms) === 1 && empty($includeTerms[0]));

                if ($hasIncludeFilter && !$isSerpPage) {
                    $passesInclude = false;
                    // Search in URL + title + description + body text (first 3000 chars)
                    $cleanBody = preg_replace('/<(script|style|noscript|nav|footer|header)[^>]*>.*?<\/\1>/si', '', $html);
                    $bodyText = mb_substr(trim(preg_replace('/\s+/', ' ', strip_tags($cleanBody))), 0, 3000);
                    $searchableContent = strtolower($url . ' ' . ($meta['title'] ?? '') . ' ' . ($meta['description'] ?? '') . ' ' . $bodyText);
                    foreach ($includeTerms as $term) {
                        if (!empty($term) && stripos($searchableContent, $term) !== false) {
                            $passesInclude = true;
                            break;
                        }
                    }
                }

                // META NOFOLLOW check: don't extract links from nofollow pages (unless SERP)
                $robotsMetaSpread = hasNoindexNofollow($html);
                $allowSpread = !$robotsMetaSpread['nofollow'] || $isSerpPage || isForcedDomain($fetchDomain, $forcedDomains);

                // ONLY SPREAD LINKS from relevant pages (or SERP pages) that allow following
                if (($passesInclude || $isSerpPage) && $allowSpread) {
                    if ($isSerpPage) {
                        $pageLinks = extractSearchResultsFromUrl($url, $html);
                        if (empty($pageLinks)) {
                            $pageLinks = extractAllLinks($html, $url, $linksPerPage, $includeTerms, $excludeTerms, $config);
                        }
                    } else {
                        $pageLinks = extractAllLinks($html, $url, $linksPerPage, $includeTerms, $excludeTerms, $config);
                    }

                    $newLinks = [];
                    foreach ($pageLinks as $pl) {
                        if (!isset($visited[$pl]) && !isset($poolSet[$pl])) {
                            $newLinks[] = $pl;
                            $poolSet[$pl] = true;
                        }
                        // Track network edges (domain → domain)
                        $targetDomain = parse_url($pl, PHP_URL_HOST) ?? '';
                        if ($sourceDomain && $targetDomain && $sourceDomain !== $targetDomain) {
                            $edgeKey = $sourceDomain . '>' . $targetDomain;
                            if (!isset($networkEdgesSeen[$edgeKey])) {
                                $networkEdgesSeen[$edgeKey] = true;
                                $stats['network'][] = [$sourceDomain, $targetDomain];
                            }
                        }
                    }

                    // Add to pool with domain diversity limit + post priority
                    if (count($pool) < $maxPool && !empty($newLinks)) {
                        $skipUrlPatterns = '/\/wp-content\/|\/wp-includes\/|\/wp-admin\/|\/cdn-cgi\/|\/static\/css\/|\/static\/js\/|\/#[^\/]*$|\/feed\/?$|\/xmlrpc\.php|\/wp-json\/|\/embed\/|^javascript:|^mailto:|^tel:/i';
                        $diverseLinks = [];
                        foreach ($newLinks as $nl) {
                            if (preg_match($skipUrlPatterns, $nl)) continue;
                            $nlHost = parse_url($nl, PHP_URL_HOST) ?? '';
                            if (isDomainSkipped($nlHost)) continue;
                            $nlBaseDomain = getBaseDomain($nlHost);
                            if (($domainFailCount[$nlHost] ?? 0) >= $MAX_DOMAIN_FAILS) continue;
                            if (($domainUrlCount[$nlBaseDomain] ?? 0) >= $MAX_URLS_PER_DOMAIN) continue;
                            // Auto-blacklist: skip domains that never produce imports
                            if (isset($domainStats[$nlHost]) &&
                                ($domainStats[$nlHost]['crawled'] ?? 0) >= $JUNK_THRESHOLD &&
                                ($domainStats[$nlHost]['imported'] ?? 0) === 0) {
                                continue;
                            }
                            // Skip domains with very low quality scores
                            if (($domainQuality[$nlHost] ?? 50) < 10) continue;
                            $domainUrlCount[$nlBaseDomain] = ($domainUrlCount[$nlBaseDomain] ?? 0) + 1;
                            $diverseLinks[] = $nl;
                        }

                        if (!empty($diverseLinks)) {
                            $postPattern = '/\/post[s]?\/|\/article[s]?\/|\/blog\/|\/news\/|\/story\/|\/stories\/|\/p\/|\/entry\/|\d{4}\/\d{2}\/|\d{4}-\d{2}-\d{2}|[a-z]+-[a-z]+-[a-z]+|\.html$|\.htm$/i';
                            if (!empty($config['posts_first'])) {
                                $postNew = array_filter($diverseLinks, fn($l) => preg_match($postPattern, $l));
                                $otherNew = array_filter($diverseLinks, fn($l) => !preg_match($postPattern, $l));
                                $space = $maxPool - count($pool);
                                $toAdd = array_slice(array_merge(array_values($postNew), array_values($otherNew)), 0, $space);
                                $postToAdd = array_filter($toAdd, fn($l) => preg_match($postPattern, $l));
                                $otherToAdd = array_filter($toAdd, fn($l) => !preg_match($postPattern, $l));
                                $pool = array_merge(array_values($postToAdd), $pool, array_values($otherToAdd));
                            } else {
                                $pool = array_merge($pool, array_slice($diverseLinks, 0, $maxPool - count($pool)));
                            }
                        }
                    }
                    $stats['discovered'] += count($newLinks);
                    // Domain stats: track links found
                    if (isset($domainStats[$fetchDomain])) {
                        $domainStats[$fetchDomain]['links'] += count($pageLinks);
                    }
                }

                // Import or skip based on relevance + quality + canonical + noindex + forced
                if ($passesInclude && !$isSerpPage) {
                    // META NOINDEX: Skip pages that request not to be indexed
                    $robotsMeta = hasNoindexNofollow($html);
                    if ($robotsMeta['noindex'] && !isForcedDomain($fetchDomain, $forcedDomains)) {
                        $stats['skipped']++;
                        $stats['log'][] = ['url' => $url, 'status' => 'skipped', 'title' => 'noindex: ' . ($meta['title'] ?? ''), 'domain' => $sourceDomain];
                        $domainQuality[$fetchDomain] = max(0, ($domainQuality[$fetchDomain] ?? 50) - 2);
                        continue;
                    }

                    // CANONICAL URL: Dedup by canonical (avoid importing same page with different URLs)
                    $canonicalUrl = getCanonicalUrl($html, $url);
                    if (isset($canonicalSeen[$canonicalUrl]) && $canonicalUrl !== $url) {
                        $stats['duplicates']++;
                        $stats['log'][] = ['url' => $url, 'status' => 'skipped', 'title' => 'Canonical dup: ' . basename($canonicalUrl), 'domain' => $sourceDomain];
                        continue;
                    }
                    $canonicalSeen[$canonicalUrl] = true;
                    // Use canonical URL for import
                    $importUrl = $canonicalUrl;

                    // FORCED DOMAINS: Always import, bypass quality check
                    $isForced = isForcedDomain($fetchDomain, $forcedDomains);

                    $quality = scoreContentQuality($url, $html, $meta);

                    // WEIGHTED RELEVANCE SCORING (term-based)
                    $relevance = calculateRelevanceScore($html, $keywords, $url);

                    if ($isForced || ($quality >= 20 && $relevance >= $MIN_RELEVANCE)) {
                        $importQueue[] = array_merge(['link' => $importUrl, '_quality' => $quality, '_relevance' => $relevance], $meta);
                        // Boost domain quality score
                        $domainQuality[$fetchDomain] = min(100, ($domainQuality[$fetchDomain] ?? 50) + 2);
                    } elseif ($quality >= 20 && $relevance < $MIN_RELEVANCE) {
                        // Good quality but low relevance - still import if quality is very high
                        if ($quality >= 60) {
                            $importQueue[] = array_merge(['link' => $importUrl, '_quality' => $quality, '_relevance' => $relevance], $meta);
                            $domainQuality[$fetchDomain] = min(100, ($domainQuality[$fetchDomain] ?? 50) + 1);
                        } else {
                            $stats['skipped']++;
                            $stats['log'][] = ['url' => $url, 'status' => 'low-relevance', 'title' => "R=$relevance Q=$quality: " . ($meta['title'] ?? ''), 'domain' => $sourceDomain];
                            $domainQuality[$fetchDomain] = max(0, ($domainQuality[$fetchDomain] ?? 50) - 1);
                        }
                    } else {
                        $stats['skipped']++;
                        $stats['log'][] = ['url' => $url, 'status' => 'low-quality', 'title' => "Q=$quality R=$relevance: " . ($meta['title'] ?? ''), 'domain' => $sourceDomain];
                        $domainQuality[$fetchDomain] = max(0, ($domainQuality[$fetchDomain] ?? 50) - 3);
                    }
                } elseif (!$passesInclude) {
                    $stats['skipped']++;
                    $stats['log'][] = ['url' => $url, 'status' => 'skipped', 'title' => 'No match in content', 'domain' => $sourceDomain];
                    // Track unproductive crawl for auto-blacklist
                    $domainQuality[$fetchDomain] = max(0, ($domainQuality[$fetchDomain] ?? 50) - 1);
                }
            }

            $stats['processed'] += count($wave);
            $stats['pool_size'] = count($pool);
            $stats['wave_size'] = $waveSize;
            $stats['unique_domains'] = count($domainStats);

            // I3: Track bandwidth
            foreach ($results as $r) {
                $bandwidthBytes += ($r['size_download'] ?? 0);
            }
            $stats['bandwidth_mb'] = round($bandwidthBytes / 1048576, 1);

            // P5: Adaptive wave size (grow on success, shrink on failure)
            $waveSuccessCount = 0;
            foreach ($results as $r) {
                if (($r['http_code'] ?? 0) >= 200 && ($r['http_code'] ?? 0) < 400) $waveSuccessCount++;
            }
            $successRate = count($results) > 0 ? $waveSuccessCount / count($results) : 0;
            if ($successRate > 0.8 && $waveSize < 100) $waveSize = min($waveSize + 3, 100);
            elseif ($successRate < 0.4 && $waveSize > 5) $waveSize = max($waveSize - 3, 5);

            // P9: Adaptive micro-batch size
            if ($stats['imported'] > 200 && $microBatchSize < 100) $microBatchSize = min($microBatchSize + 10, 100);

            // INFINITE MODE: Query rotation a cada 500 URLs processados
            if ($stats['processed'] > 0 && $stats['processed'] % $QUERY_ROTATION_INTERVAL === 0) {
                $currentQueryVariation = ($currentQueryVariation + 1) % count($queryVariations);
                $stats['query_variation'] = $queryVariations[$currentQueryVariation];
                $stats['log'][] = ['url' => '', 'status' => 'imported',
                    'title' => "Query rotated to: " . $queryVariations[$currentQueryVariation],
                    'domain' => 'ROTATION'];
            }

            // MEMORY MANAGEMENT: A cada 50 URLs, verificar uso de memoria
            if ($stats['processed'] > 0 && $stats['processed'] % $GC_INTERVAL === 0) {
                $currentMb = memory_get_usage(true) / 1024 / 1024;

                if ($currentMb > $MEMORY_LIMIT_MB * 0.8) {
                    // Trim visited cache (manter 50% mais recentes)
                    if (count($visited) > 100000) {
                        $visited = array_slice($visited, -50000, null, true);
                    }
                    // Trim poolSet
                    if (count($poolSet) > 100000) {
                        $poolSet = array_slice($poolSet, -50000, null, true);
                    }
                    gc_collect_cycles();

                    $afterMb = memory_get_usage(true) / 1024 / 1024;
                    $stats['log'][] = ['url' => '', 'status' => 'imported',
                        'title' => "Memory cleanup: " . round($currentMb, 1) . "MB -> " . round($afterMb, 1) . "MB",
                        'domain' => 'MEMORY'];
                }
                $stats['memory_mb'] = round(memory_get_usage(true) / 1024 / 1024, 1);
            }

            // ERROR RECOVERY: Pausar se muitos erros consecutivos
            if ($consecutiveErrors >= $ERROR_THRESHOLD) {
                $stats['status'] = 'error_pause';
                $stats['log'][] = ['url' => '', 'status' => 'error',
                    'title' => "Too many errors ($consecutiveErrors). Pausing {$PAUSE_ON_ERRORS}s...",
                    'domain' => 'RECOVERY'];
                outputViralStats($stats);
                sleep($PAUSE_ON_ERRORS);
                $consecutiveErrors = 0;
                $stats['status'] = 'running';
            }
        }

        // MICRO-IMPORT (adaptive batch size + P10 in-memory dedup + R3 retry)
        while (count($importQueue) >= $microBatchSize) {
            $microBatch = array_splice($importQueue, 0, $microBatchSize);

            // P10: In-memory dedup - filter out recently imported URLs before DB check
            $microBatch = array_filter($microBatch, function($r) use (&$recentImports, &$stats) {
                $urlHash = md5($r['link']);
                if (isset($recentImports[$urlHash])) {
                    $stats['duplicates']++;
                    $stats['log'][] = ['url' => $r['link'], 'status' => 'duplicate', 'title' => '(mem-cache)', 'domain' => parse_url($r['link'], PHP_URL_HOST) ?? ''];
                    return false;
                }
                return true;
            });
            if (empty($microBatch)) continue;
            $microBatch = array_values($microBatch);

            $links = array_column($microBatch, 'link');
            $existing = checkDuplicatesArray($pdo, $links);

            $toImport = array_filter($microBatch, function($r) use ($existing) {
                return !isset($existing[$r['link']]);
            });

            $dupCount = count($microBatch) - count($toImport);
            $stats['duplicates'] += $dupCount;

            // Log duplicates + add to memory cache
            foreach ($microBatch as $rec) {
                if (isset($existing[$rec['link']])) {
                    $recentImports[md5($rec['link'])] = true;
                    $stats['log'][] = ['url' => $rec['link'], 'status' => 'duplicate', 'title' => '', 'domain' => parse_url($rec['link'], PHP_URL_HOST) ?? ''];
                }
            }

            if (!empty($toImport)) {
                // R3: Retry logic with exponential backoff (3 attempts)
                $retryMax = 3;
                $importSuccess = false;
                for ($attempt = 0; $attempt < $retryMax; $attempt++) {
                    try {
                        $imported = importMicroBatch($pdo, array_values($toImport));
                        $stats['imported'] += $imported;
                        $stats['last_url'] = end($toImport)['link'] ?? '';

                        // Add to memory dedup cache
                        foreach ($toImport as $rec) {
                            $recentImports[md5($rec['link'])] = true;
                            $impDomain = parse_url($rec['link'], PHP_URL_HOST) ?? '';
                            $stats['log'][] = [
                                'url' => $rec['link'],
                                'status' => 'imported',
                                'title' => mb_substr($rec['title'] ?? '', 0, 60),
                                'domain' => $impDomain
                            ];
                            if (!isset($domainStats[$impDomain])) $domainStats[$impDomain] = ['crawled'=>0,'imported'=>0,'errors'=>0,'status'=>200,'links'=>0];
                            $domainStats[$impDomain]['imported']++;
                        }
                        $importSuccess = true;
                        break;
                    } catch (Exception $e) {
                        if ($attempt < $retryMax - 1) {
                            usleep(pow(2, $attempt) * 100000); // 100ms, 200ms, 400ms
                            // Try to reconnect DB on retry
                            try { $pdo->query('SELECT 1'); } catch (Exception $dbErr) {
                                try { $pdo = new PDO("mysql:host=$DB_HOST;dbname=$DB_NAME;charset=utf8mb4", $DB_USER, $DB_PASS, [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); } catch (PDOException $pe) {}
                            }
                        }
                    }
                }
                if (!$importSuccess) {
                    $stats['errors'] += count($toImport);
                    foreach ($toImport as $rec) {
                        $stats['log'][] = ['url' => $rec['link'], 'status' => 'error', 'title' => 'DB Error (3 retries failed)', 'domain' => ''];
                    }
                    // R6: Log structured error
                    logCrawlerError('import_failed', ['urls' => array_column($toImport, 'link'), 'wave' => $stats['waves'], 'attempts' => 3]);
                }
            }

            // P10: Keep memory cache at max 100K entries (massive dedup = less DB hits)
            if (count($recentImports) > 100000) {
                $recentImports = array_slice($recentImports, -100000, null, true);
            }
        }

        // Network stats
        $stats['total_edges'] = count($networkEdgesSeen);
        // Count unique domains from edges
        $uniqueDomains = [];
        foreach ($networkEdgesSeen as $key => $v) {
            $parts = explode('>', $key);
            $uniqueDomains[$parts[0]] = true;
            $uniqueDomains[$parts[1]] = true;
        }
        $stats['total_nodes'] = count($uniqueDomains);

        // Limit network edges per message (max 20 per wave to keep payload small)
        if (count($stats['network']) > 20) {
            $stats['network'] = array_slice($stats['network'], 0, 20);
        }

        // Send domain stats (top 30 by crawled count)
        if (!empty($domainStats)) {
            uasort($domainStats, function($a, $b) {
                return (($b['crawled'] ?? 0) + ($b['imported'] ?? 0) * 3) - (($a['crawled'] ?? 0) + ($a['imported'] ?? 0) * 3);
            });
            $stats['domain_stats'] = array_slice($domainStats, 0, 30, true);
        }

        outputViralStats($stats);

        // Reset per-wave data (only send deltas)
        $stats['network'] = [];
        $stats['log'] = [];
        unset($stats['domain_stats']);

        // Memory management: limit $visited to last 50K entries
        if (count($visited) > 50000) {
            $visited = array_slice($visited, -50000, null, true);
        }

        // Pool pruning: every 10 waves OR when pool > 5000, remove URLs from blacklisted domains
        if (($stats['waves'] % 10 === 0 || count($pool) > 5000) && count($pool) > 200) {
            $pool = array_values(array_filter($pool, function($url) use ($domainFailCount, $MAX_DOMAIN_FAILS) {
                $d = parse_url($url, PHP_URL_HOST) ?? '';
                return ($domainFailCount[$d] ?? 0) < $MAX_DOMAIN_FAILS;
            }));
            $poolSet = array_fill_keys($pool, true);
            foreach ($visited as $v => $_) $poolSet[$v] = true;
            $stats['pool_size'] = count($pool);
        }

        // Minimal delay between waves (smooth)
        usleep($waveDelayMs * 1000);
    }

    // Import remaining items in queue
    if (!empty($importQueue)) {
        $links = array_column($importQueue, 'link');
        $existing = checkDuplicatesArray($pdo, $links);
        $toImport = array_filter($importQueue, function($r) use ($existing) {
            return !isset($existing[$r['link']]);
        });
        if (!empty($toImport)) {
            try {
                $imported = importMicroBatch($pdo, array_values($toImport));
                $stats['imported'] += $imported;
            } catch (Exception $e) {}
        }
        $stats['duplicates'] += count($importQueue) - count($toImport);
    }

    $stats['status'] = ($stats['status'] === 'stopped') ? 'stopped' : 'completed';
    $stats['pool_size'] = count($pool);
    outputViralStats($stats);
    exit;
}

function extractAllLinks($html, $baseUrl, $maxLinks = 50, $includeTerms = [], $excludeTerms = [], $config = []) {
    $links = [];
    preg_match_all('/<a[^>]+href="([^"]+)"/i', $html, $matches);

    // Navigation/auth patterns to always skip
    $skipPatterns = '/login|signin|signup|register|admin|wp-admin|dashboard|cart|checkout|account|profile|settings|password|reset|logout|api\/|graphql|xmlrpc|replytocom|mailto:|javascript:|tel:|#respond|\/feed\/?$|\/rss\/?$|\/wp-json\//i';

    // Media type extensions
    $imageExts = ['jpg','jpeg','png','gif','webp','svg','bmp','ico','tiff'];
    $videoExts = ['mp4','avi','webm','mov','mkv','flv','wmv','m4v'];
    $audioExts = ['mp3','wav','ogg','flac','aac','wma','m4a'];
    $docExts = ['pdf','doc','docx','xls','xlsx','ppt','pptx','txt','csv','rtf'];
    $archiveExts = ['zip','rar','7z','tar','gz','bz2','xz'];

    // Media filters from config (default: pages only, skip media files)
    $mediaFilters = $config['media_filters'] ?? [];
    $hasMediaConfig = !empty($mediaFilters);

    // Same domain / external filter
    $baseDomain = parse_url($baseUrl, PHP_URL_HOST) ?? '';
    $sameDomainOnly = !empty($config['same_domain']);
    $followExternal = $config['follow_external'] ?? true;

    // Post priority pattern
    $postPatterns = '/\/post[s]?\/|\/article[s]?\/|\/blog\/|\/news\/|\/story\/|\/stories\/|\/p\/|\/entry\/|\d{4}\/\d{2}\/|\d{4}-\d{2}-\d{2}|\/\d+\/[a-z]|[a-z]+-[a-z]+-[a-z]+|\.html$|\.htm$/i';
    $postsFirst = !empty($config['posts_first']);

    $postLinks = [];
    $otherLinks = [];

    foreach ($matches[1] as $link) {
        $normalized = normalizeUrl($link, $baseUrl);
        if (!$normalized) continue;

        // Skip very short URLs
        if (strlen($normalized) < 15) continue;

        // Skip known bad patterns
        if (preg_match($skipPatterns, $normalized)) continue;

        // Skip major social/search/institutional domains
        $linkHost = strtolower(parse_url($normalized, PHP_URL_HOST) ?? '');
        if (isDomainSkipped($linkHost)) continue;

        // Same domain filter
        if ($sameDomainOnly) {
            $linkDomain = parse_url($normalized, PHP_URL_HOST) ?? '';
            if ($linkDomain !== $baseDomain) continue;
        }

        // External links filter
        if (!$followExternal) {
            $linkDomain = parse_url($normalized, PHP_URL_HOST) ?? '';
            if ($linkDomain !== $baseDomain) continue;
        }

        // Media type filtering
        $ext = strtolower(pathinfo(parse_url($normalized, PHP_URL_PATH) ?? '', PATHINFO_EXTENSION));
        $isImage = in_array($ext, $imageExts);
        $isVideo = in_array($ext, $videoExts);
        $isAudio = in_array($ext, $audioExts);
        $isDoc = in_array($ext, $docExts);
        $isArchive = in_array($ext, $archiveExts);

        if ($hasMediaConfig) {
            // User-configured media filters
            if ($isImage && empty($mediaFilters['images'])) continue;
            if ($isVideo && empty($mediaFilters['videos'])) continue;
            if ($isAudio && empty($mediaFilters['audio'])) continue;
            if ($isDoc && empty($mediaFilters['docs'])) continue;
            if ($isArchive && empty($mediaFilters['archives'])) continue;
        } else {
            // Default behavior: skip all media files (backward compatible)
            if ($isImage || $isVideo || $isAudio || $isDoc || $isArchive) continue;
        }

        // Skip URLs with too many query params (tracking/spam)
        $queryString = parse_url($normalized, PHP_URL_QUERY) ?? '';
        if (substr_count($queryString, '&') > 3) continue;

        // Apply exclude filters
        if (!empty($excludeTerms)) {
            $skip = false;
            foreach ($excludeTerms as $term) {
                if (!empty($term) && stripos($normalized, $term) !== false) {
                    $skip = true;
                    break;
                }
            }
            if ($skip) continue;
        }

        // Separate posts from other links for priority
        if ($postsFirst && preg_match($postPatterns, $normalized)) {
            $postLinks[] = $normalized;
        } else {
            $otherLinks[] = $normalized;
        }
    }

    // Posts first, then others
    if ($postsFirst) {
        $allLinks = array_merge($postLinks, $otherLinks);
    } else {
        $allLinks = array_merge($postLinks, $otherLinks);
    }

    return array_unique(array_slice($allLinks, 0, $maxLinks));
}

function importMicroBatch($pdo, $records) {
    if (empty($records)) return 0;

    // Static caches for performance (persist across calls within same request)
    static $domainCache = [];
    static $userCache = [];

    $columns = 'title, description, thumbnail, pubDate, link, updated, source_website, author, favicon, tags, embed_code, source_domain, user_id, source_domain_id, main_category_id, title_cat_id, description_cat_id, tag_cat_id';

    $placeholders = [];
    $values = [];

    foreach ($records as $data) {
        $link = $data['link'] ?? '';

        // URL validation - skip invalid URLs
        if (!filter_var($link, FILTER_VALIDATE_URL)) continue;

        $host = parse_url($link, PHP_URL_HOST) ?? '';
        $author = $data['author'] ?? '';

        // Author: use random if empty/generic
        if (empty($author) || $author === 'Anonymous' || $author === 'admin' || mb_strlen($author) < 3) {
            $author = generateRandomAuthor();
        }

        // User ID: cached lookup/create
        $userId = 0;
        if (isset($userCache[$author])) {
            $userId = $userCache[$author];
        } else {
            $userId = getOrCreateUserId($pdo, $author);
            $userCache[$author] = $userId;
            // Limit cache size
            if (count($userCache) > 500) {
                $userCache = array_slice($userCache, -200, null, true);
            }
        }

        // Domain ID + Category: cached lookup/create
        $domainId = 0;
        $categoryId = 0;
        if (isset($domainCache[$host])) {
            $domainId = $domainCache[$host]['domain_id'];
            $categoryId = $domainCache[$host]['category_id'];
        } else {
            $domainData = getOrCreateDomainId($pdo, $link);
            $domainId = $domainData['domain_id'];
            $categoryId = $domainData['category_id'];
            $domainCache[$host] = $domainData;
            if (count($domainCache) > 500) {
                $domainCache = array_slice($domainCache, -200, null, true);
            }
        }

        // Embed code extraction
        $embedCode = extractEmbedCode($link);

        // Tags fallback from title if empty
        $tags = $data['tags'] ?? '';
        if (empty($tags) && !empty($data['title'])) {
            $tags = extractTagsFromTitle($data['title']);
        }

        $placeholders[] = '(?, ?, ?, NOW(), ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0)';
        $values[] = mb_substr($data['title'] ?? 'No title', 0, 255);
        $values[] = mb_substr($data['description'] ?? '', 0, 1000);
        $values[] = mb_substr($data['thumbnail'] ?? '', 0, 500);
        $values[] = $link;
        $values[] = mb_substr($host, 0, 255);
        $values[] = mb_substr($author, 0, 255);
        $values[] = mb_substr($data['favicon'] ?? '', 0, 255);
        $values[] = mb_substr($tags, 0, 500);
        $values[] = mb_substr($embedCode, 0, 2000);
        $values[] = mb_substr($host, 0, 255);
        $values[] = $userId;
        $values[] = $domainId;
        $values[] = $categoryId;
    }

    if (empty($placeholders)) return 0;

    try {
        $pdo->beginTransaction();
        $sql = "INSERT IGNORE INTO pinfeeds ($columns) VALUES " . implode(', ', $placeholders);
        $stmt = $pdo->prepare($sql);
        $stmt->execute($values);
        $count = $stmt->rowCount();
        $pdo->commit();
        return $count;
    } catch (Exception $e) {
        $pdo->rollBack();
        return 0;
    }
}

function outputViralStats($stats) {
    $elapsed = max(1, time() - $stats['start_time']);
    $stats['rate'] = round($stats['imported'] / $elapsed, 1);
    $stats['elapsed'] = $elapsed;

    echo "data: " . json_encode($stats) . "\n\n";

    if (ob_get_level()) {
        ob_flush();
    }
    flush();
}

// ============================================
// PROCESS ACTIONS
// ============================================
$action = $_REQUEST['action'] ?? 'form';
// Parse JSON body for multi-process API endpoints
if (in_array($action, ['create_process','control_process','batch_control','cleanup_processes']) &&
    strpos($_SERVER['CONTENT_TYPE'] ?? '', 'application/json') !== false) {
    $jsonBody = json_decode(file_get_contents('php://input'), true);
    if (is_array($jsonBody)) {
        $_REQUEST = array_merge($_REQUEST, $jsonBody);
        // Map JS field names to PHP handler names
        if (isset($jsonBody['control'])) $_REQUEST['ctrl_action'] = $jsonBody['control'];
        if (isset($jsonBody['control'])) $_REQUEST['batch_action'] = $jsonBody['control'];
        if (isset($jsonBody['config']) && is_array($jsonBody['config'])) {
            foreach ($jsonBody['config'] as $k => $v) {
                if (!isset($_REQUEST[$k])) $_REQUEST[$k] = $v;
            }
        }
    }
}
$results = [];
$message = '';
$messageType = '';

// ============================================
// MULTI-PROCESS ACTION HANDLERS
// ============================================

// Worker mode: if called with process_id, run as background worker
if ($action === 'run_worker' && isset($_REQUEST['process_id'])) {
    $wpid = preg_replace('/[^a-f0-9]/', '', $_REQUEST['process_id']);
    if ($wpid) {
        $stmt = $pdo->prepare("SELECT * FROM crawler_processes WHERE id=? AND status='pending'");
        $stmt->execute([$wpid]);
        $proc = $stmt->fetch(PDO::FETCH_ASSOC);
        if ($proc) {
            $wKeywords = json_decode($proc['keywords'], true) ?: [];
            $wConfig = json_decode($proc['config'], true) ?: [];
            $wConfig['_start_time'] = time();
            if ($proc['mode'] === 'ultimate') {
                runUltimateWorker($wpid, $wKeywords, $wConfig, $pdo);
            } else {
                runViralWorker($wpid, $wKeywords, $wConfig, $pdo);
            }
        }
    }
    exit;
}

// Create new process
if ($action === 'create_process') {
    header('Content-Type: application/json');
    // Parse JSON body (JS sends Content-Type: application/json)
    $contentType = $_SERVER['CONTENT_TYPE'] ?? $_SERVER['HTTP_CONTENT_TYPE'] ?? '';
    if (stripos($contentType, 'application/json') !== false) {
        $jsonInput = json_decode(file_get_contents('php://input'), true);
        if (is_array($jsonInput)) {
            // Merge top-level fields (mode, keywords)
            if (isset($jsonInput['mode'])) $_REQUEST['mode'] = $jsonInput['mode'];
            if (isset($jsonInput['keywords'])) $_REQUEST['keywords'] = $jsonInput['keywords'];
            // Flatten nested config into $_REQUEST
            if (isset($jsonInput['config']) && is_array($jsonInput['config'])) {
                foreach ($jsonInput['config'] as $k => $v) {
                    $_REQUEST[$k] = is_array($v) ? implode("\n", $v) : $v;
                }
            }
        }
    }
    if (!mpCanStart($pdo)) {
        echo json_encode(['error' => 'Max processes reached (' . MP_MAX_PROCESSES . ')']);
        exit;
    }
    $mode = $_REQUEST['mode'] ?? 'viral';
    $keywords = array_filter(array_map('trim', explode("\n", $_REQUEST['keywords'] ?? '')));
    if (empty($keywords)) {
        echo json_encode(['error' => 'No keywords provided']);
        exit;
    }
    $config = [
        'wave_size' => (int)($_REQUEST['wave_size'] ?? 10),
        'max_imports' => (int)($_REQUEST['max_imports'] ?? 10000000),
        'links_per_page' => (int)($_REQUEST['links_per_page'] ?? 100),
        'include_terms' => array_filter(array_map('trim', explode("\n", $_REQUEST['include_terms'] ?? ''))),
        'exclude_terms' => array_filter(array_map('trim', explode("\n", $_REQUEST['exclude_terms'] ?? ''))),
        'forced_domains' => array_filter(array_map('trim', explode(',', $_REQUEST['forced_domains'] ?? ''))),
        'common_term' => trim($_REQUEST['common_term'] ?? ''),
        'posts_first' => ($_REQUEST['posts_first'] ?? '1') === '1',
        'follow_external' => ($_REQUEST['follow_external'] ?? '1') === '1',
        'max_depth' => (int)($_REQUEST['max_depth'] ?? 5),
        'quality_threshold' => (int)($_REQUEST['quality_threshold'] ?? 20),
        'relevance_threshold' => (int)($_REQUEST['relevance_threshold'] ?? 2),
        'enable_pagination' => ($_REQUEST['enable_pagination'] ?? '1') === '1',
        'pagination_pattern' => trim($_REQUEST['pagination_pattern'] ?? ''),
        'start_page' => (int)($_REQUEST['start_page'] ?? 1),
        'end_page' => (int)($_REQUEST['end_page'] ?? 50),
        'engines' => isset($_REQUEST['engines']) ? (is_array($_REQUEST['engines']) ? $_REQUEST['engines'] : explode(',', $_REQUEST['engines'])) : ['searxng','google','bing','bing_regional','duckduckgo','yahoo','yandex','brave','direct_sites','wikipedia','baidu'],
        'url_pattern_filter' => trim($_REQUEST['url_pattern_filter'] ?? ''),
        'http_auth_user' => trim($_REQUEST['http_auth_user'] ?? ''),
        'http_auth_pass' => trim($_REQUEST['http_auth_pass'] ?? ''),
        'fetch_delay_ms' => (int)($_REQUEST['fetch_delay_ms'] ?? 0),
    ];
    $processId = mpCreateProcess($pdo, $mode, $keywords, $config);
    // Launch background worker via HTTP (non-blocking)
    $workerUrl = 'http://localhost' . strtok($_SERVER['REQUEST_URI'], '?') . '?action=run_worker&process_id=' . $processId;
    // Use async HTTP call to launch worker
    $ctx = stream_context_create(['http' => ['method' => 'GET', 'timeout' => 1, 'ignore_errors' => true]]);
    @file_get_contents($workerUrl, false, $ctx);
    echo json_encode(['success' => true, 'process_id' => $processId]);
    exit;
}

// Control process (stop/pause/resume)
if ($action === 'control_process') {
    header('Content-Type: application/json');
    $processId = preg_replace('/[^a-f0-9]/', '', $_REQUEST['process_id'] ?? '');
    $ctrlAction = $_REQUEST['ctrl_action'] ?? '';
    if ($processId && in_array($ctrlAction, ['stop', 'pause', 'resume'])) {
        mpControlProcess($pdo, $processId, $ctrlAction);
        echo json_encode(['success' => true, 'action' => $ctrlAction]);
    } else {
        echo json_encode(['error' => 'Invalid process_id or action']);
    }
    exit;
}

// Batch control (stop_all, pause_all, resume_all)
if ($action === 'batch_control') {
    header('Content-Type: application/json');
    $batchAction = $_REQUEST['batch_action'] ?? '';
    $affected = 0;
    if ($batchAction === 'stop_all') {
        $st = $pdo->exec("UPDATE crawler_processes SET status='stopped', stopped_at=NOW() WHERE status IN ('running','paused')");
        $affected = $st ?: 0;
    } elseif ($batchAction === 'pause_all') {
        $st = $pdo->exec("UPDATE crawler_processes SET status='paused' WHERE status='running'");
        $affected = $st ?: 0;
    } elseif ($batchAction === 'resume_all') {
        $st = $pdo->exec("UPDATE crawler_processes SET status='running' WHERE status='paused'");
        $affected = $st ?: 0;
    } elseif ($batchAction === 'cleanup') {
        mpCleanupDead($pdo);
        mpCleanup($pdo);
    }
    echo json_encode(['success' => true, 'affected' => $affected]);
    exit;
}

// List processes
if ($action === 'list_processes') {
    header('Content-Type: application/json');
    $procs = $pdo->query("SELECT id, mode, status, keywords, discovered, processed, imported, duplicates, errors, skipped, pool_size, waves, rate, started_at, stopped_at, last_heartbeat, created_at FROM crawler_processes ORDER BY created_at DESC LIMIT 100")->fetchAll(PDO::FETCH_ASSOC);
    echo json_encode(['success' => true, 'processes' => $procs]);
    exit;
}

// Multi-process SSE stream
if ($action === 'multi_sse') {
    runMultiProcessSSE($pdo);
    exit;
}

// Ultimate SSE stream (enhanced with network + domain quality)
if ($action === 'ultimate_sse') {
    runUltimateSSE($pdo);
    exit;
}

// Cleanup endpoint
if ($action === 'cleanup_processes') {
    header('Content-Type: application/json');
    mpCleanupDead($pdo);
    mpCleanup($pdo);
    echo json_encode(['success' => true, 'message' => 'Cleanup complete']);
    exit;
}

// Handle infinite mode stop
if ($action === 'stop_infinite') {
    session_start();
    $_SESSION['infinite_running'] = false;
    session_write_close();
    header('Content-Type: application/json');
    echo json_encode(['success' => true, 'message' => 'Stop signal sent']);
    exit;
}

// Handle infinite mode start (streaming)
if ($action === 'infinite' && isset($_REQUEST['stream'])) {
    $keywords = array_filter(array_map('trim', explode("\n", $_REQUEST['keywords'] ?? '')));
    if (empty($keywords)) {
        header('Content-Type: application/json');
        echo json_encode(['error' => 'No keywords provided']);
        exit;
    }

    $config = [
        'max_imports' => (int)($_REQUEST['max_imports'] ?? 10000000),
        'batch_size' => (int)($_REQUEST['batch_size'] ?? 50),
        'deep_crawl' => isset($_REQUEST['deep_crawl']),
    ];

    runInfiniteMode($keywords, $config, $pdo);
    exit;
}

// Handle viral mode stop
if ($action === 'stop_viral') {
    session_start();
    $_SESSION['viral_running'] = false;
    $_SESSION['viral_paused'] = false;
    session_write_close();
    header('Content-Type: application/json');
    echo json_encode(['success' => true, 'message' => 'Viral stop signal sent']);
    exit;
}

// F2: Pause/Resume viral mode
if ($action === 'pause_viral') {
    session_start();
    $_SESSION['viral_paused'] = !($_SESSION['viral_paused'] ?? false);
    $isPaused = $_SESSION['viral_paused'];
    session_write_close();
    header('Content-Type: application/json');
    echo json_encode(['success' => true, 'paused' => $isPaused]);
    exit;
}

// Handle viral mode start (streaming)
if ($action === 'viral' && isset($_REQUEST['stream'])) {
    // S2: CSRF validation
    session_start();
    $validCsrf = isset($_REQUEST['csrf_token']) && hash_equals($_SESSION['csrf_token'] ?? '', $_REQUEST['csrf_token']);
    session_write_close();
    if (!$validCsrf) {
        header('Content-Type: application/json');
        echo json_encode(['error' => 'Invalid CSRF token. Please refresh the page.']);
        exit;
    }

    $keywords = array_filter(array_map('trim', explode("\n", $_REQUEST['keywords'] ?? '')));
    if (empty($keywords)) {
        header('Content-Type: application/json');
        echo json_encode(['error' => 'No keywords provided']);
        exit;
    }

    // COMMON TERM: append to each keyword for refined searches
    $commonTerm = trim($_REQUEST['common_term'] ?? '');
    if (!empty($commonTerm)) {
        $keywords = array_map(function($kw) use ($commonTerm) {
            // Don't append to URLs
            if (preg_match('/^https?:\/\//i', $kw)) return $kw;
            return $kw . ' ' . $commonTerm;
        }, $keywords);
    }

    // FORCED DOMAINS: always import, bypass quality/relevance checks
    $forcedDomains = array_filter(array_map('trim', explode(',', $_REQUEST['forced_domains'] ?? '')));

    $includeTerms = array_filter(array_map('trim', explode("\n", $_REQUEST['include_terms'] ?? '')));
    $excludeTerms = array_filter(array_map('trim', explode("\n", $_REQUEST['exclude_terms'] ?? '')));

    $config = [
        'wave_size' => (int)($_REQUEST['wave_size'] ?? 10),
        'micro_batch' => (int)($_REQUEST['micro_batch'] ?? 5),
        'max_pool' => (int)($_REQUEST['max_pool'] ?? 500000),
        'max_imports' => (int)($_REQUEST['max_imports'] ?? 10000000),
        'links_per_page' => (int)($_REQUEST['links_per_page'] ?? 100),
        'wave_delay_ms' => (int)($_REQUEST['wave_delay_ms'] ?? 10),
        'include_terms' => $includeTerms,
        'exclude_terms' => $excludeTerms,
        'forced_domains' => $forcedDomains,
        'media_filters' => json_decode($_REQUEST['media_filters'] ?? '{"pages":true,"articles":true}', true) ?: ['pages' => true, 'articles' => true],
        'posts_first' => ($_REQUEST['posts_first'] ?? '1') === '1',
        'follow_external' => ($_REQUEST['follow_external'] ?? '1') === '1',
        'same_domain' => ($_REQUEST['same_domain'] ?? '0') === '1',
    ];

    runViralMode($keywords, $config, $pdo);
    exit;
}

if ($_SERVER['REQUEST_METHOD'] === 'POST') {
    $mode = $_POST['mode'] ?? 'search';
    $searchTerms = array_filter(array_map('trim', explode("\n", $_POST['search_terms'] ?? '')));
    $seedUrls = array_filter(array_map('trim', explode("\n", $_POST['seed_urls'] ?? '')));
    $includeTerms = array_filter(array_map('trim', explode("\n", $_POST['include_terms'] ?? '')));
    $excludeTerms = array_filter(array_map('trim', explode("\n", $_POST['exclude_terms'] ?? '')));
    $maxPages = (int)($_POST['max_pages'] ?? 100);
    $maxDepth = (int)($_POST['max_depth'] ?? 3);
    $linkType = $_POST['link_type'] ?? 'both';
    $respectRobots = isset($_POST['respect_robots']);
    $engines = $_POST['engines'] ?? ['bing', 'duckduckgo'];

    if ($action === 'crawl') {
        $links = [];

        // Search mode
        if ($mode === 'search' || $mode === 'hybrid') {
            foreach ($searchTerms as $term) {
                if (in_array('bing', $engines)) {
                    $links = array_merge($links, getBingSearchLinks($term, 30));
                }
                if (in_array('duckduckgo', $engines)) {
                    $links = array_merge($links, getDuckDuckGoLinks($term, 30));
                }
                if (in_array('yandex', $engines)) {
                    $links = array_merge($links, getYandexSearchLinks($term, 20));
                }
                if (in_array('baidu', $engines)) {
                    $links = array_merge($links, getBaiduSearchLinks($term, 20));
                }
                if (in_array('yahoo', $engines)) {
                    $links = array_merge($links, getYahooSearchLinks($term, 20));
                }
            }
        }

        // Deep crawl mode
        if ($mode === 'deep' || $mode === 'hybrid') {
            foreach ($seedUrls as $seedUrl) {
                $deepLinks = deepCrawl($seedUrl, $maxDepth, $maxPages, $linkType, $includeTerms, $excludeTerms, $respectRobots);
                $links = array_merge($links, $deepLinks);
            }
        }

        // Pagination mode
        if ($mode === 'pagination' && !empty($_POST['pagination_pattern'])) {
            $pattern = $_POST['pagination_pattern'];
            $startPage = (int)($_POST['start_page'] ?? 1);
            $endPage = (int)($_POST['end_page'] ?? 10);
            $pageUrls = generatePaginationUrls($pattern, $startPage, $endPage);

            foreach ($pageUrls as $pageUrl) {
                $deepLinks = deepCrawl($pageUrl, 1, 50, $linkType, $includeTerms, $excludeTerms, $respectRobots);
                $links = array_merge($links, $deepLinks);
            }
        }

        // Remove duplicates and apply filters
        $links = array_unique($links);
        $links = array_filter($links, function($link) use ($includeTerms, $excludeTerms) {
            return passesFilters($link, $includeTerms, $excludeTerms);
        });
        $links = array_slice($links, 0, $maxPages);

        // Store in session for preview
        $_SESSION['crawl_results'] = $links;

        $message = "Found " . count($links) . " links. Click 'Preview & Import' to continue.";
        $messageType = 'success';
        $results = $links;
        $action = 'preview';
    }

    if ($action === 'import') {
        $selectedLinks = $_POST['selected_links'] ?? [];

        if (empty($selectedLinks)) {
            $message = "No links selected for import.";
            $messageType = 'error';
        } else {
            // Check duplicates
            $existing = checkDuplicates($pdo, $selectedLinks);
            $newLinks = array_filter($selectedLinks, function($link) use ($existing) {
                return !isset($existing[$link]);
            });

            if (empty($newLinks)) {
                $message = "All selected links already exist in the database.";
                $messageType = 'warning';
            } else {
                // Extract metadata
                $metadata = extractBatchMetadata($newLinks, 10);

                // Prepare records
                $records = [];
                foreach ($newLinks as $link) {
                    $meta = $metadata[$link] ?? [];
                    $records[] = array_merge(['link' => $link], $meta);
                }

                // Import
                try {
                    $imported = importToPinfeeds($pdo, $records);
                    $skipped = count($selectedLinks) - count($newLinks);
                    $message = "Successfully imported $imported links. Skipped $skipped duplicates.";
                    $messageType = 'success';
                } catch (Exception $e) {
                    $message = "Import failed: " . $e->getMessage();
                    $messageType = 'error';
                }
            }
        }

        $action = 'form';
    }
}

// Load preview data from session
if ($action === 'preview' && empty($results)) {
    session_start();
    $results = $_SESSION['crawl_results'] ?? [];
}
?>
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Crawler Ultimate v12.0 VIRAL+ - Flowb0t DCI</title>
    <style>
        :root {
            --bg-primary: #0a0a0f;
            --bg-secondary: #12121a;
            --bg-tertiary: #1a1a25;
            --text-primary: #ffffff;
            --text-secondary: #a0a0b0;
            --accent: #6366f1;
            --accent-hover: #818cf8;
            --success: #22c55e;
            --warning: #f59e0b;
            --error: #ef4444;
            --border: #2a2a35;
            /* A1: Expanded palette (20+ new colors) */
            --cyan: #06b6d4;
            --pink: #ec4899;
            --purple: #a855f7;
            --teal: #14b8a6;
            --lime: #84cc16;
            --rose: #f43f5e;
            --sky: #0ea5e9;
            --violet: #8b5cf6;
            --emerald: #10b981;
            --fuchsia: #d946ef;
            --orange: #f97316;
            --amber: #f59e0b;
            --indigo: #6366f1;
            --glass-bg: rgba(18,18,26,0.6);
            --glass-border: rgba(255,255,255,0.08);
            --glass-border-hover: rgba(255,255,255,0.15);
            --glow-accent: rgba(99,102,241,0.4);
            --glow-success: rgba(34,197,94,0.3);
            --glow-error: rgba(239,68,68,0.3);
            --transition-smooth: cubic-bezier(0.4,0,0.2,1);
            --transition-spring: cubic-bezier(0.175,0.885,0.32,1.275);
            --transition-bounce: cubic-bezier(0.68,-0.55,0.265,1.55);
        }

        * { box-sizing: border-box; margin: 0; padding: 0; }

        body {
            font-family: 'Inter', -apple-system, sans-serif;
            background: var(--bg-primary);
            color: var(--text-primary);
            line-height: 1.6;
            min-height: 100vh;
            overflow-x: hidden;
        }

        /* A6: Custom scrollbar */
        ::-webkit-scrollbar { width: 6px; height: 6px; }
        ::-webkit-scrollbar-track { background: var(--bg-primary); }
        ::-webkit-scrollbar-thumb { background: linear-gradient(180deg, var(--accent), var(--purple)); border-radius: 3px; }
        ::-webkit-scrollbar-thumb:hover { background: linear-gradient(180deg, var(--accent-hover), var(--fuchsia)); }

        /* A4: Pulse-glow animation */
        @keyframes pulseGlow { 0%,100% { box-shadow: 0 0 5px var(--glow-accent); } 50% { box-shadow: 0 0 20px var(--glow-accent), 0 0 40px rgba(99,102,241,0.15); } }

        /* A5: Flash/update animation */
        @keyframes statUpdate { 0% { transform: scale(1); } 30% { transform: scale(1.15); } 100% { transform: scale(1); } }
        .stat-flash { animation: statUpdate 0.4s var(--transition-spring); }

        /* A3: Gradient border rotation */
        @keyframes gradientRotate { 0% { background-position: 0% 50%; } 50% { background-position: 100% 50%; } 100% { background-position: 0% 50%; } }

        /* A11: Log entry slide-in */
        @keyframes slideInLeft { from { opacity: 0; transform: translateX(-10px); } to { opacity: 1; transform: translateX(0); } }
        .log-entry-animated { animation: slideInLeft 0.2s var(--transition-smooth); }

        /* A7: Wave processing animation */
        @keyframes waveProcess { 0% { background-position: -200% 0; } 100% { background-position: 200% 0; } }
        .wave-processing { background: linear-gradient(90deg, transparent 0%, var(--accent) 50%, transparent 100%); background-size: 200% 100%; animation: waveProcess 1.5s infinite linear; }

        /* Particle fade */
        @keyframes particleFade { from { opacity: 0; } to { opacity: 1; } }

        /* FAB animation */
        @keyframes fabPop { from { transform: scale(0) rotate(-180deg); opacity: 0; } to { transform: scale(1) rotate(0); opacity: 1; } }

        /* Theater mode */
        body.theater-mode .header { display: none; }
        body.theater-mode .container { max-width: 100vw; padding: 0.5rem; }
        body.theater-mode .grid { grid-template-columns: 1fr 280px; }
        body.theater-mode { overflow: hidden; }
        body.theater-mode .panel { border-radius: 0.5rem; }

        .header {
            background: var(--glass-bg);
            backdrop-filter: blur(12px);
            -webkit-backdrop-filter: blur(12px);
            border-bottom: 1px solid var(--glass-border);
            padding: 0.75rem 2rem;
            display: flex;
            justify-content: space-between;
            align-items: center;
            position: sticky;
            top: 0;
            z-index: 100;
            transition: all 0.4s var(--transition-smooth);
        }

        .logo {
            display: flex;
            align-items: center;
            gap: 0.75rem;
            font-size: 1.25rem;
            font-weight: 700;
        }

        .logo-icon { font-size: 1.5rem; }

        .version {
            background: var(--accent);
            color: white;
            padding: 0.125rem 0.5rem;
            border-radius: 0.25rem;
            font-size: 0.75rem;
        }

        .nav {
            display: flex;
            gap: 1rem;
        }

        .nav a {
            color: var(--text-secondary);
            text-decoration: none;
            padding: 0.5rem 1rem;
            border-radius: 0.5rem;
            transition: all 0.3s var(--transition-smooth);
            font-size: 0.85rem;
        }

        .nav a:hover {
            color: var(--text-primary);
            background: rgba(99,102,241,0.1);
            transform: translateY(-1px);
        }
        .nav a.active {
            color: var(--text-primary);
            background: rgba(99,102,241,0.15);
            box-shadow: 0 0 10px rgba(99,102,241,0.2);
        }

        .container {
            max-width: 1400px;
            margin: 0 auto;
            padding: 2rem;
        }

        .page-title {
            font-size: 1.75rem;
            margin-bottom: 0.5rem;
        }

        .page-subtitle {
            color: var(--text-secondary);
            margin-bottom: 2rem;
        }

        .grid {
            display: grid;
            grid-template-columns: 1fr 350px;
            gap: 2rem;
        }

        @media (max-width: 1024px) {
            .grid { grid-template-columns: 1fr; }
        }

        /* A2: Glassmorphism panels */
        .panel {
            background: var(--glass-bg);
            backdrop-filter: blur(12px);
            -webkit-backdrop-filter: blur(12px);
            border: 1px solid var(--glass-border);
            border-radius: 1rem;
            overflow: hidden;
            box-shadow: 0 8px 32px rgba(0,0,0,0.3);
            transition: all 0.3s var(--transition-smooth);
            position: relative;
        }
        .panel:hover {
            border-color: var(--glass-border-hover);
            box-shadow: 0 12px 40px rgba(0,0,0,0.4), 0 0 15px rgba(99,102,241,0.05);
            transform: translateY(-1px);
        }

        .panel-header {
            padding: 1rem 1.5rem;
            border-bottom: 1px solid var(--glass-border);
            display: flex;
            justify-content: space-between;
            align-items: center;
            background: rgba(255,255,255,0.02);
        }

        .panel-title {
            font-size: 1rem;
            font-weight: 600;
            display: flex;
            align-items: center;
            gap: 0.5rem;
        }

        .panel-body {
            padding: 1.5rem;
        }

        .form-group {
            margin-bottom: 1.25rem;
        }

        .form-label {
            display: block;
            font-size: 0.875rem;
            font-weight: 500;
            margin-bottom: 0.5rem;
            color: var(--text-secondary);
        }

        .form-input, .form-textarea, .form-select {
            width: 100%;
            padding: 0.75rem 1rem;
            background: rgba(26,26,37,0.8);
            border: 1px solid var(--glass-border);
            border-radius: 0.5rem;
            color: var(--text-primary);
            font-size: 0.9rem;
            transition: all 0.3s var(--transition-smooth);
        }
        .form-input:hover, .form-textarea:hover, .form-select:hover {
            border-color: rgba(255,255,255,0.12);
        }

        .form-input:focus, .form-textarea:focus, .form-select:focus {
            outline: none;
            border-color: var(--accent);
        }

        .form-textarea {
            min-height: 100px;
            resize: vertical;
            font-family: monospace;
        }

        .mode-selector {
            display: grid;
            grid-template-columns: repeat(4, 1fr);
            gap: 0.5rem;
            margin-bottom: 1.5rem;
        }

        .mode-btn {
            padding: 0.75rem;
            background: rgba(26,26,37,0.6);
            border: 2px solid transparent;
            border-radius: 0.75rem;
            color: var(--text-secondary);
            cursor: pointer;
            text-align: center;
            transition: all 0.3s var(--transition-smooth);
            backdrop-filter: blur(4px);
        }

        .mode-btn:hover {
            border-color: var(--glass-border-hover);
            transform: translateY(-2px);
            box-shadow: 0 4px 12px rgba(0,0,0,0.2);
        }

        .mode-btn.active {
            border-color: var(--accent);
            color: var(--text-primary);
            background: rgba(99, 102, 241, 0.15);
            box-shadow: 0 0 15px rgba(99,102,241,0.2);
        }

        .mode-icon { font-size: 1.25rem; display: block; margin-bottom: 0.25rem; }
        .mode-name { font-size: 0.75rem; font-weight: 500; }

        .chip-group {
            display: flex;
            flex-wrap: wrap;
            gap: 0.5rem;
        }

        .chip {
            display: flex;
            align-items: center;
            gap: 0.25rem;
            padding: 0.5rem 0.75rem;
            background: var(--bg-tertiary);
            border: 1px solid var(--border);
            border-radius: 2rem;
            cursor: pointer;
            font-size: 0.8rem;
            transition: all 0.2s;
        }

        .chip:hover {
            border-color: var(--accent);
        }

        .chip.active {
            background: rgba(99, 102, 241, 0.2);
            border-color: var(--accent);
        }

        .chip input { display: none; }

        .btn {
            display: inline-flex;
            align-items: center;
            justify-content: center;
            gap: 0.5rem;
            padding: 0.75rem 1.5rem;
            border: none;
            border-radius: 0.5rem;
            font-size: 0.9rem;
            font-weight: 600;
            cursor: pointer;
            transition: all 0.3s var(--transition-smooth);
        }

        .btn-primary {
            background: linear-gradient(135deg, var(--accent), var(--violet));
            color: white;
            box-shadow: 0 4px 15px rgba(99,102,241,0.3);
        }

        .btn-primary:hover {
            background: linear-gradient(135deg, var(--accent-hover), var(--fuchsia));
            transform: translateY(-2px);
            box-shadow: 0 6px 20px rgba(99,102,241,0.4);
        }

        .btn-success {
            background: var(--success);
            color: white;
        }

        .btn-lg {
            padding: 1rem 2rem;
            font-size: 1rem;
        }

        .btn-block {
            width: 100%;
        }

        .stats-grid {
            display: grid;
            grid-template-columns: repeat(2, 1fr);
            gap: 1rem;
        }

        /* A2+A4+A12: Glassmorphism stat cards with pulse and hover */
        .stat-card {
            background: rgba(26,26,37,0.7);
            backdrop-filter: blur(8px);
            padding: 0.85rem 0.75rem;
            border-radius: 0.75rem;
            text-align: center;
            border: 1px solid var(--glass-border);
            transition: all 0.3s var(--transition-smooth);
            position: relative;
            overflow: hidden;
        }
        .stat-card:hover {
            transform: scale(1.05) translateY(-2px);
            border-color: var(--glass-border-hover);
            box-shadow: 0 4px 20px rgba(0,0,0,0.3);
        }
        .stat-card.active-pulse { animation: pulseGlow 2s infinite; }
        .stat-card .sparkline-canvas { margin-top: 4px; display: block; margin-left: auto; margin-right: auto; opacity: 0.8; }

        .stat-value {
            font-size: 1.4rem;
            font-weight: 700;
            display: block;
            transition: all 0.3s var(--transition-smooth);
        }

        .stat-label {
            font-size: 0.7rem;
            color: var(--text-secondary);
            text-transform: uppercase;
            letter-spacing: 0.5px;
        }

        .stat-card.success .stat-value { color: var(--success); }
        .stat-card.warning .stat-value { color: var(--warning); }
        .stat-card.error .stat-value { color: var(--error); }

        .message {
            padding: 1rem;
            border-radius: 0.5rem;
            margin-bottom: 1.5rem;
        }

        .message.success {
            background: rgba(34, 197, 94, 0.1);
            border: 1px solid var(--success);
            color: var(--success);
        }

        .message.error {
            background: rgba(239, 68, 68, 0.1);
            border: 1px solid var(--error);
            color: var(--error);
        }

        .message.warning {
            background: rgba(245, 158, 11, 0.1);
            border: 1px solid var(--warning);
            color: var(--warning);
        }

        .results-list {
            max-height: 500px;
            overflow-y: auto;
        }

        .result-item {
            display: flex;
            align-items: center;
            gap: 0.75rem;
            padding: 0.75rem;
            border-bottom: 1px solid var(--border);
            transition: background 0.2s;
        }

        .result-item:hover {
            background: var(--bg-tertiary);
        }

        .result-item input[type="checkbox"] {
            width: 18px;
            height: 18px;
            accent-color: var(--accent);
        }

        .result-url {
            flex: 1;
            font-size: 0.85rem;
            color: var(--text-secondary);
            overflow: hidden;
            text-overflow: ellipsis;
            white-space: nowrap;
        }

        .result-domain {
            font-size: 0.75rem;
            padding: 0.25rem 0.5rem;
            background: var(--bg-primary);
            border-radius: 0.25rem;
        }

        .select-all {
            padding: 0.75rem 1rem;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border);
            display: flex;
            align-items: center;
            gap: 0.75rem;
        }

        .advanced-toggle {
            width: 100%;
            padding: 0.75rem;
            background: transparent;
            border: 1px dashed var(--border);
            border-radius: 0.5rem;
            color: var(--text-secondary);
            cursor: pointer;
            display: flex;
            align-items: center;
            justify-content: center;
            gap: 0.5rem;
            margin-bottom: 1rem;
        }

        .advanced-toggle:hover {
            border-color: var(--accent);
            color: var(--text-primary);
        }

        .advanced-options {
            display: none;
            padding-top: 1rem;
            border-top: 1px solid var(--border);
            margin-top: 1rem;
        }

        .advanced-options.show {
            display: block;
        }

        .checkbox-label {
            display: flex;
            align-items: center;
            gap: 0.5rem;
            cursor: pointer;
            font-size: 0.9rem;
        }

        .checkbox-label input {
            width: 16px;
            height: 16px;
            accent-color: var(--accent);
        }

        /* A10: Particle background */
        #particleCanvas { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 0; pointer-events: none; opacity: 0.4; animation: particleFade 2s ease-in; }
        .header, .container, #toastContainer, #fabContainer, #shortcutsOverlay { position: relative; z-index: 1; }

        /* B21: Floating Action Buttons */
        #fabContainer { position: fixed; bottom: 24px; right: 24px; z-index: 200; display: flex; flex-direction: column-reverse; gap: 10px; align-items: center; }
        .fab-main { width: 52px; height: 52px; border-radius: 50%; background: linear-gradient(135deg, var(--accent), var(--purple)); border: none; color: white; font-size: 1.4rem; cursor: pointer; box-shadow: 0 4px 20px rgba(99,102,241,0.4); transition: all 0.3s var(--transition-spring); display: flex; align-items: center; justify-content: center; }
        .fab-main:hover { transform: scale(1.1) rotate(45deg); box-shadow: 0 6px 25px rgba(99,102,241,0.5); }
        .fab-main.open { transform: rotate(45deg); }
        .fab-action { width: 40px; height: 40px; border-radius: 50%; border: none; color: white; font-size: 0.9rem; cursor: pointer; transition: all 0.3s var(--transition-spring); display: flex; align-items: center; justify-content: center; opacity: 0; transform: scale(0); pointer-events: none; }
        .fab-action.show { opacity: 1; transform: scale(1); pointer-events: auto; animation: fabPop 0.3s var(--transition-spring); }
        .fab-action:hover { transform: scale(1.15); }

        /* B20: Keyboard shortcuts overlay */
        #shortcutsOverlay { display: none; position: fixed; inset: 0; background: rgba(0,0,0,0.85); backdrop-filter: blur(8px); z-index: 500; align-items: center; justify-content: center; }
        #shortcutsOverlay.show { display: flex; }
        .shortcuts-content { background: var(--glass-bg); border: 1px solid var(--glass-border); border-radius: 1rem; padding: 2rem; max-width: 500px; width: 90%; backdrop-filter: blur(16px); box-shadow: 0 20px 60px rgba(0,0,0,0.5); }
        .shortcuts-content h2 { margin-bottom: 1rem; font-size: 1.2rem; color: var(--accent); }
        .shortcut-row { display: flex; justify-content: space-between; align-items: center; padding: 0.5rem 0; border-bottom: 1px solid rgba(255,255,255,0.05); }
        .shortcut-key { background: var(--bg-tertiary); padding: 0.2rem 0.6rem; border-radius: 4px; font-family: monospace; font-size: 0.8rem; border: 1px solid var(--border); }

        /* B22: Sidebar tabs */
        .sidebar-tabs { display: flex; gap: 0; border-bottom: 1px solid var(--glass-border); }
        .sidebar-tab { flex: 1; padding: 0.6rem; text-align: center; font-size: 0.75rem; font-weight: 600; cursor: pointer; color: var(--text-secondary); border-bottom: 2px solid transparent; transition: all 0.3s var(--transition-smooth); background: none; border-top: none; border-left: none; border-right: none; }
        .sidebar-tab:hover { color: var(--text-primary); background: rgba(255,255,255,0.02); }
        .sidebar-tab.active { color: var(--accent); border-bottom-color: var(--accent); }
        .sidebar-tab-content { display: none; padding: 1rem; }
        .sidebar-tab-content.active { display: block; }

        /* B23: Timeline */
        .timeline { padding: 0.5rem 0; }
        .timeline-item { display: flex; gap: 0.75rem; padding: 0.4rem 0; position: relative; }
        .timeline-dot { width: 8px; height: 8px; border-radius: 50%; margin-top: 5px; flex-shrink: 0; }
        .timeline-line { position: absolute; left: 3.5px; top: 18px; bottom: -4px; width: 1px; background: var(--border); }
        .timeline-text { font-size: 0.72rem; color: var(--text-secondary); }
        .timeline-time { font-size: 0.65rem; color: var(--text-secondary); opacity: 0.6; }

        /* B18: Domain leaderboard */
        .leaderboard { width: 100%; font-size: 0.72rem; border-collapse: collapse; }
        .leaderboard th { text-align: left; padding: 0.4rem 0.5rem; color: var(--text-secondary); font-weight: 600; border-bottom: 1px solid var(--border); font-size: 0.65rem; text-transform: uppercase; }
        .leaderboard td { padding: 0.35rem 0.5rem; border-bottom: 1px solid rgba(255,255,255,0.03); }
        .leaderboard tr:hover td { background: rgba(99,102,241,0.05); }
        .leaderboard .rank { color: var(--accent); font-weight: 700; width: 20px; }

        /* B17: Progress ring */
        .progress-ring { transform: rotate(-90deg); }
        .progress-ring-bg { fill: none; stroke: var(--bg-tertiary); stroke-width: 4; }
        .progress-ring-fill { fill: none; stroke: var(--accent); stroke-width: 4; stroke-linecap: round; transition: stroke-dashoffset 0.6s var(--transition-smooth); }

        /* B19: Rate chart container */
        .rate-chart-container { position: relative; background: var(--bg-primary); border: 1px solid var(--glass-border); border-radius: 0.5rem; overflow: hidden; margin-top: 0.75rem; }
        .rate-chart-label { position: absolute; top: 4px; right: 8px; font-size: 0.65rem; color: var(--accent); font-weight: 600; z-index: 2; }

        /* A9: Additional responsive breakpoints */
        @media (max-width: 1400px) {
            .container { padding: 1.5rem; }
            .stats-grid { gap: 0.75rem; }
        }
        @media (max-width: 640px) {
            .header { padding: 0.5rem 1rem; }
            .container { padding: 0.75rem; }
            .mode-selector { grid-template-columns: repeat(3, 1fr) !important; }
            .stats-grid { grid-template-columns: repeat(2, 1fr) !important; }
            .grid { gap: 1rem; }
            .nav { gap: 0.25rem; }
            .nav a { padding: 0.4rem 0.6rem; font-size: 0.75rem; }
            #fabContainer { bottom: 16px; right: 16px; }
        }

        /* Log filter buttons enhanced */
        .log-filter-btn { transition: all 0.2s var(--transition-smooth); }
        .log-filter-btn:hover { transform: translateY(-1px); }
        .log-filter-btn.active { box-shadow: 0 0 8px rgba(34,197,94,0.3); }

        /* Wave processing bar */
        .wave-bar { height: 3px; background: var(--bg-primary); border-radius: 2px; overflow: hidden; margin-top: 0.5rem; }
        .wave-bar-fill { height: 100%; border-radius: 2px; transition: width 0.5s var(--transition-smooth); }

        /* Pool health gauge */
        .health-gauge { display: inline-flex; align-items: center; gap: 4px; }

        /* Chart tabs */
        .chart-tab { background: var(--bg-tertiary); border: 1px solid var(--border); color: var(--text-secondary); padding: 0.15rem 0.4rem; border-radius: 0.2rem; cursor: pointer; font-size: 0.6rem; transition: all 0.2s; }
        .chart-tab:hover { border-color: var(--accent); color: var(--text-primary); }
        .chart-tab.active { background: rgba(99,102,241,0.3); color: white; border-color: var(--accent); }
        .domain-group > div:first-child:hover { background: rgba(99,102,241,0.2) !important; }
    </style>
    <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
</head>
<body>
    <!-- B14: Particle background canvas -->
    <canvas id="particleCanvas"></canvas>

    <header class="header">
        <div class="logo">
            <span class="logo-icon">🚀</span>
            Flowb0t <span class="version">v12.0</span>
            <!-- B15: Status dot -->
            <span id="headerStatusDot" style="width:8px;height:8px;border-radius:50%;background:var(--success);display:inline-block;margin-left:4px;animation:pulseGlow 2s infinite;"></span>
        </div>
        <div style="display:flex;align-items:center;gap:1.5rem;">
            <!-- B15: Real-time clock -->
            <span id="headerClock" style="font-size:0.8rem;color:var(--text-secondary);font-family:monospace;"></span>
            <nav class="nav">
                <a href="/Flowb0t_DCI/v2/public/">Dashboard</a>
                <a href="/Flowb0t_DCI/v2/public/new">Link Processor</a>
                <a href="/Flowb0t_DCI/v2/views/crawler-ultimate.php" class="active">Crawler</a>
                <a href="/Flowb0t_DCI/v2/public/history">History</a>
            </nav>
            <!-- B15: Theater mode button -->
            <button type="button" id="theaterBtn" onclick="toggleTheaterMode()" title="Theater Mode (F11)" style="background:none;border:1px solid var(--glass-border);color:var(--text-secondary);padding:4px 8px;border-radius:4px;cursor:pointer;font-size:0.8rem;transition:all 0.3s;">⛶</button>
        </div>
    </header>

    <div class="container">
        <h1 class="page-title">🔥 Crawler Ultimate v12.0</h1>
        <p class="page-subtitle">Professional crawler with 🦠 VIRAL MODE - Parallel spreading mesh + Micro-batch imports</p>

        <?php if ($message): ?>
        <div class="message <?= $messageType ?>"><?= htmlspecialchars($message) ?></div>
        <?php endif; ?>

        <?php if ($action === 'preview' && !empty($results)): ?>
        <!-- PREVIEW MODE -->
        <form method="POST">
            <input type="hidden" name="action" value="import">

            <div class="panel">
                <div class="panel-header">
                    <h2 class="panel-title">📋 Preview Results (<?= count($results) ?> links)</h2>
                    <button type="submit" class="btn btn-success">Import Selected to Pinfeeds</button>
                </div>
                <div class="select-all">
                    <input type="checkbox" id="selectAll" onchange="toggleAll(this)">
                    <label for="selectAll">Select All</label>
                    <span style="margin-left: auto; color: var(--text-secondary); font-size: 0.85rem;">
                        <span id="selectedCount">0</span> selected
                    </span>
                </div>
                <div class="results-list">
                    <?php foreach ($results as $i => $link): ?>
                    <div class="result-item">
                        <input type="checkbox" name="selected_links[]" value="<?= htmlspecialchars($link) ?>" onchange="updateCount()">
                        <span class="result-url"><?= htmlspecialchars($link) ?></span>
                        <span class="result-domain"><?= parse_url($link, PHP_URL_HOST) ?></span>
                    </div>
                    <?php endforeach; ?>
                </div>
            </div>

            <div style="margin-top: 1rem;">
                <a href="?action=form" class="btn btn-primary">← Back to Form</a>
            </div>
        </form>

        <script>
        function toggleAll(checkbox) {
            document.querySelectorAll('.result-item input[type="checkbox"]').forEach(cb => {
                cb.checked = checkbox.checked;
            });
            updateCount();
        }

        function updateCount() {
            const count = document.querySelectorAll('.result-item input[type="checkbox"]:checked').length;
            document.getElementById('selectedCount').textContent = count;
        }
        </script>

        <?php else: ?>
        <!-- FORM MODE -->
        <form method="POST">
            <input type="hidden" name="action" value="crawl">

            <div class="grid">
                <div class="panel">
                    <div class="panel-header">
                        <h2 class="panel-title">⚙️ Crawler Configuration</h2>
                    </div>
                    <div class="panel-body">
                        <!-- Mode Selection -->
                        <div class="form-group">
                            <label class="form-label">Crawl Mode</label>
                            <div class="mode-selector" style="grid-template-columns: repeat(4, 1fr);">
                                <button type="button" class="mode-btn active" data-mode="search" onclick="selectMode(this)">
                                    <span class="mode-icon">🔎</span>
                                    <span class="mode-name">Search</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="deep" onclick="selectMode(this)">
                                    <span class="mode-icon">🕸️</span>
                                    <span class="mode-name">Deep Crawl</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="pagination" onclick="selectMode(this)">
                                    <span class="mode-icon">📄</span>
                                    <span class="mode-name">Pagination</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="hybrid" onclick="selectMode(this)">
                                    <span class="mode-icon">⚡</span>
                                    <span class="mode-name">Hybrid</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="infinite" onclick="selectMode(this)" style="background: linear-gradient(135deg, #6366f1, #8b5cf6);">
                                    <span class="mode-icon">♾️</span>
                                    <span class="mode-name">INFINITE</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="viral" onclick="selectMode(this)" style="background: linear-gradient(135deg, #ef4444, #f97316);">
                                    <span class="mode-icon">🦠</span>
                                    <span class="mode-name">VIRAL</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="multi" onclick="selectMode(this)" style="background: linear-gradient(135deg, #06b6d4, #8b5cf6);">
                                    <span class="mode-icon">🚀</span>
                                    <span class="mode-name">MULTI</span>
                                </button>
                                <button type="button" class="mode-btn" data-mode="ultimate" onclick="selectMode(this)" style="background: linear-gradient(135deg, #f43f5e, #6366f1, #06b6d4); grid-column: span 4; font-size: 1.1em;">
                                    <span class="mode-icon">⚡</span>
                                    <span class="mode-name">ULTIMATE</span>
                                </button>
                            </div>
                            <input type="hidden" name="mode" id="modeInput" value="search">
                        </div>

                        <!-- Search Terms -->
                        <div class="form-group" id="searchTermsGroup">
                            <label class="form-label">Search Terms (one per line)</label>
                            <textarea name="search_terms" class="form-textarea" placeholder="web scraping tutorials&#10;python crawler guide&#10;data extraction tools"></textarea>
                        </div>

                        <!-- Seed URLs -->
                        <div class="form-group" id="seedUrlsGroup" style="display: none;">
                            <label class="form-label">Seed URLs (one per line)</label>
                            <textarea name="seed_urls" class="form-textarea" placeholder="https://example.com&#10;https://blog.example.org"></textarea>
                        </div>

                        <!-- Pagination Pattern -->
                        <div class="form-group" id="paginationGroup" style="display: none;">
                            <label class="form-label">Pagination URL Pattern</label>
                            <input type="text" name="pagination_pattern" class="form-input" placeholder="https://example.com/page/1">
                            <div style="display: flex; gap: 1rem; margin-top: 0.5rem;">
                                <div style="flex: 1;">
                                    <label class="form-label">Start Page</label>
                                    <input type="number" name="start_page" class="form-input" value="1" min="1">
                                </div>
                                <div style="flex: 1;">
                                    <label class="form-label">End Page</label>
                                    <input type="number" name="end_page" class="form-input" value="10" min="1">
                                </div>
                            </div>
                        </div>

                        <!-- Search Engines -->
                        <div class="form-group" id="enginesGroup">
                            <label class="form-label">Search Engines</label>
                            <div class="chip-group">
                                <label class="chip active">
                                    <input type="checkbox" name="engines[]" value="bing" checked> 🔵 Bing
                                </label>
                                <label class="chip active">
                                    <input type="checkbox" name="engines[]" value="duckduckgo" checked> 🦆 DuckDuckGo
                                </label>
                                <label class="chip">
                                    <input type="checkbox" name="engines[]" value="yandex"> 🔴 Yandex
                                </label>
                                <label class="chip">
                                    <input type="checkbox" name="engines[]" value="baidu"> 🟢 Baidu
                                </label>
                                <label class="chip">
                                    <input type="checkbox" name="engines[]" value="yahoo"> 🟣 Yahoo
                                </label>
                            </div>
                        </div>

                        <!-- Advanced Options Toggle -->
                        <button type="button" class="advanced-toggle" onclick="toggleAdvanced()">
                            ⚙️ Advanced Options <span id="advancedArrow">▼</span>
                        </button>

                        <!-- Advanced Options -->
                        <div class="advanced-options" id="advancedOptions">
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem;">
                                <div class="form-group">
                                    <label class="form-label">Max Pages</label>
                                    <input type="number" name="max_pages" class="form-input" value="100" min="1" max="1000">
                                </div>
                                <div class="form-group">
                                    <label class="form-label">Max Depth</label>
                                    <input type="number" name="max_depth" class="form-input" value="3" min="1" max="10">
                                </div>
                            </div>

                            <div class="form-group">
                                <label class="form-label">Link Type</label>
                                <select name="link_type" class="form-select">
                                    <option value="both">Both (Origin + External)</option>
                                    <option value="origin">Origin Site Only</option>
                                    <option value="external">External Sites Only</option>
                                </select>
                            </div>

                            <div class="form-group">
                                <label class="form-label">Include Terms (URL must contain, one per line)</label>
                                <textarea name="include_terms" class="form-textarea" rows="2" placeholder="article&#10;blog&#10;news"></textarea>
                            </div>

                            <div class="form-group">
                                <label class="form-label">Exclude Terms (URL must NOT contain, one per line)</label>
                                <textarea name="exclude_terms" class="form-textarea" rows="2" placeholder="login&#10;signup&#10;facebook.com"></textarea>
                            </div>

                            <div class="form-group">
                                <label class="checkbox-label">
                                    <input type="checkbox" name="respect_robots" checked>
                                    Respect robots.txt
                                </label>
                            </div>
                        </div>

                        <!-- Submit Button -->
                        <button type="submit" id="submitBtn" class="btn btn-primary btn-lg btn-block" style="margin-top: 1rem;">
                            🚀 Start Crawling
                        </button>

                        <!-- Infinite Mode Controls (hidden by default) -->
                        <div id="infiniteControls" style="display: none; margin-top: 1rem;">
                            <button type="button" id="startInfiniteBtn" class="btn btn-success btn-lg btn-block" onclick="startInfiniteMode()">
                                ♾️ Start Infinite Crawl + Import
                            </button>
                        </div>

                        <!-- Viral Mode Config Panel (hidden by default) -->
                        <div id="viralConfigPanel" style="display: none;">
                            <!-- Keywords + URLs side by side -->
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin-bottom: 1rem;">
                                <div class="form-group">
                                    <label class="form-label">🔍 Search Terms (one per line)</label>
                                    <textarea name="viral_keywords" class="form-textarea" rows="5" placeholder="batman&#10;python tutorials&#10;web scraping&#10;machine learning"></textarea>
                                </div>
                                <div class="form-group">
                                    <label class="form-label">🔗 Seed URLs (one per line)</label>
                                    <textarea name="viral_urls" class="form-textarea" rows="5" placeholder="https://www.google.com/search?q=batman&#10;https://www.bing.com/videos/search?q=robin&#10;https://learnbirdwatching.com/&#10;https://news.ycombinator.com/"></textarea>
                                </div>
                            </div>

                            <!-- Common Term (appended to each keyword) -->
                            <div style="margin-bottom: 1rem;">
                                <div class="form-group">
                                    <label class="form-label">🔗 Common Term (appended to EACH search term)</label>
                                    <input type="text" id="cfgCommonTerm" class="form-input" placeholder="e.g. 'latest news' → each keyword becomes 'keyword latest news'" style="width: 100%;">
                                    <small style="opacity: 0.6; font-size: 0.7rem;">Optional: added to the end of every keyword for refined searches</small>
                                </div>
                            </div>

                            <!-- Forced Domains (bypass quality check) -->
                            <div style="margin-bottom: 1rem;">
                                <div class="form-group">
                                    <label class="form-label">⭐ Forced Domains (always import, bypass quality check)</label>
                                    <input type="text" id="cfgForcedDomains" class="form-input" placeholder="e.g. bbc.com, reuters.com, nytimes.com" style="width: 100%;">
                                    <small style="opacity: 0.6; font-size: 0.7rem;">Comma-separated: these domains skip relevance/quality checks</small>
                                </div>
                            </div>

                            <!-- Content Filters -->
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin-bottom: 1rem;">
                                <div class="form-group">
                                    <label class="form-label">📋 Include Terms (page content must contain)</label>
                                    <textarea name="viral_include" class="form-textarea" rows="2" placeholder="article&#10;blog&#10;news&#10;tutorial"></textarea>
                                </div>
                                <div class="form-group">
                                    <label class="form-label">🚫 Exclude Terms (skip pages with these)</label>
                                    <textarea name="viral_exclude" class="form-textarea" rows="2" placeholder="login&#10;signup&#10;facebook.com&#10;advertisement"></textarea>
                                </div>
                            </div>

                            <!-- Link Priority -->
                            <div style="padding: 0.75rem; background: var(--bg-tertiary); border-radius: 0.5rem; margin-bottom: 1rem;">
                                <label class="form-label" style="margin-bottom: 0.5rem;">🎯 Link Priority</label>
                                <div style="display: flex; flex-wrap: wrap; gap: 1rem;">
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="postsFirst" checked>
                                        Posts/Articles first
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="followExternal" checked>
                                        Follow external links
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="sameDomainOnly">
                                        Same domain only
                                    </label>
                                </div>
                            </div>

                            <!-- Media Filters -->
                            <div style="padding: 0.75rem; background: var(--bg-tertiary); border-radius: 0.5rem; margin-bottom: 1rem;">
                                <label class="form-label" style="margin-bottom: 0.5rem;">📁 Media Filters (what to INCLUDE in pool)</label>
                                <div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 0.5rem;">
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterPages" checked>
                                        🌐 Web Pages
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterArticles" checked>
                                        📄 Articles/Posts
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterImages">
                                        🖼️ Images
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterVideos">
                                        🎬 Videos
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterAudio">
                                        🎵 Audio
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterDocs">
                                        📑 Documents
                                    </label>
                                    <label class="checkbox-label">
                                        <input type="checkbox" id="filterArchives">
                                        📦 Archives
                                    </label>
                                </div>
                            </div>

                            <!-- Speed/Scale Config -->
                            <div style="padding: 0.75rem; background: var(--bg-tertiary); border-radius: 0.5rem; margin-bottom: 1rem;">
                                <label class="form-label" style="margin-bottom: 0.5rem;">⚡ Speed & Scale</label>
                                <div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 0.75rem;">
                                    <div class="form-group" style="margin: 0;">
                                        <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Wave Size</label>
                                        <input type="number" id="cfgWaveSize" class="form-input" value="25" min="1" max="100">
                                    </div>
                                    <div class="form-group" style="margin: 0;">
                                        <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Links/Page</label>
                                        <input type="number" id="cfgLinksPerPage" class="form-input" value="100" min="10" max="500">
                                    </div>
                                    <div class="form-group" style="margin: 0;">
                                        <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Delay (ms)</label>
                                        <input type="number" id="cfgWaveDelay" class="form-input" value="10" min="0" max="5000">
                                    </div>
                                </div>
                                <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 0.75rem; margin-top: 0.75rem;">
                                    <div class="form-group" style="margin: 0;">
                                        <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Max Pool</label>
                                        <input type="number" id="cfgMaxPool" class="form-input" value="500000" min="1000">
                                    </div>
                                    <div class="form-group" style="margin: 0;">
                                        <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Max Imports</label>
                                        <input type="number" id="cfgMaxImports" class="form-input" value="10000000" min="100">
                                    </div>
                                </div>
                                <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 0.75rem; margin-top: 0.75rem;">
                                    <div class="form-group" style="margin: 0;">
                                        <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Max/Domain</label>
                                        <input type="number" id="cfgMaxPerDomain" class="form-input" value="0" min="0" max="10000000" title="0 = Ilimitado (Infinito)">
                                    </div>
                                    <div class="form-group" style="margin: 0; display: flex; align-items: flex-end;">
                                        <span style="font-size: 0.7rem; color: #94a3b8; padding-bottom: 0.5rem;">0 = Ilimitado</span>
                                    </div>
                                </div>
                            </div>
                        </div>

                        <!-- Viral Mode Controls (hidden by default) -->
                        <div id="viralControls" style="display: none; margin-top: 1rem;">
                            <button type="button" id="startViralBtn" class="btn btn-lg btn-block" style="background: linear-gradient(135deg, #ef4444, #f97316); color: white;" onclick="startViralMode()">
                                🦠 Start VIRAL Spreading
                            </button>
                        </div>

                        <!-- Multi-Process Config Panel (hidden by default) -->
                        <div id="multiConfigPanel" style="display: none;">
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin-bottom: 1rem;">
                                <div class="form-group">
                                    <label class="form-label">🔍 Keywords (one per line)</label>
                                    <textarea id="multiKeywords" class="form-textarea" rows="5" placeholder="batman&#10;python tutorials&#10;web scraping&#10;machine learning"></textarea>
                                </div>
                                <div class="form-group">
                                    <label class="form-label">🔗 Seed URLs (optional, one per line)</label>
                                    <textarea id="multiSeedUrls" class="form-textarea" rows="5" placeholder="https://example.com&#10;https://news.ycombinator.com/"></textarea>
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 0.75rem; margin-bottom: 1rem;">
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Common Term</label>
                                    <input type="text" id="multiCommonTerm" class="form-input" placeholder="e.g. latest news">
                                </div>
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Forced Domains</label>
                                    <input type="text" id="multiForcedDomains" class="form-input" placeholder="bbc.com, reuters.com">
                                </div>
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Wave Size</label>
                                    <input type="number" id="multiWaveSize" class="form-input" value="10" min="1" max="50">
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: 1fr 1fr 1fr 1fr; gap: 0.75rem; margin-bottom: 1rem;">
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Links/Page</label>
                                    <input type="number" id="multiLinksPerPage" class="form-input" value="100" min="10" max="500">
                                </div>
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Max Pool</label>
                                    <input type="number" id="multiMaxPool" class="form-input" value="500000" min="1000">
                                </div>
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;">Max Imports</label>
                                    <input type="number" id="multiMaxImports" class="form-input" value="10000000" min="100">
                                </div>
                                <div class="form-group" style="margin: 0;">
                                    <label class="form-label" style="font-size: 0.75rem; opacity: 0.7;"># Processes</label>
                                    <input type="number" id="multiNumProcesses" class="form-input" value="5" min="1" max="100">
                                </div>
                            </div>
                            <div style="padding: 0.5rem 0.75rem; background: rgba(6,182,212,0.1); border: 1px solid rgba(6,182,212,0.2); border-radius: 0.5rem; font-size: 0.75rem; color: var(--text-secondary); margin-bottom: 1rem;">
                                💡 Each process runs independently with shared URL pool. No duplicates across processes. Heartbeat monitoring auto-cleans dead workers.
                            </div>
                        </div>

                        <!-- Multi-Process Controls (hidden by default) -->
                        <div id="multiControls" style="display: none; margin-top: 1rem;">
                            <button type="button" id="startMultiBtn" class="btn btn-lg btn-block" style="background: linear-gradient(135deg, #06b6d4, #8b5cf6); color: white;" onclick="MPManager.launchBatch()">
                                🚀 Launch Multi-Process Workers
                            </button>
                        </div>

                        <!-- ULTIMATE MODE CONFIG PANEL -->
                        <div id="ultimateConfigPanel" style="display: none;">
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin-top: 1rem;">
                                <div>
                                    <label class="form-label">Keywords (one per line)</label>
                                    <textarea id="ultimateKeywords" class="form-textarea" rows="4" placeholder="web scraping&#10;python tutorials&#10;data science"></textarea>
                                </div>
                                <div>
                                    <label class="form-label">Seed URLs (optional)</label>
                                    <textarea id="ultimateSeedUrls" class="form-textarea" rows="4" placeholder="https://example.com&#10;https://blog.example.com"></textarea>
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin-top: 0.75rem;">
                                <div>
                                    <label class="form-label">Include Terms (filter)</label>
                                    <textarea id="ultimateIncludeTerms" class="form-textarea" rows="2" placeholder="Optional: only import if contains..."></textarea>
                                </div>
                                <div>
                                    <label class="form-label">Exclude Terms</label>
                                    <textarea id="ultimateExcludeTerms" class="form-textarea" rows="2" placeholder="Optional: skip if contains..."></textarea>
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: repeat(4, 1fr); gap: 0.75rem; margin-top: 0.75rem;">
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Workers</label>
                                    <input type="number" id="ultimateNumProcesses" class="form-input" value="5" min="1" max="100">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Wave Size</label>
                                    <input type="number" id="ultimateWaveSize" class="form-input" value="25" min="1" max="100">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Links/Page</label>
                                    <input type="number" id="ultimateLinksPerPage" class="form-input" value="150" min="10" max="250">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Max Depth</label>
                                    <input type="number" id="ultimateMaxDepth" class="form-input" value="5" min="1" max="10">
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: repeat(4, 1fr); gap: 0.75rem; margin-top: 0.75rem;">
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Quality Min</label>
                                    <input type="number" id="ultimateQualityThreshold" class="form-input" value="20" min="0" max="100">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Relevance Min</label>
                                    <input type="number" id="ultimateRelevanceThreshold" class="form-input" value="2" min="0" max="10">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Max Pool</label>
                                    <input type="number" id="ultimateMaxPool" class="form-input" value="500000" min="1000">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Max Imports</label>
                                    <input type="number" id="ultimateMaxImports" class="form-input" value="10000000" min="100">
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: repeat(4, 1fr); gap: 0.75rem; margin-top: 0.75rem;">
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Max/Domain</label>
                                    <input type="number" id="ultimateMaxPerDomain" class="form-input" value="0" min="0" max="10000000" title="0 = Ilimitado">
                                </div>
                                <div style="display:flex; align-items:end;">
                                    <span style="font-size:0.65rem; color:#94a3b8; padding-bottom:0.5rem;">0 = Ilimitado (Infinito)</span>
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 0.75rem; margin-top: 0.75rem;">
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Common Term</label>
                                    <input type="text" id="ultimateCommonTerm" class="form-input" placeholder="e.g. latest news">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Forced Domains</label>
                                    <input type="text" id="ultimateForcedDomains" class="form-input" placeholder="bbc.com, reuters.com">
                                </div>
                            </div>
                            <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 0.75rem; margin-top: 0.75rem;">
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">Pagination Pattern (optional)</label>
                                    <input type="text" id="ultimatePaginationPattern" class="form-input" placeholder="https://site.com/page/1">
                                </div>
                                <div style="display:flex; gap:0.5rem; align-items:end;">
                                    <div style="flex:1;">
                                        <label class="form-label" style="font-size:0.75rem;">Start</label>
                                        <input type="number" id="ultimateStartPage" class="form-input" value="1" min="1">
                                    </div>
                                    <div style="flex:1;">
                                        <label class="form-label" style="font-size:0.75rem;">End</label>
                                        <input type="number" id="ultimateEndPage" class="form-input" value="50" min="1">
                                    </div>
                                </div>
                            </div>
                            <div style="margin-top: 0.75rem; display: flex; flex-wrap: wrap; gap: 1rem;">
                                <label style="color:#e2e8f0; font-size:0.8rem; display:flex; align-items:center; gap:0.3rem;">
                                    <input type="checkbox" id="ultimatePostsFirst" checked> Posts Priority
                                </label>
                                <label style="color:#e2e8f0; font-size:0.8rem; display:flex; align-items:center; gap:0.3rem;">
                                    <input type="checkbox" id="ultimateFollowExternal" checked> Follow External
                                </label>
                                <label style="color:#e2e8f0; font-size:0.8rem; display:flex; align-items:center; gap:0.3rem;">
                                    <input type="checkbox" id="ultimateEnablePagination" checked> Pagination Discovery
                                </label>
                            </div>
                            <!-- URL Pattern Filter -->
                            <div style="margin-top:0.75rem;">
                                <label class="form-label" style="font-size:0.75rem;">URL Pattern Filter (regex, optional)</label>
                                <input type="text" id="ultimateUrlPattern" class="form-input" placeholder="ex: /video/ ou /product/\d+ (so processa URLs matching)">
                                <div style="font-size:0.65rem;color:var(--text-secondary);margin-top:2px;">Filtra URLs por pattern ANTES do check de conteudo. Vazio = todas.</div>
                            </div>
                            <!-- HTTP Auth -->
                            <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.75rem;margin-top:0.75rem;">
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">HTTP Auth User</label>
                                    <input type="text" id="ultimateAuthUser" class="form-input" placeholder="(opcional)">
                                </div>
                                <div>
                                    <label class="form-label" style="font-size:0.75rem;">HTTP Auth Pass</label>
                                    <input type="password" id="ultimateAuthPass" class="form-input" placeholder="(opcional)">
                                </div>
                            </div>
                            <!-- Rate Limit Slider -->
                            <div style="margin-top:0.75rem;">
                                <label class="form-label" style="font-size:0.75rem;">Fetch Delay: <span id="ultimateDelayLabel">0ms</span></label>
                                <input type="range" id="ultimateFetchDelay" min="0" max="5000" value="0" step="50" style="width:100%;accent-color:var(--accent);" oninput="document.getElementById('ultimateDelayLabel').textContent=this.value+'ms'">
                                <div style="display:flex;justify-content:space-between;font-size:0.6rem;color:var(--text-secondary);"><span>0ms (max speed)</span><span>5000ms (polite)</span></div>
                            </div>
                            <!-- Page Example Auto-Generator -->
                            <div style="margin-top:0.75rem;padding:0.5rem;background:rgba(99,102,241,0.05);border:1px solid rgba(99,102,241,0.15);border-radius:0.4rem;">
                                <label class="form-label" style="font-size:0.75rem;">Page Example (auto-detect pattern)</label>
                                <div style="display:flex;gap:0.5rem;align-items:end;">
                                    <div style="flex:3;">
                                        <input type="text" id="ultimatePageExample" class="form-input" placeholder="https://site.com/page/3">
                                    </div>
                                    <div style="flex:1;">
                                        <label class="form-label" style="font-size:0.65rem;">Pages</label>
                                        <input type="number" id="ultimatePageCount" class="form-input" value="20" min="1" max="200">
                                    </div>
                                    <button type="button" onclick="generatePagesFromExample()" style="background:var(--accent);border:none;color:white;padding:0.4rem 0.6rem;border-radius:0.3rem;cursor:pointer;font-size:0.7rem;white-space:nowrap;">Gen</button>
                                </div>
                                <div id="generatedPagesPreview" style="display:none;margin-top:0.4rem;font-size:0.65rem;color:var(--text-secondary);max-height:60px;overflow-y:auto;"></div>
                            </div>
                        </div>

                        <div id="ultimateControls" style="display: none; margin-top: 1rem;">
                            <button type="button" class="btn btn-lg btn-block" style="background: linear-gradient(135deg, #f43f5e, #6366f1, #06b6d4); color: white; font-weight: bold;" onclick="UltimateManager.launchBatch()">
                                ⚡ Launch ULTIMATE Workers
                            </button>
                        </div>
                    </div>
                </div>

                <!-- Infinite Mode Stats Panel -->
                <div id="infiniteStatsPanel" class="panel" style="display: none; grid-column: 1 / -1; margin-top: 1rem;">
                    <div class="panel-header" style="background: linear-gradient(135deg, #6366f1, #8b5cf6);">
                        <h2 class="panel-title" style="color: white;">♾️ INFINITE MODE - Live Stats</h2>
                        <div>
                            <button type="button" id="pauseBtn" class="btn" style="background: #f59e0b; color: white; margin-right: 0.5rem;" onclick="pauseInfinite()">⏸️ Pause</button>
                            <button type="button" id="stopBtn" class="btn" style="background: #ef4444; color: white;" onclick="stopInfinite()">⏹️ Stop</button>
                        </div>
                    </div>
                    <div class="panel-body">
                        <div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
                            <div class="stat-card">
                                <span class="stat-value" id="infDiscovered">0</span>
                                <span class="stat-label">Discovered</span>
                            </div>
                            <div class="stat-card">
                                <span class="stat-value" id="infProcessed">0</span>
                                <span class="stat-label">Processed</span>
                            </div>
                            <div class="stat-card success">
                                <span class="stat-value" id="infImported">0</span>
                                <span class="stat-label">Imported</span>
                            </div>
                            <div class="stat-card warning">
                                <span class="stat-value" id="infDuplicates">0</span>
                                <span class="stat-label">Duplicates</span>
                            </div>
                            <div class="stat-card error">
                                <span class="stat-value" id="infErrors">0</span>
                                <span class="stat-label">Errors</span>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, #6366f1, #8b5cf6);">
                                <span class="stat-value" id="infRate" style="color: white;">0/s</span>
                                <span class="stat-label" style="color: rgba(255,255,255,0.8);">Import Rate</span>
                            </div>
                        </div>
                        <div style="margin-top: 1rem; padding: 1rem; background: var(--bg-tertiary); border-radius: 0.5rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center;">
                                <div>
                                    <span style="color: var(--text-secondary);">Status: </span>
                                    <span id="infStatus" style="color: var(--success); font-weight: 600;">●  Running</span>
                                </div>
                                <div>
                                    <span style="color: var(--text-secondary);">Queue: </span>
                                    <span id="infQueue" style="font-weight: 600;">0</span>
                                </div>
                                <div>
                                    <span style="color: var(--text-secondary);">Engine: </span>
                                    <span id="infEngine" style="font-weight: 600;">-</span>
                                </div>
                                <div>
                                    <span style="color: var(--text-secondary);">Elapsed: </span>
                                    <span id="infElapsed" style="font-weight: 600;">0s</span>
                                </div>
                            </div>
                            <div style="margin-top: 0.5rem; font-size: 0.85rem; color: var(--text-secondary);">
                                Last URL: <span id="infLastUrl" style="color: var(--text-primary);">-</span>
                            </div>
                        </div>

                        <!-- Activity Log -->
                        <div style="margin-top: 1rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
                                <h3 style="font-size: 0.9rem; font-weight: 600;">Activity Log</h3>
                                <button type="button" onclick="clearLog()" style="background: var(--bg-tertiary); border: 1px solid var(--border); color: var(--text-secondary); padding: 0.25rem 0.75rem; border-radius: 0.25rem; cursor: pointer; font-size: 0.75rem;">Clear</button>
                            </div>
                            <div id="activityLog" style="max-height: 400px; overflow-y: auto; background: var(--bg-primary); border: 1px solid var(--border); border-radius: 0.5rem; font-family: 'JetBrains Mono', monospace; font-size: 0.75rem; line-height: 1.6;">
                                <div style="padding: 1rem; color: var(--text-secondary); text-align: center;">
                                    Waiting for activity...
                                </div>
                            </div>
                        </div>
                    </div>
                </div>

                <!-- Viral Mode Stats Panel -->
                <div id="viralStatsPanel" class="panel" style="display: none; grid-column: 1 / -1; margin-top: 1rem;">
                    <div class="panel-header" style="background: linear-gradient(135deg, #ef4444, #f97316);">
                        <h2 class="panel-title" style="color: white;">🦠 VIRAL MODE - Spreading Network</h2>
                        <div style="display: flex; gap: 0.5rem;">
                            <button type="button" id="viralPauseBtn" class="btn" style="background: rgba(255,255,255,0.2); color: white;" onclick="toggleViralPause()">⏸️ Pause</button>
                            <button type="button" id="viralStopBtn" class="btn" style="background: rgba(255,255,255,0.2); color: white;" onclick="stopViral()">⏹️ Stop</button>
                        </div>
                    </div>
                    <div class="panel-body">
                        <div class="stats-grid" style="grid-template-columns: repeat(4, 1fr);">
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(239,68,68,0.3), rgba(249,115,22,0.3)); border-color: rgba(239,68,68,0.3);">
                                <span class="stat-value" id="viralPoolSize" style="color: var(--orange);">0</span>
                                <span class="stat-label">Pool Size</span>
                                <canvas class="sparkline-canvas" id="sparkPool" width="80" height="24"></canvas>
                            </div>
                            <div class="stat-card" style="border-color: rgba(139,92,246,0.2);">
                                <span class="stat-value" id="viralWaves" style="color: var(--violet);">0</span>
                                <span class="stat-label">Waves</span>
                                <canvas class="sparkline-canvas" id="sparkWaves" width="80" height="24"></canvas>
                            </div>
                            <div class="stat-card" style="border-color: rgba(34,197,94,0.2);">
                                <span class="stat-value" id="viralImported" style="color: var(--success);">0</span>
                                <span class="stat-label">Imported</span>
                                <canvas class="sparkline-canvas" id="sparkImported" width="80" height="24"></canvas>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(139,92,246,0.2), rgba(99,102,241,0.2)); border-color: rgba(139,92,246,0.3);">
                                <span class="stat-value" id="viralRate" style="color: var(--pink);">0/s</span>
                                <span class="stat-label">Rate</span>
                                <canvas class="sparkline-canvas" id="sparkRate" width="80" height="24"></canvas>
                            </div>
                        </div>
                        <div class="stats-grid" style="grid-template-columns: repeat(5, 1fr); margin-top: 0.75rem;">
                            <div class="stat-card" style="border-color: rgba(6,182,212,0.2);">
                                <span class="stat-value" id="viralDiscovered" style="color: var(--cyan);">0</span>
                                <span class="stat-label">Discovered</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(20,184,166,0.2);">
                                <span class="stat-value" id="viralProcessed" style="color: var(--teal);">0</span>
                                <span class="stat-label">Processed</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(245,158,11,0.2);">
                                <span class="stat-value" id="viralDuplicates" style="color: var(--warning);">0</span>
                                <span class="stat-label">Duplicates</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(107,114,128,0.2);">
                                <span class="stat-value" id="viralSkipped" style="color: #9ca3af;">0</span>
                                <span class="stat-label">Skipped</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(239,68,68,0.2);">
                                <span class="stat-value" id="viralErrors" style="color: var(--error);">0</span>
                                <span class="stat-label">Errors</span>
                            </div>
                        </div>
                        <!-- B27: Extra derived stats row -->
                        <div class="stats-grid" style="grid-template-columns: repeat(3, 1fr); margin-top: 0.75rem;">
                            <div class="stat-card" style="border-color: rgba(132,204,22,0.2);">
                                <span class="stat-value" id="viralSuccessRate" style="color: var(--lime); font-size: 1.1rem;">0%</span>
                                <span class="stat-label">Success Rate</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(14,165,233,0.2);">
                                <span class="stat-value" id="viralUrlsMin" style="color: var(--sky); font-size: 1.1rem;">0</span>
                                <span class="stat-label">URLs/min</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(217,70,239,0.2);">
                                <span class="stat-value" id="viralPagesHour" style="color: var(--fuchsia); font-size: 1.1rem;">0</span>
                                <span class="stat-label">Pages/hr</span>
                            </div>
                        </div>
                        <div style="margin-top: 1rem; padding: 1rem; background: var(--bg-tertiary); border-radius: 0.5rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center;">
                                <div>
                                    <span style="color: var(--text-secondary);">Status: </span>
                                    <span id="viralStatus" style="color: var(--success); font-weight: 600;">● Ready</span>
                                </div>
                                <div style="display: flex; gap: 1.5rem;">
                                    <div>
                                        <span style="color: var(--text-secondary);">ETA: </span>
                                        <span id="viralETA" style="font-weight: 600; color: #a78bfa;">--</span>
                                    </div>
                                    <div>
                                        <span style="color: var(--text-secondary);">Elapsed: </span>
                                        <span id="viralElapsed" style="font-weight: 600;">0s</span>
                                    </div>
                                </div>
                            </div>
                            <div style="margin-top: 0.5rem; display: flex; justify-content: space-between; align-items: center; font-size: 0.85rem;">
                                <div style="color: var(--text-secondary);">
                                    Last URL: <span id="viralLastUrl" style="color: var(--text-primary);">-</span>
                                </div>
                                <div style="color: var(--text-secondary);">
                                    📊 <span id="viralBandwidth" style="color: #60a5fa;">0 MB</span>
                                </div>
                            </div>
                            <!-- Pool Growth Bar -->
                            <div style="margin-top: 0.75rem;">
                                <div style="display: flex; justify-content: space-between; font-size: 0.75rem; color: var(--text-secondary); margin-bottom: 0.25rem;">
                                    <span>Pool Growth</span>
                                    <span id="viralPoolPercent">0%</span>
                                </div>
                                <div style="height: 6px; background: var(--bg-primary); border-radius: 3px; overflow: hidden;">
                                    <div id="viralPoolBar" style="height: 100%; width: 0%; background: linear-gradient(90deg, #ef4444, #f97316); border-radius: 3px; transition: width 0.3s;"></div>
                                </div>
                            </div>
                            <!-- I2: Pool Health Indicator -->
                            <div style="margin-top: 0.5rem; display: flex; justify-content: space-between; align-items: center; font-size: 0.75rem;">
                                <span style="color: var(--text-secondary);">Pool Health: <span id="viralPoolHealth" style="font-weight: 600;">🟢 Healthy</span></span>
                                <span style="color: var(--text-secondary);">Domains: <strong id="viralUniqueDomains" style="color: #a78bfa;">0</strong> | Wave: <strong id="viralWaveSize" style="color: #f97316;">10</strong></span>
                            </div>
                        </div>

                        <!-- B17: Progress Ring + B19: Rate Chart -->
                        <div style="display: flex; gap: 1rem; margin-top: 1rem; align-items: center;">
                            <!-- B17: Circular progress ring -->
                            <div style="text-align: center; flex-shrink: 0;">
                                <svg class="progress-ring" width="70" height="70">
                                    <circle class="progress-ring-bg" cx="35" cy="35" r="30"/>
                                    <circle class="progress-ring-fill" id="progressRingFill" cx="35" cy="35" r="30" stroke-dasharray="188.5" stroke-dashoffset="188.5"/>
                                    <text x="35" y="35" text-anchor="middle" dy="4" fill="var(--accent)" font-size="11" font-weight="700" id="progressRingText">0%</text>
                                </svg>
                                <div style="font-size:0.6rem;color:var(--text-secondary);margin-top:2px;">Completion</div>
                            </div>
                            <!-- B19: Live rate chart -->
                            <div class="rate-chart-container" style="flex:1;height:70px;">
                                <span class="rate-chart-label" id="rateChartLabel">0/s</span>
                                <canvas id="rateChartCanvas" height="70" style="width:100%;height:70px;display:block;"></canvas>
                            </div>
                        </div>

                        <!-- B26: Wave processing bar -->
                        <div class="wave-bar" id="waveProcessBar" style="margin-top:0.75rem;">
                            <div class="wave-bar-fill wave-processing" id="waveBarFill" style="width:0%;background:linear-gradient(90deg,var(--accent),var(--purple),var(--accent));background-size:200% 100%;"></div>
                        </div>

                        <!-- Network Visualization -->
                        <div style="margin-top: 1rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
                                <h3 style="font-size: 0.9rem; font-weight: 600;">🕸️ Network Map</h3>
                                <div style="display: flex; gap: 0.8rem; font-size: 0.72rem; color: var(--text-secondary); flex-wrap:wrap;">
                                    <span>Nodes: <strong id="viralNodeCount" style="color: #f97316;">0</strong></span>
                                    <span>Edges: <strong id="viralEdgeCount" style="color: #6366f1;">0</strong></span>
                                    <span>Countries: <strong id="viralCountryCount" style="color: #06b6d4;">0</strong></span>
                                    <span>Importing: <strong id="viralImportingCount" style="color: #22c55e;">0</strong></span>
                                </div>
                            </div>
                            <!-- Controls Row 1: Actions -->
                            <div style="display:flex; gap:5px; margin-bottom:5px; flex-wrap:wrap; align-items:center;">
                                <button type="button" onclick="networkGraph.togglePause()" id="netPauseBtn" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">⏸ Pause</button>
                                <button type="button" onclick="networkGraph.resetView()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">↺ Reset</button>
                                <button type="button" onclick="networkGraph.zoomIn()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">🔍+</button>
                                <button type="button" onclick="networkGraph.zoomOut()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">🔍-</button>
                                <button type="button" onclick="networkGraph.fitAll()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">⊞ Fit</button>
                                <button type="button" onclick="networkGraph.toggleFullscreen()" id="netFullBtn" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">⛶ Fullscreen</button>
                                <button type="button" onclick="networkGraph.exportPNG()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">📸 Export</button>
                                <button type="button" onclick="networkGraph.toggleLabels()" id="netLabelsBtn" style="background:var(--bg-tertiary);border:1px solid var(--border);color:#22c55e;padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">🏷 Labels</button>
                                <button type="button" onclick="networkGraph.toggleParticles()" id="netParticlesBtn" style="background:var(--bg-tertiary);border:1px solid var(--border);color:#06b6d4;padding:3px 8px;border-radius:4px;cursor:pointer;font-size:0.68rem;">✨ Particles</button>
                            </div>
                            <!-- Controls Row 2: Filters + Search -->
                            <div style="display:flex; gap:5px; margin-bottom:6px; flex-wrap:wrap; align-items:center;">
                                <button type="button" onclick="networkGraph.setFilter('all')" class="net-filter-btn active" data-filter="all" style="background:#6366f1;border:1px solid #6366f1;color:#fff;padding:2px 7px;border-radius:10px;cursor:pointer;font-size:0.62rem;font-weight:600;">All</button>
                                <button type="button" onclick="networkGraph.setFilter('imported')" class="net-filter-btn" data-filter="imported" style="background:var(--bg-tertiary);border:1px solid var(--border);color:#22c55e;padding:2px 7px;border-radius:10px;cursor:pointer;font-size:0.62rem;">Importing</button>
                                <button type="button" onclick="networkGraph.setFilter('active')" class="net-filter-btn" data-filter="active" style="background:var(--bg-tertiary);border:1px solid var(--border);color:#6366f1;padding:2px 7px;border-radius:10px;cursor:pointer;font-size:0.62rem;">Active</button>
                                <button type="button" onclick="networkGraph.setFilter('error')" class="net-filter-btn" data-filter="error" style="background:var(--bg-tertiary);border:1px solid var(--border);color:#f59e0b;padding:2px 7px;border-radius:10px;cursor:pointer;font-size:0.62rem;">Errors</button>
                                <button type="button" onclick="networkGraph.setFilter('dead')" class="net-filter-btn" data-filter="dead" style="background:var(--bg-tertiary);border:1px solid var(--border);color:#6b7280;padding:2px 7px;border-radius:10px;cursor:pointer;font-size:0.62rem;">Dead</button>
                                <input type="text" id="netSearchInput" placeholder="🔎 Search domain..." oninput="networkGraph.searchNodes(this.value)" style="margin-left:auto;background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-primary);padding:3px 8px;border-radius:4px;font-size:0.68rem;width:160px;outline:none;" />
                            </div>
                            <div id="netGraphContainer" style="display:flex; gap:8px; position:relative;">
                                <!-- Canvas container -->
                                <div id="netCanvasWrap" style="position:relative; flex:1; background:linear-gradient(135deg, rgba(10,10,18,0.95), rgba(15,15,30,0.95)); border:1px solid var(--border); border-radius:0.5rem; overflow:hidden;">
                                    <canvas id="networkCanvas" width="800" height="600" style="width:100%; height:600px; display:block;"></canvas>
                                    <!-- Tooltip -->
                                    <div id="netTooltip" style="display:none;position:absolute;background:rgba(10,12,18,0.96);border:1px solid rgba(99,102,241,0.5);border-radius:10px;padding:10px 14px;font-size:0.72rem;color:#fff;pointer-events:none;z-index:50;min-width:200px;max-width:280px;box-shadow:0 8px 24px rgba(0,0,0,0.6);backdrop-filter:blur(8px);"></div>
                                    <!-- Minimap -->
                                    <canvas id="netMinimap" width="160" height="100" style="position:absolute;bottom:8px;right:8px;width:160px;height:100px;border:1px solid rgba(99,102,241,0.3);border-radius:6px;background:rgba(10,10,18,0.8);cursor:pointer;z-index:40;"></canvas>
                                    <!-- Legend -->
                                    <div style="position:absolute;bottom:8px;left:8px;display:flex;gap:8px;font-size:0.6rem;color:var(--text-secondary);background:rgba(10,12,18,0.85);padding:4px 10px;border-radius:6px;backdrop-filter:blur(4px);flex-wrap:wrap;">
                                        <span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#ef4444;margin-right:3px;box-shadow:0 0 4px #ef4444;"></span>Seed</span>
                                        <span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#22c55e;margin-right:3px;box-shadow:0 0 4px #22c55e;"></span>Importing</span>
                                        <span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#6366f1;margin-right:3px;box-shadow:0 0 4px #6366f1;"></span>Active</span>
                                        <span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#f59e0b;margin-right:3px;box-shadow:0 0 4px #f59e0b;"></span>Error</span>
                                        <span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#6b7280;margin-right:3px;box-shadow:0 0 4px #6b7280;"></span>Dead</span>
                                        <span style="opacity:0.5;">|</span>
                                        <span style="opacity:0.7;">Scroll:Zoom | Drag:Pan | Click:Details</span>
                                    </div>
                                    <!-- Stats overlay top-right -->
                                    <div id="netStatsOverlay" style="position:absolute;top:8px;right:8px;font-size:0.62rem;color:rgba(255,255,255,0.7);background:rgba(10,12,18,0.88);backdrop-filter:blur(8px);padding:8px 12px;border-radius:8px;line-height:1.5;text-align:right;border:1px solid rgba(255,255,255,0.06);min-width:180px;max-width:220px;box-shadow:0 4px 20px rgba(0,0,0,0.4);"></div>
                                </div>
                                <!-- Detail Panel (Enhanced) -->
                                <div id="netDetailPanel" style="display:none;width:260px;background:linear-gradient(180deg, var(--bg-primary), rgba(15,15,30,0.95));border:1px solid var(--border);border-radius:0.5rem;padding:12px;font-size:0.72rem;overflow-y:auto;max-height:600px;">
                                    <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:10px;">
                                        <div>
                                            <span id="netDetailFlag" style="font-size:1.2rem;margin-right:4px;"></span>
                                            <strong id="netDetailDomain" style="color:#6366f1;word-break:break-all;font-size:0.8rem;"></strong>
                                        </div>
                                        <button type="button" onclick="networkGraph.closeDetails()" style="background:none;border:none;color:var(--text-secondary);cursor:pointer;font-size:1.2rem;line-height:1;">&times;</button>
                                    </div>
                                    <div id="netDetailCountry" style="font-size:0.68rem;color:var(--text-secondary);margin-bottom:8px;"></div>
                                    <div id="netDetailContent" style="color:var(--text-secondary);line-height:1.8;"></div>
                                    <div id="netDetailActions" style="margin-top:10px;padding-top:8px;border-top:1px solid var(--border);"></div>
                                </div>
                            </div>
                        </div>

                        <!-- Viral Activity Log -->
                        <div style="margin-top: 1rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
                                <h3 style="font-size: 0.9rem; font-weight: 600;">🦠 Viral Activity Log</h3>
                                <div style="display: flex; gap: 0.5rem; align-items: center;">
                                    <button type="button" onclick="exportViralResults('csv')" title="Export CSV" style="background: var(--bg-tertiary); border: 1px solid var(--border); color: var(--text-secondary); padding: 0.25rem 0.5rem; border-radius: 0.25rem; cursor: pointer; font-size: 0.7rem;">CSV</button>
                                    <button type="button" onclick="exportViralResults('json')" title="Export JSON" style="background: var(--bg-tertiary); border: 1px solid var(--border); color: var(--text-secondary); padding: 0.25rem 0.5rem; border-radius: 0.25rem; cursor: pointer; font-size: 0.7rem;">JSON</button>
                                    <button type="button" onclick="clearViralLog()" style="background: var(--bg-tertiary); border: 1px solid var(--border); color: var(--text-secondary); padding: 0.25rem 0.5rem; border-radius: 0.25rem; cursor: pointer; font-size: 0.7rem;">Clear</button>
                                </div>
                            </div>
                            <!-- U4: Log filters + search -->
                            <div style="display: flex; gap: 0.5rem; margin-bottom: 0.5rem; flex-wrap: wrap; align-items: center;">
                                <input type="text" id="viralLogSearch" placeholder="Search URL/domain..." oninput="filterViralLog()" style="flex: 1; min-width: 150px; padding: 0.3rem 0.6rem; background: var(--bg-primary); border: 1px solid var(--border); border-radius: 0.25rem; color: var(--text-primary); font-size: 0.75rem;">
                                <div style="display: flex; gap: 0.25rem;">
                                    <button type="button" class="log-filter-btn active" data-filter="all" onclick="setLogFilter('all', this)" style="padding: 0.2rem 0.5rem; border-radius: 0.25rem; border: 1px solid var(--border); background: #22c55e33; color: var(--text-primary); cursor: pointer; font-size: 0.7rem;">All</button>
                                    <button type="button" class="log-filter-btn" data-filter="imported" onclick="setLogFilter('imported', this)" style="padding: 0.2rem 0.5rem; border-radius: 0.25rem; border: 1px solid var(--border); background: var(--bg-tertiary); color: var(--text-secondary); cursor: pointer; font-size: 0.7rem;">✅</button>
                                    <button type="button" class="log-filter-btn" data-filter="duplicate" onclick="setLogFilter('duplicate', this)" style="padding: 0.2rem 0.5rem; border-radius: 0.25rem; border: 1px solid var(--border); background: var(--bg-tertiary); color: var(--text-secondary); cursor: pointer; font-size: 0.7rem;">🔄</button>
                                    <button type="button" class="log-filter-btn" data-filter="skipped" onclick="setLogFilter('skipped', this)" style="padding: 0.2rem 0.5rem; border-radius: 0.25rem; border: 1px solid var(--border); background: var(--bg-tertiary); color: var(--text-secondary); cursor: pointer; font-size: 0.7rem;">⚪</button>
                                    <button type="button" class="log-filter-btn" data-filter="error" onclick="setLogFilter('error', this)" style="padding: 0.2rem 0.5rem; border-radius: 0.25rem; border: 1px solid var(--border); background: var(--bg-tertiary); color: var(--text-secondary); cursor: pointer; font-size: 0.7rem;">❌</button>
                                </div>
                            </div>
                            <div id="viralActivityLog" style="max-height: 400px; overflow-y: auto; background: var(--bg-primary); border: 1px solid var(--border); border-radius: 0.5rem; font-family: 'JetBrains Mono', monospace; font-size: 0.75rem; line-height: 1.6;">
                                <div style="padding: 1rem; color: var(--text-secondary); text-align: center;">
                                    🦠 Waiting for viral spreading to start...
                                </div>
                            </div>
                            <!-- I1: Engine Stats -->
                            <div id="engineStatsSection" style="display: none; margin-top: 0.75rem;">
                                <h4 style="font-size: 0.8rem; font-weight: 600; margin-bottom: 0.4rem; color: var(--text-secondary);">🔍 Engine Performance</h4>
                                <div id="engineStatsContent" style="display: flex; flex-wrap: wrap; gap: 0.4rem; font-size: 0.7rem;"></div>
                            </div>
                            <!-- B18: Domain Leaderboard -->
                            <div id="leaderboardSection" style="display: none; margin-top: 1rem;">
                                <h4 style="font-size: 0.85rem; font-weight: 600; margin-bottom: 0.5rem; color: var(--text-secondary);">🏆 Domain Leaderboard</h4>
                                <div style="max-height: 250px; overflow-y: auto;">
                                    <table class="leaderboard" id="leaderboardTable">
                                        <thead><tr><th>#</th><th>Domain</th><th>Imports</th><th>Crawled</th></tr></thead>
                                        <tbody id="leaderboardBody"></tbody>
                                    </table>
                                </div>
                            </div>
                        </div>
                    </div>
                </div>

                <!-- Multi-Process Manager Panel -->
                <div id="multiStatsPanel" class="panel" style="display: none; grid-column: 1 / -1; margin-top: 1rem;">
                    <div class="panel-header" style="background: linear-gradient(135deg, #06b6d4, #8b5cf6);">
                        <h2 class="panel-title" style="color: white;">🚀 MULTI-PROCESS Manager</h2>
                        <div style="display: flex; gap: 0.5rem;">
                            <button type="button" class="btn" style="background: rgba(255,255,255,0.2); color: white; font-size: 0.75rem;" onclick="MPManager.launchOne()">➕ New</button>
                            <button type="button" class="btn" style="background: rgba(255,255,255,0.2); color: white; font-size: 0.75rem;" onclick="MPManager.batchControl('pause_all')">⏸️ All</button>
                            <button type="button" class="btn" style="background: rgba(255,255,255,0.2); color: white; font-size: 0.75rem;" onclick="MPManager.batchControl('resume_all')">▶️ All</button>
                            <button type="button" class="btn" style="background: rgba(239,68,68,0.6); color: white; font-size: 0.75rem;" onclick="MPManager.batchControl('stop_all')">⏹️ All</button>
                            <button type="button" class="btn" style="background: rgba(255,255,255,0.15); color: white; font-size: 0.75rem;" onclick="MPManager.cleanup()">🧹</button>
                        </div>
                    </div>
                    <div class="panel-body">
                        <!-- Aggregate Stats -->
                        <div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(6,182,212,0.2), rgba(139,92,246,0.2)); border-color: rgba(6,182,212,0.3);">
                                <span class="stat-value" id="mpTotalProcesses" style="color: #06b6d4;">0</span>
                                <span class="stat-label">Processes</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(34,197,94,0.2);">
                                <span class="stat-value" id="mpTotalImported" style="color: var(--success);">0</span>
                                <span class="stat-label">Imported</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(99,102,241,0.2);">
                                <span class="stat-value" id="mpTotalDiscovered" style="color: var(--accent);">0</span>
                                <span class="stat-label">Discovered</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(245,158,11,0.2);">
                                <span class="stat-value" id="mpTotalDuplicates" style="color: var(--warning);">0</span>
                                <span class="stat-label">Duplicates</span>
                            </div>
                            <div class="stat-card" style="border-color: rgba(239,68,68,0.2);">
                                <span class="stat-value" id="mpTotalErrors" style="color: var(--error);">0</span>
                                <span class="stat-label">Errors</span>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(139,92,246,0.2), rgba(99,102,241,0.2)); border-color: rgba(139,92,246,0.3);">
                                <span class="stat-value" id="mpTotalRate" style="color: var(--pink);">0/s</span>
                                <span class="stat-label">Combined Rate</span>
                            </div>
                        </div>

                        <!-- Pool Stats Bar -->
                        <div style="margin-top: 0.75rem; padding: 0.6rem 1rem; background: var(--bg-tertiary); border-radius: 0.5rem; display: flex; justify-content: space-between; align-items: center; font-size: 0.8rem;">
                            <div>
                                <span style="color: var(--text-secondary);">Pool: </span>
                                <span id="mpPoolPending" style="color: #06b6d4; font-weight: 600;">0</span>
                                <span style="color: var(--text-secondary);"> pending</span>
                                <span style="margin: 0 0.5rem; opacity: 0.3;">|</span>
                                <span id="mpPoolClaimed" style="color: #f59e0b; font-weight: 600;">0</span>
                                <span style="color: var(--text-secondary);"> claimed</span>
                            </div>
                            <div>
                                <span style="color: var(--text-secondary);">Seen: </span>
                                <span id="mpSeenTotal" style="color: #a78bfa; font-weight: 600;">0</span>
                            </div>
                            <div>
                                <span style="color: var(--text-secondary);">Elapsed: </span>
                                <span id="mpElapsed" style="color: var(--text-primary); font-weight: 600;">0s</span>
                            </div>
                        </div>

                        <!-- Process List -->
                        <div style="margin-top: 1rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
                                <h3 style="font-size: 0.9rem; font-weight: 600;">📋 Active Processes</h3>
                                <span id="mpProcessCount" style="font-size: 0.75rem; color: var(--text-secondary);">0 active</span>
                            </div>
                            <div id="mpProcessList" style="max-height: 500px; overflow-y: auto; display: flex; flex-direction: column; gap: 0.5rem;">
                                <div style="padding: 2rem; text-align: center; color: var(--text-secondary); font-size: 0.85rem;">
                                    No processes running. Click "Launch Multi-Process Workers" to start.
                                </div>
                            </div>
                        </div>

                        <!-- Activity Log -->
                        <div style="margin-top: 1rem;">
                            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
                                <h3 style="font-size: 0.9rem; font-weight: 600;">📜 Multi-Process Log</h3>
                                <button type="button" onclick="MPManager.clearLog()" style="background: var(--bg-tertiary); border: 1px solid var(--border); color: var(--text-secondary); padding: 0.25rem 0.75rem; border-radius: 0.25rem; cursor: pointer; font-size: 0.75rem;">Clear</button>
                            </div>
                            <div id="mpActivityLog" style="max-height: 250px; overflow-y: auto; background: var(--bg-primary); border: 1px solid var(--border); border-radius: 0.5rem; font-family: 'JetBrains Mono', monospace; font-size: 0.72rem; line-height: 1.6; padding: 0.5rem;">
                                <div style="color: var(--text-secondary); text-align: center;">Waiting for processes to start...</div>
                            </div>
                        </div>
                    </div>
                </div>

                <!-- ULTIMATE MODE STATS PANEL -->
                <div id="ultimateStatsPanel" class="panel" style="display: none; grid-column: 1 / -1; margin-top: 1rem;">
                    <div class="panel-header" style="background: linear-gradient(135deg, #f43f5e, #6366f1, #06b6d4);">
                        <h2 class="panel-title" style="color: white;">⚡ ULTIMATE MODE - All-In-One Dashboard</h2>
                        <div style="display: flex; gap: 0.5rem;">
                            <button type="button" onclick="UltimateManager.launchOne()" style="background:rgba(255,255,255,0.2);border:none;color:white;padding:0.3rem 0.6rem;border-radius:0.25rem;cursor:pointer;font-size:0.75rem;">➕ New</button>
                            <button type="button" onclick="UltimateManager.batchControl('pause_all')" style="background:rgba(255,255,255,0.2);border:none;color:white;padding:0.3rem 0.6rem;border-radius:0.25rem;cursor:pointer;font-size:0.75rem;">⏸ All</button>
                            <button type="button" onclick="UltimateManager.batchControl('resume_all')" style="background:rgba(255,255,255,0.2);border:none;color:white;padding:0.3rem 0.6rem;border-radius:0.25rem;cursor:pointer;font-size:0.75rem;">▶ All</button>
                            <button type="button" onclick="UltimateManager.batchControl('stop_all')" style="background:rgba(239,68,68,0.6);border:none;color:white;padding:0.3rem 0.6rem;border-radius:0.25rem;cursor:pointer;font-size:0.75rem;">⏹ All</button>
                            <button type="button" onclick="UltimateManager.cleanup()" style="background:rgba(255,255,255,0.15);border:none;color:white;padding:0.3rem 0.6rem;border-radius:0.25rem;cursor:pointer;font-size:0.75rem;">🧹</button>
                        </div>
                    </div>
                    <div class="panel-body" style="padding: 1rem;">
                        <!-- Stats Grid 1: Primary with Sparklines -->
                        <div style="display: grid; grid-template-columns: repeat(6, 1fr); gap: 0.5rem; margin-bottom: 0.75rem;">
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(249,115,22,0.15), rgba(249,115,22,0.05)); border: 1px solid rgba(249,115,22,0.3); border-radius: 0.5rem; padding: 0.5rem; text-align: center;">
                                <div id="ultPoolSize" style="font-size: 1.3rem; font-weight: 700; color: #f97316;">0</div>
                                <div style="font-size: 0.65rem; color: var(--text-secondary);">Pool</div>
                                <canvas id="ultSparkPool" width="80" height="24" style="width:80px;height:24px;margin-top:2px;"></canvas>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(139,92,246,0.15), rgba(139,92,246,0.05)); border: 1px solid rgba(139,92,246,0.3); border-radius: 0.5rem; padding: 0.5rem; text-align: center;">
                                <div id="ultWaves" style="font-size: 1.3rem; font-weight: 700; color: #8b5cf6;">0</div>
                                <div style="font-size: 0.65rem; color: var(--text-secondary);">Waves</div>
                                <canvas id="ultSparkWaves" width="80" height="24" style="width:80px;height:24px;margin-top:2px;"></canvas>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(34,197,94,0.15), rgba(34,197,94,0.05)); border: 1px solid rgba(34,197,94,0.3); border-radius: 0.5rem; padding: 0.5rem; text-align: center;">
                                <div id="ultImported" style="font-size: 1.3rem; font-weight: 700; color: #22c55e;">0</div>
                                <div style="font-size: 0.65rem; color: var(--text-secondary);">Imported</div>
                                <canvas id="ultSparkImported" width="80" height="24" style="width:80px;height:24px;margin-top:2px;"></canvas>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(236,72,153,0.15), rgba(236,72,153,0.05)); border: 1px solid rgba(236,72,153,0.3); border-radius: 0.5rem; padding: 0.5rem; text-align: center;">
                                <div id="ultRate" style="font-size: 1.3rem; font-weight: 700; color: #ec4899;">0/s</div>
                                <div style="font-size: 0.65rem; color: var(--text-secondary);">Rate</div>
                                <canvas id="ultSparkRate" width="80" height="24" style="width:80px;height:24px;margin-top:2px;"></canvas>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(6,182,212,0.15), rgba(6,182,212,0.05)); border: 1px solid rgba(6,182,212,0.3); border-radius: 0.5rem; padding: 0.5rem; text-align: center;">
                                <div id="ultProcesses" style="font-size: 1.3rem; font-weight: 700; color: #06b6d4;">0</div>
                                <div style="font-size: 0.65rem; color: var(--text-secondary);">Workers</div>
                            </div>
                            <div class="stat-card" style="background: linear-gradient(135deg, rgba(99,102,241,0.15), rgba(99,102,241,0.05)); border: 1px solid rgba(99,102,241,0.3); border-radius: 0.5rem; padding: 0.5rem; text-align: center;">
                                <div id="ultDiscovered" style="font-size: 1.3rem; font-weight: 700; color: #6366f1;">0</div>
                                <div style="font-size: 0.65rem; color: var(--text-secondary);">Discovered</div>
                            </div>
                        </div>

                        <!-- Stats Grid 2: Secondary -->
                        <div style="display: grid; grid-template-columns: repeat(5, 1fr); gap: 0.5rem; margin-bottom: 0.75rem;">
                            <div style="background:var(--bg-tertiary);border-radius:0.4rem;padding:0.4rem;text-align:center;">
                                <div id="ultSuccessRate" style="font-size:1rem;font-weight:600;color:#22c55e;">0%</div>
                                <div style="font-size:0.6rem;color:var(--text-secondary);">Success</div>
                            </div>
                            <div style="background:var(--bg-tertiary);border-radius:0.4rem;padding:0.4rem;text-align:center;">
                                <div id="ultDuplicates" style="font-size:1rem;font-weight:600;color:#f59e0b;">0</div>
                                <div style="font-size:0.6rem;color:var(--text-secondary);">Duplicates</div>
                            </div>
                            <div style="background:var(--bg-tertiary);border-radius:0.4rem;padding:0.4rem;text-align:center;">
                                <div id="ultErrors" style="font-size:1rem;font-weight:600;color:#ef4444;">0</div>
                                <div style="font-size:0.6rem;color:var(--text-secondary);">Errors</div>
                            </div>
                            <div style="background:var(--bg-tertiary);border-radius:0.4rem;padding:0.4rem;text-align:center;">
                                <div id="ultNodes" style="font-size:1rem;font-weight:600;color:#06b6d4;">0</div>
                                <div style="font-size:0.6rem;color:var(--text-secondary);">Domains</div>
                            </div>
                            <div style="background:var(--bg-tertiary);border-radius:0.4rem;padding:0.4rem;text-align:center;">
                                <div id="ultEdges" style="font-size:1rem;font-weight:600;color:#8b5cf6;">0</div>
                                <div style="font-size:0.6rem;color:var(--text-secondary);">Edges</div>
                            </div>
                        </div>

                        <!-- Status Bar -->
                        <div style="display:flex;justify-content:space-between;align-items:center;background:var(--bg-tertiary);border-radius:0.4rem;padding:0.5rem 1rem;margin-bottom:0.75rem;font-size:0.75rem;">
                            <span>ETA: <span id="ultETA" style="color:#f97316;">--</span></span>
                            <span>Elapsed: <span id="ultElapsed" style="color:#06b6d4;">00:00:00</span></span>
                            <span>Pool: <span id="ultPoolPending" style="color:#22c55e;">0</span>/<span id="ultPoolClaimed" style="color:#f59e0b;">0</span></span>
                            <span>Seen: <span id="ultSeenTotal" style="color:#8b5cf6;">0</span></span>
                            <span>Health: <span id="ultPoolHealth">🟢</span></span>
                        </div>

                        <!-- Network Graph Canvas -->
                        <div style="margin-bottom:0.75rem;">
                            <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:0.5rem;">
                                <h3 style="font-size:0.85rem;font-weight:600;">🌐 Network Graph</h3>
                                <div style="display:flex;gap:0.3rem;">
                                    <button type="button" onclick="networkGraph.resetView()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.2rem 0.5rem;border-radius:0.25rem;cursor:pointer;font-size:0.7rem;">↺ Reset</button>
                                    <button type="button" onclick="networkGraph.toggleLabels()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.2rem 0.5rem;border-radius:0.25rem;cursor:pointer;font-size:0.7rem;">🏷 Labels</button>
                                    <button type="button" onclick="networkGraph.toggleParticles()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.2rem 0.5rem;border-radius:0.25rem;cursor:pointer;font-size:0.7rem;">✨ Particles</button>
                                    <button type="button" onclick="networkGraph.exportPNG()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.2rem 0.5rem;border-radius:0.25rem;cursor:pointer;font-size:0.7rem;">📸</button>
                                </div>
                            </div>
                            <div style="position:relative;background:#0a0510;border-radius:0.5rem;overflow:hidden;">
                                <canvas id="ultNetworkCanvas" width="800" height="400" style="width:100%;height:400px;display:block;"></canvas>
                                <canvas id="ultMinimap" width="160" height="100" style="position:absolute;bottom:8px;right:8px;width:160px;height:100px;border:1px solid rgba(255,255,255,0.2);border-radius:0.25rem;background:rgba(0,0,0,0.5);"></canvas>
                            </div>
                        </div>

                        <!-- Process List + Log side by side -->
                        <div style="display:grid;grid-template-columns:1fr 1fr;gap:0.75rem;">
                            <!-- Process List -->
                            <div>
                                <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:0.4rem;">
                                    <h3 style="font-size:0.85rem;font-weight:600;">📋 Workers</h3>
                                    <span id="ultProcessCount" style="font-size:0.7rem;color:var(--text-secondary);">0 active</span>
                                </div>
                                <div id="ultProcessList" style="max-height:300px;overflow-y:auto;display:flex;flex-direction:column;gap:0.4rem;background:var(--bg-primary);border:1px solid var(--border);border-radius:0.4rem;padding:0.5rem;">
                                    <div style="padding:1rem;text-align:center;color:var(--text-secondary);font-size:0.8rem;">Click "Launch ULTIMATE Workers" to start.</div>
                                </div>
                            </div>
                            <!-- Activity Log -->
                            <div>
                                <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:0.4rem;">
                                    <h3 style="font-size:0.85rem;font-weight:600;">📜 Activity Log</h3>
                                    <div style="display:flex;gap:0.3rem;">
                                        <button type="button" onclick="UltimateManager.copyResults('all')" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.15rem 0.4rem;border-radius:0.2rem;cursor:pointer;font-size:0.65rem;" title="Copy imported URLs">📋</button>
                                        <button type="button" onclick="UltimateManager.toggleGroupView()" id="ultGroupToggle" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.15rem 0.4rem;border-radius:0.2rem;cursor:pointer;font-size:0.65rem;" title="Group by domain">📁</button>
                                        <button type="button" onclick="UltimateManager.exportResults('csv')" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.15rem 0.4rem;border-radius:0.2rem;cursor:pointer;font-size:0.65rem;">CSV</button>
                                        <button type="button" onclick="UltimateManager.exportResults('json')" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.15rem 0.4rem;border-radius:0.2rem;cursor:pointer;font-size:0.65rem;">JSON</button>
                                        <button type="button" onclick="UltimateManager.exportDomainResults()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.15rem 0.4rem;border-radius:0.2rem;cursor:pointer;font-size:0.65rem;" title="Export grouped by domain">CSV/D</button>
                                        <button type="button" onclick="UltimateManager.clearLog()" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:0.15rem 0.4rem;border-radius:0.2rem;cursor:pointer;font-size:0.65rem;">Clear</button>
                                    </div>
                                </div>
                                <input type="text" id="ultLogSearch" class="form-input" placeholder="Filter: URL, domain, status..." oninput="UltimateManager.filterLog(this.value)" style="margin-bottom:0.4rem;font-size:0.7rem;padding:0.3rem 0.5rem;">
                                <div id="ultActivityLog" style="max-height:300px;overflow-y:auto;background:var(--bg-primary);border:1px solid var(--border);border-radius:0.4rem;font-family:'JetBrains Mono',monospace;font-size:0.7rem;line-height:1.5;padding:0.5rem;">
                                    <div style="color:var(--text-secondary);text-align:center;">Waiting for workers...</div>
                                </div>
                            </div>
                        </div>

                        <!-- Domain Leaderboard -->
                        <div style="margin-top:0.75rem;">
                            <h3 style="font-size:0.85rem;font-weight:600;margin-bottom:0.4rem;">🏆 Domain Leaderboard</h3>
                            <div id="ultLeaderboard" style="max-height:200px;overflow-y:auto;background:var(--bg-primary);border:1px solid var(--border);border-radius:0.4rem;padding:0.5rem;font-size:0.72rem;">
                                <div style="color:var(--text-secondary);text-align:center;">No data yet...</div>
                            </div>
                        </div>
                        <!-- Charts Section -->
                        <div style="margin-top:0.75rem;">
                            <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:0.4rem;">
                                <h3 style="font-size:0.85rem;font-weight:600;">📊 Charts</h3>
                                <div style="display:flex;gap:0.3rem;">
                                    <button type="button" class="chart-tab active" onclick="switchChart('progress',this)">Progress</button>
                                    <button type="button" class="chart-tab" onclick="switchChart('domains',this)">Domains</button>
                                    <button type="button" class="chart-tab" onclick="switchChart('types',this)">Types</button>
                                    <button type="button" class="chart-tab" onclick="switchChart('rate',this)">Rate</button>
                                </div>
                            </div>
                            <div style="background:var(--bg-primary);border:1px solid var(--border);border-radius:0.4rem;padding:0.5rem;height:200px;position:relative;">
                                <canvas id="ultChartProgress" style="width:100%;height:100%;"></canvas>
                                <canvas id="ultChartDomains" style="display:none;width:100%;height:100%;"></canvas>
                                <canvas id="ultChartTypes" style="display:none;width:100%;height:100%;"></canvas>
                                <canvas id="ultChartRate" style="display:none;width:100%;height:100%;"></canvas>
                            </div>
                        </div>
                    </div>
                </div>

                <!-- B22: Enhanced Sidebar with Tabs -->
                <div>
                    <div class="panel" style="margin-bottom: 1rem;">
                        <!-- B22: Sidebar tabs -->
                        <div class="sidebar-tabs">
                            <button type="button" class="sidebar-tab active" onclick="switchSidebarTab('stats', this)">📊 Stats</button>
                            <button type="button" class="sidebar-tab" onclick="switchSidebarTab('config', this)">⚙️ Info</button>
                            <button type="button" class="sidebar-tab" onclick="switchSidebarTab('timeline', this)">📅 Timeline</button>
                        </div>

                        <!-- Tab: Stats -->
                        <div class="sidebar-tab-content active" id="sidebarStats">
                            <div class="stats-grid" style="margin-bottom: 0.75rem;">
                                <div class="stat-card" style="border-color: rgba(99,102,241,0.2);">
                                    <span class="stat-value" id="totalPinfeeds" style="color: var(--accent);">-</span>
                                    <span class="stat-label">Total Pinfeeds</span>
                                </div>
                                <div class="stat-card" style="border-color: rgba(34,197,94,0.2);">
                                    <span class="stat-value" id="todayImports" style="color: var(--success);">-</span>
                                    <span class="stat-label">Today's Imports</span>
                                </div>
                            </div>
                            <!-- B25: Pool Health Gauge SVG -->
                            <div style="text-align:center; padding: 0.5rem 0;">
                                <svg width="120" height="70" id="poolGaugeSvg">
                                    <path d="M 15 60 A 45 45 0 0 1 105 60" fill="none" stroke="var(--bg-tertiary)" stroke-width="8" stroke-linecap="round"/>
                                    <path d="M 15 60 A 45 45 0 0 1 105 60" fill="none" stroke="var(--emerald)" stroke-width="8" stroke-linecap="round" id="gaugeArc" stroke-dasharray="141.4" stroke-dashoffset="0"/>
                                    <text x="60" y="55" text-anchor="middle" fill="var(--text-primary)" font-size="14" font-weight="700" id="gaugeText">100%</text>
                                    <text x="60" y="68" text-anchor="middle" fill="var(--text-secondary)" font-size="8">POOL HEALTH</text>
                                </svg>
                            </div>
                            <!-- Elapsed time -->
                            <div style="text-align:center;font-size:0.75rem;color:var(--text-secondary);margin-top:0.5rem;">
                                Session: <span id="sidebarElapsed" style="color:var(--accent);font-weight:600;">00:00:00</span>
                            </div>
                        </div>

                        <!-- Tab: Config/Info -->
                        <div class="sidebar-tab-content" id="sidebarConfig">
                            <div id="modeInfo" style="font-size: 0.85rem; line-height: 1.7;">
                                <p><strong>Search Mode:</strong> Search across multiple engines (Bing, DuckDuckGo, Yandex, Baidu, Yahoo) and collect URLs from results.</p>
                            </div>
                            <div style="margin-top:1rem;padding-top:0.75rem;border-top:1px solid var(--glass-border);">
                                <h4 style="font-size:0.75rem;color:var(--text-secondary);margin-bottom:0.5rem;">⌨️ Shortcuts</h4>
                                <div style="font-size:0.7rem;color:var(--text-secondary);line-height:2;">
                                    <div><kbd style="background:var(--bg-tertiary);padding:1px 5px;border-radius:3px;border:1px solid var(--border);">Ctrl+Enter</kbd> Start</div>
                                    <div><kbd style="background:var(--bg-tertiary);padding:1px 5px;border-radius:3px;border:1px solid var(--border);">Esc</kbd> Stop</div>
                                    <div><kbd style="background:var(--bg-tertiary);padding:1px 5px;border-radius:3px;border:1px solid var(--border);">Space</kbd> Pause</div>
                                    <div><kbd style="background:var(--bg-tertiary);padding:1px 5px;border-radius:3px;border:1px solid var(--border);">?</kbd> All Shortcuts</div>
                                </div>
                            </div>
                        </div>

                        <!-- Tab: Timeline -->
                        <div class="sidebar-tab-content" id="sidebarTimeline">
                            <div class="timeline" id="timelineContainer">
                                <div class="timeline-item">
                                    <div class="timeline-dot" style="background:var(--text-secondary);"></div>
                                    <div>
                                        <div class="timeline-text">Session started</div>
                                        <div class="timeline-time" id="sessionStartTime">--</div>
                                    </div>
                                </div>
                            </div>
                        </div>
                    </div>
                </div>
            </div>
        </form>

        <script>
        // S2: CSRF token for form protection
        const csrfToken = '<?php echo $csrfToken; ?>';
        const modeInfos = {
            search: '<p><strong>Search Mode:</strong> Search across multiple engines (Bing, DuckDuckGo, Yandex, Baidu, Yahoo) and collect URLs from results.</p>',
            deep: '<p><strong>Deep Crawl Mode:</strong> Start from seed URLs and recursively follow links up to the specified depth. Great for scraping entire websites.</p>',
            pagination: '<p><strong>Pagination Mode:</strong> Follow a URL pattern with page numbers (e.g., /page/1, /page/2). Perfect for paginated listings.</p>',
            hybrid: '<p><strong>Hybrid Mode:</strong> Combine search results with deep crawling. Search for terms, then deeply crawl the found pages.</p>',
            infinite: '<p><strong>♾️ INFINITE Mode:</strong> <span style="color:#22c55e;font-weight:bold;">CRAWL + IMPORT SIMULTANEOUSLY!</span> Continuously discovers links from multiple search engines, extracts metadata, and imports directly to pinfeeds. Processes millions of links without stopping. Real-time stats and controls.</p>',
            viral: '<p><strong>🦠 VIRAL Mode:</strong> <span style="color:#ef4444;font-weight:bold;">PARALLEL SPREADING MESH!</span> Enter keywords (searches 10+ engines worldwide) OR paste URLs directly (Google/Bing search pages, any website). Spreads like a virus - each page spawns parallel crawls, follows ALL links infinitely layer by layer. Pool grows exponentially like a network mesh.</p>',
            multi: '<p><strong>🚀 MULTI Mode:</strong> <span style="color:#06b6d4;font-weight:bold;">1000+ SIMULTANEOUS PROCESSES!</span> Launch multiple independent crawl workers that run in parallel. Each process claims URLs from a shared pool. No conflicts, no duplicates. Database-coordinated with heartbeat monitoring, auto-cleanup of dead processes, and real-time aggregated stats.</p>',
            ultimate: '<p><strong>⚡ ULTIMATE Mode:</strong> <span style="color:#f43f5e;font-weight:bold;">ALL MODES COMBINED!</span> Multi-process architecture + ALL 11 search engines + Deep Crawl + Pagination + Relevance Scoring + Canonical Dedup + Noindex Detection + Domain Quality + Exponential Backoff + Network Graph + Sparklines + Export. The most powerful crawl mode.</p>'
        };

        // ========================
        // C29: AnimatedCounter Class
        // ========================
        class AnimatedCounter {
            constructor(element, duration = 600) {
                this.el = element;
                this.duration = duration;
                this.current = 0;
                this.animFrame = null;
            }
            update(newVal) {
                if (newVal === this.current) return;
                const start = this.current;
                const diff = newVal - start;
                const startTime = performance.now();
                if (this.animFrame) cancelAnimationFrame(this.animFrame);
                const animate = (now) => {
                    const elapsed = now - startTime;
                    const progress = Math.min(elapsed / this.duration, 1);
                    const eased = 1 - Math.pow(1 - progress, 3); // ease-out cubic
                    const val = Math.round(start + diff * eased);
                    this.el.textContent = val.toLocaleString();
                    if (progress < 1) {
                        this.animFrame = requestAnimationFrame(animate);
                    } else {
                        this.current = newVal;
                        this.el.classList.add('stat-flash');
                        setTimeout(() => this.el.classList.remove('stat-flash'), 400);
                    }
                };
                this.animFrame = requestAnimationFrame(animate);
            }
        }

        // ========================
        // C30: Sparkline Class
        // ========================
        class Sparkline {
            constructor(canvasId, color = '#6366f1', maxPoints = 30) {
                this.canvas = document.getElementById(canvasId);
                if (!this.canvas) return;
                this.ctx = this.canvas.getContext('2d');
                this.color = color;
                this.maxPoints = maxPoints;
                this.data = [];
            }
            push(val) {
                if (!this.ctx) return;
                this.data.push(val);
                if (this.data.length > this.maxPoints) this.data.shift();
                this.draw();
            }
            draw() {
                const w = this.canvas.width, h = this.canvas.height;
                this.ctx.clearRect(0, 0, w, h);
                if (this.data.length < 2) return;
                const max = Math.max(...this.data, 1);
                const min = Math.min(...this.data, 0);
                const range = max - min || 1;
                const stepX = w / (this.maxPoints - 1);
                // Fill area
                this.ctx.beginPath();
                this.ctx.moveTo(0, h);
                this.data.forEach((v, i) => {
                    const x = i * stepX;
                    const y = h - ((v - min) / range) * (h - 4) - 2;
                    this.ctx.lineTo(x, y);
                });
                this.ctx.lineTo((this.data.length - 1) * stepX, h);
                this.ctx.closePath();
                const grad = this.ctx.createLinearGradient(0, 0, 0, h);
                grad.addColorStop(0, this.color + '40');
                grad.addColorStop(1, 'transparent');
                this.ctx.fillStyle = grad;
                this.ctx.fill();
                // Line
                this.ctx.beginPath();
                this.data.forEach((v, i) => {
                    const x = i * stepX;
                    const y = h - ((v - min) / range) * (h - 4) - 2;
                    i === 0 ? this.ctx.moveTo(x, y) : this.ctx.lineTo(x, y);
                });
                this.ctx.strokeStyle = this.color;
                this.ctx.lineWidth = 1.5;
                this.ctx.stroke();
                // Dot on latest
                const lastX = (this.data.length - 1) * stepX;
                const lastY = h - ((this.data[this.data.length - 1] - min) / range) * (h - 4) - 2;
                this.ctx.beginPath();
                this.ctx.arc(lastX, lastY, 2.5, 0, Math.PI * 2);
                this.ctx.fillStyle = this.color;
                this.ctx.fill();
            }
        }

        // ========================
        // C31: ParticleSystem Class
        // ========================
        class ParticleSystem {
            constructor(canvasId, count = 50) {
                this.canvas = document.getElementById(canvasId);
                if (!this.canvas) return;
                this.ctx = this.canvas.getContext('2d');
                this.particles = [];
                this.colors = ['#6366f1', '#8b5cf6', '#06b6d4', '#22c55e', '#a855f7'];
                this.resize();
                window.addEventListener('resize', () => this.resize());
                for (let i = 0; i < count; i++) this.particles.push(this.createParticle());
                this.animate();
            }
            resize() {
                this.canvas.width = window.innerWidth;
                this.canvas.height = window.innerHeight;
            }
            createParticle() {
                return {
                    x: Math.random() * this.canvas.width,
                    y: Math.random() * this.canvas.height,
                    vx: (Math.random() - 0.5) * 0.4,
                    vy: (Math.random() - 0.5) * 0.4,
                    size: Math.random() * 2 + 1,
                    color: this.colors[Math.floor(Math.random() * this.colors.length)],
                    alpha: Math.random() * 0.3 + 0.1
                };
            }
            animate() {
                this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
                this.particles.forEach(p => {
                    p.x += p.vx;
                    p.y += p.vy;
                    if (p.x < 0 || p.x > this.canvas.width) p.vx *= -1;
                    if (p.y < 0 || p.y > this.canvas.height) p.vy *= -1;
                    this.ctx.beginPath();
                    this.ctx.arc(p.x, p.y, p.size, 0, Math.PI * 2);
                    this.ctx.fillStyle = p.color;
                    this.ctx.globalAlpha = p.alpha;
                    this.ctx.fill();
                });
                // Connection lines
                this.ctx.globalAlpha = 0.06;
                this.ctx.strokeStyle = '#6366f1';
                this.ctx.lineWidth = 0.5;
                for (let i = 0; i < this.particles.length; i++) {
                    for (let j = i + 1; j < this.particles.length; j++) {
                        const dx = this.particles[i].x - this.particles[j].x;
                        const dy = this.particles[i].y - this.particles[j].y;
                        const dist = Math.sqrt(dx * dx + dy * dy);
                        if (dist < 120) {
                            this.ctx.beginPath();
                            this.ctx.moveTo(this.particles[i].x, this.particles[i].y);
                            this.ctx.lineTo(this.particles[j].x, this.particles[j].y);
                            this.ctx.stroke();
                        }
                    }
                }
                this.ctx.globalAlpha = 1;
                requestAnimationFrame(() => this.animate());
            }
        }

        // ========================
        // C32: LiveRateChart Class
        // ========================
        class LiveRateChart {
            constructor(canvasId, maxPoints = 60) {
                this.canvas = document.getElementById(canvasId);
                if (!this.canvas) return;
                this.ctx = this.canvas.getContext('2d');
                this.maxPoints = maxPoints;
                this.data = new Array(maxPoints).fill(0);
                this.maxVal = 10;
            }
            push(val) {
                if (!this.ctx) return;
                this.data.push(val);
                if (this.data.length > this.maxPoints) this.data.shift();
                this.maxVal = Math.max(Math.max(...this.data) * 1.2, 10);
                this.draw();
            }
            draw() {
                const w = this.canvas.width = this.canvas.offsetWidth;
                const h = this.canvas.height;
                this.ctx.clearRect(0, 0, w, h);
                // Grid lines
                this.ctx.strokeStyle = 'rgba(255,255,255,0.03)';
                this.ctx.lineWidth = 0.5;
                for (let i = 1; i <= 3; i++) {
                    const y = h * (i / 4);
                    this.ctx.beginPath();
                    this.ctx.moveTo(0, y);
                    this.ctx.lineTo(w, y);
                    this.ctx.stroke();
                }
                // Data line + fill
                const stepX = w / (this.maxPoints - 1);
                this.ctx.beginPath();
                this.ctx.moveTo(0, h);
                this.data.forEach((v, i) => {
                    const x = i * stepX;
                    const y = h - (v / this.maxVal) * (h - 8) - 4;
                    this.ctx.lineTo(x, y);
                });
                this.ctx.lineTo(w, h);
                this.ctx.closePath();
                const grad = this.ctx.createLinearGradient(0, 0, 0, h);
                grad.addColorStop(0, 'rgba(99,102,241,0.3)');
                grad.addColorStop(1, 'transparent');
                this.ctx.fillStyle = grad;
                this.ctx.fill();
                // Line
                this.ctx.beginPath();
                this.data.forEach((v, i) => {
                    const x = i * stepX;
                    const y = h - (v / this.maxVal) * (h - 8) - 4;
                    i === 0 ? this.ctx.moveTo(x, y) : this.ctx.lineTo(x, y);
                });
                this.ctx.strokeStyle = '#6366f1';
                this.ctx.lineWidth = 1.5;
                this.ctx.stroke();
            }
        }

        // ========================
        // C33: Theater Mode
        // ========================
        let theaterMode = false;
        function toggleTheaterMode() {
            theaterMode = !theaterMode;
            document.body.classList.toggle('theater-mode', theaterMode);
            const btn = document.getElementById('theaterBtn');
            if (btn) btn.style.color = theaterMode ? 'var(--accent)' : 'var(--text-secondary)';
            addTimelineEvent(theaterMode ? 'Theater mode ON' : 'Theater mode OFF', 'var(--purple)');
        }

        // ========================
        // C34: Real-time Clock
        // ========================
        let sessionStartTimestamp = Date.now();
        function updateClock() {
            const now = new Date();
            const hh = String(now.getHours()).padStart(2, '0');
            const mm = String(now.getMinutes()).padStart(2, '0');
            const ss = String(now.getSeconds()).padStart(2, '0');
            const el = document.getElementById('headerClock');
            if (el) el.textContent = hh + ':' + mm + ':' + ss;
            // Sidebar elapsed
            const elapsed = Math.floor((Date.now() - sessionStartTimestamp) / 1000);
            const eh = String(Math.floor(elapsed / 3600)).padStart(2, '0');
            const em = String(Math.floor((elapsed % 3600) / 60)).padStart(2, '0');
            const es = String(elapsed % 60).padStart(2, '0');
            const sel = document.getElementById('sidebarElapsed');
            if (sel) sel.textContent = eh + ':' + em + ':' + es;
        }
        setInterval(updateClock, 1000);
        updateClock();
        const startEl = document.getElementById('sessionStartTime');
        if (startEl) startEl.textContent = new Date().toLocaleTimeString();

        // ========================
        // C35: Sound Effects
        // ========================
        let soundEnabled = false;
        let audioCtx = null;
        function toggleSound() {
            soundEnabled = !soundEnabled;
            const btn = document.getElementById('fabSoundBtn');
            if (btn) btn.textContent = soundEnabled ? '🔊' : '🔇';
            if (soundEnabled && !audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
            showToast(soundEnabled ? 'Sound ON' : 'Sound OFF', 'info');
        }
        function playBeep(freq = 800, duration = 100) {
            if (!soundEnabled || !audioCtx) return;
            const osc = audioCtx.createOscillator();
            const gain = audioCtx.createGain();
            osc.connect(gain);
            gain.connect(audioCtx.destination);
            osc.frequency.value = freq;
            gain.gain.setValueAtTime(0.1, audioCtx.currentTime);
            gain.gain.exponentialRampToValueAtTime(0.001, audioCtx.currentTime + duration / 1000);
            osc.start();
            osc.stop(audioCtx.currentTime + duration / 1000);
        }

        // ========================
        // C36: Keyboard Shortcuts Overlay
        // ========================
        function openShortcuts() { document.getElementById('shortcutsOverlay').classList.add('show'); }
        function closeShortcuts() { document.getElementById('shortcutsOverlay').classList.remove('show'); }

        // ========================
        // C40: Sidebar Tab Switching
        // ========================
        function switchSidebarTab(tab, btn) {
            document.querySelectorAll('.sidebar-tab').forEach(t => t.classList.remove('active'));
            document.querySelectorAll('.sidebar-tab-content').forEach(c => c.classList.remove('active'));
            btn.classList.add('active');
            const content = document.getElementById('sidebar' + tab.charAt(0).toUpperCase() + tab.slice(1));
            if (content) content.classList.add('active');
        }

        // ========================
        // C41: Timeline Event Logger
        // ========================
        let timelineEvents = [];
        function addTimelineEvent(text, color = 'var(--accent)') {
            const container = document.getElementById('timelineContainer');
            if (!container) return;
            const time = new Date().toLocaleTimeString();
            timelineEvents.push({ text, time, color });
            if (timelineEvents.length > 50) timelineEvents.shift();
            const item = document.createElement('div');
            item.className = 'timeline-item';
            item.innerHTML = '<div class="timeline-dot" style="background:' + color + ';"></div><div class="timeline-line"></div><div><div class="timeline-text">' + text + '</div><div class="timeline-time">' + time + '</div></div>';
            container.appendChild(item);
            container.scrollTop = container.scrollHeight;
        }

        // ========================
        // C43: FAB Menu Toggle
        // ========================
        let fabOpen = false;
        function toggleFAB() {
            fabOpen = !fabOpen;
            const main = document.getElementById('fabMainBtn');
            main.classList.toggle('open', fabOpen);
            document.querySelectorAll('.fab-action').forEach((btn, i) => {
                setTimeout(() => btn.classList.toggle('show', fabOpen), i * 50);
            });
        }
        function scrollToTop() { window.scrollTo({ top: 0, behavior: 'smooth' }); }

        // ========================
        // C44: Pool Gauge Update
        // ========================
        function updatePoolGauge(healthPercent) {
            const arc = document.getElementById('gaugeArc');
            const text = document.getElementById('gaugeText');
            if (!arc || !text) return;
            const maxDash = 141.4;
            const offset = maxDash * (1 - healthPercent / 100);
            arc.setAttribute('stroke-dashoffset', offset);
            text.textContent = Math.round(healthPercent) + '%';
            // Color based on health
            let color = 'var(--emerald)';
            if (healthPercent < 40) color = 'var(--error)';
            else if (healthPercent < 70) color = 'var(--warning)';
            arc.setAttribute('stroke', color);
        }

        // ========================
        // C48: Progress Ring Update
        // ========================
        function updateProgressRing(percent) {
            const fill = document.getElementById('progressRingFill');
            const text = document.getElementById('progressRingText');
            if (!fill || !text) return;
            const circumference = 188.5;
            const offset = circumference * (1 - percent / 100);
            fill.setAttribute('stroke-dashoffset', offset);
            text.textContent = Math.round(percent) + '%';
        }

        // ========================
        // C37: Domain Leaderboard Updater
        // ========================
        let domainStats = {};
        let leaderboardWaveCount = 0;
        function updateLeaderboard() {
            const section = document.getElementById('leaderboardSection');
            const body = document.getElementById('leaderboardBody');
            if (!section || !body) return;
            const sorted = Object.entries(domainStats).sort((a, b) => b[1].imports - a[1].imports).slice(0, 15);
            if (sorted.length === 0) { section.style.display = 'none'; return; }
            section.style.display = 'block';
            body.innerHTML = sorted.map((entry, i) => {
                const [domain, stats] = entry;
                const colors = ['#ffd700', '#c0c0c0', '#cd7f32'];
                const rankColor = i < 3 ? colors[i] : 'var(--text-secondary)';
                return '<tr><td class="rank" style="color:' + rankColor + ';">' + (i + 1) + '</td><td style="color:var(--text-primary);max-width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="' + domain + '">' + domain + '</td><td style="color:var(--success);">' + (stats.imports || 0) + '</td><td style="color:var(--cyan);">' + (stats.crawled || 0) + '</td></tr>';
            }).join('');
        }

        // ========================
        // Initialize Systems
        // ========================
        const particleSystem = new ParticleSystem('particleCanvas', 50);
        const rateChart = new LiveRateChart('rateChartCanvas', 60);
        const sparkPool = new Sparkline('sparkPool', '#f97316');
        const sparkWaves = new Sparkline('sparkWaves', '#8b5cf6');
        const sparkImported = new Sparkline('sparkImported', '#22c55e');
        const sparkRate = new Sparkline('sparkRate', '#ec4899');

        // Animated counters for stat values
        const counters = {};
        ['viralPoolSize', 'viralWaves', 'viralImported', 'viralDiscovered', 'viralProcessed', 'viralDuplicates', 'viralSkipped', 'viralErrors'].forEach(id => {
            const el = document.getElementById(id);
            if (el) counters[id] = new AnimatedCounter(el);
        });

        // Rate chart auto-push (1/sec)
        let currentRate = 0;
        setInterval(() => { rateChart.push(currentRate); }, 1000);

        // Update rate chart label
        setInterval(() => {
            const label = document.getElementById('rateChartLabel');
            if (label) label.textContent = currentRate.toFixed(1) + '/s';
        }, 1000);

        let infiniteController = null;
        let infiniteRunning = false;

        function selectMode(btn) {
            document.querySelectorAll('.mode-btn').forEach(b => b.classList.remove('active'));
            btn.classList.add('active');

            const mode = btn.dataset.mode;
            document.getElementById('modeInput').value = mode;
            document.getElementById('modeInfo').innerHTML = modeInfos[mode];

            // Toggle visibility based on mode
            const isInfinite = mode === 'infinite';
            const isViral = mode === 'viral';
            const isMulti = mode === 'multi';
            const isUltimate = mode === 'ultimate';
            document.getElementById('searchTermsGroup').style.display = (mode === 'search' || mode === 'hybrid' || mode === 'infinite') ? 'block' : 'none';
            document.getElementById('seedUrlsGroup').style.display = (mode === 'deep' || mode === 'hybrid') ? 'block' : 'none';
            document.getElementById('paginationGroup').style.display = (mode === 'pagination') ? 'block' : 'none';
            document.getElementById('enginesGroup').style.display = (mode === 'search' || mode === 'hybrid') ? 'block' : 'none';
            document.getElementById('viralConfigPanel').style.display = isViral ? 'block' : 'none';
            document.getElementById('multiConfigPanel').style.display = isMulti ? 'block' : 'none';
            document.getElementById('ultimateConfigPanel').style.display = isUltimate ? 'block' : 'none';

            // Hide advanced options for viral/multi/ultimate (have their own config)
            document.querySelector('.advanced-toggle').style.display = (isViral || isMulti || isUltimate) ? 'none' : 'block';
            document.getElementById('advancedOptions').style.display = (isViral || isMulti || isUltimate) ? 'none' : '';

            // Toggle buttons
            document.getElementById('submitBtn').style.display = (isInfinite || isViral || isMulti || isUltimate) ? 'none' : 'block';
            document.getElementById('infiniteControls').style.display = isInfinite ? 'block' : 'none';
            document.getElementById('viralControls').style.display = isViral ? 'block' : 'none';
            document.getElementById('multiControls').style.display = isMulti ? 'block' : 'none';
            document.getElementById('ultimateControls').style.display = isUltimate ? 'block' : 'none';

            // Show/hide multi stats panel
            if (isMulti) {
                document.getElementById('multiStatsPanel').style.display = 'block';
                MPManager.init();
            } else {
                document.getElementById('multiStatsPanel').style.display = 'none';
                MPManager.stop();
            }

            // Show/hide ultimate stats panel
            if (isUltimate) {
                document.getElementById('ultimateStatsPanel').style.display = 'block';
                UltimateManager.init();
            } else {
                document.getElementById('ultimateStatsPanel').style.display = 'none';
                UltimateManager.stop();
            }

            // Update search terms label/placeholder for infinite mode
            const searchLabel = document.querySelector('#searchTermsGroup .form-label');
            const searchTextarea = document.querySelector('textarea[name="search_terms"]');
            if (isInfinite) {
                searchLabel.textContent = 'Keywords or URLs (one per line)';
                searchTextarea.placeholder = 'batman\nhttps://www.google.com/search?q=batman\nhttps://learnbirdwatching.com/\npython tutorials';
            } else if (!isViral && !isMulti) {
                searchLabel.textContent = 'Search Terms (one per line)';
                searchTextarea.placeholder = 'web scraping tutorials\npython crawler guide\ndata extraction tools';
            }
        }

        async function startInfiniteMode() {
            const keywords = document.querySelector('textarea[name="search_terms"]').value.trim();
            if (!keywords) {
                alert('Please enter at least one keyword');
                return;
            }

            // Show stats panel
            document.getElementById('infiniteStatsPanel').style.display = 'block';
            document.getElementById('startInfiniteBtn').disabled = true;
            document.getElementById('startInfiniteBtn').textContent = '⏳ Starting...';
            infiniteRunning = true;

            // Build URL with parameters
            const params = new URLSearchParams({
                action: 'infinite',
                stream: '1',
                keywords: keywords,
                max_imports: document.querySelector('input[name="max_pages"]')?.value || 10000000,
                batch_size: 50,
                deep_crawl: document.querySelector('input[name="respect_robots"]')?.checked ? '1' : ''
            });

            try {
                const response = await fetch('?' + params.toString());
                const reader = response.body.getReader();
                const decoder = new TextDecoder();

                while (infiniteRunning) {
                    const {value, done} = await reader.read();
                    if (done) break;

                    const chunk = decoder.decode(value);
                    const lines = chunk.split('\n');

                    for (const line of lines) {
                        if (line.startsWith('data: ')) {
                            try {
                                const stats = JSON.parse(line.substring(6));
                                updateInfiniteStats(stats);
                            } catch (e) {}
                        }
                    }
                }
            } catch (error) {
                console.error('Infinite mode error:', error);
                document.getElementById('infStatus').textContent = '● Error';
                document.getElementById('infStatus').style.color = '#ef4444';
            }

            document.getElementById('startInfiniteBtn').disabled = false;
            document.getElementById('startInfiniteBtn').textContent = '♾️ Start Infinite Crawl + Import';
        }

        let logCount = 0;

        function updateInfiniteStats(stats) {
            document.getElementById('infDiscovered').textContent = stats.discovered.toLocaleString();
            document.getElementById('infProcessed').textContent = stats.processed.toLocaleString();
            document.getElementById('infImported').textContent = stats.imported.toLocaleString();
            document.getElementById('infDuplicates').textContent = stats.duplicates.toLocaleString();
            document.getElementById('infErrors').textContent = stats.errors.toLocaleString();
            document.getElementById('infRate').textContent = stats.rate + '/s';
            document.getElementById('infQueue').textContent = stats.queue_size.toLocaleString();
            document.getElementById('infEngine').textContent = stats.current_engine || '-';
            document.getElementById('infElapsed').textContent = formatTime(stats.elapsed);
            document.getElementById('infLastUrl').textContent = stats.last_url ? (stats.last_url.length > 80 ? stats.last_url.substring(0, 80) + '...' : stats.last_url) : '-';

            // Update activity log
            if (stats.log && stats.log.length > 0) {
                const logEl = document.getElementById('activityLog');

                // Clear initial message
                if (logCount === 0) {
                    logEl.innerHTML = '';
                }

                stats.log.forEach(entry => {
                    const div = document.createElement('div');
                    div.style.cssText = 'padding: 0.35rem 0.75rem; border-bottom: 1px solid rgba(255,255,255,0.05); display: flex; align-items: center; gap: 0.5rem;';

                    let statusIcon, statusColor;
                    if (entry.status === 'imported') {
                        statusIcon = '✅';
                        statusColor = '#22c55e';
                    } else if (entry.status === 'duplicate') {
                        statusIcon = '🔄';
                        statusColor = '#f59e0b';
                    } else if (entry.status === 'skipped') {
                        statusIcon = '⚪';
                        statusColor = '#6b7280';
                    } else {
                        statusIcon = '❌';
                        statusColor = '#ef4444';
                    }

                    const domain = entry.domain || (entry.url ? new URL(entry.url).hostname : '');
                    const title = entry.title || '';
                    const shortUrl = entry.url ? (entry.url.length > 60 ? entry.url.substring(0, 60) + '...' : entry.url) : '';

                    div.innerHTML = `
                        <span>${statusIcon}</span>
                        <span style="color: ${statusColor}; font-weight: 600; min-width: 65px; font-size: 0.7rem;">${entry.status.toUpperCase()}</span>
                        <span style="color: #6366f1; min-width: 120px; font-size: 0.7rem;">${domain}</span>
                        <span style="color: var(--text-primary); flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">${title || shortUrl}</span>
                    `;

                    logEl.insertBefore(div, logEl.firstChild);
                    logCount++;
                });

                // Keep max 500 entries
                while (logEl.children.length > 500) {
                    logEl.removeChild(logEl.lastChild);
                }
            }

            // Update status
            const statusEl = document.getElementById('infStatus');
            if (stats.status === 'running') {
                statusEl.textContent = '● Running';
                statusEl.style.color = '#22c55e';
            } else if (stats.status === 'stopped') {
                statusEl.textContent = '● Stopped';
                statusEl.style.color = '#f59e0b';
                infiniteRunning = false;
            } else if (stats.status === 'completed') {
                statusEl.textContent = '● Completed';
                statusEl.style.color = '#6366f1';
                infiniteRunning = false;
            }
        }

        function clearLog() {
            const logEl = document.getElementById('activityLog');
            logEl.innerHTML = '<div style="padding: 1rem; color: var(--text-secondary); text-align: center;">Log cleared</div>';
            logCount = 0;
        }

        function formatTime(seconds) {
            if (seconds < 60) return seconds + 's';
            if (seconds < 3600) return Math.floor(seconds / 60) + 'm ' + (seconds % 60) + 's';
            return Math.floor(seconds / 3600) + 'h ' + Math.floor((seconds % 3600) / 60) + 'm';
        }

        function pauseInfinite() {
            // Toggle pause (not fully implemented - would need backend support)
            alert('Pause functionality coming soon. Use Stop to halt the process.');
        }

        async function stopInfinite() {
            if (!confirm('Are you sure you want to stop the infinite crawler?')) return;

            infiniteRunning = false;
            document.getElementById('infStatus').textContent = '● Stopping...';
            document.getElementById('infStatus').style.color = '#f59e0b';

            try {
                await fetch('?action=stop_infinite');
            } catch (e) {}
        }

        // ========================
        // VIRAL MODE
        // ========================
        let viralRunning = false;
        let viralReader = null;
        let viralLogCount = 0;
        let viralLogFilter = 'all';           // U4: current filter
        let viralLogData = [];                // U4: all log entries for filtering
        let viralMilestones = new Set();      // U3: track shown milestones
        let viralStartTime = null;            // U1: for ETA calculation
        let viralMaxImports = 10000000;       // U1: target for ETA

        async function startViralMode() {
            // Read from viral-specific fields
            const keywords = document.querySelector('textarea[name="viral_keywords"]')?.value.trim() || '';
            const seedUrls = document.querySelector('textarea[name="viral_urls"]')?.value.trim() || '';
            const combined = [keywords, seedUrls].filter(Boolean).join('\n');

            if (!combined) {
                alert('Please enter at least one keyword or URL for viral spreading');
                return;
            }

            // Show stats panel
            document.getElementById('viralStatsPanel').style.display = 'block';
            document.getElementById('startViralBtn').disabled = true;
            document.getElementById('startViralBtn').textContent = '⏳ Seeding...';
            viralRunning = true;
            viralLogCount = 0;
            viralStartTime = Date.now();
            addTimelineEvent('🦠 Viral mode started', 'var(--error)');
            document.getElementById('headerStatusDot').style.background = 'var(--error)';

            // Initialize network graph
            networkGraph.init();

            // Get include/exclude terms from viral config panel
            const includeTerms = document.querySelector('textarea[name="viral_include"]')?.value || '';
            const excludeTerms = document.querySelector('textarea[name="viral_exclude"]')?.value || '';

            // Media filters
            const mediaFilters = JSON.stringify({
                pages: document.getElementById('filterPages')?.checked ?? true,
                articles: document.getElementById('filterArticles')?.checked ?? true,
                images: document.getElementById('filterImages')?.checked ?? false,
                videos: document.getElementById('filterVideos')?.checked ?? false,
                audio: document.getElementById('filterAudio')?.checked ?? false,
                docs: document.getElementById('filterDocs')?.checked ?? false,
                archives: document.getElementById('filterArchives')?.checked ?? false
            });

            // Priority settings
            const postsFirst = document.getElementById('postsFirst')?.checked ? '1' : '0';
            const followExternal = document.getElementById('followExternal')?.checked ? '1' : '0';
            const sameDomain = document.getElementById('sameDomainOnly')?.checked ? '1' : '0';

            // Common term (appended to each keyword)
            const commonTerm = document.getElementById('cfgCommonTerm')?.value.trim() || '';

            // Forced domains (bypass quality check)
            const forcedDomains = document.getElementById('cfgForcedDomains')?.value.trim() || '';

            // Speed config from UI
            const waveSize = document.getElementById('cfgWaveSize')?.value || '25';
            const linksPerPage = document.getElementById('cfgLinksPerPage')?.value || '100';
            const waveDelay = document.getElementById('cfgWaveDelay')?.value || '10';
            const maxPool = document.getElementById('cfgMaxPool')?.value || '500000';
            const maxImports = document.getElementById('cfgMaxImports')?.value || '10000000';
            const maxPerDomain = document.getElementById('cfgMaxPerDomain')?.value || '0';

            // Build URL with all parameters (S2: includes CSRF token)
            const params = new URLSearchParams({
                action: 'viral',
                stream: '1',
                csrf_token: csrfToken,
                keywords: combined,
                common_term: commonTerm,
                forced_domains: forcedDomains,
                wave_size: waveSize,
                micro_batch: 5,
                max_pool: maxPool,
                max_imports: maxImports,
                max_per_domain: maxPerDomain,
                links_per_page: linksPerPage,
                wave_delay_ms: waveDelay,
                include_terms: includeTerms,
                exclude_terms: excludeTerms,
                media_filters: mediaFilters,
                posts_first: postsFirst,
                follow_external: followExternal,
                same_domain: sameDomain
            });

            // U1: Track start time and max imports for ETA
            viralStartTime = Date.now();
            viralMaxImports = parseInt(maxImports) || 10000000;
            viralMilestones.clear();
            viralLogData = [];

            let retryCount = 0;
            const maxRetries = 5;

            while (viralRunning && retryCount <= maxRetries) {
                try {
                    if (retryCount > 0) {
                        document.getElementById('viralStatus').textContent = '● Reconnecting... (' + retryCount + '/' + maxRetries + ')';
                        document.getElementById('viralStatus').style.color = '#f59e0b';
                        await new Promise(r => setTimeout(r, 3000 * retryCount));
                    }

                    const response = await fetch('?' + params.toString());
                    if (!response.ok) throw new Error('HTTP ' + response.status);
                    const reader = response.body.getReader();
                    viralReader = reader;
                    const decoder = new TextDecoder();
                    retryCount = 0; // Reset on successful connection

                    document.getElementById('viralStatus').textContent = '● Running';
                    document.getElementById('viralStatus').style.color = '#22c55e';

                    while (viralRunning) {
                        const {value, done} = await reader.read();
                        if (done) break;

                        const chunk = decoder.decode(value);
                        const lines = chunk.split('\n');

                        for (const line of lines) {
                            if (line.startsWith('data: ')) {
                                try {
                                    const stats = JSON.parse(line.substring(6));
                                    updateViralStats(stats);
                                } catch (e) {}
                            }
                        }
                    }
                    break; // Clean exit
                } catch (error) {
                    if (viralRunning) {
                        retryCount++;
                        console.error('Viral stream error (retry ' + retryCount + '):', error);
                        if (retryCount > maxRetries) {
                            document.getElementById('viralStatus').textContent = '● Connection Lost';
                            document.getElementById('viralStatus').style.color = '#ef4444';
                        }
                    }
                }
            }

            viralReader = null;
            viralRunning = false;
            document.getElementById('startViralBtn').disabled = false;
            document.getElementById('startViralBtn').textContent = '🦠 Start VIRAL Spreading';
        }

        function updateViralStats(stats) {
            // C29: Use AnimatedCounters for smooth number transitions
            if (counters.viralPoolSize) counters.viralPoolSize.update(stats.pool_size);
            else document.getElementById('viralPoolSize').textContent = stats.pool_size.toLocaleString();
            if (counters.viralWaves) counters.viralWaves.update(stats.waves);
            else document.getElementById('viralWaves').textContent = stats.waves.toLocaleString();
            if (counters.viralImported) counters.viralImported.update(stats.imported);
            else document.getElementById('viralImported').textContent = stats.imported.toLocaleString();
            document.getElementById('viralRate').textContent = stats.rate + '/s';
            if (counters.viralDiscovered) counters.viralDiscovered.update(stats.discovered);
            else document.getElementById('viralDiscovered').textContent = stats.discovered.toLocaleString();
            if (counters.viralProcessed) counters.viralProcessed.update(stats.processed);
            else document.getElementById('viralProcessed').textContent = stats.processed.toLocaleString();
            if (counters.viralDuplicates) counters.viralDuplicates.update(stats.duplicates);
            else document.getElementById('viralDuplicates').textContent = stats.duplicates.toLocaleString();
            if (counters.viralSkipped) counters.viralSkipped.update(stats.skipped || 0);
            else document.getElementById('viralSkipped').textContent = (stats.skipped || 0).toLocaleString();
            if (counters.viralErrors) counters.viralErrors.update(stats.errors);
            else document.getElementById('viralErrors').textContent = stats.errors.toLocaleString();
            document.getElementById('viralElapsed').textContent = formatTime(stats.elapsed);
            document.getElementById('viralLastUrl').textContent = stats.last_url ? (stats.last_url.length > 80 ? stats.last_url.substring(0, 80) + '...' : stats.last_url) : '-';

            // C30: Update sparklines
            sparkPool.push(stats.pool_size);
            sparkWaves.push(stats.waves);
            sparkImported.push(stats.imported);
            currentRate = parseFloat(stats.rate) || 0;
            sparkRate.push(currentRate);

            // C42: Derived stats (success rate, URLs/min, pages/hour)
            if (stats.processed > 0) {
                const successRate = ((stats.imported / stats.processed) * 100).toFixed(1);
                document.getElementById('viralSuccessRate').textContent = successRate + '%';
            }
            if (stats.elapsed > 0) {
                const urlsMin = Math.round((stats.processed / stats.elapsed) * 60);
                const pagesHour = Math.round((stats.imported / stats.elapsed) * 3600);
                document.getElementById('viralUrlsMin').textContent = urlsMin.toLocaleString();
                document.getElementById('viralPagesHour').textContent = pagesHour.toLocaleString();
            }

            // C48: Progress ring (based on imports vs target)
            const importPercent = Math.min(100, (stats.imported / viralMaxImports) * 100);
            updateProgressRing(importPercent);

            // B26: Wave processing bar animation
            const waveBar = document.getElementById('waveBarFill');
            if (waveBar && stats.status === 'running') {
                waveBar.style.width = '100%';
            } else if (waveBar) {
                waveBar.style.width = '0%';
            }

            // Pool growth bar (max 50000)
            const poolPercent = Math.min(100, (stats.pool_size / 50000) * 100);
            document.getElementById('viralPoolBar').style.width = poolPercent + '%';
            document.getElementById('viralPoolPercent').textContent = Math.round(poolPercent) + '%';

            // I2: Pool health indicator + wave size + domains
            let healthPercent = 100;
            if (stats.unique_domains !== undefined) {
                document.getElementById('viralUniqueDomains').textContent = stats.unique_domains;
            }
            if (stats.wave_size !== undefined) {
                document.getElementById('viralWaveSize').textContent = stats.wave_size;
            }
            if (stats.pool_size !== undefined && stats.unique_domains !== undefined) {
                const diversity = stats.unique_domains > 0 ? stats.unique_domains / Math.max(1, stats.pool_size) : 0;
                const errorRate = stats.errors / Math.max(1, stats.processed);
                let health, healthColor;
                if (diversity > 0.05 && errorRate < 0.3 && stats.pool_size > 10) {
                    health = '🟢 Healthy'; healthColor = '#22c55e'; healthPercent = 90;
                } else if (diversity > 0.02 || errorRate < 0.5) {
                    health = '🟡 Warning'; healthColor = '#f59e0b'; healthPercent = 55;
                } else {
                    health = '🔴 Degraded'; healthColor = '#ef4444'; healthPercent = 20;
                }
                const el = document.getElementById('viralPoolHealth');
                el.textContent = health; el.style.color = healthColor;
                // C44: Update pool gauge
                updatePoolGauge(healthPercent);
            }

            // I1: Engine stats display with Circuit Breaker status
            if (stats.engine_stats && Object.keys(stats.engine_stats).length > 0) {
                const section = document.getElementById('engineStatsSection');
                section.style.display = 'block';
                const content = document.getElementById('engineStatsContent');
                content.innerHTML = Object.entries(stats.engine_stats)
                    .sort((a, b) => b[1].seeds - a[1].seeds)
                    .map(([engine, data]) => {
                        const failures = data.failures || 0;
                        const isBlocked = failures >= 3;
                        let color, icon, status;
                        if (isBlocked) {
                            color = '#ef4444'; // Red for blocked
                            icon = '🚫';
                            status = `BLOCKED (${failures}x fails)`;
                        } else if (failures > 0) {
                            color = '#f59e0b'; // Orange for warning
                            icon = '⚠️';
                            status = `${data.seeds} seeds (${failures} fails)`;
                        } else if (data.seeds > 50) {
                            color = '#22c55e'; // Green for healthy
                            icon = '✓';
                            status = `${data.seeds} seeds`;
                        } else if (data.seeds > 0) {
                            color = '#3b82f6'; // Blue for active
                            icon = '⚡';
                            status = `${data.seeds} seeds`;
                        } else {
                            color = '#6b7280'; // Gray for inactive
                            icon = '○';
                            status = 'no seeds';
                        }
                        return `<span style="padding: 0.2rem 0.5rem; background: ${color}22; border: 1px solid ${color}44; border-radius: 0.25rem; color: ${color}; ${isBlocked ? 'text-decoration: line-through; opacity: 0.7;' : ''}" title="${data.reseeds || 0} re-seeds">${icon} ${engine}: ${status}</span>`;
                    }).join('');
            }

            // U1: ETA calculation
            if (stats.elapsed > 10 && stats.imported > 0 && stats.rate && parseFloat(stats.rate) > 0) {
                const rate = parseFloat(stats.rate);
                const remaining = viralMaxImports - stats.imported;
                if (remaining > 0 && rate > 0) {
                    const etaSeconds = remaining / rate;
                    document.getElementById('viralETA').textContent = etaSeconds > 86400 ? '> 24h' : formatTime(Math.round(etaSeconds));
                } else {
                    document.getElementById('viralETA').textContent = 'Done';
                }
            }

            // Bandwidth display
            if (stats.bandwidth_mb !== undefined) {
                document.getElementById('viralBandwidth').textContent = stats.bandwidth_mb >= 1024 ? (stats.bandwidth_mb / 1024).toFixed(1) + ' GB' : stats.bandwidth_mb + ' MB';
            }

            // C37: Domain leaderboard (update every 5 waves)
            if (stats.domain_stats) {
                for (const [domain, ds] of Object.entries(stats.domain_stats)) {
                    if (!domainStats[domain]) domainStats[domain] = { imports: 0, crawled: 0 };
                    domainStats[domain].imports = ds.imported || domainStats[domain].imports;
                    domainStats[domain].crawled = ds.crawled || domainStats[domain].crawled;
                }
            }
            if (stats.waves % 5 === 0 && stats.waves !== leaderboardWaveCount) {
                leaderboardWaveCount = stats.waves;
                updateLeaderboard();
            }

            // U3: Milestone toast notifications + C35: Sound
            const milestones = [100, 500, 1000, 2500, 5000, 10000, 25000, 50000, 100000];
            milestones.forEach(m => {
                if (stats.imported >= m && !viralMilestones.has(m)) {
                    viralMilestones.add(m);
                    showToast('🎉 ' + m.toLocaleString() + ' imports reached!', 'success');
                    playBeep(1000 + m / 100, 200);
                    addTimelineEvent('🎉 Milestone: ' + m.toLocaleString() + ' imports', 'var(--success)');
                }
            });

            // Network graph data
            if (stats.network && stats.network.length > 0) {
                stats.network.forEach(edge => {
                    networkGraph.addEdge(edge[0], edge[1]);
                });
            }
            // Domain stats update (per-domain crawl/import/error data)
            if (stats.domain_stats) {
                for (const [domain, ds] of Object.entries(stats.domain_stats)) {
                    if (networkGraph.nodes[domain]) {
                        networkGraph.nodes[domain].urlsCrawled = ds.crawled || 0;
                        networkGraph.nodes[domain].imports = ds.imported || 0;
                        networkGraph.nodes[domain].errors = ds.errors || 0;
                        networkGraph.nodes[domain].lastStatus = ds.status || 0;
                        networkGraph.nodes[domain].linksFound = ds.links || 0;
                        networkGraph.nodes[domain].status = (ds.errors > 5 && (ds.imported||0) === 0) ? 'dead' : ds.errors > 2 ? 'error' : ds.imported > 0 ? 'imported' : 'active';
                    }
                }
            }
            // Update counters
            document.getElementById('viralNodeCount').textContent = (stats.total_nodes || Object.keys(networkGraph.nodes).length).toLocaleString();
            document.getElementById('viralEdgeCount').textContent = (stats.total_edges || networkGraph.edges.length).toLocaleString();

            // Update activity log
            if (stats.log && stats.log.length > 0) {
                const logEl = document.getElementById('viralActivityLog');

                if (viralLogCount === 0) {
                    logEl.innerHTML = '';
                }

                stats.log.forEach(entry => {
                    // U4: Store for filtering + F3: export
                    viralLogData.push(entry);
                    if (viralLogData.length > 5000) viralLogData = viralLogData.slice(-5000);

                    const div = document.createElement('div');
                    div.className = 'log-entry-animated';
                    div.setAttribute('data-status', entry.status || '');
                    div.setAttribute('data-domain', (entry.domain || '').toLowerCase());
                    div.setAttribute('data-url', (entry.url || '').toLowerCase());
                    div.style.cssText = 'padding: 0.35rem 0.75rem; border-bottom: 1px solid rgba(255,255,255,0.03); display: flex; align-items: center; gap: 0.5rem; transition: background 0.2s;';
                    div.addEventListener('mouseenter', function() { this.style.background = 'rgba(99,102,241,0.05)'; });
                    div.addEventListener('mouseleave', function() { this.style.background = 'none'; });

                    // U4: Apply current filter visibility
                    if (viralLogFilter !== 'all' && entry.status !== viralLogFilter) {
                        div.style.display = 'none';
                    }
                    const searchVal = (document.getElementById('viralLogSearch')?.value || '').toLowerCase();
                    if (searchVal && !(entry.url || '').toLowerCase().includes(searchVal) && !(entry.domain || '').toLowerCase().includes(searchVal)) {
                        div.style.display = 'none';
                    }

                    let statusIcon, statusColor;
                    if (entry.status === 'imported') {
                        statusIcon = '✅';
                        statusColor = '#22c55e';
                    } else if (entry.status === 'duplicate') {
                        statusIcon = '🔄';
                        statusColor = '#f59e0b';
                    } else if (entry.status === 'skipped') {
                        statusIcon = '⚪';
                        statusColor = '#6b7280';
                    } else {
                        statusIcon = '❌';
                        statusColor = '#ef4444';
                    }

                    const domain = entry.domain || '';
                    const title = entry.title || '';
                    const shortUrl = entry.url ? (entry.url.length > 60 ? entry.url.substring(0, 60) + '...' : entry.url) : '';

                    div.innerHTML = `
                        <span>${statusIcon}</span>
                        <span style="color: ${statusColor}; font-weight: 600; min-width: 65px; font-size: 0.7rem;">${entry.status.toUpperCase()}</span>
                        <span style="color: #ef4444; min-width: 120px; font-size: 0.7rem;">${domain}</span>
                        <span style="color: var(--text-primary); flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">${title || shortUrl}</span>
                    `;

                    logEl.insertBefore(div, logEl.firstChild);
                    viralLogCount++;
                });

                while (logEl.children.length > 500) {
                    logEl.removeChild(logEl.lastChild);
                }
            }

            // Update status
            const statusEl = document.getElementById('viralStatus');
            if (stats.status === 'seeding') {
                statusEl.textContent = '● Seeding...';
                statusEl.style.color = '#f97316';
            } else if (stats.status === 'running') {
                statusEl.textContent = '● Spreading';
                statusEl.style.color = '#22c55e';
            } else if (stats.status === 'paused') {
                statusEl.textContent = '● Paused';
                statusEl.style.color = '#a78bfa';
            } else if (stats.status === 'cooldown') {
                statusEl.textContent = '● Waiting (domains cooling)';
                statusEl.style.color = '#f59e0b';
            } else if (stats.status === 'error_pause') {
                statusEl.textContent = '● Error Recovery...';
                statusEl.style.color = '#ef4444';
            } else if (stats.status === 'stopped') {
                statusEl.textContent = '● Stopped';
                statusEl.style.color = '#f59e0b';
                viralRunning = false;
            } else if (stats.status === 'completed') {
                statusEl.textContent = '● Completed';
                statusEl.style.color = '#6366f1';
                viralRunning = false;
            }
        }

        function clearViralLog() {
            const logEl = document.getElementById('viralActivityLog');
            logEl.innerHTML = '<div style="padding: 1rem; color: var(--text-secondary); text-align: center;">Log cleared</div>';
            viralLogCount = 0;
        }

        async function stopViral() {
            if (!confirm('Stop the viral spreading?')) return;

            viralRunning = false;
            addTimelineEvent('⏹ Viral mode stopped', 'var(--warning)');
            document.getElementById('headerStatusDot').style.background = 'var(--warning)';
            document.getElementById('viralStatus').textContent = '● Stopping...';
            document.getElementById('viralStatus').style.color = '#f59e0b';

            // Cancel the stream reader immediately
            if (viralReader) {
                try { viralReader.cancel(); } catch (e) {}
                viralReader = null;
            }

            try {
                await fetch('?action=stop_viral');
            } catch (e) {}

            // Force stopped state
            document.getElementById('viralStatus').textContent = '● Stopped';
            document.getElementById('viralStatus').style.color = '#ef4444';
            document.getElementById('startViralBtn').disabled = false;
            document.getElementById('startViralBtn').textContent = '🦠 Start VIRAL Spreading';
        }

        // F2: Pause/Resume toggle
        let viralPaused = false;
        async function toggleViralPause() {
            try {
                const resp = await fetch('?action=pause_viral');
                const data = await resp.json();
                viralPaused = data.paused;
                const btn = document.getElementById('viralPauseBtn');
                if (viralPaused) {
                    btn.innerHTML = '▶️ Resume';
                    btn.style.background = 'rgba(34,197,94,0.3)';
                    showToast('Crawl paused', 'warning');
                } else {
                    btn.innerHTML = '⏸️ Pause';
                    btn.style.background = 'rgba(255,255,255,0.2)';
                    showToast('Crawl resumed', 'success');
                }
            } catch (e) {}
        }

        // U3: Toast notification system
        function showToast(message, type = 'info') {
            const container = document.getElementById('toastContainer');
            const toast = document.createElement('div');
            const colors = { success: '#22c55e', error: '#ef4444', info: '#6366f1', warning: '#f59e0b' };
            toast.style.cssText = `pointer-events: auto; padding: 0.75rem 1.25rem; border-radius: 0.5rem; background: ${colors[type] || colors.info}; color: white; font-weight: 600; font-size: 0.85rem; box-shadow: 0 4px 12px rgba(0,0,0,0.3); opacity: 0; transform: translateX(100px); transition: all 0.3s ease;`;
            toast.textContent = message;
            container.appendChild(toast);
            requestAnimationFrame(() => { toast.style.opacity = '1'; toast.style.transform = 'translateX(0)'; });
            setTimeout(() => {
                toast.style.opacity = '0'; toast.style.transform = 'translateX(100px)';
                setTimeout(() => toast.remove(), 300);
            }, 4000);
        }

        // U4: Log filter functions
        function setLogFilter(filter, btn) {
            viralLogFilter = filter;
            document.querySelectorAll('.log-filter-btn').forEach(b => {
                b.style.background = 'var(--bg-tertiary)';
                b.style.color = 'var(--text-secondary)';
            });
            btn.style.background = filter === 'all' ? '#22c55e33' : filter === 'imported' ? '#22c55e33' : filter === 'duplicate' ? '#f59e0b33' : filter === 'error' ? '#ef444433' : '#6b728033';
            btn.style.color = 'var(--text-primary)';
            filterViralLog();
        }

        function filterViralLog() {
            const searchVal = (document.getElementById('viralLogSearch')?.value || '').toLowerCase();
            const logEl = document.getElementById('viralActivityLog');
            const entries = logEl.querySelectorAll('div[data-status]');
            entries.forEach(div => {
                const status = div.getAttribute('data-status');
                const domain = div.getAttribute('data-domain') || '';
                const url = div.getAttribute('data-url') || '';
                let show = true;
                if (viralLogFilter !== 'all' && status !== viralLogFilter) show = false;
                if (searchVal && !domain.includes(searchVal) && !url.includes(searchVal)) show = false;
                div.style.display = show ? 'flex' : 'none';
            });
        }

        // F3: Export results
        function exportViralResults(format) {
            if (viralLogData.length === 0) { showToast('No data to export', 'warning'); return; }
            let content, filename, mime;
            const imported = viralLogData.filter(e => e.status === 'imported');
            if (format === 'csv') {
                content = 'URL,Status,Domain,Title\n';
                viralLogData.forEach(e => {
                    content += `"${(e.url||'').replace(/"/g,'""')}","${e.status}","${e.domain||''}","${(e.title||'').replace(/"/g,'""')}"\n`;
                });
                filename = 'viral_results_' + new Date().toISOString().slice(0,10) + '.csv';
                mime = 'text/csv';
            } else {
                content = JSON.stringify({ exported: new Date().toISOString(), total: viralLogData.length, imported: imported.length, entries: viralLogData }, null, 2);
                filename = 'viral_results_' + new Date().toISOString().slice(0,10) + '.json';
                mime = 'application/json';
            }
            const blob = new Blob([content], { type: mime });
            const a = document.createElement('a');
            a.href = URL.createObjectURL(blob);
            a.download = filename;
            a.click();
            URL.revokeObjectURL(a.href);
            showToast('Exported ' + viralLogData.length + ' entries as ' + format.toUpperCase(), 'success');
        }

        // U9: Keyboard shortcuts (enhanced)
        document.addEventListener('keydown', function(e) {
            // Close shortcuts overlay on any key
            if (document.getElementById('shortcutsOverlay').classList.contains('show')) {
                closeShortcuts();
                return;
            }
            // Don't trigger shortcuts when typing in inputs/textareas
            if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA' || e.target.tagName === 'SELECT') return;
            if (e.ctrlKey && e.key === 'Enter') { e.preventDefault(); if (!viralRunning) document.getElementById('startViralBtn')?.click(); }
            if (e.key === 'Escape') { if (viralRunning) stopViral(); }
            if (e.key === ' ' && viralRunning) { e.preventDefault(); toggleViralPause(); }
            if (e.ctrlKey && e.key === 'e') { e.preventDefault(); exportViralResults('csv'); }
            if (e.ctrlKey && e.key === 'l') { e.preventDefault(); clearViralLog(); }
            if (e.key === 'F11') { e.preventDefault(); toggleTheaterMode(); }
            if (e.key === 'm' || e.key === 'M') { toggleSound(); }
            if (e.key === '?') { openShortcuts(); }
        });

        // ========================
        // NETWORK GRAPH VISUALIZATION v3.0 (Ultra Interactive)
        // ========================
        // TLD to country/flag mapping
        const TLD_COUNTRY = {'.br':'BR','.uk':'GB','.de':'DE','.fr':'FR','.it':'IT','.es':'ES','.pt':'PT','.nl':'NL','.be':'BE','.ch':'CH','.at':'AT','.pl':'PL','.cz':'CZ','.ru':'RU','.ua':'UA','.jp':'JP','.cn':'CN','.kr':'KR','.in':'IN','.au':'AU','.nz':'NZ','.ca':'CA','.mx':'MX','.ar':'AR','.cl':'CL','.co':'CO','.pe':'PE','.ve':'VE','.za':'ZA','.ng':'NG','.ke':'KE','.eg':'EG','.tr':'TR','.sa':'SA','.ae':'AE','.il':'IL','.se':'SE','.no':'NO','.dk':'DK','.fi':'FI','.ie':'IE','.ro':'RO','.hu':'HU','.gr':'GR','.bg':'BG','.hr':'HR','.rs':'RS','.sk':'SK','.si':'SI','.lt':'LT','.lv':'LV','.ee':'EE','.th':'TH','.vn':'VN','.ph':'PH','.my':'MY','.sg':'SG','.id':'ID','.tw':'TW','.hk':'HK','.us':'US','.io':'GB','.ai':'AI','.dev':'US','.app':'US','.com':'INT','.org':'INT','.net':'INT','.info':'INT','.edu':'US','.gov':'US'};
        const COUNTRY_FLAG = {'BR':'\ud83c\udde7\ud83c\uddf7','GB':'\ud83c\uddec\ud83c\udde7','DE':'\ud83c\udde9\ud83c\uddea','FR':'\ud83c\uddeb\ud83c\uddf7','IT':'\ud83c\uddee\ud83c\uddf9','ES':'\ud83c\uddea\ud83c\uddf8','PT':'\ud83c\uddf5\ud83c\uddf9','NL':'\ud83c\uddf3\ud83c\uddf1','PL':'\ud83c\uddf5\ud83c\uddf1','RU':'\ud83c\uddf7\ud83c\uddfa','JP':'\ud83c\uddef\ud83c\uddf5','CN':'\ud83c\udde8\ud83c\uddf3','KR':'\ud83c\uddf0\ud83c\uddf7','IN':'\ud83c\uddee\ud83c\uddf3','AU':'\ud83c\udde6\ud83c\uddfa','CA':'\ud83c\udde8\ud83c\udde6','MX':'\ud83c\uddf2\ud83c\uddfd','AR':'\ud83c\udde6\ud83c\uddf7','US':'\ud83c\uddfa\ud83c\uddf8','INT':'\ud83c\udf10','SE':'\ud83c\uddf8\ud83c\uddea','NO':'\ud83c\uddf3\ud83c\uddf4','DK':'\ud83c\udde9\ud83c\uddf0','FI':'\ud83c\uddeb\ud83c\uddee','TR':'\ud83c\uddf9\ud83c\uddf7','ZA':'\ud83c\uddff\ud83c\udde6','CO':'\ud83c\udde8\ud83c\uddf4','CL':'\ud83c\udde8\ud83c\uddf1','PE':'\ud83c\uddf5\ud83c\uddea','TH':'\ud83c\uddf9\ud83c\udded','VN':'\ud83c\uddfb\ud83c\uddf3','PH':'\ud83c\uddf5\ud83c\udded','ID':'\ud83c\uddee\ud83c\udde9','TW':'\ud83c\uddf9\ud83c\uddfc','AI':'\ud83e\udd16','UA':'\ud83c\uddfa\ud83c\udde6','RO':'\ud83c\uddf7\ud83c\uddf4','GR':'\ud83c\uddec\ud83c\uddf7','HU':'\ud83c\udded\ud83c\uddfa','CZ':'\ud83c\udde8\ud83c\uddff','IE':'\ud83c\uddee\ud83c\uddea','BE':'\ud83c\udde7\ud83c\uddea','CH':'\ud83c\udde8\ud83c\udded','AT':'\ud83c\udde6\ud83c\uddf9','SG':'\ud83c\uddf8\ud83c\uddec','MY':'\ud83c\uddf2\ud83c\uddfe','NZ':'\ud83c\uddf3\ud83c\uddff','HK':'\ud83c\udded\ud83c\uddf0'};
        const COUNTRY_NAMES = {'BR':'Brazil','GB':'United Kingdom','DE':'Germany','FR':'France','IT':'Italy','ES':'Spain','PT':'Portugal','NL':'Netherlands','PL':'Poland','RU':'Russia','JP':'Japan','CN':'China','KR':'South Korea','IN':'India','AU':'Australia','CA':'Canada','MX':'Mexico','AR':'Argentina','US':'United States','INT':'International','SE':'Sweden','NO':'Norway','DK':'Denmark','FI':'Finland','TR':'Turkey','ZA':'South Africa','CO':'Colombia','CL':'Chile','PE':'Peru','TH':'Thailand','VN':'Vietnam','PH':'Philippines','ID':'Indonesia','TW':'Taiwan','AI':'AI/Tech','UA':'Ukraine','RO':'Romania','GR':'Greece','HU':'Hungary','CZ':'Czech Rep.','IE':'Ireland','BE':'Belgium','CH':'Switzerland','AT':'Austria','SG':'Singapore','MY':'Malaysia','NZ':'New Zealand','HK':'Hong Kong'};
        const COUNTRY_COLORS = {'BR':'#22c55e','US':'#3b82f6','GB':'#a855f7','DE':'#f59e0b','FR':'#6366f1','ES':'#ef4444','PT':'#22c55e','JP':'#ec4899','CN':'#ef4444','IN':'#f97316','RU':'#6366f1','INT':'#6366f1','AU':'#06b6d4','CA':'#ef4444','MX':'#22c55e','AR':'#06b6d4'};

        // GALAXY TIER SYSTEM: Nodes classified by importance
        const NODE_TIERS = {
            supernova: {min: 50, label: '🌟 Supernova', color: '#fbbf24', glow: 'rgba(251,191,36,0.4)', size: 28, ring: 3},
            star:      {min: 20, label: '⭐ Star', color: '#f59e0b', glow: 'rgba(245,158,11,0.3)', size: 22, ring: 2},
            planet:    {min: 5,  label: '🪐 Planet', color: '#22c55e', glow: 'rgba(34,197,94,0.2)', size: 16, ring: 1},
            moon:      {min: 1,  label: '🌙 Moon', color: '#06b6d4', glow: 'rgba(6,182,212,0.15)', size: 12, ring: 0},
            asteroid:  {min: 0,  label: '☄️ Asteroid', color: '#6366f1', glow: 'rgba(99,102,241,0.1)', size: 8, ring: 0}
        };
        // DOMAIN CATEGORY detection from URL patterns
        const DOMAIN_CATEGORIES = {
            news:     {pattern: /news|jornal|noticia|gazette|times|herald|tribune|post|daily|press|reuters|bbc|cnn|folha|globo|uol/i, label: '📰 News', color: '#ef4444'},
            blog:     {pattern: /blog|medium|wordpress|blogger|substack|dev\.to|hashnode/i, label: '📝 Blog', color: '#a855f7'},
            tech:     {pattern: /tech|dev|code|github|stack|digital|web|soft|app|cyber|ai|data|cloud/i, label: '💻 Tech', color: '#06b6d4'},
            commerce: {pattern: /shop|store|buy|sell|market|commerce|loja|mercado|price|deal|product/i, label: '🛒 Commerce', color: '#f97316'},
            edu:      {pattern: /edu|university|univ|school|college|academy|learn|course|study/i, label: '🎓 Education', color: '#3b82f6'},
            gov:      {pattern: /gov|government|governo|municipal|prefeitura|senado|camara/i, label: '🏛️ Government', color: '#64748b'},
            health:   {pattern: /health|saude|medical|hospital|clinic|doctor|med|pharma/i, label: '🏥 Health', color: '#10b981'},
            social:   {pattern: /forum|community|social|chat|discuss|grupo|rede/i, label: '💬 Social', color: '#ec4899'},
            media:    {pattern: /video|music|audio|podcast|tv|radio|stream|film|cinema|foto|photo/i, label: '🎬 Media', color: '#8b5cf6'},
            sports:   {pattern: /sport|esporte|futebol|soccer|football|basketball|tennis|game/i, label: '⚽ Sports', color: '#22c55e'},
            science:  {pattern: /science|research|journal|paper|academic|ciencia|nature|physics/i, label: '🔬 Science', color: '#14b8a6'},
            travel:   {pattern: /travel|viagem|tourism|hotel|flight|trip|destination|turismo/i, label: '✈️ Travel', color: '#0ea5e9'},
            food:     {pattern: /food|recipe|cook|restaurant|cuisine|culinaria|receita|gastro/i, label: '🍽️ Food', color: '#f43f5e'},
            finance:  {pattern: /finance|bank|money|invest|crypto|stock|economy|financ|dinhe/i, label: '💰 Finance', color: '#eab308'}
        };
        // COSMIC COLOR PALETTES per tier
        const COSMIC_COLORS = [
            '#ff6b6b','#feca57','#48dbfb','#ff9ff3','#54a0ff','#5f27cd','#01a3a4','#00d2d3',
            '#e056fd','#7bed9f','#70a1ff','#ffa502','#ff4757','#2ed573','#1e90ff','#ff6348',
            '#5352ed','#2bcbba','#fa8231','#45aaf2','#a55eea','#26de81','#fd9644','#4bcffa'
        ];
        // Starfield data (generated once)
        const STARS = Array.from({length: 200}, () => ({
            x: Math.random(), y: Math.random(),
            size: Math.random() * 1.5 + 0.3,
            alpha: Math.random() * 0.5 + 0.1,
            twinkle: Math.random() * 3 + 1
        }));

        function getNodeTier(node) {
            const imp = node.imports || 0;
            if (imp >= NODE_TIERS.supernova.min) return 'supernova';
            if (imp >= NODE_TIERS.star.min) return 'star';
            if (imp >= NODE_TIERS.planet.min) return 'planet';
            if (imp >= NODE_TIERS.moon.min) return 'moon';
            return 'asteroid';
        }

        function getNodeCategory(domain) {
            const d = domain.toLowerCase();
            for (const [cat, info] of Object.entries(DOMAIN_CATEGORIES)) {
                if (info.pattern.test(d)) return cat;
            }
            return 'general';
        }

        function getCountryFromDomain(domain) {
            const d = domain.toLowerCase();
            // Check multi-part TLDs first (com.br, co.uk, etc.)
            const multiMatch = d.match(/\.(com|co|org|net|gov|edu)\.[a-z]{2}$/);
            if (multiMatch) {
                const cc = '.' + d.split('.').pop();
                return TLD_COUNTRY[cc] || 'INT';
            }
            const tld = '.' + d.split('.').pop();
            return TLD_COUNTRY[tld] || 'INT';
        }

        const networkGraph = {
            nodes: {},
            edges: [],
            edgeSet: new Set(),
            canvas: null,
            ctx: null,
            minimap: null,
            minimapCtx: null,
            animFrame: null,
            nodeCount: 0,
            firstNode: null,
            // Interaction state
            scale: 1,
            offsetX: 0,
            offsetY: 0,
            isDragging: false,
            dragStart: {x:0, y:0},
            lastOffset: {x:0, y:0},
            hoveredNode: null,
            selectedNode: null,
            paused: false,
            dpr: 1,
            // Enhanced features
            showLabels: true,
            showParticles: true,
            filter: 'all',
            searchQuery: '',
            isFullscreen: false,
            countryStats: {},
            particles: [],
            lastParticleSpawn: 0,

            init() {
                this.canvas = document.getElementById('networkCanvas');
                if (!this.canvas) return;
                this.ctx = this.canvas.getContext('2d');
                this.minimap = document.getElementById('netMinimap');
                if (this.minimap) this.minimapCtx = this.minimap.getContext('2d');
                this.dpr = window.devicePixelRatio || 1;
                const rect = this.canvas.getBoundingClientRect();
                this.canvas.width = rect.width * this.dpr;
                this.canvas.height = rect.height * this.dpr;
                this.width = rect.width;
                this.height = rect.height;
                this.offsetX = 0;
                this.offsetY = 0;
                this.scale = 1;
                this.setupEvents();
                this.startAnimation();
            },

            setupEvents() {
                const c = this.canvas;
                c.addEventListener('wheel', (e) => {
                    e.preventDefault();
                    const rect = c.getBoundingClientRect();
                    const mx = e.clientX - rect.left;
                    const my = e.clientY - rect.top;
                    const zoom = e.deltaY > 0 ? 0.9 : 1.1;
                    const newScale = Math.max(0.2, Math.min(8, this.scale * zoom));
                    const factor = newScale / this.scale;
                    this.offsetX = mx - (mx - this.offsetX) * factor;
                    this.offsetY = my - (my - this.offsetY) * factor;
                    this.scale = newScale;
                }, {passive: false});
                c.addEventListener('mousedown', (e) => {
                    if (e.button !== 0) return;
                    this.isDragging = true;
                    this.dragStart = {x: e.clientX, y: e.clientY};
                    this.lastOffset = {x: this.offsetX, y: this.offsetY};
                    c.style.cursor = 'grabbing';
                });
                c.addEventListener('mousemove', (e) => {
                    if (this.isDragging) {
                        this.offsetX = this.lastOffset.x + (e.clientX - this.dragStart.x);
                        this.offsetY = this.lastOffset.y + (e.clientY - this.dragStart.y);
                        return;
                    }
                    const rect = c.getBoundingClientRect();
                    const mx = (e.clientX - rect.left - this.offsetX) / this.scale;
                    const my = (e.clientY - rect.top - this.offsetY) / this.scale;
                    this.hoveredNode = this.findNodeAt(mx, my);
                    c.style.cursor = this.hoveredNode ? 'pointer' : 'grab';
                    this.showTooltip(e, this.hoveredNode);
                });
                c.addEventListener('mouseup', (e) => {
                    if (this.isDragging) {
                        const moved = Math.abs(e.clientX - this.dragStart.x) + Math.abs(e.clientY - this.dragStart.y);
                        this.isDragging = false;
                        c.style.cursor = 'grab';
                        if (moved > 5) return;
                    }
                    const rect = c.getBoundingClientRect();
                    const mx = (e.clientX - rect.left - this.offsetX) / this.scale;
                    const my = (e.clientY - rect.top - this.offsetY) / this.scale;
                    const clicked = this.findNodeAt(mx, my);
                    if (clicked) {
                        this.selectedNode = clicked;
                        this.showDetails(clicked);
                    } else {
                        this.selectedNode = null;
                        this.closeDetails();
                    }
                });
                c.addEventListener('mouseleave', () => {
                    this.isDragging = false;
                    this.hoveredNode = null;
                    c.style.cursor = 'grab';
                    document.getElementById('netTooltip').style.display = 'none';
                });
                // Double-click to focus on node
                c.addEventListener('dblclick', (e) => {
                    const rect = c.getBoundingClientRect();
                    const mx = (e.clientX - rect.left - this.offsetX) / this.scale;
                    const my = (e.clientY - rect.top - this.offsetY) / this.scale;
                    const node = this.findNodeAt(mx, my);
                    if (node && this.nodes[node]) {
                        this.scale = 2.5;
                        this.offsetX = this.width/2 - this.nodes[node].x * this.scale;
                        this.offsetY = this.height/2 - this.nodes[node].y * this.scale;
                    }
                });
                // Minimap click
                if (this.minimap) {
                    this.minimap.addEventListener('click', (e) => {
                        const rect = this.minimap.getBoundingClientRect();
                        const mx = (e.clientX - rect.left) / rect.width;
                        const my = (e.clientY - rect.top) / rect.height;
                        const keys = Object.keys(this.nodes);
                        if (keys.length === 0) return;
                        let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity;
                        keys.forEach(k => { const n = this.nodes[k]; minX = Math.min(minX,n.x); maxX = Math.max(maxX,n.x); minY = Math.min(minY,n.y); maxY = Math.max(maxY,n.y); });
                        const targetX = minX + (maxX - minX) * mx;
                        const targetY = minY + (maxY - minY) * my;
                        this.offsetX = this.width/2 - targetX * this.scale;
                        this.offsetY = this.height/2 - targetY * this.scale;
                    });
                }
            },

            findNodeAt(mx, my) {
                let closest = null, closestDist = 25;
                for (const [key, node] of Object.entries(this.nodes)) {
                    if (!this.isNodeVisible(key)) continue;
                    const size = this.getNodeSize(node);
                    const dx = node.x - mx, dy = node.y - my;
                    const dist = Math.sqrt(dx*dx + dy*dy);
                    if (dist < size + 6 && dist < closestDist) {
                        closest = key;
                        closestDist = dist;
                    }
                }
                return closest;
            },

            isNodeVisible(key) {
                if (this.filter === 'all' && !this.searchQuery) return true;
                const node = this.nodes[key];
                if (!node) return false;
                if (this.searchQuery && !key.toLowerCase().includes(this.searchQuery.toLowerCase())) return false;
                if (this.filter === 'all') return true;
                if (this.filter === 'imported') return node.imports > 0;
                if (this.filter === 'active') return node.status === 'active' && node.imports === 0;
                if (this.filter === 'error') return node.status === 'error' || node.errors > 0;
                if (this.filter === 'dead') return node.status === 'dead';
                return true;
            },

            showTooltip(e, nodeKey) {
                const tip = document.getElementById('netTooltip');
                if (!nodeKey || !this.nodes[nodeKey]) { tip.style.display = 'none'; return; }
                const n = this.nodes[nodeKey];
                const country = getCountryFromDomain(nodeKey);
                const flag = COUNTRY_FLAG[country] || '\ud83c\udf10';
                const countryName = COUNTRY_NAMES[country] || 'Unknown';
                const statusColors = {imported:'#22c55e', active:'#6366f1', error:'#f59e0b', dead:'#6b7280'};
                const statusColor = n.isFirst ? '#ef4444' : (statusColors[n.status] || '#6366f1');
                const age = Math.round((Date.now() - n.firstSeen) / 1000);
                const ageStr = age > 60 ? Math.round(age/60) + 'm' : age + 's';
                const successRate = n.urlsCrawled > 0 ? Math.round((n.imports / n.urlsCrawled) * 100) : 0;
                tip.innerHTML = `<div style="display:flex;align-items:center;gap:6px;margin-bottom:6px;">
                    <span style="font-size:1.3rem;">${flag}</span>
                    <div><div style="font-weight:700;color:${statusColor};font-size:0.78rem;">${nodeKey}</div>
                    <div style="opacity:0.6;font-size:0.62rem;">${countryName} \u2022 ${ageStr} ago</div></div></div>
                    <div style="display:grid;grid-template-columns:1fr 1fr;gap:3px 12px;margin-top:4px;">
                    <div>Crawled: <strong>${n.urlsCrawled || 0}</strong></div>
                    <div>Links: <strong>${n.linksFound || 0}</strong></div>
                    <div style="color:#22c55e;">Imports: <strong>${n.imports || 0}</strong></div>
                    <div style="color:#f59e0b;">Errors: <strong>${n.errors || 0}</strong></div>
                    <div>Edges: <strong>${n.connections}</strong></div>
                    <div>Rate: <strong>${successRate}%</strong></div></div>`
                    + (n.lastStatus ? `<div style="margin-top:4px;opacity:0.5;font-size:0.6rem;">HTTP ${n.lastStatus} \u2022 Click for details</div>` : '');
                const rect = this.canvas.getBoundingClientRect();
                let tx = e.clientX - rect.left + 15;
                let ty = e.clientY - rect.top - 10;
                if (tx + 220 > rect.width) tx = e.clientX - rect.left - 230;
                if (ty + 140 > rect.height) ty = rect.height - 150;
                tip.style.left = tx + 'px';
                tip.style.top = ty + 'px';
                tip.style.display = 'block';
            },

            showDetails(nodeKey) {
                const panel = document.getElementById('netDetailPanel');
                const domainEl = document.getElementById('netDetailDomain');
                const flagEl = document.getElementById('netDetailFlag');
                const countryEl = document.getElementById('netDetailCountry');
                const content = document.getElementById('netDetailContent');
                const actions = document.getElementById('netDetailActions');
                if (!this.nodes[nodeKey]) return;
                const n = this.nodes[nodeKey];
                const country = getCountryFromDomain(nodeKey);
                const flag = COUNTRY_FLAG[country] || '\ud83c\udf10';
                const countryName = COUNTRY_NAMES[country] || 'Unknown';
                domainEl.textContent = nodeKey;
                flagEl.textContent = flag;
                countryEl.innerHTML = `${countryName} \u2022 <span style="opacity:0.6;">${country}</span>`;
                const outgoing = this.edges.filter(e => e.source === nodeKey).map(e => e.target);
                const incoming = this.edges.filter(e => e.target === nodeKey).map(e => e.source);
                const statusLabel = n.isFirst ? 'Seed' : (n.imports > 0 ? 'Importing' : n.status === 'error' ? 'Error' : n.status === 'dead' ? 'Dead' : 'Active');
                const statusColors = {Seed:'#ef4444', Importing:'#22c55e', Active:'#6366f1', Error:'#f59e0b', Dead:'#6b7280'};
                const successRate = n.urlsCrawled > 0 ? Math.round((n.imports / n.urlsCrawled) * 100) : 0;
                const age = Math.round((Date.now() - n.firstSeen) / 1000);
                const ageStr = age > 3600 ? Math.round(age/3600)+'h '+Math.round((age%3600)/60)+'m' : age > 60 ? Math.round(age/60)+'m '+age%60+'s' : age+'s';
                let html = `<div style="margin-bottom:8px;padding:6px;background:rgba(99,102,241,0.08);border-radius:6px;">
                    <span style="color:${statusColors[statusLabel]||'#6366f1'};font-size:1.1rem;">\u25cf</span>
                    <strong style="color:${statusColors[statusLabel]||'#6366f1'};">${statusLabel}</strong>
                    ${n.lastStatus ? '<span style="opacity:0.5;margin-left:4px;">HTTP '+n.lastStatus+'</span>' : ''}
                    <div style="opacity:0.6;font-size:0.65rem;margin-top:2px;">Active for ${ageStr}</div></div>`;
                // Stats grid
                html += `<div style="display:grid;grid-template-columns:1fr 1fr;gap:4px;margin-bottom:8px;">
                    <div style="background:rgba(34,197,94,0.1);padding:4px 6px;border-radius:4px;text-align:center;">
                        <div style="font-size:1rem;font-weight:700;color:#22c55e;">${n.imports||0}</div><div style="font-size:0.58rem;opacity:0.7;">Imports</div></div>
                    <div style="background:rgba(99,102,241,0.1);padding:4px 6px;border-radius:4px;text-align:center;">
                        <div style="font-size:1rem;font-weight:700;color:#6366f1;">${n.urlsCrawled||0}</div><div style="font-size:0.58rem;opacity:0.7;">Crawled</div></div>
                    <div style="background:rgba(245,158,11,0.1);padding:4px 6px;border-radius:4px;text-align:center;">
                        <div style="font-size:1rem;font-weight:700;color:#f59e0b;">${n.errors||0}</div><div style="font-size:0.58rem;opacity:0.7;">Errors</div></div>
                    <div style="background:rgba(6,182,212,0.1);padding:4px 6px;border-radius:4px;text-align:center;">
                        <div style="font-size:1rem;font-weight:700;color:#06b6d4;">${n.linksFound||0}</div><div style="font-size:0.58rem;opacity:0.7;">Links</div></div></div>`;
                // Success rate bar
                html += `<div style="margin-bottom:8px;"><div style="display:flex;justify-content:space-between;font-size:0.62rem;margin-bottom:2px;">
                    <span>Success Rate</span><span style="font-weight:700;color:${successRate>50?'#22c55e':successRate>20?'#f59e0b':'#ef4444'};">${successRate}%</span></div>
                    <div style="height:4px;background:rgba(255,255,255,0.1);border-radius:2px;overflow:hidden;">
                    <div style="height:100%;width:${successRate}%;background:${successRate>50?'#22c55e':successRate>20?'#f59e0b':'#ef4444'};border-radius:2px;transition:width 0.3s;"></div></div></div>`;
                // Connections
                html += `<div style="font-size:0.65rem;margin-bottom:4px;opacity:0.7;">Connections: <strong>${n.connections}</strong> (${outgoing.length} out, ${incoming.length} in)</div>`;
                if (outgoing.length > 0) {
                    html += `<div style="border-top:1px solid var(--border);padding-top:6px;margin-top:6px;"><div style="font-weight:600;margin-bottom:4px;font-size:0.68rem;">\u2192 Outgoing (${outgoing.length}):</div>`;
                    outgoing.slice(0, 15).forEach(d => {
                        const df = COUNTRY_FLAG[getCountryFromDomain(d)] || '\ud83c\udf10';
                        const dn = this.nodes[d];
                        const imp = dn ? dn.imports : 0;
                        html += `<div style="display:flex;align-items:center;gap:4px;padding:1px 0;font-size:0.63rem;cursor:pointer;opacity:0.8;" onclick="networkGraph.focusNode('${d}')">
                            <span>${df}</span><span style="flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${d}</span>${imp>0?'<span style="color:#22c55e;font-weight:700;">'+imp+'</span>':''}</div>`;
                    });
                    if (outgoing.length > 15) html += `<div style="opacity:0.4;font-size:0.6rem;">+${outgoing.length-15} more</div>`;
                    html += `</div>`;
                }
                if (incoming.length > 0) {
                    html += `<div style="border-top:1px solid var(--border);padding-top:6px;margin-top:6px;"><div style="font-weight:600;margin-bottom:4px;font-size:0.68rem;">\u2190 Incoming (${incoming.length}):</div>`;
                    incoming.slice(0, 15).forEach(d => {
                        const df = COUNTRY_FLAG[getCountryFromDomain(d)] || '\ud83c\udf10';
                        html += `<div style="display:flex;align-items:center;gap:4px;padding:1px 0;font-size:0.63rem;cursor:pointer;opacity:0.8;" onclick="networkGraph.focusNode('${d}')">
                            <span>${df}</span><span style="flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${d}</span></div>`;
                    });
                    if (incoming.length > 15) html += `<div style="opacity:0.4;font-size:0.6rem;">+${incoming.length-15} more</div>`;
                    html += `</div>`;
                }
                content.innerHTML = html;
                // Actions
                actions.innerHTML = `<div style="display:flex;gap:5px;flex-wrap:wrap;">
                    <button onclick="window.open('https://'+networkGraph.selectedNode,'_blank')" style="background:#6366f1;border:none;color:#fff;padding:4px 8px;border-radius:4px;cursor:pointer;font-size:0.62rem;">Open Site \u2197</button>
                    <button onclick="networkGraph.focusNode('${nodeKey}')" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:4px 8px;border-radius:4px;cursor:pointer;font-size:0.62rem;">Focus \u25ce</button>
                    <button onclick="networkGraph.highlightConnections('${nodeKey}')" style="background:var(--bg-tertiary);border:1px solid var(--border);color:var(--text-secondary);padding:4px 8px;border-radius:4px;cursor:pointer;font-size:0.62rem;">Highlight</button></div>`;
                panel.style.display = 'block';
            },

            closeDetails() {
                document.getElementById('netDetailPanel').style.display = 'none';
                this.selectedNode = null;
                this.highlightedNodes = null;
            },

            focusNode(key) {
                if (!this.nodes[key]) return;
                this.selectedNode = key;
                this.scale = 2.5;
                this.offsetX = this.width/2 - this.nodes[key].x * this.scale;
                this.offsetY = this.height/2 - this.nodes[key].y * this.scale;
                this.showDetails(key);
            },

            highlightConnections(key) {
                const connected = new Set([key]);
                this.edges.forEach(e => {
                    if (e.source === key) connected.add(e.target);
                    if (e.target === key) connected.add(e.source);
                });
                this.highlightedNodes = connected;
                setTimeout(() => { this.highlightedNodes = null; }, 5000);
            },

            getNodeSize(node) {
                const tier = getNodeTier(node);
                const tierData = NODE_TIERS[tier];
                const base = tierData.size * 0.5;
                const connBonus = Math.min(node.connections * 0.4, 8);
                const crawlBonus = Math.min((node.urlsCrawled || 0) * 0.2, 6);
                return Math.min(base + connBonus + crawlBonus, tierData.size);
            },

            getNodeColor(node, key) {
                if (node.isFirst) return '#ef4444';
                if (node.status === 'dead') return '#4b5563';
                if (node.status === 'error') return '#f59e0b';
                const tier = getNodeTier(node);
                if (tier === 'supernova') return '#fbbf24';
                if (tier === 'star') return '#f59e0b';
                if (tier === 'planet') return '#22c55e';
                if (tier === 'moon') return '#06b6d4';
                // Asteroid: color by category
                const cat = node.category || getNodeCategory(key);
                if (cat !== 'general' && DOMAIN_CATEGORIES[cat]) return DOMAIN_CATEGORIES[cat].color;
                // Fallback: color by country
                const country = node.country || getCountryFromDomain(key);
                return COUNTRY_COLORS[country] || COSMIC_COLORS[Math.abs(key.split('').reduce((h,c)=>((h<<5)-h)+c.charCodeAt(0),0)) % COSMIC_COLORS.length];
            },

            createNode(domain, x, y, isFirst) {
                const country = getCountryFromDomain(domain);
                const category = getNodeCategory(domain);
                if (!this.countryStats[country]) this.countryStats[country] = 0;
                this.countryStats[country]++;
                if (!this.categoryStats) this.categoryStats = {};
                if (!this.categoryStats[category]) this.categoryStats[category] = 0;
                this.categoryStats[category]++;
                return {
                    x, y, vx: 0, vy: 0,
                    connections: 0,
                    isFirst,
                    label: domain.replace('www.', '').substring(0, 22),
                    fullDomain: domain,
                    country,
                    category,
                    urlsCrawled: 0,
                    imports: 0,
                    errors: 0,
                    lastStatus: 0,
                    status: 'active',
                    firstSeen: Date.now(),
                    linksFound: 0,
                    spawnTime: Date.now(),
                    explosionRadius: 0,
                    ring: 0
                };
            },

            addEdge(source, target) {
                const key = source + '>' + target;
                if (this.edgeSet.has(key)) return;
                this.edgeSet.add(key);
                if (!this.nodes[source]) {
                    const isFirst = this.nodeCount === 0;
                    this.nodes[source] = this.createNode(source,
                        this.width/2 + (Math.random()-0.5)*300,
                        this.height/2 + (Math.random()-0.5)*300, isFirst);
                    if (isFirst) this.firstNode = source;
                    this.nodeCount++;
                }
                if (!this.nodes[target]) {
                    const src = this.nodes[source];
                    // Place new nodes near their source with some spread
                    const angle = Math.random() * Math.PI * 2;
                    const dist = 60 + Math.random() * 80;
                    this.nodes[target] = this.createNode(target,
                        src.x + Math.cos(angle) * dist,
                        src.y + Math.sin(angle) * dist, false);
                    this.nodeCount++;
                }
                this.nodes[source].connections++;
                this.nodes[target].connections++;
                this.edges.push({source, target, time: Date.now()});
                // Spawn particles on new edge
                if (this.showParticles && this.particles.length < 100) {
                    this.particles.push({source, target, t: 0, speed: 0.005 + Math.random() * 0.01, color: this.getNodeColor(this.nodes[source], source)});
                }
                // Update country counter
                const cCount = Object.keys(this.countryStats).length;
                const el = document.getElementById('viralCountryCount');
                if (el) el.textContent = cCount;
            },

            simulate() {
                if (this.paused) return;
                const nodeKeys = Object.keys(this.nodes);
                if (nodeKeys.length < 2) return;
                const centerX = this.width / 2;
                const centerY = this.height / 2;
                // Repulsion + center gravity
                for (let i = 0; i < nodeKeys.length; i++) {
                    const nodeA = this.nodes[nodeKeys[i]];
                    nodeA.vx += (centerX - nodeA.x) * 0.0008;
                    nodeA.vy += (centerY - nodeA.y) * 0.0008;
                    for (let j = i + 1; j < nodeKeys.length; j++) {
                        const nodeB = this.nodes[nodeKeys[j]];
                        const dx = nodeA.x - nodeB.x;
                        const dy = nodeA.y - nodeB.y;
                        const dist = Math.sqrt(dx*dx + dy*dy) || 1;
                        // Stronger repulsion for closer nodes, weaker for far
                        const force = Math.min(800 / (dist*dist), 3);
                        nodeA.vx += dx/dist*force;
                        nodeA.vy += dy/dist*force;
                        nodeB.vx -= dx/dist*force;
                        nodeB.vy -= dy/dist*force;
                    }
                }
                // Spring force for edges (natural distance 100px)
                for (const edge of this.edges) {
                    const a = this.nodes[edge.source];
                    const b = this.nodes[edge.target];
                    if (!a || !b) continue;
                    const dx = b.x - a.x, dy = b.y - a.y;
                    const dist = Math.sqrt(dx*dx + dy*dy) || 1;
                    const force = (dist - 100) * 0.004;
                    a.vx += dx/dist*force; a.vy += dy/dist*force;
                    b.vx -= dx/dist*force; b.vy -= dy/dist*force;
                }
                // Country clustering: same country nodes attract slightly
                for (let i = 0; i < nodeKeys.length; i++) {
                    const nodeA = this.nodes[nodeKeys[i]];
                    for (let j = i + 1; j < nodeKeys.length; j++) {
                        const nodeB = this.nodes[nodeKeys[j]];
                        if (nodeA.country && nodeA.country === nodeB.country && nodeA.country !== 'INT') {
                            const dx = nodeB.x - nodeA.x, dy = nodeB.y - nodeA.y;
                            const dist = Math.sqrt(dx*dx + dy*dy) || 1;
                            if (dist > 50) {
                                const attract = 0.0003;
                                nodeA.vx += dx * attract; nodeA.vy += dy * attract;
                                nodeB.vx -= dx * attract; nodeB.vy -= dy * attract;
                            }
                        }
                    }
                }
                // Apply velocity with damping
                for (const key of nodeKeys) {
                    const node = this.nodes[key];
                    node.vx *= 0.88; node.vy *= 0.88;
                    node.x += node.vx; node.y += node.vy;
                    node.x = Math.max(30, Math.min(this.width-30, node.x));
                    node.y = Math.max(30, Math.min(this.height-30, node.y));
                }
                // Update particles
                if (this.showParticles) {
                    this.particles = this.particles.filter(p => {
                        p.t += p.speed;
                        return p.t < 1;
                    });
                }
            },

            render() {
                if (!this.ctx) return;
                const ctx = this.ctx;
                const w = this.width, h = this.height;
                ctx.setTransform(this.dpr, 0, 0, this.dpr, 0, 0);
                ctx.clearRect(0, 0, w, h);
                const now = Date.now();

                // GALAXY BACKGROUND: Starfield + nebula
                this.renderStarfield(ctx, w, h, now);

                ctx.save();
                ctx.translate(this.offsetX, this.offsetY);
                ctx.scale(this.scale, this.scale);
                const nodeKeys = Object.keys(this.nodes);
                const visibleNodes = new Set(nodeKeys.filter(k => this.isNodeVisible(k)));

                // NEBULA GLOW: Soft glow around high-tier node clusters
                for (const key of nodeKeys) {
                    const node = this.nodes[key];
                    if (!visibleNodes.has(key)) continue;
                    const tier = getNodeTier(node);
                    if (tier === 'supernova' || tier === 'star') {
                        const tierData = NODE_TIERS[tier];
                        const glowSize = tierData.size * 2.5;
                        const grad = ctx.createRadialGradient(node.x, node.y, 0, node.x, node.y, glowSize);
                        grad.addColorStop(0, tierData.glow);
                        grad.addColorStop(1, 'rgba(0,0,0,0)');
                        ctx.beginPath();
                        ctx.arc(node.x, node.y, glowSize, 0, Math.PI*2);
                        ctx.fillStyle = grad;
                        ctx.fill();
                    }
                }

                // Draw edges (with thickness by traffic)
                for (const edge of this.edges) {
                    const a = this.nodes[edge.source];
                    const b = this.nodes[edge.target];
                    if (!a || !b) continue;
                    const sourceVisible = visibleNodes.has(edge.source);
                    const targetVisible = visibleNodes.has(edge.target);
                    if (!sourceVisible && !targetVisible) continue;
                    const isSelected = (this.selectedNode === edge.source || this.selectedNode === edge.target);
                    const isHighlighted = this.highlightedNodes && (this.highlightedNodes.has(edge.source) || this.highlightedNodes.has(edge.target));
                    // Edge thickness based on node connections
                    const maxConn = Math.max(a.connections, b.connections);
                    const thickness = isSelected ? 2 : isHighlighted ? 1.5 : Math.min(0.4 + maxConn * 0.15, 2);
                    const alpha = isSelected ? 0.6 : isHighlighted ? 0.5 : Math.min(0.08 + maxConn * 0.03, 0.25);
                    // Color by source node
                    const edgeColor = isSelected || isHighlighted ? this.getNodeColor(a, edge.source) : '#6366f1';
                    ctx.strokeStyle = edgeColor;
                    ctx.globalAlpha = alpha;
                    ctx.lineWidth = thickness;
                    ctx.beginPath();
                    ctx.moveTo(a.x, a.y);
                    ctx.lineTo(b.x, b.y);
                    ctx.stroke();
                    ctx.globalAlpha = 1;
                    // Directional arrow for selected/highlighted edges
                    if (isSelected || isHighlighted) {
                        const angle = Math.atan2(b.y-a.y, b.x-a.x);
                        const arrowSize = 6;
                        const mx = a.x + (b.x-a.x)*0.65, my = a.y + (b.y-a.y)*0.65;
                        ctx.fillStyle = edgeColor;
                        ctx.globalAlpha = 0.7;
                        ctx.beginPath();
                        ctx.moveTo(mx + Math.cos(angle)*arrowSize, my + Math.sin(angle)*arrowSize);
                        ctx.lineTo(mx + Math.cos(angle+2.5)*arrowSize*0.6, my + Math.sin(angle+2.5)*arrowSize*0.6);
                        ctx.lineTo(mx + Math.cos(angle-2.5)*arrowSize*0.6, my + Math.sin(angle-2.5)*arrowSize*0.6);
                        ctx.fill();
                        ctx.globalAlpha = 1;
                    }
                }

                // Animated particles along edges
                if (this.showParticles) {
                    for (const p of this.particles) {
                        const a = this.nodes[p.source];
                        const b = this.nodes[p.target];
                        if (!a || !b) continue;
                        const px = a.x + (b.x - a.x) * p.t;
                        const py = a.y + (b.y - a.y) * p.t;
                        const pAlpha = p.t < 0.1 ? p.t*10 : p.t > 0.9 ? (1-p.t)*10 : 1;
                        ctx.beginPath();
                        ctx.arc(px, py, 2, 0, Math.PI * 2);
                        ctx.globalAlpha = pAlpha * 0.7;
                        ctx.fillStyle = p.color || '#6366f1';
                        ctx.fill();
                        ctx.globalAlpha = 1;
                    }
                }

                // Constellation lines (connect same-category nodes with subtle dashed lines)
                if (this.showLabels && nodeKeys.length < 200) {
                    const catGroups = {};
                    for (const key of nodeKeys) {
                        if (!visibleNodes.has(key)) continue;
                        const node = this.nodes[key];
                        const cat = getNodeCategory(node.domain || node.label);
                        if (cat === 'other') continue;
                        if (!catGroups[cat]) catGroups[cat] = [];
                        catGroups[cat].push(node);
                    }
                    ctx.save();
                    ctx.setLineDash([3, 6]);
                    ctx.lineWidth = 0.5;
                    for (const [cat, nodes] of Object.entries(catGroups)) {
                        if (nodes.length < 2 || nodes.length > 15) continue;
                        const catData = DOMAIN_CATEGORIES[cat];
                        const catColor = catData ? catData.color : '#6366f1';
                        ctx.strokeStyle = catColor;
                        ctx.globalAlpha = 0.12;
                        // Connect nearest neighbors within category (max 3 per node)
                        for (let i = 0; i < nodes.length; i++) {
                            const dists = [];
                            for (let j = 0; j < nodes.length; j++) {
                                if (i === j) continue;
                                const dx = nodes[i].x - nodes[j].x;
                                const dy = nodes[i].y - nodes[j].y;
                                dists.push({j, d: Math.sqrt(dx*dx + dy*dy)});
                            }
                            dists.sort((a,b) => a.d - b.d);
                            const nearest = dists.slice(0, 3);
                            for (const {j, d} of nearest) {
                                if (d > 250) continue;
                                ctx.beginPath();
                                ctx.moveTo(nodes[i].x, nodes[i].y);
                                ctx.lineTo(nodes[j].x, nodes[j].y);
                                ctx.stroke();
                            }
                        }
                    }
                    ctx.restore();
                }

                // Draw nodes
                for (const key of nodeKeys) {
                    const node = this.nodes[key];
                    if (!visibleNodes.has(key)) continue;
                    const rawSize = this.getNodeSize(node);
                    const color = this.getNodeColor(node, key);
                    const isHovered = (this.hoveredNode === key);
                    const isSelected = (this.selectedNode === key);
                    const isHighlighted = this.highlightedNodes && this.highlightedNodes.has(key);
                    const isDimmed = this.highlightedNodes && !this.highlightedNodes.has(key);

                    // Spawn animation
                    const age = now - (node.spawnTime || 0);
                    const spawnProgress = Math.min(age / 500, 1);
                    const spawnScale = spawnProgress < 1 ? (1 - Math.pow(1 - spawnProgress, 3)) : 1;
                    const size = rawSize * spawnScale;

                    if (isDimmed) ctx.globalAlpha = 0.2;

                    // Explosion ring on new node
                    if (age < 1000) {
                        const ringProgress = age / 1000;
                        const ringRadius = size + ringProgress * 30;
                        ctx.beginPath();
                        ctx.arc(node.x, node.y, ringRadius, 0, Math.PI * 2);
                        ctx.strokeStyle = color;
                        ctx.lineWidth = 2 * (1 - ringProgress);
                        ctx.globalAlpha = isDimmed ? 0.05 : 0.4 * (1 - ringProgress);
                        ctx.stroke();
                        ctx.globalAlpha = isDimmed ? 0.2 : 1;
                    }

                    // Pulse glow for importing nodes
                    if (node.imports > 0) {
                        const pulse = 0.2 + Math.sin(now/500 + node.x) * 0.1;
                        ctx.beginPath();
                        ctx.arc(node.x, node.y, size + 6, 0, Math.PI*2);
                        ctx.globalAlpha = isDimmed ? 0.02 : pulse;
                        ctx.fillStyle = '#22c55e';
                        ctx.fill();
                        ctx.globalAlpha = isDimmed ? 0.2 : 1;
                    } else if (node.urlsCrawled > 0 && node.status === 'active') {
                        const pulse = 0.1 + Math.sin(now/800 + node.x) * 0.05;
                        ctx.beginPath();
                        ctx.arc(node.x, node.y, size + 4, 0, Math.PI*2);
                        ctx.globalAlpha = isDimmed ? 0.02 : pulse;
                        ctx.fillStyle = color;
                        ctx.fill();
                        ctx.globalAlpha = isDimmed ? 0.2 : 1;
                    }

                    // Hover/selection/highlight ring
                    if (isHovered || isSelected || isHighlighted) {
                        ctx.beginPath();
                        ctx.arc(node.x, node.y, size + 5, 0, Math.PI*2);
                        ctx.strokeStyle = isSelected ? '#fff' : isHighlighted ? color : 'rgba(255,255,255,0.6)';
                        ctx.lineWidth = isSelected ? 2.5 : 1.5;
                        ctx.stroke();
                    }

                    // ORBITAL RINGS for high-tier nodes (rotating)
                    const tier = getNodeTier(node);
                    const tierData = NODE_TIERS[tier];
                    if (tierData.ring >= 2) {
                        // Double orbital ring for supernovas/stars
                        const orbitAngle = now / 3000 + node.x;
                        for (let r = 1; r <= tierData.ring; r++) {
                            const orbitRadius = size + 6 + r * 5;
                            ctx.beginPath();
                            ctx.ellipse(node.x, node.y, orbitRadius, orbitRadius * 0.4, orbitAngle + r * 0.5, 0, Math.PI * 2);
                            ctx.strokeStyle = tierData.glow.replace(/[\d.]+\)$/, (0.3/r) + ')');
                            ctx.lineWidth = 1;
                            ctx.stroke();
                            // Orbiting dot
                            const dotAngle = (now / (1000 + r*500)) % (Math.PI*2);
                            const dotX = node.x + Math.cos(dotAngle) * orbitRadius;
                            const dotY = node.y + Math.sin(dotAngle) * orbitRadius * 0.4;
                            ctx.beginPath();
                            ctx.arc(dotX, dotY, 1.5, 0, Math.PI*2);
                            ctx.fillStyle = tierData.color;
                            ctx.fill();
                        }
                    } else if (tierData.ring === 1) {
                        // Single ring for planets
                        const orbitRadius = size + 5;
                        ctx.beginPath();
                        ctx.ellipse(node.x, node.y, orbitRadius, orbitRadius * 0.35, now/4000 + node.y, 0, Math.PI*2);
                        ctx.strokeStyle = 'rgba(34,197,94,0.2)';
                        ctx.lineWidth = 0.8;
                        ctx.stroke();
                    }

                    // Node circle with enhanced gradient
                    const grad = ctx.createRadialGradient(node.x - size*0.3, node.y - size*0.3, 0, node.x, node.y, size);
                    grad.addColorStop(0, this.lightenColor(color, 50));
                    grad.addColorStop(0.7, color);
                    grad.addColorStop(1, this.darkenColor(color, 30));
                    ctx.beginPath();
                    ctx.arc(node.x, node.y, size, 0, Math.PI*2);
                    ctx.fillStyle = grad;
                    ctx.fill();
                    // Glowing border based on tier
                    ctx.strokeStyle = tier === 'supernova' ? 'rgba(251,191,36,0.6)' : tier === 'star' ? 'rgba(245,158,11,0.4)' : 'rgba(255,255,255,0.15)';
                    ctx.lineWidth = tier === 'supernova' ? 1.5 : tier === 'star' ? 1 : 0.5;
                    ctx.stroke();

                    // Import count badge
                    if (node.imports > 0 && size >= 7) {
                        ctx.font = 'bold ' + Math.max(7, size*0.6) + 'px Inter,sans-serif';
                        ctx.fillStyle = '#fff';
                        ctx.textAlign = 'center';
                        ctx.textBaseline = 'middle';
                        ctx.fillText(node.imports > 999 ? Math.round(node.imports/1000)+'k' : node.imports, node.x, node.y);
                    }

                    // Country flag + tier icon (above node)
                    if (size >= 8 && (isHovered || isSelected || node.imports > 2)) {
                        const flag = COUNTRY_FLAG[node.country] || '';
                        const tierIcon = tier === 'supernova' ? '\ud83c\udf1f' : tier === 'star' ? '\u2b50' : tier === 'planet' ? '\ud83e\ude90' : '';
                        ctx.font = Math.max(8, size*0.55) + 'px sans-serif';
                        ctx.textAlign = 'center';
                        ctx.textBaseline = 'bottom';
                        ctx.fillText((tierIcon||flag), node.x, node.y - size - 2);
                    }

                    // Label + category badge
                    if (this.showLabels || isHovered || isSelected || node.imports > 0 || node.connections > 2) {
                        const labelAlpha = (isHovered || isSelected) ? 0.95 : node.imports > 0 ? 0.85 : this.showLabels ? 0.5 : 0.4;
                        const fontSize = (isHovered || isSelected) ? 11 : node.imports > 0 ? 9 : 7;
                        ctx.font = ((isHovered || isSelected || node.imports > 0) ? 'bold ' : '') + fontSize + 'px Inter,sans-serif';
                        ctx.fillStyle = `rgba(255,255,255,${labelAlpha})`;
                        ctx.textAlign = 'center';
                        ctx.textBaseline = 'top';
                        ctx.fillText(node.label, node.x, node.y + size + 3);
                        // Category label under domain name
                        if ((isHovered || isSelected) && node.category && node.category !== 'general') {
                            const catInfo = DOMAIN_CATEGORIES[node.category];
                            if (catInfo) {
                                ctx.font = '7px Inter,sans-serif';
                                ctx.fillStyle = catInfo.color;
                                ctx.fillText(catInfo.label, node.x, node.y + size + 14);
                            }
                        }
                        // Tier label for high-value nodes
                        if (tier !== 'asteroid' && tier !== 'moon' && !isHovered && !isSelected) {
                            ctx.font = '6px Inter,sans-serif';
                            ctx.fillStyle = `rgba(255,255,255,0.35)`;
                            ctx.fillText(tierData.label, node.x, node.y + size + 13);
                        }
                    }

                    if (isDimmed) ctx.globalAlpha = 1;
                }

                ctx.restore();

                // Update stats overlay (HTML element for better rendering)
                if (nodeKeys.length > 0 && now - (this._lastStatsUpdate || 0) > 1000) {
                    this._lastStatsUpdate = now;
                    const allNodes = Object.values(this.nodes);
                    const imported = allNodes.filter(n => n.imports > 0).length;
                    const errors = allNodes.filter(n => n.errors > 0).length;
                    const totalImports = allNodes.reduce((s,n) => s + (n.imports||0), 0);
                    const totalCrawled = allNodes.reduce((s,n) => s + (n.crawled||0), 0);
                    const totalErrors = allNodes.reduce((s,n) => s + (n.errors||0), 0);

                    // Tier breakdown
                    const tierCounts = {supernova:0, star:0, planet:0, moon:0, asteroid:0};
                    allNodes.forEach(n => { tierCounts[getNodeTier(n)]++; });

                    // Category breakdown (top 5)
                    const catCounts = {};
                    allNodes.forEach(n => {
                        const cat = getNodeCategory(n.domain || n.label);
                        catCounts[cat] = (catCounts[cat]||0) + 1;
                    });
                    const topCats = Object.entries(catCounts).sort((a,b) => b[1]-a[1]).slice(0,5);

                    // Calculations
                    const successRate = totalCrawled > 0 ? ((totalImports / totalCrawled) * 100).toFixed(1) : '0.0';
                    const avgImportsPerNode = imported > 0 ? (totalImports / imported).toFixed(1) : '0';
                    const errorRate = totalCrawled > 0 ? ((totalErrors / totalCrawled) * 100).toFixed(1) : '0.0';
                    const density = nodeKeys.length > 1 ? ((this.edges.length * 2) / (nodeKeys.length * (nodeKeys.length-1)) * 100).toFixed(2) : '0';
                    const countryCt = Object.keys(this.countryStats).length;

                    const overlay = document.getElementById('netStatsOverlay');
                    if (overlay) {
                        overlay.innerHTML = `
                            <div style="font-weight:700;font-size:0.68rem;margin-bottom:4px;color:#fff;border-bottom:1px solid rgba(255,255,255,0.1);padding-bottom:3px;">🌌 Galaxy Overview</div>
                            <div style="display:grid;grid-template-columns:1fr 1fr;gap:1px 8px;margin-bottom:4px;">
                                <span>${nodeKeys.length} nodes</span><span>${this.edges.length} edges</span>
                                <span style="color:#22c55e;">${totalImports} imports</span><span style="color:#ef4444;">${totalErrors} errors</span>
                            </div>
                            <div style="border-top:1px solid rgba(255,255,255,0.06);padding-top:3px;margin-bottom:3px;">
                                <div style="font-weight:600;color:#fbbf24;font-size:0.58rem;">⚡ TIERS</div>
                                ${tierCounts.supernova > 0 ? `<div style="color:#fbbf24;">🌟 ${tierCounts.supernova} Supernova</div>` : ''}
                                ${tierCounts.star > 0 ? `<div style="color:#f59e0b;">⭐ ${tierCounts.star} Star</div>` : ''}
                                ${tierCounts.planet > 0 ? `<div style="color:#22c55e;">🪐 ${tierCounts.planet} Planet</div>` : ''}
                                ${tierCounts.moon > 0 ? `<div style="color:#06b6d4;">🌙 ${tierCounts.moon} Moon</div>` : ''}
                                <div style="color:#6366f1;">☄️ ${tierCounts.asteroid} Asteroid</div>
                            </div>
                            <div style="border-top:1px solid rgba(255,255,255,0.06);padding-top:3px;margin-bottom:3px;">
                                <div style="font-weight:600;color:#a855f7;font-size:0.58rem;">📂 TOP CATEGORIES</div>
                                ${topCats.map(([cat,ct]) => {
                                    const catData = DOMAIN_CATEGORIES[cat];
                                    return `<div style="color:${catData?catData.color:'#888'};">${catData?catData.label:cat} ${ct}</div>`;
                                }).join('')}
                            </div>
                            <div style="border-top:1px solid rgba(255,255,255,0.06);padding-top:3px;">
                                <div style="font-weight:600;color:#06b6d4;font-size:0.58rem;">📊 CALCULATIONS</div>
                                <div>Success: <span style="color:${parseFloat(successRate)>10?'#22c55e':'#f59e0b'};">${successRate}%</span></div>
                                <div>Avg/node: <span style="color:#8b5cf6;">${avgImportsPerNode}</span></div>
                                <div>Err rate: <span style="color:${parseFloat(errorRate)>30?'#ef4444':'#06b6d4'};">${errorRate}%</span></div>
                                <div>Density: <span style="color:#ec4899;">${density}%</span></div>
                                <div>🌍 ${countryCt} countries</div>
                            </div>`;
                    }
                    // Update header counters
                    const nc = document.getElementById('viralNodeCount');
                    const ec = document.getElementById('viralEdgeCount');
                    const ic = document.getElementById('viralImportingCount');
                    if (nc) nc.textContent = nodeKeys.length.toLocaleString();
                    if (ec) ec.textContent = this.edges.length.toLocaleString();
                    if (ic) ic.textContent = imported.toLocaleString();
                }

                // Render minimap
                this.renderMinimap(nodeKeys);
            },

            lightenColor(hex, percent) {
                const num = parseInt(hex.replace('#',''), 16);
                const r = Math.min(255, (num >> 16) + percent);
                const g = Math.min(255, ((num >> 8) & 0xff) + percent);
                const b = Math.min(255, (num & 0xff) + percent);
                return `rgb(${r},${g},${b})`;
            },

            darkenColor(hex, percent) {
                const num = parseInt(hex.replace('#',''), 16);
                const r = Math.max(0, (num >> 16) - percent);
                const g = Math.max(0, ((num >> 8) & 0xff) - percent);
                const b = Math.max(0, (num & 0xff) - percent);
                return `rgb(${r},${g},${b})`;
            },

            renderStarfield(ctx, w, h, now) {
                // Deep space gradient background
                const bgGrad = ctx.createRadialGradient(w/2, h/2, 0, w/2, h/2, Math.max(w,h)*0.7);
                bgGrad.addColorStop(0, 'rgba(15,10,30,0.3)');
                bgGrad.addColorStop(0.5, 'rgba(8,5,20,0.2)');
                bgGrad.addColorStop(1, 'rgba(3,2,10,0.1)');
                ctx.fillStyle = bgGrad;
                ctx.fillRect(0, 0, w, h);

                // Twinkling stars
                for (const star of STARS) {
                    const twinkle = Math.sin(now / (star.twinkle * 1000) + star.x * 100) * 0.3 + 0.7;
                    ctx.beginPath();
                    ctx.arc(star.x * w, star.y * h, star.size, 0, Math.PI * 2);
                    ctx.fillStyle = `rgba(255,255,255,${star.alpha * twinkle})`;
                    ctx.fill();
                }

                // Cosmic nebula clouds (very subtle, 3 clouds)
                const nebulas = [
                    {x: w*0.2, y: h*0.3, r: 120, color: 'rgba(99,102,241,0.015)'},
                    {x: w*0.7, y: h*0.6, r: 150, color: 'rgba(236,72,153,0.012)'},
                    {x: w*0.5, y: h*0.8, r: 100, color: 'rgba(34,197,94,0.01)'}
                ];
                for (const neb of nebulas) {
                    const nebGrad = ctx.createRadialGradient(neb.x, neb.y, 0, neb.x, neb.y, neb.r);
                    nebGrad.addColorStop(0, neb.color);
                    nebGrad.addColorStop(1, 'rgba(0,0,0,0)');
                    ctx.fillStyle = nebGrad;
                    ctx.beginPath();
                    ctx.arc(neb.x, neb.y, neb.r, 0, Math.PI*2);
                    ctx.fill();
                }
            },

            renderMinimap(nodeKeys) {
                if (!this.minimapCtx || nodeKeys.length === 0) return;
                const ctx = this.minimapCtx;
                const mw = 160, mh = 100;
                ctx.clearRect(0, 0, mw, mh);
                // Find bounds
                let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity;
                nodeKeys.forEach(k => { const n = this.nodes[k]; minX = Math.min(minX,n.x); maxX = Math.max(maxX,n.x); minY = Math.min(minY,n.y); maxY = Math.max(maxY,n.y); });
                const pw = (maxX - minX) || 1, ph = (maxY - minY) || 1;
                const scale = Math.min((mw-10)/pw, (mh-10)/ph);
                const ox = (mw - pw*scale)/2 - minX*scale;
                const oy = (mh - ph*scale)/2 - minY*scale;
                // Draw edges
                ctx.strokeStyle = 'rgba(99,102,241,0.15)';
                ctx.lineWidth = 0.5;
                for (const edge of this.edges) {
                    const a = this.nodes[edge.source], b = this.nodes[edge.target];
                    if (!a || !b) continue;
                    ctx.beginPath();
                    ctx.moveTo(a.x*scale+ox, a.y*scale+oy);
                    ctx.lineTo(b.x*scale+ox, b.y*scale+oy);
                    ctx.stroke();
                }
                // Draw nodes
                for (const key of nodeKeys) {
                    const n = this.nodes[key];
                    ctx.beginPath();
                    ctx.arc(n.x*scale+ox, n.y*scale+oy, Math.max(1.5, this.getNodeSize(n)*scale*0.3), 0, Math.PI*2);
                    ctx.fillStyle = this.getNodeColor(n, key);
                    ctx.fill();
                }
                // Draw viewport rectangle
                const vx = (-this.offsetX / this.scale) * scale + ox;
                const vy = (-this.offsetY / this.scale) * scale + oy;
                const vw = (this.width / this.scale) * scale;
                const vh = (this.height / this.scale) * scale;
                ctx.strokeStyle = 'rgba(255,255,255,0.4)';
                ctx.lineWidth = 1;
                ctx.strokeRect(vx, vy, vw, vh);
            },

            startAnimation() {
                const tick = () => {
                    this.simulate();
                    this.render();
                    this.animFrame = requestAnimationFrame(tick);
                };
                tick();
            },

            stop() {
                if (this.animFrame) cancelAnimationFrame(this.animFrame);
            },

            // Controls
            togglePause() {
                this.paused = !this.paused;
                document.getElementById('netPauseBtn').textContent = this.paused ? '\u25b6 Resume' : '\u23f8 Pause';
            },

            resetView() {
                this.scale = 1; this.offsetX = 0; this.offsetY = 0;
            },

            zoomIn() { this.scale = Math.min(8, this.scale * 1.3); },
            zoomOut() { this.scale = Math.max(0.2, this.scale * 0.7); },

            fitAll() {
                const keys = Object.keys(this.nodes);
                if (keys.length === 0) return;
                let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity;
                keys.forEach(k => { const n = this.nodes[k]; minX = Math.min(minX,n.x); maxX = Math.max(maxX,n.x); minY = Math.min(minY,n.y); maxY = Math.max(maxY,n.y); });
                const pw = maxX - minX + 80, ph = maxY - minY + 80;
                this.scale = Math.min(this.width / pw, this.height / ph, 4);
                this.offsetX = (this.width - pw*this.scale)/2 - minX*this.scale + 40*this.scale;
                this.offsetY = (this.height - ph*this.scale)/2 - minY*this.scale + 40*this.scale;
            },

            toggleFullscreen() {
                const container = document.getElementById('netGraphContainer');
                const wrap = document.getElementById('netCanvasWrap');
                this.isFullscreen = !this.isFullscreen;
                if (this.isFullscreen) {
                    container.style.cssText = 'position:fixed;top:0;left:0;right:0;bottom:0;z-index:9999;background:rgba(5,5,15,0.98);padding:10px;display:flex;gap:8px;';
                    wrap.querySelector('canvas#networkCanvas').style.height = '100%';
                    document.getElementById('netFullBtn').textContent = '\u2716 Exit';
                } else {
                    container.style.cssText = 'display:flex;gap:8px;position:relative;';
                    wrap.querySelector('canvas#networkCanvas').style.height = '600px';
                    document.getElementById('netFullBtn').textContent = '\u26f6 Fullscreen';
                }
                // Resize canvas
                setTimeout(() => {
                    const rect = this.canvas.getBoundingClientRect();
                    this.canvas.width = rect.width * this.dpr;
                    this.canvas.height = rect.height * this.dpr;
                    this.width = rect.width;
                    this.height = rect.height;
                }, 100);
            },

            toggleLabels() {
                this.showLabels = !this.showLabels;
                document.getElementById('netLabelsBtn').style.color = this.showLabels ? '#22c55e' : 'var(--text-secondary)';
            },

            toggleParticles() {
                this.showParticles = !this.showParticles;
                document.getElementById('netParticlesBtn').style.color = this.showParticles ? '#06b6d4' : 'var(--text-secondary)';
            },

            setFilter(f) {
                this.filter = f;
                document.querySelectorAll('.net-filter-btn').forEach(btn => {
                    const isActive = btn.dataset.filter === f;
                    btn.style.background = isActive ? '#6366f1' : 'var(--bg-tertiary)';
                    btn.style.color = isActive ? '#fff' : btn.style.color;
                    btn.style.borderColor = isActive ? '#6366f1' : 'var(--border)';
                });
            },

            searchNodes(query) {
                this.searchQuery = query.trim();
                if (this.searchQuery && this.nodes[this.searchQuery]) {
                    this.focusNode(this.searchQuery);
                }
            },

            exportPNG() {
                const link = document.createElement('a');
                link.download = 'network-map-' + new Date().toISOString().slice(0,19).replace(/:/g,'-') + '.png';
                link.href = this.canvas.toDataURL('image/png');
                link.click();
            }
        };

        function toggleAdvanced() {
            const panel = document.getElementById('advancedOptions');
            const arrow = document.getElementById('advancedArrow');
            panel.classList.toggle('show');
            arrow.textContent = panel.classList.contains('show') ? '▲' : '▼';
        }

        // Chip toggle
        document.querySelectorAll('.chip').forEach(chip => {
            chip.addEventListener('click', function(e) {
                if (e.target.tagName !== 'INPUT') {
                    const checkbox = this.querySelector('input');
                    checkbox.checked = !checkbox.checked;
                }
                this.classList.toggle('active', this.querySelector('input').checked);
            });
        });

        // Load stats
        fetch('/Flowb0t_DCI/v2/public/api/v1/dashboard/stats')
            .then(r => r.json())
            .then(data => {
                if (data.success) {
                    document.getElementById('totalPinfeeds').textContent = data.data.total_urls || 0;
                    document.getElementById('todayImports').textContent = data.data.today || 0;
                }
            })
            .catch(() => {
                document.getElementById('totalPinfeeds').textContent = '?';
                document.getElementById('todayImports').textContent = '?';
            });

        // ========================================
        // MULTI-PROCESS MANAGER (MPManager)
        // ========================================
        const MPManager = {
            sse: null,
            running: false,
            startTime: null,
            elapsedTimer: null,
            processes: {},
            logEntries: [],

            init() {
                if (this.running) return;
                this.running = true;
                this.startTime = Date.now();
                this.connectSSE();
                this.elapsedTimer = setInterval(() => this.updateElapsed(), 1000);
                this.refreshList();
            },

            stop() {
                this.running = false;
                if (this.sse) { this.sse.close(); this.sse = null; }
                if (this.elapsedTimer) { clearInterval(this.elapsedTimer); this.elapsedTimer = null; }
            },

            connectSSE() {
                if (this.sse) this.sse.close();
                this.sse = new EventSource('?action=multi_sse');
                this.sse.onmessage = (e) => {
                    try {
                        const data = JSON.parse(e.data);
                        this.updateDashboard(data);
                    } catch(err) {}
                };
                this.sse.onerror = () => {
                    if (this.running) {
                        setTimeout(() => this.connectSSE(), 3000);
                    }
                };
            },

            updateDashboard(data) {
                // Aggregate stats
                document.getElementById('mpTotalProcesses').textContent = data.total_running || 0;
                document.getElementById('mpTotalImported').textContent = this.formatNum(data.total_imported || 0);
                document.getElementById('mpTotalDiscovered').textContent = this.formatNum(data.total_discovered || 0);
                document.getElementById('mpTotalDuplicates').textContent = this.formatNum(data.total_duplicates || 0);
                document.getElementById('mpTotalErrors').textContent = this.formatNum(data.total_errors || 0);
                document.getElementById('mpTotalRate').textContent = (data.total_rate || 0).toFixed(1) + '/s';

                // Pool stats
                if (data.pool) {
                    document.getElementById('mpPoolPending').textContent = this.formatNum(data.pool.pending || 0);
                    document.getElementById('mpPoolClaimed').textContent = this.formatNum(data.pool.claimed || 0);
                }
                if (data.seen_total !== undefined) {
                    document.getElementById('mpSeenTotal').textContent = this.formatNum(data.seen_total);
                }

                // Process count
                const procs = data.processes || [];
                const activeCount = procs.filter(p => p.status === 'running').length;
                document.getElementById('mpProcessCount').textContent = activeCount + ' active / ' + procs.length + ' total';

                // Render process list
                this.renderProcessList(procs);
            },

            renderProcessList(procs) {
                const container = document.getElementById('mpProcessList');
                if (!procs.length) {
                    container.innerHTML = '<div style="padding: 2rem; text-align: center; color: var(--text-secondary); font-size: 0.85rem;">No processes. Launch workers to start.</div>';
                    return;
                }

                const statusColors = {
                    running: '#22c55e', paused: '#f59e0b', stopped: '#ef4444',
                    completed: '#6366f1', dead: '#6b7280', pending: '#06b6d4'
                };
                const statusIcons = {
                    running: '●', paused: '⏸', stopped: '⏹',
                    completed: '✓', dead: '💀', pending: '○'
                };

                container.innerHTML = procs.map(p => {
                    const color = statusColors[p.status] || '#6b7280';
                    const icon = statusIcons[p.status] || '?';
                    const keywords = (p.keywords || '').substring(0, 50);
                    const rate = p.rate ? parseFloat(p.rate).toFixed(1) + '/s' : '0/s';
                    const logHtml = (p.recent_log || []).slice(0, 3).map(l =>
                        '<div style="opacity:0.7;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">' +
                        (l.status === 'imported' ? '✅' : l.status === 'error' ? '❌' : '⚪') +
                        ' ' + (l.url || '').substring(0, 60) + '</div>'
                    ).join('');

                    return `<div style="padding: 0.6rem 0.8rem; background: var(--bg-tertiary); border: 1px solid var(--border); border-radius: 0.5rem; border-left: 3px solid ${color};">
                        <div style="display: flex; justify-content: space-between; align-items: center;">
                            <div style="display: flex; align-items: center; gap: 0.5rem;">
                                <span style="color: ${color}; font-weight: 700;">${icon}</span>
                                <span style="font-size: 0.72rem; color: var(--text-secondary); font-family: monospace;">${p.id.substring(0,8)}</span>
                                <span style="font-size: 0.72rem; color: ${color}; font-weight: 600; text-transform: uppercase;">${p.status}</span>
                            </div>
                            <div style="display: flex; gap: 0.3rem;">
                                ${p.status === 'running' ? '<button onclick="MPManager.control(\'' + p.id + '\',\'pause\')" style="background:none;border:1px solid var(--border);color:#f59e0b;padding:2px 6px;border-radius:3px;cursor:pointer;font-size:0.65rem;">⏸</button>' : ''}
                                ${p.status === 'paused' ? '<button onclick="MPManager.control(\'' + p.id + '\',\'resume\')" style="background:none;border:1px solid var(--border);color:#22c55e;padding:2px 6px;border-radius:3px;cursor:pointer;font-size:0.65rem;">▶</button>' : ''}
                                ${(p.status === 'running' || p.status === 'paused') ? '<button onclick="MPManager.control(\'' + p.id + '\',\'stop\')" style="background:none;border:1px solid var(--border);color:#ef4444;padding:2px 6px;border-radius:3px;cursor:pointer;font-size:0.65rem;">⏹</button>' : ''}
                            </div>
                        </div>
                        <div style="display: flex; gap: 1rem; margin-top: 0.4rem; font-size: 0.72rem; color: var(--text-secondary);">
                            <span>📥 <strong style="color:var(--success);">${this.formatNum(p.imported || 0)}</strong></span>
                            <span>🔍 <strong style="color:var(--accent);">${this.formatNum(p.discovered || 0)}</strong></span>
                            <span>🔄 <strong>${this.formatNum(p.duplicates || 0)}</strong></span>
                            <span>⚡ <strong style="color:var(--pink);">${rate}</strong></span>
                            <span>🌊 <strong>${p.waves || 0}</strong></span>
                        </div>
                        <div style="margin-top: 0.3rem; font-size: 0.68rem; color: var(--text-secondary); white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">
                            🔑 ${keywords}${(p.keywords||'').length > 50 ? '...' : ''}
                        </div>
                        ${logHtml ? '<div style="margin-top: 0.3rem; font-size: 0.65rem; font-family: monospace; color: var(--text-secondary);">' + logHtml + '</div>' : ''}
                    </div>`;
                }).join('');
            },

            async launchBatch() {
                const keywords = document.getElementById('multiKeywords').value.trim();
                const seedUrls = document.getElementById('multiSeedUrls').value.trim();
                if (!keywords && !seedUrls) {
                    alert('Enter keywords or seed URLs');
                    return;
                }

                const numProc = parseInt(document.getElementById('multiNumProcesses').value) || 5;
                const btn = document.getElementById('startMultiBtn');
                btn.disabled = true;
                btn.textContent = '⏳ Launching...';

                // Show stats panel
                document.getElementById('multiStatsPanel').style.display = 'block';
                this.init();

                const config = {
                    seed_urls: seedUrls,
                    common_term: document.getElementById('multiCommonTerm').value.trim(),
                    forced_domains: document.getElementById('multiForcedDomains').value.trim(),
                    wave_size: parseInt(document.getElementById('multiWaveSize').value) || 10,
                    links_per_page: parseInt(document.getElementById('multiLinksPerPage').value) || 100,
                    max_pool: parseInt(document.getElementById('multiMaxPool').value) || 500000,
                    max_imports: parseInt(document.getElementById('multiMaxImports').value) || 10000000,
                    max_per_domain: 0 // Ilimitado por padrao no MULTI mode
                };

                let launched = 0;
                for (let i = 0; i < numProc; i++) {
                    try {
                        const resp = await fetch('?action=create_process', {
                            method: 'POST',
                            headers: {'Content-Type': 'application/json'},
                            body: JSON.stringify({ mode: 'viral', keywords, config })
                        });
                        const data = await resp.json();
                        if (data.success) {
                            launched++;
                            this.addLog('✅ Process ' + data.process_id.substring(0,8) + ' launched (' + launched + '/' + numProc + ')');
                        } else {
                            this.addLog('❌ Failed: ' + (data.error || 'unknown'));
                        }
                    } catch(err) {
                        this.addLog('❌ Error launching: ' + err.message);
                    }
                    // Small delay between launches
                    if (i < numProc - 1) await new Promise(r => setTimeout(r, 500));
                }

                btn.disabled = false;
                btn.textContent = '🚀 Launch Multi-Process Workers';
                this.addLog('🎉 Batch complete: ' + launched + '/' + numProc + ' launched');
            },

            async launchOne() {
                const keywords = document.getElementById('multiKeywords').value.trim();
                if (!keywords) { alert('Enter keywords first'); return; }

                const config = {
                    seed_urls: document.getElementById('multiSeedUrls').value.trim(),
                    common_term: document.getElementById('multiCommonTerm').value.trim(),
                    forced_domains: document.getElementById('multiForcedDomains').value.trim(),
                    wave_size: parseInt(document.getElementById('multiWaveSize').value) || 10,
                    links_per_page: parseInt(document.getElementById('multiLinksPerPage').value) || 100,
                    max_pool: parseInt(document.getElementById('multiMaxPool').value) || 500000,
                    max_imports: parseInt(document.getElementById('multiMaxImports').value) || 10000000,
                    max_per_domain: 0 // Ilimitado por padrao no MULTI mode
                };

                try {
                    const resp = await fetch('?action=create_process', {
                        method: 'POST',
                        headers: {'Content-Type': 'application/json'},
                        body: JSON.stringify({ mode: 'viral', keywords, config })
                    });
                    const data = await resp.json();
                    if (data.success) {
                        this.addLog('✅ New process: ' + data.process_id.substring(0,8));
                    } else {
                        this.addLog('❌ Failed: ' + (data.error || 'unknown'));
                    }
                } catch(err) {
                    this.addLog('❌ Error: ' + err.message);
                }
            },

            async control(processId, action) {
                try {
                    const resp = await fetch('?action=control_process', {
                        method: 'POST',
                        headers: {'Content-Type': 'application/json'},
                        body: JSON.stringify({ process_id: processId, control: action })
                    });
                    const data = await resp.json();
                    this.addLog((data.success ? '✅' : '❌') + ' ' + action + ' ' + processId.substring(0,8));
                } catch(err) {
                    this.addLog('❌ Control error: ' + err.message);
                }
            },

            async batchControl(action) {
                try {
                    const resp = await fetch('?action=batch_control', {
                        method: 'POST',
                        headers: {'Content-Type': 'application/json'},
                        body: JSON.stringify({ control: action })
                    });
                    const data = await resp.json();
                    this.addLog((data.success ? '✅' : '❌') + ' Batch: ' + action + (data.affected ? ' (' + data.affected + ' affected)' : ''));
                } catch(err) {
                    this.addLog('❌ Batch error: ' + err.message);
                }
            },

            async cleanup() {
                try {
                    const resp = await fetch('?action=cleanup_processes', { method: 'POST' });
                    const data = await resp.json();
                    this.addLog('🧹 Cleanup: ' + JSON.stringify(data.cleaned || {}));
                } catch(err) {
                    this.addLog('❌ Cleanup error: ' + err.message);
                }
            },

            async refreshList() {
                try {
                    const resp = await fetch('?action=list_processes');
                    const data = await resp.json();
                    if (data.success) {
                        this.renderProcessList(data.processes || []);
                    }
                } catch(err) {}
            },

            addLog(msg) {
                const now = new Date().toLocaleTimeString();
                this.logEntries.unshift({ time: now, msg });
                if (this.logEntries.length > 100) this.logEntries.length = 100;
                const container = document.getElementById('mpActivityLog');
                container.innerHTML = this.logEntries.map(l =>
                    '<div style="padding: 0 0.5rem;"><span style="color: var(--text-secondary);">[' + l.time + ']</span> ' + l.msg + '</div>'
                ).join('');
            },

            clearLog() {
                this.logEntries = [];
                document.getElementById('mpActivityLog').innerHTML = '<div style="color: var(--text-secondary); text-align: center;">Log cleared.</div>';
            },

            updateElapsed() {
                if (!this.startTime) return;
                const secs = Math.floor((Date.now() - this.startTime) / 1000);
                const h = Math.floor(secs / 3600);
                const m = Math.floor((secs % 3600) / 60);
                const s = secs % 60;
                document.getElementById('mpElapsed').textContent =
                    (h > 0 ? h + 'h ' : '') + (m > 0 ? m + 'm ' : '') + s + 's';
            },

            formatNum(n) {
                if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M';
                if (n >= 1000) return (n / 1000).toFixed(1) + 'K';
                return n.toString();
            }
        };

        // ===== ULTIMATE MANAGER =====
        const UltimateManager = {
            sse: null,
            running: false,
            startTime: null,
            elapsedTimer: null,
            processes: {},
            logEntries: [],
            logData: [],
            sparklines: {},
            milestoneReached: new Set(),
            maxImports: 10000000,

            init() {
                if (this.running) return;
                this.running = true;
                this.startTime = Date.now();
                this.maxImports = parseInt(document.getElementById('ultimateMaxImports')?.value) || 10000000;
                this.connectSSE();
                this.elapsedTimer = setInterval(() => this.updateElapsed(), 1000);
                // Init sparklines
                this.sparklines.pool = new Sparkline('ultSparkPool', '#f97316');
                this.sparklines.waves = new Sparkline('ultSparkWaves', '#8b5cf6');
                this.sparklines.imported = new Sparkline('ultSparkImported', '#22c55e');
                this.sparklines.rate = new Sparkline('ultSparkRate', '#ec4899');
                // Init Chart.js charts
                this.initCharts();
            },

            stop() {
                this.running = false;
                if (this.sse) { this.sse.close(); this.sse = null; }
                if (this.elapsedTimer) { clearInterval(this.elapsedTimer); this.elapsedTimer = null; }
            },

            connectSSE() {
                if (this.sse) this.sse.close();
                this.sse = new EventSource('?action=ultimate_sse');
                this.sse.onmessage = (e) => {
                    try { this.updateDashboard(JSON.parse(e.data)); } catch(err) {}
                };
                this.sse.onerror = () => {
                    if (this.running) setTimeout(() => this.connectSSE(), 3000);
                };
            },

            updateDashboard(data) {
                // 1. Aggregate stats
                const el = (id) => document.getElementById(id);
                el('ultPoolSize').textContent = this.fmtNum(data.pool?.pending || 0);
                el('ultWaves').textContent = this.fmtNum(data.processes?.reduce((s,p) => s + (parseInt(p.waves)||0), 0) || 0);
                el('ultImported').textContent = this.fmtNum(data.total_imported || 0);
                el('ultRate').textContent = (data.total_rate || 0).toFixed(1) + '/s';
                el('ultProcesses').textContent = data.total_running || 0;
                el('ultDiscovered').textContent = this.fmtNum(data.total_discovered || 0);
                el('ultDuplicates').textContent = this.fmtNum(data.total_duplicates || 0);
                el('ultErrors').textContent = this.fmtNum(data.total_errors || 0);
                el('ultNodes').textContent = data.total_nodes || 0;
                el('ultEdges').textContent = data.total_edges || 0;

                // 2. Sparklines
                this.sparklines.pool?.push(data.pool?.pending || 0);
                this.sparklines.waves?.push(data.processes?.reduce((s,p) => s + (parseInt(p.waves)||0), 0) || 0);
                this.sparklines.imported?.push(data.total_imported || 0);
                this.sparklines.rate?.push(data.total_rate || 0);

                // 3. Derived stats
                const processed = data.total_processed || 1;
                const imported = data.total_imported || 0;
                el('ultSuccessRate').textContent = Math.round((imported / Math.max(1, processed)) * 100) + '%';

                // 4. Pool info
                el('ultPoolPending').textContent = this.fmtNum(data.pool?.pending || 0);
                el('ultPoolClaimed').textContent = this.fmtNum(data.pool?.claimed || 0);
                el('ultSeenTotal').textContent = this.fmtNum(data.seen_total || 0);

                // 5. Pool health
                const poolPending = data.pool?.pending || 0;
                const errorRate = (data.total_errors || 0) / Math.max(1, processed);
                el('ultPoolHealth').textContent = poolPending > 100 && errorRate < 0.3 ? '🟢' : poolPending > 10 ? '🟡' : '🔴';

                // 6. ETA
                const rate = data.total_rate || 0;
                if (rate > 0 && imported < this.maxImports) {
                    const remaining = this.maxImports - imported;
                    const secs = remaining / rate;
                    if (secs > 86400) el('ultETA').textContent = '> 24h';
                    else {
                        const h = Math.floor(secs / 3600);
                        const m = Math.floor((secs % 3600) / 60);
                        el('ultETA').textContent = h + 'h ' + m + 'm';
                    }
                } else {
                    el('ultETA').textContent = rate > 0 ? 'Done' : '--';
                }

                // 7. Network Graph edges
                if (data.network && data.network.length > 0 && typeof networkGraph !== 'undefined') {
                    data.network.forEach(edge => {
                        if (edge[0] && edge[1]) networkGraph.addEdge(edge[0], edge[1]);
                    });
                }

                // 8. Domain stats → update node properties + leaderboard
                if (data.domain_stats && data.domain_stats.length > 0) {
                    this.updateLeaderboard(data.domain_stats);
                    // Update network graph nodes
                    if (typeof networkGraph !== 'undefined') {
                        data.domain_stats.forEach(d => {
                            const node = networkGraph.nodes?.[d.domain];
                            if (node) {
                                node.imports = parseInt(d.imported) || 0;
                                node.urlsCrawled = parseInt(d.crawled) || 0;
                                node.errors = parseInt(d.errors) || 0;
                                node.status = d.errors > 5 && d.imported == 0 ? 'dead' : d.errors > 2 ? 'error' : d.imported > 0 ? 'imported' : 'active';
                            }
                        });
                    }
                }

                // 9. Process list
                this.renderProcessList(data.processes || []);

                // 10. Activity log from process logs
                if (data.processes) {
                    data.processes.forEach(p => {
                        if (p.recent_log) {
                            p.recent_log.forEach(log => {
                                if (log.url && !this.logData.find(l => l.url === log.url)) {
                                    this.logData.push(log);
                                    if (this.logData.length > 2000) this.logData.shift();
                                }
                            });
                        }
                    });
                    this.renderLog();
                }

                // 11. Update Chart.js charts
                this.updateCharts(data);

                // 12. Milestones
                const milestones = [100, 500, 1000, 5000, 10000, 25000, 50000, 100000];
                milestones.forEach(m => {
                    if (imported >= m && !this.milestoneReached.has(m)) {
                        this.milestoneReached.add(m);
                        this.addLog('🎉 Milestone: ' + this.fmtNum(m) + ' imports!');
                    }
                });

                el('ultProcessCount').textContent = (data.total_running || 0) + ' active';
            },

            renderProcessList(procs) {
                const container = document.getElementById('ultProcessList');
                if (!procs.length) {
                    container.innerHTML = '<div style="padding:1rem;text-align:center;color:var(--text-secondary);font-size:0.8rem;">No workers running.</div>';
                    return;
                }
                container.innerHTML = procs.map(p => {
                    const statusColor = p.status === 'running' ? '#22c55e' : p.status === 'paused' ? '#f59e0b' : '#6b7280';
                    const elapsed = p.elapsed ? Math.floor(p.elapsed / 60) + 'm' : '0m';
                    return `<div style="background:var(--bg-tertiary);border:1px solid var(--border);border-radius:0.3rem;padding:0.4rem 0.6rem;display:flex;justify-content:space-between;align-items:center;">
                        <div style="display:flex;align-items:center;gap:0.4rem;">
                            <span style="color:${statusColor};font-size:0.7rem;">●</span>
                            <span style="font-size:0.7rem;color:var(--text-primary);font-family:monospace;">${p.id.substring(0,8)}</span>
                            <span style="font-size:0.65rem;color:var(--text-secondary);">${this.fmtNum(p.imported||0)} imp | ${elapsed}</span>
                        </div>
                        <div style="display:flex;gap:0.2rem;">
                            ${p.status==='running'?'<button onclick="UltimateManager.control(\''+p.id+'\',\'pause\')" style="background:none;border:none;cursor:pointer;font-size:0.7rem;">⏸</button>':''}
                            ${p.status==='paused'?'<button onclick="UltimateManager.control(\''+p.id+'\',\'resume\')" style="background:none;border:none;cursor:pointer;font-size:0.7rem;">▶</button>':''}
                            <button onclick="UltimateManager.control('${p.id}','stop')" style="background:none;border:none;cursor:pointer;font-size:0.7rem;">⏹</button>
                        </div>
                    </div>`;
                }).join('');
            },

            renderLog() {
                const container = document.getElementById('ultActivityLog');
                const recent = this.logData.slice(-100).reverse();
                if (!recent.length) return;
                container.innerHTML = recent.map(log => {
                    const icon = log.status === 'imported' ? '✅' : log.status === 'error' ? '❌' : log.status === 'duplicate' ? '🔄' : '⚪';
                    const color = log.status === 'imported' ? '#22c55e' : log.status === 'error' ? '#ef4444' : '#6b7280';
                    return `<div class="log-entry ${log.status||''}" data-domain="${log.domain||''}" style="display:flex;gap:0.3rem;align-items:baseline;padding:1px 0;">
                        <span>${icon}</span>
                        <span style="color:${color};min-width:55px;font-size:0.65rem;">${log.status||''}</span>
                        <span class="log-domain" style="color:#06b6d4;min-width:100px;font-size:0.65rem;">${log.domain||''}</span>
                        <span class="log-url" style="color:var(--text-secondary);flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:0.65rem;" title="${log.url||''}">${log.title||log.url||''}</span>
                    </div>`;
                }).join('');
                // Re-apply filter if active
                if (this.currentFilter) this.filterLog(this.currentFilter);
            },

            updateLeaderboard(domainStats) {
                const container = document.getElementById('ultLeaderboard');
                const top15 = domainStats.slice(0, 15);
                if (!top15.length) return;
                container.innerHTML = '<table style="width:100%;border-collapse:collapse;font-size:0.7rem;">' +
                    '<tr style="color:var(--text-secondary);"><th style="text-align:left;padding:2px 4px;">#</th><th style="text-align:left;padding:2px 4px;">Domain</th><th style="text-align:right;padding:2px 4px;">Imp</th><th style="text-align:right;padding:2px 4px;">Crawl</th><th style="text-align:right;padding:2px 4px;">Q</th></tr>' +
                    top15.map((d, i) => {
                        const medal = i === 0 ? '🥇' : i === 1 ? '🥈' : i === 2 ? '🥉' : (i+1);
                        const qColor = d.quality >= 70 ? '#22c55e' : d.quality >= 40 ? '#f59e0b' : '#ef4444';
                        return `<tr style="border-top:1px solid var(--border);"><td style="padding:2px 4px;">${medal}</td><td style="padding:2px 4px;color:#06b6d4;max-width:150px;overflow:hidden;text-overflow:ellipsis;">${d.domain}</td><td style="text-align:right;padding:2px 4px;color:#22c55e;">${d.imported||0}</td><td style="text-align:right;padding:2px 4px;">${d.crawled||0}</td><td style="text-align:right;padding:2px 4px;color:${qColor};">${d.quality||0}</td></tr>`;
                    }).join('') + '</table>';
            },

            async launchBatch() {
                const keywords = (document.getElementById('ultimateKeywords')?.value || '').trim();
                const seedUrls = (document.getElementById('ultimateSeedUrls')?.value || '').trim();
                const allKeywords = (keywords + '\n' + seedUrls).trim();
                if (!allKeywords) { alert('Enter keywords or seed URLs'); return; }

                const numProc = parseInt(document.getElementById('ultimateNumProcesses')?.value) || 5;
                const config = {
                    wave_size: parseInt(document.getElementById('ultimateWaveSize')?.value) || 25,
                    max_imports: parseInt(document.getElementById('ultimateMaxImports')?.value) || 10000000,
                    max_per_domain: parseInt(document.getElementById('ultimateMaxPerDomain')?.value) || 0,
                    links_per_page: parseInt(document.getElementById('ultimateLinksPerPage')?.value) || 150,
                    max_depth: parseInt(document.getElementById('ultimateMaxDepth')?.value) || 5,
                    quality_threshold: parseInt(document.getElementById('ultimateQualityThreshold')?.value) || 20,
                    relevance_threshold: parseInt(document.getElementById('ultimateRelevanceThreshold')?.value) || 2,
                    include_terms: (document.getElementById('ultimateIncludeTerms')?.value || '').trim(),
                    exclude_terms: (document.getElementById('ultimateExcludeTerms')?.value || '').trim(),
                    forced_domains: (document.getElementById('ultimateForcedDomains')?.value || '').trim(),
                    common_term: (document.getElementById('ultimateCommonTerm')?.value || '').trim(),
                    posts_first: document.getElementById('ultimatePostsFirst')?.checked ? '1' : '0',
                    follow_external: document.getElementById('ultimateFollowExternal')?.checked ? '1' : '0',
                    enable_pagination: document.getElementById('ultimateEnablePagination')?.checked ? '1' : '0',
                    pagination_pattern: (document.getElementById('ultimatePaginationPattern')?.value || '').trim(),
                    start_page: parseInt(document.getElementById('ultimateStartPage')?.value) || 1,
                    end_page: parseInt(document.getElementById('ultimateEndPage')?.value) || 50,
                    // v2.0 enhancements
                    url_pattern_filter: (document.getElementById('ultimateUrlPattern')?.value || '').trim(),
                    http_auth_user: (document.getElementById('ultimateAuthUser')?.value || '').trim(),
                    http_auth_pass: (document.getElementById('ultimateAuthPass')?.value || '').trim(),
                    fetch_delay_ms: parseInt(document.getElementById('ultimateFetchDelay')?.value) || 0,
                };
                this.maxImports = config.max_imports;

                if (!this.running) this.init();
                let launched = 0;
                for (let i = 0; i < numProc; i++) {
                    try {
                        const resp = await fetch('?action=create_process', {
                            method: 'POST',
                            headers: {'Content-Type': 'application/json'},
                            body: JSON.stringify({ mode: 'ultimate', keywords: allKeywords, config })
                        });
                        const data = await resp.json();
                        if (data.success) {
                            launched++;
                            this.addLog('⚡ Worker ' + data.process_id.substring(0,8) + ' launched');
                        } else {
                            this.addLog('❌ ' + (data.error || 'Unknown error'));
                        }
                    } catch(err) { this.addLog('❌ ' + err.message); }
                    if (i < numProc - 1) await new Promise(r => setTimeout(r, 500));
                }
                this.addLog('🚀 Launched ' + launched + '/' + numProc + ' ULTIMATE workers');
            },

            async launchOne() {
                const keywords = (document.getElementById('ultimateKeywords')?.value || '').trim();
                const seedUrls = (document.getElementById('ultimateSeedUrls')?.value || '').trim();
                const allKeywords = (keywords + '\n' + seedUrls).trim();
                if (!allKeywords) { alert('Enter keywords or seed URLs'); return; }
                const config = {
                    wave_size: parseInt(document.getElementById('ultimateWaveSize')?.value) || 25,
                    max_imports: parseInt(document.getElementById('ultimateMaxImports')?.value) || 10000000,
                    max_per_domain: parseInt(document.getElementById('ultimateMaxPerDomain')?.value) || 0,
                    links_per_page: parseInt(document.getElementById('ultimateLinksPerPage')?.value) || 150,
                    max_depth: parseInt(document.getElementById('ultimateMaxDepth')?.value) || 5,
                    quality_threshold: parseInt(document.getElementById('ultimateQualityThreshold')?.value) || 20,
                    relevance_threshold: parseInt(document.getElementById('ultimateRelevanceThreshold')?.value) || 2,
                };
                try {
                    const resp = await fetch('?action=create_process', {
                        method: 'POST', headers: {'Content-Type': 'application/json'},
                        body: JSON.stringify({ mode: 'ultimate', keywords: allKeywords, config })
                    });
                    const data = await resp.json();
                    if (data.success) this.addLog('⚡ Worker ' + data.process_id.substring(0,8) + ' launched');
                    else this.addLog('❌ ' + (data.error || 'Failed'));
                } catch(err) { this.addLog('❌ ' + err.message); }
            },

            async control(processId, action) {
                try {
                    await fetch('?action=control_process', {
                        method: 'POST', headers: {'Content-Type': 'application/json'},
                        body: JSON.stringify({ process_id: processId, control: action })
                    });
                    this.addLog((action === 'stop' ? '⏹' : action === 'pause' ? '⏸' : '▶') + ' ' + processId.substring(0,8));
                } catch(err) {}
            },

            async batchControl(action) {
                try {
                    const resp = await fetch('?action=batch_control', {
                        method: 'POST', headers: {'Content-Type': 'application/json'},
                        body: JSON.stringify({ control: action })
                    });
                    const data = await resp.json();
                    this.addLog('🔧 ' + action + ' → ' + (data.affected || 0) + ' affected');
                } catch(err) {}
            },

            async cleanup() {
                try {
                    await fetch('?action=cleanup_processes', { method: 'POST' });
                    this.addLog('🧹 Cleanup done');
                } catch(err) {}
            },

            exportResults(format) {
                if (!this.logData.length) { alert('No data to export'); return; }
                const imported = this.logData.filter(l => l.status === 'imported');
                let content, mime, ext;
                if (format === 'csv') {
                    content = 'URL,Status,Domain,Title\n' + this.logData.map(l =>
                        `"${(l.url||'').replace(/"/g,'""')}","${l.status||''}","${l.domain||''}","${(l.title||'').replace(/"/g,'""')}"`
                    ).join('\n');
                    mime = 'text/csv'; ext = 'csv';
                } else {
                    content = JSON.stringify({exported: new Date().toISOString(), total: this.logData.length, imported: imported.length, entries: this.logData}, null, 2);
                    mime = 'application/json'; ext = 'json';
                }
                const blob = new Blob([content], {type: mime});
                const a = document.createElement('a');
                a.href = URL.createObjectURL(blob);
                a.download = 'ultimate_results_' + new Date().toISOString().slice(0,10) + '.' + ext;
                a.click();
                URL.revokeObjectURL(a.href);
                this.addLog('📥 Exported ' + this.logData.length + ' entries as ' + ext.toUpperCase());
            },

            addLog(msg) {
                this.logEntries.unshift({time: new Date().toLocaleTimeString(), msg});
                if (this.logEntries.length > 100) this.logEntries.pop();
                // Also render visually in the activity log
                const container = document.getElementById('ultActivityLog');
                if (container) {
                    const div = document.createElement('div');
                    div.className = 'log-entry info';
                    div.style.cssText = 'padding:2px 4px;font-size:0.7rem;color:var(--text-primary);border-bottom:1px solid var(--border);';
                    div.innerHTML = '<span style="color:var(--text-secondary);margin-right:0.3rem;">' +
                        new Date().toLocaleTimeString().slice(0,5) + '</span>' + msg;
                    container.insertBefore(div, container.firstChild);
                }
            },

            clearLog() {
                this.logData = [];
                this.logEntries = [];
                document.getElementById('ultActivityLog').innerHTML = '<div style="color:var(--text-secondary);text-align:center;">Log cleared.</div>';
            },

            updateElapsed() {
                if (!this.startTime) return;
                const secs = Math.floor((Date.now() - this.startTime) / 1000);
                const h = Math.floor(secs / 3600);
                const m = Math.floor((secs % 3600) / 60);
                const s = secs % 60;
                const el = document.getElementById('ultElapsed');
                if (el) el.textContent = h.toString().padStart(2,'0') + ':' + m.toString().padStart(2,'0') + ':' + s.toString().padStart(2,'0');
            },

            // --- v2.0 Enhancement Properties ---
            charts: {},
            chartData: { progress: [], rate: [], types: [0,0,0,0] },
            chartUpdateCounter: 0,
            currentFilter: '',
            groupByDomain: false,

            // --- Chart.js Methods ---
            initCharts() {
                if (typeof Chart === 'undefined') return; // Chart.js not loaded
                const chartOpts = { responsive: true, maintainAspectRatio: false, animation: false, plugins: { legend: { display: false } }, scales: { x: { display: false }, y: { display: true, ticks: { font: { size: 9 }, color: '#6b7280' } } } };
                try {
                    this.charts.progress = new Chart(document.getElementById('ultChartProgress'), {
                        type: 'line', data: { labels: [], datasets: [{ data: [], borderColor: '#22c55e', borderWidth: 1.5, fill: true, backgroundColor: 'rgba(34,197,94,0.1)', pointRadius: 0, tension: 0.3 }] }, options: chartOpts
                    });
                    this.charts.domains = new Chart(document.getElementById('ultChartDomains'), {
                        type: 'bar', data: { labels: [], datasets: [{ data: [], backgroundColor: 'rgba(99,102,241,0.6)', borderRadius: 3 }] }, options: { ...chartOpts, indexAxis: 'y', scales: { x: { display: false }, y: { ticks: { font: { size: 8 }, color: '#a0a0b0' } } } }
                    });
                    this.charts.types = new Chart(document.getElementById('ultChartTypes'), {
                        type: 'doughnut', data: { labels: ['Imported','Skipped','Error','Duplicate'], datasets: [{ data: [0,0,0,0], backgroundColor: ['#22c55e','#eab308','#ef4444','#6b7280'], borderWidth: 0 }] }, options: { responsive: true, maintainAspectRatio: false, animation: false, plugins: { legend: { position: 'right', labels: { font: { size: 9 }, color: '#a0a0b0', boxWidth: 10 } } } }
                    });
                    this.charts.rate = new Chart(document.getElementById('ultChartRate'), {
                        type: 'line', data: { labels: [], datasets: [{ data: [], borderColor: '#6366f1', borderWidth: 1.5, fill: true, backgroundColor: 'rgba(99,102,241,0.1)', pointRadius: 0, tension: 0.3 }] }, options: chartOpts
                    });
                } catch(e) { console.warn('Chart init error:', e); }
            },

            updateCharts(data) {
                if (typeof Chart === 'undefined' || !this.charts.progress) return;
                if (++this.chartUpdateCounter % 3 !== 0) return; // Update every ~6s
                try {
                    // Progress chart
                    this.chartData.progress.push(data.total_imported || 0);
                    if (this.chartData.progress.length > 60) this.chartData.progress.shift();
                    this.charts.progress.data.labels = this.chartData.progress.map(() => '');
                    this.charts.progress.data.datasets[0].data = this.chartData.progress;
                    this.charts.progress.update('none');
                    // Rate chart
                    this.chartData.rate.push(data.total_rate || 0);
                    if (this.chartData.rate.length > 60) this.chartData.rate.shift();
                    this.charts.rate.data.labels = this.chartData.rate.map(() => '');
                    this.charts.rate.data.datasets[0].data = this.chartData.rate;
                    this.charts.rate.update('none');
                    // Types chart (doughnut)
                    const imp = data.total_imported || 0;
                    const err = data.total_errors || 0;
                    const dup = data.total_duplicates || 0;
                    const skip = Math.max(0, (data.total_processed || 0) - imp - err - dup);
                    this.charts.types.data.datasets[0].data = [imp, skip, err, dup];
                    this.charts.types.update('none');
                    // Domains chart (top 10)
                    if (data.domain_stats && data.domain_stats.length > 0) {
                        const top = data.domain_stats.slice(0, 10);
                        this.charts.domains.data.labels = top.map(d => (d.domain || '').substring(0, 20));
                        this.charts.domains.data.datasets[0].data = top.map(d => parseInt(d.imported) || 0);
                        this.charts.domains.update('none');
                    }
                } catch(e) {}
            },

            // --- Filter/Copy/Group Methods ---
            filterLog(query) {
                this.currentFilter = query;
                const container = document.getElementById('ultActivityLog');
                const entries = container.querySelectorAll('.log-entry');
                const q = (query || '').toLowerCase();
                entries.forEach(el => {
                    if (!q) { el.style.display = ''; return; }
                    el.style.display = el.textContent.toLowerCase().includes(q) ? '' : 'none';
                });
            },

            async copyResults(scope) {
                const entries = this.logData.filter(l => l.status === 'imported');
                let urls = entries.map(l => l.url || l.title || '');
                if (scope === 'filtered' && this.currentFilter) {
                    const q = this.currentFilter.toLowerCase();
                    urls = urls.filter(u => u.toLowerCase().includes(q));
                }
                const text = urls.join('\n');
                if (!text) { this.addLog('📋 Nothing to copy'); return; }
                try {
                    await navigator.clipboard.writeText(text);
                } catch(e) {
                    const ta = document.createElement('textarea');
                    ta.value = text; ta.style.position = 'fixed'; ta.style.left = '-9999px';
                    document.body.appendChild(ta); ta.select(); document.execCommand('copy');
                    document.body.removeChild(ta);
                }
                this.addLog('📋 Copied ' + urls.length + ' URLs to clipboard');
            },

            toggleGroupView() {
                this.groupByDomain = !this.groupByDomain;
                const btn = document.getElementById('ultGroupToggle');
                if (btn) btn.style.background = this.groupByDomain ? 'rgba(99,102,241,0.3)' : '';
                const container = document.getElementById('ultActivityLog');
                if (this.groupByDomain) {
                    const groups = {};
                    this.logData.forEach(l => {
                        const d = l.domain || 'unknown';
                        if (!groups[d]) groups[d] = [];
                        groups[d].push(l);
                    });
                    let html = '';
                    Object.keys(groups).sort((a,b) => groups[b].length - groups[a].length).forEach(d => {
                        const entries = groups[d];
                        const imported = entries.filter(e => e.status === 'imported').length;
                        html += `<div class="domain-group" style="margin:0.3rem 0;">
                            <div onclick="this.nextElementSibling.style.display=this.nextElementSibling.style.display==='none'?'block':'none'"
                                style="cursor:pointer;font-size:0.7rem;font-weight:600;padding:0.3rem 0.4rem;background:rgba(99,102,241,0.1);border-radius:0.2rem;display:flex;justify-content:space-between;">
                                <span style="color:#06b6d4;">${d}</span>
                                <span style="color:var(--text-secondary);">${imported}/${entries.length}</span>
                            </div>
                            <div style="padding-left:0.5rem;">` +
                            entries.map(l => {
                                const icon = l.status === 'imported' ? '✅' : l.status === 'error' ? '❌' : '⚪';
                                return `<div style="font-size:0.65rem;padding:1px 0;color:var(--text-secondary);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">${icon} ${l.title||l.url||''}</div>`;
                            }).join('') +
                            `</div></div>`;
                    });
                    container.innerHTML = html || '<div style="color:var(--text-secondary);text-align:center;">No data.</div>';
                } else {
                    this.renderLog();
                }
            },

            exportDomainResults() {
                if (!this.logData.length) { alert('No data to export'); return; }
                const groups = {};
                this.logData.forEach(l => {
                    const d = l.domain || 'unknown';
                    if (!groups[d]) groups[d] = [];
                    groups[d].push(l);
                });
                let csv = 'Domain,Status,URL,Title\n';
                Object.keys(groups).sort().forEach(d => {
                    groups[d].forEach(l => {
                        csv += `"${d}","${l.status||''}","${(l.url||'').replace(/"/g,'""')}","${(l.title||'').replace(/"/g,'""')}"\n`;
                    });
                });
                const blob = new Blob([csv], {type: 'text/csv'});
                const a = document.createElement('a');
                a.href = URL.createObjectURL(blob);
                a.download = 'ultimate_domains_' + Date.now() + '.csv';
                a.click();
                URL.revokeObjectURL(a.href);
                this.addLog('📥 Exported ' + Object.keys(groups).length + ' domains as CSV');
            },

            fmtNum(n) {
                if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M';
                if (n >= 1000) return (n / 1000).toFixed(1) + 'K';
                return n.toString();
            }
        };

        // --- Global Functions: Page Example Generator & Chart Switcher ---
        function generatePagesFromExample() {
            const example = (document.getElementById('ultimatePageExample')?.value || '').trim();
            const count = parseInt(document.getElementById('ultimatePageCount')?.value) || 20;
            if (!example) { alert('Enter an example URL containing a page number'); return; }
            // Find the LAST number in the URL (most likely the page number)
            const match = example.match(/^(.*?)(\d+)([^0-9]*)$/);
            if (!match) { alert('No page number detected in the URL'); return; }
            const prefix = match[1];
            const suffix = match[3];
            const urls = [];
            for (let i = 1; i <= count; i++) urls.push(prefix + i + suffix);
            // Add to seed URLs textarea
            const seedArea = document.getElementById('ultimateSeedUrls');
            if (seedArea) {
                const existing = seedArea.value.trim();
                seedArea.value = (existing ? existing + '\n' : '') + urls.join('\n');
            }
            // Show preview
            const preview = document.getElementById('generatedPagesPreview');
            if (preview) {
                preview.style.display = 'block';
                preview.innerHTML = '<strong>Generated ' + count + ' URLs:</strong><br>' + urls[0] + '<br>...<br>' + urls[urls.length-1];
            }
        }

        function switchChart(type, btn) {
            ['Progress','Domains','Types','Rate'].forEach(t => {
                const el = document.getElementById('ultChart' + t);
                if (el) el.style.display = (t.toLowerCase() === type) ? 'block' : 'none';
            });
            document.querySelectorAll('.chart-tab').forEach(b => b.classList.remove('active'));
            if (btn) btn.classList.add('active');
        }
        </script>
        <?php endif; ?>
    </div>
    <!-- U3: Toast notification container -->
    <div id="toastContainer" style="position: fixed; top: 20px; right: 20px; z-index: 9999; display: flex; flex-direction: column; gap: 0.5rem; pointer-events: none;"></div>

    <!-- B21: Floating Action Buttons -->
    <div id="fabContainer">
        <button type="button" class="fab-action" style="background:var(--purple);" onclick="toggleTheaterMode()" title="Theater Mode">⛶</button>
        <button type="button" class="fab-action" style="background:var(--success);" onclick="exportViralResults('csv')" title="Export CSV">📥</button>
        <button type="button" class="fab-action" style="background:var(--sky);" onclick="toggleSound()" title="Toggle Sound" id="fabSoundBtn">🔇</button>
        <button type="button" class="fab-action" style="background:var(--orange);" onclick="scrollToTop()" title="Scroll to Top">⬆️</button>
        <button type="button" class="fab-main" onclick="toggleFAB()" id="fabMainBtn">+</button>
    </div>

    <!-- B20: Keyboard Shortcuts Overlay -->
    <div id="shortcutsOverlay" onclick="if(event.target===this)closeShortcuts()">
        <div class="shortcuts-content">
            <h2>⌨️ Keyboard Shortcuts</h2>
            <div class="shortcut-row"><span>Start Crawling</span><span class="shortcut-key">Ctrl+Enter</span></div>
            <div class="shortcut-row"><span>Stop Crawling</span><span class="shortcut-key">Esc</span></div>
            <div class="shortcut-row"><span>Pause/Resume</span><span class="shortcut-key">Space</span></div>
            <div class="shortcut-row"><span>Export Results</span><span class="shortcut-key">Ctrl+E</span></div>
            <div class="shortcut-row"><span>Clear Log</span><span class="shortcut-key">Ctrl+L</span></div>
            <div class="shortcut-row"><span>Theater Mode</span><span class="shortcut-key">F11</span></div>
            <div class="shortcut-row"><span>Toggle Sound</span><span class="shortcut-key">M</span></div>
            <div class="shortcut-row"><span>Show Shortcuts</span><span class="shortcut-key">?</span></div>
            <div style="margin-top:1rem;text-align:center;color:var(--text-secondary);font-size:0.8rem;">Press any key to close</div>
        </div>
    </div>
</body>
</html>
