import axios from 'axios';
import * as cheerio from 'cheerio';
import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js';
import { BrowserPool } from './browser-pool.js';
export class EnhancedContentExtractor {
    defaultTimeout;
    maxContentLength;
    browserPool;
    fallbackThreshold;
    constructor() {
        this.defaultTimeout = parseInt(process.env.DEFAULT_TIMEOUT || '6000', 10);
        // Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB
        const envMaxLength = process.env.MAX_CONTENT_LENGTH;
        this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000;
        // Validate the parsed value
        if (isNaN(this.maxContentLength) || this.maxContentLength < 0) {
            console.warn(`[EnhancedContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`);
            this.maxContentLength = 500000;
        }
        this.browserPool = new BrowserPool();
        this.fallbackThreshold = parseInt(process.env.BROWSER_FALLBACK_THRESHOLD || '3', 10);
        console.log(`[EnhancedContentExtractor] Configuration: timeout=${this.defaultTimeout}, maxContentLength=${this.maxContentLength}, fallbackThreshold=${this.fallbackThreshold}`);
    }
    async extractContent(options) {
        const { url } = options;
        console.log(`[EnhancedContentExtractor] Starting extraction for: ${url}`);
        // First, try with regular HTTP client (faster)
        try {
            const content = await this.extractWithAxios(options);
            console.log(`[EnhancedContentExtractor] Successfully extracted with axios: ${content.length} chars`);
            return content;
        }
        catch (error) {
            console.log(`[EnhancedContentExtractor] Axios failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
            // Check if this looks like a case where browser would help
            if (this.shouldUseBrowser(error, url)) {
                console.log(`[EnhancedContentExtractor] Falling back to headless browser for: ${url}`);
                try {
                    const content = await this.extractWithBrowser(options);
                    console.log(`[EnhancedContentExtractor] Successfully extracted with browser: ${content.length} chars`);
                    return content;
                }
                catch (browserError) {
                    console.error(`[EnhancedContentExtractor] Browser extraction also failed:`, browserError);
                    throw new Error(`Both axios and browser extraction failed for ${url}`);
                }
            }
            else {
                throw error;
            }
        }
    }
    async extractWithAxios(options) {
        const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options;
        const response = await axios.get(url, {
            headers: this.getRandomHeaders(),
            timeout,
            // Remove maxContentLength from axios config - handle truncation manually
            validateStatus: (status) => status < 400,
        });
        let content = this.parseContent(response.data);
        // Truncate content if it exceeds the limit (instead of axios throwing an error)
        if (maxContentLength && content.length > maxContentLength) {
            console.log(`[EnhancedContentExtractor] Content truncated from ${content.length} to ${maxContentLength} characters for ${url}`);
            content = content.substring(0, maxContentLength);
        }
        // Check if we got a meaningful response
        if (this.isLowQualityContent(content)) {
            throw new Error('Low quality content detected - likely bot detection');
        }
        return content;
    }
    async extractWithBrowser(options) {
        const { url, timeout = this.defaultTimeout } = options;
        const browser = await this.browserPool.getBrowser();
        const browserType = this.browserPool.getLastUsedBrowserType();
        try {
            // Create context options based on browser capabilities
            const baseContextOptions = {
                userAgent: this.getRandomUserAgent(),
                viewport: this.getRandomViewport(),
                locale: 'en-US',
                timezoneId: this.getRandomTimezone(),
                // Simulate real device characteristics
                deviceScaleFactor: Math.random() > 0.5 ? 1 : 2,
                hasTouch: Math.random() > 0.7,
            };
            // Firefox doesn't support isMobile option - check multiple ways to ensure detection
            const isFirefox = browserType === 'firefox' ||
                browserType.includes('firefox') ||
                browser.constructor.name.toLowerCase().includes('firefox');
            const contextOptions = isFirefox
                ? baseContextOptions
                : { ...baseContextOptions, isMobile: Math.random() > 0.8 };
            // Create a new context for each request (isolation)
            const context = await browser.newContext(contextOptions);
            // Add stealth scripts to avoid detection
            await context.addInitScript(() => {
                // Remove webdriver property
                Object.defineProperty(navigator, 'webdriver', {
                    get: () => undefined,
                });
                // Mock plugins
                Object.defineProperty(navigator, 'plugins', {
                    get: () => [1, 2, 3, 4, 5],
                });
                // Mock languages
                Object.defineProperty(navigator, 'languages', {
                    get: () => ['en-US', 'en'],
                });
                // Mock permissions
                const originalQuery = window.navigator.permissions.query;
                window.navigator.permissions.query = (parameters) => (parameters.name === 'notifications' ?
                    Promise.resolve({ state: 'default' }) :
                    originalQuery(parameters));
                // Remove automation indicators
                const windowWithChrome = window;
                if (windowWithChrome.chrome) {
                    delete windowWithChrome.chrome.app;
                    delete windowWithChrome.chrome.runtime;
                }
            });
            const page = await context.newPage();
            // Set up request interception to block unnecessary resources
            await page.route('**/*', (route) => {
                const resourceType = route.request().resourceType();
                // Block images, fonts, and other non-essential resources for faster loading
                if (['image', 'font', 'media'].includes(resourceType)) {
                    route.abort();
                }
                else {
                    route.continue();
                }
            });
            // Navigate with realistic options and better error handling
            console.log(`[BrowserExtractor] Navigating to ${url}`);
            try {
                await page.goto(url, {
                    waitUntil: 'domcontentloaded', // Don't wait for all resources
                    timeout: Math.min(timeout, 8000) // Reduced timeout, max 8 seconds
                });
            }
            catch (gotoError) {
                // Handle specific protocol errors
                const errorMessage = gotoError instanceof Error ? gotoError.message : String(gotoError);
                if (errorMessage.includes('ERR_HTTP2_PROTOCOL_ERROR') || errorMessage.includes('HTTP2')) {
                    console.log(`[BrowserExtractor] HTTP/2 error detected, trying with HTTP/1.1`);
                    // Create a new context with HTTP/1.1 preference
                    await context.close();
                    const http1Context = await browser.newContext({
                        userAgent: this.getRandomUserAgent(),
                        viewport: this.getRandomViewport(),
                        locale: 'en-US',
                        timezoneId: this.getRandomTimezone(),
                        extraHTTPHeaders: {
                            'Connection': 'keep-alive',
                            'Upgrade-Insecure-Requests': '1'
                        }
                    });
                    const http1Page = await http1Context.newPage();
                    // Disable HTTP/2 by intercepting requests
                    await http1Page.route('**/*', (route) => {
                        const resourceType = route.request().resourceType();
                        if (['image', 'font', 'media'].includes(resourceType)) {
                            route.abort();
                        }
                        else {
                            route.continue();
                        }
                    });
                    await http1Page.goto(url, {
                        waitUntil: 'domcontentloaded',
                        timeout: Math.min(timeout, 6000)
                    });
                    // Quick content extraction
                    const html = await http1Page.content();
                    const content = this.parseContent(html);
                    await http1Context.close();
                    return content;
                }
                else {
                    throw gotoError;
                }
            }
            // Quick human simulation - reduced time
            await page.mouse.move(Math.random() * 100, Math.random() * 100);
            // Reduced wait time for dynamic content
            await page.waitForTimeout(500 + Math.random() * 1000);
            // Quick check for main content without long wait
            try {
                await page.waitForSelector('article, main, .content, .post-content, .entry-content', {
                    timeout: 2000
                });
            }
            catch {
                console.log(`[BrowserExtractor] No main content selector found, proceeding anyway`);
            }
            // Extract content using the same logic as axios version
            const html = await page.content();
            const content = this.parseContent(html);
            await context.close();
            return content;
        }
        catch (error) {
            console.error(`[BrowserExtractor] Browser extraction failed for ${url}:`, error);
            throw error;
        }
    }
    async simulateHumanBehavior(page) {
        try {
            // Random mouse movements
            await page.mouse.move(Math.random() * 800, Math.random() * 600);
            // Random scroll (common human behavior)
            const scrollY = Math.random() * 500;
            await page.evaluate((y) => window.scrollTo(0, y), scrollY);
            // Small random delay
            await page.waitForTimeout(500 + Math.random() * 1000);
            // Sometimes click somewhere innocuous
            if (Math.random() > 0.7) {
                try {
                    await page.click('body', { timeout: 1000 });
                }
                catch {
                    // Ignore click failures
                }
            }
        }
        catch {
            // Ignore simulation errors
            console.log(`[BrowserExtractor] Behavior simulation failed, continuing`);
        }
    }
    shouldUseBrowser(error, url) {
        // Conditions where browser is likely to succeed where axios failed
        const indicators = [
            // HTTP status codes that suggest bot detection
            error.response?.status === 403,
            error.response?.status === 429,
            error.response?.status === 503,
            // Error messages suggesting JS requirement
            error.message?.includes('timeout'),
            error.message?.includes('Access denied'),
            error.message?.includes('Forbidden'),
            error.message?.includes('Low quality content detected'),
            // Response content suggesting bot detection
            error.response?.data?.includes('Please enable JavaScript'),
            error.response?.data?.includes('captcha'),
            error.response?.data?.includes('unusual traffic'),
            error.response?.data?.includes('robot'),
            // Sites known to be JS-heavy
            url.includes('twitter.com'),
            url.includes('facebook.com'),
            url.includes('instagram.com'),
            url.includes('linkedin.com'),
            url.includes('reddit.com'),
            url.includes('medium.com'),
        ];
        return indicators.some(indicator => indicator === true);
    }
    isLowQualityContent(content) {
        const lowQualityIndicators = [
            content.length < 100,
            content.includes('Please enable JavaScript'),
            content.includes('Access Denied'),
            content.includes('403 Forbidden'),
            content.includes('captcha'),
            content.includes('unusual traffic'),
            content.includes('robot'),
            content.trim() === '',
        ];
        return lowQualityIndicators.some(indicator => indicator === true);
    }
    getRandomHeaders() {
        const browsers = [
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
                'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
                'sec-ch-ua-platform': '"Windows"',
            },
            {
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
                'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
                'sec-ch-ua-platform': '"macOS"',
            },
            {
                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
                'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
                'sec-ch-ua-platform': '"Linux"',
            }
        ];
        const browser = browsers[Math.floor(Math.random() * browsers.length)];
        return {
            ...browser,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'DNT': '1',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Cache-Control': 'max-age=0',
            'sec-ch-ua-mobile': '?0',
        };
    }
    getRandomUserAgent() {
        const userAgents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0',
        ];
        return userAgents[Math.floor(Math.random() * userAgents.length)];
    }
    getRandomViewport() {
        const viewports = [
            { width: 1920, height: 1080 },
            { width: 1366, height: 768 },
            { width: 1440, height: 900 },
            { width: 1536, height: 864 },
            { width: 1280, height: 720 },
        ];
        return viewports[Math.floor(Math.random() * viewports.length)];
    }
    getRandomTimezone() {
        const timezones = [
            'America/New_York',
            'America/Los_Angeles',
            'America/Chicago',
            'Europe/London',
            'Europe/Berlin',
            'Asia/Tokyo',
        ];
        return timezones[Math.floor(Math.random() * timezones.length)];
    }
    async extractContentForResults(results, targetCount = results.length) {
        console.log(`[EnhancedContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`);
        // Filter out PDF files first
        const nonPdfResults = results.filter(result => !isPdfUrl(result.url));
        const resultsToProcess = nonPdfResults.slice(0, Math.min(targetCount * 2, 10)); // Process extra to account for failures
        console.log(`[EnhancedContentExtractor] Processing ${resultsToProcess.length} non-PDF results concurrently`);
        // Process results concurrently with timeout
        const extractionPromises = resultsToProcess.map(async (result) => {
            try {
                // Use a race condition with timeout to prevent hanging
                const extractionPromise = this.extractContent({
                    url: result.url,
                    timeout: 6000 // Reduced timeout to 6 seconds per page
                });
                const timeoutPromise = new Promise((_, reject) => {
                    setTimeout(() => reject(new Error('Content extraction timeout')), 8000);
                });
                const content = await Promise.race([extractionPromise, timeoutPromise]);
                const cleanedContent = cleanText(content, this.maxContentLength);
                console.log(`[EnhancedContentExtractor] Successfully extracted: ${result.url}`);
                return {
                    ...result,
                    fullContent: cleanedContent,
                    contentPreview: getContentPreview(cleanedContent),
                    wordCount: getWordCount(cleanedContent),
                    timestamp: generateTimestamp(),
                    fetchStatus: 'success',
                };
            }
            catch (error) {
                console.log(`[EnhancedContentExtractor] Failed to extract: ${result.url} - ${error instanceof Error ? error.message : 'Unknown error'}`);
                return {
                    ...result,
                    fullContent: '',
                    contentPreview: '',
                    wordCount: 0,
                    timestamp: generateTimestamp(),
                    fetchStatus: 'error',
                    error: this.getSpecificErrorMessage(error),
                };
            }
        });
        // Wait for all extractions to complete
        const allResults = await Promise.all(extractionPromises);
        // Return successful results first, up to targetCount
        const successfulResults = allResults.filter(r => r.fetchStatus === 'success');
        const failedResults = allResults.filter(r => r.fetchStatus === 'error');
        // Combine successful and failed results, prioritizing successful ones
        const enhancedResults = [
            ...successfulResults.slice(0, targetCount),
            ...failedResults.slice(0, Math.max(0, targetCount - successfulResults.length))
        ].slice(0, targetCount);
        console.log(`[EnhancedContentExtractor] Completed processing ${resultsToProcess.length} results, extracted ${successfulResults.length} successful/${failedResults.length} failed`);
        return enhancedResults;
    }
    parseContent(html) {
        const $ = cheerio.load(html);
        // Remove all script, style, and other non-content elements
        $('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove();
        // Remove navigation, header, footer, and other non-content elements
        $('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove();
        // Remove elements with common ad/tracking classes
        $('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove();
        // Remove elements with common non-content IDs
        $('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove();
        // Remove image-related elements and attributes
        $('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove();
        $('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove();
        $('[style*="background-image"]').remove();
        // Remove empty elements and whitespace-only elements
        $('*').each(function () {
            const $this = $(this);
            if ($this.children().length === 0 && $this.text().trim() === '') {
                $this.remove();
            }
        });
        // Try to find the main content area first
        let mainContent = '';
        // Priority selectors for main content
        const contentSelectors = [
            'article',
            'main',
            '[role="main"]',
            '.content',
            '.post-content',
            '.entry-content',
            '.article-content',
            '.story-content',
            '.news-content',
            '.main-content',
            '.page-content',
            '.text-content',
            '.body-content',
            '.copy',
            '.text',
            '.body'
        ];
        for (const selector of contentSelectors) {
            const $content = $(selector).first();
            if ($content.length > 0) {
                mainContent = $content.text().trim();
                if (mainContent.length > 100) { // Ensure we have substantial content
                    console.log(`[EnhancedContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`);
                    break;
                }
            }
        }
        // If no main content found, try body content
        if (!mainContent || mainContent.length < 100) {
            console.log(`[EnhancedContentExtractor] No main content found, using body content`);
            mainContent = $('body').text().trim();
        }
        // Clean up the text
        const cleanedContent = this.cleanTextContent(mainContent);
        return cleanText(cleanedContent, this.maxContentLength);
    }
    cleanTextContent(text) {
        // Remove excessive whitespace
        text = text.replace(/\s+/g, ' ');
        // Remove image-related text and data URLs
        text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data
        text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs
        text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions
        text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words
        text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text
        // Remove common non-content patterns
        text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, '');
        // Remove excessive line breaks and spacing
        text = text.replace(/\n\s*\n/g, '\n');
        text = text.replace(/\r\n/g, '\n');
        text = text.replace(/\r/g, '\n');
        // Remove leading/trailing whitespace
        text = text.trim();
        return text;
    }
    getSpecificErrorMessage(error) {
        if (axios.isAxiosError(error)) {
            if (error.code === 'ECONNABORTED') {
                return 'Request timeout';
            }
            if (error.response?.status === 403) {
                return '403 Forbidden - Access denied';
            }
            if (error.response?.status === 404) {
                return '404 Not found';
            }
            if (error.message.includes('maxContentLength')) {
                return 'Content too long';
            }
            if (error.response?.status) {
                return `HTTP ${error.response.status}: ${error.message}`;
            }
            return `Network error: ${error.message}`;
        }
        return error instanceof Error ? error.message : 'Unknown error';
    }
    async closeAll() {
        await this.browserPool.closeAll();
    }
}
//# sourceMappingURL=enhanced-content-extractor.js.map