import axios from 'axios';
import * as cheerio from 'cheerio';
import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js';
export class ContentExtractor {
    defaultTimeout;
    maxContentLength;
    constructor() {
        this.defaultTimeout = 10000;
        // Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB
        const envMaxLength = process.env.MAX_CONTENT_LENGTH;
        this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000;
        // Validate the parsed value
        if (isNaN(this.maxContentLength) || this.maxContentLength < 0) {
            console.warn(`[ContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`);
            this.maxContentLength = 500000;
        }
    }
    async extractContent(options) {
        const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options;
        try {
            const response = await axios.get(url, {
                headers: {
                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                    'Accept-Language': 'en-US,en;q=0.9',
                    'Accept-Encoding': 'gzip, deflate, br',
                    'DNT': '1',
                    'Connection': 'keep-alive',
                    'Upgrade-Insecure-Requests': '1',
                    'Sec-Fetch-Dest': 'document',
                    'Sec-Fetch-Mode': 'navigate',
                    'Sec-Fetch-Site': 'none',
                    'Sec-Fetch-User': '?1',
                    'Cache-Control': 'max-age=0',
                    'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
                    'sec-ch-ua-mobile': '?0',
                    'sec-ch-ua-platform': '"macOS"',
                },
                timeout,
                maxContentLength,
                validateStatus: (status) => status < 400,
            });
            return this.parseContent(response.data);
        }
        catch (error) {
            console.error(`Content extraction error for ${url}:`, error);
            // If it's a 403 error, try with different headers
            if (axios.isAxiosError(error) && error.response?.status === 403) {
                console.log(`[ContentExtractor] Trying alternative headers for ${url}`);
                try {
                    const response = await axios.get(url, {
                        headers: {
                            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                            'Accept-Language': 'en-US,en;q=0.5',
                            'Accept-Encoding': 'gzip, deflate',
                            'DNT': '1',
                            'Connection': 'keep-alive',
                            'Upgrade-Insecure-Requests': '1',
                            'Cache-Control': 'no-cache',
                            'Pragma': 'no-cache',
                        },
                        timeout,
                        maxContentLength,
                        validateStatus: (status) => status < 400,
                    });
                    console.log(`[ContentExtractor] Alternative headers worked for ${url}`);
                    return this.parseContent(response.data);
                }
                catch (retryError) {
                    console.error(`[ContentExtractor] Alternative headers also failed for ${url}:`, retryError);
                }
            }
            throw new Error(`Failed to extract content from ${url}: ${this.getSpecificErrorMessage(error)}`);
        }
    }
    async extractContentForResults(results, targetCount = results.length) {
        const enhancedResults = [];
        let processedCount = 0;
        console.log(`[ContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`);
        for (const result of results) {
            if (enhancedResults.length >= targetCount) {
                console.log(`[ContentExtractor] Reached target count of ${targetCount} results`);
                break;
            }
            processedCount++;
            // Skip PDF files
            if (isPdfUrl(result.url)) {
                console.log(`[ContentExtractor] Skipping PDF file: ${result.url}`);
                continue;
            }
            try {
                console.log(`[ContentExtractor] Extracting content from: ${result.url}`);
                const content = await this.extractContent({ url: result.url });
                const cleanedContent = cleanText(content, this.maxContentLength);
                enhancedResults.push({
                    ...result,
                    fullContent: cleanedContent,
                    contentPreview: getContentPreview(cleanedContent),
                    wordCount: getWordCount(cleanedContent),
                    timestamp: generateTimestamp(),
                    fetchStatus: 'success',
                });
                console.log(`[ContentExtractor] Successfully extracted content (${enhancedResults.length}/${targetCount})`);
            }
            catch (error) {
                console.log(`[ContentExtractor] Failed to extract content from ${result.url}: ${error instanceof Error ? error.message : 'Unknown error'}`);
                enhancedResults.push({
                    ...result,
                    fullContent: '',
                    contentPreview: '',
                    wordCount: 0,
                    timestamp: generateTimestamp(),
                    fetchStatus: 'error',
                    error: this.getSpecificErrorMessage(error),
                });
            }
        }
        console.log(`[ContentExtractor] Processed ${processedCount} results, extracted ${enhancedResults.length} non-PDF results`);
        return enhancedResults;
    }
    parseContent(html) {
        const $ = cheerio.load(html);
        // Remove all script, style, and other non-content elements
        $('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove();
        // Remove navigation, header, footer, and other non-content elements
        $('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove();
        // Remove elements with common ad/tracking classes
        $('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove();
        // Remove elements with common non-content IDs
        $('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove();
        // Remove image-related elements and attributes
        $('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove();
        $('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove();
        $('[style*="background-image"]').remove();
        // Remove empty elements and whitespace-only elements
        $('*').each(function () {
            const $this = $(this);
            if ($this.children().length === 0 && $this.text().trim() === '') {
                $this.remove();
            }
        });
        // Try to find the main content area first
        let mainContent = '';
        // Priority selectors for main content
        const contentSelectors = [
            'article',
            'main',
            '[role="main"]',
            '.content',
            '.post-content',
            '.entry-content',
            '.article-content',
            '.story-content',
            '.news-content',
            '.main-content',
            '.page-content',
            '.text-content',
            '.body-content',
            '.copy',
            '.text',
            '.body'
        ];
        for (const selector of contentSelectors) {
            const $content = $(selector).first();
            if ($content.length > 0) {
                mainContent = $content.text().trim();
                if (mainContent.length > 100) { // Ensure we have substantial content
                    console.log(`[ContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`);
                    break;
                }
            }
        }
        // If no main content found, try body content
        if (!mainContent || mainContent.length < 100) {
            console.log(`[ContentExtractor] No main content found, using body content`);
            mainContent = $('body').text().trim();
        }
        // Clean up the text
        const cleanedContent = this.cleanTextContent(mainContent);
        return cleanText(cleanedContent, this.maxContentLength);
    }
    cleanTextContent(text) {
        // Remove excessive whitespace
        text = text.replace(/\s+/g, ' ');
        // Remove image-related text and data URLs
        text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data
        text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs
        text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions
        text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words
        text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text
        // Remove common non-content patterns
        text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, '');
        // Remove excessive line breaks and spacing
        text = text.replace(/\n\s*\n/g, '\n');
        text = text.replace(/\r\n/g, '\n');
        text = text.replace(/\r/g, '\n');
        // Remove leading/trailing whitespace
        text = text.trim();
        return text;
    }
    getSpecificErrorMessage(error) {
        if (axios.isAxiosError(error)) {
            if (error.code === 'ECONNABORTED') {
                return 'Request timeout';
            }
            if (error.response?.status === 403) {
                return '403 Forbidden - Access denied';
            }
            if (error.response?.status === 404) {
                return '404 Not found';
            }
            if (error.message.includes('maxContentLength')) {
                return 'Content too long';
            }
            if (error.response?.status) {
                return `HTTP ${error.response.status}: ${error.message}`;
            }
            return `Network error: ${error.message}`;
        }
        return error instanceof Error ? error.message : 'Unknown error';
    }
}
//# sourceMappingURL=content-extractor.js.map