224 lines
12 KiB
JavaScript
224 lines
12 KiB
JavaScript
import axios from 'axios';
|
|
import * as cheerio from 'cheerio';
|
|
import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js';
|
|
export class ContentExtractor {
|
|
defaultTimeout;
|
|
maxContentLength;
|
|
constructor() {
|
|
this.defaultTimeout = 10000;
|
|
// Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB
|
|
const envMaxLength = process.env.MAX_CONTENT_LENGTH;
|
|
this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000;
|
|
// Validate the parsed value
|
|
if (isNaN(this.maxContentLength) || this.maxContentLength < 0) {
|
|
console.warn(`[ContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`);
|
|
this.maxContentLength = 500000;
|
|
}
|
|
}
|
|
async extractContent(options) {
|
|
const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options;
|
|
try {
|
|
const response = await axios.get(url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
'Sec-Fetch-Dest': 'document',
|
|
'Sec-Fetch-Mode': 'navigate',
|
|
'Sec-Fetch-Site': 'none',
|
|
'Sec-Fetch-User': '?1',
|
|
'Cache-Control': 'max-age=0',
|
|
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
|
|
'sec-ch-ua-mobile': '?0',
|
|
'sec-ch-ua-platform': '"macOS"',
|
|
},
|
|
timeout,
|
|
maxContentLength,
|
|
validateStatus: (status) => status < 400,
|
|
});
|
|
return this.parseContent(response.data);
|
|
}
|
|
catch (error) {
|
|
console.error(`Content extraction error for ${url}:`, error);
|
|
// If it's a 403 error, try with different headers
|
|
if (axios.isAxiosError(error) && error.response?.status === 403) {
|
|
console.log(`[ContentExtractor] Trying alternative headers for ${url}`);
|
|
try {
|
|
const response = await axios.get(url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
'Cache-Control': 'no-cache',
|
|
'Pragma': 'no-cache',
|
|
},
|
|
timeout,
|
|
maxContentLength,
|
|
validateStatus: (status) => status < 400,
|
|
});
|
|
console.log(`[ContentExtractor] Alternative headers worked for ${url}`);
|
|
return this.parseContent(response.data);
|
|
}
|
|
catch (retryError) {
|
|
console.error(`[ContentExtractor] Alternative headers also failed for ${url}:`, retryError);
|
|
}
|
|
}
|
|
throw new Error(`Failed to extract content from ${url}: ${this.getSpecificErrorMessage(error)}`);
|
|
}
|
|
}
|
|
async extractContentForResults(results, targetCount = results.length) {
|
|
const enhancedResults = [];
|
|
let processedCount = 0;
|
|
console.log(`[ContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`);
|
|
for (const result of results) {
|
|
if (enhancedResults.length >= targetCount) {
|
|
console.log(`[ContentExtractor] Reached target count of ${targetCount} results`);
|
|
break;
|
|
}
|
|
processedCount++;
|
|
// Skip PDF files
|
|
if (isPdfUrl(result.url)) {
|
|
console.log(`[ContentExtractor] Skipping PDF file: ${result.url}`);
|
|
continue;
|
|
}
|
|
try {
|
|
console.log(`[ContentExtractor] Extracting content from: ${result.url}`);
|
|
const content = await this.extractContent({ url: result.url });
|
|
const cleanedContent = cleanText(content, this.maxContentLength);
|
|
enhancedResults.push({
|
|
...result,
|
|
fullContent: cleanedContent,
|
|
contentPreview: getContentPreview(cleanedContent),
|
|
wordCount: getWordCount(cleanedContent),
|
|
timestamp: generateTimestamp(),
|
|
fetchStatus: 'success',
|
|
});
|
|
console.log(`[ContentExtractor] Successfully extracted content (${enhancedResults.length}/${targetCount})`);
|
|
}
|
|
catch (error) {
|
|
console.log(`[ContentExtractor] Failed to extract content from ${result.url}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
enhancedResults.push({
|
|
...result,
|
|
fullContent: '',
|
|
contentPreview: '',
|
|
wordCount: 0,
|
|
timestamp: generateTimestamp(),
|
|
fetchStatus: 'error',
|
|
error: this.getSpecificErrorMessage(error),
|
|
});
|
|
}
|
|
}
|
|
console.log(`[ContentExtractor] Processed ${processedCount} results, extracted ${enhancedResults.length} non-PDF results`);
|
|
return enhancedResults;
|
|
}
|
|
parseContent(html) {
|
|
const $ = cheerio.load(html);
|
|
// Remove all script, style, and other non-content elements
|
|
$('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove();
|
|
// Remove navigation, header, footer, and other non-content elements
|
|
$('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove();
|
|
// Remove elements with common ad/tracking classes
|
|
$('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove();
|
|
// Remove elements with common non-content IDs
|
|
$('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove();
|
|
// Remove image-related elements and attributes
|
|
$('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove();
|
|
$('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove();
|
|
$('[style*="background-image"]').remove();
|
|
// Remove empty elements and whitespace-only elements
|
|
$('*').each(function () {
|
|
const $this = $(this);
|
|
if ($this.children().length === 0 && $this.text().trim() === '') {
|
|
$this.remove();
|
|
}
|
|
});
|
|
// Try to find the main content area first
|
|
let mainContent = '';
|
|
// Priority selectors for main content
|
|
const contentSelectors = [
|
|
'article',
|
|
'main',
|
|
'[role="main"]',
|
|
'.content',
|
|
'.post-content',
|
|
'.entry-content',
|
|
'.article-content',
|
|
'.story-content',
|
|
'.news-content',
|
|
'.main-content',
|
|
'.page-content',
|
|
'.text-content',
|
|
'.body-content',
|
|
'.copy',
|
|
'.text',
|
|
'.body'
|
|
];
|
|
for (const selector of contentSelectors) {
|
|
const $content = $(selector).first();
|
|
if ($content.length > 0) {
|
|
mainContent = $content.text().trim();
|
|
if (mainContent.length > 100) { // Ensure we have substantial content
|
|
console.log(`[ContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
// If no main content found, try body content
|
|
if (!mainContent || mainContent.length < 100) {
|
|
console.log(`[ContentExtractor] No main content found, using body content`);
|
|
mainContent = $('body').text().trim();
|
|
}
|
|
// Clean up the text
|
|
const cleanedContent = this.cleanTextContent(mainContent);
|
|
return cleanText(cleanedContent, this.maxContentLength);
|
|
}
|
|
cleanTextContent(text) {
|
|
// Remove excessive whitespace
|
|
text = text.replace(/\s+/g, ' ');
|
|
// Remove image-related text and data URLs
|
|
text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data
|
|
text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs
|
|
text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions
|
|
text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words
|
|
text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text
|
|
// Remove common non-content patterns
|
|
text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, '');
|
|
// Remove excessive line breaks and spacing
|
|
text = text.replace(/\n\s*\n/g, '\n');
|
|
text = text.replace(/\r\n/g, '\n');
|
|
text = text.replace(/\r/g, '\n');
|
|
// Remove leading/trailing whitespace
|
|
text = text.trim();
|
|
return text;
|
|
}
|
|
getSpecificErrorMessage(error) {
|
|
if (axios.isAxiosError(error)) {
|
|
if (error.code === 'ECONNABORTED') {
|
|
return 'Request timeout';
|
|
}
|
|
if (error.response?.status === 403) {
|
|
return '403 Forbidden - Access denied';
|
|
}
|
|
if (error.response?.status === 404) {
|
|
return '404 Not found';
|
|
}
|
|
if (error.message.includes('maxContentLength')) {
|
|
return 'Content too long';
|
|
}
|
|
if (error.response?.status) {
|
|
return `HTTP ${error.response.status}: ${error.message}`;
|
|
}
|
|
return `Network error: ${error.message}`;
|
|
}
|
|
return error instanceof Error ? error.message : 'Unknown error';
|
|
}
|
|
}
|
|
//# sourceMappingURL=content-extractor.js.map
|