1002 lines
50 KiB
JavaScript
1002 lines
50 KiB
JavaScript
import axios from 'axios';
|
|
import * as cheerio from 'cheerio';
|
|
import { generateTimestamp, sanitizeQuery } from './utils.js';
|
|
import { RateLimiter } from './rate-limiter.js';
|
|
import { BrowserPool } from './browser-pool.js';
|
|
export class SearchEngine {
|
|
rateLimiter;
|
|
browserPool;
|
|
constructor() {
|
|
this.rateLimiter = new RateLimiter(10); // 10 requests per minute
|
|
this.browserPool = new BrowserPool();
|
|
}
|
|
async search(options) {
|
|
const { query, numResults = 5, timeout = 10000 } = options;
|
|
const sanitizedQuery = sanitizeQuery(query);
|
|
console.log(`[SearchEngine] Starting search for query: "${sanitizedQuery}"`);
|
|
try {
|
|
return await this.rateLimiter.execute(async () => {
|
|
console.log(`[SearchEngine] Starting search with multiple engines...`);
|
|
// Configuration from environment variables
|
|
const enableQualityCheck = process.env.ENABLE_RELEVANCE_CHECKING !== 'false';
|
|
const qualityThreshold = parseFloat(process.env.RELEVANCE_THRESHOLD || '0.3');
|
|
const forceMultiEngine = process.env.FORCE_MULTI_ENGINE_SEARCH === 'true';
|
|
const debugBrowsers = process.env.DEBUG_BROWSER_LIFECYCLE === 'true';
|
|
console.log(`[SearchEngine] Quality checking: ${enableQualityCheck}, threshold: ${qualityThreshold}, multi-engine: ${forceMultiEngine}, debug: ${debugBrowsers}`);
|
|
// Try multiple approaches to get search results, starting with most reliable
|
|
const approaches = [
|
|
{ method: this.tryBrowserBingSearch.bind(this), name: 'Browser Bing' },
|
|
{ method: this.tryBrowserBraveSearch.bind(this), name: 'Browser Brave' },
|
|
{ method: this.tryDuckDuckGoSearch.bind(this), name: 'Axios DuckDuckGo' }
|
|
];
|
|
let bestResults = [];
|
|
let bestEngine = 'None';
|
|
let bestQuality = 0;
|
|
for (let i = 0; i < approaches.length; i++) {
|
|
const approach = approaches[i];
|
|
try {
|
|
console.log(`[SearchEngine] Attempting ${approach.name} (${i + 1}/${approaches.length})...`);
|
|
// Use more aggressive timeouts for faster fallback
|
|
const approachTimeout = Math.min(timeout / 3, 4000); // Max 4 seconds per approach for faster fallback
|
|
const results = await approach.method(sanitizedQuery, numResults, approachTimeout);
|
|
if (results.length > 0) {
|
|
console.log(`[SearchEngine] Found ${results.length} results with ${approach.name}`);
|
|
// Validate result quality to detect irrelevant results
|
|
const qualityScore = enableQualityCheck ? this.assessResultQuality(results, sanitizedQuery) : 1.0;
|
|
console.log(`[SearchEngine] ${approach.name} quality score: ${qualityScore.toFixed(2)}/1.0`);
|
|
// Track the best results so far
|
|
if (qualityScore > bestQuality) {
|
|
bestResults = results;
|
|
bestEngine = approach.name;
|
|
bestQuality = qualityScore;
|
|
}
|
|
// If quality is excellent, return immediately (unless forcing multi-engine)
|
|
if (qualityScore >= 0.8 && !forceMultiEngine) {
|
|
console.log(`[SearchEngine] Excellent quality results from ${approach.name}, returning immediately`);
|
|
return { results, engine: approach.name };
|
|
}
|
|
// If quality is acceptable and this isn't Bing (first engine), return
|
|
if (qualityScore >= qualityThreshold && approach.name !== 'Browser Bing' && !forceMultiEngine) {
|
|
console.log(`[SearchEngine] Good quality results from ${approach.name}, using as primary`);
|
|
return { results, engine: approach.name };
|
|
}
|
|
// If this is the last engine or quality is acceptable, prepare to return
|
|
if (i === approaches.length - 1) {
|
|
if (bestQuality >= qualityThreshold || !enableQualityCheck) {
|
|
console.log(`[SearchEngine] Using best results from ${bestEngine} (quality: ${bestQuality.toFixed(2)})`);
|
|
return { results: bestResults, engine: bestEngine };
|
|
}
|
|
else if (bestResults.length > 0) {
|
|
console.log(`[SearchEngine] Warning: Low quality results from all engines, using best available from ${bestEngine}`);
|
|
return { results: bestResults, engine: bestEngine };
|
|
}
|
|
}
|
|
else {
|
|
console.log(`[SearchEngine] ${approach.name} results quality: ${qualityScore.toFixed(2)}, continuing to try other engines...`);
|
|
}
|
|
}
|
|
}
|
|
catch (error) {
|
|
console.error(`[SearchEngine] ${approach.name} approach failed:`, error);
|
|
// Handle browser-specific errors (no cleanup needed since each engine uses dedicated browsers)
|
|
await this.handleBrowserError(error, approach.name);
|
|
}
|
|
}
|
|
console.log(`[SearchEngine] All approaches failed, returning empty results`);
|
|
return { results: [], engine: 'None' };
|
|
});
|
|
}
|
|
catch (error) {
|
|
console.error('[SearchEngine] Search error:', error);
|
|
if (axios.isAxiosError(error)) {
|
|
console.error('[SearchEngine] Axios error details:', {
|
|
status: error.response?.status,
|
|
statusText: error.response?.statusText,
|
|
data: error.response?.data?.substring(0, 500),
|
|
});
|
|
}
|
|
throw new Error(`Failed to perform search: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
}
|
|
}
|
|
async tryBrowserBraveSearch(query, numResults, timeout) {
|
|
console.log(`[SearchEngine] Trying browser-based Brave search with dedicated browser...`);
|
|
// Try with retry mechanism
|
|
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
let browser;
|
|
try {
|
|
// Create a dedicated browser instance for Brave search only
|
|
const { firefox } = await import('playwright');
|
|
browser = await firefox.launch({
|
|
headless: process.env.BROWSER_HEADLESS !== 'false',
|
|
args: [
|
|
'--no-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
],
|
|
});
|
|
console.log(`[SearchEngine] Brave search attempt ${attempt}/2 with fresh browser`);
|
|
const results = await this.tryBrowserBraveSearchInternal(browser, query, numResults, timeout);
|
|
return results;
|
|
}
|
|
catch (error) {
|
|
console.error(`[SearchEngine] Brave search attempt ${attempt}/2 failed:`, error);
|
|
if (attempt === 2) {
|
|
throw error; // Re-throw on final attempt
|
|
}
|
|
// Small delay before retry
|
|
await new Promise(resolve => setTimeout(resolve, 500));
|
|
}
|
|
finally {
|
|
// Always close the dedicated browser
|
|
if (browser) {
|
|
try {
|
|
await browser.close();
|
|
}
|
|
catch (closeError) {
|
|
console.log(`[SearchEngine] Error closing Brave browser:`, closeError);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
throw new Error('All Brave search attempts failed');
|
|
}
|
|
async tryBrowserBraveSearchInternal(browser, query, numResults, timeout) {
|
|
// Validate browser is still functional before proceeding
|
|
if (!browser.isConnected()) {
|
|
throw new Error('Browser is not connected');
|
|
}
|
|
try {
|
|
const context = await browser.newContext({
|
|
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
viewport: { width: 1366, height: 768 },
|
|
locale: 'en-US',
|
|
timezoneId: 'America/New_York',
|
|
});
|
|
try {
|
|
const page = await context.newPage();
|
|
// Navigate to Brave search
|
|
const searchUrl = `https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web`;
|
|
console.log(`[SearchEngine] Browser navigating to Brave: ${searchUrl}`);
|
|
await page.goto(searchUrl, {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: timeout
|
|
});
|
|
// Wait for search results to load
|
|
try {
|
|
await page.waitForSelector('[data-type="web"]', { timeout: 3000 });
|
|
}
|
|
catch {
|
|
console.log(`[SearchEngine] Browser Brave results selector not found, proceeding anyway`);
|
|
}
|
|
// Get the page content
|
|
const html = await page.content();
|
|
console.log(`[SearchEngine] Browser Brave got HTML with length: ${html.length}`);
|
|
const results = this.parseBraveResults(html, numResults);
|
|
console.log(`[SearchEngine] Browser Brave parsed ${results.length} results`);
|
|
await context.close();
|
|
return results;
|
|
}
|
|
catch (error) {
|
|
// Ensure context is closed even on error
|
|
await context.close();
|
|
throw error;
|
|
}
|
|
}
|
|
catch (error) {
|
|
console.error(`[SearchEngine] Browser Brave search failed:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
async tryBrowserBingSearch(query, numResults, timeout) {
|
|
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
|
|
console.error(`[SearchEngine] BING: Starting browser-based search with dedicated browser for query: "${query}"`);
|
|
// Try with retry mechanism
|
|
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
let browser;
|
|
try {
|
|
console.error(`[SearchEngine] BING: Attempt ${attempt}/2 - Launching Chromium browser...`);
|
|
// Create a dedicated browser instance for Bing search only
|
|
const { chromium } = await import('playwright');
|
|
const startTime = Date.now();
|
|
browser = await chromium.launch({
|
|
headless: process.env.BROWSER_HEADLESS !== 'false',
|
|
args: [
|
|
'--no-sandbox',
|
|
'--disable-blink-features=AutomationControlled',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-gpu',
|
|
],
|
|
});
|
|
const launchTime = Date.now() - startTime;
|
|
console.error(`[SearchEngine] BING: Browser launched successfully in ${launchTime}ms, connected: ${browser.isConnected()}`);
|
|
const results = await this.tryBrowserBingSearchInternal(browser, query, numResults, timeout);
|
|
console.error(`[SearchEngine] BING: Search completed successfully with ${results.length} results`);
|
|
return results;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
|
console.error(`[SearchEngine] BING: Attempt ${attempt}/2 FAILED with error: ${errorMessage}`);
|
|
if (debugBing) {
|
|
console.error(`[SearchEngine] BING: Full error details:`, error);
|
|
}
|
|
if (attempt === 2) {
|
|
console.error(`[SearchEngine] BING: All attempts exhausted, giving up`);
|
|
throw error; // Re-throw on final attempt
|
|
}
|
|
// Small delay before retry
|
|
console.error(`[SearchEngine] BING: Waiting 500ms before retry...`);
|
|
await new Promise(resolve => setTimeout(resolve, 500));
|
|
}
|
|
finally {
|
|
// Always close the dedicated browser
|
|
if (browser) {
|
|
try {
|
|
await browser.close();
|
|
if (debugBing) {
|
|
console.error(`[SearchEngine] BING: Browser closed successfully`);
|
|
}
|
|
}
|
|
catch (closeError) {
|
|
console.error(`[SearchEngine] BING: Error closing browser:`, closeError);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
throw new Error('All Bing search attempts failed');
|
|
}
|
|
async tryBrowserBingSearchInternal(browser, query, numResults, timeout) {
|
|
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
|
|
// Validate browser is still functional before proceeding
|
|
if (!browser.isConnected()) {
|
|
console.error(`[SearchEngine] BING: Browser is not connected`);
|
|
throw new Error('Browser is not connected');
|
|
}
|
|
console.error(`[SearchEngine] BING: Creating browser context with enhanced fingerprinting...`);
|
|
try {
|
|
// Enhanced browser context with more realistic fingerprinting
|
|
const context = await browser.newContext({
|
|
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
viewport: { width: 1366, height: 768 },
|
|
locale: 'en-US',
|
|
timezoneId: 'America/New_York',
|
|
colorScheme: 'light',
|
|
deviceScaleFactor: 1,
|
|
hasTouch: false,
|
|
isMobile: false,
|
|
extraHTTPHeaders: {
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'DNT': '1',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
'Sec-Fetch-Dest': 'document',
|
|
'Sec-Fetch-Mode': 'navigate',
|
|
'Sec-Fetch-Site': 'none'
|
|
}
|
|
});
|
|
console.error(`[SearchEngine] BING: Context created, opening new page...`);
|
|
const page = await context.newPage();
|
|
console.error(`[SearchEngine] BING: Page opened successfully`);
|
|
try {
|
|
// Try enhanced Bing search with proper web interface flow
|
|
try {
|
|
console.error(`[SearchEngine] BING: Attempting enhanced search (homepage → form submission)...`);
|
|
const results = await this.tryEnhancedBingSearch(page, query, numResults, timeout);
|
|
console.error(`[SearchEngine] BING: Enhanced search succeeded with ${results.length} results`);
|
|
await context.close();
|
|
return results;
|
|
}
|
|
catch (enhancedError) {
|
|
const errorMessage = enhancedError instanceof Error ? enhancedError.message : 'Unknown error';
|
|
console.error(`[SearchEngine] BING: Enhanced search failed: ${errorMessage}`);
|
|
if (debugBing) {
|
|
console.error(`[SearchEngine] BING: Enhanced search error details:`, enhancedError);
|
|
}
|
|
console.error(`[SearchEngine] BING: Falling back to direct URL search...`);
|
|
// Fallback to direct URL approach with enhanced parameters
|
|
const results = await this.tryDirectBingSearch(page, query, numResults, timeout);
|
|
console.error(`[SearchEngine] BING: Direct search succeeded with ${results.length} results`);
|
|
await context.close();
|
|
return results;
|
|
}
|
|
}
|
|
catch (error) {
|
|
// Ensure context is closed even on error
|
|
console.error(`[SearchEngine] BING: All search methods failed, closing context...`);
|
|
await context.close();
|
|
throw error;
|
|
}
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
|
console.error(`[SearchEngine] BING: Internal search failed: ${errorMessage}`);
|
|
if (debugBing) {
|
|
console.error(`[SearchEngine] BING: Internal search error details:`, error);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
async tryEnhancedBingSearch(page, query, numResults, timeout) {
|
|
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
|
|
console.error(`[SearchEngine] BING: Enhanced search - navigating to Bing homepage...`);
|
|
// Navigate to Bing homepage first to establish proper session
|
|
const startTime = Date.now();
|
|
await page.goto('https://www.bing.com', {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: timeout / 2
|
|
});
|
|
const loadTime = Date.now() - startTime;
|
|
const pageTitle = await page.title();
|
|
const currentUrl = page.url();
|
|
console.error(`[SearchEngine] BING: Homepage loaded in ${loadTime}ms, title: "${pageTitle}", URL: ${currentUrl}`);
|
|
// Wait a moment for page to fully load
|
|
await page.waitForTimeout(500);
|
|
// Find and use the search box (more realistic than direct URL)
|
|
try {
|
|
console.error(`[SearchEngine] BING: Looking for search form elements...`);
|
|
await page.waitForSelector('#sb_form_q', { timeout: 2000 });
|
|
console.error(`[SearchEngine] BING: Search box found, filling with query: "${query}"`);
|
|
await page.fill('#sb_form_q', query);
|
|
console.error(`[SearchEngine] BING: Clicking search button and waiting for navigation...`);
|
|
// Submit the search form
|
|
await Promise.all([
|
|
page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }),
|
|
page.click('#search_icon')
|
|
]);
|
|
const searchLoadTime = Date.now() - startTime;
|
|
const searchPageTitle = await page.title();
|
|
const searchPageUrl = page.url();
|
|
console.error(`[SearchEngine] BING: Search completed in ${searchLoadTime}ms total, title: "${searchPageTitle}", URL: ${searchPageUrl}`);
|
|
}
|
|
catch (formError) {
|
|
const errorMessage = formError instanceof Error ? formError.message : 'Unknown error';
|
|
console.error(`[SearchEngine] BING: Search form submission failed: ${errorMessage}`);
|
|
if (debugBing) {
|
|
console.error(`[SearchEngine] BING: Form error details:`, formError);
|
|
}
|
|
throw formError;
|
|
}
|
|
// Wait for search results to load
|
|
try {
|
|
console.error(`[SearchEngine] BING: Waiting for search results to appear...`);
|
|
await page.waitForSelector('.b_algo, .b_result', { timeout: 3000 });
|
|
console.error(`[SearchEngine] BING: Search results selector found`);
|
|
}
|
|
catch {
|
|
console.error(`[SearchEngine] BING: Search results selector not found, proceeding with page content anyway`);
|
|
}
|
|
const html = await page.content();
|
|
console.error(`[SearchEngine] BING: Got page HTML with length: ${html.length} characters`);
|
|
if (debugBing && html.length < 10000) {
|
|
console.error(`[SearchEngine] BING: WARNING - HTML seems short, possible bot detection or error page`);
|
|
}
|
|
const results = this.parseBingResults(html, numResults);
|
|
console.error(`[SearchEngine] BING: Enhanced search parsed ${results.length} results`);
|
|
if (results.length === 0) {
|
|
console.error(`[SearchEngine] BING: WARNING - No results found, possible parsing failure or empty search`);
|
|
if (debugBing) {
|
|
const sampleHtml = html.substring(0, 1000);
|
|
console.error(`[SearchEngine] BING: Sample HTML for debugging:`, sampleHtml);
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
async tryDirectBingSearch(page, query, numResults, timeout) {
|
|
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
|
|
console.error(`[SearchEngine] BING: Direct search with enhanced parameters...`);
|
|
// Generate a conversation ID (cvid) similar to what Bing uses
|
|
const cvid = this.generateConversationId();
|
|
// Construct URL with enhanced parameters based on successful manual searches
|
|
const searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&count=${Math.min(numResults, 10)}&form=QBLH&sp=-1&qs=n&cvid=${cvid}`;
|
|
console.error(`[SearchEngine] BING: Navigating to direct URL: ${searchUrl}`);
|
|
const startTime = Date.now();
|
|
await page.goto(searchUrl, {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: timeout
|
|
});
|
|
const loadTime = Date.now() - startTime;
|
|
const pageTitle = await page.title();
|
|
const currentUrl = page.url();
|
|
console.error(`[SearchEngine] BING: Direct page loaded in ${loadTime}ms, title: "${pageTitle}", URL: ${currentUrl}`);
|
|
// Wait for search results to load
|
|
try {
|
|
console.error(`[SearchEngine] BING: Waiting for search results to appear...`);
|
|
await page.waitForSelector('.b_algo, .b_result', { timeout: 3000 });
|
|
console.error(`[SearchEngine] BING: Search results selector found`);
|
|
}
|
|
catch {
|
|
console.error(`[SearchEngine] BING: Search results selector not found, proceeding with page content anyway`);
|
|
}
|
|
const html = await page.content();
|
|
console.error(`[SearchEngine] BING: Got page HTML with length: ${html.length} characters`);
|
|
if (debugBing && html.length < 10000) {
|
|
console.error(`[SearchEngine] BING: WARNING - HTML seems short, possible bot detection or error page`);
|
|
}
|
|
const results = this.parseBingResults(html, numResults);
|
|
console.error(`[SearchEngine] BING: Direct search parsed ${results.length} results`);
|
|
if (results.length === 0) {
|
|
console.error(`[SearchEngine] BING: WARNING - No results found, possible parsing failure or empty search`);
|
|
if (debugBing) {
|
|
const sampleHtml = html.substring(0, 1000);
|
|
console.error(`[SearchEngine] BING: Sample HTML for debugging:`, sampleHtml);
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
generateConversationId() {
|
|
// Generate a conversation ID similar to Bing's format (32 hex characters)
|
|
const chars = '0123456789ABCDEF';
|
|
let cvid = '';
|
|
for (let i = 0; i < 32; i++) {
|
|
cvid += chars[Math.floor(Math.random() * chars.length)];
|
|
}
|
|
return cvid;
|
|
}
|
|
async tryDuckDuckGoSearch(query, numResults, timeout) {
|
|
console.log(`[SearchEngine] Trying DuckDuckGo as fallback...`);
|
|
try {
|
|
const response = await axios.get('https://html.duckduckgo.com/html/', {
|
|
params: {
|
|
q: query,
|
|
},
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
},
|
|
timeout,
|
|
validateStatus: (status) => status < 400,
|
|
});
|
|
console.log(`[SearchEngine] DuckDuckGo got response with status: ${response.status}`);
|
|
const results = this.parseDuckDuckGoResults(response.data, numResults);
|
|
console.log(`[SearchEngine] DuckDuckGo parsed ${results.length} results`);
|
|
return results;
|
|
}
|
|
catch {
|
|
console.error(`[SearchEngine] DuckDuckGo search failed`);
|
|
throw new Error('DuckDuckGo search failed');
|
|
}
|
|
}
|
|
parseSearchResults(html, maxResults) {
|
|
console.log(`[SearchEngine] Parsing HTML with length: ${html.length}`);
|
|
const $ = cheerio.load(html);
|
|
const results = [];
|
|
const timestamp = generateTimestamp();
|
|
// Log what selectors we find - more comprehensive debugging
|
|
const gElements = $('div.g');
|
|
const sokobanElements = $('div[data-sokoban-container]');
|
|
const tF2CxcElements = $('.tF2Cxc');
|
|
const rcElements = $('.rc');
|
|
const vedElements = $('[data-ved]');
|
|
const h3Elements = $('h3');
|
|
const linkElements = $('a[href]');
|
|
console.log(`[SearchEngine] Found elements:`);
|
|
console.log(` - div.g: ${gElements.length}`);
|
|
console.log(` - div[data-sokoban-container]: ${sokobanElements.length}`);
|
|
console.log(` - .tF2Cxc: ${tF2CxcElements.length}`);
|
|
console.log(` - .rc: ${rcElements.length}`);
|
|
console.log(` - [data-ved]: ${vedElements.length}`);
|
|
console.log(` - h3: ${h3Elements.length}`);
|
|
console.log(` - a[href]: ${linkElements.length}`);
|
|
// Try multiple approaches to find search results
|
|
const searchResultSelectors = [
|
|
'div.g',
|
|
'div[data-sokoban-container]',
|
|
'.tF2Cxc',
|
|
'.rc',
|
|
'[data-ved]',
|
|
'div[jscontroller]'
|
|
];
|
|
let foundResults = false;
|
|
for (const selector of searchResultSelectors) {
|
|
if (foundResults)
|
|
break;
|
|
console.log(`[SearchEngine] Trying selector: ${selector}`);
|
|
const elements = $(selector);
|
|
console.log(`[SearchEngine] Found ${elements.length} elements with selector ${selector}`);
|
|
elements.each((_index, element) => {
|
|
if (results.length >= maxResults)
|
|
return false;
|
|
const $element = $(element);
|
|
// Try multiple title selectors
|
|
const titleSelectors = ['h3', '.LC20lb', '.DKV0Md', 'a[data-ved]', '.r', '.s'];
|
|
let title = '';
|
|
let url = '';
|
|
for (const titleSelector of titleSelectors) {
|
|
const $title = $element.find(titleSelector).first();
|
|
if ($title.length) {
|
|
title = $title.text().trim();
|
|
console.log(`[SearchEngine] Found title with ${titleSelector}: "${title}"`);
|
|
// Try to find the link
|
|
const $link = $title.closest('a');
|
|
if ($link.length) {
|
|
url = $link.attr('href') || '';
|
|
console.log(`[SearchEngine] Found URL: "${url}"`);
|
|
}
|
|
else {
|
|
// Try to find any link in the element
|
|
const $anyLink = $element.find('a[href]').first();
|
|
if ($anyLink.length) {
|
|
url = $anyLink.attr('href') || '';
|
|
console.log(`[SearchEngine] Found URL from any link: "${url}"`);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
// Try multiple snippet selectors
|
|
const snippetSelectors = ['.VwiC3b', '.st', '.aCOpRe', '.IsZvec', '.s3v9rd', '.MUxGbd', '.aCOpRe', '.snippet-content'];
|
|
let snippet = '';
|
|
for (const snippetSelector of snippetSelectors) {
|
|
const $snippet = $element.find(snippetSelector).first();
|
|
if ($snippet.length) {
|
|
snippet = $snippet.text().trim();
|
|
console.log(`[SearchEngine] Found snippet with ${snippetSelector}: "${snippet.substring(0, 100)}..."`);
|
|
break;
|
|
}
|
|
}
|
|
if (title && url && this.isValidSearchUrl(url)) {
|
|
console.log(`[SearchEngine] Adding result: ${title}`);
|
|
results.push({
|
|
title,
|
|
url: this.cleanGoogleUrl(url),
|
|
description: snippet || 'No description available',
|
|
fullContent: '',
|
|
contentPreview: '',
|
|
wordCount: 0,
|
|
timestamp,
|
|
fetchStatus: 'success',
|
|
});
|
|
foundResults = true;
|
|
}
|
|
else {
|
|
console.log(`[SearchEngine] Skipping result: title="${title}", url="${url}", isValid=${this.isValidSearchUrl(url)}`);
|
|
}
|
|
});
|
|
}
|
|
console.log(`[SearchEngine] Found ${results.length} results with all selectors`);
|
|
// If still no results, try a more aggressive approach - look for any h3 with links
|
|
if (results.length === 0) {
|
|
console.log(`[SearchEngine] No results found, trying aggressive h3 search...`);
|
|
$('h3').each((_index, element) => {
|
|
if (results.length >= maxResults)
|
|
return false;
|
|
const $h3 = $(element);
|
|
const title = $h3.text().trim();
|
|
const $link = $h3.closest('a');
|
|
if ($link.length && title) {
|
|
const url = $link.attr('href') || '';
|
|
console.log(`[SearchEngine] Aggressive search found: "${title}" -> "${url}"`);
|
|
if (this.isValidSearchUrl(url)) {
|
|
results.push({
|
|
title,
|
|
url: this.cleanGoogleUrl(url),
|
|
description: 'No description available',
|
|
fullContent: '',
|
|
contentPreview: '',
|
|
wordCount: 0,
|
|
timestamp,
|
|
fetchStatus: 'success',
|
|
});
|
|
}
|
|
}
|
|
});
|
|
console.log(`[SearchEngine] Aggressive search found ${results.length} results`);
|
|
}
|
|
return results;
|
|
}
|
|
parseBraveResults(html, maxResults) {
|
|
console.log(`[SearchEngine] Parsing Brave HTML with length: ${html.length}`);
|
|
const $ = cheerio.load(html);
|
|
const results = [];
|
|
const timestamp = generateTimestamp();
|
|
// Brave result selectors
|
|
const resultSelectors = [
|
|
'[data-type="web"]', // Main Brave results
|
|
'.result', // Alternative format
|
|
'.fdb' // Brave specific format
|
|
];
|
|
let foundResults = false;
|
|
for (const selector of resultSelectors) {
|
|
if (foundResults && results.length >= maxResults)
|
|
break;
|
|
console.log(`[SearchEngine] Trying Brave selector: ${selector}`);
|
|
const elements = $(selector);
|
|
console.log(`[SearchEngine] Found ${elements.length} elements with selector ${selector}`);
|
|
elements.each((_index, element) => {
|
|
if (results.length >= maxResults)
|
|
return false;
|
|
const $element = $(element);
|
|
// Try multiple title selectors for Brave
|
|
const titleSelectors = [
|
|
'.title a', // Brave specific
|
|
'h2 a', // Common format
|
|
'.result-title a', // Alternative format
|
|
'a[href*="://"]', // Any external link
|
|
'.snippet-title a' // Snippet title
|
|
];
|
|
let title = '';
|
|
let url = '';
|
|
for (const titleSelector of titleSelectors) {
|
|
const $titleElement = $element.find(titleSelector).first();
|
|
if ($titleElement.length) {
|
|
title = $titleElement.text().trim();
|
|
url = $titleElement.attr('href') || '';
|
|
console.log(`[SearchEngine] Brave found title with ${titleSelector}: "${title}"`);
|
|
if (title && url && url.startsWith('http')) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
// If still no title, try getting it from any text content
|
|
if (!title) {
|
|
const textContent = $element.text().trim();
|
|
const lines = textContent.split('\n').filter(line => line.trim().length > 0);
|
|
if (lines.length > 0) {
|
|
title = lines[0].trim();
|
|
console.log(`[SearchEngine] Brave found title from text content: "${title}"`);
|
|
}
|
|
}
|
|
// Try multiple snippet selectors for Brave
|
|
const snippetSelectors = [
|
|
'.snippet-content', // Brave specific
|
|
'.snippet', // Generic
|
|
'.description', // Alternative
|
|
'p' // Fallback paragraph
|
|
];
|
|
let snippet = '';
|
|
for (const snippetSelector of snippetSelectors) {
|
|
const $snippetElement = $element.find(snippetSelector).first();
|
|
if ($snippetElement.length) {
|
|
snippet = $snippetElement.text().trim();
|
|
break;
|
|
}
|
|
}
|
|
if (title && url && this.isValidSearchUrl(url)) {
|
|
console.log(`[SearchEngine] Brave found: "${title}" -> "${url}"`);
|
|
results.push({
|
|
title,
|
|
url: this.cleanBraveUrl(url),
|
|
description: snippet || 'No description available',
|
|
fullContent: '',
|
|
contentPreview: '',
|
|
wordCount: 0,
|
|
timestamp,
|
|
fetchStatus: 'success',
|
|
});
|
|
foundResults = true;
|
|
}
|
|
});
|
|
}
|
|
console.log(`[SearchEngine] Brave found ${results.length} results`);
|
|
return results;
|
|
}
|
|
parseBingResults(html, maxResults) {
|
|
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
|
|
console.error(`[SearchEngine] BING: Parsing HTML with length: ${html.length}`);
|
|
const $ = cheerio.load(html);
|
|
const results = [];
|
|
const timestamp = generateTimestamp();
|
|
// Check for common Bing error indicators
|
|
const pageTitle = $('title').text();
|
|
console.error(`[SearchEngine] BING: Page title: "${pageTitle}"`);
|
|
if (pageTitle.includes('Access Denied') || pageTitle.includes('blocked') || pageTitle.includes('captcha')) {
|
|
console.error(`[SearchEngine] BING: ERROR - Bot detection or access denied detected in page title`);
|
|
}
|
|
// Bing result selectors
|
|
const resultSelectors = [
|
|
'.b_algo', // Main Bing results
|
|
'.b_result', // Alternative Bing format
|
|
'.b_card' // Card format
|
|
];
|
|
console.error(`[SearchEngine] BING: Checking for result elements...`);
|
|
// Log counts for all selectors first
|
|
for (const selector of resultSelectors) {
|
|
const elements = $(selector);
|
|
console.error(`[SearchEngine] BING: Found ${elements.length} elements with selector "${selector}"`);
|
|
}
|
|
let foundResults = false;
|
|
for (const selector of resultSelectors) {
|
|
if (foundResults && results.length >= maxResults)
|
|
break;
|
|
const elements = $(selector);
|
|
if (elements.length === 0)
|
|
continue;
|
|
elements.each((_index, element) => {
|
|
if (results.length >= maxResults)
|
|
return false;
|
|
const $element = $(element);
|
|
// Try multiple title selectors for Bing
|
|
const titleSelectors = [
|
|
'h2 a', // Standard Bing format
|
|
'.b_title a', // Alternative format
|
|
'a[data-seid]' // Bing specific
|
|
];
|
|
let title = '';
|
|
let url = '';
|
|
for (const titleSelector of titleSelectors) {
|
|
const $titleElement = $element.find(titleSelector).first();
|
|
if ($titleElement.length) {
|
|
title = $titleElement.text().trim();
|
|
url = $titleElement.attr('href') || '';
|
|
console.log(`[SearchEngine] Bing found title with ${titleSelector}: "${title}"`);
|
|
break;
|
|
}
|
|
}
|
|
// Try multiple snippet selectors for Bing
|
|
const snippetSelectors = [
|
|
'.b_caption p', // Standard Bing snippet
|
|
'.b_snippet', // Alternative format
|
|
'.b_descript', // Description format
|
|
'.b_caption', // Caption without p tag
|
|
'.b_caption > span', // Caption span
|
|
'.b_excerpt', // Excerpt format
|
|
'p', // Any paragraph in the result
|
|
'.b_algo_content p', // Content paragraph
|
|
'.b_algo_content', // Full content area
|
|
'.b_context' // Context information
|
|
];
|
|
let snippet = '';
|
|
for (const snippetSelector of snippetSelectors) {
|
|
const $snippetElement = $element.find(snippetSelector).first();
|
|
if ($snippetElement.length) {
|
|
const candidateSnippet = $snippetElement.text().trim();
|
|
// Skip very short snippets or those that look like metadata
|
|
if (candidateSnippet.length > 20 && !candidateSnippet.match(/^\d+\s*(min|sec|hour|day|week|month|year)/i)) {
|
|
snippet = candidateSnippet;
|
|
console.log(`[SearchEngine] Bing found snippet with ${snippetSelector}: "${snippet.substring(0, 100)}..."`);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (title && url && this.isValidSearchUrl(url)) {
|
|
console.log(`[SearchEngine] Bing found: "${title}" -> "${url}"`);
|
|
results.push({
|
|
title,
|
|
url: this.cleanBingUrl(url),
|
|
description: snippet || 'No description available',
|
|
fullContent: '',
|
|
contentPreview: '',
|
|
wordCount: 0,
|
|
timestamp,
|
|
fetchStatus: 'success',
|
|
});
|
|
foundResults = true;
|
|
}
|
|
});
|
|
}
|
|
console.log(`[SearchEngine] Bing found ${results.length} results`);
|
|
return results;
|
|
}
|
|
parseDuckDuckGoResults(html, maxResults) {
|
|
console.log(`[SearchEngine] Parsing DuckDuckGo HTML with length: ${html.length}`);
|
|
const $ = cheerio.load(html);
|
|
const results = [];
|
|
const timestamp = generateTimestamp();
|
|
// DuckDuckGo results are in .result elements
|
|
$('.result').each((_index, element) => {
|
|
if (results.length >= maxResults)
|
|
return false;
|
|
const $element = $(element);
|
|
// Extract title and URL
|
|
const $titleElement = $element.find('.result__title a');
|
|
const title = $titleElement.text().trim();
|
|
const url = $titleElement.attr('href');
|
|
// Extract snippet
|
|
const snippet = $element.find('.result__snippet').text().trim();
|
|
if (title && url) {
|
|
console.log(`[SearchEngine] DuckDuckGo found: "${title}" -> "${url}"`);
|
|
results.push({
|
|
title,
|
|
url: this.cleanDuckDuckGoUrl(url),
|
|
description: snippet || 'No description available',
|
|
fullContent: '',
|
|
contentPreview: '',
|
|
wordCount: 0,
|
|
timestamp,
|
|
fetchStatus: 'success',
|
|
});
|
|
}
|
|
});
|
|
console.log(`[SearchEngine] DuckDuckGo found ${results.length} results`);
|
|
return results;
|
|
}
|
|
isValidSearchUrl(url) {
|
|
// Google search results URLs can be in various formats
|
|
return url.startsWith('/url?') ||
|
|
url.startsWith('http://') ||
|
|
url.startsWith('https://') ||
|
|
url.startsWith('//') ||
|
|
url.startsWith('/search?') ||
|
|
url.startsWith('/') ||
|
|
url.includes('google.com') ||
|
|
url.length > 10; // Accept any reasonably long URL
|
|
}
|
|
cleanGoogleUrl(url) {
|
|
// Handle Google's redirect URLs
|
|
if (url.startsWith('/url?')) {
|
|
try {
|
|
const urlParams = new URLSearchParams(url.substring(5));
|
|
const actualUrl = urlParams.get('q') || urlParams.get('url');
|
|
if (actualUrl) {
|
|
return actualUrl;
|
|
}
|
|
}
|
|
catch {
|
|
console.warn('Failed to parse Google redirect URL:', url);
|
|
}
|
|
}
|
|
// Handle protocol-relative URLs
|
|
if (url.startsWith('//')) {
|
|
return 'https:' + url;
|
|
}
|
|
return url;
|
|
}
|
|
cleanBraveUrl(url) {
|
|
// Brave URLs are usually direct, but check for any redirect patterns
|
|
if (url.startsWith('//')) {
|
|
return 'https:' + url;
|
|
}
|
|
// If it's already a full URL, return as-is
|
|
if (url.startsWith('http://') || url.startsWith('https://')) {
|
|
return url;
|
|
}
|
|
return url;
|
|
}
|
|
cleanBingUrl(url) {
|
|
// Bing URLs are usually direct, but check for any redirect patterns
|
|
if (url.startsWith('//')) {
|
|
return 'https:' + url;
|
|
}
|
|
// If it's already a full URL, return as-is
|
|
if (url.startsWith('http://') || url.startsWith('https://')) {
|
|
return url;
|
|
}
|
|
return url;
|
|
}
|
|
cleanDuckDuckGoUrl(url) {
|
|
// DuckDuckGo URLs are redirect URLs that need to be decoded
|
|
if (url.startsWith('//duckduckgo.com/l/')) {
|
|
try {
|
|
// Extract the uddg parameter which contains the actual URL
|
|
const urlParams = new URLSearchParams(url.substring(url.indexOf('?') + 1));
|
|
const actualUrl = urlParams.get('uddg');
|
|
if (actualUrl) {
|
|
// Decode the URL
|
|
const decodedUrl = decodeURIComponent(actualUrl);
|
|
console.log(`[SearchEngine] Decoded DuckDuckGo URL: ${decodedUrl}`);
|
|
return decodedUrl;
|
|
}
|
|
}
|
|
catch {
|
|
console.log(`[SearchEngine] Failed to decode DuckDuckGo URL: ${url}`);
|
|
}
|
|
}
|
|
// If it's a protocol-relative URL, add https:
|
|
if (url.startsWith('//')) {
|
|
return 'https:' + url;
|
|
}
|
|
return url;
|
|
}
|
|
assessResultQuality(results, originalQuery) {
|
|
if (results.length === 0)
|
|
return 0;
|
|
// Extract keywords from the original query (ignore common words)
|
|
const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'can', 'group', 'members']);
|
|
const queryWords = originalQuery.toLowerCase()
|
|
.replace(/[^\w\s]/g, ' ')
|
|
.split(/\s+/)
|
|
.filter(word => word.length > 2 && !commonWords.has(word));
|
|
if (queryWords.length === 0)
|
|
return 0.5; // Default score if no meaningful keywords
|
|
console.log(`[SearchEngine] Quality assessment - Query keywords: [${queryWords.join(', ')}]`);
|
|
let totalScore = 0;
|
|
let scoredResults = 0;
|
|
for (const result of results) {
|
|
const titleText = result.title.toLowerCase();
|
|
const descText = result.description.toLowerCase();
|
|
const urlText = result.url.toLowerCase();
|
|
const combinedText = `${titleText} ${descText} ${urlText}`;
|
|
// Count keyword matches
|
|
let keywordMatches = 0;
|
|
let phraseMatches = 0;
|
|
// Check for exact phrase matches (higher value)
|
|
if (queryWords.length >= 2) {
|
|
const queryPhrases = [];
|
|
for (let i = 0; i < queryWords.length - 1; i++) {
|
|
queryPhrases.push(queryWords.slice(i, i + 2).join(' '));
|
|
}
|
|
if (queryWords.length >= 3) {
|
|
queryPhrases.push(queryWords.slice(0, 3).join(' '));
|
|
}
|
|
for (const phrase of queryPhrases) {
|
|
if (combinedText.includes(phrase)) {
|
|
phraseMatches++;
|
|
}
|
|
}
|
|
}
|
|
// Check individual keyword matches
|
|
for (const keyword of queryWords) {
|
|
if (combinedText.includes(keyword)) {
|
|
keywordMatches++;
|
|
}
|
|
}
|
|
// Calculate score for this result
|
|
const keywordRatio = keywordMatches / queryWords.length;
|
|
const phraseBonus = phraseMatches * 0.3; // Bonus for phrase matches
|
|
const resultScore = Math.min(1.0, keywordRatio + phraseBonus);
|
|
// Penalty for obvious irrelevant content
|
|
const irrelevantPatterns = [
|
|
/recipe/i, /cooking/i, /food/i, /restaurant/i, /menu/i,
|
|
/weather/i, /temperature/i, /forecast/i,
|
|
/shopping/i, /sale/i, /price/i, /buy/i, /store/i,
|
|
/movie/i, /film/i, /tv show/i, /entertainment/i,
|
|
/sports/i, /game/i, /score/i, /team/i,
|
|
/fashion/i, /clothing/i, /style/i,
|
|
/travel/i, /hotel/i, /flight/i, /vacation/i,
|
|
/car/i, /vehicle/i, /automotive/i,
|
|
/real estate/i, /property/i, /house/i, /apartment/i
|
|
];
|
|
let penalty = 0;
|
|
for (const pattern of irrelevantPatterns) {
|
|
if (pattern.test(combinedText)) {
|
|
penalty += 0.2;
|
|
}
|
|
}
|
|
const finalScore = Math.max(0, resultScore - penalty);
|
|
console.log(`[SearchEngine] Result "${result.title.substring(0, 50)}..." - Score: ${finalScore.toFixed(2)} (keywords: ${keywordMatches}/${queryWords.length}, phrases: ${phraseMatches}, penalty: ${penalty.toFixed(2)})`);
|
|
totalScore += finalScore;
|
|
scoredResults++;
|
|
}
|
|
const averageScore = scoredResults > 0 ? totalScore / scoredResults : 0;
|
|
return averageScore;
|
|
}
|
|
async validateBrowserHealth(browser) {
|
|
const debugBrowsers = process.env.DEBUG_BROWSER_LIFECYCLE === 'true';
|
|
try {
|
|
if (debugBrowsers)
|
|
console.log(`[SearchEngine] Validating browser health...`);
|
|
// Check if browser is still connected
|
|
if (!browser.isConnected()) {
|
|
if (debugBrowsers)
|
|
console.log(`[SearchEngine] Browser is not connected`);
|
|
return false;
|
|
}
|
|
// Try to create a simple context to test browser responsiveness
|
|
const testContext = await browser.newContext();
|
|
await testContext.close();
|
|
if (debugBrowsers)
|
|
console.log(`[SearchEngine] Browser health check passed`);
|
|
return true;
|
|
}
|
|
catch (error) {
|
|
console.log(`[SearchEngine] Browser health check failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
return false;
|
|
}
|
|
}
|
|
async handleBrowserError(error, engineName, attemptNumber = 1) {
|
|
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
|
console.error(`[SearchEngine] ${engineName} browser error (attempt ${attemptNumber}): ${errorMessage}`);
|
|
// Check for specific browser-related errors
|
|
if (errorMessage.includes('Target page, context or browser has been closed') ||
|
|
errorMessage.includes('Browser has been closed') ||
|
|
errorMessage.includes('Session has been closed')) {
|
|
console.log(`[SearchEngine] Detected browser session closure, attempting to refresh browser pool`);
|
|
// Try to refresh the browser pool for subsequent attempts
|
|
try {
|
|
await this.browserPool.closeAll();
|
|
console.log(`[SearchEngine] Browser pool refreshed for ${engineName}`);
|
|
}
|
|
catch (refreshError) {
|
|
console.error(`[SearchEngine] Failed to refresh browser pool: ${refreshError instanceof Error ? refreshError.message : 'Unknown error'}`);
|
|
}
|
|
}
|
|
}
|
|
async closeAll() {
|
|
await this.browserPool.closeAll();
|
|
}
|
|
}
|
|
//# sourceMappingURL=search-engine.js.map
|