web-search-mcp
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
node_modules/
|
||||
package-lock.json
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Browser } from 'playwright';
|
||||
export declare class BrowserPool {
|
||||
private browsers;
|
||||
private maxBrowsers;
|
||||
private browserTypes;
|
||||
private currentBrowserIndex;
|
||||
private headless;
|
||||
private lastUsedBrowserType;
|
||||
constructor();
|
||||
getBrowser(): Promise<Browser>;
|
||||
closeAll(): Promise<void>;
|
||||
getLastUsedBrowserType(): string;
|
||||
}
|
||||
//# sourceMappingURL=browser-pool.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,OAAO,EAAE,MAAM,YAAY,CAAC;AAEhE,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAW;IAC/B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,mBAAmB,CAAc;;IAcnC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IA4F9B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAa/B,sBAAsB,IAAI,MAAM;CAGjC"}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { chromium, firefox, webkit } from 'playwright';
|
||||
export class BrowserPool {
|
||||
browsers = new Map();
|
||||
maxBrowsers;
|
||||
browserTypes;
|
||||
currentBrowserIndex = 0;
|
||||
headless;
|
||||
lastUsedBrowserType = '';
|
||||
constructor() {
|
||||
// Read configuration from environment variables
|
||||
this.maxBrowsers = parseInt(process.env.MAX_BROWSERS || '3', 10);
|
||||
this.headless = process.env.BROWSER_HEADLESS !== 'false'; // Default to true
|
||||
// Configure browser types based on environment
|
||||
const browserTypesEnv = process.env.BROWSER_TYPES || 'chromium,firefox';
|
||||
this.browserTypes = browserTypesEnv.split(',').map(type => type.trim());
|
||||
console.log(`[BrowserPool] Configuration: maxBrowsers=${this.maxBrowsers}, headless=${this.headless}, types=${this.browserTypes.join(',')}`);
|
||||
}
|
||||
async getBrowser() {
|
||||
// Rotate between browser types for variety
|
||||
const browserType = this.browserTypes[this.currentBrowserIndex % this.browserTypes.length];
|
||||
this.currentBrowserIndex++;
|
||||
this.lastUsedBrowserType = browserType;
|
||||
if (this.browsers.has(browserType)) {
|
||||
const browser = this.browsers.get(browserType);
|
||||
// Check if browser is still connected and healthy
|
||||
try {
|
||||
if (browser.isConnected()) {
|
||||
// Quick health check by trying to create and close a context
|
||||
// Use minimal options to avoid Firefox isMobile issues
|
||||
const testContext = await browser.newContext({
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
|
||||
});
|
||||
await testContext.close();
|
||||
return browser;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.log(`[BrowserPool] Browser ${browserType} health check failed:`, error);
|
||||
// Browser is unhealthy, remove it and close if possible
|
||||
this.browsers.delete(browserType);
|
||||
try {
|
||||
await browser.close();
|
||||
}
|
||||
catch (closeError) {
|
||||
console.log(`[BrowserPool] Error closing unhealthy browser:`, closeError);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Launch new browser
|
||||
console.log(`[BrowserPool] Launching new ${browserType} browser`);
|
||||
const launchOptions = {
|
||||
headless: this.headless,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--disable-default-apps',
|
||||
'--disable-extensions',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-features=TranslateUI',
|
||||
'--disable-ipc-flooding-protection',
|
||||
],
|
||||
};
|
||||
let browser;
|
||||
try {
|
||||
switch (browserType) {
|
||||
case 'chromium':
|
||||
browser = await chromium.launch(launchOptions);
|
||||
break;
|
||||
case 'firefox':
|
||||
browser = await firefox.launch(launchOptions);
|
||||
break;
|
||||
case 'webkit':
|
||||
browser = await webkit.launch(launchOptions);
|
||||
break;
|
||||
default:
|
||||
browser = await chromium.launch(launchOptions);
|
||||
}
|
||||
this.browsers.set(browserType, browser);
|
||||
// Clean up old browsers if we have too many
|
||||
if (this.browsers.size > this.maxBrowsers) {
|
||||
const oldestBrowser = this.browsers.entries().next().value;
|
||||
if (oldestBrowser) {
|
||||
try {
|
||||
await oldestBrowser[1].close();
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`[BrowserPool] Error closing old browser:`, error);
|
||||
}
|
||||
this.browsers.delete(oldestBrowser[0]);
|
||||
}
|
||||
}
|
||||
return browser;
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`[BrowserPool] Failed to launch ${browserType} browser:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
async closeAll() {
|
||||
console.log(`[BrowserPool] Closing ${this.browsers.size} browsers`);
|
||||
const closePromises = Array.from(this.browsers.values()).map(browser => browser.close().catch(error => console.error('Error closing browser:', error)));
|
||||
await Promise.all(closePromises);
|
||||
this.browsers.clear();
|
||||
}
|
||||
getLastUsedBrowserType() {
|
||||
return this.lastUsedBrowserType;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=browser-pool.js.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"browser-pool.js","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAW,MAAM,YAAY,CAAC;AAEhE,MAAM,OAAO,WAAW;IACd,QAAQ,GAAyB,IAAI,GAAG,EAAE,CAAC;IAC3C,WAAW,CAAS;IACpB,YAAY,CAAW;IACvB,mBAAmB,GAAG,CAAC,CAAC;IACxB,QAAQ,CAAU;IAClB,mBAAmB,GAAW,EAAE,CAAC;IAEzC;QACE,gDAAgD;QAChD,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO,CAAC,CAAC,kBAAkB;QAE5E,+CAA+C;QAC/C,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,kBAAkB,CAAC;QACxE,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAExE,OAAO,CAAC,GAAG,CAAC,4CAA4C,IAAI,CAAC,WAAW,cAAc,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/I,CAAC;IAED,KAAK,CAAC,UAAU;QACd,2CAA2C;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC3F,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,WAAW,CAAC;QAEvC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC;YAEhD,kDAAkD;YAClD,IAAI,CAAC;gBACH,IAAI,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC1B,6DAA6D;oBAC7D,uDAAuD;oBACvD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;wBAC3C,SAAS,EAAE,uHAAuH;qBACnI,CAAC,CAAC;oBACH,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;oBAC1B,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,uBAAuB,EAAE,KAAK,CAAC,CAAC;gBAChF,wDAAwD;gBACxD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACxB,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,OAAO,CAAC,GAAG,CAAC,gDAAgD,EAAE,UAAU,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,OAAO,CAAC,GAAG,CAAC,+BAA+B,WAAW,UAAU,CAAC,CAAC;QAElE,MAAM,aAAa,GAAG;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE;gBACJ,cAAc;gBACd,+CAA+C;gBAC/C,yBAAyB;gBACzB,eAAe;gBACf,gBAAgB;gBAChB,4BAA4B;gBAC5B,wBAAwB;gBACxB,sBAAsB;gBACtB,uCAAuC;gBACvC,0CAA0C;gBAC1C,kCAAkC;gBAClC,gCAAgC;gBAChC,mCAAmC;aACpC;SACF,CAAC;QAEF,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,WAAW,EAAE,CAAC;gBACpB,KAAK,UAAU;oBACb,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC/C,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC9C,MAAM;gBACR,KAAK,QAAQ;oBACX,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC7C,MAAM;gBACR;oBACE,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAExC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;gBAC3D,IAAI,aAAa,EAAE,CAAC;oBAClB,IAAI,CAAC;wBACH,MAAM,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;oBACjC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;oBACnE,CAAC;oBACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kCAAkC,WAAW,WAAW,EAAE,KAAK,CAAC,CAAC;YAC/E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,CAAC;QAEpE,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACrE,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAC5B,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAC/C,CACF,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;CACF"}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { ContentExtractionOptions, SearchResult } from './types.js';
|
||||
export declare class ContentExtractor {
|
||||
private readonly defaultTimeout;
|
||||
private readonly maxContentLength;
|
||||
constructor();
|
||||
extractContent(options: ContentExtractionOptions): Promise<string>;
|
||||
extractContentForResults(results: SearchResult[], targetCount?: number): Promise<SearchResult[]>;
|
||||
private parseContent;
|
||||
private cleanTextContent;
|
||||
private getSpecificErrorMessage;
|
||||
}
|
||||
//# sourceMappingURL=content-extractor.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"content-extractor.d.ts","sourceRoot":"","sources":["../src/content-extractor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGpE,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;;IAepC,cAAc,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC;IA+DlE,wBAAwB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,WAAW,GAAE,MAAuB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqDtH,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,uBAAuB;CAsBhC"}
|
||||
@@ -0,0 +1,224 @@
|
||||
import axios from 'axios';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js';
|
||||
export class ContentExtractor {
|
||||
defaultTimeout;
|
||||
maxContentLength;
|
||||
constructor() {
|
||||
this.defaultTimeout = 10000;
|
||||
// Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB
|
||||
const envMaxLength = process.env.MAX_CONTENT_LENGTH;
|
||||
this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000;
|
||||
// Validate the parsed value
|
||||
if (isNaN(this.maxContentLength) || this.maxContentLength < 0) {
|
||||
console.warn(`[ContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`);
|
||||
this.maxContentLength = 500000;
|
||||
}
|
||||
}
|
||||
async extractContent(options) {
|
||||
const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options;
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'DNT': '1',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"macOS"',
|
||||
},
|
||||
timeout,
|
||||
maxContentLength,
|
||||
validateStatus: (status) => status < 400,
|
||||
});
|
||||
return this.parseContent(response.data);
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Content extraction error for ${url}:`, error);
|
||||
// If it's a 403 error, try with different headers
|
||||
if (axios.isAxiosError(error) && error.response?.status === 403) {
|
||||
console.log(`[ContentExtractor] Trying alternative headers for ${url}`);
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'DNT': '1',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Pragma': 'no-cache',
|
||||
},
|
||||
timeout,
|
||||
maxContentLength,
|
||||
validateStatus: (status) => status < 400,
|
||||
});
|
||||
console.log(`[ContentExtractor] Alternative headers worked for ${url}`);
|
||||
return this.parseContent(response.data);
|
||||
}
|
||||
catch (retryError) {
|
||||
console.error(`[ContentExtractor] Alternative headers also failed for ${url}:`, retryError);
|
||||
}
|
||||
}
|
||||
throw new Error(`Failed to extract content from ${url}: ${this.getSpecificErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
async extractContentForResults(results, targetCount = results.length) {
|
||||
const enhancedResults = [];
|
||||
let processedCount = 0;
|
||||
console.log(`[ContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`);
|
||||
for (const result of results) {
|
||||
if (enhancedResults.length >= targetCount) {
|
||||
console.log(`[ContentExtractor] Reached target count of ${targetCount} results`);
|
||||
break;
|
||||
}
|
||||
processedCount++;
|
||||
// Skip PDF files
|
||||
if (isPdfUrl(result.url)) {
|
||||
console.log(`[ContentExtractor] Skipping PDF file: ${result.url}`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
console.log(`[ContentExtractor] Extracting content from: ${result.url}`);
|
||||
const content = await this.extractContent({ url: result.url });
|
||||
const cleanedContent = cleanText(content, this.maxContentLength);
|
||||
enhancedResults.push({
|
||||
...result,
|
||||
fullContent: cleanedContent,
|
||||
contentPreview: getContentPreview(cleanedContent),
|
||||
wordCount: getWordCount(cleanedContent),
|
||||
timestamp: generateTimestamp(),
|
||||
fetchStatus: 'success',
|
||||
});
|
||||
console.log(`[ContentExtractor] Successfully extracted content (${enhancedResults.length}/${targetCount})`);
|
||||
}
|
||||
catch (error) {
|
||||
console.log(`[ContentExtractor] Failed to extract content from ${result.url}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
enhancedResults.push({
|
||||
...result,
|
||||
fullContent: '',
|
||||
contentPreview: '',
|
||||
wordCount: 0,
|
||||
timestamp: generateTimestamp(),
|
||||
fetchStatus: 'error',
|
||||
error: this.getSpecificErrorMessage(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
console.log(`[ContentExtractor] Processed ${processedCount} results, extracted ${enhancedResults.length} non-PDF results`);
|
||||
return enhancedResults;
|
||||
}
|
||||
parseContent(html) {
|
||||
const $ = cheerio.load(html);
|
||||
// Remove all script, style, and other non-content elements
|
||||
$('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove();
|
||||
// Remove navigation, header, footer, and other non-content elements
|
||||
$('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove();
|
||||
// Remove elements with common ad/tracking classes
|
||||
$('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove();
|
||||
// Remove elements with common non-content IDs
|
||||
$('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove();
|
||||
// Remove image-related elements and attributes
|
||||
$('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove();
|
||||
$('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove();
|
||||
$('[style*="background-image"]').remove();
|
||||
// Remove empty elements and whitespace-only elements
|
||||
$('*').each(function () {
|
||||
const $this = $(this);
|
||||
if ($this.children().length === 0 && $this.text().trim() === '') {
|
||||
$this.remove();
|
||||
}
|
||||
});
|
||||
// Try to find the main content area first
|
||||
let mainContent = '';
|
||||
// Priority selectors for main content
|
||||
const contentSelectors = [
|
||||
'article',
|
||||
'main',
|
||||
'[role="main"]',
|
||||
'.content',
|
||||
'.post-content',
|
||||
'.entry-content',
|
||||
'.article-content',
|
||||
'.story-content',
|
||||
'.news-content',
|
||||
'.main-content',
|
||||
'.page-content',
|
||||
'.text-content',
|
||||
'.body-content',
|
||||
'.copy',
|
||||
'.text',
|
||||
'.body'
|
||||
];
|
||||
for (const selector of contentSelectors) {
|
||||
const $content = $(selector).first();
|
||||
if ($content.length > 0) {
|
||||
mainContent = $content.text().trim();
|
||||
if (mainContent.length > 100) { // Ensure we have substantial content
|
||||
console.log(`[ContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// If no main content found, try body content
|
||||
if (!mainContent || mainContent.length < 100) {
|
||||
console.log(`[ContentExtractor] No main content found, using body content`);
|
||||
mainContent = $('body').text().trim();
|
||||
}
|
||||
// Clean up the text
|
||||
const cleanedContent = this.cleanTextContent(mainContent);
|
||||
return cleanText(cleanedContent, this.maxContentLength);
|
||||
}
|
||||
cleanTextContent(text) {
|
||||
// Remove excessive whitespace
|
||||
text = text.replace(/\s+/g, ' ');
|
||||
// Remove image-related text and data URLs
|
||||
text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data
|
||||
text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs
|
||||
text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions
|
||||
text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words
|
||||
text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text
|
||||
// Remove common non-content patterns
|
||||
text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, '');
|
||||
// Remove excessive line breaks and spacing
|
||||
text = text.replace(/\n\s*\n/g, '\n');
|
||||
text = text.replace(/\r\n/g, '\n');
|
||||
text = text.replace(/\r/g, '\n');
|
||||
// Remove leading/trailing whitespace
|
||||
text = text.trim();
|
||||
return text;
|
||||
}
|
||||
getSpecificErrorMessage(error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
if (error.code === 'ECONNABORTED') {
|
||||
return 'Request timeout';
|
||||
}
|
||||
if (error.response?.status === 403) {
|
||||
return '403 Forbidden - Access denied';
|
||||
}
|
||||
if (error.response?.status === 404) {
|
||||
return '404 Not found';
|
||||
}
|
||||
if (error.message.includes('maxContentLength')) {
|
||||
return 'Content too long';
|
||||
}
|
||||
if (error.response?.status) {
|
||||
return `HTTP ${error.response.status}: ${error.message}`;
|
||||
}
|
||||
return `Network error: ${error.message}`;
|
||||
}
|
||||
return error instanceof Error ? error.message : 'Unknown error';
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=content-extractor.js.map
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,24 @@
|
||||
import { ContentExtractionOptions, SearchResult } from './types.js';
|
||||
export declare class EnhancedContentExtractor {
|
||||
private readonly defaultTimeout;
|
||||
private readonly maxContentLength;
|
||||
private browserPool;
|
||||
private fallbackThreshold;
|
||||
constructor();
|
||||
extractContent(options: ContentExtractionOptions): Promise<string>;
|
||||
private extractWithAxios;
|
||||
private extractWithBrowser;
|
||||
private simulateHumanBehavior;
|
||||
private shouldUseBrowser;
|
||||
private isLowQualityContent;
|
||||
private getRandomHeaders;
|
||||
private getRandomUserAgent;
|
||||
private getRandomViewport;
|
||||
private getRandomTimezone;
|
||||
extractContentForResults(results: SearchResult[], targetCount?: number): Promise<SearchResult[]>;
|
||||
private parseContent;
|
||||
private cleanTextContent;
|
||||
private getSpecificErrorMessage;
|
||||
closeAll(): Promise<void>;
|
||||
}
|
||||
//# sourceMappingURL=enhanced-content-extractor.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"enhanced-content-extractor.d.ts","sourceRoot":"","sources":["../src/enhanced-content-extractor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAIpE,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,iBAAiB,CAAS;;IAqB5B,cAAc,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC;YA8B1D,gBAAgB;YA0BhB,kBAAkB;YAgKlB,qBAAqB;IA6BnC,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,mBAAmB;IAe3B,OAAO,CAAC,gBAAgB;IAsCxB,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,iBAAiB;IAanB,wBAAwB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,WAAW,GAAE,MAAuB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAiEtH,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,uBAAuB;IAuBzB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGhC"}
|
||||
@@ -0,0 +1,499 @@
|
||||
import axios from 'axios';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js';
|
||||
import { BrowserPool } from './browser-pool.js';
|
||||
export class EnhancedContentExtractor {
|
||||
defaultTimeout;
|
||||
maxContentLength;
|
||||
browserPool;
|
||||
fallbackThreshold;
|
||||
constructor() {
|
||||
this.defaultTimeout = parseInt(process.env.DEFAULT_TIMEOUT || '6000', 10);
|
||||
// Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB
|
||||
const envMaxLength = process.env.MAX_CONTENT_LENGTH;
|
||||
this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000;
|
||||
// Validate the parsed value
|
||||
if (isNaN(this.maxContentLength) || this.maxContentLength < 0) {
|
||||
console.warn(`[EnhancedContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`);
|
||||
this.maxContentLength = 500000;
|
||||
}
|
||||
this.browserPool = new BrowserPool();
|
||||
this.fallbackThreshold = parseInt(process.env.BROWSER_FALLBACK_THRESHOLD || '3', 10);
|
||||
console.log(`[EnhancedContentExtractor] Configuration: timeout=${this.defaultTimeout}, maxContentLength=${this.maxContentLength}, fallbackThreshold=${this.fallbackThreshold}`);
|
||||
}
|
||||
async extractContent(options) {
|
||||
const { url } = options;
|
||||
console.log(`[EnhancedContentExtractor] Starting extraction for: ${url}`);
|
||||
// First, try with regular HTTP client (faster)
|
||||
try {
|
||||
const content = await this.extractWithAxios(options);
|
||||
console.log(`[EnhancedContentExtractor] Successfully extracted with axios: ${content.length} chars`);
|
||||
return content;
|
||||
}
|
||||
catch (error) {
|
||||
console.log(`[EnhancedContentExtractor] Axios failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
// Check if this looks like a case where browser would help
|
||||
if (this.shouldUseBrowser(error, url)) {
|
||||
console.log(`[EnhancedContentExtractor] Falling back to headless browser for: ${url}`);
|
||||
try {
|
||||
const content = await this.extractWithBrowser(options);
|
||||
console.log(`[EnhancedContentExtractor] Successfully extracted with browser: ${content.length} chars`);
|
||||
return content;
|
||||
}
|
||||
catch (browserError) {
|
||||
console.error(`[EnhancedContentExtractor] Browser extraction also failed:`, browserError);
|
||||
throw new Error(`Both axios and browser extraction failed for ${url}`);
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
async extractWithAxios(options) {
|
||||
const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options;
|
||||
const response = await axios.get(url, {
|
||||
headers: this.getRandomHeaders(),
|
||||
timeout,
|
||||
// Remove maxContentLength from axios config - handle truncation manually
|
||||
validateStatus: (status) => status < 400,
|
||||
});
|
||||
let content = this.parseContent(response.data);
|
||||
// Truncate content if it exceeds the limit (instead of axios throwing an error)
|
||||
if (maxContentLength && content.length > maxContentLength) {
|
||||
console.log(`[EnhancedContentExtractor] Content truncated from ${content.length} to ${maxContentLength} characters for ${url}`);
|
||||
content = content.substring(0, maxContentLength);
|
||||
}
|
||||
// Check if we got a meaningful response
|
||||
if (this.isLowQualityContent(content)) {
|
||||
throw new Error('Low quality content detected - likely bot detection');
|
||||
}
|
||||
return content;
|
||||
}
|
||||
async extractWithBrowser(options) {
|
||||
const { url, timeout = this.defaultTimeout } = options;
|
||||
const browser = await this.browserPool.getBrowser();
|
||||
const browserType = this.browserPool.getLastUsedBrowserType();
|
||||
try {
|
||||
// Create context options based on browser capabilities
|
||||
const baseContextOptions = {
|
||||
userAgent: this.getRandomUserAgent(),
|
||||
viewport: this.getRandomViewport(),
|
||||
locale: 'en-US',
|
||||
timezoneId: this.getRandomTimezone(),
|
||||
// Simulate real device characteristics
|
||||
deviceScaleFactor: Math.random() > 0.5 ? 1 : 2,
|
||||
hasTouch: Math.random() > 0.7,
|
||||
};
|
||||
// Firefox doesn't support isMobile option - check multiple ways to ensure detection
|
||||
const isFirefox = browserType === 'firefox' ||
|
||||
browserType.includes('firefox') ||
|
||||
browser.constructor.name.toLowerCase().includes('firefox');
|
||||
const contextOptions = isFirefox
|
||||
? baseContextOptions
|
||||
: { ...baseContextOptions, isMobile: Math.random() > 0.8 };
|
||||
// Create a new context for each request (isolation)
|
||||
const context = await browser.newContext(contextOptions);
|
||||
// Add stealth scripts to avoid detection
|
||||
await context.addInitScript(() => {
|
||||
// Remove webdriver property
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => undefined,
|
||||
});
|
||||
// Mock plugins
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [1, 2, 3, 4, 5],
|
||||
});
|
||||
// Mock languages
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['en-US', 'en'],
|
||||
});
|
||||
// Mock permissions
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => (parameters.name === 'notifications' ?
|
||||
Promise.resolve({ state: 'default' }) :
|
||||
originalQuery(parameters));
|
||||
// Remove automation indicators
|
||||
const windowWithChrome = window;
|
||||
if (windowWithChrome.chrome) {
|
||||
delete windowWithChrome.chrome.app;
|
||||
delete windowWithChrome.chrome.runtime;
|
||||
}
|
||||
});
|
||||
const page = await context.newPage();
|
||||
// Set up request interception to block unnecessary resources
|
||||
await page.route('**/*', (route) => {
|
||||
const resourceType = route.request().resourceType();
|
||||
// Block images, fonts, and other non-essential resources for faster loading
|
||||
if (['image', 'font', 'media'].includes(resourceType)) {
|
||||
route.abort();
|
||||
}
|
||||
else {
|
||||
route.continue();
|
||||
}
|
||||
});
|
||||
// Navigate with realistic options and better error handling
|
||||
console.log(`[BrowserExtractor] Navigating to ${url}`);
|
||||
try {
|
||||
await page.goto(url, {
|
||||
waitUntil: 'domcontentloaded', // Don't wait for all resources
|
||||
timeout: Math.min(timeout, 8000) // Reduced timeout, max 8 seconds
|
||||
});
|
||||
}
|
||||
catch (gotoError) {
|
||||
// Handle specific protocol errors
|
||||
const errorMessage = gotoError instanceof Error ? gotoError.message : String(gotoError);
|
||||
if (errorMessage.includes('ERR_HTTP2_PROTOCOL_ERROR') || errorMessage.includes('HTTP2')) {
|
||||
console.log(`[BrowserExtractor] HTTP/2 error detected, trying with HTTP/1.1`);
|
||||
// Create a new context with HTTP/1.1 preference
|
||||
await context.close();
|
||||
const http1Context = await browser.newContext({
|
||||
userAgent: this.getRandomUserAgent(),
|
||||
viewport: this.getRandomViewport(),
|
||||
locale: 'en-US',
|
||||
timezoneId: this.getRandomTimezone(),
|
||||
extraHTTPHeaders: {
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1'
|
||||
}
|
||||
});
|
||||
const http1Page = await http1Context.newPage();
|
||||
// Disable HTTP/2 by intercepting requests
|
||||
await http1Page.route('**/*', (route) => {
|
||||
const resourceType = route.request().resourceType();
|
||||
if (['image', 'font', 'media'].includes(resourceType)) {
|
||||
route.abort();
|
||||
}
|
||||
else {
|
||||
route.continue();
|
||||
}
|
||||
});
|
||||
await http1Page.goto(url, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: Math.min(timeout, 6000)
|
||||
});
|
||||
// Quick content extraction
|
||||
const html = await http1Page.content();
|
||||
const content = this.parseContent(html);
|
||||
await http1Context.close();
|
||||
return content;
|
||||
}
|
||||
else {
|
||||
throw gotoError;
|
||||
}
|
||||
}
|
||||
// Quick human simulation - reduced time
|
||||
await page.mouse.move(Math.random() * 100, Math.random() * 100);
|
||||
// Reduced wait time for dynamic content
|
||||
await page.waitForTimeout(500 + Math.random() * 1000);
|
||||
// Quick check for main content without long wait
|
||||
try {
|
||||
await page.waitForSelector('article, main, .content, .post-content, .entry-content', {
|
||||
timeout: 2000
|
||||
});
|
||||
}
|
||||
catch {
|
||||
console.log(`[BrowserExtractor] No main content selector found, proceeding anyway`);
|
||||
}
|
||||
// Extract content using the same logic as axios version
|
||||
const html = await page.content();
|
||||
const content = this.parseContent(html);
|
||||
await context.close();
|
||||
return content;
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`[BrowserExtractor] Browser extraction failed for ${url}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
async simulateHumanBehavior(page) {
|
||||
try {
|
||||
// Random mouse movements
|
||||
await page.mouse.move(Math.random() * 800, Math.random() * 600);
|
||||
// Random scroll (common human behavior)
|
||||
const scrollY = Math.random() * 500;
|
||||
await page.evaluate((y) => window.scrollTo(0, y), scrollY);
|
||||
// Small random delay
|
||||
await page.waitForTimeout(500 + Math.random() * 1000);
|
||||
// Sometimes click somewhere innocuous
|
||||
if (Math.random() > 0.7) {
|
||||
try {
|
||||
await page.click('body', { timeout: 1000 });
|
||||
}
|
||||
catch {
|
||||
// Ignore click failures
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Ignore simulation errors
|
||||
console.log(`[BrowserExtractor] Behavior simulation failed, continuing`);
|
||||
}
|
||||
}
|
||||
shouldUseBrowser(error, url) {
|
||||
// Conditions where browser is likely to succeed where axios failed
|
||||
const indicators = [
|
||||
// HTTP status codes that suggest bot detection
|
||||
error.response?.status === 403,
|
||||
error.response?.status === 429,
|
||||
error.response?.status === 503,
|
||||
// Error messages suggesting JS requirement
|
||||
error.message?.includes('timeout'),
|
||||
error.message?.includes('Access denied'),
|
||||
error.message?.includes('Forbidden'),
|
||||
error.message?.includes('Low quality content detected'),
|
||||
// Response content suggesting bot detection
|
||||
error.response?.data?.includes('Please enable JavaScript'),
|
||||
error.response?.data?.includes('captcha'),
|
||||
error.response?.data?.includes('unusual traffic'),
|
||||
error.response?.data?.includes('robot'),
|
||||
// Sites known to be JS-heavy
|
||||
url.includes('twitter.com'),
|
||||
url.includes('facebook.com'),
|
||||
url.includes('instagram.com'),
|
||||
url.includes('linkedin.com'),
|
||||
url.includes('reddit.com'),
|
||||
url.includes('medium.com'),
|
||||
];
|
||||
return indicators.some(indicator => indicator === true);
|
||||
}
|
||||
isLowQualityContent(content) {
|
||||
const lowQualityIndicators = [
|
||||
content.length < 100,
|
||||
content.includes('Please enable JavaScript'),
|
||||
content.includes('Access Denied'),
|
||||
content.includes('403 Forbidden'),
|
||||
content.includes('captcha'),
|
||||
content.includes('unusual traffic'),
|
||||
content.includes('robot'),
|
||||
content.trim() === '',
|
||||
];
|
||||
return lowQualityIndicators.some(indicator => indicator === true);
|
||||
}
|
||||
getRandomHeaders() {
|
||||
const browsers = [
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
},
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
|
||||
'sec-ch-ua-platform': '"macOS"',
|
||||
},
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
|
||||
'sec-ch-ua-platform': '"Linux"',
|
||||
}
|
||||
];
|
||||
const browser = browsers[Math.floor(Math.random() * browsers.length)];
|
||||
return {
|
||||
...browser,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'DNT': '1',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
};
|
||||
}
|
||||
getRandomUserAgent() {
|
||||
const userAgents = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0',
|
||||
];
|
||||
return userAgents[Math.floor(Math.random() * userAgents.length)];
|
||||
}
|
||||
getRandomViewport() {
|
||||
const viewports = [
|
||||
{ width: 1920, height: 1080 },
|
||||
{ width: 1366, height: 768 },
|
||||
{ width: 1440, height: 900 },
|
||||
{ width: 1536, height: 864 },
|
||||
{ width: 1280, height: 720 },
|
||||
];
|
||||
return viewports[Math.floor(Math.random() * viewports.length)];
|
||||
}
|
||||
getRandomTimezone() {
|
||||
const timezones = [
|
||||
'America/New_York',
|
||||
'America/Los_Angeles',
|
||||
'America/Chicago',
|
||||
'Europe/London',
|
||||
'Europe/Berlin',
|
||||
'Asia/Tokyo',
|
||||
];
|
||||
return timezones[Math.floor(Math.random() * timezones.length)];
|
||||
}
|
||||
async extractContentForResults(results, targetCount = results.length) {
|
||||
console.log(`[EnhancedContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`);
|
||||
// Filter out PDF files first
|
||||
const nonPdfResults = results.filter(result => !isPdfUrl(result.url));
|
||||
const resultsToProcess = nonPdfResults.slice(0, Math.min(targetCount * 2, 10)); // Process extra to account for failures
|
||||
console.log(`[EnhancedContentExtractor] Processing ${resultsToProcess.length} non-PDF results concurrently`);
|
||||
// Process results concurrently with timeout
|
||||
const extractionPromises = resultsToProcess.map(async (result) => {
|
||||
try {
|
||||
// Use a race condition with timeout to prevent hanging
|
||||
const extractionPromise = this.extractContent({
|
||||
url: result.url,
|
||||
timeout: 6000 // Reduced timeout to 6 seconds per page
|
||||
});
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error('Content extraction timeout')), 8000);
|
||||
});
|
||||
const content = await Promise.race([extractionPromise, timeoutPromise]);
|
||||
const cleanedContent = cleanText(content, this.maxContentLength);
|
||||
console.log(`[EnhancedContentExtractor] Successfully extracted: ${result.url}`);
|
||||
return {
|
||||
...result,
|
||||
fullContent: cleanedContent,
|
||||
contentPreview: getContentPreview(cleanedContent),
|
||||
wordCount: getWordCount(cleanedContent),
|
||||
timestamp: generateTimestamp(),
|
||||
fetchStatus: 'success',
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
console.log(`[EnhancedContentExtractor] Failed to extract: ${result.url} - ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
return {
|
||||
...result,
|
||||
fullContent: '',
|
||||
contentPreview: '',
|
||||
wordCount: 0,
|
||||
timestamp: generateTimestamp(),
|
||||
fetchStatus: 'error',
|
||||
error: this.getSpecificErrorMessage(error),
|
||||
};
|
||||
}
|
||||
});
|
||||
// Wait for all extractions to complete
|
||||
const allResults = await Promise.all(extractionPromises);
|
||||
// Return successful results first, up to targetCount
|
||||
const successfulResults = allResults.filter(r => r.fetchStatus === 'success');
|
||||
const failedResults = allResults.filter(r => r.fetchStatus === 'error');
|
||||
// Combine successful and failed results, prioritizing successful ones
|
||||
const enhancedResults = [
|
||||
...successfulResults.slice(0, targetCount),
|
||||
...failedResults.slice(0, Math.max(0, targetCount - successfulResults.length))
|
||||
].slice(0, targetCount);
|
||||
console.log(`[EnhancedContentExtractor] Completed processing ${resultsToProcess.length} results, extracted ${successfulResults.length} successful/${failedResults.length} failed`);
|
||||
return enhancedResults;
|
||||
}
|
||||
parseContent(html) {
|
||||
const $ = cheerio.load(html);
|
||||
// Remove all script, style, and other non-content elements
|
||||
$('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove();
|
||||
// Remove navigation, header, footer, and other non-content elements
|
||||
$('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove();
|
||||
// Remove elements with common ad/tracking classes
|
||||
$('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove();
|
||||
// Remove elements with common non-content IDs
|
||||
$('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove();
|
||||
// Remove image-related elements and attributes
|
||||
$('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove();
|
||||
$('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove();
|
||||
$('[style*="background-image"]').remove();
|
||||
// Remove empty elements and whitespace-only elements
|
||||
$('*').each(function () {
|
||||
const $this = $(this);
|
||||
if ($this.children().length === 0 && $this.text().trim() === '') {
|
||||
$this.remove();
|
||||
}
|
||||
});
|
||||
// Try to find the main content area first
|
||||
let mainContent = '';
|
||||
// Priority selectors for main content
|
||||
const contentSelectors = [
|
||||
'article',
|
||||
'main',
|
||||
'[role="main"]',
|
||||
'.content',
|
||||
'.post-content',
|
||||
'.entry-content',
|
||||
'.article-content',
|
||||
'.story-content',
|
||||
'.news-content',
|
||||
'.main-content',
|
||||
'.page-content',
|
||||
'.text-content',
|
||||
'.body-content',
|
||||
'.copy',
|
||||
'.text',
|
||||
'.body'
|
||||
];
|
||||
for (const selector of contentSelectors) {
|
||||
const $content = $(selector).first();
|
||||
if ($content.length > 0) {
|
||||
mainContent = $content.text().trim();
|
||||
if (mainContent.length > 100) { // Ensure we have substantial content
|
||||
console.log(`[EnhancedContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// If no main content found, try body content
|
||||
if (!mainContent || mainContent.length < 100) {
|
||||
console.log(`[EnhancedContentExtractor] No main content found, using body content`);
|
||||
mainContent = $('body').text().trim();
|
||||
}
|
||||
// Clean up the text
|
||||
const cleanedContent = this.cleanTextContent(mainContent);
|
||||
return cleanText(cleanedContent, this.maxContentLength);
|
||||
}
|
||||
cleanTextContent(text) {
|
||||
// Remove excessive whitespace
|
||||
text = text.replace(/\s+/g, ' ');
|
||||
// Remove image-related text and data URLs
|
||||
text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data
|
||||
text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs
|
||||
text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions
|
||||
text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words
|
||||
text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text
|
||||
// Remove common non-content patterns
|
||||
text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, '');
|
||||
// Remove excessive line breaks and spacing
|
||||
text = text.replace(/\n\s*\n/g, '\n');
|
||||
text = text.replace(/\r\n/g, '\n');
|
||||
text = text.replace(/\r/g, '\n');
|
||||
// Remove leading/trailing whitespace
|
||||
text = text.trim();
|
||||
return text;
|
||||
}
|
||||
getSpecificErrorMessage(error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
if (error.code === 'ECONNABORTED') {
|
||||
return 'Request timeout';
|
||||
}
|
||||
if (error.response?.status === 403) {
|
||||
return '403 Forbidden - Access denied';
|
||||
}
|
||||
if (error.response?.status === 404) {
|
||||
return '404 Not found';
|
||||
}
|
||||
if (error.message.includes('maxContentLength')) {
|
||||
return 'Content too long';
|
||||
}
|
||||
if (error.response?.status) {
|
||||
return `HTTP ${error.response.status}: ${error.message}`;
|
||||
}
|
||||
return `Network error: ${error.message}`;
|
||||
}
|
||||
return error instanceof Error ? error.message : 'Unknown error';
|
||||
}
|
||||
async closeAll() {
|
||||
await this.browserPool.closeAll();
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=enhanced-content-extractor.js.map
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env node
|
||||
export {};
|
||||
//# sourceMappingURL=index.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}
|
||||
+449
@@ -0,0 +1,449 @@
|
||||
#!/usr/bin/env node
|
||||
console.log('Web Search MCP Server starting...');
|
||||
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import { z } from 'zod';
|
||||
import { SearchEngine } from './search-engine.js';
|
||||
import { EnhancedContentExtractor } from './enhanced-content-extractor.js';
|
||||
import { isPdfUrl } from './utils.js';
|
||||
class WebSearchMCPServer {
|
||||
server;
|
||||
searchEngine;
|
||||
contentExtractor;
|
||||
constructor() {
|
||||
this.server = new McpServer({
|
||||
name: 'web-search-mcp',
|
||||
version: '0.3.1',
|
||||
});
|
||||
this.searchEngine = new SearchEngine();
|
||||
this.contentExtractor = new EnhancedContentExtractor();
|
||||
this.setupTools();
|
||||
this.setupGracefulShutdown();
|
||||
}
|
||||
setupTools() {
|
||||
// Register the main web search tool (primary choice for comprehensive searches)
|
||||
this.server.tool('full-web-search', 'Search the web and fetch complete page content from top results. This is the most comprehensive web search tool. It searches the web and then follows the resulting links to extract their full page content, providing the most detailed and complete information available. Use get-web-search-summaries for a lightweight alternative.', {
|
||||
query: z.string().describe('Search query to execute (recommended for comprehensive research)'),
|
||||
limit: z.union([z.number(), z.string()]).transform((val) => {
|
||||
const num = typeof val === 'string' ? parseInt(val, 10) : val;
|
||||
if (isNaN(num) || num < 1 || num > 10) {
|
||||
throw new Error('Invalid limit: must be a number between 1 and 10');
|
||||
}
|
||||
return num;
|
||||
}).default(5).describe('Number of results to return with full content (1-10)'),
|
||||
includeContent: z.union([z.boolean(), z.string()]).transform((val) => {
|
||||
if (typeof val === 'string') {
|
||||
return val.toLowerCase() === 'true';
|
||||
}
|
||||
return Boolean(val);
|
||||
}).default(true).describe('Whether to fetch full page content (default: true)'),
|
||||
maxContentLength: z.union([z.number(), z.string()]).transform((val) => {
|
||||
const num = typeof val === 'string' ? parseInt(val, 10) : val;
|
||||
if (isNaN(num) || num < 0) {
|
||||
throw new Error('Invalid maxContentLength: must be a non-negative number');
|
||||
}
|
||||
return num;
|
||||
}).optional().describe('Maximum characters per result content (0 = no limit). Usually not needed - content length is automatically optimized.'),
|
||||
}, async (args) => {
|
||||
console.log(`[MCP] Tool call received: full-web-search`);
|
||||
console.log(`[MCP] Raw arguments:`, JSON.stringify(args, null, 2));
|
||||
try {
|
||||
// Convert and validate arguments
|
||||
const validatedArgs = this.validateAndConvertArgs(args);
|
||||
// Auto-detect model types based on parameter formats
|
||||
// Llama models often send string parameters and struggle with large responses
|
||||
const isLikelyLlama = typeof args === 'object' && args !== null && (('limit' in args && typeof args.limit === 'string') ||
|
||||
('includeContent' in args && typeof args.includeContent === 'string'));
|
||||
// Detect models that handle large responses well (Qwen, Gemma, recent Deepseek)
|
||||
const isLikelyRobustModel = typeof args === 'object' && args !== null && (('limit' in args && typeof args.limit === 'number') &&
|
||||
('includeContent' in args && typeof args.includeContent === 'boolean'));
|
||||
// Only apply auto-limit if maxContentLength is not explicitly set (including 0)
|
||||
const hasExplicitMaxLength = typeof args === 'object' && args !== null && 'maxContentLength' in args;
|
||||
if (!hasExplicitMaxLength && isLikelyLlama) {
|
||||
console.log(`[MCP] Detected potential Llama model (string parameters), applying content length limit`);
|
||||
validatedArgs.maxContentLength = 2000; // Reasonable limit for Llama
|
||||
}
|
||||
// For robust models (Qwen, Gemma, recent Deepseek), remove maxContentLength if it's set to a low value
|
||||
if (isLikelyRobustModel && validatedArgs.maxContentLength && validatedArgs.maxContentLength < 5000) {
|
||||
console.log(`[MCP] Detected robust model (numeric parameters), removing unnecessary content length limit`);
|
||||
validatedArgs.maxContentLength = undefined;
|
||||
}
|
||||
console.log(`[MCP] Validated args:`, JSON.stringify(validatedArgs, null, 2));
|
||||
console.log(`[MCP] Starting web search...`);
|
||||
const result = await this.handleWebSearch(validatedArgs);
|
||||
console.log(`[MCP] Search completed, found ${result.results.length} results`);
|
||||
// Format the results as a comprehensive text response
|
||||
let responseText = `Search completed for "${result.query}" with ${result.total_results} results:\n\n`;
|
||||
// Add status line if available
|
||||
if (result.status) {
|
||||
responseText += `**Status:** ${result.status}\n\n`;
|
||||
}
|
||||
const maxLength = validatedArgs.maxContentLength;
|
||||
result.results.forEach((searchResult, idx) => {
|
||||
responseText += `**${idx + 1}. ${searchResult.title}**\n`;
|
||||
responseText += `URL: ${searchResult.url}\n`;
|
||||
responseText += `Description: ${searchResult.description}\n`;
|
||||
if (searchResult.fullContent && searchResult.fullContent.trim()) {
|
||||
let content = searchResult.fullContent;
|
||||
if (maxLength && maxLength > 0 && content.length > maxLength) {
|
||||
content = content.substring(0, maxLength) + `\n\n[Content truncated at ${maxLength} characters]`;
|
||||
}
|
||||
responseText += `\n**Full Content:**\n${content}\n`;
|
||||
}
|
||||
else if (searchResult.contentPreview && searchResult.contentPreview.trim()) {
|
||||
let content = searchResult.contentPreview;
|
||||
if (maxLength && maxLength > 0 && content.length > maxLength) {
|
||||
content = content.substring(0, maxLength) + `\n\n[Content truncated at ${maxLength} characters]`;
|
||||
}
|
||||
responseText += `\n**Content Preview:**\n${content}\n`;
|
||||
}
|
||||
else if (searchResult.fetchStatus === 'error') {
|
||||
responseText += `\n**Content Extraction Failed:** ${searchResult.error}\n`;
|
||||
}
|
||||
responseText += `\n---\n\n`;
|
||||
});
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: responseText,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`[MCP] Error in tool handler:`, error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
// Register the lightweight web search summaries tool (secondary choice for quick results)
|
||||
this.server.tool('get-web-search-summaries', 'Search the web and return only the search result snippets/descriptions without following links to extract full page content. This is a lightweight alternative to full-web-search for when you only need brief search results. For comprehensive information, use full-web-search instead.', {
|
||||
query: z.string().describe('Search query to execute (lightweight alternative)'),
|
||||
limit: z.union([z.number(), z.string()]).transform((val) => {
|
||||
const num = typeof val === 'string' ? parseInt(val, 10) : val;
|
||||
if (isNaN(num) || num < 1 || num > 10) {
|
||||
throw new Error('Invalid limit: must be a number between 1 and 10');
|
||||
}
|
||||
return num;
|
||||
}).default(5).describe('Number of search results to return (1-10)'),
|
||||
}, async (args) => {
|
||||
console.log(`[MCP] Tool call received: get-web-search-summaries`);
|
||||
console.log(`[MCP] Raw arguments:`, JSON.stringify(args, null, 2));
|
||||
try {
|
||||
// Validate arguments
|
||||
if (typeof args !== 'object' || args === null) {
|
||||
throw new Error('Invalid arguments: args must be an object');
|
||||
}
|
||||
const obj = args;
|
||||
if (!obj.query || typeof obj.query !== 'string') {
|
||||
throw new Error('Invalid arguments: query is required and must be a string');
|
||||
}
|
||||
let limit = 5; // default
|
||||
if (obj.limit !== undefined) {
|
||||
const limitValue = typeof obj.limit === 'string' ? parseInt(obj.limit, 10) : obj.limit;
|
||||
if (typeof limitValue !== 'number' || isNaN(limitValue) || limitValue < 1 || limitValue > 10) {
|
||||
throw new Error('Invalid limit: must be a number between 1 and 10');
|
||||
}
|
||||
limit = limitValue;
|
||||
}
|
||||
console.log(`[MCP] Starting web search summaries...`);
|
||||
try {
|
||||
// Use existing search engine to get results with snippets
|
||||
const searchResponse = await this.searchEngine.search({
|
||||
query: obj.query,
|
||||
numResults: limit,
|
||||
});
|
||||
// const searchTime = Date.now() - startTime; // Unused for now
|
||||
// Convert to summary format (no content extraction)
|
||||
const summaryResults = searchResponse.results.map(item => ({
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
description: item.description,
|
||||
timestamp: item.timestamp,
|
||||
}));
|
||||
console.log(`[MCP] Search summaries completed, found ${summaryResults.length} results`);
|
||||
// Format the results as text
|
||||
let responseText = `Search summaries for "${obj.query}" with ${summaryResults.length} results:\n\n`;
|
||||
summaryResults.forEach((summary, i) => {
|
||||
responseText += `**${i + 1}. ${summary.title}**\n`;
|
||||
responseText += `URL: ${summary.url}\n`;
|
||||
responseText += `Description: ${summary.description}\n`;
|
||||
responseText += `\n---\n\n`;
|
||||
});
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: responseText,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
finally {
|
||||
// Ensure browsers are cleaned up after search-only operations
|
||||
// This prevents EventEmitter memory leaks when browsers accumulate listeners
|
||||
try {
|
||||
await this.searchEngine.closeAll();
|
||||
}
|
||||
catch (cleanupError) {
|
||||
console.error(`[MCP] Error during browser cleanup:`, cleanupError);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`[MCP] Error in get-web-search-summaries tool handler:`, error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
// Register the single page content extraction tool
|
||||
this.server.tool('get-single-web-page-content', 'Extract and return the full content from a single web page URL. This tool follows a provided URL and extracts the main page content. Useful for getting detailed content from a specific webpage without performing a search.', {
|
||||
url: z.string().url().describe('The URL of the web page to extract content from'),
|
||||
maxContentLength: z.union([z.number(), z.string()]).transform((val) => {
|
||||
const num = typeof val === 'string' ? parseInt(val, 10) : val;
|
||||
if (isNaN(num) || num < 0) {
|
||||
throw new Error('Invalid maxContentLength: must be a non-negative number');
|
||||
}
|
||||
return num;
|
||||
}).optional().describe('Maximum characters for the extracted content (0 = no limit, undefined = use default limit). Usually not needed - content length is automatically optimized.'),
|
||||
}, async (args) => {
|
||||
console.log(`[MCP] Tool call received: get-single-web-page-content`);
|
||||
console.log(`[MCP] Raw arguments:`, JSON.stringify(args, null, 2));
|
||||
try {
|
||||
// Validate arguments
|
||||
if (typeof args !== 'object' || args === null) {
|
||||
throw new Error('Invalid arguments: args must be an object');
|
||||
}
|
||||
const obj = args;
|
||||
if (!obj.url || typeof obj.url !== 'string') {
|
||||
throw new Error('Invalid arguments: url is required and must be a string');
|
||||
}
|
||||
let maxContentLength;
|
||||
if (obj.maxContentLength !== undefined) {
|
||||
const maxLengthValue = typeof obj.maxContentLength === 'string' ? parseInt(obj.maxContentLength, 10) : obj.maxContentLength;
|
||||
if (typeof maxLengthValue !== 'number' || isNaN(maxLengthValue) || maxLengthValue < 0) {
|
||||
throw new Error('Invalid maxContentLength: must be a non-negative number');
|
||||
}
|
||||
// If maxContentLength is 0, treat it as "no limit" (undefined)
|
||||
maxContentLength = maxLengthValue === 0 ? undefined : maxLengthValue;
|
||||
}
|
||||
console.log(`[MCP] Starting single page content extraction for: ${obj.url}`);
|
||||
// Use existing content extractor to get page content
|
||||
const content = await this.contentExtractor.extractContent({
|
||||
url: obj.url,
|
||||
maxContentLength,
|
||||
});
|
||||
// Get page title from URL (simple extraction)
|
||||
const urlObj = new URL(obj.url);
|
||||
const title = urlObj.hostname + urlObj.pathname;
|
||||
// Create content preview and word count
|
||||
// const contentPreview = content.length > 200 ? content.substring(0, 200) + '...' : content; // Unused for now
|
||||
const wordCount = content.split(/\s+/).filter(word => word.length > 0).length;
|
||||
console.log(`[MCP] Single page content extraction completed, extracted ${content.length} characters`);
|
||||
// Format the result as text
|
||||
let responseText = `**Page Content from: ${obj.url}**\n\n`;
|
||||
responseText += `**Title:** ${title}\n`;
|
||||
responseText += `**Word Count:** ${wordCount}\n`;
|
||||
responseText += `**Content Length:** ${content.length} characters\n\n`;
|
||||
if (maxContentLength && maxContentLength > 0 && content.length > maxContentLength) {
|
||||
responseText += `**Content (truncated at ${maxContentLength} characters):**\n${content.substring(0, maxContentLength)}\n\n[Content truncated at ${maxContentLength} characters]`;
|
||||
}
|
||||
else {
|
||||
responseText += `**Content:**\n${content}`;
|
||||
}
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: responseText,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`[MCP] Error in get-single-web-page-content tool handler:`, error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
validateAndConvertArgs(args) {
|
||||
if (typeof args !== 'object' || args === null) {
|
||||
throw new Error('Invalid arguments: args must be an object');
|
||||
}
|
||||
const obj = args;
|
||||
// Ensure query is a string
|
||||
if (!obj.query || typeof obj.query !== 'string') {
|
||||
throw new Error('Invalid arguments: query is required and must be a string');
|
||||
}
|
||||
// Convert limit to number if it's a string
|
||||
let limit = 5; // default
|
||||
if (obj.limit !== undefined) {
|
||||
const limitValue = typeof obj.limit === 'string' ? parseInt(obj.limit, 10) : obj.limit;
|
||||
if (typeof limitValue !== 'number' || isNaN(limitValue) || limitValue < 1 || limitValue > 10) {
|
||||
throw new Error('Invalid limit: must be a number between 1 and 10');
|
||||
}
|
||||
limit = limitValue;
|
||||
}
|
||||
// Convert includeContent to boolean if it's a string
|
||||
let includeContent = true; // default
|
||||
if (obj.includeContent !== undefined) {
|
||||
if (typeof obj.includeContent === 'string') {
|
||||
includeContent = obj.includeContent.toLowerCase() === 'true';
|
||||
}
|
||||
else {
|
||||
includeContent = Boolean(obj.includeContent);
|
||||
}
|
||||
}
|
||||
return {
|
||||
query: obj.query,
|
||||
limit,
|
||||
includeContent,
|
||||
};
|
||||
}
|
||||
async handleWebSearch(input) {
|
||||
const startTime = Date.now();
|
||||
const { query, limit = 5, includeContent = true } = input;
|
||||
console.error(`[web-search-mcp] DEBUG: handleWebSearch called with limit=${limit}, includeContent=${includeContent}`);
|
||||
try {
|
||||
// Request extra search results to account for potential PDF files that will be skipped
|
||||
// Request up to 2x the limit or at least 5 extra results, capped at 10 (Google's max)
|
||||
const searchLimit = includeContent ? Math.min(limit * 2 + 2, 10) : limit;
|
||||
console.log(`[web-search-mcp] DEBUG: Requesting ${searchLimit} search results to get ${limit} non-PDF content results`);
|
||||
// Perform the search
|
||||
const searchResponse = await this.searchEngine.search({
|
||||
query,
|
||||
numResults: searchLimit,
|
||||
});
|
||||
const searchResults = searchResponse.results;
|
||||
// Log search summary
|
||||
const pdfCount = searchResults.filter(result => isPdfUrl(result.url)).length;
|
||||
const followedCount = searchResults.length - pdfCount;
|
||||
console.error(`[web-search-mcp] DEBUG: Search engine: ${searchResponse.engine}; ${limit} requested/${searchResults.length} obtained; PDF: ${pdfCount}; ${followedCount} followed.`);
|
||||
// Extract content from each result if requested, with target count
|
||||
const enhancedResults = includeContent
|
||||
? await this.contentExtractor.extractContentForResults(searchResults, limit)
|
||||
: searchResults.slice(0, limit); // If not extracting content, just take the first 'limit' results
|
||||
// Log extraction summary with failure reasons and generate combined status
|
||||
let combinedStatus = `Search engine: ${searchResponse.engine}; ${limit} result requested/${searchResults.length} obtained; PDF: ${pdfCount}; ${followedCount} followed`;
|
||||
if (includeContent) {
|
||||
const successCount = enhancedResults.filter(r => r.fetchStatus === 'success').length;
|
||||
const failedResults = enhancedResults.filter(r => r.fetchStatus === 'error');
|
||||
const failedCount = failedResults.length;
|
||||
const failureReasons = this.categorizeFailureReasons(failedResults);
|
||||
const failureReasonText = failureReasons.length > 0 ? ` (${failureReasons.join(', ')})` : '';
|
||||
console.error(`[web-search-mcp] DEBUG: Links requested: ${limit}; Successfully extracted: ${successCount}; Failed: ${failedCount}${failureReasonText}; Results: ${enhancedResults.length}.`);
|
||||
// Add extraction info to combined status
|
||||
combinedStatus += `; Successfully extracted: ${successCount}; Failed: ${failedCount}; Results: ${enhancedResults.length}`;
|
||||
}
|
||||
const searchTime = Date.now() - startTime;
|
||||
return {
|
||||
results: enhancedResults,
|
||||
total_results: enhancedResults.length,
|
||||
search_time_ms: searchTime,
|
||||
query,
|
||||
status: combinedStatus,
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Web search error:', error);
|
||||
throw new Error(`Web search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
categorizeFailureReasons(failedResults) {
|
||||
const reasonCounts = new Map();
|
||||
failedResults.forEach(result => {
|
||||
if (result.error) {
|
||||
const category = this.categorizeError(result.error);
|
||||
reasonCounts.set(category, (reasonCounts.get(category) || 0) + 1);
|
||||
}
|
||||
});
|
||||
return Array.from(reasonCounts.entries()).map(([reason, count]) => count > 1 ? `${reason} (${count})` : reason);
|
||||
}
|
||||
categorizeError(errorMessage) {
|
||||
const lowerError = errorMessage.toLowerCase();
|
||||
if (lowerError.includes('timeout') || lowerError.includes('timed out')) {
|
||||
return 'Timeout';
|
||||
}
|
||||
if (lowerError.includes('403') || lowerError.includes('forbidden')) {
|
||||
return 'Access denied';
|
||||
}
|
||||
if (lowerError.includes('404') || lowerError.includes('not found')) {
|
||||
return 'Not found';
|
||||
}
|
||||
if (lowerError.includes('bot') || lowerError.includes('captcha') || lowerError.includes('unusual traffic')) {
|
||||
return 'Bot detection';
|
||||
}
|
||||
if (lowerError.includes('too large') || lowerError.includes('content length') || lowerError.includes('maxcontentlength')) {
|
||||
return 'Content too long';
|
||||
}
|
||||
if (lowerError.includes('ssl') || lowerError.includes('certificate') || lowerError.includes('tls')) {
|
||||
return 'SSL error';
|
||||
}
|
||||
if (lowerError.includes('network') || lowerError.includes('connection') || lowerError.includes('econnrefused')) {
|
||||
return 'Network error';
|
||||
}
|
||||
if (lowerError.includes('dns') || lowerError.includes('hostname')) {
|
||||
return 'DNS error';
|
||||
}
|
||||
return 'Other error';
|
||||
}
|
||||
setupGracefulShutdown() {
|
||||
// Handle unhandled promise rejections
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
// Don't exit on unhandled rejections, just log them
|
||||
});
|
||||
// Handle uncaught exceptions
|
||||
process.on('uncaughtException', (error) => {
|
||||
console.error('Uncaught Exception:', error);
|
||||
// Don't exit on uncaught exceptions in MCP context
|
||||
});
|
||||
// Graceful shutdown - close browsers when process exits
|
||||
process.on('SIGINT', async () => {
|
||||
console.log('Shutting down gracefully...');
|
||||
try {
|
||||
await Promise.all([
|
||||
this.contentExtractor.closeAll(),
|
||||
this.searchEngine.closeAll()
|
||||
]);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error during graceful shutdown:', error);
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
process.on('SIGTERM', async () => {
|
||||
console.log('Shutting down gracefully...');
|
||||
try {
|
||||
await Promise.all([
|
||||
this.contentExtractor.closeAll(),
|
||||
this.searchEngine.closeAll()
|
||||
]);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error during graceful shutdown:', error);
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
async run() {
|
||||
console.log('Setting up MCP server...');
|
||||
const transport = new StdioServerTransport();
|
||||
console.log('Connecting to transport...');
|
||||
await this.server.connect(transport);
|
||||
console.log('Web Search MCP Server started');
|
||||
console.log('Server timestamp:', new Date().toISOString());
|
||||
console.log('Waiting for MCP messages...');
|
||||
}
|
||||
}
|
||||
// Start the server
|
||||
const server = new WebSearchMCPServer();
|
||||
server.run().catch((error) => {
|
||||
if (error instanceof Error) {
|
||||
console.error('Server error:', error.message);
|
||||
}
|
||||
else {
|
||||
console.error('Server error:', error);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
//# sourceMappingURL=index.js.map
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,15 @@
|
||||
export declare class RateLimiter {
|
||||
private limit;
|
||||
private requestCount;
|
||||
private lastResetTime;
|
||||
private readonly maxRequestsPerMinute;
|
||||
private readonly resetIntervalMs;
|
||||
constructor(maxRequestsPerMinute?: number);
|
||||
execute<T>(fn: () => Promise<T>): Promise<T>;
|
||||
getStatus(): {
|
||||
requestCount: number;
|
||||
maxRequests: number;
|
||||
resetTime: number;
|
||||
};
|
||||
}
|
||||
//# sourceMappingURL=rate-limiter.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../src/rate-limiter.ts"],"names":[],"mappings":"AAEA,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAiB;gBAErC,oBAAoB,GAAE,MAAW;IAKvC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuBlD,SAAS,IAAI;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAO9E"}
|
||||
@@ -0,0 +1,39 @@
|
||||
import pLimit from 'p-limit';
|
||||
export class RateLimiter {
|
||||
limit;
|
||||
requestCount = 0;
|
||||
lastResetTime = Date.now();
|
||||
maxRequestsPerMinute;
|
||||
resetIntervalMs = 60000; // 1 minute
|
||||
constructor(maxRequestsPerMinute = 10) {
|
||||
this.maxRequestsPerMinute = maxRequestsPerMinute;
|
||||
this.limit = pLimit(5); // Max 5 concurrent requests
|
||||
}
|
||||
async execute(fn) {
|
||||
// Check if we need to reset the counter
|
||||
const now = Date.now();
|
||||
if (now - this.lastResetTime >= this.resetIntervalMs) {
|
||||
this.requestCount = 0;
|
||||
this.lastResetTime = now;
|
||||
}
|
||||
// Check rate limit
|
||||
if (this.requestCount >= this.maxRequestsPerMinute) {
|
||||
const waitTime = this.resetIntervalMs - (now - this.lastResetTime);
|
||||
throw new Error(`Rate limit exceeded. Please wait ${Math.ceil(waitTime / 1000)} seconds.`);
|
||||
}
|
||||
// Execute with concurrency limit
|
||||
const result = await this.limit(async () => {
|
||||
this.requestCount++;
|
||||
return await fn();
|
||||
});
|
||||
return result;
|
||||
}
|
||||
getStatus() {
|
||||
return {
|
||||
requestCount: this.requestCount,
|
||||
maxRequests: this.maxRequestsPerMinute,
|
||||
resetTime: this.lastResetTime + this.resetIntervalMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=rate-limiter.js.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../src/rate-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,MAAM,OAAO,WAAW;IACd,KAAK,CAA4B;IACjC,YAAY,GAAW,CAAC,CAAC;IACzB,aAAa,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAC1B,oBAAoB,CAAS;IAC7B,eAAe,GAAW,KAAK,CAAC,CAAC,WAAW;IAE7D,YAAY,uBAA+B,EAAE;QAC3C,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,4BAA4B;IACtD,CAAC;IAED,KAAK,CAAC,OAAO,CAAI,EAAoB;QACnC,wCAAwC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACrD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;QAC3B,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7F,CAAC;QAED,iCAAiC;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;YACzC,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,SAAS;QACP,OAAO;YACL,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,WAAW,EAAE,IAAI,CAAC,oBAAoB;YACtC,SAAS,EAAE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,eAAe;SACrD,CAAC;IACJ,CAAC;CACF"}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { SearchOptions, SearchResultWithMetadata } from './types.js';
|
||||
export declare class SearchEngine {
|
||||
private readonly rateLimiter;
|
||||
private browserPool;
|
||||
constructor();
|
||||
search(options: SearchOptions): Promise<SearchResultWithMetadata>;
|
||||
private tryBrowserBraveSearch;
|
||||
private tryBrowserBraveSearchInternal;
|
||||
private tryBrowserBingSearch;
|
||||
private tryBrowserBingSearchInternal;
|
||||
private tryEnhancedBingSearch;
|
||||
private tryDirectBingSearch;
|
||||
private generateConversationId;
|
||||
private tryDuckDuckGoSearch;
|
||||
private parseSearchResults;
|
||||
private parseBraveResults;
|
||||
private parseBingResults;
|
||||
private parseDuckDuckGoResults;
|
||||
private isValidSearchUrl;
|
||||
private cleanGoogleUrl;
|
||||
private cleanBraveUrl;
|
||||
private cleanBingUrl;
|
||||
private cleanDuckDuckGoUrl;
|
||||
private assessResultQuality;
|
||||
private validateBrowserHealth;
|
||||
private handleBrowserError;
|
||||
closeAll(): Promise<void>;
|
||||
}
|
||||
//# sourceMappingURL=search-engine.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"search-engine.d.ts","sourceRoot":"","sources":["../src/search-engine.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAgB,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAKnF,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,WAAW,CAAc;;IAO3B,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,wBAAwB,CAAC;YAuGzD,qBAAqB;YA0CrB,6BAA6B;YAsD7B,oBAAoB;YA8DpB,4BAA4B;YAgF5B,qBAAqB;YAgFrB,mBAAmB;IAqDjC,OAAO,CAAC,sBAAsB;YAWhB,mBAAmB;IAiCjC,OAAO,CAAC,kBAAkB;IAkJ1B,OAAO,CAAC,iBAAiB;IAoGzB,OAAO,CAAC,gBAAgB;IAiHxB,OAAO,CAAC,sBAAsB;IAwC9B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,mBAAmB;YAwFb,qBAAqB;YAwBrB,kBAAkB;IAqB1B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGhC"}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,69 @@
|
||||
export interface SearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
description: string;
|
||||
fullContent: string;
|
||||
contentPreview: string;
|
||||
wordCount: number;
|
||||
timestamp: string;
|
||||
fetchStatus: 'success' | 'error' | 'timeout';
|
||||
error?: string;
|
||||
}
|
||||
export interface SearchResponse {
|
||||
query: string;
|
||||
limit: number;
|
||||
results: SearchResult[];
|
||||
totalFound: number;
|
||||
searchTimestamp: string;
|
||||
processingTimeMs: number;
|
||||
}
|
||||
export interface SearchOptions {
|
||||
query: string;
|
||||
numResults?: number;
|
||||
timeout?: number;
|
||||
}
|
||||
export interface ContentExtractionOptions {
|
||||
url: string;
|
||||
timeout?: number;
|
||||
maxContentLength?: number;
|
||||
}
|
||||
export interface WebSearchToolInput {
|
||||
query: string;
|
||||
limit?: number;
|
||||
includeContent?: boolean;
|
||||
maxContentLength?: number;
|
||||
}
|
||||
export interface WebSearchToolOutput {
|
||||
results: SearchResult[];
|
||||
total_results: number;
|
||||
search_time_ms: number;
|
||||
query: string;
|
||||
status?: string;
|
||||
}
|
||||
export interface SearchSummaryResult {
|
||||
title: string;
|
||||
url: string;
|
||||
description: string;
|
||||
timestamp: string;
|
||||
}
|
||||
export interface SearchSummaryOutput {
|
||||
results: SearchSummaryResult[];
|
||||
total_results: number;
|
||||
search_time_ms: number;
|
||||
query: string;
|
||||
}
|
||||
export interface SinglePageContentOutput {
|
||||
url: string;
|
||||
title: string;
|
||||
content: string;
|
||||
contentPreview: string;
|
||||
wordCount: number;
|
||||
timestamp: string;
|
||||
fetchStatus: 'success' | 'error';
|
||||
error?: string;
|
||||
}
|
||||
export interface SearchResultWithMetadata {
|
||||
results: SearchResult[];
|
||||
engine: string;
|
||||
}
|
||||
//# sourceMappingURL=types.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAGD,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf;AAGD,MAAM,WAAW,uBAAuB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,SAAS,GAAG,OAAO,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAGD,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB"}
|
||||
@@ -0,0 +1,2 @@
|
||||
export {};
|
||||
//# sourceMappingURL=types.js.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Utility functions for the web search MCP server
|
||||
*/
|
||||
export declare function cleanText(text: string, maxLength?: number): string;
|
||||
export declare function getWordCount(text: string): number;
|
||||
export declare function getContentPreview(text: string, maxLength?: number): string;
|
||||
export declare function generateTimestamp(): string;
|
||||
export declare function validateUrl(url: string): boolean;
|
||||
export declare function sanitizeQuery(query: string): string;
|
||||
export declare function getRandomUserAgent(): string;
|
||||
export declare function delay(ms: number): Promise<void>;
|
||||
export declare function isPdfUrl(url: string): boolean;
|
||||
//# sourceMappingURL=utils.d.ts.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAc,GAAG,MAAM,CAMzE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAY,GAAG,MAAM,CAG/E;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAOhD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAQ3C;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAQ7C"}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Utility functions for the web search MCP server
|
||||
*/
|
||||
export function cleanText(text, maxLength = 10000) {
|
||||
return text
|
||||
.replace(/\s+/g, ' ') // Replace multiple whitespace with single space
|
||||
.replace(/\n\s*\n/g, '\n') // Replace multiple newlines with single newline
|
||||
.trim()
|
||||
.substring(0, maxLength);
|
||||
}
|
||||
export function getWordCount(text) {
|
||||
return text.trim().split(/\s+/).filter(word => word.length > 0).length;
|
||||
}
|
||||
export function getContentPreview(text, maxLength = 500) {
|
||||
const cleaned = cleanText(text, maxLength);
|
||||
return cleaned.length === maxLength ? cleaned + '...' : cleaned;
|
||||
}
|
||||
export function generateTimestamp() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
export function validateUrl(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === 'http:' || parsed.protocol === 'https:';
|
||||
}
|
||||
catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
export function sanitizeQuery(query) {
|
||||
return query.trim().substring(0, 1000); // Limit query length
|
||||
}
|
||||
export function getRandomUserAgent() {
|
||||
const userAgents = [
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
||||
];
|
||||
return userAgents[Math.floor(Math.random() * userAgents.length)];
|
||||
}
|
||||
export function delay(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
export function isPdfUrl(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.pathname.toLowerCase().endsWith('.pdf');
|
||||
}
|
||||
catch {
|
||||
// If URL parsing fails, check the raw string as fallback
|
||||
return url.toLowerCase().endsWith('.pdf');
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=utils.js.map
|
||||
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,YAAoB,KAAK;IAC/D,OAAO,IAAI;SACR,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,gDAAgD;SACrE,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,gDAAgD;SAC1E,IAAI,EAAE;SACN,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,YAAoB,GAAG;IACrE,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,qBAAqB;AAC/D,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,UAAU,GAAG;QACjB,uHAAuH;QACvH,iHAAiH;QACjH,uGAAuG;QACvG,uHAAuH;KACxH,CAAC;IACF,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,yDAAyD;QACzD,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC"}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"name": "web-search-mcp-server",
|
||||
"version": "0.3.1",
|
||||
"description": "MCP server for web search with full page content extraction, search summaries, and single page content extraction",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"bin": {
|
||||
"web-search-mcp": "./dist/index.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc && echo '✅ TypeScript compilation complete: dist/index.js'",
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"start": "node ./dist/index.js",
|
||||
"lint": "eslint \"src/**/*.ts\"",
|
||||
"format": "prettier --write ."
|
||||
},
|
||||
"keywords": [
|
||||
"mcp",
|
||||
"web-search",
|
||||
"ai",
|
||||
"llm"
|
||||
],
|
||||
"author": "Mark Russell",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/mrkrsl/web-search-mcp.git"
|
||||
},
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.15.0",
|
||||
"axios": "^1.6.8",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"p-limit": "^6.2.0",
|
||||
"p-retry": "^6.2.1",
|
||||
"playwright": "^1.48.0",
|
||||
"zod": "^3.22.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.30.1",
|
||||
"@types/node": "^24.0.10",
|
||||
"@typescript-eslint/eslint-plugin": "^8.35.1",
|
||||
"@typescript-eslint/parser": "^8.35.1",
|
||||
"esbuild": "^0.25.5",
|
||||
"eslint": "^9.30.1",
|
||||
"prettier": "^3.2.5",
|
||||
"tsx": "^4.7.0",
|
||||
"typescript": "^5.4.5"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user