From f94fe6894473f2d3330a0c9aaa06186979565407 Mon Sep 17 00:00:00 2001 From: Mathias Scheider Date: Fri, 10 Apr 2026 23:00:16 +0200 Subject: [PATCH] web-search-mcp --- .../.config/mcp/web-search-mcp/.gitignore | 2 + .../mcp/web-search-mcp/dist/browser-pool.d.ts | 14 + .../web-search-mcp/dist/browser-pool.d.ts.map | 1 + .../mcp/web-search-mcp/dist/browser-pool.js | 115 ++ .../web-search-mcp/dist/browser-pool.js.map | 1 + .../dist/content-extractor.d.ts | 12 + .../dist/content-extractor.d.ts.map | 1 + .../web-search-mcp/dist/content-extractor.js | 224 ++++ .../dist/content-extractor.js.map | 1 + .../dist/enhanced-content-extractor.d.ts | 24 + .../dist/enhanced-content-extractor.d.ts.map | 1 + .../dist/enhanced-content-extractor.js | 499 ++++++++ .../dist/enhanced-content-extractor.js.map | 1 + .../mcp/web-search-mcp/dist/index.d.ts | 3 + .../mcp/web-search-mcp/dist/index.d.ts.map | 1 + .../.config/mcp/web-search-mcp/dist/index.js | 449 ++++++++ .../mcp/web-search-mcp/dist/index.js.map | 1 + .../mcp/web-search-mcp/dist/rate-limiter.d.ts | 15 + .../web-search-mcp/dist/rate-limiter.d.ts.map | 1 + .../mcp/web-search-mcp/dist/rate-limiter.js | 39 + .../web-search-mcp/dist/rate-limiter.js.map | 1 + .../web-search-mcp/dist/search-engine.d.ts | 29 + .../dist/search-engine.d.ts.map | 1 + .../mcp/web-search-mcp/dist/search-engine.js | 1002 +++++++++++++++++ .../web-search-mcp/dist/search-engine.js.map | 1 + .../mcp/web-search-mcp/dist/types.d.ts | 69 ++ .../mcp/web-search-mcp/dist/types.d.ts.map | 1 + .../.config/mcp/web-search-mcp/dist/types.js | 2 + .../mcp/web-search-mcp/dist/types.js.map | 1 + .../mcp/web-search-mcp/dist/utils.d.ts | 13 + .../mcp/web-search-mcp/dist/utils.d.ts.map | 1 + .../.config/mcp/web-search-mcp/dist/utils.js | 55 + .../mcp/web-search-mcp/dist/utils.js.map | 1 + .../.config/mcp/web-search-mcp/package.json | 49 + 34 files changed, 2631 insertions(+) create mode 100644 local_ai/.config/mcp/web-search-mcp/.gitignore create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/index.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/index.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/index.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/index.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/search-engine.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/search-engine.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/types.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/types.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/types.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/types.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts.map create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/utils.js create mode 100644 local_ai/.config/mcp/web-search-mcp/dist/utils.js.map create mode 100644 local_ai/.config/mcp/web-search-mcp/package.json diff --git a/local_ai/.config/mcp/web-search-mcp/.gitignore b/local_ai/.config/mcp/web-search-mcp/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts new file mode 100644 index 0000000..6518e28 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts @@ -0,0 +1,14 @@ +import { Browser } from 'playwright'; +export declare class BrowserPool { + private browsers; + private maxBrowsers; + private browserTypes; + private currentBrowserIndex; + private headless; + private lastUsedBrowserType; + constructor(); + getBrowser(): Promise; + closeAll(): Promise; + getLastUsedBrowserType(): string; +} +//# sourceMappingURL=browser-pool.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts.map new file mode 100644 index 0000000..54f87aa --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAA6B,OAAO,EAAE,MAAM,YAAY,CAAC;AAEhE,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAW;IAC/B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,mBAAmB,CAAc;;IAcnC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IA4F9B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAa/B,sBAAsB,IAAI,MAAM;CAGjC"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js new file mode 100644 index 0000000..f0b2ee1 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js @@ -0,0 +1,115 @@ +import { chromium, firefox, webkit } from 'playwright'; +export class BrowserPool { + browsers = new Map(); + maxBrowsers; + browserTypes; + currentBrowserIndex = 0; + headless; + lastUsedBrowserType = ''; + constructor() { + // Read configuration from environment variables + this.maxBrowsers = parseInt(process.env.MAX_BROWSERS || '3', 10); + this.headless = process.env.BROWSER_HEADLESS !== 'false'; // Default to true + // Configure browser types based on environment + const browserTypesEnv = process.env.BROWSER_TYPES || 'chromium,firefox'; + this.browserTypes = browserTypesEnv.split(',').map(type => type.trim()); + console.log(`[BrowserPool] Configuration: maxBrowsers=${this.maxBrowsers}, headless=${this.headless}, types=${this.browserTypes.join(',')}`); + } + async getBrowser() { + // Rotate between browser types for variety + const browserType = this.browserTypes[this.currentBrowserIndex % this.browserTypes.length]; + this.currentBrowserIndex++; + this.lastUsedBrowserType = browserType; + if (this.browsers.has(browserType)) { + const browser = this.browsers.get(browserType); + // Check if browser is still connected and healthy + try { + if (browser.isConnected()) { + // Quick health check by trying to create and close a context + // Use minimal options to avoid Firefox isMobile issues + const testContext = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36' + }); + await testContext.close(); + return browser; + } + } + catch (error) { + console.log(`[BrowserPool] Browser ${browserType} health check failed:`, error); + // Browser is unhealthy, remove it and close if possible + this.browsers.delete(browserType); + try { + await browser.close(); + } + catch (closeError) { + console.log(`[BrowserPool] Error closing unhealthy browser:`, closeError); + } + } + } + // Launch new browser + console.log(`[BrowserPool] Launching new ${browserType} browser`); + const launchOptions = { + headless: this.headless, + args: [ + '--no-sandbox', + '--disable-blink-features=AutomationControlled', + '--disable-dev-shm-usage', + '--disable-gpu', + '--no-first-run', + '--no-default-browser-check', + '--disable-default-apps', + '--disable-extensions', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-renderer-backgrounding', + '--disable-features=TranslateUI', + '--disable-ipc-flooding-protection', + ], + }; + let browser; + try { + switch (browserType) { + case 'chromium': + browser = await chromium.launch(launchOptions); + break; + case 'firefox': + browser = await firefox.launch(launchOptions); + break; + case 'webkit': + browser = await webkit.launch(launchOptions); + break; + default: + browser = await chromium.launch(launchOptions); + } + this.browsers.set(browserType, browser); + // Clean up old browsers if we have too many + if (this.browsers.size > this.maxBrowsers) { + const oldestBrowser = this.browsers.entries().next().value; + if (oldestBrowser) { + try { + await oldestBrowser[1].close(); + } + catch (error) { + console.error(`[BrowserPool] Error closing old browser:`, error); + } + this.browsers.delete(oldestBrowser[0]); + } + } + return browser; + } + catch (error) { + console.error(`[BrowserPool] Failed to launch ${browserType} browser:`, error); + throw error; + } + } + async closeAll() { + console.log(`[BrowserPool] Closing ${this.browsers.size} browsers`); + const closePromises = Array.from(this.browsers.values()).map(browser => browser.close().catch(error => console.error('Error closing browser:', error))); + await Promise.all(closePromises); + this.browsers.clear(); + } + getLastUsedBrowserType() { + return this.lastUsedBrowserType; + } +} +//# sourceMappingURL=browser-pool.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js.map b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js.map new file mode 100644 index 0000000..8291baa --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/browser-pool.js.map @@ -0,0 +1 @@ +{"version":3,"file":"browser-pool.js","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAW,MAAM,YAAY,CAAC;AAEhE,MAAM,OAAO,WAAW;IACd,QAAQ,GAAyB,IAAI,GAAG,EAAE,CAAC;IAC3C,WAAW,CAAS;IACpB,YAAY,CAAW;IACvB,mBAAmB,GAAG,CAAC,CAAC;IACxB,QAAQ,CAAU;IAClB,mBAAmB,GAAW,EAAE,CAAC;IAEzC;QACE,gDAAgD;QAChD,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO,CAAC,CAAC,kBAAkB;QAE5E,+CAA+C;QAC/C,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,kBAAkB,CAAC;QACxE,IAAI,CAAC,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAExE,OAAO,CAAC,GAAG,CAAC,4CAA4C,IAAI,CAAC,WAAW,cAAc,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/I,CAAC;IAED,KAAK,CAAC,UAAU;QACd,2CAA2C;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC3F,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,WAAW,CAAC;QAEvC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAE,CAAC;YAEhD,kDAAkD;YAClD,IAAI,CAAC;gBACH,IAAI,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC1B,6DAA6D;oBAC7D,uDAAuD;oBACvD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;wBAC3C,SAAS,EAAE,uHAAuH;qBACnI,CAAC,CAAC;oBACH,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;oBAC1B,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,yBAAyB,WAAW,uBAAuB,EAAE,KAAK,CAAC,CAAC;gBAChF,wDAAwD;gBACxD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACxB,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,OAAO,CAAC,GAAG,CAAC,gDAAgD,EAAE,UAAU,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,OAAO,CAAC,GAAG,CAAC,+BAA+B,WAAW,UAAU,CAAC,CAAC;QAElE,MAAM,aAAa,GAAG;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE;gBACJ,cAAc;gBACd,+CAA+C;gBAC/C,yBAAyB;gBACzB,eAAe;gBACf,gBAAgB;gBAChB,4BAA4B;gBAC5B,wBAAwB;gBACxB,sBAAsB;gBACtB,uCAAuC;gBACvC,0CAA0C;gBAC1C,kCAAkC;gBAClC,gCAAgC;gBAChC,mCAAmC;aACpC;SACF,CAAC;QAEF,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,WAAW,EAAE,CAAC;gBACpB,KAAK,UAAU;oBACb,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC/C,MAAM;gBACR,KAAK,SAAS;oBACZ,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC9C,MAAM;gBACR,KAAK,QAAQ;oBACX,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBAC7C,MAAM;gBACR;oBACE,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAExC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;gBAC3D,IAAI,aAAa,EAAE,CAAC;oBAClB,IAAI,CAAC;wBACH,MAAM,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;oBACjC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;oBACnE,CAAC;oBACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kCAAkC,WAAW,WAAW,EAAE,KAAK,CAAC,CAAC;YAC/E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,CAAC;QAEpE,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACrE,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAC5B,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAC/C,CACF,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;CACF"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts new file mode 100644 index 0000000..c5bfa98 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts @@ -0,0 +1,12 @@ +import { ContentExtractionOptions, SearchResult } from './types.js'; +export declare class ContentExtractor { + private readonly defaultTimeout; + private readonly maxContentLength; + constructor(); + extractContent(options: ContentExtractionOptions): Promise; + extractContentForResults(results: SearchResult[], targetCount?: number): Promise; + private parseContent; + private cleanTextContent; + private getSpecificErrorMessage; +} +//# sourceMappingURL=content-extractor.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts.map new file mode 100644 index 0000000..c48dbd9 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"content-extractor.d.ts","sourceRoot":"","sources":["../src/content-extractor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGpE,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;;IAepC,cAAc,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC;IA+DlE,wBAAwB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,WAAW,GAAE,MAAuB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqDtH,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,uBAAuB;CAsBhC"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js new file mode 100644 index 0000000..78b3c24 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js @@ -0,0 +1,224 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; +import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js'; +export class ContentExtractor { + defaultTimeout; + maxContentLength; + constructor() { + this.defaultTimeout = 10000; + // Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB + const envMaxLength = process.env.MAX_CONTENT_LENGTH; + this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000; + // Validate the parsed value + if (isNaN(this.maxContentLength) || this.maxContentLength < 0) { + console.warn(`[ContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`); + this.maxContentLength = 500000; + } + } + async extractContent(options) { + const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options; + try { + const response = await axios.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Cache-Control': 'max-age=0', + 'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + }, + timeout, + maxContentLength, + validateStatus: (status) => status < 400, + }); + return this.parseContent(response.data); + } + catch (error) { + console.error(`Content extraction error for ${url}:`, error); + // If it's a 403 error, try with different headers + if (axios.isAxiosError(error) && error.response?.status === 403) { + console.log(`[ContentExtractor] Trying alternative headers for ${url}`); + try { + const response = await axios.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Cache-Control': 'no-cache', + 'Pragma': 'no-cache', + }, + timeout, + maxContentLength, + validateStatus: (status) => status < 400, + }); + console.log(`[ContentExtractor] Alternative headers worked for ${url}`); + return this.parseContent(response.data); + } + catch (retryError) { + console.error(`[ContentExtractor] Alternative headers also failed for ${url}:`, retryError); + } + } + throw new Error(`Failed to extract content from ${url}: ${this.getSpecificErrorMessage(error)}`); + } + } + async extractContentForResults(results, targetCount = results.length) { + const enhancedResults = []; + let processedCount = 0; + console.log(`[ContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`); + for (const result of results) { + if (enhancedResults.length >= targetCount) { + console.log(`[ContentExtractor] Reached target count of ${targetCount} results`); + break; + } + processedCount++; + // Skip PDF files + if (isPdfUrl(result.url)) { + console.log(`[ContentExtractor] Skipping PDF file: ${result.url}`); + continue; + } + try { + console.log(`[ContentExtractor] Extracting content from: ${result.url}`); + const content = await this.extractContent({ url: result.url }); + const cleanedContent = cleanText(content, this.maxContentLength); + enhancedResults.push({ + ...result, + fullContent: cleanedContent, + contentPreview: getContentPreview(cleanedContent), + wordCount: getWordCount(cleanedContent), + timestamp: generateTimestamp(), + fetchStatus: 'success', + }); + console.log(`[ContentExtractor] Successfully extracted content (${enhancedResults.length}/${targetCount})`); + } + catch (error) { + console.log(`[ContentExtractor] Failed to extract content from ${result.url}: ${error instanceof Error ? error.message : 'Unknown error'}`); + enhancedResults.push({ + ...result, + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp: generateTimestamp(), + fetchStatus: 'error', + error: this.getSpecificErrorMessage(error), + }); + } + } + console.log(`[ContentExtractor] Processed ${processedCount} results, extracted ${enhancedResults.length} non-PDF results`); + return enhancedResults; + } + parseContent(html) { + const $ = cheerio.load(html); + // Remove all script, style, and other non-content elements + $('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove(); + // Remove navigation, header, footer, and other non-content elements + $('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove(); + // Remove elements with common ad/tracking classes + $('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove(); + // Remove elements with common non-content IDs + $('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove(); + // Remove image-related elements and attributes + $('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove(); + $('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove(); + $('[style*="background-image"]').remove(); + // Remove empty elements and whitespace-only elements + $('*').each(function () { + const $this = $(this); + if ($this.children().length === 0 && $this.text().trim() === '') { + $this.remove(); + } + }); + // Try to find the main content area first + let mainContent = ''; + // Priority selectors for main content + const contentSelectors = [ + 'article', + 'main', + '[role="main"]', + '.content', + '.post-content', + '.entry-content', + '.article-content', + '.story-content', + '.news-content', + '.main-content', + '.page-content', + '.text-content', + '.body-content', + '.copy', + '.text', + '.body' + ]; + for (const selector of contentSelectors) { + const $content = $(selector).first(); + if ($content.length > 0) { + mainContent = $content.text().trim(); + if (mainContent.length > 100) { // Ensure we have substantial content + console.log(`[ContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`); + break; + } + } + } + // If no main content found, try body content + if (!mainContent || mainContent.length < 100) { + console.log(`[ContentExtractor] No main content found, using body content`); + mainContent = $('body').text().trim(); + } + // Clean up the text + const cleanedContent = this.cleanTextContent(mainContent); + return cleanText(cleanedContent, this.maxContentLength); + } + cleanTextContent(text) { + // Remove excessive whitespace + text = text.replace(/\s+/g, ' '); + // Remove image-related text and data URLs + text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data + text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs + text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions + text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words + text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text + // Remove common non-content patterns + text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, ''); + // Remove excessive line breaks and spacing + text = text.replace(/\n\s*\n/g, '\n'); + text = text.replace(/\r\n/g, '\n'); + text = text.replace(/\r/g, '\n'); + // Remove leading/trailing whitespace + text = text.trim(); + return text; + } + getSpecificErrorMessage(error) { + if (axios.isAxiosError(error)) { + if (error.code === 'ECONNABORTED') { + return 'Request timeout'; + } + if (error.response?.status === 403) { + return '403 Forbidden - Access denied'; + } + if (error.response?.status === 404) { + return '404 Not found'; + } + if (error.message.includes('maxContentLength')) { + return 'Content too long'; + } + if (error.response?.status) { + return `HTTP ${error.response.status}: ${error.message}`; + } + return `Network error: ${error.message}`; + } + return error instanceof Error ? error.message : 'Unknown error'; + } +} +//# sourceMappingURL=content-extractor.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js.map b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js.map new file mode 100644 index 0000000..19d8a51 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/content-extractor.js.map @@ -0,0 +1 @@ +{"version":3,"file":"content-extractor.js","sourceRoot":"","sources":["../src/content-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErG,MAAM,OAAO,gBAAgB;IACV,cAAc,CAAS;IACvB,gBAAgB,CAAS;IAE1C;QACE,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,uEAAuE;QACvE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACpD,IAAI,CAAC,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAE3E,4BAA4B;QAC5B,IAAI,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC,wDAAwD,YAAY,wBAAwB,CAAC,CAAC;YAC3G,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC;QACjC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,OAAiC;QACpD,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAEjG,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;gBACpC,OAAO,EAAE;oBACP,YAAY,EAAE,uHAAuH;oBACrI,QAAQ,EAAE,yIAAyI;oBACnJ,iBAAiB,EAAE,gBAAgB;oBACnC,iBAAiB,EAAE,mBAAmB;oBACtC,KAAK,EAAE,GAAG;oBACV,YAAY,EAAE,YAAY;oBAC1B,2BAA2B,EAAE,GAAG;oBAChC,gBAAgB,EAAE,UAAU;oBAC5B,gBAAgB,EAAE,UAAU;oBAC5B,gBAAgB,EAAE,MAAM;oBACxB,gBAAgB,EAAE,IAAI;oBACtB,eAAe,EAAE,WAAW;oBAC5B,WAAW,EAAE,kEAAkE;oBAC/E,kBAAkB,EAAE,IAAI;oBACxB,oBAAoB,EAAE,SAAS;iBAChC;gBACD,OAAO;gBACP,gBAAgB;gBAChB,cAAc,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG;aACjD,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAE7D,kDAAkD;YAClD,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBAChE,OAAO,CAAC,GAAG,CAAC,qDAAqD,GAAG,EAAE,CAAC,CAAC;gBACxE,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;wBACpC,OAAO,EAAE;4BACP,YAAY,EAAE,iHAAiH;4BAC/H,QAAQ,EAAE,iEAAiE;4BAC3E,iBAAiB,EAAE,gBAAgB;4BACnC,iBAAiB,EAAE,eAAe;4BAClC,KAAK,EAAE,GAAG;4BACV,YAAY,EAAE,YAAY;4BAC1B,2BAA2B,EAAE,GAAG;4BAChC,eAAe,EAAE,UAAU;4BAC3B,QAAQ,EAAE,UAAU;yBACrB;wBACD,OAAO;wBACP,gBAAgB;wBAChB,cAAc,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG;qBACjD,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,qDAAqD,GAAG,EAAE,CAAC,CAAC;oBACxE,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,OAAO,CAAC,KAAK,CAAC,0DAA0D,GAAG,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC9F,CAAC;YACH,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,KAAK,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,OAAuB,EAAE,cAAsB,OAAO,CAAC,MAAM;QAC1F,MAAM,eAAe,GAAmB,EAAE,CAAC;QAC3C,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,OAAO,CAAC,GAAG,CAAC,uCAAuC,OAAO,CAAC,MAAM,mBAAmB,WAAW,kBAAkB,CAAC,CAAC;QAEnH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,eAAe,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,8CAA8C,WAAW,UAAU,CAAC,CAAC;gBACjF,MAAM;YACR,CAAC;YAED,cAAc,EAAE,CAAC;YAEjB,iBAAiB;YACjB,IAAI,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,yCAAyC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnE,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,+CAA+C,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBACzE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC/D,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAEjE,eAAe,CAAC,IAAI,CAAC;oBACnB,GAAG,MAAM;oBACT,WAAW,EAAE,cAAc;oBAC3B,cAAc,EAAE,iBAAiB,CAAC,cAAc,CAAC;oBACjD,SAAS,EAAE,YAAY,CAAC,cAAc,CAAC;oBACvC,SAAS,EAAE,iBAAiB,EAAE;oBAC9B,WAAW,EAAE,SAAkB;iBAChC,CAAC,CAAC;gBAEH,OAAO,CAAC,GAAG,CAAC,sDAAsD,eAAe,CAAC,MAAM,IAAI,WAAW,GAAG,CAAC,CAAC;YAC9G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,qDAAqD,MAAM,CAAC,GAAG,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;gBAC5I,eAAe,CAAC,IAAI,CAAC;oBACnB,GAAG,MAAM;oBACT,WAAW,EAAE,EAAE;oBACf,cAAc,EAAE,EAAE;oBAClB,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,iBAAiB,EAAE;oBAC9B,WAAW,EAAE,OAAgB;oBAC7B,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC;iBAC3C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,cAAc,uBAAuB,eAAe,CAAC,MAAM,kBAAkB,CAAC,CAAC;QAC3H,OAAO,eAAe,CAAC;IACzB,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,2DAA2D;QAC3D,CAAC,CAAC,0KAA0K,CAAC,CAAC,MAAM,EAAE,CAAC;QAEvL,oEAAoE;QACpE,CAAC,CAAC,knBAAknB,CAAC,CAAC,MAAM,EAAE,CAAC;QAE/nB,kDAAkD;QAClD,CAAC,CAAC,6UAA6U,CAAC,CAAC,MAAM,EAAE,CAAC;QAE1V,8CAA8C;QAC9C,CAAC,CAAC,6WAA6W,CAAC,CAAC,MAAM,EAAE,CAAC;QAE1X,+CAA+C;QAC/C,CAAC,CAAC,8GAA8G,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3H,CAAC,CAAC,oFAAoF,CAAC,CAAC,MAAM,EAAE,CAAC;QACjG,CAAC,CAAC,6BAA6B,CAAC,CAAC,MAAM,EAAE,CAAC;QAE1C,qDAAqD;QACrD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;YACV,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAChE,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0CAA0C;QAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,sCAAsC;QACtC,MAAM,gBAAgB,GAAG;YACvB,SAAS;YACT,MAAM;YACN,eAAe;YACf,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,kBAAkB;YAClB,gBAAgB;YAChB,eAAe;YACf,eAAe;YACf,eAAe;YACf,eAAe;YACf,eAAe;YACf,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC;QAEF,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,qCAAqC;oBACnE,OAAO,CAAC,GAAG,CAAC,mDAAmD,QAAQ,KAAK,WAAW,CAAC,MAAM,SAAS,CAAC,CAAC;oBACzG,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;YAC5E,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC;QAED,oBAAoB;QACpB,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAE1D,OAAO,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1D,CAAC;IAEO,gBAAgB,CAAC,IAAY;QACnC,8BAA8B;QAC9B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAEjC,0CAA0C;QAC1C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,2CAA2C,EAAE,EAAE,CAAC,CAAC,CAAC,2BAA2B;QACjG,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,0EAA0E,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;QACzH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,8CAA8C,EAAE,EAAE,CAAC,CAAC,CAAC,+BAA+B;QACxG,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,sDAAsD,EAAE,EAAE,CAAC,CAAC,CAAC,6BAA6B;QAC9G,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,mEAAmE,EAAE,EAAE,CAAC,CAAC,CAAC,2BAA2B;QAEzH,qCAAqC;QACrC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kFAAkF,EAAE,EAAE,CAAC,CAAC;QAE5G,2CAA2C;QAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAEjC,qCAAqC;QACrC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,uBAAuB,CAAC,KAAc;QAC5C,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAClC,OAAO,iBAAiB,CAAC;YAC3B,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnC,OAAO,+BAA+B,CAAC;YACzC,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnC,OAAO,eAAe,CAAC;YACzB,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC/C,OAAO,kBAAkB,CAAC;YAC5B,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;gBAC3B,OAAO,QAAQ,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3D,CAAC;YACD,OAAO,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;QAED,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IAClE,CAAC;CACF"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts new file mode 100644 index 0000000..33774c9 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts @@ -0,0 +1,24 @@ +import { ContentExtractionOptions, SearchResult } from './types.js'; +export declare class EnhancedContentExtractor { + private readonly defaultTimeout; + private readonly maxContentLength; + private browserPool; + private fallbackThreshold; + constructor(); + extractContent(options: ContentExtractionOptions): Promise; + private extractWithAxios; + private extractWithBrowser; + private simulateHumanBehavior; + private shouldUseBrowser; + private isLowQualityContent; + private getRandomHeaders; + private getRandomUserAgent; + private getRandomViewport; + private getRandomTimezone; + extractContentForResults(results: SearchResult[], targetCount?: number): Promise; + private parseContent; + private cleanTextContent; + private getSpecificErrorMessage; + closeAll(): Promise; +} +//# sourceMappingURL=enhanced-content-extractor.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts.map new file mode 100644 index 0000000..41c9fda --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"enhanced-content-extractor.d.ts","sourceRoot":"","sources":["../src/enhanced-content-extractor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAIpE,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,iBAAiB,CAAS;;IAqB5B,cAAc,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC;YA8B1D,gBAAgB;YA0BhB,kBAAkB;YAgKlB,qBAAqB;IA6BnC,OAAO,CAAC,gBAAgB;IAgCxB,OAAO,CAAC,mBAAmB;IAe3B,OAAO,CAAC,gBAAgB;IAsCxB,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,iBAAiB;IAanB,wBAAwB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,WAAW,GAAE,MAAuB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAiEtH,OAAO,CAAC,YAAY;IA0EpB,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,uBAAuB;IAuBzB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGhC"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js new file mode 100644 index 0000000..ef344a9 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js @@ -0,0 +1,499 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; +import { cleanText, getWordCount, getContentPreview, generateTimestamp, isPdfUrl } from './utils.js'; +import { BrowserPool } from './browser-pool.js'; +export class EnhancedContentExtractor { + defaultTimeout; + maxContentLength; + browserPool; + fallbackThreshold; + constructor() { + this.defaultTimeout = parseInt(process.env.DEFAULT_TIMEOUT || '6000', 10); + // Read MAX_CONTENT_LENGTH from environment variable, fallback to 500KB + const envMaxLength = process.env.MAX_CONTENT_LENGTH; + this.maxContentLength = envMaxLength ? parseInt(envMaxLength, 10) : 500000; + // Validate the parsed value + if (isNaN(this.maxContentLength) || this.maxContentLength < 0) { + console.warn(`[EnhancedContentExtractor] Invalid MAX_CONTENT_LENGTH value: ${envMaxLength}, using default 500000`); + this.maxContentLength = 500000; + } + this.browserPool = new BrowserPool(); + this.fallbackThreshold = parseInt(process.env.BROWSER_FALLBACK_THRESHOLD || '3', 10); + console.log(`[EnhancedContentExtractor] Configuration: timeout=${this.defaultTimeout}, maxContentLength=${this.maxContentLength}, fallbackThreshold=${this.fallbackThreshold}`); + } + async extractContent(options) { + const { url } = options; + console.log(`[EnhancedContentExtractor] Starting extraction for: ${url}`); + // First, try with regular HTTP client (faster) + try { + const content = await this.extractWithAxios(options); + console.log(`[EnhancedContentExtractor] Successfully extracted with axios: ${content.length} chars`); + return content; + } + catch (error) { + console.log(`[EnhancedContentExtractor] Axios failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + // Check if this looks like a case where browser would help + if (this.shouldUseBrowser(error, url)) { + console.log(`[EnhancedContentExtractor] Falling back to headless browser for: ${url}`); + try { + const content = await this.extractWithBrowser(options); + console.log(`[EnhancedContentExtractor] Successfully extracted with browser: ${content.length} chars`); + return content; + } + catch (browserError) { + console.error(`[EnhancedContentExtractor] Browser extraction also failed:`, browserError); + throw new Error(`Both axios and browser extraction failed for ${url}`); + } + } + else { + throw error; + } + } + } + async extractWithAxios(options) { + const { url, timeout = this.defaultTimeout, maxContentLength = this.maxContentLength } = options; + const response = await axios.get(url, { + headers: this.getRandomHeaders(), + timeout, + // Remove maxContentLength from axios config - handle truncation manually + validateStatus: (status) => status < 400, + }); + let content = this.parseContent(response.data); + // Truncate content if it exceeds the limit (instead of axios throwing an error) + if (maxContentLength && content.length > maxContentLength) { + console.log(`[EnhancedContentExtractor] Content truncated from ${content.length} to ${maxContentLength} characters for ${url}`); + content = content.substring(0, maxContentLength); + } + // Check if we got a meaningful response + if (this.isLowQualityContent(content)) { + throw new Error('Low quality content detected - likely bot detection'); + } + return content; + } + async extractWithBrowser(options) { + const { url, timeout = this.defaultTimeout } = options; + const browser = await this.browserPool.getBrowser(); + const browserType = this.browserPool.getLastUsedBrowserType(); + try { + // Create context options based on browser capabilities + const baseContextOptions = { + userAgent: this.getRandomUserAgent(), + viewport: this.getRandomViewport(), + locale: 'en-US', + timezoneId: this.getRandomTimezone(), + // Simulate real device characteristics + deviceScaleFactor: Math.random() > 0.5 ? 1 : 2, + hasTouch: Math.random() > 0.7, + }; + // Firefox doesn't support isMobile option - check multiple ways to ensure detection + const isFirefox = browserType === 'firefox' || + browserType.includes('firefox') || + browser.constructor.name.toLowerCase().includes('firefox'); + const contextOptions = isFirefox + ? baseContextOptions + : { ...baseContextOptions, isMobile: Math.random() > 0.8 }; + // Create a new context for each request (isolation) + const context = await browser.newContext(contextOptions); + // Add stealth scripts to avoid detection + await context.addInitScript(() => { + // Remove webdriver property + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + // Mock plugins + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + // Mock languages + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + }); + // Mock permissions + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => (parameters.name === 'notifications' ? + Promise.resolve({ state: 'default' }) : + originalQuery(parameters)); + // Remove automation indicators + const windowWithChrome = window; + if (windowWithChrome.chrome) { + delete windowWithChrome.chrome.app; + delete windowWithChrome.chrome.runtime; + } + }); + const page = await context.newPage(); + // Set up request interception to block unnecessary resources + await page.route('**/*', (route) => { + const resourceType = route.request().resourceType(); + // Block images, fonts, and other non-essential resources for faster loading + if (['image', 'font', 'media'].includes(resourceType)) { + route.abort(); + } + else { + route.continue(); + } + }); + // Navigate with realistic options and better error handling + console.log(`[BrowserExtractor] Navigating to ${url}`); + try { + await page.goto(url, { + waitUntil: 'domcontentloaded', // Don't wait for all resources + timeout: Math.min(timeout, 8000) // Reduced timeout, max 8 seconds + }); + } + catch (gotoError) { + // Handle specific protocol errors + const errorMessage = gotoError instanceof Error ? gotoError.message : String(gotoError); + if (errorMessage.includes('ERR_HTTP2_PROTOCOL_ERROR') || errorMessage.includes('HTTP2')) { + console.log(`[BrowserExtractor] HTTP/2 error detected, trying with HTTP/1.1`); + // Create a new context with HTTP/1.1 preference + await context.close(); + const http1Context = await browser.newContext({ + userAgent: this.getRandomUserAgent(), + viewport: this.getRandomViewport(), + locale: 'en-US', + timezoneId: this.getRandomTimezone(), + extraHTTPHeaders: { + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1' + } + }); + const http1Page = await http1Context.newPage(); + // Disable HTTP/2 by intercepting requests + await http1Page.route('**/*', (route) => { + const resourceType = route.request().resourceType(); + if (['image', 'font', 'media'].includes(resourceType)) { + route.abort(); + } + else { + route.continue(); + } + }); + await http1Page.goto(url, { + waitUntil: 'domcontentloaded', + timeout: Math.min(timeout, 6000) + }); + // Quick content extraction + const html = await http1Page.content(); + const content = this.parseContent(html); + await http1Context.close(); + return content; + } + else { + throw gotoError; + } + } + // Quick human simulation - reduced time + await page.mouse.move(Math.random() * 100, Math.random() * 100); + // Reduced wait time for dynamic content + await page.waitForTimeout(500 + Math.random() * 1000); + // Quick check for main content without long wait + try { + await page.waitForSelector('article, main, .content, .post-content, .entry-content', { + timeout: 2000 + }); + } + catch { + console.log(`[BrowserExtractor] No main content selector found, proceeding anyway`); + } + // Extract content using the same logic as axios version + const html = await page.content(); + const content = this.parseContent(html); + await context.close(); + return content; + } + catch (error) { + console.error(`[BrowserExtractor] Browser extraction failed for ${url}:`, error); + throw error; + } + } + async simulateHumanBehavior(page) { + try { + // Random mouse movements + await page.mouse.move(Math.random() * 800, Math.random() * 600); + // Random scroll (common human behavior) + const scrollY = Math.random() * 500; + await page.evaluate((y) => window.scrollTo(0, y), scrollY); + // Small random delay + await page.waitForTimeout(500 + Math.random() * 1000); + // Sometimes click somewhere innocuous + if (Math.random() > 0.7) { + try { + await page.click('body', { timeout: 1000 }); + } + catch { + // Ignore click failures + } + } + } + catch { + // Ignore simulation errors + console.log(`[BrowserExtractor] Behavior simulation failed, continuing`); + } + } + shouldUseBrowser(error, url) { + // Conditions where browser is likely to succeed where axios failed + const indicators = [ + // HTTP status codes that suggest bot detection + error.response?.status === 403, + error.response?.status === 429, + error.response?.status === 503, + // Error messages suggesting JS requirement + error.message?.includes('timeout'), + error.message?.includes('Access denied'), + error.message?.includes('Forbidden'), + error.message?.includes('Low quality content detected'), + // Response content suggesting bot detection + error.response?.data?.includes('Please enable JavaScript'), + error.response?.data?.includes('captcha'), + error.response?.data?.includes('unusual traffic'), + error.response?.data?.includes('robot'), + // Sites known to be JS-heavy + url.includes('twitter.com'), + url.includes('facebook.com'), + url.includes('instagram.com'), + url.includes('linkedin.com'), + url.includes('reddit.com'), + url.includes('medium.com'), + ]; + return indicators.some(indicator => indicator === true); + } + isLowQualityContent(content) { + const lowQualityIndicators = [ + content.length < 100, + content.includes('Please enable JavaScript'), + content.includes('Access Denied'), + content.includes('403 Forbidden'), + content.includes('captcha'), + content.includes('unusual traffic'), + content.includes('robot'), + content.trim() === '', + ]; + return lowQualityIndicators.some(indicator => indicator === true); + } + getRandomHeaders() { + const browsers = [ + { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"', + 'sec-ch-ua-platform': '"Windows"', + }, + { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"', + 'sec-ch-ua-platform': '"macOS"', + }, + { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'sec-ch-ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"', + 'sec-ch-ua-platform': '"Linux"', + } + ]; + const browser = browsers[Math.floor(Math.random() * browsers.length)]; + return { + ...browser, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Cache-Control': 'max-age=0', + 'sec-ch-ua-mobile': '?0', + }; + } + getRandomUserAgent() { + const userAgents = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0', + ]; + return userAgents[Math.floor(Math.random() * userAgents.length)]; + } + getRandomViewport() { + const viewports = [ + { width: 1920, height: 1080 }, + { width: 1366, height: 768 }, + { width: 1440, height: 900 }, + { width: 1536, height: 864 }, + { width: 1280, height: 720 }, + ]; + return viewports[Math.floor(Math.random() * viewports.length)]; + } + getRandomTimezone() { + const timezones = [ + 'America/New_York', + 'America/Los_Angeles', + 'America/Chicago', + 'Europe/London', + 'Europe/Berlin', + 'Asia/Tokyo', + ]; + return timezones[Math.floor(Math.random() * timezones.length)]; + } + async extractContentForResults(results, targetCount = results.length) { + console.log(`[EnhancedContentExtractor] Processing up to ${results.length} results to get ${targetCount} non-PDF results`); + // Filter out PDF files first + const nonPdfResults = results.filter(result => !isPdfUrl(result.url)); + const resultsToProcess = nonPdfResults.slice(0, Math.min(targetCount * 2, 10)); // Process extra to account for failures + console.log(`[EnhancedContentExtractor] Processing ${resultsToProcess.length} non-PDF results concurrently`); + // Process results concurrently with timeout + const extractionPromises = resultsToProcess.map(async (result) => { + try { + // Use a race condition with timeout to prevent hanging + const extractionPromise = this.extractContent({ + url: result.url, + timeout: 6000 // Reduced timeout to 6 seconds per page + }); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Content extraction timeout')), 8000); + }); + const content = await Promise.race([extractionPromise, timeoutPromise]); + const cleanedContent = cleanText(content, this.maxContentLength); + console.log(`[EnhancedContentExtractor] Successfully extracted: ${result.url}`); + return { + ...result, + fullContent: cleanedContent, + contentPreview: getContentPreview(cleanedContent), + wordCount: getWordCount(cleanedContent), + timestamp: generateTimestamp(), + fetchStatus: 'success', + }; + } + catch (error) { + console.log(`[EnhancedContentExtractor] Failed to extract: ${result.url} - ${error instanceof Error ? error.message : 'Unknown error'}`); + return { + ...result, + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp: generateTimestamp(), + fetchStatus: 'error', + error: this.getSpecificErrorMessage(error), + }; + } + }); + // Wait for all extractions to complete + const allResults = await Promise.all(extractionPromises); + // Return successful results first, up to targetCount + const successfulResults = allResults.filter(r => r.fetchStatus === 'success'); + const failedResults = allResults.filter(r => r.fetchStatus === 'error'); + // Combine successful and failed results, prioritizing successful ones + const enhancedResults = [ + ...successfulResults.slice(0, targetCount), + ...failedResults.slice(0, Math.max(0, targetCount - successfulResults.length)) + ].slice(0, targetCount); + console.log(`[EnhancedContentExtractor] Completed processing ${resultsToProcess.length} results, extracted ${successfulResults.length} successful/${failedResults.length} failed`); + return enhancedResults; + } + parseContent(html) { + const $ = cheerio.load(html); + // Remove all script, style, and other non-content elements + $('script, style, noscript, iframe, img, video, audio, canvas, svg, object, embed, applet, form, input, textarea, select, button, label, fieldset, legend, optgroup, option').remove(); + // Remove navigation, header, footer, and other non-content elements + $('nav, header, footer, .nav, .header, .footer, .sidebar, .menu, .breadcrumb, aside, .ad, .advertisement, .ads, .advertisement-container, .social-share, .share-buttons, .comments, .comment-section, .related-posts, .recommendations, .newsletter-signup, .cookie-notice, .privacy-notice, .terms-notice, .disclaimer, .legal, .copyright, .meta, .metadata, .author-info, .publish-date, .tags, .categories, .navigation, .pagination, .search-box, .search-form, .login-form, .signup-form, .newsletter, .popup, .modal, .overlay, .tooltip, .toolbar, .ribbon, .banner, .promo, .sponsored, .affiliate, .tracking, .analytics, .pixel, .beacon').remove(); + // Remove elements with common ad/tracking classes + $('[class*="ad"], [class*="ads"], [class*="advertisement"], [class*="tracking"], [class*="analytics"], [class*="pixel"], [class*="beacon"], [class*="sponsored"], [class*="affiliate"], [class*="promo"], [class*="banner"], [class*="popup"], [class*="modal"], [class*="overlay"], [class*="tooltip"], [class*="toolbar"], [class*="ribbon"]').remove(); + // Remove elements with common non-content IDs + $('[id*="ad"], [id*="ads"], [id*="advertisement"], [id*="tracking"], [id*="analytics"], [id*="pixel"], [id*="beacon"], [id*="sponsored"], [id*="affiliate"], [id*="promo"], [id*="banner"], [id*="popup"], [id*="modal"], [id*="overlay"], [id*="tooltip"], [id*="toolbar"], [id*="ribbon"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"]').remove(); + // Remove image-related elements and attributes + $('picture, source, figure, figcaption, .image, .img, .photo, .picture, .media, .gallery, .slideshow, .carousel').remove(); + $('[data-src*="image"], [data-src*="img"], [data-src*="photo"], [data-src*="picture"]').remove(); + $('[style*="background-image"]').remove(); + // Remove empty elements and whitespace-only elements + $('*').each(function () { + const $this = $(this); + if ($this.children().length === 0 && $this.text().trim() === '') { + $this.remove(); + } + }); + // Try to find the main content area first + let mainContent = ''; + // Priority selectors for main content + const contentSelectors = [ + 'article', + 'main', + '[role="main"]', + '.content', + '.post-content', + '.entry-content', + '.article-content', + '.story-content', + '.news-content', + '.main-content', + '.page-content', + '.text-content', + '.body-content', + '.copy', + '.text', + '.body' + ]; + for (const selector of contentSelectors) { + const $content = $(selector).first(); + if ($content.length > 0) { + mainContent = $content.text().trim(); + if (mainContent.length > 100) { // Ensure we have substantial content + console.log(`[EnhancedContentExtractor] Found content with selector: ${selector} (${mainContent.length} chars)`); + break; + } + } + } + // If no main content found, try body content + if (!mainContent || mainContent.length < 100) { + console.log(`[EnhancedContentExtractor] No main content found, using body content`); + mainContent = $('body').text().trim(); + } + // Clean up the text + const cleanedContent = this.cleanTextContent(mainContent); + return cleanText(cleanedContent, this.maxContentLength); + } + cleanTextContent(text) { + // Remove excessive whitespace + text = text.replace(/\s+/g, ' '); + // Remove image-related text and data URLs + text = text.replace(/data:image\/[^;]+;base64,[A-Za-z0-9+/=]+/g, ''); // Remove base64 image data + text = text.replace(/https?:\/\/[^\s]+\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)(\?[^\s]*)?/gi, ''); // Remove image URLs + text = text.replace(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff)/gi, ''); // Remove image file extensions + text = text.replace(/image|img|photo|picture|gallery|slideshow|carousel/gi, ''); // Remove image-related words + text = text.replace(/click to enlarge|click for full size|view larger|download image/gi, ''); // Remove image action text + // Remove common non-content patterns + text = text.replace(/cookie|privacy|terms|conditions|disclaimer|legal|copyright|all rights reserved/gi, ''); + // Remove excessive line breaks and spacing + text = text.replace(/\n\s*\n/g, '\n'); + text = text.replace(/\r\n/g, '\n'); + text = text.replace(/\r/g, '\n'); + // Remove leading/trailing whitespace + text = text.trim(); + return text; + } + getSpecificErrorMessage(error) { + if (axios.isAxiosError(error)) { + if (error.code === 'ECONNABORTED') { + return 'Request timeout'; + } + if (error.response?.status === 403) { + return '403 Forbidden - Access denied'; + } + if (error.response?.status === 404) { + return '404 Not found'; + } + if (error.message.includes('maxContentLength')) { + return 'Content too long'; + } + if (error.response?.status) { + return `HTTP ${error.response.status}: ${error.message}`; + } + return `Network error: ${error.message}`; + } + return error instanceof Error ? error.message : 'Unknown error'; + } + async closeAll() { + await this.browserPool.closeAll(); + } +} +//# sourceMappingURL=enhanced-content-extractor.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js.map b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js.map new file mode 100644 index 0000000..26010e0 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/enhanced-content-extractor.js.map @@ -0,0 +1 @@ +{"version":3,"file":"enhanced-content-extractor.js","sourceRoot":"","sources":["../src/enhanced-content-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACrG,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,OAAO,wBAAwB;IAClB,cAAc,CAAS;IACvB,gBAAgB,CAAS;IAClC,WAAW,CAAc;IACzB,iBAAiB,CAAS;IAElC;QACE,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1E,uEAAuE;QACvE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACpD,IAAI,CAAC,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAE3E,4BAA4B;QAC5B,IAAI,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC,gEAAgE,YAAY,wBAAwB,CAAC,CAAC;YACnH,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAErF,OAAO,CAAC,GAAG,CAAC,qDAAqD,IAAI,CAAC,cAAc,sBAAsB,IAAI,CAAC,gBAAgB,uBAAuB,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAClL,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,OAAiC;QACpD,MAAM,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;QAExB,OAAO,CAAC,GAAG,CAAC,uDAAuD,GAAG,EAAE,CAAC,CAAC;QAE1E,+CAA+C;QAC/C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,iEAAiE,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;YACrG,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,4CAA4C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YAEpH,2DAA2D;YAC3D,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,oEAAoE,GAAG,EAAE,CAAC,CAAC;gBACvF,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;oBACvD,OAAO,CAAC,GAAG,CAAC,mEAAmE,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;oBACvG,OAAO,OAAO,CAAC;gBACjB,CAAC;gBAAC,OAAO,YAAY,EAAE,CAAC;oBACtB,OAAO,CAAC,KAAK,CAAC,4DAA4D,EAAE,YAAY,CAAC,CAAC;oBAC1F,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,EAAE,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAiC;QAC9D,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAEjG,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YACpC,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE;YAChC,OAAO;YACP,yEAAyE;YACzE,cAAc,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG;SACjD,CAAC,CAAC;QAEH,IAAI,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE/C,gFAAgF;QAChF,IAAI,gBAAgB,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,qDAAqD,OAAO,CAAC,MAAM,OAAO,gBAAgB,mBAAmB,GAAG,EAAE,CAAC,CAAC;YAChI,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACnD,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,OAAiC;QAChE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC;QAE9D,IAAI,CAAC;YACH,uDAAuD;YACvD,MAAM,kBAAkB,GAAG;gBACzB,SAAS,EAAE,IAAI,CAAC,kBAAkB,EAAE;gBACpC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,EAAE;gBAClC,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE;gBACpC,uCAAuC;gBACvC,iBAAiB,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG;aAC9B,CAAC;YAEF,oFAAoF;YACpF,MAAM,SAAS,GAAG,WAAW,KAAK,SAAS;gBAC1B,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC/B,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAE5E,MAAM,cAAc,GAAG,SAAS;gBAC9B,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,EAAE,GAAG,kBAAkB,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;YAE7D,oDAAoD;YACpD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAEzD,yCAAyC;YACzC,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE;gBAC/B,4BAA4B;gBAC5B,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE;oBAC5C,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS;iBACrB,CAAC,CAAC;gBAEH,eAAe;gBACf,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE;oBAC1C,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;iBAC3B,CAAC,CAAC;gBAEH,iBAAiB;gBACjB,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE;oBAC5C,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC;iBAC3B,CAAC,CAAC;gBAEH,mBAAmB;gBACnB,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC;gBACzD,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,CACnD,UAAU,CAAC,IAAI,KAAK,eAAe,CAAC,CAAC;oBACnC,OAAO,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,SAAS,EAAiC,CAAC,CAAC,CAAC;oBACtE,aAAa,CAAC,UAAU,CAAC,CAC5B,CAAC;gBAEF,+BAA+B;gBAC/B,MAAM,gBAAgB,GAAG,MAAa,CAAC;gBACvC,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;oBAC5B,OAAO,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC;oBACnC,OAAO,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC;gBACzC,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAErC,6DAA6D;YAC7D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBAEpD,4EAA4E;gBAC5E,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACtD,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,4DAA4D;YAC5D,OAAO,CAAC,GAAG,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;YAEvD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;oBACnB,SAAS,EAAE,kBAAkB,EAAE,+BAA+B;oBAC9D,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,iCAAiC;iBACnE,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,kCAAkC;gBAClC,MAAM,YAAY,GAAG,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAExF,IAAI,YAAY,CAAC,QAAQ,CAAC,0BAA0B,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxF,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;oBAE9E,gDAAgD;oBAChD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtB,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;wBAC5C,SAAS,EAAE,IAAI,CAAC,kBAAkB,EAAE;wBACpC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,EAAE;wBAClC,MAAM,EAAE,OAAO;wBACf,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE;wBACpC,gBAAgB,EAAE;4BAChB,YAAY,EAAE,YAAY;4BAC1B,2BAA2B,EAAE,GAAG;yBACjC;qBACF,CAAC,CAAC;oBAEH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;oBAE/C,0CAA0C;oBAC1C,MAAM,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;wBACtC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;wBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;4BACtD,KAAK,CAAC,KAAK,EAAE,CAAC;wBAChB,CAAC;6BAAM,CAAC;4BACN,KAAK,CAAC,QAAQ,EAAE,CAAC;wBACnB,CAAC;oBACH,CAAC,CAAC,CAAC;oBAEH,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;wBACxB,SAAS,EAAE,kBAAkB;wBAC7B,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC;qBACjC,CAAC,CAAC;oBAEH,2BAA2B;oBAC3B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;oBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;oBACxC,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;oBAC3B,OAAO,OAAO,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACN,MAAM,SAAS,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,wCAAwC;YACxC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;YAEhE,wCAAwC;YACxC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;YAEtD,iDAAiD;YACjD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC,wDAAwD,EAAE;oBACnF,OAAO,EAAE,IAAI;iBACd,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,GAAG,CAAC,sEAAsE,CAAC,CAAC;YACtF,CAAC;YAED,wDAAwD;YACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC;QAEjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,oDAAoD,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YACjF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,IAAU;QAC5C,IAAI,CAAC;YACH,yBAAyB;YACzB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,EACnB,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CACpB,CAAC;YAEF,wCAAwC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;YACpC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAE3D,qBAAqB;YACrB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;YAEtD,sCAAsC;YACtC,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;gBACxB,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC9C,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,2BAA2B;YAC3B,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,KAAU,EAAE,GAAW;QAC9C,mEAAmE;QACnE,MAAM,UAAU,GAAG;YACjB,+CAA+C;YAC/C,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG;YAC9B,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG;YAC9B,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG;YAE9B,2CAA2C;YAC3C,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC;YAClC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,CAAC;YACxC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC;YACpC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,8BAA8B,CAAC;YAEvD,4CAA4C;YAC5C,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,0BAA0B,CAAC;YAC1D,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC;YACzC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,iBAAiB,CAAC;YACjD,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC;YAEvC,6BAA6B;YAC7B,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC5B,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC7B,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC5B,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC1B,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC3B,CAAC;QAEF,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC;IAC1D,CAAC;IAEO,mBAAmB,CAAC,OAAe;QACzC,MAAM,oBAAoB,GAAG;YAC3B,OAAO,CAAC,MAAM,GAAG,GAAG;YACpB,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC;YAC5C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;YACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;YACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC3B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzB,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;SACtB,CAAC;QAEF,OAAO,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC;IACpE,CAAC;IAEO,gBAAgB;QACtB,MAAM,QAAQ,GAAG;YACf;gBACE,YAAY,EAAE,iHAAiH;gBAC/H,WAAW,EAAE,mEAAmE;gBAChF,oBAAoB,EAAE,WAAW;aAClC;YACD;gBACE,YAAY,EAAE,uHAAuH;gBACrI,WAAW,EAAE,mEAAmE;gBAChF,oBAAoB,EAAE,SAAS;aAChC;YACD;gBACE,YAAY,EAAE,uGAAuG;gBACrH,WAAW,EAAE,mEAAmE;gBAChF,oBAAoB,EAAE,SAAS;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAEtE,OAAO;YACL,GAAG,OAAO;YACV,QAAQ,EAAE,kGAAkG;YAC5G,iBAAiB,EAAE,gBAAgB;YACnC,iBAAiB,EAAE,mBAAmB;YACtC,KAAK,EAAE,GAAG;YACV,YAAY,EAAE,YAAY;YAC1B,2BAA2B,EAAE,GAAG;YAChC,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,MAAM;YACxB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,WAAW;YAC5B,kBAAkB,EAAE,IAAI;SACzB,CAAC;IACJ,CAAC;IAEO,kBAAkB;QACxB,MAAM,UAAU,GAAG;YACjB,iHAAiH;YACjH,uHAAuH;YACvH,uGAAuG;YACvG,kFAAkF;YAClF,sFAAsF;SACvF,CAAC;QAEF,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB;QACvB,MAAM,SAAS,GAAG;YAChB,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;YAC7B,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5B,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5B,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5B,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;SAC7B,CAAC;QAEF,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,CAAC;IAEO,iBAAiB;QACvB,MAAM,SAAS,GAAG;YAChB,kBAAkB;YAClB,qBAAqB;YACrB,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,YAAY;SACb,CAAC;QAEF,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,OAAuB,EAAE,cAAsB,OAAO,CAAC,MAAM;QAC1F,OAAO,CAAC,GAAG,CAAC,+CAA+C,OAAO,CAAC,MAAM,mBAAmB,WAAW,kBAAkB,CAAC,CAAC;QAE3H,6BAA6B;QAC7B,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,wCAAwC;QAExH,OAAO,CAAC,GAAG,CAAC,yCAAyC,gBAAgB,CAAC,MAAM,+BAA+B,CAAC,CAAC;QAE7G,4CAA4C;QAC5C,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAyB,EAAE;YACtF,IAAI,CAAC;gBACH,uDAAuD;gBACvD,MAAM,iBAAiB,GAAG,IAAI,CAAC,cAAc,CAAC;oBAC5C,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,OAAO,EAAE,IAAI,CAAC,wCAAwC;iBACvD,CAAC,CAAC;gBAEH,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;oBACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC1E,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC;gBACxE,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAEjE,OAAO,CAAC,GAAG,CAAC,sDAAsD,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChF,OAAO;oBACL,GAAG,MAAM;oBACT,WAAW,EAAE,cAAc;oBAC3B,cAAc,EAAE,iBAAiB,CAAC,cAAc,CAAC;oBACjD,SAAS,EAAE,YAAY,CAAC,cAAc,CAAC;oBACvC,SAAS,EAAE,iBAAiB,EAAE;oBAC9B,WAAW,EAAE,SAAkB;iBAChC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,iDAAiD,MAAM,CAAC,GAAG,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;gBACzI,OAAO;oBACL,GAAG,MAAM;oBACT,WAAW,EAAE,EAAE;oBACf,cAAc,EAAE,EAAE;oBAClB,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,iBAAiB,EAAE;oBAC9B,WAAW,EAAE,OAAgB;oBAC7B,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC;iBAC3C,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,uCAAuC;QACvC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAEzD,qDAAqD;QACrD,MAAM,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC;QAC9E,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC;QAExE,sEAAsE;QACtE,MAAM,eAAe,GAAG;YACtB,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC;YAC1C,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;SAC/E,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAExB,OAAO,CAAC,GAAG,CAAC,mDAAmD,gBAAgB,CAAC,MAAM,uBAAuB,iBAAiB,CAAC,MAAM,eAAe,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QACnL,OAAO,eAAe,CAAC;IACzB,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,2DAA2D;QAC3D,CAAC,CAAC,0KAA0K,CAAC,CAAC,MAAM,EAAE,CAAC;QAEvL,oEAAoE;QACpE,CAAC,CAAC,knBAAknB,CAAC,CAAC,MAAM,EAAE,CAAC;QAE/nB,kDAAkD;QAClD,CAAC,CAAC,6UAA6U,CAAC,CAAC,MAAM,EAAE,CAAC;QAE1V,8CAA8C;QAC9C,CAAC,CAAC,6WAA6W,CAAC,CAAC,MAAM,EAAE,CAAC;QAE1X,+CAA+C;QAC/C,CAAC,CAAC,8GAA8G,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3H,CAAC,CAAC,oFAAoF,CAAC,CAAC,MAAM,EAAE,CAAC;QACjG,CAAC,CAAC,6BAA6B,CAAC,CAAC,MAAM,EAAE,CAAC;QAE1C,qDAAqD;QACrD,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;YACV,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAChE,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0CAA0C;QAC1C,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,sCAAsC;QACtC,MAAM,gBAAgB,GAAG;YACvB,SAAS;YACT,MAAM;YACN,eAAe;YACf,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,kBAAkB;YAClB,gBAAgB;YAChB,eAAe;YACf,eAAe;YACf,eAAe;YACf,eAAe;YACf,eAAe;YACf,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC;QAEF,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,qCAAqC;oBACnE,OAAO,CAAC,GAAG,CAAC,2DAA2D,QAAQ,KAAK,WAAW,CAAC,MAAM,SAAS,CAAC,CAAC;oBACjH,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,sEAAsE,CAAC,CAAC;YACpF,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC;QAED,oBAAoB;QACpB,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAE1D,OAAO,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1D,CAAC;IAEO,gBAAgB,CAAC,IAAY;QACnC,8BAA8B;QAC9B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAEjC,0CAA0C;QAC1C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,2CAA2C,EAAE,EAAE,CAAC,CAAC,CAAC,2BAA2B;QACjG,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,0EAA0E,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;QACzH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,8CAA8C,EAAE,EAAE,CAAC,CAAC,CAAC,+BAA+B;QACxG,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,sDAAsD,EAAE,EAAE,CAAC,CAAC,CAAC,6BAA6B;QAC9G,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,mEAAmE,EAAE,EAAE,CAAC,CAAC,CAAC,2BAA2B;QAEzH,qCAAqC;QACrC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kFAAkF,EAAE,EAAE,CAAC,CAAC;QAE5G,2CAA2C;QAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACtC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAEjC,qCAAqC;QACrC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,uBAAuB,CAAC,KAAc;QAC5C,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAClC,OAAO,iBAAiB,CAAC;YAC3B,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnC,OAAO,+BAA+B,CAAC;YACzC,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnC,OAAO,eAAe,CAAC;YACzB,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC/C,OAAO,kBAAkB,CAAC;YAC5B,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;gBAC3B,OAAO,QAAQ,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3D,CAAC;YACD,OAAO,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;QAED,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;IACpC,CAAC;CACF"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/index.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/index.d.ts new file mode 100644 index 0000000..dc1ec89 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/index.d.ts @@ -0,0 +1,3 @@ +#!/usr/bin/env node +export {}; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/index.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/index.d.ts.map new file mode 100644 index 0000000..535b86d --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/index.js b/local_ai/.config/mcp/web-search-mcp/dist/index.js new file mode 100644 index 0000000..f654941 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/index.js @@ -0,0 +1,449 @@ +#!/usr/bin/env node +console.log('Web Search MCP Server starting...'); +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { z } from 'zod'; +import { SearchEngine } from './search-engine.js'; +import { EnhancedContentExtractor } from './enhanced-content-extractor.js'; +import { isPdfUrl } from './utils.js'; +class WebSearchMCPServer { + server; + searchEngine; + contentExtractor; + constructor() { + this.server = new McpServer({ + name: 'web-search-mcp', + version: '0.3.1', + }); + this.searchEngine = new SearchEngine(); + this.contentExtractor = new EnhancedContentExtractor(); + this.setupTools(); + this.setupGracefulShutdown(); + } + setupTools() { + // Register the main web search tool (primary choice for comprehensive searches) + this.server.tool('full-web-search', 'Search the web and fetch complete page content from top results. This is the most comprehensive web search tool. It searches the web and then follows the resulting links to extract their full page content, providing the most detailed and complete information available. Use get-web-search-summaries for a lightweight alternative.', { + query: z.string().describe('Search query to execute (recommended for comprehensive research)'), + limit: z.union([z.number(), z.string()]).transform((val) => { + const num = typeof val === 'string' ? parseInt(val, 10) : val; + if (isNaN(num) || num < 1 || num > 10) { + throw new Error('Invalid limit: must be a number between 1 and 10'); + } + return num; + }).default(5).describe('Number of results to return with full content (1-10)'), + includeContent: z.union([z.boolean(), z.string()]).transform((val) => { + if (typeof val === 'string') { + return val.toLowerCase() === 'true'; + } + return Boolean(val); + }).default(true).describe('Whether to fetch full page content (default: true)'), + maxContentLength: z.union([z.number(), z.string()]).transform((val) => { + const num = typeof val === 'string' ? parseInt(val, 10) : val; + if (isNaN(num) || num < 0) { + throw new Error('Invalid maxContentLength: must be a non-negative number'); + } + return num; + }).optional().describe('Maximum characters per result content (0 = no limit). Usually not needed - content length is automatically optimized.'), + }, async (args) => { + console.log(`[MCP] Tool call received: full-web-search`); + console.log(`[MCP] Raw arguments:`, JSON.stringify(args, null, 2)); + try { + // Convert and validate arguments + const validatedArgs = this.validateAndConvertArgs(args); + // Auto-detect model types based on parameter formats + // Llama models often send string parameters and struggle with large responses + const isLikelyLlama = typeof args === 'object' && args !== null && (('limit' in args && typeof args.limit === 'string') || + ('includeContent' in args && typeof args.includeContent === 'string')); + // Detect models that handle large responses well (Qwen, Gemma, recent Deepseek) + const isLikelyRobustModel = typeof args === 'object' && args !== null && (('limit' in args && typeof args.limit === 'number') && + ('includeContent' in args && typeof args.includeContent === 'boolean')); + // Only apply auto-limit if maxContentLength is not explicitly set (including 0) + const hasExplicitMaxLength = typeof args === 'object' && args !== null && 'maxContentLength' in args; + if (!hasExplicitMaxLength && isLikelyLlama) { + console.log(`[MCP] Detected potential Llama model (string parameters), applying content length limit`); + validatedArgs.maxContentLength = 2000; // Reasonable limit for Llama + } + // For robust models (Qwen, Gemma, recent Deepseek), remove maxContentLength if it's set to a low value + if (isLikelyRobustModel && validatedArgs.maxContentLength && validatedArgs.maxContentLength < 5000) { + console.log(`[MCP] Detected robust model (numeric parameters), removing unnecessary content length limit`); + validatedArgs.maxContentLength = undefined; + } + console.log(`[MCP] Validated args:`, JSON.stringify(validatedArgs, null, 2)); + console.log(`[MCP] Starting web search...`); + const result = await this.handleWebSearch(validatedArgs); + console.log(`[MCP] Search completed, found ${result.results.length} results`); + // Format the results as a comprehensive text response + let responseText = `Search completed for "${result.query}" with ${result.total_results} results:\n\n`; + // Add status line if available + if (result.status) { + responseText += `**Status:** ${result.status}\n\n`; + } + const maxLength = validatedArgs.maxContentLength; + result.results.forEach((searchResult, idx) => { + responseText += `**${idx + 1}. ${searchResult.title}**\n`; + responseText += `URL: ${searchResult.url}\n`; + responseText += `Description: ${searchResult.description}\n`; + if (searchResult.fullContent && searchResult.fullContent.trim()) { + let content = searchResult.fullContent; + if (maxLength && maxLength > 0 && content.length > maxLength) { + content = content.substring(0, maxLength) + `\n\n[Content truncated at ${maxLength} characters]`; + } + responseText += `\n**Full Content:**\n${content}\n`; + } + else if (searchResult.contentPreview && searchResult.contentPreview.trim()) { + let content = searchResult.contentPreview; + if (maxLength && maxLength > 0 && content.length > maxLength) { + content = content.substring(0, maxLength) + `\n\n[Content truncated at ${maxLength} characters]`; + } + responseText += `\n**Content Preview:**\n${content}\n`; + } + else if (searchResult.fetchStatus === 'error') { + responseText += `\n**Content Extraction Failed:** ${searchResult.error}\n`; + } + responseText += `\n---\n\n`; + }); + return { + content: [ + { + type: 'text', + text: responseText, + }, + ], + }; + } + catch (error) { + console.error(`[MCP] Error in tool handler:`, error); + throw error; + } + }); + // Register the lightweight web search summaries tool (secondary choice for quick results) + this.server.tool('get-web-search-summaries', 'Search the web and return only the search result snippets/descriptions without following links to extract full page content. This is a lightweight alternative to full-web-search for when you only need brief search results. For comprehensive information, use full-web-search instead.', { + query: z.string().describe('Search query to execute (lightweight alternative)'), + limit: z.union([z.number(), z.string()]).transform((val) => { + const num = typeof val === 'string' ? parseInt(val, 10) : val; + if (isNaN(num) || num < 1 || num > 10) { + throw new Error('Invalid limit: must be a number between 1 and 10'); + } + return num; + }).default(5).describe('Number of search results to return (1-10)'), + }, async (args) => { + console.log(`[MCP] Tool call received: get-web-search-summaries`); + console.log(`[MCP] Raw arguments:`, JSON.stringify(args, null, 2)); + try { + // Validate arguments + if (typeof args !== 'object' || args === null) { + throw new Error('Invalid arguments: args must be an object'); + } + const obj = args; + if (!obj.query || typeof obj.query !== 'string') { + throw new Error('Invalid arguments: query is required and must be a string'); + } + let limit = 5; // default + if (obj.limit !== undefined) { + const limitValue = typeof obj.limit === 'string' ? parseInt(obj.limit, 10) : obj.limit; + if (typeof limitValue !== 'number' || isNaN(limitValue) || limitValue < 1 || limitValue > 10) { + throw new Error('Invalid limit: must be a number between 1 and 10'); + } + limit = limitValue; + } + console.log(`[MCP] Starting web search summaries...`); + try { + // Use existing search engine to get results with snippets + const searchResponse = await this.searchEngine.search({ + query: obj.query, + numResults: limit, + }); + // const searchTime = Date.now() - startTime; // Unused for now + // Convert to summary format (no content extraction) + const summaryResults = searchResponse.results.map(item => ({ + title: item.title, + url: item.url, + description: item.description, + timestamp: item.timestamp, + })); + console.log(`[MCP] Search summaries completed, found ${summaryResults.length} results`); + // Format the results as text + let responseText = `Search summaries for "${obj.query}" with ${summaryResults.length} results:\n\n`; + summaryResults.forEach((summary, i) => { + responseText += `**${i + 1}. ${summary.title}**\n`; + responseText += `URL: ${summary.url}\n`; + responseText += `Description: ${summary.description}\n`; + responseText += `\n---\n\n`; + }); + return { + content: [ + { + type: 'text', + text: responseText, + }, + ], + }; + } + finally { + // Ensure browsers are cleaned up after search-only operations + // This prevents EventEmitter memory leaks when browsers accumulate listeners + try { + await this.searchEngine.closeAll(); + } + catch (cleanupError) { + console.error(`[MCP] Error during browser cleanup:`, cleanupError); + } + } + } + catch (error) { + console.error(`[MCP] Error in get-web-search-summaries tool handler:`, error); + throw error; + } + }); + // Register the single page content extraction tool + this.server.tool('get-single-web-page-content', 'Extract and return the full content from a single web page URL. This tool follows a provided URL and extracts the main page content. Useful for getting detailed content from a specific webpage without performing a search.', { + url: z.string().url().describe('The URL of the web page to extract content from'), + maxContentLength: z.union([z.number(), z.string()]).transform((val) => { + const num = typeof val === 'string' ? parseInt(val, 10) : val; + if (isNaN(num) || num < 0) { + throw new Error('Invalid maxContentLength: must be a non-negative number'); + } + return num; + }).optional().describe('Maximum characters for the extracted content (0 = no limit, undefined = use default limit). Usually not needed - content length is automatically optimized.'), + }, async (args) => { + console.log(`[MCP] Tool call received: get-single-web-page-content`); + console.log(`[MCP] Raw arguments:`, JSON.stringify(args, null, 2)); + try { + // Validate arguments + if (typeof args !== 'object' || args === null) { + throw new Error('Invalid arguments: args must be an object'); + } + const obj = args; + if (!obj.url || typeof obj.url !== 'string') { + throw new Error('Invalid arguments: url is required and must be a string'); + } + let maxContentLength; + if (obj.maxContentLength !== undefined) { + const maxLengthValue = typeof obj.maxContentLength === 'string' ? parseInt(obj.maxContentLength, 10) : obj.maxContentLength; + if (typeof maxLengthValue !== 'number' || isNaN(maxLengthValue) || maxLengthValue < 0) { + throw new Error('Invalid maxContentLength: must be a non-negative number'); + } + // If maxContentLength is 0, treat it as "no limit" (undefined) + maxContentLength = maxLengthValue === 0 ? undefined : maxLengthValue; + } + console.log(`[MCP] Starting single page content extraction for: ${obj.url}`); + // Use existing content extractor to get page content + const content = await this.contentExtractor.extractContent({ + url: obj.url, + maxContentLength, + }); + // Get page title from URL (simple extraction) + const urlObj = new URL(obj.url); + const title = urlObj.hostname + urlObj.pathname; + // Create content preview and word count + // const contentPreview = content.length > 200 ? content.substring(0, 200) + '...' : content; // Unused for now + const wordCount = content.split(/\s+/).filter(word => word.length > 0).length; + console.log(`[MCP] Single page content extraction completed, extracted ${content.length} characters`); + // Format the result as text + let responseText = `**Page Content from: ${obj.url}**\n\n`; + responseText += `**Title:** ${title}\n`; + responseText += `**Word Count:** ${wordCount}\n`; + responseText += `**Content Length:** ${content.length} characters\n\n`; + if (maxContentLength && maxContentLength > 0 && content.length > maxContentLength) { + responseText += `**Content (truncated at ${maxContentLength} characters):**\n${content.substring(0, maxContentLength)}\n\n[Content truncated at ${maxContentLength} characters]`; + } + else { + responseText += `**Content:**\n${content}`; + } + return { + content: [ + { + type: 'text', + text: responseText, + }, + ], + }; + } + catch (error) { + console.error(`[MCP] Error in get-single-web-page-content tool handler:`, error); + throw error; + } + }); + } + validateAndConvertArgs(args) { + if (typeof args !== 'object' || args === null) { + throw new Error('Invalid arguments: args must be an object'); + } + const obj = args; + // Ensure query is a string + if (!obj.query || typeof obj.query !== 'string') { + throw new Error('Invalid arguments: query is required and must be a string'); + } + // Convert limit to number if it's a string + let limit = 5; // default + if (obj.limit !== undefined) { + const limitValue = typeof obj.limit === 'string' ? parseInt(obj.limit, 10) : obj.limit; + if (typeof limitValue !== 'number' || isNaN(limitValue) || limitValue < 1 || limitValue > 10) { + throw new Error('Invalid limit: must be a number between 1 and 10'); + } + limit = limitValue; + } + // Convert includeContent to boolean if it's a string + let includeContent = true; // default + if (obj.includeContent !== undefined) { + if (typeof obj.includeContent === 'string') { + includeContent = obj.includeContent.toLowerCase() === 'true'; + } + else { + includeContent = Boolean(obj.includeContent); + } + } + return { + query: obj.query, + limit, + includeContent, + }; + } + async handleWebSearch(input) { + const startTime = Date.now(); + const { query, limit = 5, includeContent = true } = input; + console.error(`[web-search-mcp] DEBUG: handleWebSearch called with limit=${limit}, includeContent=${includeContent}`); + try { + // Request extra search results to account for potential PDF files that will be skipped + // Request up to 2x the limit or at least 5 extra results, capped at 10 (Google's max) + const searchLimit = includeContent ? Math.min(limit * 2 + 2, 10) : limit; + console.log(`[web-search-mcp] DEBUG: Requesting ${searchLimit} search results to get ${limit} non-PDF content results`); + // Perform the search + const searchResponse = await this.searchEngine.search({ + query, + numResults: searchLimit, + }); + const searchResults = searchResponse.results; + // Log search summary + const pdfCount = searchResults.filter(result => isPdfUrl(result.url)).length; + const followedCount = searchResults.length - pdfCount; + console.error(`[web-search-mcp] DEBUG: Search engine: ${searchResponse.engine}; ${limit} requested/${searchResults.length} obtained; PDF: ${pdfCount}; ${followedCount} followed.`); + // Extract content from each result if requested, with target count + const enhancedResults = includeContent + ? await this.contentExtractor.extractContentForResults(searchResults, limit) + : searchResults.slice(0, limit); // If not extracting content, just take the first 'limit' results + // Log extraction summary with failure reasons and generate combined status + let combinedStatus = `Search engine: ${searchResponse.engine}; ${limit} result requested/${searchResults.length} obtained; PDF: ${pdfCount}; ${followedCount} followed`; + if (includeContent) { + const successCount = enhancedResults.filter(r => r.fetchStatus === 'success').length; + const failedResults = enhancedResults.filter(r => r.fetchStatus === 'error'); + const failedCount = failedResults.length; + const failureReasons = this.categorizeFailureReasons(failedResults); + const failureReasonText = failureReasons.length > 0 ? ` (${failureReasons.join(', ')})` : ''; + console.error(`[web-search-mcp] DEBUG: Links requested: ${limit}; Successfully extracted: ${successCount}; Failed: ${failedCount}${failureReasonText}; Results: ${enhancedResults.length}.`); + // Add extraction info to combined status + combinedStatus += `; Successfully extracted: ${successCount}; Failed: ${failedCount}; Results: ${enhancedResults.length}`; + } + const searchTime = Date.now() - startTime; + return { + results: enhancedResults, + total_results: enhancedResults.length, + search_time_ms: searchTime, + query, + status: combinedStatus, + }; + } + catch (error) { + console.error('Web search error:', error); + throw new Error(`Web search failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + categorizeFailureReasons(failedResults) { + const reasonCounts = new Map(); + failedResults.forEach(result => { + if (result.error) { + const category = this.categorizeError(result.error); + reasonCounts.set(category, (reasonCounts.get(category) || 0) + 1); + } + }); + return Array.from(reasonCounts.entries()).map(([reason, count]) => count > 1 ? `${reason} (${count})` : reason); + } + categorizeError(errorMessage) { + const lowerError = errorMessage.toLowerCase(); + if (lowerError.includes('timeout') || lowerError.includes('timed out')) { + return 'Timeout'; + } + if (lowerError.includes('403') || lowerError.includes('forbidden')) { + return 'Access denied'; + } + if (lowerError.includes('404') || lowerError.includes('not found')) { + return 'Not found'; + } + if (lowerError.includes('bot') || lowerError.includes('captcha') || lowerError.includes('unusual traffic')) { + return 'Bot detection'; + } + if (lowerError.includes('too large') || lowerError.includes('content length') || lowerError.includes('maxcontentlength')) { + return 'Content too long'; + } + if (lowerError.includes('ssl') || lowerError.includes('certificate') || lowerError.includes('tls')) { + return 'SSL error'; + } + if (lowerError.includes('network') || lowerError.includes('connection') || lowerError.includes('econnrefused')) { + return 'Network error'; + } + if (lowerError.includes('dns') || lowerError.includes('hostname')) { + return 'DNS error'; + } + return 'Other error'; + } + setupGracefulShutdown() { + // Handle unhandled promise rejections + process.on('unhandledRejection', (reason, promise) => { + console.error('Unhandled Rejection at:', promise, 'reason:', reason); + // Don't exit on unhandled rejections, just log them + }); + // Handle uncaught exceptions + process.on('uncaughtException', (error) => { + console.error('Uncaught Exception:', error); + // Don't exit on uncaught exceptions in MCP context + }); + // Graceful shutdown - close browsers when process exits + process.on('SIGINT', async () => { + console.log('Shutting down gracefully...'); + try { + await Promise.all([ + this.contentExtractor.closeAll(), + this.searchEngine.closeAll() + ]); + } + catch (error) { + console.error('Error during graceful shutdown:', error); + } + process.exit(0); + }); + process.on('SIGTERM', async () => { + console.log('Shutting down gracefully...'); + try { + await Promise.all([ + this.contentExtractor.closeAll(), + this.searchEngine.closeAll() + ]); + } + catch (error) { + console.error('Error during graceful shutdown:', error); + } + process.exit(0); + }); + } + async run() { + console.log('Setting up MCP server...'); + const transport = new StdioServerTransport(); + console.log('Connecting to transport...'); + await this.server.connect(transport); + console.log('Web Search MCP Server started'); + console.log('Server timestamp:', new Date().toISOString()); + console.log('Waiting for MCP messages...'); + } +} +// Start the server +const server = new WebSearchMCPServer(); +server.run().catch((error) => { + if (error instanceof Error) { + console.error('Server error:', error.message); + } + else { + console.error('Server error:', error); + } + process.exit(1); +}); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/index.js.map b/local_ai/.config/mcp/web-search-mcp/dist/index.js.map new file mode 100644 index 0000000..6579721 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAE3E,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,kBAAkB;IACd,MAAM,CAAY;IAClB,YAAY,CAAe;IAC3B,gBAAgB,CAA2B;IAEnD;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC;YAC1B,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,gBAAgB,GAAG,IAAI,wBAAwB,EAAE,CAAC;QAEvD,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,IAAI,CAAC,qBAAqB,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU;QAChB,gFAAgF;QAChF,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iBAAiB,EACjB,2UAA2U,EAC3U;YACE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kEAAkE,CAAC;YAC9F,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzD,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9D,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,EAAE,EAAE,CAAC;oBACtC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACtE,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,sDAAsD,CAAC;YAC9E,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnE,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC5B,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;gBACtC,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,oDAAoD,CAAC;YAC/E,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;gBACpE,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9D,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uHAAuH,CAAC;SAChJ,EACD,KAAK,EAAE,IAAa,EAAE,EAAE;YACtB,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAEnE,IAAI,CAAC;gBACH,iCAAiC;gBACjC,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;gBAExD,qDAAqD;gBACrD,8EAA8E;gBAC9E,MAAM,aAAa,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CACjE,CAAC,OAAO,IAAI,IAAI,IAAI,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ,CAAC;oBAChF,CAAC,gBAAgB,IAAI,IAAI,IAAI,OAAQ,IAAgC,CAAC,cAAc,KAAK,QAAQ,CAAC,CACnG,CAAC;gBAEF,gFAAgF;gBAChF,MAAM,mBAAmB,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CACvE,CAAC,OAAO,IAAI,IAAI,IAAI,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ,CAAC;oBAChF,CAAC,gBAAgB,IAAI,IAAI,IAAI,OAAQ,IAAgC,CAAC,cAAc,KAAK,SAAS,CAAC,CACpG,CAAC;gBAEF,gFAAgF;gBAChF,MAAM,oBAAoB,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,kBAAkB,IAAI,IAAI,CAAC;gBAErG,IAAI,CAAC,oBAAoB,IAAI,aAAa,EAAE,CAAC;oBAC3C,OAAO,CAAC,GAAG,CAAC,yFAAyF,CAAC,CAAC;oBACvG,aAAa,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,6BAA6B;gBACtE,CAAC;gBAED,uGAAuG;gBACvG,IAAI,mBAAmB,IAAI,aAAa,CAAC,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,GAAG,IAAI,EAAE,CAAC;oBACnG,OAAO,CAAC,GAAG,CAAC,6FAA6F,CAAC,CAAC;oBAC3G,aAAa,CAAC,gBAAgB,GAAG,SAAS,CAAC;gBAC7C,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBAE7E,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;gBAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;gBAEzD,OAAO,CAAC,GAAG,CAAC,iCAAiC,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;gBAE9E,sDAAsD;gBACtD,IAAI,YAAY,GAAG,yBAAyB,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,aAAa,eAAe,CAAC;gBAEtG,+BAA+B;gBAC/B,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClB,YAAY,IAAI,eAAe,MAAM,CAAC,MAAM,MAAM,CAAC;gBACrD,CAAC;gBAED,MAAM,SAAS,GAAG,aAAa,CAAC,gBAAgB,CAAC;gBAEjD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE;oBAC3C,YAAY,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,YAAY,CAAC,KAAK,MAAM,CAAC;oBAC1D,YAAY,IAAI,QAAQ,YAAY,CAAC,GAAG,IAAI,CAAC;oBAC7C,YAAY,IAAI,gBAAgB,YAAY,CAAC,WAAW,IAAI,CAAC;oBAE7D,IAAI,YAAY,CAAC,WAAW,IAAI,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;wBAChE,IAAI,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC;wBACvC,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;4BAC7D,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,6BAA6B,SAAS,cAAc,CAAC;wBACnG,CAAC;wBACD,YAAY,IAAI,wBAAwB,OAAO,IAAI,CAAC;oBACtD,CAAC;yBAAM,IAAI,YAAY,CAAC,cAAc,IAAI,YAAY,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;wBAC7E,IAAI,OAAO,GAAG,YAAY,CAAC,cAAc,CAAC;wBAC1C,IAAI,SAAS,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;4BAC7D,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,6BAA6B,SAAS,cAAc,CAAC;wBACnG,CAAC;wBACD,YAAY,IAAI,2BAA2B,OAAO,IAAI,CAAC;oBACzD,CAAC;yBAAM,IAAI,YAAY,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;wBAChD,YAAY,IAAI,oCAAoC,YAAY,CAAC,KAAK,IAAI,CAAC;oBAC7E,CAAC;oBAED,YAAY,IAAI,WAAW,CAAC;gBAC9B,CAAC,CAAC,CAAC;gBAEH,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,YAAY;yBACnB;qBACF;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;gBACrD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CACF,CAAC;QAEF,0FAA0F;QAC1F,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,0BAA0B,EAC1B,4RAA4R,EAC5R;YACE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mDAAmD,CAAC;YAC/E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzD,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9D,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,EAAE,EAAE,CAAC;oBACtC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACtE,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,2CAA2C,CAAC;SACpE,EACD,KAAK,EAAE,IAAa,EAAE,EAAE;YACtB,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAClE,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAEnE,IAAI,CAAC;gBACH,qBAAqB;gBACrB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC9C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAC/D,CAAC;gBACD,MAAM,GAAG,GAAG,IAA+B,CAAC;gBAE5C,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;gBAC/E,CAAC;gBAED,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU;gBACzB,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;oBAC5B,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;oBACvF,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,EAAE,EAAE,CAAC;wBAC7F,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;oBACtE,CAAC;oBACD,KAAK,GAAG,UAAU,CAAC;gBACrB,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;gBAEtD,IAAI,CAAC;oBACH,0DAA0D;oBAC1D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;wBACpD,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,UAAU,EAAE,KAAK;qBAClB,CAAC,CAAC;oBAEH,+DAA+D;oBAE/D,oDAAoD;oBACpD,MAAM,cAAc,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACzD,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,WAAW,EAAE,IAAI,CAAC,WAAW;wBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;qBAC1B,CAAC,CAAC,CAAC;oBAEJ,OAAO,CAAC,GAAG,CAAC,2CAA2C,cAAc,CAAC,MAAM,UAAU,CAAC,CAAC;oBAExF,6BAA6B;oBAC7B,IAAI,YAAY,GAAG,yBAAyB,GAAG,CAAC,KAAK,UAAU,cAAc,CAAC,MAAM,eAAe,CAAC;oBAEpG,cAAc,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;wBACpC,YAAY,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,KAAK,MAAM,CAAC;wBACnD,YAAY,IAAI,QAAQ,OAAO,CAAC,GAAG,IAAI,CAAC;wBACxC,YAAY,IAAI,gBAAgB,OAAO,CAAC,WAAW,IAAI,CAAC;wBACxD,YAAY,IAAI,WAAW,CAAC;oBAC9B,CAAC,CAAC,CAAC;oBAEH,OAAO;wBACL,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAe;gCACrB,IAAI,EAAE,YAAY;6BACnB;yBACF;qBACF,CAAC;gBACJ,CAAC;wBAAS,CAAC;oBACT,8DAA8D;oBAC9D,6EAA6E;oBAC7E,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;oBACrC,CAAC;oBAAC,OAAO,YAAY,EAAE,CAAC;wBACtB,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,YAAY,CAAC,CAAC;oBACrE,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,uDAAuD,EAAE,KAAK,CAAC,CAAC;gBAC9E,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CACF,CAAC;QAEF,mDAAmD;QACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,6BAA6B,EAC7B,+NAA+N,EAC/N;YACE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,iDAAiD,CAAC;YACjF,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;gBACpE,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9D,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6JAA6J,CAAC;SACtL,EACD,KAAK,EAAE,IAAa,EAAE,EAAE;YACtB,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;YACrE,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAEnE,IAAI,CAAC;gBACH,qBAAqB;gBACrB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC9C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAC/D,CAAC;gBACD,MAAM,GAAG,GAAG,IAA+B,CAAC;gBAE5C,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC5C,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;gBAED,IAAI,gBAAoC,CAAC;gBACzC,IAAI,GAAG,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;oBACvC,MAAM,cAAc,GAAG,OAAO,GAAG,CAAC,gBAAgB,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC;oBAC5H,IAAI,OAAO,cAAc,KAAK,QAAQ,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;wBACtF,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;oBAC7E,CAAC;oBACD,+DAA+D;oBAC/D,gBAAgB,GAAG,cAAc,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC;gBACvE,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,sDAAsD,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBAE7E,qDAAqD;gBACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC;oBACzD,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,gBAAgB;iBACjB,CAAC,CAAC;gBAEH,8CAA8C;gBAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;gBAEhD,wCAAwC;gBACxC,+GAA+G;gBAC/G,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;gBAE9E,OAAO,CAAC,GAAG,CAAC,6DAA6D,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;gBAEtG,4BAA4B;gBAC5B,IAAI,YAAY,GAAG,wBAAwB,GAAG,CAAC,GAAG,QAAQ,CAAC;gBAC3D,YAAY,IAAI,cAAc,KAAK,IAAI,CAAC;gBACxC,YAAY,IAAI,mBAAmB,SAAS,IAAI,CAAC;gBACjD,YAAY,IAAI,uBAAuB,OAAO,CAAC,MAAM,iBAAiB,CAAC;gBAEvE,IAAI,gBAAgB,IAAI,gBAAgB,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;oBAClF,YAAY,IAAI,2BAA2B,gBAAgB,oBAAoB,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,gBAAgB,CAAC,6BAA6B,gBAAgB,cAAc,CAAC;gBACnL,CAAC;qBAAM,CAAC;oBACN,YAAY,IAAI,iBAAiB,OAAO,EAAE,CAAC;gBAC7C,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,YAAY;yBACnB;qBACF;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,0DAA0D,EAAE,KAAK,CAAC,CAAC;gBACjF,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CACF,CAAC;IACJ,CAAC;IAEO,sBAAsB,CAAC,IAAa;QAC1C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,2BAA2B;QAC3B,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QAED,2CAA2C;QAC3C,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU;QACzB,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;YACvF,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,EAAE,EAAE,CAAC;gBAC7F,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACtE,CAAC;YACD,KAAK,GAAG,UAAU,CAAC;QACrB,CAAC;QAED,qDAAqD;QACrD,IAAI,cAAc,GAAG,IAAI,CAAC,CAAC,UAAU;QACrC,IAAI,GAAG,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;gBAC3C,cAAc,GAAG,GAAG,CAAC,cAAc,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,KAAK;YACL,cAAc;SACf,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAyB;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,cAAc,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;QAE1D,OAAO,CAAC,KAAK,CAAC,6DAA6D,KAAK,oBAAoB,cAAc,EAAE,CAAC,CAAC;QAEtH,IAAI,CAAC;YACH,uFAAuF;YACvF,sFAAsF;YACtF,MAAM,WAAW,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAEzE,OAAO,CAAC,GAAG,CAAC,sCAAsC,WAAW,0BAA0B,KAAK,0BAA0B,CAAC,CAAC;YAExH,qBAAqB;YACrB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;gBACpD,KAAK;gBACL,UAAU,EAAE,WAAW;aACxB,CAAC,CAAC;YACH,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,CAAC;YAE7C,qBAAqB;YACrB,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YAC7E,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,GAAG,QAAQ,CAAC;YACtD,OAAO,CAAC,KAAK,CAAC,0CAA0C,cAAc,CAAC,MAAM,KAAK,KAAK,cAAc,aAAa,CAAC,MAAM,mBAAmB,QAAQ,KAAK,aAAa,YAAY,CAAC,CAAC;YAEpL,mEAAmE;YACnE,MAAM,eAAe,GAAG,cAAc;gBACpC,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,aAAa,EAAE,KAAK,CAAC;gBAC5E,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,iEAAiE;YAEpG,2EAA2E;YAC3E,IAAI,cAAc,GAAG,kBAAkB,cAAc,CAAC,MAAM,KAAK,KAAK,qBAAqB,aAAa,CAAC,MAAM,mBAAmB,QAAQ,KAAK,aAAa,WAAW,CAAC;YAExK,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;gBACrF,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC;gBAC7E,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;gBAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;gBACpE,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAE7F,OAAO,CAAC,KAAK,CAAC,4CAA4C,KAAK,6BAA6B,YAAY,aAAa,WAAW,GAAG,iBAAiB,cAAc,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;gBAE7L,yCAAyC;gBACzC,cAAc,IAAI,6BAA6B,YAAY,aAAa,WAAW,cAAc,eAAe,CAAC,MAAM,EAAE,CAAC;YAC5H,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE1C,OAAO;gBACL,OAAO,EAAE,eAAe;gBACxB,aAAa,EAAE,eAAe,CAAC,MAAM;gBACrC,cAAc,EAAE,UAAU;gBAC1B,KAAK;gBACL,MAAM,EAAE,cAAc;aACvB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IAEO,wBAAwB,CAAC,aAA6B;QAC5D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE/C,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC7B,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAChE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAC5C,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,YAAoB;QAC1C,MAAM,UAAU,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;QAE9C,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACvE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnE,OAAO,eAAe,CAAC;QACzB,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnE,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC3G,OAAO,eAAe,CAAC;QACzB,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzH,OAAO,kBAAkB,CAAC;QAC5B,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnG,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/G,OAAO,eAAe,CAAC;QACzB,CAAC;QACD,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAClE,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,qBAAqB;QAC3B,sCAAsC;QACtC,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;YACnD,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACrE,oDAAoD;QACtD,CAAC,CAAC,CAAC;QAEH,6BAA6B;QAC7B,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE;YACxC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAC5C,mDAAmD;QACrD,CAAC,CAAC,CAAC;QAEH,wDAAwD;QACxD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC9B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC;oBAChB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE;oBAChC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAC/B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC;oBAChB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE;oBAChC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG;QACP,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;QAE7C,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAC7C,CAAC;CACF;AAED,mBAAmB;AACnB,MAAM,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;AACxC,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACpC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts new file mode 100644 index 0000000..a0808d3 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts @@ -0,0 +1,15 @@ +export declare class RateLimiter { + private limit; + private requestCount; + private lastResetTime; + private readonly maxRequestsPerMinute; + private readonly resetIntervalMs; + constructor(maxRequestsPerMinute?: number); + execute(fn: () => Promise): Promise; + getStatus(): { + requestCount: number; + maxRequests: number; + resetTime: number; + }; +} +//# sourceMappingURL=rate-limiter.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts.map new file mode 100644 index 0000000..5dc718f --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../src/rate-limiter.ts"],"names":[],"mappings":"AAEA,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAiB;gBAErC,oBAAoB,GAAE,MAAW;IAKvC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuBlD,SAAS,IAAI;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAO9E"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js new file mode 100644 index 0000000..e764829 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js @@ -0,0 +1,39 @@ +import pLimit from 'p-limit'; +export class RateLimiter { + limit; + requestCount = 0; + lastResetTime = Date.now(); + maxRequestsPerMinute; + resetIntervalMs = 60000; // 1 minute + constructor(maxRequestsPerMinute = 10) { + this.maxRequestsPerMinute = maxRequestsPerMinute; + this.limit = pLimit(5); // Max 5 concurrent requests + } + async execute(fn) { + // Check if we need to reset the counter + const now = Date.now(); + if (now - this.lastResetTime >= this.resetIntervalMs) { + this.requestCount = 0; + this.lastResetTime = now; + } + // Check rate limit + if (this.requestCount >= this.maxRequestsPerMinute) { + const waitTime = this.resetIntervalMs - (now - this.lastResetTime); + throw new Error(`Rate limit exceeded. Please wait ${Math.ceil(waitTime / 1000)} seconds.`); + } + // Execute with concurrency limit + const result = await this.limit(async () => { + this.requestCount++; + return await fn(); + }); + return result; + } + getStatus() { + return { + requestCount: this.requestCount, + maxRequests: this.maxRequestsPerMinute, + resetTime: this.lastResetTime + this.resetIntervalMs, + }; + } +} +//# sourceMappingURL=rate-limiter.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js.map b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js.map new file mode 100644 index 0000000..70128d9 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/rate-limiter.js.map @@ -0,0 +1 @@ +{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../src/rate-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,MAAM,OAAO,WAAW;IACd,KAAK,CAA4B;IACjC,YAAY,GAAW,CAAC,CAAC;IACzB,aAAa,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAC1B,oBAAoB,CAAS;IAC7B,eAAe,GAAW,KAAK,CAAC,CAAC,WAAW;IAE7D,YAAY,uBAA+B,EAAE;QAC3C,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,4BAA4B;IACtD,CAAC;IAED,KAAK,CAAC,OAAO,CAAI,EAAoB;QACnC,wCAAwC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACrD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;QAC3B,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7F,CAAC;QAED,iCAAiC;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;YACzC,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,SAAS;QACP,OAAO;YACL,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,WAAW,EAAE,IAAI,CAAC,oBAAoB;YACtC,SAAS,EAAE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,eAAe;SACrD,CAAC;IACJ,CAAC;CACF"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts new file mode 100644 index 0000000..f5e1933 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts @@ -0,0 +1,29 @@ +import { SearchOptions, SearchResultWithMetadata } from './types.js'; +export declare class SearchEngine { + private readonly rateLimiter; + private browserPool; + constructor(); + search(options: SearchOptions): Promise; + private tryBrowserBraveSearch; + private tryBrowserBraveSearchInternal; + private tryBrowserBingSearch; + private tryBrowserBingSearchInternal; + private tryEnhancedBingSearch; + private tryDirectBingSearch; + private generateConversationId; + private tryDuckDuckGoSearch; + private parseSearchResults; + private parseBraveResults; + private parseBingResults; + private parseDuckDuckGoResults; + private isValidSearchUrl; + private cleanGoogleUrl; + private cleanBraveUrl; + private cleanBingUrl; + private cleanDuckDuckGoUrl; + private assessResultQuality; + private validateBrowserHealth; + private handleBrowserError; + closeAll(): Promise; +} +//# sourceMappingURL=search-engine.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts.map new file mode 100644 index 0000000..333bd3e --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"search-engine.d.ts","sourceRoot":"","sources":["../src/search-engine.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAgB,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAKnF,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,WAAW,CAAc;;IAO3B,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,wBAAwB,CAAC;YAuGzD,qBAAqB;YA0CrB,6BAA6B;YAsD7B,oBAAoB;YA8DpB,4BAA4B;YAgF5B,qBAAqB;YAgFrB,mBAAmB;IAqDjC,OAAO,CAAC,sBAAsB;YAWhB,mBAAmB;IAiCjC,OAAO,CAAC,kBAAkB;IAkJ1B,OAAO,CAAC,iBAAiB;IAoGzB,OAAO,CAAC,gBAAgB;IAiHxB,OAAO,CAAC,sBAAsB;IAwC9B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,mBAAmB;YAwFb,qBAAqB;YAwBrB,kBAAkB;IAqB1B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGhC"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/search-engine.js b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.js new file mode 100644 index 0000000..5915201 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.js @@ -0,0 +1,1002 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; +import { generateTimestamp, sanitizeQuery } from './utils.js'; +import { RateLimiter } from './rate-limiter.js'; +import { BrowserPool } from './browser-pool.js'; +export class SearchEngine { + rateLimiter; + browserPool; + constructor() { + this.rateLimiter = new RateLimiter(10); // 10 requests per minute + this.browserPool = new BrowserPool(); + } + async search(options) { + const { query, numResults = 5, timeout = 10000 } = options; + const sanitizedQuery = sanitizeQuery(query); + console.log(`[SearchEngine] Starting search for query: "${sanitizedQuery}"`); + try { + return await this.rateLimiter.execute(async () => { + console.log(`[SearchEngine] Starting search with multiple engines...`); + // Configuration from environment variables + const enableQualityCheck = process.env.ENABLE_RELEVANCE_CHECKING !== 'false'; + const qualityThreshold = parseFloat(process.env.RELEVANCE_THRESHOLD || '0.3'); + const forceMultiEngine = process.env.FORCE_MULTI_ENGINE_SEARCH === 'true'; + const debugBrowsers = process.env.DEBUG_BROWSER_LIFECYCLE === 'true'; + console.log(`[SearchEngine] Quality checking: ${enableQualityCheck}, threshold: ${qualityThreshold}, multi-engine: ${forceMultiEngine}, debug: ${debugBrowsers}`); + // Try multiple approaches to get search results, starting with most reliable + const approaches = [ + { method: this.tryBrowserBingSearch.bind(this), name: 'Browser Bing' }, + { method: this.tryBrowserBraveSearch.bind(this), name: 'Browser Brave' }, + { method: this.tryDuckDuckGoSearch.bind(this), name: 'Axios DuckDuckGo' } + ]; + let bestResults = []; + let bestEngine = 'None'; + let bestQuality = 0; + for (let i = 0; i < approaches.length; i++) { + const approach = approaches[i]; + try { + console.log(`[SearchEngine] Attempting ${approach.name} (${i + 1}/${approaches.length})...`); + // Use more aggressive timeouts for faster fallback + const approachTimeout = Math.min(timeout / 3, 4000); // Max 4 seconds per approach for faster fallback + const results = await approach.method(sanitizedQuery, numResults, approachTimeout); + if (results.length > 0) { + console.log(`[SearchEngine] Found ${results.length} results with ${approach.name}`); + // Validate result quality to detect irrelevant results + const qualityScore = enableQualityCheck ? this.assessResultQuality(results, sanitizedQuery) : 1.0; + console.log(`[SearchEngine] ${approach.name} quality score: ${qualityScore.toFixed(2)}/1.0`); + // Track the best results so far + if (qualityScore > bestQuality) { + bestResults = results; + bestEngine = approach.name; + bestQuality = qualityScore; + } + // If quality is excellent, return immediately (unless forcing multi-engine) + if (qualityScore >= 0.8 && !forceMultiEngine) { + console.log(`[SearchEngine] Excellent quality results from ${approach.name}, returning immediately`); + return { results, engine: approach.name }; + } + // If quality is acceptable and this isn't Bing (first engine), return + if (qualityScore >= qualityThreshold && approach.name !== 'Browser Bing' && !forceMultiEngine) { + console.log(`[SearchEngine] Good quality results from ${approach.name}, using as primary`); + return { results, engine: approach.name }; + } + // If this is the last engine or quality is acceptable, prepare to return + if (i === approaches.length - 1) { + if (bestQuality >= qualityThreshold || !enableQualityCheck) { + console.log(`[SearchEngine] Using best results from ${bestEngine} (quality: ${bestQuality.toFixed(2)})`); + return { results: bestResults, engine: bestEngine }; + } + else if (bestResults.length > 0) { + console.log(`[SearchEngine] Warning: Low quality results from all engines, using best available from ${bestEngine}`); + return { results: bestResults, engine: bestEngine }; + } + } + else { + console.log(`[SearchEngine] ${approach.name} results quality: ${qualityScore.toFixed(2)}, continuing to try other engines...`); + } + } + } + catch (error) { + console.error(`[SearchEngine] ${approach.name} approach failed:`, error); + // Handle browser-specific errors (no cleanup needed since each engine uses dedicated browsers) + await this.handleBrowserError(error, approach.name); + } + } + console.log(`[SearchEngine] All approaches failed, returning empty results`); + return { results: [], engine: 'None' }; + }); + } + catch (error) { + console.error('[SearchEngine] Search error:', error); + if (axios.isAxiosError(error)) { + console.error('[SearchEngine] Axios error details:', { + status: error.response?.status, + statusText: error.response?.statusText, + data: error.response?.data?.substring(0, 500), + }); + } + throw new Error(`Failed to perform search: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + async tryBrowserBraveSearch(query, numResults, timeout) { + console.log(`[SearchEngine] Trying browser-based Brave search with dedicated browser...`); + // Try with retry mechanism + for (let attempt = 1; attempt <= 2; attempt++) { + let browser; + try { + // Create a dedicated browser instance for Brave search only + const { firefox } = await import('playwright'); + browser = await firefox.launch({ + headless: process.env.BROWSER_HEADLESS !== 'false', + args: [ + '--no-sandbox', + '--disable-dev-shm-usage', + ], + }); + console.log(`[SearchEngine] Brave search attempt ${attempt}/2 with fresh browser`); + const results = await this.tryBrowserBraveSearchInternal(browser, query, numResults, timeout); + return results; + } + catch (error) { + console.error(`[SearchEngine] Brave search attempt ${attempt}/2 failed:`, error); + if (attempt === 2) { + throw error; // Re-throw on final attempt + } + // Small delay before retry + await new Promise(resolve => setTimeout(resolve, 500)); + } + finally { + // Always close the dedicated browser + if (browser) { + try { + await browser.close(); + } + catch (closeError) { + console.log(`[SearchEngine] Error closing Brave browser:`, closeError); + } + } + } + } + throw new Error('All Brave search attempts failed'); + } + async tryBrowserBraveSearchInternal(browser, query, numResults, timeout) { + // Validate browser is still functional before proceeding + if (!browser.isConnected()) { + throw new Error('Browser is not connected'); + } + try { + const context = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + viewport: { width: 1366, height: 768 }, + locale: 'en-US', + timezoneId: 'America/New_York', + }); + try { + const page = await context.newPage(); + // Navigate to Brave search + const searchUrl = `https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web`; + console.log(`[SearchEngine] Browser navigating to Brave: ${searchUrl}`); + await page.goto(searchUrl, { + waitUntil: 'domcontentloaded', + timeout: timeout + }); + // Wait for search results to load + try { + await page.waitForSelector('[data-type="web"]', { timeout: 3000 }); + } + catch { + console.log(`[SearchEngine] Browser Brave results selector not found, proceeding anyway`); + } + // Get the page content + const html = await page.content(); + console.log(`[SearchEngine] Browser Brave got HTML with length: ${html.length}`); + const results = this.parseBraveResults(html, numResults); + console.log(`[SearchEngine] Browser Brave parsed ${results.length} results`); + await context.close(); + return results; + } + catch (error) { + // Ensure context is closed even on error + await context.close(); + throw error; + } + } + catch (error) { + console.error(`[SearchEngine] Browser Brave search failed:`, error); + throw error; + } + } + async tryBrowserBingSearch(query, numResults, timeout) { + const debugBing = process.env.DEBUG_BING_SEARCH === 'true'; + console.error(`[SearchEngine] BING: Starting browser-based search with dedicated browser for query: "${query}"`); + // Try with retry mechanism + for (let attempt = 1; attempt <= 2; attempt++) { + let browser; + try { + console.error(`[SearchEngine] BING: Attempt ${attempt}/2 - Launching Chromium browser...`); + // Create a dedicated browser instance for Bing search only + const { chromium } = await import('playwright'); + const startTime = Date.now(); + browser = await chromium.launch({ + headless: process.env.BROWSER_HEADLESS !== 'false', + args: [ + '--no-sandbox', + '--disable-blink-features=AutomationControlled', + '--disable-dev-shm-usage', + '--disable-gpu', + ], + }); + const launchTime = Date.now() - startTime; + console.error(`[SearchEngine] BING: Browser launched successfully in ${launchTime}ms, connected: ${browser.isConnected()}`); + const results = await this.tryBrowserBingSearchInternal(browser, query, numResults, timeout); + console.error(`[SearchEngine] BING: Search completed successfully with ${results.length} results`); + return results; + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error(`[SearchEngine] BING: Attempt ${attempt}/2 FAILED with error: ${errorMessage}`); + if (debugBing) { + console.error(`[SearchEngine] BING: Full error details:`, error); + } + if (attempt === 2) { + console.error(`[SearchEngine] BING: All attempts exhausted, giving up`); + throw error; // Re-throw on final attempt + } + // Small delay before retry + console.error(`[SearchEngine] BING: Waiting 500ms before retry...`); + await new Promise(resolve => setTimeout(resolve, 500)); + } + finally { + // Always close the dedicated browser + if (browser) { + try { + await browser.close(); + if (debugBing) { + console.error(`[SearchEngine] BING: Browser closed successfully`); + } + } + catch (closeError) { + console.error(`[SearchEngine] BING: Error closing browser:`, closeError); + } + } + } + } + throw new Error('All Bing search attempts failed'); + } + async tryBrowserBingSearchInternal(browser, query, numResults, timeout) { + const debugBing = process.env.DEBUG_BING_SEARCH === 'true'; + // Validate browser is still functional before proceeding + if (!browser.isConnected()) { + console.error(`[SearchEngine] BING: Browser is not connected`); + throw new Error('Browser is not connected'); + } + console.error(`[SearchEngine] BING: Creating browser context with enhanced fingerprinting...`); + try { + // Enhanced browser context with more realistic fingerprinting + const context = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + viewport: { width: 1366, height: 768 }, + locale: 'en-US', + timezoneId: 'America/New_York', + colorScheme: 'light', + deviceScaleFactor: 1, + hasTouch: false, + isMobile: false, + extraHTTPHeaders: { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'DNT': '1', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none' + } + }); + console.error(`[SearchEngine] BING: Context created, opening new page...`); + const page = await context.newPage(); + console.error(`[SearchEngine] BING: Page opened successfully`); + try { + // Try enhanced Bing search with proper web interface flow + try { + console.error(`[SearchEngine] BING: Attempting enhanced search (homepage → form submission)...`); + const results = await this.tryEnhancedBingSearch(page, query, numResults, timeout); + console.error(`[SearchEngine] BING: Enhanced search succeeded with ${results.length} results`); + await context.close(); + return results; + } + catch (enhancedError) { + const errorMessage = enhancedError instanceof Error ? enhancedError.message : 'Unknown error'; + console.error(`[SearchEngine] BING: Enhanced search failed: ${errorMessage}`); + if (debugBing) { + console.error(`[SearchEngine] BING: Enhanced search error details:`, enhancedError); + } + console.error(`[SearchEngine] BING: Falling back to direct URL search...`); + // Fallback to direct URL approach with enhanced parameters + const results = await this.tryDirectBingSearch(page, query, numResults, timeout); + console.error(`[SearchEngine] BING: Direct search succeeded with ${results.length} results`); + await context.close(); + return results; + } + } + catch (error) { + // Ensure context is closed even on error + console.error(`[SearchEngine] BING: All search methods failed, closing context...`); + await context.close(); + throw error; + } + } + catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error(`[SearchEngine] BING: Internal search failed: ${errorMessage}`); + if (debugBing) { + console.error(`[SearchEngine] BING: Internal search error details:`, error); + } + throw error; + } + } + async tryEnhancedBingSearch(page, query, numResults, timeout) { + const debugBing = process.env.DEBUG_BING_SEARCH === 'true'; + console.error(`[SearchEngine] BING: Enhanced search - navigating to Bing homepage...`); + // Navigate to Bing homepage first to establish proper session + const startTime = Date.now(); + await page.goto('https://www.bing.com', { + waitUntil: 'domcontentloaded', + timeout: timeout / 2 + }); + const loadTime = Date.now() - startTime; + const pageTitle = await page.title(); + const currentUrl = page.url(); + console.error(`[SearchEngine] BING: Homepage loaded in ${loadTime}ms, title: "${pageTitle}", URL: ${currentUrl}`); + // Wait a moment for page to fully load + await page.waitForTimeout(500); + // Find and use the search box (more realistic than direct URL) + try { + console.error(`[SearchEngine] BING: Looking for search form elements...`); + await page.waitForSelector('#sb_form_q', { timeout: 2000 }); + console.error(`[SearchEngine] BING: Search box found, filling with query: "${query}"`); + await page.fill('#sb_form_q', query); + console.error(`[SearchEngine] BING: Clicking search button and waiting for navigation...`); + // Submit the search form + await Promise.all([ + page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }), + page.click('#search_icon') + ]); + const searchLoadTime = Date.now() - startTime; + const searchPageTitle = await page.title(); + const searchPageUrl = page.url(); + console.error(`[SearchEngine] BING: Search completed in ${searchLoadTime}ms total, title: "${searchPageTitle}", URL: ${searchPageUrl}`); + } + catch (formError) { + const errorMessage = formError instanceof Error ? formError.message : 'Unknown error'; + console.error(`[SearchEngine] BING: Search form submission failed: ${errorMessage}`); + if (debugBing) { + console.error(`[SearchEngine] BING: Form error details:`, formError); + } + throw formError; + } + // Wait for search results to load + try { + console.error(`[SearchEngine] BING: Waiting for search results to appear...`); + await page.waitForSelector('.b_algo, .b_result', { timeout: 3000 }); + console.error(`[SearchEngine] BING: Search results selector found`); + } + catch { + console.error(`[SearchEngine] BING: Search results selector not found, proceeding with page content anyway`); + } + const html = await page.content(); + console.error(`[SearchEngine] BING: Got page HTML with length: ${html.length} characters`); + if (debugBing && html.length < 10000) { + console.error(`[SearchEngine] BING: WARNING - HTML seems short, possible bot detection or error page`); + } + const results = this.parseBingResults(html, numResults); + console.error(`[SearchEngine] BING: Enhanced search parsed ${results.length} results`); + if (results.length === 0) { + console.error(`[SearchEngine] BING: WARNING - No results found, possible parsing failure or empty search`); + if (debugBing) { + const sampleHtml = html.substring(0, 1000); + console.error(`[SearchEngine] BING: Sample HTML for debugging:`, sampleHtml); + } + } + return results; + } + async tryDirectBingSearch(page, query, numResults, timeout) { + const debugBing = process.env.DEBUG_BING_SEARCH === 'true'; + console.error(`[SearchEngine] BING: Direct search with enhanced parameters...`); + // Generate a conversation ID (cvid) similar to what Bing uses + const cvid = this.generateConversationId(); + // Construct URL with enhanced parameters based on successful manual searches + const searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&count=${Math.min(numResults, 10)}&form=QBLH&sp=-1&qs=n&cvid=${cvid}`; + console.error(`[SearchEngine] BING: Navigating to direct URL: ${searchUrl}`); + const startTime = Date.now(); + await page.goto(searchUrl, { + waitUntil: 'domcontentloaded', + timeout: timeout + }); + const loadTime = Date.now() - startTime; + const pageTitle = await page.title(); + const currentUrl = page.url(); + console.error(`[SearchEngine] BING: Direct page loaded in ${loadTime}ms, title: "${pageTitle}", URL: ${currentUrl}`); + // Wait for search results to load + try { + console.error(`[SearchEngine] BING: Waiting for search results to appear...`); + await page.waitForSelector('.b_algo, .b_result', { timeout: 3000 }); + console.error(`[SearchEngine] BING: Search results selector found`); + } + catch { + console.error(`[SearchEngine] BING: Search results selector not found, proceeding with page content anyway`); + } + const html = await page.content(); + console.error(`[SearchEngine] BING: Got page HTML with length: ${html.length} characters`); + if (debugBing && html.length < 10000) { + console.error(`[SearchEngine] BING: WARNING - HTML seems short, possible bot detection or error page`); + } + const results = this.parseBingResults(html, numResults); + console.error(`[SearchEngine] BING: Direct search parsed ${results.length} results`); + if (results.length === 0) { + console.error(`[SearchEngine] BING: WARNING - No results found, possible parsing failure or empty search`); + if (debugBing) { + const sampleHtml = html.substring(0, 1000); + console.error(`[SearchEngine] BING: Sample HTML for debugging:`, sampleHtml); + } + } + return results; + } + generateConversationId() { + // Generate a conversation ID similar to Bing's format (32 hex characters) + const chars = '0123456789ABCDEF'; + let cvid = ''; + for (let i = 0; i < 32; i++) { + cvid += chars[Math.floor(Math.random() * chars.length)]; + } + return cvid; + } + async tryDuckDuckGoSearch(query, numResults, timeout) { + console.log(`[SearchEngine] Trying DuckDuckGo as fallback...`); + try { + const response = await axios.get('https://html.duckduckgo.com/html/', { + params: { + q: query, + }, + headers: { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + }, + timeout, + validateStatus: (status) => status < 400, + }); + console.log(`[SearchEngine] DuckDuckGo got response with status: ${response.status}`); + const results = this.parseDuckDuckGoResults(response.data, numResults); + console.log(`[SearchEngine] DuckDuckGo parsed ${results.length} results`); + return results; + } + catch { + console.error(`[SearchEngine] DuckDuckGo search failed`); + throw new Error('DuckDuckGo search failed'); + } + } + parseSearchResults(html, maxResults) { + console.log(`[SearchEngine] Parsing HTML with length: ${html.length}`); + const $ = cheerio.load(html); + const results = []; + const timestamp = generateTimestamp(); + // Log what selectors we find - more comprehensive debugging + const gElements = $('div.g'); + const sokobanElements = $('div[data-sokoban-container]'); + const tF2CxcElements = $('.tF2Cxc'); + const rcElements = $('.rc'); + const vedElements = $('[data-ved]'); + const h3Elements = $('h3'); + const linkElements = $('a[href]'); + console.log(`[SearchEngine] Found elements:`); + console.log(` - div.g: ${gElements.length}`); + console.log(` - div[data-sokoban-container]: ${sokobanElements.length}`); + console.log(` - .tF2Cxc: ${tF2CxcElements.length}`); + console.log(` - .rc: ${rcElements.length}`); + console.log(` - [data-ved]: ${vedElements.length}`); + console.log(` - h3: ${h3Elements.length}`); + console.log(` - a[href]: ${linkElements.length}`); + // Try multiple approaches to find search results + const searchResultSelectors = [ + 'div.g', + 'div[data-sokoban-container]', + '.tF2Cxc', + '.rc', + '[data-ved]', + 'div[jscontroller]' + ]; + let foundResults = false; + for (const selector of searchResultSelectors) { + if (foundResults) + break; + console.log(`[SearchEngine] Trying selector: ${selector}`); + const elements = $(selector); + console.log(`[SearchEngine] Found ${elements.length} elements with selector ${selector}`); + elements.each((_index, element) => { + if (results.length >= maxResults) + return false; + const $element = $(element); + // Try multiple title selectors + const titleSelectors = ['h3', '.LC20lb', '.DKV0Md', 'a[data-ved]', '.r', '.s']; + let title = ''; + let url = ''; + for (const titleSelector of titleSelectors) { + const $title = $element.find(titleSelector).first(); + if ($title.length) { + title = $title.text().trim(); + console.log(`[SearchEngine] Found title with ${titleSelector}: "${title}"`); + // Try to find the link + const $link = $title.closest('a'); + if ($link.length) { + url = $link.attr('href') || ''; + console.log(`[SearchEngine] Found URL: "${url}"`); + } + else { + // Try to find any link in the element + const $anyLink = $element.find('a[href]').first(); + if ($anyLink.length) { + url = $anyLink.attr('href') || ''; + console.log(`[SearchEngine] Found URL from any link: "${url}"`); + } + } + break; + } + } + // Try multiple snippet selectors + const snippetSelectors = ['.VwiC3b', '.st', '.aCOpRe', '.IsZvec', '.s3v9rd', '.MUxGbd', '.aCOpRe', '.snippet-content']; + let snippet = ''; + for (const snippetSelector of snippetSelectors) { + const $snippet = $element.find(snippetSelector).first(); + if ($snippet.length) { + snippet = $snippet.text().trim(); + console.log(`[SearchEngine] Found snippet with ${snippetSelector}: "${snippet.substring(0, 100)}..."`); + break; + } + } + if (title && url && this.isValidSearchUrl(url)) { + console.log(`[SearchEngine] Adding result: ${title}`); + results.push({ + title, + url: this.cleanGoogleUrl(url), + description: snippet || 'No description available', + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp, + fetchStatus: 'success', + }); + foundResults = true; + } + else { + console.log(`[SearchEngine] Skipping result: title="${title}", url="${url}", isValid=${this.isValidSearchUrl(url)}`); + } + }); + } + console.log(`[SearchEngine] Found ${results.length} results with all selectors`); + // If still no results, try a more aggressive approach - look for any h3 with links + if (results.length === 0) { + console.log(`[SearchEngine] No results found, trying aggressive h3 search...`); + $('h3').each((_index, element) => { + if (results.length >= maxResults) + return false; + const $h3 = $(element); + const title = $h3.text().trim(); + const $link = $h3.closest('a'); + if ($link.length && title) { + const url = $link.attr('href') || ''; + console.log(`[SearchEngine] Aggressive search found: "${title}" -> "${url}"`); + if (this.isValidSearchUrl(url)) { + results.push({ + title, + url: this.cleanGoogleUrl(url), + description: 'No description available', + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp, + fetchStatus: 'success', + }); + } + } + }); + console.log(`[SearchEngine] Aggressive search found ${results.length} results`); + } + return results; + } + parseBraveResults(html, maxResults) { + console.log(`[SearchEngine] Parsing Brave HTML with length: ${html.length}`); + const $ = cheerio.load(html); + const results = []; + const timestamp = generateTimestamp(); + // Brave result selectors + const resultSelectors = [ + '[data-type="web"]', // Main Brave results + '.result', // Alternative format + '.fdb' // Brave specific format + ]; + let foundResults = false; + for (const selector of resultSelectors) { + if (foundResults && results.length >= maxResults) + break; + console.log(`[SearchEngine] Trying Brave selector: ${selector}`); + const elements = $(selector); + console.log(`[SearchEngine] Found ${elements.length} elements with selector ${selector}`); + elements.each((_index, element) => { + if (results.length >= maxResults) + return false; + const $element = $(element); + // Try multiple title selectors for Brave + const titleSelectors = [ + '.title a', // Brave specific + 'h2 a', // Common format + '.result-title a', // Alternative format + 'a[href*="://"]', // Any external link + '.snippet-title a' // Snippet title + ]; + let title = ''; + let url = ''; + for (const titleSelector of titleSelectors) { + const $titleElement = $element.find(titleSelector).first(); + if ($titleElement.length) { + title = $titleElement.text().trim(); + url = $titleElement.attr('href') || ''; + console.log(`[SearchEngine] Brave found title with ${titleSelector}: "${title}"`); + if (title && url && url.startsWith('http')) { + break; + } + } + } + // If still no title, try getting it from any text content + if (!title) { + const textContent = $element.text().trim(); + const lines = textContent.split('\n').filter(line => line.trim().length > 0); + if (lines.length > 0) { + title = lines[0].trim(); + console.log(`[SearchEngine] Brave found title from text content: "${title}"`); + } + } + // Try multiple snippet selectors for Brave + const snippetSelectors = [ + '.snippet-content', // Brave specific + '.snippet', // Generic + '.description', // Alternative + 'p' // Fallback paragraph + ]; + let snippet = ''; + for (const snippetSelector of snippetSelectors) { + const $snippetElement = $element.find(snippetSelector).first(); + if ($snippetElement.length) { + snippet = $snippetElement.text().trim(); + break; + } + } + if (title && url && this.isValidSearchUrl(url)) { + console.log(`[SearchEngine] Brave found: "${title}" -> "${url}"`); + results.push({ + title, + url: this.cleanBraveUrl(url), + description: snippet || 'No description available', + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp, + fetchStatus: 'success', + }); + foundResults = true; + } + }); + } + console.log(`[SearchEngine] Brave found ${results.length} results`); + return results; + } + parseBingResults(html, maxResults) { + const debugBing = process.env.DEBUG_BING_SEARCH === 'true'; + console.error(`[SearchEngine] BING: Parsing HTML with length: ${html.length}`); + const $ = cheerio.load(html); + const results = []; + const timestamp = generateTimestamp(); + // Check for common Bing error indicators + const pageTitle = $('title').text(); + console.error(`[SearchEngine] BING: Page title: "${pageTitle}"`); + if (pageTitle.includes('Access Denied') || pageTitle.includes('blocked') || pageTitle.includes('captcha')) { + console.error(`[SearchEngine] BING: ERROR - Bot detection or access denied detected in page title`); + } + // Bing result selectors + const resultSelectors = [ + '.b_algo', // Main Bing results + '.b_result', // Alternative Bing format + '.b_card' // Card format + ]; + console.error(`[SearchEngine] BING: Checking for result elements...`); + // Log counts for all selectors first + for (const selector of resultSelectors) { + const elements = $(selector); + console.error(`[SearchEngine] BING: Found ${elements.length} elements with selector "${selector}"`); + } + let foundResults = false; + for (const selector of resultSelectors) { + if (foundResults && results.length >= maxResults) + break; + const elements = $(selector); + if (elements.length === 0) + continue; + elements.each((_index, element) => { + if (results.length >= maxResults) + return false; + const $element = $(element); + // Try multiple title selectors for Bing + const titleSelectors = [ + 'h2 a', // Standard Bing format + '.b_title a', // Alternative format + 'a[data-seid]' // Bing specific + ]; + let title = ''; + let url = ''; + for (const titleSelector of titleSelectors) { + const $titleElement = $element.find(titleSelector).first(); + if ($titleElement.length) { + title = $titleElement.text().trim(); + url = $titleElement.attr('href') || ''; + console.log(`[SearchEngine] Bing found title with ${titleSelector}: "${title}"`); + break; + } + } + // Try multiple snippet selectors for Bing + const snippetSelectors = [ + '.b_caption p', // Standard Bing snippet + '.b_snippet', // Alternative format + '.b_descript', // Description format + '.b_caption', // Caption without p tag + '.b_caption > span', // Caption span + '.b_excerpt', // Excerpt format + 'p', // Any paragraph in the result + '.b_algo_content p', // Content paragraph + '.b_algo_content', // Full content area + '.b_context' // Context information + ]; + let snippet = ''; + for (const snippetSelector of snippetSelectors) { + const $snippetElement = $element.find(snippetSelector).first(); + if ($snippetElement.length) { + const candidateSnippet = $snippetElement.text().trim(); + // Skip very short snippets or those that look like metadata + if (candidateSnippet.length > 20 && !candidateSnippet.match(/^\d+\s*(min|sec|hour|day|week|month|year)/i)) { + snippet = candidateSnippet; + console.log(`[SearchEngine] Bing found snippet with ${snippetSelector}: "${snippet.substring(0, 100)}..."`); + break; + } + } + } + if (title && url && this.isValidSearchUrl(url)) { + console.log(`[SearchEngine] Bing found: "${title}" -> "${url}"`); + results.push({ + title, + url: this.cleanBingUrl(url), + description: snippet || 'No description available', + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp, + fetchStatus: 'success', + }); + foundResults = true; + } + }); + } + console.log(`[SearchEngine] Bing found ${results.length} results`); + return results; + } + parseDuckDuckGoResults(html, maxResults) { + console.log(`[SearchEngine] Parsing DuckDuckGo HTML with length: ${html.length}`); + const $ = cheerio.load(html); + const results = []; + const timestamp = generateTimestamp(); + // DuckDuckGo results are in .result elements + $('.result').each((_index, element) => { + if (results.length >= maxResults) + return false; + const $element = $(element); + // Extract title and URL + const $titleElement = $element.find('.result__title a'); + const title = $titleElement.text().trim(); + const url = $titleElement.attr('href'); + // Extract snippet + const snippet = $element.find('.result__snippet').text().trim(); + if (title && url) { + console.log(`[SearchEngine] DuckDuckGo found: "${title}" -> "${url}"`); + results.push({ + title, + url: this.cleanDuckDuckGoUrl(url), + description: snippet || 'No description available', + fullContent: '', + contentPreview: '', + wordCount: 0, + timestamp, + fetchStatus: 'success', + }); + } + }); + console.log(`[SearchEngine] DuckDuckGo found ${results.length} results`); + return results; + } + isValidSearchUrl(url) { + // Google search results URLs can be in various formats + return url.startsWith('/url?') || + url.startsWith('http://') || + url.startsWith('https://') || + url.startsWith('//') || + url.startsWith('/search?') || + url.startsWith('/') || + url.includes('google.com') || + url.length > 10; // Accept any reasonably long URL + } + cleanGoogleUrl(url) { + // Handle Google's redirect URLs + if (url.startsWith('/url?')) { + try { + const urlParams = new URLSearchParams(url.substring(5)); + const actualUrl = urlParams.get('q') || urlParams.get('url'); + if (actualUrl) { + return actualUrl; + } + } + catch { + console.warn('Failed to parse Google redirect URL:', url); + } + } + // Handle protocol-relative URLs + if (url.startsWith('//')) { + return 'https:' + url; + } + return url; + } + cleanBraveUrl(url) { + // Brave URLs are usually direct, but check for any redirect patterns + if (url.startsWith('//')) { + return 'https:' + url; + } + // If it's already a full URL, return as-is + if (url.startsWith('http://') || url.startsWith('https://')) { + return url; + } + return url; + } + cleanBingUrl(url) { + // Bing URLs are usually direct, but check for any redirect patterns + if (url.startsWith('//')) { + return 'https:' + url; + } + // If it's already a full URL, return as-is + if (url.startsWith('http://') || url.startsWith('https://')) { + return url; + } + return url; + } + cleanDuckDuckGoUrl(url) { + // DuckDuckGo URLs are redirect URLs that need to be decoded + if (url.startsWith('//duckduckgo.com/l/')) { + try { + // Extract the uddg parameter which contains the actual URL + const urlParams = new URLSearchParams(url.substring(url.indexOf('?') + 1)); + const actualUrl = urlParams.get('uddg'); + if (actualUrl) { + // Decode the URL + const decodedUrl = decodeURIComponent(actualUrl); + console.log(`[SearchEngine] Decoded DuckDuckGo URL: ${decodedUrl}`); + return decodedUrl; + } + } + catch { + console.log(`[SearchEngine] Failed to decode DuckDuckGo URL: ${url}`); + } + } + // If it's a protocol-relative URL, add https: + if (url.startsWith('//')) { + return 'https:' + url; + } + return url; + } + assessResultQuality(results, originalQuery) { + if (results.length === 0) + return 0; + // Extract keywords from the original query (ignore common words) + const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'can', 'group', 'members']); + const queryWords = originalQuery.toLowerCase() + .replace(/[^\w\s]/g, ' ') + .split(/\s+/) + .filter(word => word.length > 2 && !commonWords.has(word)); + if (queryWords.length === 0) + return 0.5; // Default score if no meaningful keywords + console.log(`[SearchEngine] Quality assessment - Query keywords: [${queryWords.join(', ')}]`); + let totalScore = 0; + let scoredResults = 0; + for (const result of results) { + const titleText = result.title.toLowerCase(); + const descText = result.description.toLowerCase(); + const urlText = result.url.toLowerCase(); + const combinedText = `${titleText} ${descText} ${urlText}`; + // Count keyword matches + let keywordMatches = 0; + let phraseMatches = 0; + // Check for exact phrase matches (higher value) + if (queryWords.length >= 2) { + const queryPhrases = []; + for (let i = 0; i < queryWords.length - 1; i++) { + queryPhrases.push(queryWords.slice(i, i + 2).join(' ')); + } + if (queryWords.length >= 3) { + queryPhrases.push(queryWords.slice(0, 3).join(' ')); + } + for (const phrase of queryPhrases) { + if (combinedText.includes(phrase)) { + phraseMatches++; + } + } + } + // Check individual keyword matches + for (const keyword of queryWords) { + if (combinedText.includes(keyword)) { + keywordMatches++; + } + } + // Calculate score for this result + const keywordRatio = keywordMatches / queryWords.length; + const phraseBonus = phraseMatches * 0.3; // Bonus for phrase matches + const resultScore = Math.min(1.0, keywordRatio + phraseBonus); + // Penalty for obvious irrelevant content + const irrelevantPatterns = [ + /recipe/i, /cooking/i, /food/i, /restaurant/i, /menu/i, + /weather/i, /temperature/i, /forecast/i, + /shopping/i, /sale/i, /price/i, /buy/i, /store/i, + /movie/i, /film/i, /tv show/i, /entertainment/i, + /sports/i, /game/i, /score/i, /team/i, + /fashion/i, /clothing/i, /style/i, + /travel/i, /hotel/i, /flight/i, /vacation/i, + /car/i, /vehicle/i, /automotive/i, + /real estate/i, /property/i, /house/i, /apartment/i + ]; + let penalty = 0; + for (const pattern of irrelevantPatterns) { + if (pattern.test(combinedText)) { + penalty += 0.2; + } + } + const finalScore = Math.max(0, resultScore - penalty); + console.log(`[SearchEngine] Result "${result.title.substring(0, 50)}..." - Score: ${finalScore.toFixed(2)} (keywords: ${keywordMatches}/${queryWords.length}, phrases: ${phraseMatches}, penalty: ${penalty.toFixed(2)})`); + totalScore += finalScore; + scoredResults++; + } + const averageScore = scoredResults > 0 ? totalScore / scoredResults : 0; + return averageScore; + } + async validateBrowserHealth(browser) { + const debugBrowsers = process.env.DEBUG_BROWSER_LIFECYCLE === 'true'; + try { + if (debugBrowsers) + console.log(`[SearchEngine] Validating browser health...`); + // Check if browser is still connected + if (!browser.isConnected()) { + if (debugBrowsers) + console.log(`[SearchEngine] Browser is not connected`); + return false; + } + // Try to create a simple context to test browser responsiveness + const testContext = await browser.newContext(); + await testContext.close(); + if (debugBrowsers) + console.log(`[SearchEngine] Browser health check passed`); + return true; + } + catch (error) { + console.log(`[SearchEngine] Browser health check failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + return false; + } + } + async handleBrowserError(error, engineName, attemptNumber = 1) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error(`[SearchEngine] ${engineName} browser error (attempt ${attemptNumber}): ${errorMessage}`); + // Check for specific browser-related errors + if (errorMessage.includes('Target page, context or browser has been closed') || + errorMessage.includes('Browser has been closed') || + errorMessage.includes('Session has been closed')) { + console.log(`[SearchEngine] Detected browser session closure, attempting to refresh browser pool`); + // Try to refresh the browser pool for subsequent attempts + try { + await this.browserPool.closeAll(); + console.log(`[SearchEngine] Browser pool refreshed for ${engineName}`); + } + catch (refreshError) { + console.error(`[SearchEngine] Failed to refresh browser pool: ${refreshError instanceof Error ? refreshError.message : 'Unknown error'}`); + } + } + } + async closeAll() { + await this.browserPool.closeAll(); + } +} +//# sourceMappingURL=search-engine.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/search-engine.js.map b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.js.map new file mode 100644 index 0000000..26532bf --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/search-engine.js.map @@ -0,0 +1 @@ +{"version":3,"file":"search-engine.js","sourceRoot":"","sources":["../src/search-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IAClC,WAAW,CAAc;IAEjC;QACE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,yBAAyB;QACjE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,CAAC,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAC3D,MAAM,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAE5C,OAAO,CAAC,GAAG,CAAC,8CAA8C,cAAc,GAAG,CAAC,CAAC;QAE7E,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBAC/C,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;gBAEvE,2CAA2C;gBAC3C,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,OAAO,CAAC;gBAC7E,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,KAAK,CAAC,CAAC;gBAC9E,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,MAAM,CAAC;gBAC1E,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,MAAM,CAAC;gBAErE,OAAO,CAAC,GAAG,CAAC,oCAAoC,kBAAkB,gBAAgB,gBAAgB,mBAAmB,gBAAgB,YAAY,aAAa,EAAE,CAAC,CAAC;gBAElK,6EAA6E;gBAC7E,MAAM,UAAU,GAAG;oBACjB,EAAE,MAAM,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE;oBACtE,EAAE,MAAM,EAAE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE;oBACxE,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE;iBAC1E,CAAC;gBAEF,IAAI,WAAW,GAAmB,EAAE,CAAC;gBACrC,IAAI,UAAU,GAAG,MAAM,CAAC;gBACxB,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAC/B,IAAI,CAAC;wBACH,OAAO,CAAC,GAAG,CAAC,6BAA6B,QAAQ,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,MAAM,CAAC,CAAC;wBAE7F,mDAAmD;wBACnD,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,iDAAiD;wBACtG,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;wBACnF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACvB,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,MAAM,iBAAiB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;4BAEpF,uDAAuD;4BACvD,MAAM,YAAY,GAAG,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;4BAClG,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,IAAI,mBAAmB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;4BAE7F,gCAAgC;4BAChC,IAAI,YAAY,GAAG,WAAW,EAAE,CAAC;gCAC/B,WAAW,GAAG,OAAO,CAAC;gCACtB,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;gCAC3B,WAAW,GAAG,YAAY,CAAC;4BAC7B,CAAC;4BAED,4EAA4E;4BAC5E,IAAI,YAAY,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gCAC7C,OAAO,CAAC,GAAG,CAAC,iDAAiD,QAAQ,CAAC,IAAI,yBAAyB,CAAC,CAAC;gCACrG,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;4BAC5C,CAAC;4BAED,sEAAsE;4BACtE,IAAI,YAAY,IAAI,gBAAgB,IAAI,QAAQ,CAAC,IAAI,KAAK,cAAc,IAAI,CAAC,gBAAgB,EAAE,CAAC;gCAC9F,OAAO,CAAC,GAAG,CAAC,4CAA4C,QAAQ,CAAC,IAAI,oBAAoB,CAAC,CAAC;gCAC3F,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;4BAC5C,CAAC;4BAED,yEAAyE;4BACzE,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gCAChC,IAAI,WAAW,IAAI,gBAAgB,IAAI,CAAC,kBAAkB,EAAE,CAAC;oCAC3D,OAAO,CAAC,GAAG,CAAC,0CAA0C,UAAU,cAAc,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oCACzG,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;gCACtD,CAAC;qCAAM,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCAClC,OAAO,CAAC,GAAG,CAAC,2FAA2F,UAAU,EAAE,CAAC,CAAC;oCACrH,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;gCACtD,CAAC;4BACH,CAAC;iCAAM,CAAC;gCACN,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,IAAI,qBAAqB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC;4BACjI,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,QAAQ,CAAC,IAAI,mBAAmB,EAAE,KAAK,CAAC,CAAC;wBAEzE,+FAA+F;wBAC/F,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;oBACtD,CAAC;gBACH,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;gBAC7E,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;YACrD,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE;oBACnD,MAAM,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM;oBAC9B,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU;oBACtC,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;iBAC9C,CAAC,CAAC;YACL,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,qBAAqB,CAAC,KAAa,EAAE,UAAkB,EAAE,OAAe;QACpF,OAAO,CAAC,GAAG,CAAC,4EAA4E,CAAC,CAAC;QAE1F,2BAA2B;QAC3B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9C,IAAI,OAAO,CAAC;YACZ,IAAI,CAAC;gBACH,4DAA4D;gBAC5D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC/C,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC;oBAC7B,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;oBAClD,IAAI,EAAE;wBACJ,cAAc;wBACd,yBAAyB;qBAC1B;iBACF,CAAC,CAAC;gBAEH,OAAO,CAAC,GAAG,CAAC,uCAAuC,OAAO,uBAAuB,CAAC,CAAC;gBACnF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;gBAC9F,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,uCAAuC,OAAO,YAAY,EAAE,KAAK,CAAC,CAAC;gBACjF,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;oBAClB,MAAM,KAAK,CAAC,CAAC,4BAA4B;gBAC3C,CAAC;gBACD,2BAA2B;gBAC3B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACzD,CAAC;oBAAS,CAAC;gBACT,qCAAqC;gBACrC,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACxB,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,OAAO,CAAC,GAAG,CAAC,6CAA6C,EAAE,UAAU,CAAC,CAAC;oBACzE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,OAAY,EAAE,KAAa,EAAE,UAAkB,EAAE,OAAe;QAC1G,yDAAyD;QACzD,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;gBACvC,SAAS,EAAE,uHAAuH;gBAClI,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;gBACtC,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBAErC,2BAA2B;gBAC3B,MAAM,SAAS,GAAG,qCAAqC,kBAAkB,CAAC,KAAK,CAAC,aAAa,CAAC;gBAC9F,OAAO,CAAC,GAAG,CAAC,+CAA+C,SAAS,EAAE,CAAC,CAAC;gBAExE,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;oBACzB,SAAS,EAAE,kBAAkB;oBAC7B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;gBAEH,kCAAkC;gBAClC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,CAAC,GAAG,CAAC,4EAA4E,CAAC,CAAC;gBAC5F,CAAC;gBAED,uBAAuB;gBACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBAElC,OAAO,CAAC,GAAG,CAAC,sDAAsD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAEjF,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,uCAAuC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;gBAE7E,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtB,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,yCAAyC;gBACzC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,KAAa,EAAE,UAAkB,EAAE,OAAe;QACnF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,yFAAyF,KAAK,GAAG,CAAC,CAAC;QAEjH,2BAA2B;QAC3B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9C,IAAI,OAAO,CAAC;YACZ,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,gCAAgC,OAAO,oCAAoC,CAAC,CAAC;gBAE3F,2DAA2D;gBAC3D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;oBAC9B,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;oBAClD,IAAI,EAAE;wBACJ,cAAc;wBACd,+CAA+C;wBAC/C,yBAAyB;wBACzB,eAAe;qBAChB;iBACF,CAAC,CAAC;gBAEH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAC1C,OAAO,CAAC,KAAK,CAAC,yDAAyD,UAAU,kBAAkB,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;gBAE5H,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;gBAC7F,OAAO,CAAC,KAAK,CAAC,2DAA2D,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;gBACnG,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBAC9E,OAAO,CAAC,KAAK,CAAC,gCAAgC,OAAO,yBAAyB,YAAY,EAAE,CAAC,CAAC;gBAE9F,IAAI,SAAS,EAAE,CAAC;oBACd,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;gBACnE,CAAC;gBAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;oBAClB,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;oBACxE,MAAM,KAAK,CAAC,CAAC,4BAA4B;gBAC3C,CAAC;gBACD,2BAA2B;gBAC3B,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;gBACpE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACzD,CAAC;oBAAS,CAAC;gBACT,qCAAqC;gBACrC,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;wBACtB,IAAI,SAAS,EAAE,CAAC;4BACd,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;wBACpE,CAAC;oBACH,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,UAAU,CAAC,CAAC;oBAC3E,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,4BAA4B,CAAC,OAAY,EAAE,KAAa,EAAE,UAAkB,EAAE,OAAe;QACzG,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,CAAC;QAE3D,yDAAyD;QACzD,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;QAE/F,IAAI,CAAC;YACH,8DAA8D;YAC9D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;gBACvC,SAAS,EAAE,uHAAuH;gBAClI,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;gBACtC,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,kBAAkB;gBAC9B,WAAW,EAAE,OAAO;gBACpB,iBAAiB,EAAE,CAAC;gBACpB,QAAQ,EAAE,KAAK;gBACf,QAAQ,EAAE,KAAK;gBACf,gBAAgB,EAAE;oBAChB,QAAQ,EAAE,uFAAuF;oBACjG,iBAAiB,EAAE,gBAAgB;oBACnC,iBAAiB,EAAE,mBAAmB;oBACtC,KAAK,EAAE,GAAG;oBACV,2BAA2B,EAAE,GAAG;oBAChC,gBAAgB,EAAE,UAAU;oBAC5B,gBAAgB,EAAE,UAAU;oBAC5B,gBAAgB,EAAE,MAAM;iBACzB;aACF,CAAC,CAAC;YAEH,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAE/D,IAAI,CAAC;gBACH,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,OAAO,CAAC,KAAK,CAAC,iFAAiF,CAAC,CAAC;oBACjG,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;oBACnF,OAAO,CAAC,KAAK,CAAC,uDAAuD,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;oBAC/F,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtB,OAAO,OAAO,CAAC;gBACjB,CAAC;gBAAC,OAAO,aAAa,EAAE,CAAC;oBACvB,MAAM,YAAY,GAAG,aAAa,YAAY,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;oBAC9F,OAAO,CAAC,KAAK,CAAC,gDAAgD,YAAY,EAAE,CAAC,CAAC;oBAE9E,IAAI,SAAS,EAAE,CAAC;wBACd,OAAO,CAAC,KAAK,CAAC,qDAAqD,EAAE,aAAa,CAAC,CAAC;oBACtF,CAAC;oBAED,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;oBAE3E,2DAA2D;oBAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;oBACjF,OAAO,CAAC,KAAK,CAAC,qDAAqD,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;oBAC7F,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtB,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,yCAAyC;gBACzC,OAAO,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;gBACpF,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9E,OAAO,CAAC,KAAK,CAAC,gDAAgD,YAAY,EAAE,CAAC,CAAC;YAE9E,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC;YAC9E,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,IAAS,EAAE,KAAa,EAAE,UAAkB,EAAE,OAAe;QAC/F,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAEvF,8DAA8D;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE;YACtC,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,OAAO,GAAG,CAAC;SACrB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,2CAA2C,QAAQ,eAAe,SAAS,WAAW,UAAU,EAAE,CAAC,CAAC;QAElH,uCAAuC;QACvC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAE/B,+DAA+D;QAC/D,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC,CAAC;YAC1E,MAAM,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,+DAA+D,KAAK,GAAG,CAAC,CAAC;YACvF,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YAErC,OAAO,CAAC,KAAK,CAAC,2EAA2E,CAAC,CAAC;YAC3F,yBAAyB;YACzB,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,IAAI,CAAC,iBAAiB,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;gBAC3E,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC;aAC3B,CAAC,CAAC;YAEH,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC9C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACjC,OAAO,CAAC,KAAK,CAAC,4CAA4C,cAAc,qBAAqB,eAAe,WAAW,aAAa,EAAE,CAAC,CAAC;QAE1I,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,MAAM,YAAY,GAAG,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACtF,OAAO,CAAC,KAAK,CAAC,uDAAuD,YAAY,EAAE,CAAC,CAAC;YAErF,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,SAAS,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,SAAS,CAAC;QAClB,CAAC;QAED,kCAAkC;QAClC,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,eAAe,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,6FAA6F,CAAC,CAAC;QAC/G,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,mDAAmD,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;QAE3F,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACrC,OAAO,CAAC,KAAK,CAAC,uFAAuF,CAAC,CAAC;QACzG,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,+CAA+C,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QAEvF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,2FAA2F,CAAC,CAAC;YAE3G,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC3C,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,UAAU,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,IAAS,EAAE,KAAa,EAAE,UAAkB,EAAE,OAAe;QAC7F,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAEhF,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE3C,6EAA6E;QAC7E,MAAM,SAAS,GAAG,iCAAiC,kBAAkB,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,8BAA8B,IAAI,EAAE,CAAC;QACnJ,OAAO,CAAC,KAAK,CAAC,kDAAkD,SAAS,EAAE,CAAC,CAAC;QAE7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACzB,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,8CAA8C,QAAQ,eAAe,SAAS,WAAW,UAAU,EAAE,CAAC,CAAC;QAErH,kCAAkC;QAClC,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,eAAe,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,6FAA6F,CAAC,CAAC;QAC/G,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,mDAAmD,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;QAE3F,IAAI,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACrC,OAAO,CAAC,KAAK,CAAC,uFAAuF,CAAC,CAAC;QACzG,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,6CAA6C,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QAErF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,2FAA2F,CAAC,CAAC;YAE3G,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC3C,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,UAAU,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,sBAAsB;QAC5B,0EAA0E;QAC1E,MAAM,KAAK,GAAG,kBAAkB,CAAC;QACjC,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAGO,KAAK,CAAC,mBAAmB,CAAC,KAAa,EAAE,UAAkB,EAAE,OAAe;QAClF,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,mCAAmC,EAAE;gBACpE,MAAM,EAAE;oBACN,CAAC,EAAE,KAAK;iBACT;gBACD,OAAO,EAAE;oBACP,YAAY,EAAE,uHAAuH;oBACrI,QAAQ,EAAE,iEAAiE;oBAC3E,iBAAiB,EAAE,gBAAgB;oBACnC,iBAAiB,EAAE,eAAe;oBAClC,KAAK,EAAE,GAAG;oBACV,YAAY,EAAE,YAAY;oBAC1B,2BAA2B,EAAE,GAAG;iBACjC;gBACD,OAAO;gBACP,cAAc,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG;aACjD,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,uDAAuD,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAEtF,MAAM,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACvE,OAAO,CAAC,GAAG,CAAC,oCAAoC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;YAE1E,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACzD,OAAO,CAAC,GAAG,CAAC,4CAA4C,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEvE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAC;QAEtC,4DAA4D;QAC5D,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7B,MAAM,eAAe,GAAG,CAAC,CAAC,6BAA6B,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC5B,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3B,MAAM,YAAY,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QAElC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,oCAAoC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,gBAAgB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,YAAY,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,mBAAmB,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,gBAAgB,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAEnD,iDAAiD;QACjD,MAAM,qBAAqB,GAAG;YAC5B,OAAO;YACP,6BAA6B;YAC7B,SAAS;YACT,KAAK;YACL,YAAY;YACZ,mBAAmB;SACpB,CAAC;QAEF,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;YAC7C,IAAI,YAAY;gBAAE,MAAM;YAExB,OAAO,CAAC,GAAG,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,CAAC,MAAM,2BAA2B,QAAQ,EAAE,CAAC,CAAC;YAE1F,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;gBAChC,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;oBAAE,OAAO,KAAK,CAAC;gBAE/C,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBAE5B,+BAA+B;gBAC/B,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC/E,IAAI,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,GAAG,GAAG,EAAE,CAAC;gBAEb,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;oBAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC;oBACpD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;wBAClB,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBAC7B,OAAO,CAAC,GAAG,CAAC,mCAAmC,aAAa,MAAM,KAAK,GAAG,CAAC,CAAC;wBAE5E,uBAAuB;wBACvB,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;4BACjB,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;4BAC/B,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,GAAG,CAAC,CAAC;wBACpD,CAAC;6BAAM,CAAC;4BACN,sCAAsC;4BACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;4BAClD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gCACpB,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gCAClC,OAAO,CAAC,GAAG,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;4BAClE,CAAC;wBACH,CAAC;wBACD,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,iCAAiC;gBACjC,MAAM,gBAAgB,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,kBAAkB,CAAC,CAAC;gBACvH,IAAI,OAAO,GAAG,EAAE,CAAC;gBAEjB,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;oBAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC;oBACxD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;wBACpB,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACjC,OAAO,CAAC,GAAG,CAAC,qCAAqC,eAAe,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;wBACvG,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/C,OAAO,CAAC,GAAG,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC;oBACtD,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK;wBACL,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;wBAC7B,WAAW,EAAE,OAAO,IAAI,0BAA0B;wBAClD,WAAW,EAAE,EAAE;wBACf,cAAc,EAAE,EAAE;wBAClB,SAAS,EAAE,CAAC;wBACZ,SAAS;wBACT,WAAW,EAAE,SAAS;qBACvB,CAAC,CAAC;oBACH,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,0CAA0C,KAAK,WAAW,GAAG,cAAc,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAEjF,mFAAmF;QACnF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;YAC/E,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;gBAC/B,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;oBAAE,OAAO,KAAK,CAAC;gBAE/C,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBACvB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAE/B,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;oBAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACrC,OAAO,CAAC,GAAG,CAAC,4CAA4C,KAAK,SAAS,GAAG,GAAG,CAAC,CAAC;oBAE9E,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/B,OAAO,CAAC,IAAI,CAAC;4BACX,KAAK;4BACL,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;4BAC7B,WAAW,EAAE,0BAA0B;4BACvC,WAAW,EAAE,EAAE;4BACf,cAAc,EAAE,EAAE;4BAClB,SAAS,EAAE,CAAC;4BACZ,SAAS;4BACT,WAAW,EAAE,SAAS;yBACvB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,0CAA0C,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,iBAAiB,CAAC,IAAY,EAAE,UAAkB;QACxD,OAAO,CAAC,GAAG,CAAC,kDAAkD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7E,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAC;QAEtC,yBAAyB;QACzB,MAAM,eAAe,GAAG;YACtB,mBAAmB,EAAM,qBAAqB;YAC9C,SAAS,EAAgB,qBAAqB;YAC9C,MAAM,CAAmB,wBAAwB;SAClD,CAAC;QAEF,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,IAAI,YAAY,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;gBAAE,MAAM;YAExD,OAAO,CAAC,GAAG,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,CAAC,MAAM,2BAA2B,QAAQ,EAAE,CAAC,CAAC;YAE1F,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;gBAChC,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;oBAAE,OAAO,KAAK,CAAC;gBAE/C,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBAE5B,yCAAyC;gBACzC,MAAM,cAAc,GAAG;oBACrB,UAAU,EAAe,iBAAiB;oBAC1C,MAAM,EAAmB,kBAAkB;oBAC3C,iBAAiB,EAAQ,qBAAqB;oBAC9C,gBAAgB,EAAS,oBAAoB;oBAC7C,kBAAkB,CAAO,gBAAgB;iBAC1C,CAAC;gBAEF,IAAI,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,GAAG,GAAG,EAAE,CAAC;gBAEb,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;oBAC3C,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC;oBAC3D,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;wBACzB,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACpC,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACvC,OAAO,CAAC,GAAG,CAAC,yCAAyC,aAAa,MAAM,KAAK,GAAG,CAAC,CAAC;wBAClF,IAAI,KAAK,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;4BAC3C,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAC7E,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACrB,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBACxB,OAAO,CAAC,GAAG,CAAC,wDAAwD,KAAK,GAAG,CAAC,CAAC;oBAChF,CAAC;gBACH,CAAC;gBAED,2CAA2C;gBAC3C,MAAM,gBAAgB,GAAG;oBACvB,kBAAkB,EAAO,iBAAiB;oBAC1C,UAAU,EAAe,UAAU;oBACnC,cAAc,EAAW,cAAc;oBACvC,GAAG,CAAsB,qBAAqB;iBAC/C,CAAC;gBAEF,IAAI,OAAO,GAAG,EAAE,CAAC;gBACjB,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;oBAC/C,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC;oBAC/D,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC;wBAC3B,OAAO,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACxC,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/C,OAAO,CAAC,GAAG,CAAC,gCAAgC,KAAK,SAAS,GAAG,GAAG,CAAC,CAAC;oBAClE,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK;wBACL,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;wBAC5B,WAAW,EAAE,OAAO,IAAI,0BAA0B;wBAClD,WAAW,EAAE,EAAE;wBACf,cAAc,EAAE,EAAE;wBAClB,SAAS,EAAE,CAAC;wBACZ,SAAS;wBACT,WAAW,EAAE,SAAS;qBACvB,CAAC,CAAC;oBACH,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,8BAA8B,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACpE,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,gBAAgB,CAAC,IAAY,EAAE,UAAkB;QACvD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,kDAAkD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE/E,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAC;QAEtC,yCAAyC;QACzC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,qCAAqC,SAAS,GAAG,CAAC,CAAC;QAEjE,IAAI,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1G,OAAO,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;QACtG,CAAC;QAED,wBAAwB;QACxB,MAAM,eAAe,GAAG;YACtB,SAAS,EAAM,oBAAoB;YACnC,WAAW,EAAI,0BAA0B;YACzC,SAAS,CAAM,cAAc;SAC9B,CAAC;QAEF,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAEtE,qCAAqC;QACrC,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,8BAA8B,QAAQ,CAAC,MAAM,4BAA4B,QAAQ,GAAG,CAAC,CAAC;QACtG,CAAC;QAED,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,IAAI,YAAY,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;gBAAE,MAAM;YAExD,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEpC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;gBAChC,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;oBAAE,OAAO,KAAK,CAAC;gBAE/C,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBAE5B,wCAAwC;gBACxC,MAAM,cAAc,GAAG;oBACrB,MAAM,EAAY,uBAAuB;oBACzC,YAAY,EAAM,qBAAqB;oBACvC,cAAc,CAAI,gBAAgB;iBACnC,CAAC;gBAEF,IAAI,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,GAAG,GAAG,EAAE,CAAC;gBAEb,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;oBAC3C,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC;oBAC3D,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;wBACzB,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACpC,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBACvC,OAAO,CAAC,GAAG,CAAC,wCAAwC,aAAa,MAAM,KAAK,GAAG,CAAC,CAAC;wBACjF,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,0CAA0C;gBAC1C,MAAM,gBAAgB,GAAG;oBACvB,cAAc,EAAY,wBAAwB;oBAClD,YAAY,EAAc,qBAAqB;oBAC/C,aAAa,EAAa,qBAAqB;oBAC/C,YAAY,EAAc,wBAAwB;oBAClD,mBAAmB,EAAO,eAAe;oBACzC,YAAY,EAAc,iBAAiB;oBAC3C,GAAG,EAAuB,8BAA8B;oBACxD,mBAAmB,EAAO,oBAAoB;oBAC9C,iBAAiB,EAAS,oBAAoB;oBAC9C,YAAY,CAAc,sBAAsB;iBACjD,CAAC;gBAEF,IAAI,OAAO,GAAG,EAAE,CAAC;gBACjB,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;oBAC/C,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC;oBAC/D,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC;wBAC3B,MAAM,gBAAgB,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;wBACvD,4DAA4D;wBAC5D,IAAI,gBAAgB,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,4CAA4C,CAAC,EAAE,CAAC;4BAC1G,OAAO,GAAG,gBAAgB,CAAC;4BAC3B,OAAO,CAAC,GAAG,CAAC,0CAA0C,eAAe,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;4BAC5G,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/C,OAAO,CAAC,GAAG,CAAC,+BAA+B,KAAK,SAAS,GAAG,GAAG,CAAC,CAAC;oBACjE,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK;wBACL,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC;wBAC3B,WAAW,EAAE,OAAO,IAAI,0BAA0B;wBAClD,WAAW,EAAE,EAAE;wBACf,cAAc,EAAE,EAAE;wBAClB,SAAS,EAAE,CAAC;wBACZ,SAAS;wBACT,WAAW,EAAE,SAAS;qBACvB,CAAC,CAAC;oBACH,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,6BAA6B,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACnE,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,sBAAsB,CAAC,IAAY,EAAE,UAAkB;QAC7D,OAAO,CAAC,GAAG,CAAC,uDAAuD,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAElF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAC;QAEtC,6CAA6C;QAC7C,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;YACpC,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;gBAAE,OAAO,KAAK,CAAC;YAE/C,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YAE5B,wBAAwB;YACxB,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACxD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEvC,kBAAkB;YAClB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAEhE,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,qCAAqC,KAAK,SAAS,GAAG,GAAG,CAAC,CAAC;gBACvE,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,GAAG,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC;oBACjC,WAAW,EAAE,OAAO,IAAI,0BAA0B;oBAClD,WAAW,EAAE,EAAE;oBACf,cAAc,EAAE,EAAE;oBAClB,SAAS,EAAE,CAAC;oBACZ,SAAS;oBACT,WAAW,EAAE,SAAS;iBACvB,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,mCAAmC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QACzE,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,uDAAuD;QACvD,OAAO,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;YACvB,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;YACzB,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC;YAC1B,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC;YACpB,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC;YAC1B,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YACnB,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC1B,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,iCAAiC;IAC3D,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,gCAAgC;QAChC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7D,IAAI,SAAS,EAAE,CAAC;oBACd,OAAO,SAAS,CAAC;gBACnB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,QAAQ,GAAG,GAAG,CAAC;QACxB,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,qEAAqE;QACrE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,QAAQ,GAAG,GAAG,CAAC;QACxB,CAAC;QAED,2CAA2C;QAC3C,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5D,OAAO,GAAG,CAAC;QACb,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,oEAAoE;QACpE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,QAAQ,GAAG,GAAG,CAAC;QACxB,CAAC;QAED,2CAA2C;QAC3C,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5D,OAAO,GAAG,CAAC;QACb,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,kBAAkB,CAAC,GAAW;QACpC,4DAA4D;QAC5D,IAAI,GAAG,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,2DAA2D;gBAC3D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC3E,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACxC,IAAI,SAAS,EAAE,CAAC;oBACd,iBAAiB;oBACjB,MAAM,UAAU,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;oBACjD,OAAO,CAAC,GAAG,CAAC,0CAA0C,UAAU,EAAE,CAAC,CAAC;oBACpE,OAAO,UAAU,CAAC;gBACpB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,GAAG,CAAC,mDAAmD,GAAG,EAAE,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QAED,8CAA8C;QAC9C,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,QAAQ,GAAG,GAAG,CAAC;QACxB,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,mBAAmB,CAAC,OAAuB,EAAE,aAAqB;QACxE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEnC,iEAAiE;QACjE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;QACnS,MAAM,UAAU,GAAG,aAAa,CAAC,WAAW,EAAE;aAC3C,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAE7D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,0CAA0C;QAEnF,OAAO,CAAC,GAAG,CAAC,wDAAwD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE9F,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,YAAY,GAAG,GAAG,SAAS,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAE3D,wBAAwB;YACxB,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,gDAAgD;YAChD,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,EAAE,CAAC;gBACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/C,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBACD,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC3B,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtD,CAAC;gBAED,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;oBAClC,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClC,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;gBACjC,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACnC,cAAc,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC;YAED,kCAAkC;YAClC,MAAM,YAAY,GAAG,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;YACxD,MAAM,WAAW,GAAG,aAAa,GAAG,GAAG,CAAC,CAAC,2BAA2B;YACpE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,CAAC,CAAC;YAE9D,yCAAyC;YACzC,MAAM,kBAAkB,GAAG;gBACzB,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO;gBACtD,UAAU,EAAE,cAAc,EAAE,WAAW;gBACvC,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ;gBAChD,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB;gBAC/C,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO;gBACrC,UAAU,EAAE,WAAW,EAAE,QAAQ;gBACjC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW;gBAC3C,MAAM,EAAE,UAAU,EAAE,aAAa;gBACjC,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY;aACpD,CAAC;YAEF,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;gBACzC,IAAI,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC/B,OAAO,IAAI,GAAG,CAAC;gBACjB,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC;YAEtD,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,cAAc,IAAI,UAAU,CAAC,MAAM,cAAc,aAAa,cAAc,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE3N,UAAU,IAAI,UAAU,CAAC;YACzB,aAAa,EAAE,CAAC;QAClB,CAAC;QAED,MAAM,YAAY,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,OAAO,YAAY,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,OAAY;QAC9C,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,MAAM,CAAC;QAErE,IAAI,CAAC;YACH,IAAI,aAAa;gBAAE,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;YAE9E,sCAAsC;YACtC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC3B,IAAI,aAAa;oBAAE,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;gBAC1E,OAAO,KAAK,CAAC;YACf,CAAC;YAED,gEAAgE;YAChE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAC/C,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;YAE1B,IAAI,aAAa;gBAAE,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC7E,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,+CAA+C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YACvH,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAU,EAAE,UAAkB,EAAE,gBAAwB,CAAC;QACxF,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC9E,OAAO,CAAC,KAAK,CAAC,kBAAkB,UAAU,2BAA2B,aAAa,MAAM,YAAY,EAAE,CAAC,CAAC;QAExG,4CAA4C;QAC5C,IAAI,YAAY,CAAC,QAAQ,CAAC,iDAAiD,CAAC;YACxE,YAAY,CAAC,QAAQ,CAAC,yBAAyB,CAAC;YAChD,YAAY,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;YAErD,OAAO,CAAC,GAAG,CAAC,qFAAqF,CAAC,CAAC;YAEnG,0DAA0D;YAC1D,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,6CAA6C,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;YAAC,OAAO,YAAY,EAAE,CAAC;gBACtB,OAAO,CAAC,KAAK,CAAC,kDAAkD,YAAY,YAAY,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YAC5I,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;IACpC,CAAC;CACF"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/types.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/types.d.ts new file mode 100644 index 0000000..3302b25 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/types.d.ts @@ -0,0 +1,69 @@ +export interface SearchResult { + title: string; + url: string; + description: string; + fullContent: string; + contentPreview: string; + wordCount: number; + timestamp: string; + fetchStatus: 'success' | 'error' | 'timeout'; + error?: string; +} +export interface SearchResponse { + query: string; + limit: number; + results: SearchResult[]; + totalFound: number; + searchTimestamp: string; + processingTimeMs: number; +} +export interface SearchOptions { + query: string; + numResults?: number; + timeout?: number; +} +export interface ContentExtractionOptions { + url: string; + timeout?: number; + maxContentLength?: number; +} +export interface WebSearchToolInput { + query: string; + limit?: number; + includeContent?: boolean; + maxContentLength?: number; +} +export interface WebSearchToolOutput { + results: SearchResult[]; + total_results: number; + search_time_ms: number; + query: string; + status?: string; +} +export interface SearchSummaryResult { + title: string; + url: string; + description: string; + timestamp: string; +} +export interface SearchSummaryOutput { + results: SearchSummaryResult[]; + total_results: number; + search_time_ms: number; + query: string; +} +export interface SinglePageContentOutput { + url: string; + title: string; + content: string; + contentPreview: string; + wordCount: number; + timestamp: string; + fetchStatus: 'success' | 'error'; + error?: string; +} +export interface SearchResultWithMetadata { + results: SearchResult[]; + engine: string; +} +//# sourceMappingURL=types.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/types.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/types.d.ts.map new file mode 100644 index 0000000..51771cc --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/types.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAGD,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf;AAGD,MAAM,WAAW,uBAAuB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,SAAS,GAAG,OAAO,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAGD,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/types.js b/local_ai/.config/mcp/web-search-mcp/dist/types.js new file mode 100644 index 0000000..718fd38 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/types.js @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=types.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/types.js.map b/local_ai/.config/mcp/web-search-mcp/dist/types.js.map new file mode 100644 index 0000000..c768b79 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/types.js.map @@ -0,0 +1 @@ +{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts b/local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts new file mode 100644 index 0000000..b036707 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts @@ -0,0 +1,13 @@ +/** + * Utility functions for the web search MCP server + */ +export declare function cleanText(text: string, maxLength?: number): string; +export declare function getWordCount(text: string): number; +export declare function getContentPreview(text: string, maxLength?: number): string; +export declare function generateTimestamp(): string; +export declare function validateUrl(url: string): boolean; +export declare function sanitizeQuery(query: string): string; +export declare function getRandomUserAgent(): string; +export declare function delay(ms: number): Promise; +export declare function isPdfUrl(url: string): boolean; +//# sourceMappingURL=utils.d.ts.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts.map b/local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts.map new file mode 100644 index 0000000..2468494 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/utils.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAc,GAAG,MAAM,CAMzE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAY,GAAG,MAAM,CAG/E;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAOhD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAQ3C;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAQ7C"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/utils.js b/local_ai/.config/mcp/web-search-mcp/dist/utils.js new file mode 100644 index 0000000..f234622 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/utils.js @@ -0,0 +1,55 @@ +/** + * Utility functions for the web search MCP server + */ +export function cleanText(text, maxLength = 10000) { + return text + .replace(/\s+/g, ' ') // Replace multiple whitespace with single space + .replace(/\n\s*\n/g, '\n') // Replace multiple newlines with single newline + .trim() + .substring(0, maxLength); +} +export function getWordCount(text) { + return text.trim().split(/\s+/).filter(word => word.length > 0).length; +} +export function getContentPreview(text, maxLength = 500) { + const cleaned = cleanText(text, maxLength); + return cleaned.length === maxLength ? cleaned + '...' : cleaned; +} +export function generateTimestamp() { + return new Date().toISOString(); +} +export function validateUrl(url) { + try { + const parsed = new URL(url); + return parsed.protocol === 'http:' || parsed.protocol === 'https:'; + } + catch { + return false; + } +} +export function sanitizeQuery(query) { + return query.trim().substring(0, 1000); // Limit query length +} +export function getRandomUserAgent() { + const userAgents = [ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15', + ]; + return userAgents[Math.floor(Math.random() * userAgents.length)]; +} +export function delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} +export function isPdfUrl(url) { + try { + const parsed = new URL(url); + return parsed.pathname.toLowerCase().endsWith('.pdf'); + } + catch { + // If URL parsing fails, check the raw string as fallback + return url.toLowerCase().endsWith('.pdf'); + } +} +//# sourceMappingURL=utils.js.map \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/dist/utils.js.map b/local_ai/.config/mcp/web-search-mcp/dist/utils.js.map new file mode 100644 index 0000000..4730008 --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/dist/utils.js.map @@ -0,0 +1 @@ +{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,YAAoB,KAAK;IAC/D,OAAO,IAAI;SACR,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,gDAAgD;SACrE,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,gDAAgD;SAC1E,IAAI,EAAE;SACN,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,YAAoB,GAAG;IACrE,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,qBAAqB;AAC/D,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,UAAU,GAAG;QACjB,uHAAuH;QACvH,iHAAiH;QACjH,uGAAuG;QACvG,uHAAuH;KACxH,CAAC;IACF,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,yDAAyD;QACzD,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/local_ai/.config/mcp/web-search-mcp/package.json b/local_ai/.config/mcp/web-search-mcp/package.json new file mode 100644 index 0000000..fd7724d --- /dev/null +++ b/local_ai/.config/mcp/web-search-mcp/package.json @@ -0,0 +1,49 @@ +{ + "name": "web-search-mcp-server", + "version": "0.3.1", + "description": "MCP server for web search with full page content extraction, search summaries, and single page content extraction", + "license": "MIT", + "type": "module", + "main": "./dist/index.js", + "bin": { + "web-search-mcp": "./dist/index.js" + }, + "scripts": { + "build": "tsc && echo '✅ TypeScript compilation complete: dist/index.js'", + "dev": "tsx watch src/index.ts", + "start": "node ./dist/index.js", + "lint": "eslint \"src/**/*.ts\"", + "format": "prettier --write ." + }, + "keywords": [ + "mcp", + "web-search", + "ai", + "llm" + ], + "author": "Mark Russell", + "repository": { + "type": "git", + "url": "https://github.com/mrkrsl/web-search-mcp.git" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "^1.15.0", + "axios": "^1.6.8", + "cheerio": "^1.0.0-rc.12", + "p-limit": "^6.2.0", + "p-retry": "^6.2.1", + "playwright": "^1.48.0", + "zod": "^3.22.0" + }, + "devDependencies": { + "@eslint/js": "^9.30.1", + "@types/node": "^24.0.10", + "@typescript-eslint/eslint-plugin": "^8.35.1", + "@typescript-eslint/parser": "^8.35.1", + "esbuild": "^0.25.5", + "eslint": "^9.30.1", + "prettier": "^3.2.5", + "tsx": "^4.7.0", + "typescript": "^5.4.5" + } +}