Compare commits
1 Commits
main
...
fix/websea
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
149b1eb8fb |
@@ -1,6 +1,23 @@
|
|||||||
import type { SearchInput, SearchProvider } from './types.js'
|
import type { SearchInput, SearchProvider } from './types.js'
|
||||||
import { applyDomainFilters, type ProviderOutput } from './types.js'
|
import { applyDomainFilters, type ProviderOutput } from './types.js'
|
||||||
|
|
||||||
|
// DuckDuckGo's HTML scraper aggressively blocks datacenter / repeat IPs with
|
||||||
|
// an "anomaly in the request" response. When that happens we surface an
|
||||||
|
// actionable error instead of the opaque scraper message so users know how
|
||||||
|
// to configure a working backend.
|
||||||
|
const DDG_ANOMALY_HINT =
|
||||||
|
'DuckDuckGo scraping is rate-limited from this network. ' +
|
||||||
|
'Configure a search backend with one of: ' +
|
||||||
|
'FIRECRAWL_API_KEY, TAVILY_API_KEY, EXA_API_KEY, YOU_API_KEY, ' +
|
||||||
|
'JINA_API_KEY, BING_API_KEY, MOJEEK_API_KEY, LINKUP_API_KEY — ' +
|
||||||
|
'or use an Anthropic / Vertex / Foundry provider for native web search.'
|
||||||
|
|
||||||
|
function isAnomalyError(message: string): boolean {
|
||||||
|
return /anomaly in the request|likely making requests too quickly/i.test(
|
||||||
|
message,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
export const duckduckgoProvider: SearchProvider = {
|
export const duckduckgoProvider: SearchProvider = {
|
||||||
name: 'duckduckgo',
|
name: 'duckduckgo',
|
||||||
|
|
||||||
@@ -20,7 +37,16 @@ export const duckduckgoProvider: SearchProvider = {
|
|||||||
}
|
}
|
||||||
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError')
|
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError')
|
||||||
// TODO: duck-duck-scrape doesn't accept AbortSignal — can't cancel in-flight searches
|
// TODO: duck-duck-scrape doesn't accept AbortSignal — can't cancel in-flight searches
|
||||||
const response = await search(input.query, { safeSearch: SafeSearchType.STRICT })
|
let response: Awaited<ReturnType<typeof search>>
|
||||||
|
try {
|
||||||
|
response = await search(input.query, { safeSearch: SafeSearchType.STRICT })
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err)
|
||||||
|
if (isAnomalyError(msg)) {
|
||||||
|
throw new Error(DDG_ANOMALY_HINT)
|
||||||
|
}
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
|
||||||
const hits = applyDomainFilters(
|
const hits = applyDomainFilters(
|
||||||
response.results.map(r => ({
|
response.results.map(r => ({
|
||||||
|
|||||||
Reference in New Issue
Block a user