Compare commits
1 Commits
fix/issue-
...
fix/websea
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
149b1eb8fb |
@@ -1,6 +1,23 @@
|
||||
import type { SearchInput, SearchProvider } from './types.js'
|
||||
import { applyDomainFilters, type ProviderOutput } from './types.js'
|
||||
|
||||
// DuckDuckGo's HTML scraper aggressively blocks datacenter / repeat IPs with
|
||||
// an "anomaly in the request" response. When that happens we surface an
|
||||
// actionable error instead of the opaque scraper message so users know how
|
||||
// to configure a working backend.
|
||||
const DDG_ANOMALY_HINT =
|
||||
'DuckDuckGo scraping is rate-limited from this network. ' +
|
||||
'Configure a search backend with one of: ' +
|
||||
'FIRECRAWL_API_KEY, TAVILY_API_KEY, EXA_API_KEY, YOU_API_KEY, ' +
|
||||
'JINA_API_KEY, BING_API_KEY, MOJEEK_API_KEY, LINKUP_API_KEY — ' +
|
||||
'or use an Anthropic / Vertex / Foundry provider for native web search.'
|
||||
|
||||
function isAnomalyError(message: string): boolean {
|
||||
return /anomaly in the request|likely making requests too quickly/i.test(
|
||||
message,
|
||||
)
|
||||
}
|
||||
|
||||
export const duckduckgoProvider: SearchProvider = {
|
||||
name: 'duckduckgo',
|
||||
|
||||
@@ -20,7 +37,16 @@ export const duckduckgoProvider: SearchProvider = {
|
||||
}
|
||||
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError')
|
||||
// TODO: duck-duck-scrape doesn't accept AbortSignal — can't cancel in-flight searches
|
||||
const response = await search(input.query, { safeSearch: SafeSearchType.STRICT })
|
||||
let response: Awaited<ReturnType<typeof search>>
|
||||
try {
|
||||
response = await search(input.query, { safeSearch: SafeSearchType.STRICT })
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
if (isAnomalyError(msg)) {
|
||||
throw new Error(DDG_ANOMALY_HINT)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
const hits = applyDomainFilters(
|
||||
response.results.map(r => ({
|
||||
|
||||
Reference in New Issue
Block a user