fix(agent): provider-aware fallback for haiku/sonnet aliases (#908)
* fix(agent): provider-aware fallback for haiku/sonnet aliases Explore agent fails on custom providers (Z.AI GLM, Alibaba Anthropic-compatible, local OpenAI endpoints) because 'haiku' alias resolves to a non-existent model. Changes: - Add isClaudeNativeProvider check (Bedrock, Vertex, Foundry, official Anthropic) - For non-Claude-native providers, haiku/sonnet aliases inherit parent model - Add 8 tests for provider-aware fallback behavior Fixes Explore agent "model not found" errors on custom Anthropic-compatible APIs. * test(agent): use Bun mock.module() for provider tests Replace env manipulation with proper Bun mock.module() to reliably mock getAPIProvider() and isFirstPartyAnthropicBaseUrl() functions. This ensures tests work correctly on CI where module caching caused false negatives. --------- Co-authored-by: Ali Alakbarli <ali.alakbarli@users.noreply.github.com>
This commit is contained in:
emsanakhchivan
GitHub
261
src/utils/model/agent.test.ts
Normal file
261
src/utils/model/agent.test.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
import { describe, test, expect, beforeEach, afterEach, mock } from 'bun:test'
|
||||
|
||||
describe('getAgentModel provider-aware fallback', () => {
|
||||
// Restore all mocks after each test
|
||||
afterEach(() => {
|
||||
mock.restore()
|
||||
})
|
||||
|
||||
describe('Claude-native providers', () => {
|
||||
test('haiku alias resolves to haiku model for official Anthropic API', async () => {
|
||||
// Mock providers to return firstParty with official URL
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'firstParty',
|
||||
isFirstPartyAnthropicBaseUrl: () => true,
|
||||
}))
|
||||
|
||||
// Import after mock is set up
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'claude-sonnet-4-6', undefined, 'default')
|
||||
|
||||
// Should resolve haiku alias, not inherit parent
|
||||
expect(result).toContain('haiku')
|
||||
expect(result).not.toBe('claude-sonnet-4-6')
|
||||
})
|
||||
|
||||
test('haiku alias resolves for Bedrock provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'bedrock',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'claude-sonnet-4-6', undefined, 'default')
|
||||
|
||||
// Should resolve haiku alias for Bedrock
|
||||
expect(result).toContain('haiku')
|
||||
})
|
||||
|
||||
test('haiku alias resolves for Vertex provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'vertex',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'claude-sonnet-4-6', undefined, 'default')
|
||||
|
||||
// Should resolve haiku alias for Vertex
|
||||
expect(result).toContain('haiku')
|
||||
})
|
||||
|
||||
test('haiku alias resolves for Foundry provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'foundry',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'claude-sonnet-4-6', undefined, 'default')
|
||||
|
||||
// Should resolve haiku alias for Foundry
|
||||
expect(result).toContain('haiku')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Non-Claude-native providers', () => {
|
||||
test('haiku alias inherits parent model for OpenAI provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'openai',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'gpt-4o-mini', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for OpenAI (no haiku concept)
|
||||
expect(result).toBe('gpt-4o-mini')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for Gemini provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'gemini',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'gemini-2.5-pro', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for Gemini
|
||||
expect(result).toBe('gemini-2.5-pro')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for custom Anthropic-compatible URL', async () => {
|
||||
// firstParty provider but with custom URL (not official Anthropic)
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'firstParty',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'claude-sonnet-4-6', undefined, 'default')
|
||||
|
||||
// Should inherit parent for custom Anthropic-compatible URL
|
||||
expect(result).toBe('claude-sonnet-4-6')
|
||||
})
|
||||
|
||||
test('sonnet alias inherits parent model for OpenAI provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'openai',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('sonnet', 'gpt-4o-mini', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for OpenAI
|
||||
expect(result).toBe('gpt-4o-mini')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for Mistral provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'mistral',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'mistral-small-latest', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for Mistral (no haiku concept)
|
||||
expect(result).toBe('mistral-small-latest')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for GitHub Copilot provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'github',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'gpt-4o-mini', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for GitHub Copilot
|
||||
expect(result).toBe('gpt-4o-mini')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for NVIDIA NIM provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'nvidia-nim',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'meta/llama-3.1-8b-instruct', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for NVIDIA NIM (no haiku concept)
|
||||
expect(result).toBe('meta/llama-3.1-8b-instruct')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for MiniMax provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'minimax',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'MiniMax-M2.5-highspeed', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for MiniMax (no haiku concept)
|
||||
expect(result).toBe('MiniMax-M2.5-highspeed')
|
||||
})
|
||||
|
||||
test('haiku alias inherits parent model for Codex provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'codex',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('haiku', 'gpt-5.5-mini', undefined, 'default')
|
||||
|
||||
// Should inherit parent model for Codex provider (no haiku concept)
|
||||
expect(result).toBe('gpt-5.5-mini')
|
||||
})
|
||||
})
|
||||
|
||||
describe('inherit behavior unchanged', () => {
|
||||
test('inherit always returns parent model regardless of provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'openai',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { getAgentModel } = await import('./agent.js')
|
||||
const result = getAgentModel('inherit', 'gpt-4o', undefined, 'default')
|
||||
|
||||
expect(result).toBe('gpt-4o')
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkIsClaudeNativeProvider helper', () => {
|
||||
test('returns true for official Anthropic API', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'firstParty',
|
||||
isFirstPartyAnthropicBaseUrl: () => true,
|
||||
}))
|
||||
|
||||
const { checkIsClaudeNativeProvider } = await import('./agent.js')
|
||||
expect(checkIsClaudeNativeProvider()).toBe(true)
|
||||
})
|
||||
|
||||
test('returns true for Bedrock provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'bedrock',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { checkIsClaudeNativeProvider } = await import('./agent.js')
|
||||
expect(checkIsClaudeNativeProvider()).toBe(true)
|
||||
})
|
||||
|
||||
test('returns true for Vertex provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'vertex',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { checkIsClaudeNativeProvider } = await import('./agent.js')
|
||||
expect(checkIsClaudeNativeProvider()).toBe(true)
|
||||
})
|
||||
|
||||
test('returns true for Foundry provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'foundry',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { checkIsClaudeNativeProvider } = await import('./agent.js')
|
||||
expect(checkIsClaudeNativeProvider()).toBe(true)
|
||||
})
|
||||
|
||||
test('returns false for OpenAI provider', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'openai',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { checkIsClaudeNativeProvider } = await import('./agent.js')
|
||||
expect(checkIsClaudeNativeProvider()).toBe(false)
|
||||
})
|
||||
|
||||
test('returns false for custom Anthropic URL', async () => {
|
||||
mock.module('./providers.js', () => ({
|
||||
getAPIProvider: () => 'firstParty',
|
||||
isFirstPartyAnthropicBaseUrl: () => false,
|
||||
}))
|
||||
|
||||
const { checkIsClaudeNativeProvider } = await import('./agent.js')
|
||||
expect(checkIsClaudeNativeProvider()).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
getRuntimeMainLoopModel,
|
||||
parseUserSpecifiedModel,
|
||||
} from './model.js'
|
||||
import { getAPIProvider } from './providers.js'
|
||||
import { getAPIProvider, isFirstPartyAnthropicBaseUrl } from './providers.js'
|
||||
|
||||
export const AGENT_MODEL_OPTIONS = [...MODEL_ALIASES, 'inherit'] as const
|
||||
export type AgentModelAlias = (typeof AGENT_MODEL_OPTIONS)[number]
|
||||
@@ -77,6 +77,26 @@ export function getAgentModel(
|
||||
|
||||
const agentModelWithExp = agentModel ?? getDefaultSubagentModel()
|
||||
|
||||
// Provider-aware model alias fallback for agents.
|
||||
// Claude-native providers (Bedrock, Vertex, Foundry, official Anthropic API)
|
||||
// have guaranteed haiku/sonnet model availability. Custom Anthropic-compatible
|
||||
// endpoints, OpenAI-shim, Gemini, Mistral, and other providers may not have
|
||||
// equivalent models, causing "model not found" errors when resolving aliases.
|
||||
// For haiku/sonnet aliases on non-Claude-native providers, inherit parent model.
|
||||
// Note: 'opus' is NOT included here because it's handled separately by
|
||||
// aliasMatchesParentTier() which checks if parent's tier matches the alias.
|
||||
if (
|
||||
(agentModelWithExp === 'haiku' || agentModelWithExp === 'sonnet') &&
|
||||
!checkIsClaudeNativeProvider()
|
||||
) {
|
||||
// Non-Claude-native provider → inherit parent model
|
||||
return getRuntimeMainLoopModel({
|
||||
permissionMode: permissionMode ?? 'default',
|
||||
mainLoopModel: parentModel,
|
||||
exceeds200kTokens: false,
|
||||
})
|
||||
}
|
||||
|
||||
if (agentModelWithExp === 'inherit') {
|
||||
// Apply runtime model resolution for inherit to get the effective model
|
||||
// This ensures agents using 'inherit' get opusplan→Opus resolution in plan mode
|
||||
@@ -121,6 +141,22 @@ function aliasMatchesParentTier(alias: string, parentModel: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the current provider is Claude-native (has guaranteed haiku/sonnet models).
|
||||
* Claude-native providers: Bedrock, Vertex, Foundry, official Anthropic API.
|
||||
* Non-Claude-native: OpenAI, Gemini, Mistral, GitHub, NVIDIA NIM, MiniMax,
|
||||
* and custom Anthropic-compatible endpoints (proxies, self-hosted).
|
||||
*/
|
||||
export function checkIsClaudeNativeProvider(): boolean {
|
||||
const provider = getAPIProvider()
|
||||
return (
|
||||
provider === 'bedrock' ||
|
||||
provider === 'vertex' ||
|
||||
provider === 'foundry' ||
|
||||
(provider === 'firstParty' && isFirstPartyAnthropicBaseUrl())
|
||||
)
|
||||
}
|
||||
|
||||
export function getAgentModelDisplay(model: string | undefined): string {
|
||||
// When model is omitted, getDefaultSubagentModel() returns 'inherit' at runtime
|
||||
if (!model) return 'Inherit from parent (default)'
|
||||
|
||||
Reference in New Issue
Block a user