Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions apps/sim/executor/handlers/agent/agent-handler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { SIM_AUTO_MODEL_ID } from '@/providers/models'
import { getProviderFromModel, transformBlockTool } from '@/providers/utils'
import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
import { executeTool } from '@/tools'
import { ToolSchemaEnrichmentError } from '@/tools/params'

process.env.NEXT_PUBLIC_APP_URL = 'http://localhost:3000'

Expand Down Expand Up @@ -289,6 +290,24 @@ describe('AgentBlockHandler', () => {
expect(result).toEqual(expectedOutput)
})

it('fails fast when a configured tool schema cannot be enriched', async () => {
const error = new ToolSchemaEnrichmentError(
'table_query_rows',
new Error('table metadata unavailable')
)
mockTransformBlockTool.mockRejectedValueOnce(error)

await expect(
handler.execute(mockContext, mockBlock, {
model: 'gpt-4o',
userPrompt: 'Query the table',
apiKey: 'test-api-key',
tools: [{ type: 'table', operation: 'query_rows', usageControl: 'auto' }],
})
).rejects.toBe(error)
expect(mockExecuteProviderRequest).not.toHaveBeenCalled()
})

it('reports a sim-auto run under the sim-auto identity, not the model that served it', async () => {
mockExecuteProviderRequest.mockResolvedValue({
content: 'Mocked response content',
Expand Down
9 changes: 8 additions & 1 deletion apps/sim/executor/handlers/agent/agent-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ import {
import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models'
import { getProviderFromModel, transformBlockTool } from '@/providers/utils'
import type { SerializedBlock } from '@/serializer/types'
import { filterSchemaForLLM, type ToolSchema } from '@/tools/params'
import { filterSchemaForLLM, type ToolSchema, ToolSchemaEnrichmentError } from '@/tools/params'
import { getTool } from '@/tools/utils'
import { getToolAsync } from '@/tools/utils.server'

Expand Down Expand Up @@ -526,6 +526,7 @@ export class AgentBlockHandler implements BlockHandler {
}
return this.transformBlockTool(ctx, tool, canonicalModes, toolIndex)
} catch (error) {
if (error instanceof ToolSchemaEnrichmentError) throw error
Comment thread
TheodoreSpeaks marked this conversation as resolved.
logger.error(
'[AgentHandler] Error creating tool',
projectAgentDiagnosticMetadata(
Expand Down Expand Up @@ -952,6 +953,12 @@ export class AgentBlockHandler implements BlockHandler {
}),
getTool,
canonicalModes,
enrichmentContext: {
workflowId: ctx.workflowId,
workspaceId: ctx.workspaceId,
executionId: ctx.executionId,
userId: ctx.userId,
},
toolIndex,
resolveCustomBlockBinding: (blockType: string) =>
resolveCustomBlockToolBinding(blockType, ctx.workspaceId),
Expand Down
15 changes: 15 additions & 0 deletions apps/sim/executor/handlers/pi/sim-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ vi.mock('@/tools/utils.server', () => ({ getToolAsync: vi.fn() }))
import { buildSimToolSpecs } from '@/executor/handlers/pi/sim-tools'
import type { ExecutionContext } from '@/executor/types'
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
import { ToolSchemaEnrichmentError } from '@/tools/params'

function executionContext(registry: ResolvedSecretTraceRegistry | undefined): ExecutionContext {
return {
Expand Down Expand Up @@ -76,6 +77,20 @@ describe('buildSimToolSpecs', () => {
expect(mockTransformBlockTool).not.toHaveBeenCalled()
})

it('fails fast when a tool schema cannot be enriched', async () => {
const error = new ToolSchemaEnrichmentError(
'table_query_rows',
new Error('table metadata unavailable')
)
mockTransformBlockTool.mockRejectedValueOnce(error)

await expect(
buildSimToolSpecs(completeExecutionContext(), [
{ type: 'table', operation: 'query_rows', usageControl: 'auto' },
])
).rejects.toBe(error)
})

it('forwards a trusted _context that an LLM-supplied _context cannot override', async () => {
mockTransformBlockTool.mockResolvedValue({
id: 'exa_search',
Expand Down
8 changes: 8 additions & 0 deletions apps/sim/executor/handlers/pi/sim-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secr
import { transformBlockTool } from '@/providers/utils'
import { executeTool } from '@/tools'
import { mergeToolParameters } from '@/tools/merge-params'
import { ToolSchemaEnrichmentError } from '@/tools/params'
import type { ToolResponse } from '@/tools/types'
import { getTool } from '@/tools/utils'
import { getToolAsync } from '@/tools/utils.server'
Expand Down Expand Up @@ -97,6 +98,12 @@ export async function buildSimToolSpecs(
getAllBlocks,
getTool,
getToolAsync,
enrichmentContext: {
workflowId: ctx.workflowId,
workspaceId: ctx.workspaceId,
executionId: ctx.executionId,
userId: ctx.userId,
},
resolveCustomBlockBinding: (blockType: string) =>
resolveCustomBlockToolBinding(blockType, ctx.workspaceId),
})
Expand Down Expand Up @@ -171,6 +178,7 @@ export async function buildSimToolSpecs(
},
})
} catch (error) {
if (error instanceof ToolSchemaEnrichmentError) throw error
logger.warn('Failed to adapt Sim tool for Pi', {
type: tool.type,
error: getErrorMessage(error),
Expand Down
72 changes: 72 additions & 0 deletions apps/sim/providers/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1622,6 +1622,78 @@ describe('transformBlockTool multi-instance unique IDs', () => {
expect(result?.id).toBe('table_query_rows_tbl_abc')
})

it('resolves the canonical table id before enriching the LLM tool schema', async () => {
const enrichTool = vi.fn(
async (
tableId: string,
schema: {
type: 'object'
properties: Record<string, unknown>
required: string[]
}
) => ({
description: `Query rows from ${tableId}`,
parameters: {
...schema,
properties: {
...schema.properties,
customer_name: { type: 'string' },
},
},
})
)
const result = await transformBlockTool(
{
type: 'table',
operation: 'query_rows',
params: { tableSelector: 'tbl_abc' },
},
{
selectedOperation: 'query_rows',
getAllBlocks,
enrichmentContext: {
workspaceId: 'workspace-1',
userId: 'user-1',
},
getTool: (id: string) => ({
id,
name: 'Query Rows',
description: 'Query table rows',
params: {
tableId: { type: 'string', required: true, visibility: 'user-only' },
filter: { type: 'object', visibility: 'user-or-llm' },
},
toolEnrichment: {
dependsOn: 'tableId',
enrichTool,
},
}),
}
)

expect(enrichTool).toHaveBeenCalledWith(
'tbl_abc',
expect.objectContaining({
properties: expect.objectContaining({ filter: expect.any(Object) }),
}),
'Query table rows',
{
workspaceId: 'workspace-1',
userId: 'user-1',
}
)
expect(result).toMatchObject({
id: 'table_query_rows_tbl_abc',
description: 'Query rows from tbl_abc',
params: { tableSelector: 'tbl_abc' },
parameters: {
properties: {
customer_name: { type: 'string' },
},
},
})
})

it('appends the table id resolved from the advanced manual input', async () => {
const result = await transformTable(
{ manualTableId: 'tbl_xyz' },
Expand Down
25 changes: 17 additions & 8 deletions apps/sim/providers/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import {
import type { ProviderId, ProviderToolConfig } from '@/providers/types'
import { useProvidersStore } from '@/stores/providers/store'
import { mergeToolParameters } from '@/tools/merge-params'
import type { WorkflowToolExecutionContext } from '@/tools/types'

const logger = createLogger('ProviderUtils')

Expand Down Expand Up @@ -629,6 +630,7 @@ export async function transformBlockTool(
getTool: (toolId: string) => any
getToolAsync?: (toolId: string) => Promise<any>
canonicalModes?: Record<string, 'basic' | 'advanced'>
enrichmentContext?: WorkflowToolExecutionContext
/**
* Server-only resolver for a custom (deploy-as-block) tool's binding (bound
* workflow + input schema), org-scoped to the consumer. Injected as a dependency
Expand All @@ -646,8 +648,15 @@ export async function transformBlockTool(
toolIndex?: number
}
): Promise<ProviderToolConfig | null> {
const { selectedOperation, getAllBlocks, getTool, getToolAsync, canonicalModes, toolIndex } =
options
const {
selectedOperation,
getAllBlocks,
getTool,
getToolAsync,
canonicalModes,
enrichmentContext,
toolIndex,
} = options
const scopedCanonicalModes = scopeCanonicalModesForTool(canonicalModes, toolIndex, block.type)

const blockDef = getAllBlocks().find((b: any) => b.type === block.type)
Expand Down Expand Up @@ -755,12 +764,6 @@ export async function transformBlockTool(

const userProvidedParams = block.params || {}

const {
schema: llmSchema,
enrichedDescription,
modelBlockedParams,
} = await createLLMToolSchema(toolConfig, userProvidedParams)

const canonicalGroups: CanonicalGroup[] = blockDef?.subBlocks
? Object.values(buildCanonicalIndex(blockDef.subBlocks).groupsById).filter(isCanonicalPair)
: []
Expand All @@ -771,6 +774,12 @@ export async function transformBlockTool(
scopedCanonicalModes
)

const {
schema: llmSchema,
enrichedDescription,
modelBlockedParams,
} = await createLLMToolSchema(toolConfig, resolvedResourceParams, enrichmentContext)

let uniqueToolId = toolConfig.id
let toolName = toolConfig.name
let toolDescription = enrichedDescription || toolConfig.description
Expand Down
22 changes: 22 additions & 0 deletions apps/sim/tools/params.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
isPasswordParameter,
type ToolParameterConfig,
type ToolSchema,
ToolSchemaEnrichmentError,
type ValidationResult,
validateToolParameters,
} from '@/tools/params'
Expand Down Expand Up @@ -130,6 +131,27 @@ describe('Tool Parameters Utils', () => {
expect(schema.required).not.toContain('apiKey') // user-only, never required for LLM
expect(schema.required).toContain('message') // user-or-llm + required: true
})

it('wraps tool enrichment failures so execution boundaries can fail fast', async () => {
const cause = new Error('table metadata unavailable')
const toolConfig = {
...mockToolConfig,
toolEnrichment: {
dependsOn: 'tableId',
enrichTool: vi.fn().mockRejectedValue(cause),
},
}

const error = await createLLMToolSchema(toolConfig, { tableId: 'tbl_123' }).catch(
(caught) => caught
)

expect(error).toBeInstanceOf(ToolSchemaEnrichmentError)
expect(error).toMatchObject({
message: 'Failed to enrich schema for tool "test_tool"',
cause,
})
})
})

describe('createUserToolSchema', () => {
Expand Down
27 changes: 21 additions & 6 deletions apps/sim/tools/params.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import type {
ParameterVisibility,
ToolConfig,
ToolParameterItemSchema,
WorkflowToolExecutionContext,
} from '@/tools/types'

const logger = createLogger('ToolsParams')
Expand Down Expand Up @@ -155,6 +156,13 @@ export interface LLMToolSchemaResult {
modelBlockedParams?: string[]
}

export class ToolSchemaEnrichmentError extends Error {
constructor(toolId: string, cause: unknown) {
super(`Failed to enrich schema for tool "${toolId}"`, { cause })
this.name = 'ToolSchemaEnrichmentError'
}
}

export interface ValidationResult {
valid: boolean
missingParams: string[]
Expand Down Expand Up @@ -630,7 +638,8 @@ export function createUserToolSchema(

export async function createLLMToolSchema(
toolConfig: ToolConfig,
userProvidedParams: Record<string, unknown>
userProvidedParams: Record<string, unknown>,
enrichmentContext: WorkflowToolExecutionContext = {}
): Promise<LLMToolSchemaResult> {
const schema: ToolSchema = {
type: 'object',
Expand Down Expand Up @@ -704,11 +713,17 @@ export async function createLLMToolSchema(
if (toolConfig.toolEnrichment) {
const dependencyValue = userProvidedParams[toolConfig.toolEnrichment.dependsOn] as string
if (dependencyValue) {
const enriched = await toolConfig.toolEnrichment.enrichTool(
dependencyValue,
schema,
toolConfig.description
)
let enriched
try {
enriched = await toolConfig.toolEnrichment.enrichTool(
dependencyValue,
schema,
toolConfig.description,
enrichmentContext
)
} catch (error) {
throw new ToolSchemaEnrichmentError(toolConfig.id, error)
}
if (enriched) {
return {
schema: enriched.parameters as ToolSchema,
Expand Down
Loading
Loading