From e706b6a9760963fa4d55526550b85e48bd87c548 Mon Sep 17 00:00:00 2001 From: John Costa Date: Fri, 18 Apr 2025 15:32:07 +0100 Subject: [PATCH] feat(location): correctly updating an image if it contains a duplicate locatino --- backend/agents/client/client.go | 2 +- backend/agents/location_agent.go | 74 ++++++++++++++++++++++++++------ 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/backend/agents/client/client.go b/backend/agents/client/client.go index ed260d0..5f1b322 100644 --- a/backend/agents/client/client.go +++ b/backend/agents/client/client.go @@ -242,7 +242,7 @@ func (client *AgentClient) RunAgent(userId uuid.UUID, imageId uuid.UUID, imageNa request := AgentRequestBody{ Tools: &tools, ToolChoice: &toolChoice, - Model: "pixtral-12b-2409", + Model: "pixtral-large-latest", Temperature: 0.3, EndToolCall: client.Options.EndToolCall, ResponseFormat: ResponseFormat{ diff --git a/backend/agents/location_agent.go b/backend/agents/location_agent.go index dcc972d..640d7bc 100644 --- a/backend/agents/location_agent.go +++ b/backend/agents/location_agent.go @@ -9,6 +9,7 @@ import ( "screenmark/screenmark/models" "github.com/charmbracelet/log" + "github.com/google/uuid" ) const locationPrompt = ` @@ -25,14 +26,21 @@ Core Logic: **Check for Existing Location:** If details *were* extracted: * Use listLocations with the extracted InputName and/or InputAddress to search for potentially matching locations already saved in the list. - * If you find an existing location, you shouldn't create a duplicate. Call stopAgent when this happens. + +Action loop: +**Thinking** + * Use the think tool to analytise the image. + * You should think about whether listLocations already contains this location, or if it is a new location. + * You should always call this after listLocations. + * You must think about whether or not listLocations already has this location. **Decide Action based on Search Results:** - * If no existing location looks like the location on the input. You should use doesLocationExist to think about whether or not this location is a duplicate. - * If you determine it is not a duplicate, then use createLocation to create a new location for the user. - * Else, you should call stopAgent. - * If the image does not contain any location, you should use stopAgent. - * You should repeat this loop of doesLocationExist and createLocation until you've completed all locations on the image. + * If no existing location looks like the location on the input. You should use createLocation. + * Do not use this tool if this location already exists. + * If the input contains a location that already exists, you should use createExistingLocation. + +You should repeat the action loop until all locations on the image are done. +Once you are done, use stopAgent. **Reply to user querys** * If the user asks you a specific question, you should use the reply tool to reply to them. @@ -62,17 +70,17 @@ const locationTools = ` { "type": "function", "function": { - "name": "doesLocationExist", - "description": "", + "name": "think", + "description": "Use this tool to think through the image, evaluating the location and whether or not it exists in the users listLocations.", "parameters": { "type": "object", "properties": { - "rationale": { + "thought": { "type": "string", - "description": "Your reasoning as to whether or not this image contains a location that already exists in listLocations" + "description": "A singular thought about the image" } }, - "required": ["rationale"] + "required": ["thought"] } } }, @@ -108,6 +116,23 @@ const locationTools = ` "required": ["name"] } } + }, + { + "type": "function", + "function": { + "name": "createExistingLocation", + "description": "Called when a location already exists in the users list, from listLocations. Only call this to indicate this image contains a duplicate. And only after using the doesLocationExist tol", + "parameters": { + "type": "object", + "properties": { + "locationId": { + "type": "string", + "description": "The UUID of the location, from listLocations" + } + }, + "required": ["locationId"] + } + } }, %s { @@ -137,6 +162,9 @@ type createLocationArguments struct { Name string `json:"name"` Address *string `json:"address"` } +type createExistingLocationArguments struct { + LocationID string `json:"locationId"` +} func NewLocationAgentWithComm(log *log.Logger, locationModel models.LocationModel) client.AgentClient { client := NewLocationAgent(log, locationModel) @@ -186,11 +214,33 @@ func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) clien return location, nil }) + agentClient.ToolHandler.AddTool("createExistingLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) { + args := createExistingLocationArguments{} + err := json.Unmarshal([]byte(_args), &args) + if err != nil { + return "", err + } + + ctx := context.Background() + + locationId, err := uuid.Parse(args.LocationID) + if err != nil { + return "", err + } + + _, err = locationModel.SaveToImage(ctx, info.ImageId, locationId) + if err != nil { + return "", err + } + + return "", nil + }) + agentClient.ToolHandler.AddTool("reply", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) { return "ok", nil }) - agentClient.ToolHandler.AddTool("doesLocationExist", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) { + agentClient.ToolHandler.AddTool("think", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) { return "ok", nil })