diff --git a/backend/agents/event_agent.go b/backend/agents/event_agent.go index 831976a..f275a6f 100644 --- a/backend/agents/event_agent.go +++ b/backend/agents/event_agent.go @@ -154,7 +154,7 @@ func NewEventAgent(log *log.Logger, eventsModel models.EventModel, locationModel EndToolCall: "stopAgent", }) - locationAgent := NewLocationAgent(log.WithPrefix("Events 📅 > Locations 📍"), locationModel) + locationAgent := NewLocationAgentWithComm(log.WithPrefix("Events 📅 > Locations 📍"), locationModel) locationQuery := "Can you get me the ID of the location present in this image?" locationAgent.Options.Query = &locationQuery diff --git a/backend/agents/location_agent.go b/backend/agents/location_agent.go index 52ee305..7069ff7 100644 --- a/backend/agents/location_agent.go +++ b/backend/agents/location_agent.go @@ -3,6 +3,7 @@ package agents import ( "context" "encoding/json" + "fmt" "screenmark/screenmark/.gen/haystack/haystack/model" "screenmark/screenmark/agents/client" "screenmark/screenmark/models" @@ -14,37 +15,58 @@ import ( const locationPrompt = ` Role: Location AI Assistant -Objective: Identify locations from images/text, manage a saved list, and answer user queries about saved locations using the provided tools. +Objective: Identify locations from images/text, manage a saved list (create, update), and answer user queries about saved locations using the provided tools. Core Logic: -1. **Analyze Input:** Look for location details (Name, Address) in the image and check for any user query about a location. +**Handle Image/Text Location (if no query was handled in Step 2):** + * If location details (InputName, InputAddress, etc.) were successfully extracted from the input in Step 1: + * Use listLocations to check if a location matching InputName or InputAddress already exists in the saved list. + * **If *no match*** is found: + * Use createLocation, providing the extracted InputName (required) and any other details like InputAddress. + * *(Proceed to step 5)* + * **If a *match*** is found (meaning it's a potential duplicate/update candidate - let the matching saved location have ExistingLocationId): + * Use updateLocation. Provide the locationId = ExistingLocationId. + * Also provide any *new or potentially refined* details extracted from the current input (e.g., name = InputName, address = InputAddress). The updateLocation tool should handle updating the record with these details and/or linking the new input context (like the image) to this existing location. + * *(Proceed to step 5)* + * If no location details could be extracted from the input in Step 1. + * *(Proceed to step 5)* -2. **Handle User Query First:** - * If the user asks about a *specific* location: - * Use listLocations to find its locationId. - * If found, use reply with the locationId. - * If not found, prepare summary (no matching location). - * *(Proceed to step 4)* - -3. **Handle Image Location (if no query was handled):** - * If location details were found in the image: - * Use listLocations to check if it's already saved. - * If *new*, use createLocation (Name is required). - * If *duplicate*, prepare summary (location already exists). - * *(Proceed to step 4)* - -4. **Summarize & Stop:** Always finish by writing a message explaining what you did (e.g., called reply, called createLocation, found a duplicate, couldn't find a match) or if no location information was found. After providing the summary message, call stopAgent to signal the end of processing for this turn. +**Summarize & Stop:** Always finish by writing a concise message explaining the outcome of the turn. Examples: + * "Okay, I've answered your query about [Location Name]." (after calling reply) + * "I couldn't find [Queried Location Name] in my saved list." (after failing to find a match for a query) + * "I've saved '[InputName]' as a new location." (after calling createLocation) + * "I found that '[InputName]' was already saved, so I've updated its information/context based on your latest input." (after calling updateLocation) + * "I couldn't identify a specific location from your input." (if Step 1 failed or no action was taken) + * After providing the summary message, call stopAgent to signal the end of processing for this turn. Tool Usage: -* listLocations: Check saved locations (for queries or before saving). -* createLocation: Save a *new* location (requires Name). -* reply: Answer a query about a *known*, *saved* location using its locationId. +* listLocations: Check saved locations. Used to find matches for user queries or to detect existing entries before creating/updating. Returns matching location(s) including their locationId. +* createLocation: Save a *new* location. Requires name, can include address, etc. +* updateLocation: Update an *existing* location. Requires locationId. Can include name, address, etc., to update specific fields or simply to associate the new input context with the existing location. * stopAgent: Signals the end of the agent's processing for the current turn. Call this *after* providing the summary message. -* **Constraint:** Typically, only one main action tool (listLocations, createLocation, or reply) will be called per turn before summarizing and stopping. listLocations might precede createLocation or reply within the logic. ` +const replyTool = ` +{ + "type": "function", + "function": { + "name": "reply", + "description": "Signals intent to provide information about a specific known location in response to a user's query. Use only if the user asked a question and the location's ID was found via listLocations.", + "parameters": { + "type": "object", + "properties": { + "locationId": { + "type": "string", + "description": "The unique identifier of the saved location that the user is asking about." + } + }, + "required": ["locationId"] + } + } +},` + const locationTools = ` [ { @@ -83,20 +105,21 @@ const locationTools = ` { "type": "function", "function": { - "name": "reply", - "description": "Signals intent to provide information about a specific known location in response to a user's query. Use only if the user asked a question and the location's ID was found via listLocations.", + "name": "updateLocation", + "description": "Updates an existing saved location identified by its locationId. Use when input matches a pre-existing location. Pass locationId and any new details (name, address) to update.", "parameters": { "type": "object", "properties": { "locationId": { "type": "string", - "description": "The unique identifier of the saved location that the user is asking about." + "description": "The UUID of the location you are trying to update" } }, "required": ["locationId"] } } }, + %s { "type": "function", "function": { @@ -111,19 +134,35 @@ const locationTools = ` } ]` +func getLocationAgentTools(allowReply bool) string { + if allowReply { + return fmt.Sprintf(locationTools, replyTool) + } else { + return fmt.Sprintf(locationTools, "") + } +} + type listLocationArguments struct{} type createLocationArguments struct { Name string `json:"name"` Address *string `json:"address"` } -type linkLocationArguments struct { +type updateLocationArguments struct { LocationID string `json:"locationId"` } +func NewLocationAgentWithComm(log *log.Logger, locationModel models.LocationModel) client.AgentClient { + client := NewLocationAgent(log, locationModel) + + client.Options.JsonTools = getLocationAgentTools(true) + + return client +} + func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) client.AgentClient { agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{ SystemPrompt: locationPrompt, - JsonTools: locationTools, + JsonTools: getLocationAgentTools(false), Log: log, EndToolCall: "stopAgent", }) @@ -158,8 +197,8 @@ func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) clien return location, nil }) - agentClient.ToolHandler.AddTool("linkLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) { - args := linkLocationArguments{} + agentClient.ToolHandler.AddTool("updateLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) { + args := updateLocationArguments{} err := json.Unmarshal([]byte(_args), &args) if err != nil { return "", err @@ -167,12 +206,12 @@ func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) clien ctx := context.Background() - contactUuid, err := uuid.Parse(args.LocationID) + locationId, err := uuid.Parse(args.LocationID) if err != nil { return "", err } - locationModel.SaveToImage(ctx, info.ImageId, contactUuid) + locationModel.SaveToImage(ctx, info.ImageId, locationId) return "Saved", nil }) diff --git a/backend/agents/orchestrator.go b/backend/agents/orchestrator.go index f399216..13d0df2 100644 --- a/backend/agents/orchestrator.go +++ b/backend/agents/orchestrator.go @@ -125,7 +125,7 @@ func NewOrchestratorAgent(log *log.Logger, noteAgent NoteAgent, contactAgent cli }) agent.ToolHandler.AddTool("noteAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) { - // go noteAgent.GetNotes(info.UserId, info.ImageId, imageName, imageData) + go noteAgent.GetNotes(info.UserId, info.ImageId, imageName, imageData) return "noteAgent called successfully", nil })