JohnCosta27/Haystack
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat(location-agent): seperating the tool to allow for replying

Browse Source 
This means it makes less mistakes and doesnt get as confused.
This commit is contained in:
John Costa
2025-04-17 18:09:00 +01:00
parent 7002b05aae
commit 2b7206c29e
3 changed files with 71 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/agents/event_agent.go
View File

@ -154,7 +154,7 @@ func NewEventAgent(log *log.Logger, eventsModel models.EventModel, locationModel
EndToolCall: "stopAgent",
})
locationAgent := NewLocationAgent(log.WithPrefix("Events 📅 > Locations 📍"), locationModel)
locationAgent := NewLocationAgentWithComm(log.WithPrefix("Events 📅 > Locations 📍"), locationModel)
locationQuery := "Can you get me the ID of the location present in this image?"
locationAgent.Options.Query = &locationQuery

99
backend/agents/location_agent.go
View File

@ -3,6 +3,7 @@ package agents
import (
"context"
"encoding/json"
"fmt"
"screenmark/screenmark/.gen/haystack/haystack/model"
"screenmark/screenmark/agents/client"
"screenmark/screenmark/models"
@ -14,37 +15,58 @@ import (
const locationPrompt = `
Role: Location AI Assistant
Objective: Identify locations from images/text, manage a saved list, and answer user queries about saved locations using the provided tools.
Objective: Identify locations from images/text, manage a saved list (create, update), and answer user queries about saved locations using the provided tools.
Core Logic:
1. **Analyze Input:** Look for location details (Name, Address) in the image and check for any user query about a location.
**Handle Image/Text Location (if no query was handled in Step 2):**
* If location details (InputName, InputAddress, etc.) were successfully extracted from the input in Step 1:
* Use listLocations to check if a location matching InputName or InputAddress already exists in the saved list.
* **If *no match*** is found:
* Use createLocation, providing the extracted InputName (required) and any other details like InputAddress.
* *(Proceed to step 5)*
* **If a *match*** is found (meaning it's a potential duplicate/update candidate - let the matching saved location have ExistingLocationId):
* Use updateLocation. Provide the locationId = ExistingLocationId.
* Also provide any *new or potentially refined* details extracted from the current input (e.g., name = InputName, address = InputAddress). The updateLocation tool should handle updating the record with these details and/or linking the new input context (like the image) to this existing location.
* *(Proceed to step 5)*
* If no location details could be extracted from the input in Step 1.
* *(Proceed to step 5)*
2. **Handle User Query First:**
* If the user asks about a *specific* location:
* Use listLocations to find its locationId.
* If found, use reply with the locationId.
* If not found, prepare summary (no matching location).
* *(Proceed to step 4)*
3. **Handle Image Location (if no query was handled):**
* If location details were found in the image:
* Use listLocations to check if it's already saved.
* If *new*, use createLocation (Name is required).
* If *duplicate*, prepare summary (location already exists).
* *(Proceed to step 4)*
4. **Summarize & Stop:** Always finish by writing a message explaining what you did (e.g., called reply, called createLocation, found a duplicate, couldn't find a match) or if no location information was found. After providing the summary message, call stopAgent to signal the end of processing for this turn.
**Summarize & Stop:** Always finish by writing a concise message explaining the outcome of the turn. Examples:
* "Okay, I've answered your query about [Location Name]." (after calling reply)
* "I couldn't find [Queried Location Name] in my saved list." (after failing to find a match for a query)
* "I've saved '[InputName]' as a new location." (after calling createLocation)
* "I found that '[InputName]' was already saved, so I've updated its information/context based on your latest input." (after calling updateLocation)
* "I couldn't identify a specific location from your input." (if Step 1 failed or no action was taken)
* After providing the summary message, call stopAgent to signal the end of processing for this turn.
Tool Usage:
* listLocations: Check saved locations (for queries or before saving).
* createLocation: Save a *new* location (requires Name).
* reply: Answer a query about a *known*, *saved* location using its locationId.
* listLocations: Check saved locations. Used to find matches for user queries or to detect existing entries before creating/updating. Returns matching location(s) including their locationId.
* createLocation: Save a *new* location. Requires name, can include address, etc.
* updateLocation: Update an *existing* location. Requires locationId. Can include name, address, etc., to update specific fields or simply to associate the new input context with the existing location.
* stopAgent: Signals the end of the agent's processing for the current turn. Call this *after* providing the summary message.
* **Constraint:** Typically, only one main action tool (listLocations, createLocation, or reply) will be called per turn before summarizing and stopping. listLocations might precede createLocation or reply within the logic.
`
const replyTool = `
{
"type": "function",
"function": {
"name": "reply",
"description": "Signals intent to provide information about a specific known location in response to a user's query. Use only if the user asked a question and the location's ID was found via listLocations.",
"parameters": {
"type": "object",
"properties": {
"locationId": {
"type": "string",
"description": "The unique identifier of the saved location that the user is asking about."
}
},
"required": ["locationId"]
}
}
},`
const locationTools = `
[
{
@ -83,20 +105,21 @@ const locationTools = `
{
"type": "function",
"function": {
"name": "reply",
"description": "Signals intent to provide information about a specific known location in response to a user's query. Use only if the user asked a question and the location's ID was found via listLocations.",
"name": "updateLocation",
"description": "Updates an existing saved location identified by its locationId. Use when input matches a pre-existing location. Pass locationId and any new details (name, address) to update.",
"parameters": {
"type": "object",
"properties": {
"locationId": {
"type": "string",
"description": "The unique identifier of the saved location that the user is asking about."
"description": "The UUID of the location you are trying to update"
}
},
"required": ["locationId"]
}
}
},
%s
{
"type": "function",
"function": {
@ -111,19 +134,35 @@ const locationTools = `
}
]`
func getLocationAgentTools(allowReply bool) string {
if allowReply {
return fmt.Sprintf(locationTools, replyTool)
} else {
return fmt.Sprintf(locationTools, "")
}
}
type listLocationArguments struct{}
type createLocationArguments struct {
Name string `json:"name"`
Address *string `json:"address"`
}
type linkLocationArguments struct {
type updateLocationArguments struct {
LocationID string `json:"locationId"`
}
func NewLocationAgentWithComm(log *log.Logger, locationModel models.LocationModel) client.AgentClient {
client := NewLocationAgent(log, locationModel)
client.Options.JsonTools = getLocationAgentTools(true)
return client
}
func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) client.AgentClient {
agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{
SystemPrompt: locationPrompt,
JsonTools: locationTools,
JsonTools: getLocationAgentTools(false),
Log: log,
EndToolCall: "stopAgent",
})
@ -158,8 +197,8 @@ func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) clien
return location, nil
})
agentClient.ToolHandler.AddTool("linkLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
args := linkLocationArguments{}
agentClient.ToolHandler.AddTool("updateLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
args := updateLocationArguments{}
err := json.Unmarshal([]byte(_args), &args)
if err != nil {
return "", err
@ -167,12 +206,12 @@ func NewLocationAgent(log *log.Logger, locationModel models.LocationModel) clien
ctx := context.Background()
contactUuid, err := uuid.Parse(args.LocationID)
locationId, err := uuid.Parse(args.LocationID)
if err != nil {
return "", err
}
locationModel.SaveToImage(ctx, info.ImageId, contactUuid)
locationModel.SaveToImage(ctx, info.ImageId, locationId)
return "Saved", nil
})

2
backend/agents/orchestrator.go
View File

@ -125,7 +125,7 @@ func NewOrchestratorAgent(log *log.Logger, noteAgent NoteAgent, contactAgent cli
})
agent.ToolHandler.AddTool("noteAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
// go noteAgent.GetNotes(info.UserId, info.ImageId, imageName, imageData)
go noteAgent.GetNotes(info.UserId, info.ImageId, imageName, imageData)
return "noteAgent called successfully", nil
})