JohnCosta27/Haystack
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

refactor: changing notes to be a simple image description

Browse Source 
Notes would generate too often and not be very useful. This is much
better.
This commit is contained in:
John Costa
2025-07-24 13:59:24 +01:00
parent 2ac996db73
commit 59bf884f5d
45 changed files with 102 additions and 1260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
backend/agents/note_agent.go → backend/agents/description_agent.go
View File

@@ -2,7 +2,6 @@ package agents
import (
"context"
"screenmark/screenmark/.gen/haystack/haystack/model"
"screenmark/screenmark/agents/client"
"screenmark/screenmark/models"
@@ -22,13 +21,13 @@ Do not return anything except markdown.
If the image contains code, add this inside code blocks. You must try and correctly guess the language too.
`
type NoteAgent struct {
type DescriptionAgent struct {
client client.AgentClient
noteModel models.NoteModel
imageModel models.ImageModel
}
func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
func (agent DescriptionAgent) Describe(imageId uuid.UUID, imageName string, imageData []byte) error {
request := client.AgentRequestBody{
Model: "gpt-4.1-nano",
Temperature: 0.3,
@@ -52,15 +51,7 @@ func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName s
markdown := resp.Choices[0].Message.Content
note, err := agent.noteModel.Save(ctx, userId, model.Notes{
Name: "the note", // TODO: add some json schema
Content: markdown,
})
if err != nil {
return err
}
_, err = agent.noteModel.SaveToImage(ctx, imageId, note.ID)
err = agent.imageModel.AddDescription(ctx, imageId, markdown)
if err != nil {
return err
}
@@ -68,15 +59,15 @@ func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName s
return nil
}
func NewNoteAgent(log *log.Logger, noteModel models.NoteModel) NoteAgent {
func NewDescriptionAgent(log *log.Logger, imageModel models.ImageModel) DescriptionAgent {
client := client.CreateAgentClient(client.CreateAgentClientOptions{
SystemPrompt: noteAgentPrompt,
Log: log,
})
agent := NoteAgent{
client: client,
noteModel: noteModel,
agent := DescriptionAgent{
client: client,
imageModel: imageModel,
}
return agent

8
backend/agents/orchestrator.go
View File

@@ -127,7 +127,7 @@ type Status struct {
Ok bool `json:"ok"`
}
func NewOrchestratorAgent(log *log.Logger, noteAgent NoteAgent, contactAgent client.AgentClient, locationAgent client.AgentClient, eventAgent client.AgentClient, imageName string, imageData []byte) client.AgentClient {
func NewOrchestratorAgent(log *log.Logger, contactAgent client.AgentClient, locationAgent client.AgentClient, eventAgent client.AgentClient, imageName string, imageData []byte) client.AgentClient {
agent := client.CreateAgentClient(client.CreateAgentClientOptions{
SystemPrompt: orchestratorPrompt,
JsonTools: orchestratorTools,
@@ -139,12 +139,6 @@ func NewOrchestratorAgent(log *log.Logger, noteAgent NoteAgent, contactAgent cli
return "Thought", nil
})
agent.ToolHandler.AddTool("noteAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
noteAgent.GetNotes(info.UserId, info.ImageId, imageName, imageData)
return "noteAgent called successfully", nil
})
agent.ToolHandler.AddTool("contactAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
contactAgent.RunAgent(info.UserId, info.ImageId, imageName, imageData)