From b046a928b09d025c5f4436181d47cf2630fa87d9 Mon Sep 17 00:00:00 2001 From: John Costa Date: Sat, 3 May 2025 18:07:37 +0100 Subject: [PATCH] feat: using gpt-4.1-mini feat: createExistingContact feat: using nano instead of mini so I don't run out of money instantly --- backend/agents/client/chat.go | 13 +++++-- backend/agents/client/client.go | 10 +++--- backend/agents/contact_agent.go | 45 ++++++++++++++++++++++- backend/agents/event_agent.go | 61 ++++++++++++++++++++------------ backend/agents/location_agent.go | 4 +-- backend/agents/note_agent.go | 2 +- 6 files changed, 101 insertions(+), 34 deletions(-) diff --git a/backend/agents/client/chat.go b/backend/agents/client/chat.go index 36909c5..6f8cadd 100644 --- a/backend/agents/client/chat.go +++ b/backend/agents/client/chat.go @@ -142,8 +142,12 @@ func (m TextMessageContent) IsImageMessage() bool { } type ImageMessageContent struct { - ImageType string `json:"type"` - ImageUrl string `json:"image_url"` + ImageType string `json:"type"` + ImageUrl ImageMessageUrl `json:"image_url"` +} + +type ImageMessageUrl struct { + Url string `json:"url"` } func (m ImageMessageContent) IsImageMessage() bool { @@ -161,6 +165,7 @@ type ImageContentUrl struct { type ToolCall struct { Index int `json:"index"` Id string `json:"id"` + Type string `json:"type,omitzero"` Function FunctionCall `json:"function"` } @@ -213,7 +218,9 @@ func (chat *Chat) AddImage(imageName string, image []byte, query *string) error messageContent.Content[index] = ImageMessageContent{ ImageType: "image_url", - ImageUrl: fmt.Sprintf("data:image/%s;base64,%s", extension, encodedString), + ImageUrl: ImageMessageUrl{ + Url: fmt.Sprintf("data:image/%s;base64,%s", extension, encodedString), + }, } arrayMessage := ChatUserMessage{Role: User, MessageContent: messageContent} diff --git a/backend/agents/client/client.go b/backend/agents/client/client.go index 1158abb..e817e7a 100644 --- a/backend/agents/client/client.go +++ b/backend/agents/client/client.go @@ -14,7 +14,7 @@ import ( type ResponseFormat struct { Type string `json:"type"` - JsonSchema any `json:"json_schema"` + JsonSchema any `json:"json_schema,omitzero"` } type AgentRequestBody struct { @@ -82,7 +82,7 @@ type AgentClient struct { Options CreateAgentClientOptions } -const OPENAI_API_KEY = "OPENAI_API_KEY" +const OPENAI_API_KEY = "REAL_OPEN_AI_KEY" type CreateAgentClientOptions struct { Log *log.Logger @@ -101,7 +101,7 @@ func CreateAgentClient(options CreateAgentClientOptions) AgentClient { return AgentClient{ apiKey: apiKey, - url: "https://api.mistral.ai/v1/chat/completions", + url: "https://api.openai.com/v1/chat/completions", Do: func(req *http.Request) (*http.Response, error) { client := &http.Client{} return client.Do(req) @@ -239,13 +239,13 @@ func (client *AgentClient) RunAgent(userId uuid.UUID, imageId uuid.UUID, imageNa panic(err) } - toolChoice := "any" + toolChoice := "auto" seed := 42 request := AgentRequestBody{ Tools: &tools, ToolChoice: &toolChoice, - Model: "pixtral-12b-2409", + Model: "gpt-4.1-nano", RandomSeed: &seed, Temperature: 0.3, EndToolCall: client.Options.EndToolCall, diff --git a/backend/agents/contact_agent.go b/backend/agents/contact_agent.go index a57d286..bf9e6eb 100644 --- a/backend/agents/contact_agent.go +++ b/backend/agents/contact_agent.go @@ -21,13 +21,14 @@ const contactPrompt = ` **Workflow:** 1. **Scan Image:** Extract all contact details. If none, call stopAgent. 2. **Think:** Using the think tool, you must layout your thoughts about the contacts on the image. If they are duplicates or not, and what your next action should be, -3. **Check Duplicates:** If contacts found, *first* call listContacts. Compare extracted info to list. If all found contacts already exist, call stopAgent. +3. **Check Duplicates:** If contacts found, *first* call listContacts. Compare extracted info to list. If all found contacts already exist, use createExistingContact. 4. **Add New:** If you detect a new contact on the image, call createContact to create a new contact. 5. **Finish:** Call stopAgent once all new contacts are created OR if steps 1 or 2 determined no action/creation was needed. **Tools:** * listContacts: Check existing contacts (Use first if contacts found). * createContact: Add a NEW contact (Name required). +* createExistingContact: Adds this image to an existing contact, if one is found in listContacts. * stopAgent: Signal task completion. ` @@ -95,6 +96,23 @@ const contactTools = ` } } }, + { + "type": "function", + "function": { + "name": "createExistingContact", + "description": "Called when a contact already exists in the users list, from listContas. Only call this to indicate this image contains a duplicate.", + "parameters": { + "type": "object", + "properties": { + "contactId": { + "type": "string", + "description": "The UUID of the contact" + } + }, + "required": ["contactId"] + } + } + }, { "type": "function", "function": { @@ -118,6 +136,9 @@ type createContactsArguments struct { Address *string `json:"address"` Email *string `json:"email"` } +type createExistingContactArguments struct { + ContactID string `json:"contactId"` +} func NewContactAgent(log *log.Logger, contactModel models.ContactModel) client.AgentClient { agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{ @@ -173,5 +194,27 @@ func NewContactAgent(log *log.Logger, contactModel models.ContactModel) client.A return contact, nil }) + agentClient.ToolHandler.AddTool("createExistingContact", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) { + args := createExistingContactArguments{} + err := json.Unmarshal([]byte(_args), &args) + if err != nil { + return "", err + } + + ctx := context.Background() + + contactId, err := uuid.Parse(args.ContactID) + if err != nil { + return "", err + } + + _, err = contactModel.SaveToImage(ctx, info.ImageId, contactId) + if err != nil { + return "", err + } + + return "", nil + }) + return agentClient } diff --git a/backend/agents/event_agent.go b/backend/agents/event_agent.go index f0bc682..4b3a023 100644 --- a/backend/agents/event_agent.go +++ b/backend/agents/event_agent.go @@ -153,6 +153,43 @@ type updateEventArguments struct { EventID string `json:"eventId"` } +const layout = "2006-01-02T15:04:05Z" + +func getArguments(args createEventArguments) (model.Events, error) { + event := model.Events{ + Name: args.Name, + } + + if args.StartDateTime != nil { + startTime, err := time.Parse(layout, *args.StartDateTime) + if err != nil { + return event, err + } + + event.StartDateTime = &startTime + } + + if args.EndDateTime != nil { + endTime, err := time.Parse(layout, *args.EndDateTime) + if err != nil { + return event, err + } + + event.EndDateTime = &endTime + } + + if args.LocationID != nil { + locationId, err := uuid.Parse(*args.LocationID) + if err != nil { + return model.Events{}, err + } + + event.LocationID = &locationId + } + + return event, nil +} + func NewEventAgent(log *log.Logger, eventsModel models.EventModel, locationModel models.LocationModel) client.AgentClient { agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{ SystemPrompt: eventPrompt, @@ -181,32 +218,12 @@ func NewEventAgent(log *log.Logger, eventsModel models.EventModel, locationModel } ctx := context.Background() - - layout := "2006-01-02T15:04:05Z" - - // TODO: check for nil pointers. - - startTime, err := time.Parse(layout, *args.StartDateTime) + event, err := getArguments(args) if err != nil { return model.Events{}, err } - endTime, err := time.Parse(layout, *args.EndDateTime) - if err != nil { - return model.Events{}, err - } - - locationId, err := uuid.Parse(*args.LocationID) - if err != nil { - return model.Events{}, err - } - - events, err := eventsModel.Save(ctx, info.UserId, model.Events{ - Name: args.Name, - StartDateTime: &startTime, - EndDateTime: &endTime, - LocationID: &locationId, - }) + events, err := eventsModel.Save(ctx, info.UserId, event) if err != nil { return model.Events{}, err diff --git a/backend/agents/location_agent.go b/backend/agents/location_agent.go index ccaacae..bddec6e 100644 --- a/backend/agents/location_agent.go +++ b/backend/agents/location_agent.go @@ -57,7 +57,7 @@ const replyTool = ` "properties": { "locationId": { "type": "string", - "description": "The unique identifier of the saved location that the user is asking about." + "description": "The UUID of the saved location that the user is asking about." } }, "required": ["locationId"] @@ -121,7 +121,7 @@ const locationTools = ` "type": "function", "function": { "name": "createExistingLocation", - "description": "Called when a location already exists in the users list, from listLocations. Only call this to indicate this image contains a duplicate. And only after using the doesLocationExist tol", + "description": "Called when a location already exists in the users list, from listLocations. Only call this to indicate this image contains a duplicate.", "parameters": { "type": "object", "properties": { diff --git a/backend/agents/note_agent.go b/backend/agents/note_agent.go index 3b06c59..a0555ec 100644 --- a/backend/agents/note_agent.go +++ b/backend/agents/note_agent.go @@ -30,7 +30,7 @@ type NoteAgent struct { func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error { request := client.AgentRequestBody{ - Model: "pixtral-12b-2409", + Model: "gpt-4.1-nano", Temperature: 0.3, ResponseFormat: client.ResponseFormat{ Type: "text",