JohnCosta27/Haystack
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: using gpt-4.1-mini

Browse Source 
feat: createExistingContact

feat: using nano instead of mini so I don't run out of money instantly
This commit is contained in:
John Costa
2025-05-03 18:07:37 +01:00
parent 9860dd2dc5
commit b046a928b0
6 changed files with 101 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/agents/client/chat.go
View File

@ -142,8 +142,12 @@ func (m TextMessageContent) IsImageMessage() bool {
}
type ImageMessageContent struct {
ImageType string `json:"type"`
ImageUrl string `json:"image_url"`
ImageType string `json:"type"`
ImageUrl ImageMessageUrl `json:"image_url"`
}
type ImageMessageUrl struct {
Url string `json:"url"`
}
func (m ImageMessageContent) IsImageMessage() bool {
@ -161,6 +165,7 @@ type ImageContentUrl struct {
type ToolCall struct {
Index int `json:"index"`
Id string `json:"id"`
Type string `json:"type,omitzero"`
Function FunctionCall `json:"function"`
}
@ -213,7 +218,9 @@ func (chat *Chat) AddImage(imageName string, image []byte, query *string) error
messageContent.Content[index] = ImageMessageContent{
ImageType: "image_url",
ImageUrl: fmt.Sprintf("data:image/%s;base64,%s", extension, encodedString),
ImageUrl: ImageMessageUrl{
Url: fmt.Sprintf("data:image/%s;base64,%s", extension, encodedString),
},
}
arrayMessage := ChatUserMessage{Role: User, MessageContent: messageContent}

10
backend/agents/client/client.go
View File

@ -14,7 +14,7 @@ import (
type ResponseFormat struct {
Type string `json:"type"`
JsonSchema any `json:"json_schema"`
JsonSchema any `json:"json_schema,omitzero"`
}
type AgentRequestBody struct {
@ -82,7 +82,7 @@ type AgentClient struct {
Options CreateAgentClientOptions
}
const OPENAI_API_KEY = "OPENAI_API_KEY"
const OPENAI_API_KEY = "REAL_OPEN_AI_KEY"
type CreateAgentClientOptions struct {
Log *log.Logger
@ -101,7 +101,7 @@ func CreateAgentClient(options CreateAgentClientOptions) AgentClient {
return AgentClient{
apiKey: apiKey,
url: "https://api.mistral.ai/v1/chat/completions",
url: "https://api.openai.com/v1/chat/completions",
Do: func(req *http.Request) (*http.Response, error) {
client := &http.Client{}
return client.Do(req)
@ -239,13 +239,13 @@ func (client *AgentClient) RunAgent(userId uuid.UUID, imageId uuid.UUID, imageNa
panic(err)
}
toolChoice := "any"
toolChoice := "auto"
seed := 42
request := AgentRequestBody{
Tools: &tools,
ToolChoice: &toolChoice,
Model: "pixtral-12b-2409",
Model: "gpt-4.1-nano",
RandomSeed: &seed,
Temperature: 0.3,
EndToolCall: client.Options.EndToolCall,

45
backend/agents/contact_agent.go
View File

@ -21,13 +21,14 @@ const contactPrompt = `
**Workflow:**
1. **Scan Image:** Extract all contact details. If none, call stopAgent.
2. **Think:** Using the think tool, you must layout your thoughts about the contacts on the image. If they are duplicates or not, and what your next action should be,
3. **Check Duplicates:** If contacts found, *first* call listContacts. Compare extracted info to list. If all found contacts already exist, call stopAgent.
3. **Check Duplicates:** If contacts found, *first* call listContacts. Compare extracted info to list. If all found contacts already exist, use createExistingContact.
4. **Add New:** If you detect a new contact on the image, call createContact to create a new contact.
5. **Finish:** Call stopAgent once all new contacts are created OR if steps 1 or 2 determined no action/creation was needed.
**Tools:**
* listContacts: Check existing contacts (Use first if contacts found).
* createContact: Add a NEW contact (Name required).
* createExistingContact: Adds this image to an existing contact, if one is found in listContacts.
* stopAgent: Signal task completion.
`
@ -95,6 +96,23 @@ const contactTools = `
}
}
},
{
"type": "function",
"function": {
"name": "createExistingContact",
"description": "Called when a contact already exists in the users list, from listContas. Only call this to indicate this image contains a duplicate.",
"parameters": {
"type": "object",
"properties": {
"contactId": {
"type": "string",
"description": "The UUID of the contact"
}
},
"required": ["contactId"]
}
}
},
{
"type": "function",
"function": {
@ -118,6 +136,9 @@ type createContactsArguments struct {
Address *string `json:"address"`
Email *string `json:"email"`
}
type createExistingContactArguments struct {
ContactID string `json:"contactId"`
}
func NewContactAgent(log *log.Logger, contactModel models.ContactModel) client.AgentClient {
agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{
@ -173,5 +194,27 @@ func NewContactAgent(log *log.Logger, contactModel models.ContactModel) client.A
return contact, nil
})
agentClient.ToolHandler.AddTool("createExistingContact", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
args := createExistingContactArguments{}
err := json.Unmarshal([]byte(_args), &args)
if err != nil {
return "", err
}
ctx := context.Background()
contactId, err := uuid.Parse(args.ContactID)
if err != nil {
return "", err
}
_, err = contactModel.SaveToImage(ctx, info.ImageId, contactId)
if err != nil {
return "", err
}
return "", nil
})
return agentClient
}

61
backend/agents/event_agent.go
View File

@ -153,6 +153,43 @@ type updateEventArguments struct {
EventID string `json:"eventId"`
}
const layout = "2006-01-02T15:04:05Z"
func getArguments(args createEventArguments) (model.Events, error) {
event := model.Events{
Name: args.Name,
}
if args.StartDateTime != nil {
startTime, err := time.Parse(layout, *args.StartDateTime)
if err != nil {
return event, err
}
event.StartDateTime = &startTime
}
if args.EndDateTime != nil {
endTime, err := time.Parse(layout, *args.EndDateTime)
if err != nil {
return event, err
}
event.EndDateTime = &endTime
}
if args.LocationID != nil {
locationId, err := uuid.Parse(*args.LocationID)
if err != nil {
return model.Events{}, err
}
event.LocationID = &locationId
}
return event, nil
}
func NewEventAgent(log *log.Logger, eventsModel models.EventModel, locationModel models.LocationModel) client.AgentClient {
agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{
SystemPrompt: eventPrompt,
@ -181,32 +218,12 @@ func NewEventAgent(log *log.Logger, eventsModel models.EventModel, locationModel
}
ctx := context.Background()
layout := "2006-01-02T15:04:05Z"
// TODO: check for nil pointers.
startTime, err := time.Parse(layout, *args.StartDateTime)
event, err := getArguments(args)
if err != nil {
return model.Events{}, err
}
endTime, err := time.Parse(layout, *args.EndDateTime)
if err != nil {
return model.Events{}, err
}
locationId, err := uuid.Parse(*args.LocationID)
if err != nil {
return model.Events{}, err
}
events, err := eventsModel.Save(ctx, info.UserId, model.Events{
Name: args.Name,
StartDateTime: &startTime,
EndDateTime: &endTime,
LocationID: &locationId,
})
events, err := eventsModel.Save(ctx, info.UserId, event)
if err != nil {
return model.Events{}, err

4
backend/agents/location_agent.go
View File

@ -57,7 +57,7 @@ const replyTool = `
"properties": {
"locationId": {
"type": "string",
"description": "The unique identifier of the saved location that the user is asking about."
"description": "The UUID of the saved location that the user is asking about."
}
},
"required": ["locationId"]
@ -121,7 +121,7 @@ const locationTools = `
"type": "function",
"function": {
"name": "createExistingLocation",
"description": "Called when a location already exists in the users list, from listLocations. Only call this to indicate this image contains a duplicate. And only after using the doesLocationExist tol",
"description": "Called when a location already exists in the users list, from listLocations. Only call this to indicate this image contains a duplicate.",
"parameters": {
"type": "object",
"properties": {

2
backend/agents/note_agent.go
View File

@ -30,7 +30,7 @@ type NoteAgent struct {
func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
request := client.AgentRequestBody{
Model: "pixtral-12b-2409",
Model: "gpt-4.1-nano",
Temperature: 0.3,
ResponseFormat: client.ResponseFormat{
Type: "text",