chore: removing old agent that was messy and too coupled

chore

backend/.gen/haystack/haystack/enum/progress.go

@@ -0,0 +1,18 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package enum
+
+import "github.com/go-jet/jet/v2/postgres"
+
+var Progress = &struct {
+	NotStarted postgres.StringExpression
+	InProgress postgres.StringExpression
+}{
+	NotStarted: postgres.NewEnumValue("not-started"),
+	InProgress: postgres.NewEnumValue("in-progress"),
+}

backend/.gen/haystack/haystack/model/progress.go

@@ -0,0 +1,49 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import "errors"
+
+type Progress string
+
+const (
+	Progress_NotStarted Progress = "not-started"
+	Progress_InProgress Progress = "in-progress"
+)
+
+var ProgressAllValues = []Progress{
+	Progress_NotStarted,
+	Progress_InProgress,
+}
+
+func (e *Progress) Scan(value interface{}) error {
+	var enumValue string
+	switch val := value.(type) {
+	case string:
+		enumValue = val
+	case []byte:
+		enumValue = string(val)
+	default:
+		return errors.New("jet: Invalid scan value for AllTypesEnum enum. Enum value has to be of type string or []byte")
+	}
+
+	switch enumValue {
+	case "not-started":
+		*e = Progress_NotStarted
+	case "in-progress":
+		*e = Progress_InProgress
+	default:
+		return errors.New("jet: Invalid scan value '" + enumValue + "' for Progress enum")
+	}
+
+	return nil
+}
+
+func (e Progress) String() string {
+	return string(e)
+}

backend/.gen/haystack/haystack/model/user_images_to_process.go

@@ -13,6 +13,7 @@ import (
 
 type UserImagesToProcess struct {
 	ID      uuid.UUID `sql:"primary_key"`
+	Status  Progress
 	ImageID uuid.UUID
 	UserID  uuid.UUID
 }

backend/.gen/haystack/haystack/table/user_images_to_process.go

@@ -18,6 +18,7 @@ type userImagesToProcessTable struct {
 
 	// Columns
 	ID      postgres.ColumnString
+	Status  postgres.ColumnString
 	ImageID postgres.ColumnString
 	UserID  postgres.ColumnString
 
@@ -61,10 +62,11 @@ func newUserImagesToProcessTable(schemaName, tableName, alias string) *UserImage
 func newUserImagesToProcessTableImpl(schemaName, tableName, alias string) userImagesToProcessTable {
 	var (
 		IDColumn       = postgres.StringColumn("id")
+		StatusColumn   = postgres.StringColumn("status")
 		ImageIDColumn  = postgres.StringColumn("image_id")
 		UserIDColumn   = postgres.StringColumn("user_id")
-		allColumns     = postgres.ColumnList{IDColumn, ImageIDColumn, UserIDColumn}
-		mutableColumns = postgres.ColumnList{ImageIDColumn, UserIDColumn}
+		allColumns     = postgres.ColumnList{IDColumn, StatusColumn, ImageIDColumn, UserIDColumn}
+		mutableColumns = postgres.ColumnList{StatusColumn, ImageIDColumn, UserIDColumn}
 	)
 
 	return userImagesToProcessTable{
@@ -72,6 +74,7 @@ func newUserImagesToProcessTableImpl(schemaName, tableName, alias string) userIm
 
 		//Columns
 		ID:      IDColumn,
+		Status:  StatusColumn,
 		ImageID: ImageIDColumn,
 		UserID:  UserIDColumn,
 

backend/agents/client/client.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
+
+	"github.com/charmbracelet/log"
+	"github.com/google/uuid"
 )
 
 type ResponseFormat struct {
@@ -69,12 +71,14 @@ type AgentClient struct {
 
 	ToolHandler ToolsHandlers
 
+	Log *log.Logger
+
 	Do func(req *http.Request) (*http.Response, error)
 }
 
 const OPENAI_API_KEY = "OPENAI_API_KEY"
 
-func CreateAgentClient() (AgentClient, error) {
+func CreateAgentClient(log *log.Logger) (AgentClient, error) {
 	apiKey := os.Getenv(OPENAI_API_KEY)
 
 	if len(apiKey) == 0 {
@@ -89,6 +93,8 @@ func CreateAgentClient() (AgentClient, error) {
 			return client.Do(req)
 		},
 
+		Log: log,
+
 		ToolHandler: ToolsHandlers{
 			handlers: map[string]ToolHandler{},
 		},
@@ -128,8 +134,6 @@ func (client AgentClient) Request(req *AgentRequestBody) (AgentResponse, error)
 		return AgentResponse{}, err
 	}
 
-	fmt.Println(string(response))
-
 	agentResponse := AgentResponse{}
 	err = json.Unmarshal(response, &agentResponse)
 
@@ -138,9 +142,29 @@ func (client AgentClient) Request(req *AgentRequestBody) (AgentResponse, error)
 	}
 
 	if len(agentResponse.Choices) != 1 {
+		client.Log.Errorf("Received more than 1 choice from AI \n %s\n", string(response))
 		return AgentResponse{}, errors.New("Unsupported. We currently only accept 1 choice from AI.")
 	}
 
+	client.Log.SetLevel(log.DebugLevel)
+
+	msg := agentResponse.Choices[0].Message
+
+	if len(msg.Content) > 0 {
+		client.Log.Debugf("Content: %s", msg.Content)
+	}
+
+	if msg.ToolCalls != nil && len(*msg.ToolCalls) > 0 {
+		client.Log.Debugf("Tool Call: %s", (*msg.ToolCalls)[0].Function.Name)
+
+		prettyJson, err := json.MarshalIndent((*msg.ToolCalls)[0].Function.Arguments, "", "	")
+		if err != nil {
+			return AgentResponse{}, err
+		}
+
+		client.Log.Debugf("Arguments: %s", string(prettyJson))
+	}
+
 	req.Chat.AddAiResponse(agentResponse.Choices[0].Message)
 
 	return agentResponse, nil
@@ -187,8 +211,47 @@ func (client AgentClient) Process(info ToolHandlerInfo, req *AgentRequestBody) e
 
 		toolResponse := client.ToolHandler.Handle(info, toolCall)
 
+		client.Log.SetLevel(log.DebugLevel)
+		client.Log.Debugf("Response: %s", toolResponse.Content)
+
 		req.Chat.AddToolResponse(toolResponse)
 	}
 
 	return err
 }
+
+func (client AgentClient) RunAgent(systemPrompt string, jsonTools string, endToolCall string, userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
+	var tools any
+	err := json.Unmarshal([]byte(jsonTools), &tools)
+
+	toolChoice := "any"
+
+	request := AgentRequestBody{
+		Tools:       &tools,
+		ToolChoice:  &toolChoice,
+		Model:       "pixtral-12b-2409",
+		Temperature: 0.3,
+		EndToolCall: endToolCall,
+		ResponseFormat: ResponseFormat{
+			Type: "text",
+		},
+		Chat: &Chat{
+			Messages: make([]ChatMessage, 0),
+		},
+	}
+
+	request.Chat.AddSystem(systemPrompt)
+	request.Chat.AddImage(imageName, imageData)
+
+	_, err = client.Request(&request)
+	if err != nil {
+		return err
+	}
+
+	toolHandlerInfo := ToolHandlerInfo{
+		ImageId: imageId,
+		UserId:  userId,
+	}
+
+	return client.ToolLoop(toolHandlerInfo, &request)
+}

backend/agents/client/tools_test.go

@@ -2,8 +2,10 @@ package client
 
 import (
 	"errors"
+	"os"
 	"testing"
 
+	"github.com/charmbracelet/log"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/suite"
 )
@@ -28,6 +30,7 @@ func (suite *ToolTestSuite) SetupTest() {
 		return false, errors.New("I will always error")
 	})
 
+	suite.client.Log = log.New(os.Stdout)
 	suite.client.ToolHandler = suite.handler
 }
 

backend/agents/contact_agent.go

@@ -0,0 +1,188 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"screenmark/screenmark/.gen/haystack/haystack/model"
+	"screenmark/screenmark/agents/client"
+	"screenmark/screenmark/models"
+	"time"
+
+	"github.com/charmbracelet/log"
+	"github.com/google/uuid"
+)
+
+const contactPrompt = `
+You are an agent that performs actions on contacts and people you find on an image.
+
+You can use tools to achieve your task.
+
+You should use listContacts to make sure that you don't create duplicate contacts.
+
+Call createContact when you see there is a new contact on this image. Do not create duplicate contacts.
+Or call linkContact when you think this image contains an existing contact.
+
+Call finish if you dont think theres anything else to do.
+`
+
+const contactTools = `
+[
+    {
+        "type": "function",
+        "function": {
+            "name": "listContacts",
+            "description": "List the users existing contacts",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "createContact",
+            "description": "Creates a new contact",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string",
+						"description": "the name of the person"
+                    },
+                    "phoneNumber": {
+                        "type": "string"
+                    },
+                    "address": {
+                        "type": "string",
+						"description": "their physical address"
+                    },
+                    "email": {
+                        "type": "string"
+                    }
+                },
+                "required": ["name"]
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "linkContact",
+            "description": "Links an existing contact with this image",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "contactId": {
+                        "type": "string",
+						"description": "The UUID of the existing contact"
+                    }
+                },
+                "required": ["contactId"]
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "finish",
+            "description": "Call when you dont think theres anything to do",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    }
+]
+`
+
+type ContactAgent struct {
+	client client.AgentClient
+
+	contactModel models.ContactModel
+}
+
+type listContactsArguments struct{}
+type createContactsArguments struct {
+	Name        string  `json:"name"`
+	PhoneNumber *string `json:"phoneNumber"`
+	Address     *string `json:"address"`
+	Email       *string `json:"email"`
+}
+type linkContactArguments struct {
+	ContactID string `json:"contactId"`
+}
+
+func NewContactAgent(contactModel models.ContactModel) (ContactAgent, error) {
+	agentClient, err := client.CreateAgentClient(log.NewWithOptions(os.Stdout, log.Options{
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Contacts 👥",
+	}))
+	if err != nil {
+		return ContactAgent{}, err
+	}
+
+	agent := ContactAgent{
+		client:       agentClient,
+		contactModel: contactModel,
+	}
+
+	agentClient.ToolHandler.AddTool("listContacts", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+		return agent.contactModel.List(context.Background(), info.UserId)
+	})
+
+	agentClient.ToolHandler.AddTool("createContact", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
+		args := createContactsArguments{}
+		err := json.Unmarshal([]byte(_args), &args)
+		if err != nil {
+			return model.Contacts{}, err
+		}
+
+		ctx := context.Background()
+
+		contact, err := agent.contactModel.Save(ctx, info.UserId, model.Contacts{
+			Name:        args.Name,
+			PhoneNumber: args.PhoneNumber,
+			Email:       args.Email,
+		})
+
+		if err != nil {
+			return model.Contacts{}, err
+		}
+
+		_, err = agent.contactModel.SaveToImage(ctx, info.ImageId, contact.ID)
+		if err != nil {
+			return model.Contacts{}, err
+		}
+
+		return contact, nil
+	})
+
+	agentClient.ToolHandler.AddTool("linkContact", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
+		args := linkContactArguments{}
+		err := json.Unmarshal([]byte(_args), &args)
+		if err != nil {
+			return "", err
+		}
+
+		ctx := context.Background()
+
+		contactUuid, err := uuid.Parse(args.ContactID)
+		if err != nil {
+			return "", err
+		}
+
+		_, err = agent.contactModel.SaveToImage(ctx, info.ImageId, contactUuid)
+		if err != nil {
+			return "", err
+		}
+
+		return "Saved", nil
+	})
+
+	return agent, nil
+}

backend/agents/event_agent.go

@@ -0,0 +1,200 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"screenmark/screenmark/.gen/haystack/haystack/model"
+	"screenmark/screenmark/agents/client"
+	"screenmark/screenmark/models"
+	"time"
+
+	"github.com/charmbracelet/log"
+	"github.com/google/uuid"
+)
+
+const eventPrompt = `
+You are an agent.
+
+The user will send you images and you have to identify if they have any events or a place.
+This could be a friend suggesting to meet, a conference, or anything that looks like an event.
+
+There are various tools you can use to perform this task.
+
+listEvents
+Lists the users already existing events, you should do this before using createEvents to avoid creating duplicates. 
+
+createEvent
+Use this to create a new events.
+
+linkEvent
+Links an image to a events.
+
+finish
+Call when there is nothing else to do.
+`
+
+const eventTools = `
+[
+    {
+        "type": "function",
+        "function": {
+            "name": "listEvents",
+            "description": "List the events the user already has.",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "createEvent",
+            "description": "Use to create a new events",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "startDateTime": {
+                        "type": "string",
+                        "description": "The start time as an ISO string"
+                    },
+                    "endDateTime": {
+                        "type": "string",
+                        "description": "The end time as an ISO string"
+                    }
+				},
+                "required": ["name"]
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "linkEvent",
+            "description": "Use to link an already existing events to the image you were sent",
+            "parameters": {
+                "type": "object",
+                "properties": {
+					"eventId": {
+						"type": "string"
+					}
+				},
+                "required": ["eventsId"]
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "finish",
+            "description": "Call this when there is nothing left to do.",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    }
+]`
+
+type EventAgent struct {
+	client client.AgentClient
+
+	eventsModel models.EventModel
+}
+
+type listEventArguments struct{}
+type createEventArguments struct {
+	Name          string  `json:"name"`
+	StartDateTime *string `json:"startDateTime"`
+	EndDateTime   *string `json:"endDateTime"`
+	OrganizerName *string `json:"organizerName"`
+}
+type linkEventArguments struct {
+	EventID string `json:"eventId"`
+}
+
+func NewEventAgent(eventsModel models.EventModel) (EventAgent, error) {
+	agentClient, err := client.CreateAgentClient(log.NewWithOptions(os.Stdout, log.Options{
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Events 📍",
+	}))
+
+	if err != nil {
+		return EventAgent{}, err
+	}
+
+	agent := EventAgent{
+		client:      agentClient,
+		eventsModel: eventsModel,
+	}
+
+	agentClient.ToolHandler.AddTool("listEvents", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+		return agent.eventsModel.List(context.Background(), info.UserId)
+	})
+
+	agentClient.ToolHandler.AddTool("createEvent", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
+		args := createEventArguments{}
+		err := json.Unmarshal([]byte(_args), &args)
+		if err != nil {
+			return model.Events{}, err
+		}
+
+		ctx := context.Background()
+
+		layout := "2006-01-02T15:04:05Z"
+
+		startTime, err := time.Parse(layout, *args.StartDateTime)
+		if err != nil {
+			return model.Events{}, err
+		}
+
+		endTime, err := time.Parse(layout, *args.EndDateTime)
+		if err != nil {
+			return model.Events{}, err
+		}
+
+		events, err := agent.eventsModel.Save(ctx, info.UserId, model.Events{
+			Name:          args.Name,
+			StartDateTime: &startTime,
+			EndDateTime:   &endTime,
+		})
+
+		if err != nil {
+			return model.Events{}, err
+		}
+
+		_, err = agent.eventsModel.SaveToImage(ctx, info.ImageId, events.ID)
+		if err != nil {
+			return model.Events{}, err
+		}
+
+		return events, nil
+	})
+
+	agentClient.ToolHandler.AddTool("linkEvent", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
+		args := linkEventArguments{}
+		err := json.Unmarshal([]byte(_args), &args)
+		if err != nil {
+			return "", err
+		}
+
+		ctx := context.Background()
+
+		contactUuid, err := uuid.Parse(args.EventID)
+		if err != nil {
+			return "", err
+		}
+
+		agent.eventsModel.SaveToImage(ctx, info.ImageId, contactUuid)
+		return "Saved", nil
+	})
+
+	return agent, nil
+}

backend/agents/event_location_agent.go

@@ -1,295 +0,0 @@
-package agents
-
-import (
-	"context"
-	"encoding/json"
-	"screenmark/screenmark/.gen/haystack/haystack/model"
-	"screenmark/screenmark/agents/client"
-	"screenmark/screenmark/models"
-	"time"
-
-	"github.com/google/uuid"
-)
-
-// This prompt is probably shit.
-const eventLocationPrompt = `
-You are an agent that extracts events, locations, and organizers from an image. Your primary tasks are to identify and create locations and organizers before creating events. Follow these steps:
-
-Identify and Create Locations:
-
-Check if the image contains a location.
-If a location is found, check if it exists in the listLocations.
-If the location does not exist, create it first.
-Always reuse existing locations from listLocations to avoid duplicates.
-
-Identify and Create Events:
-
-Check if the image contains an event. An event should have a name and a date.
-If an event is found, ensure you have a location (from step 1) and an organizer (from step 2) before creating the event.
-Events must have an associated location and organizer. Do not create an event without these.
-If possible, return a start time and an end time as ISO datetime strings.
-Handling Images Without Events or Locations:
-
-It is possible that the image does not contain an event or a location. In such cases, do not create an event.
-Always prioritize the creation of locations and organizers before events. Ensure that all events have an associated location and organizer.
-`
-
-// TODO: this should be read directly from a file on load.
-const TOOLS = `
-[
-    {
-        "type": "function",
-        "function": {
-            "name": "createLocation",
-            "description": "Creates a location. No not use if you think an existing location is suitable!",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string"
-                    },
-                    "address": {
-                        "type": "string"
-                    }
-                },
-                "required": ["name"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "listLocations",
-            "description": "Lists the locations available",
-            "parameters": {
-                "type": "object",
-                "properties": {}
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "createEvent",
-            "description": "Creates a new event",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string"
-                    },
-                    "startDateTime": {
-                        "type": "string",
-                        "description": "The start time as an ISO string"
-                    },
-                    "endDateTime": {
-                        "type": "string",
-                        "description": "The end time as an ISO string"
-                    },
-                    "locationId": {
-                        "type": "string",
-                        "description": "The ID of the location, available by listLocations"
-                    },
-                    "organizerName": {
-                        "type": "string",
-                        "description": "The name of the organizer"
-                    }
-                },
-                "required": ["name"]
-            }
-        }
-    },
-	{
-		"type": "function",
-		"function": {
-			"name": "finish",
-			"description": "Nothing else to do. call this function.",
-			"parameters": {}
-		}
-	}
-]
-`
-
-type EventLocationAgent struct {
-	client client.AgentClient
-
-	eventModel    models.EventModel
-	locationModel models.LocationModel
-	contactModel  models.ContactModel
-
-	toolHandler client.ToolsHandlers
-}
-
-type ListLocationArguments struct{}
-type ListOrganizerArguments struct{}
-
-type CreateLocationArguments struct {
-	Name        string  `json:"name"`
-	Address     *string `json:"address,omitempty"`
-	Coordinates *string `json:"coordinates,omitempty"`
-}
-
-type CreateOrganizerArguments struct {
-	Name        string  `json:"name"`
-	PhoneNumber *string `json:"phoneNumber,omitempty"`
-	Email       *string `json:"email,omitempty"`
-}
-
-type AttachImageLocationArguments struct {
-	LocationId string `json:"locationId"`
-}
-
-type CreateEventArguments struct {
-	Name          string `json:"name"`
-	StartDateTime string `json:"startDateTime"`
-	EndDateTime   string `json:"endDateTime"`
-	LocationId    string `json:"locationId"`
-	OrganizerName string `json:"organizerName"`
-}
-
-func (agent EventLocationAgent) GetLocations(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
-	var tools any
-	err := json.Unmarshal([]byte(TOOLS), &tools)
-
-	toolChoice := "any"
-
-	request := client.AgentRequestBody{
-		Tools:       &tools,
-		ToolChoice:  &toolChoice,
-		Model:       "pixtral-12b-2409",
-		Temperature: 0.3,
-		EndToolCall: "finish",
-		ResponseFormat: client.ResponseFormat{
-			Type: "text",
-		},
-		Chat: &client.Chat{
-			Messages: make([]client.ChatMessage, 0),
-		},
-	}
-
-	request.Chat.AddSystem(eventLocationPrompt)
-	request.Chat.AddImage(imageName, imageData)
-
-	_, err = agent.client.Request(&request)
-	if err != nil {
-		return err
-	}
-
-	toolHandlerInfo := client.ToolHandlerInfo{
-		ImageId: imageId,
-		UserId:  userId,
-	}
-
-	return agent.client.ToolLoop(toolHandlerInfo, &request)
-}
-
-func NewLocationEventAgent(locationModel models.LocationModel, eventModel models.EventModel, contactModel models.ContactModel) (EventLocationAgent, error) {
-	agentClient, err := client.CreateAgentClient()
-	if err != nil {
-		return EventLocationAgent{}, err
-	}
-
-	agent := EventLocationAgent{
-		client:        agentClient,
-		locationModel: locationModel,
-		eventModel:    eventModel,
-		contactModel:  contactModel,
-	}
-
-	agentClient.ToolHandler.AddTool("listLocations",
-		func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
-			return agent.locationModel.List(context.Background(), info.UserId)
-		},
-	)
-
-	agentClient.ToolHandler.AddTool("createLocation",
-		func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
-			args := CreateLocationArguments{}
-			err := json.Unmarshal([]byte(_args), &args)
-			if err != nil {
-				return model.Locations{}, err
-			}
-
-			ctx := context.Background()
-
-			location, err := agent.locationModel.Save(ctx, info.UserId, model.Locations{
-				Name:    args.Name,
-				Address: args.Address,
-			})
-
-			if err != nil {
-				return location, err
-			}
-
-			_, err = agent.locationModel.SaveToImage(ctx, info.ImageId, location.ID)
-
-			return location, err
-		},
-	)
-
-	agentClient.ToolHandler.AddTool("createEvent",
-		func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
-			args := CreateEventArguments{}
-			err := json.Unmarshal([]byte(_args), &args)
-			if err != nil {
-				return model.Locations{}, err
-			}
-
-			ctx := context.Background()
-
-			layout := "2006-01-02T15:04:05Z"
-
-			startTime, err := time.Parse(layout, args.StartDateTime)
-			if err != nil {
-				return model.Events{}, err
-			}
-
-			endTime, err := time.Parse(layout, args.EndDateTime)
-			if err != nil {
-				return model.Events{}, err
-			}
-
-			event, err := agent.eventModel.Save(ctx, info.UserId, model.Events{
-				Name:          args.Name,
-				StartDateTime: &startTime,
-				EndDateTime:   &endTime,
-			})
-
-			if err != nil {
-				return event, err
-			}
-
-			organizer, err := agent.contactModel.Save(ctx, info.UserId, model.Contacts{
-				Name: args.Name,
-			})
-
-			if err != nil {
-				return event, err
-			}
-
-			_, err = agent.eventModel.SaveToImage(ctx, info.ImageId, event.ID)
-			if err != nil {
-				return event, err
-			}
-
-			_, err = agent.contactModel.SaveToImage(ctx, info.ImageId, organizer.ID)
-			if err != nil {
-				return event, err
-			}
-
-			locationId, err := uuid.Parse(args.LocationId)
-			if err != nil {
-				return event, err
-			}
-
-			event, err = agent.eventModel.UpdateLocation(ctx, event.ID, locationId)
-			if err != nil {
-				return event, err
-			}
-
-			return agent.eventModel.UpdateOrganizer(ctx, event.ID, organizer.ID)
-		},
-	)
-
-	return agent, nil
-}

backend/agents/location_agent.go

@@ -0,0 +1,179 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"screenmark/screenmark/.gen/haystack/haystack/model"
+	"screenmark/screenmark/agents/client"
+	"screenmark/screenmark/models"
+	"time"
+
+	"github.com/charmbracelet/log"
+	"github.com/google/uuid"
+)
+
+const locationPrompt = `
+You are an agent.
+
+The user will send you images and you have to identify if they have any location or a place. This could a picture of a real place, an address, or it's name.
+
+There are various tools you can use to perform this task.
+
+listLocations
+Lists the users already existing locations, you should do this before using createLocation to avoid creating duplicates. 
+
+createLocation
+Use this to create a new location. Avoid making duplicates and only create a new location if listLocations doesnt contain the location on the image.
+
+linkLocation
+Links an image to a location.
+
+finish
+Call when there is nothing else to do.
+`
+
+const locationTools = `
+[
+    {
+        "type": "function",
+        "function": {
+            "name": "listLocations",
+            "description": "List the locations the user already has.",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "createLocation",
+            "description": "Use to create a new location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "address": {
+                        "type": "string"
+                    }
+				},
+                "required": ["name"]
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "linkLocation",
+            "description": "Use to link an already existing location to the image you were sent",
+            "parameters": {
+                "type": "object",
+                "properties": {
+					"locationId": {
+						"type": "string"
+					}
+				},
+                "required": ["locationId"]
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "finish",
+            "description": "Call this when there is nothing left to do.",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    }
+]`
+
+type LocationAgent struct {
+	client client.AgentClient
+
+	locationModel models.LocationModel
+}
+
+type listLocationArguments struct{}
+type createLocationArguments struct {
+	Name    string  `json:"name"`
+	Address *string `json:"address"`
+}
+type linkLocationArguments struct {
+	LocationID string `json:"locationId"`
+}
+
+func NewLocationAgent(locationModel models.LocationModel) (LocationAgent, error) {
+	agentClient, err := client.CreateAgentClient(log.NewWithOptions(os.Stdout, log.Options{
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Locations 📍",
+	}))
+
+	if err != nil {
+		return LocationAgent{}, err
+	}
+
+	agent := LocationAgent{
+		client:        agentClient,
+		locationModel: locationModel,
+	}
+
+	agentClient.ToolHandler.AddTool("listLocations", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+		return agent.locationModel.List(context.Background(), info.UserId)
+	})
+
+	agentClient.ToolHandler.AddTool("createLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
+		args := createLocationArguments{}
+		err := json.Unmarshal([]byte(_args), &args)
+		if err != nil {
+			return model.Locations{}, err
+		}
+
+		ctx := context.Background()
+
+		location, err := agent.locationModel.Save(ctx, info.UserId, model.Locations{
+			Name:    args.Name,
+			Address: args.Address,
+		})
+
+		if err != nil {
+			return model.Locations{}, err
+		}
+
+		_, err = agent.locationModel.SaveToImage(ctx, info.ImageId, location.ID)
+		if err != nil {
+			return model.Locations{}, err
+		}
+
+		return location, nil
+	})
+
+	agentClient.ToolHandler.AddTool("linkLocation", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) {
+		args := linkLocationArguments{}
+		err := json.Unmarshal([]byte(_args), &args)
+		if err != nil {
+			return "", err
+		}
+
+		ctx := context.Background()
+
+		contactUuid, err := uuid.Parse(args.LocationID)
+		if err != nil {
+			return "", err
+		}
+
+		agent.locationModel.SaveToImage(ctx, info.ImageId, contactUuid)
+		return "Saved", nil
+	})
+
+	return agent, nil
+}

backend/agents/note_agent.go

@@ -2,10 +2,13 @@ package agents
 
 import (
 	"context"
+	"os"
 	"screenmark/screenmark/.gen/haystack/haystack/model"
 	"screenmark/screenmark/agents/client"
 	"screenmark/screenmark/models"
+	"time"
 
+	"github.com/charmbracelet/log"
 	"github.com/google/uuid"
 )
 
@@ -17,6 +20,8 @@ An image can have more than one note.
 
 You must return markdown, and adapt the text to best fit markdown.
 Do not return anything except markdown.
+
+If the image contains code, add this inside code blocks. You must try and correctly guess the language too.
 `
 
 type NoteAgent struct {
@@ -66,7 +71,11 @@ func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName s
 }
 
 func NewNoteAgent(noteModel models.NoteModel) (NoteAgent, error) {
-	client, err := client.CreateAgentClient()
+	client, err := client.CreateAgentClient(log.NewWithOptions(os.Stdout, log.Options{
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Notes 📝",
+	}))
 	if err != nil {
 		return NoteAgent{}, err
 	}

backend/agents/orchestrator.go

@@ -1,15 +1,15 @@
 package agents
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
+	"os"
 	"screenmark/screenmark/agents/client"
+	"time"
 
-	"github.com/google/uuid"
+	"github.com/charmbracelet/log"
 )
 
-const orchestratorPrompt = `
+const OrchestratorPrompt = `
 You are an Orchestrator for various AI agents.
 
 The user will send you images and you have to determine which agents you have to call, in order to best help the user.
@@ -20,40 +20,33 @@ The agents are available as tool calls.
 
 Agents available:
 
-eventLocationAgent
-
-Use it when you think the image contains an event or a location of any sort. This can be an event page, a map, an address or a date.
-
 noteAgent
+Use when there is ANY text on the image.
 
-Use it when there is text on the screen. Any text, always use this. Use me!
+contactAgent
+Use it when the image contains information relating a person.
 
-defaultAgent
+locationAgent
+Use it when the image contains some address or a place.
 
-When none of the above apply.
+eventAgent
+Use it when the image contains an event, this can be a date, a message suggesting an event.
+
+noAction
+When you think there is no more information to extract from the image.
 
 Always call agents in parallel if you need to call more than 1.
+
+Do not call the agent if you do not think it is relevant for the image.
 `
 
-const MY_TOOLS = `
+const OrchestratorTools = `
 [
-    {
-        "type": "function",
-        "function": {
-            "name": "eventLocationAgent",
-            "description": "Uses the event location agent",
-            "parameters": {
-                "type": "object",
-                "properties": {},
-                "required": []
-            }
-        }
-    },
 	{
         "type": "function",
         "function": {
             "name": "noteAgent",
-            "description": "Uses the note agent",
+            "description": "Use when there is any text on the image, this can be code/text/formulas any writing",
             "parameters": {
                 "type": "object",
                 "properties": {},
@@ -64,8 +57,44 @@ const MY_TOOLS = `
 	{
         "type": "function",
         "function": {
-            "name": "defaultAgent",
-            "description": "Used when you dont think its a good idea to call other agents",
+            "name": "contactAgent",
+            "description": "Use when then image contains some person or contact",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "locationAgent",
+            "description": "Use when then image contains some place, location or address",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "eventAgent",
+            "description": "Use when then image contains some event",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    },
+	{
+        "type": "function",
+        "function": {
+            "name": "noAction",
+            "description": "Use when you are sure nothing can be done about this image anymore",
             "parameters": {
                 "type": "object",
                 "properties": {},
@@ -76,75 +105,26 @@ const MY_TOOLS = `
 ]`
 
 type OrchestratorAgent struct {
-	client client.AgentClient
+	Client client.AgentClient
+
+	log log.Logger
 }
 
 type Status struct {
 	Ok bool `json:"ok"`
 }
 
-// TODO: the primary function of the agent could be extracted outwards.
-// This is basically the same function as we have in the `event_location_agent.go`
-func (agent OrchestratorAgent) Orchestrate(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
-	toolChoice := "any"
+func NewOrchestratorAgent(noteAgent NoteAgent, contactAgent ContactAgent, locationAgent LocationAgent, eventAgent EventAgent, imageName string, imageData []byte) (OrchestratorAgent, error) {
+	agent, err := client.CreateAgentClient(log.NewWithOptions(os.Stdout, log.Options{
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Orchestrator 🎼",
+	}))
 
-	var tools any
-	err := json.Unmarshal([]byte(MY_TOOLS), &tools)
-	if err != nil {
-		return err
-	}
-
-	request := client.AgentRequestBody{
-		Model:       "pixtral-12b-2409",
-		Temperature: 0.3,
-		ResponseFormat: client.ResponseFormat{
-			Type: "text",
-		},
-		ToolChoice: &toolChoice,
-		Tools:      &tools,
-
-		EndToolCall: "defaultAgent",
-
-		Chat: &client.Chat{
-			Messages: make([]client.ChatMessage, 0),
-		},
-	}
-
-	request.Chat.AddSystem(orchestratorPrompt)
-	request.Chat.AddImage(imageName, imageData)
-
-	res, err := agent.client.Request(&request)
-	if err != nil {
-		return err
-	}
-
-	fmt.Println(res)
-
-	toolHandlerInfo := client.ToolHandlerInfo{
-		ImageId: imageId,
-		UserId:  userId,
-	}
-
-	return agent.client.ToolLoop(toolHandlerInfo, &request)
-}
-
-func NewOrchestratorAgent(eventLocationAgent EventLocationAgent, noteAgent NoteAgent, imageName string, imageData []byte) (OrchestratorAgent, error) {
-	agent, err := client.CreateAgentClient()
 	if err != nil {
 		return OrchestratorAgent{}, err
 	}
 
-	agent.ToolHandler.AddTool("eventLocationAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
-		// We need a way to keep track of this async?
-		// Probably just a DB, because we don't want to wait. The orchistrator shouldnt wait for this stuff to finish.
-
-		go eventLocationAgent.GetLocations(info.UserId, info.ImageId, imageName, imageData)
-
-		return Status{
-			Ok: true,
-		}, nil
-	})
-
 	agent.ToolHandler.AddTool("noteAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
 		go noteAgent.GetNotes(info.UserId, info.ImageId, imageName, imageData)
 
@@ -153,7 +133,31 @@ func NewOrchestratorAgent(eventLocationAgent EventLocationAgent, noteAgent NoteA
 		}, nil
 	})
 
-	agent.ToolHandler.AddTool("defaultAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+	agent.ToolHandler.AddTool("contactAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+		go contactAgent.client.RunAgent(contactPrompt, contactTools, "finish", info.UserId, info.ImageId, imageName, imageData)
+
+		return Status{
+			Ok: true,
+		}, nil
+	})
+
+	agent.ToolHandler.AddTool("locationAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+		go locationAgent.client.RunAgent(locationPrompt, locationTools, "finish", info.UserId, info.ImageId, imageName, imageData)
+
+		return Status{
+			Ok: true,
+		}, nil
+	})
+
+	agent.ToolHandler.AddTool("eventAgent", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
+		go eventAgent.client.RunAgent(eventPrompt, eventTools, "finish", info.UserId, info.ImageId, imageName, imageData)
+
+		return Status{
+			Ok: true,
+		}, nil
+	})
+
+	agent.ToolHandler.AddTool("noAction", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) {
 		// To nothing
 
 		return Status{
@@ -162,6 +166,6 @@ func NewOrchestratorAgent(eventLocationAgent EventLocationAgent, noteAgent NoteA
 	})
 
 	return OrchestratorAgent{
-		client: agent,
+		Client: agent,
 	}, nil
 }

backend/ai_response.json

@@ -1,108 +0,0 @@
-{
-    "name": "image_info",
-    "strict": true,
-    "schema": {
-        "type": "object",
-        "title": "image",
-        "required": ["tags", "text", "links"],
-        "additionalProperties": false,
-        "properties": {
-            "tags": {
-                "type": "array",
-                "title": "tags",
-                "description": "A list of tags you think the image is relevant to.",
-                "items": {
-                    "type": "string"
-                }
-            },
-            "text": {
-                "type": "array",
-                "title": "text",
-                "description": "A list of sentences the image contains.",
-                "items": {
-                    "type": "string"
-                }
-            },
-            "links": {
-                "type": "array",
-                "title": "links",
-                "description": "A list of all the links you can find in the image.",
-                "items": {
-                    "type": "string"
-                }
-            },
-            "locations": {
-                "title": "locations",
-                "type": "array",
-                "description": "A list of locations you can find on the image, if any",
-                "items": {
-                    "type": "object",
-                    "required": ["name"],
-                    "additionalProperties": false,
-                    "properties": {
-                        "name": {
-                            "title": "name",
-                            "type": "string"
-                        },
-                        "coordinates": {
-                            "title": "coordinates",
-                            "type": "string"
-                        },
-                        "address": {
-                            "title": "address",
-                            "type": "string"
-                        },
-                        "description": {
-                            "title": "description",
-                            "type": "string"
-                        }
-                    }
-                }
-            },
-            "events": {
-                "title": "events",
-                "type": "array",
-                "description": "A list of events you find on the image, if any",
-                "items": {
-                    "type": "object",
-                    "required": ["name"],
-                    "additionalProperties": false,
-                    "properties": {
-                        "name": {
-                            "type": "string",
-                            "title": "name"
-                        },
-                        "locations": {
-                            "title": "locations",
-                            "type": "array",
-                            "description": "A list of locations on this event, if any",
-                            "items": {
-                                "type": "object",
-                                "required": ["name"],
-                                "additionalProperties": false,
-                                "properties": {
-                                    "name": {
-                                        "title": "name",
-                                        "type": "string"
-                                    },
-                                    "coordinates": {
-                                        "title": "coordinates",
-                                        "type": "string"
-                                    },
-                                    "address": {
-                                        "title": "address",
-                                        "type": "string"
-                                    },
-                                    "description": {
-                                        "title": "description",
-                                        "type": "string"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

backend/email.go

@@ -56,6 +56,7 @@ func CreateMailClient() (Mailer, error) {
 
 	client, err := mail.NewClient(
 		"smtp.mailbox.org",
+		mail.WithTLSPortPolicy(mail.TLSMandatory),
 		mail.WithSMTPAuth(mail.SMTPAuthPlain),
 		mail.WithUsername(os.Getenv("EMAIL_USERNAME")),
 		mail.WithPassword(os.Getenv("EMAIL_PASSWORD")),

backend/events.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"database/sql"
+	"fmt"
 	"log"
 	"os"
 	"screenmark/screenmark/agents"
@@ -13,7 +14,7 @@ import (
 	"github.com/lib/pq"
 )
 
-func ListenNewImageEvents(db *sql.DB) {
+func ListenNewImageEvents(db *sql.DB, eventManager *EventManager) {
 	listener := pq.NewListener(os.Getenv("DB_CONNECTION"), time.Second, time.Second, func(event pq.ListenerEventType, err error) {
 		if err != nil {
 			panic(err)
@@ -36,16 +37,27 @@ func ListenNewImageEvents(db *sql.DB) {
 		select {
 		case parameters := <-listener.Notify:
 			imageId := uuid.MustParse(parameters.Extra)
+			eventManager.listeners[parameters.Extra] = make(chan string)
 
 			ctx := context.Background()
 
 			go func() {
-				locationAgent, err := agents.NewLocationEventAgent(locationModel, eventModel, contactModel)
+				noteAgent, err := agents.NewNoteAgent(noteModel)
 				if err != nil {
 					panic(err)
 				}
 
-				noteAgent, err := agents.NewNoteAgent(noteModel)
+				contactAgent, err := agents.NewContactAgent(contactModel)
+				if err != nil {
+					panic(err)
+				}
+
+				locationAgent, err := agents.NewLocationAgent(locationModel)
+				if err != nil {
+					panic(err)
+				}
+
+				eventAgent, err := agents.NewEventAgent(eventModel)
 				if err != nil {
 					panic(err)
 				}
@@ -57,23 +69,71 @@ func ListenNewImageEvents(db *sql.DB) {
 					return
 				}
 
-				_, err = imageModel.FinishProcessing(ctx, image.ID)
-				if err != nil {
+				if err := imageModel.StartProcessing(ctx, image.ID); err != nil {
 					log.Println("Failed to FinishProcessing")
 					log.Println(err)
 					return
 				}
 
-				orchestrator, err := agents.NewOrchestratorAgent(locationAgent, noteAgent, image.Image.ImageName, image.Image.Image)
+				orchestrator, err := agents.NewOrchestratorAgent(noteAgent, contactAgent, locationAgent, eventAgent, image.Image.ImageName, image.Image.Image)
 				if err != nil {
 					panic(err)
 				}
 
-				err = orchestrator.Orchestrate(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image)
+				// Still need to find some way to hide this complexity away.
+				// I don't think wrapping agents in structs actually works too well.
+				err = orchestrator.Client.RunAgent(agents.OrchestratorPrompt, agents.OrchestratorTools, "noAction", image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image)
 				if err != nil {
 					log.Println(err)
 				}
+
+				imageModel.FinishProcessing(ctx, image.ID)
 			}()
 		}
 	}
 }
+
+type EventManager struct {
+	// Maps processing image UUID to a channel
+	listeners map[string]chan string
+}
+
+func NewEventManager() EventManager {
+	return EventManager{
+		listeners: make(map[string]chan string),
+	}
+}
+
+func ListenProcessingImageStatus(db *sql.DB, eventManager *EventManager) {
+	listener := pq.NewListener(os.Getenv("DB_CONNECTION"), time.Second, time.Second, func(event pq.ListenerEventType, err error) {
+		if err != nil {
+			panic(err)
+		}
+	})
+	defer listener.Close()
+
+	if err := listener.Listen("new_processing_image_status"); err != nil {
+		panic(err)
+	}
+
+	for {
+		select {
+		case data := <-listener.Notify:
+			stringUuid := data.Extra[0:36]
+			status := data.Extra[36:]
+
+			fmt.Printf("UUID: %s\n", stringUuid)
+			fmt.Printf("Receiving :s\n", data.Extra)
+
+			imageListener, exists := eventManager.listeners[stringUuid]
+			if !exists {
+				continue
+			}
+
+			imageListener <- status
+
+			close(imageListener)
+			delete(eventManager.listeners, stringUuid)
+		}
+	}
+}

backend/go.mod

@@ -3,17 +3,28 @@ module screenmark/screenmark
 go 1.24.0
 
 require (
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/charmbracelet/lipgloss v1.0.0 // indirect
+	github.com/charmbracelet/log v0.4.1 // indirect
+	github.com/charmbracelet/x/ansi v0.4.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-chi/chi/v5 v5.2.1 // indirect
 	github.com/go-jet/jet/v2 v2.12.0 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/lib/pq v1.10.9 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/wneessen/go-mail v0.6.2 // indirect
 	golang.org/x/crypto v0.33.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

backend/go.sum

@@ -1,9 +1,19 @@
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
+github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
+github.com/charmbracelet/log v0.4.1 h1:6AYnoHKADkghm/vt4neaNEXkxcXLSV2g1rdyFDOpTyk=
+github.com/charmbracelet/log v0.4.1/go.mod h1:pXgyTsqsVu4N9hGdHmQ0xEA4RsXof402LX9ZgiITn2I=
+github.com/charmbracelet/x/ansi v0.4.2 h1:0JM6Aj/g/KC154/gOP4vfxun0ff6itogDYk41kof+qk=
+github.com/charmbracelet/x/ansi v0.4.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
 github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
 github.com/go-jet/jet/v2 v2.12.0 h1:z2JfvBAZgsfxlQz6NXBYdZTXc7ep3jhbszTLtETv1JE=
 github.com/go-jet/jet/v2 v2.12.0/go.mod h1:ufQVRQeI1mbcO5R8uCEVcVf3Foej9kReBdwDx7YMWUM=
+github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
+github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -13,8 +23,16 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
+github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
+github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
@@ -29,6 +47,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -55,10 +75,12 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

backend/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -12,6 +13,7 @@ import (
 	"screenmark/screenmark/.gen/haystack/haystack/model"
 	"screenmark/screenmark/agents/client"
 	"screenmark/screenmark/models"
+	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
@@ -48,7 +50,10 @@ func main() {
 
 	auth := CreateAuth(mail)
 
-	go ListenNewImageEvents(db)
+	eventManager := NewEventManager()
+
+	go ListenNewImageEvents(db, &eventManager)
+	go ListenProcessingImageStatus(db, &eventManager)
 
 	r := chi.NewRouter()
 
@@ -107,6 +112,13 @@ func main() {
 						Data: note,
 					})
 				}
+
+				for _, contact := range image.Contacts {
+					dataTypes = append(dataTypes, DataType{
+						Type: "contact",
+						Data: contact,
+					})
+				}
 			}
 
 			jsonImages, err := json.Marshal(dataTypes)
@@ -210,7 +222,7 @@ func main() {
 				return
 			}
 
-			userImage, err := imageModel.Process(r.Context(), uuid.MustParse(userId), model.Image{
+			userImage, err := imageModel.Process(r.Context(), userId, model.Image{
 				Image:     image,
 				ImageName: imageName,
 			})
@@ -239,6 +251,41 @@ func main() {
 
 	})
 
+	r.Get("/image-events/{id}", func(w http.ResponseWriter, r *http.Request) {
+		// TODO: authentication :)
+
+		id := r.PathValue("id")
+
+		imageNotifier, exists := eventManager.listeners[id]
+		if !exists {
+			fmt.Println("Not found!")
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		w.Header().Set("Content-Type", "text/event-stream")
+		w.Header().Set("Cache-Control", "no-cache")
+		w.Header().Set("Connection", "keep-alive")
+		w.(http.Flusher).Flush()
+
+		ctx, cancel := context.WithCancel(r.Context())
+
+		for {
+			select {
+			case <-ctx.Done():
+				fmt.Fprint(w, "event: close\ndata: Connection closed\n\n")
+				w.(http.Flusher).Flush()
+				cancel()
+				return
+			case data := <-imageNotifier:
+				fmt.Printf("Status received: %s\n", data)
+				fmt.Fprintf(w, "data: %s-%s\n", data, time.Now().String())
+				w.(http.Flusher).Flush()
+				cancel()
+			}
+		}
+	})
+
 	r.Post("/login", func(w http.ResponseWriter, r *http.Request) {
 		type LoginBody struct {
 			Email string `json:"email"`

backend/middleware.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 )
 
@@ -26,7 +25,6 @@ func ProtectedRoute(next http.Handler) http.Handler {
 			return
 		}
 
-		fmt.Println(token[len("Bearer "):])
 		userId, err := GetUserIdFromAccess(token[len("Bearer "):])
 		if err != nil {
 			w.WriteHeader(http.StatusUnauthorized)

backend/models/events.go

@@ -14,6 +14,20 @@ type EventModel struct {
 	dbPool *sql.DB
 }
 
+func (m EventModel) List(ctx context.Context, userId uuid.UUID) ([]model.Events, error) {
+	listEventsStmt := SELECT(Events.AllColumns).
+		FROM(
+			Events.
+				INNER_JOIN(UserEvents, UserEvents.EventID.EQ(Events.ID)),
+		).
+		WHERE(UserEvents.UserID.EQ(UUID(userId)))
+
+	events := []model.Events{}
+
+	err := listEventsStmt.QueryContext(ctx, m.dbPool, &events)
+	return events, err
+}
+
 func (m EventModel) Save(ctx context.Context, userId uuid.UUID, event model.Events) (model.Events, error) {
 	// TODO tx here
 	insertEventStmt := Events.

backend/models/image.go

@@ -130,6 +130,17 @@ func (m ImageModel) FinishProcessing(ctx context.Context, imageId uuid.UUID) (mo
 	return userImage, err
 }
 
+func (m ImageModel) StartProcessing(ctx context.Context, processingImageId uuid.UUID) error {
+	startProcessingStmt := UserImagesToProcess.
+		UPDATE(UserImagesToProcess.Status).
+		SET(model.Progress_InProgress).
+		WHERE(UserImagesToProcess.ID.EQ(UUID(processingImageId)))
+
+	_, err := startProcessingStmt.ExecContext(ctx, m.dbPool)
+
+	return err
+}
+
 func (m ImageModel) Get(ctx context.Context, imageId uuid.UUID) (ImageData, error) {
 	getImageStmt := SELECT(UserImages.AllColumns, Image.AllColumns).
 		FROM(

backend/models/locations.go

@@ -7,6 +7,7 @@ import (
 	. "screenmark/screenmark/.gen/haystack/haystack/table"
 
 	. "github.com/go-jet/jet/v2/postgres"
+	"github.com/go-jet/jet/v2/qrm"
 
 	"github.com/google/uuid"
 )
@@ -51,13 +52,32 @@ func (m LocationModel) Save(ctx context.Context, userId uuid.UUID, location mode
 }
 
 func (m LocationModel) SaveToImage(ctx context.Context, imageId uuid.UUID, locationId uuid.UUID) (model.ImageLocations, error) {
+	imageLocation := model.ImageLocations{}
+
+	checkExistingStmt := ImageLocations.
+		SELECT(ImageLocations.AllColumns).
+		WHERE(
+			ImageLocations.ImageID.EQ(UUID(imageId)).
+				AND(ImageLocations.LocationID.EQ(UUID(locationId))),
+		)
+
+	err := checkExistingStmt.QueryContext(ctx, m.dbPool, &imageLocation)
+	if err != nil && err != qrm.ErrNoRows {
+		// A real error
+		return model.ImageLocations{}, err
+	}
+
+	if err == nil {
+		// Already exists.
+		return imageLocation, nil
+	}
+
 	insertImageLocationStmt := ImageLocations.
 		INSERT(ImageLocations.ImageID, ImageLocations.LocationID).
 		VALUES(imageId, locationId).
 		RETURNING(ImageLocations.AllColumns)
 
-	imageLocation := model.ImageLocations{}
-	err := insertImageLocationStmt.QueryContext(ctx, m.dbPool, &imageLocation)
+	err = insertImageLocationStmt.QueryContext(ctx, m.dbPool, &imageLocation)
 
 	return imageLocation, err
 }

backend/models/user.go

@@ -39,6 +39,8 @@ type ImageWithProperties struct {
 	}
 
 	Notes []model.Notes
+
+	Contacts []model.Contacts
 }
 
 func getUserIdFromImage(ctx context.Context, dbPool *sql.DB, imageId uuid.UUID) (uuid.UUID, error) {

backend/schema.sql

@@ -2,6 +2,10 @@ DROP SCHEMA IF EXISTS haystack CASCADE;
 
 CREATE SCHEMA haystack;
 
+/* -----| Enums |----- */
+
+CREATE TYPE haystack.progress AS ENUM('not-started','in-progress');
+
 /* -----| Schema tables |----- */
 
 CREATE TABLE haystack.users (
@@ -17,6 +21,7 @@ CREATE TABLE haystack.image (
 
 CREATE TABLE haystack.user_images_to_process (
 	id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    status haystack.progress NOT NULL DEFAULT 'not-started',
 	image_id uuid NOT NULL UNIQUE REFERENCES haystack.image (id),
 	user_id uuid NOT NULL REFERENCES haystack.users (id)
 );
@@ -155,6 +160,14 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;
 
+CREATE OR REPLACE FUNCTION notify_new_processing_image_status()
+RETURNS TRIGGER AS $$
+BEGIN
+PERFORM pg_notify('new_processing_image_status', NEW.id::text || NEW.status::text);
+	RETURN NEW;
+END
+$$ LANGUAGE plpgsql;
+
 /* -----| Triggers |----- */
 
 CREATE OR REPLACE TRIGGER on_new_image AFTER INSERT
@@ -162,6 +175,12 @@ ON haystack.user_images_to_process
 FOR EACH ROW
 EXECUTE PROCEDURE notify_new_image();
 
+CREATE OR REPLACE TRIGGER on_update_image_progress 
+AFTER UPDATE OF status
+ON haystack.user_images_to_process
+FOR EACH ROW
+EXECUTE PROCEDURE notify_new_processing_image_status();
+
 /* -----| Test Data |----- */
 
 -- Insert a user

backend/tools.json

@@ -1,75 +0,0 @@
-[
-    {
-        "type": "function",
-        "function": {
-            "name": "createLocation",
-            "description": "Creates a location. No not use if you think an existing location is suitable!",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string"
-                    },
-                    "address": {
-                        "type": "string"
-                    }
-                },
-                "required": ["name"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "listLocations",
-            "description": "Lists the locations available",
-            "parameters": {
-                "type": "object",
-                "properties": {}
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "createEvent",
-            "description": "Creates a new event",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string"
-                    },
-                    "startDateTime": {
-                        "type": "string",
-                        "description": "The start time as an ISO string"
-                    },
-                    "endDateTime": {
-                        "type": "string",
-                        "description": "The end time as an ISO string"
-                    },
-                    "locationId": {
-                        "type": "string",
-                        "description": "The ID of the location, available by listLocations"
-                    },
-                    "organizerName": {
-                        "type": "string",
-                        "description": "The name of the organizer"
-                    }
-                },
-                "required": ["name"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "finish",
-            "description": "Nothing else to do, call this function.",
-            "parameters": {
-                "type": "object",
-                "properties": {}
-            }
-        }
-    }
-]

frontend/src/App.tsx

@@ -1,13 +1,14 @@
+import { A } from "@solidjs/router";
 import { IconSearch } from "@tabler/icons-solidjs";
 import clsx from "clsx";
 import Fuse from "fuse.js";
 import { For, createEffect, createResource, createSignal } from "solid-js";
 import { SearchCardEvent } from "./components/search-card/SearchCardEvent";
 import { SearchCardLocation } from "./components/search-card/SearchCardLocation";
-import { UserImage, getUserImages } from "./network";
-import { getCardSize } from "./utils/getCardSize";
 import { SearchCardNote } from "./components/search-card/SearchCardNote";
-import { A } from "@solidjs/router";
+import { type UserImage, getUserImages } from "./network";
+import { getCardSize } from "./utils/getCardSize";
+import { SearchCardContact } from "./components/search-card/SearchCardContact";
 
 const getCardComponent = (item: UserImage) => {
     switch (item.type) {
@@ -17,8 +18,8 @@ const getCardComponent = (item: UserImage) => {
             return <SearchCardEvent item={item} />;
         case "note":
             return <SearchCardNote item={item} />;
-        // case "Contact":
-        //     return <SearchCardContact item={item} />;
+        case "contact":
+            return <SearchCardContact item={item} />;
         // case "Website":
         //     return <SearchCardWebsite item={item} />;
         // case "Note":

frontend/src/components/search-card/SearchCardContact.tsx

@@ -1,10 +1,10 @@
 import { Separator } from "@kobalte/core/separator";
 
 import { IconUser } from "@tabler/icons-solidjs";
-import type { Contact } from "../../network/types";
+import type { UserImage } from "../../network";
 
 type Props = {
-    item: Contact;
+    item: Extract<UserImage, { type: "contact" }>;
 };
 
 export const SearchCardContact = ({ item }: Props) => {
@@ -13,13 +13,15 @@ export const SearchCardContact = ({ item }: Props) => {
     return (
         <div class="absolute inset-0 p-3 bg-orange-50">
             <div class="grid grid-cols-[auto_20px] gap-1 mb-1">
-                <p class="text-sm text-neutral-900 font-bold">{data.name}</p>
+                <p class="text-sm text-neutral-900 font-bold">{data.Name}</p>
                 <IconUser size={20} class="text-neutral-500 mt-1" />
             </div>
-            <p class="text-xs text-neutral-500">{data.phoneNumber}</p>
+            <p class="text-xs text-neutral-500">{data.PhoneNumber}</p>
+            <Separator class="my-2" />
+            <p class="text-xs text-neutral-500">{data.Email}</p>
             <Separator class="my-2" />
             <p class="text-xs text-neutral-500 line-clamp-2 overflow-hidden">
-                {data.notes}
+                {data.Description}
             </p>
         </div>
     );

frontend/src/network/index.ts

@@ -41,6 +41,7 @@ const sendImageResponseValidator = strictObject({
     ID: pipe(string(), uuid()),
     ImageID: pipe(string(), uuid()),
     UserID: pipe(string(), uuid()),
+    Status: string(),
 });
 
 export const sendImage = async (
@@ -109,10 +110,16 @@ const noteDataType = strictObject({
     data: noteValidator,
 });
 
+const contactDataType = strictObject({
+    type: literal("contact"),
+    data: contactValidator,
+});
+
 const dataTypeValidator = variant("type", [
     locationDataType,
     eventDataType,
     noteDataType,
+    contactDataType,
 ]);
 const getUserImagesResponseValidator = array(dataTypeValidator);