From a283bc1bcdbfa74c30a590aee7dc26ad3a301ce6 Mon Sep 17 00:00:00 2001 From: John Costa Date: Tue, 22 Jul 2025 19:40:56 +0100 Subject: [PATCH] feat: new AI generated lists I think this could be how we generate other lists Problems: - Knowing it's a location is good because you can do nice stuff on the frontend. - Same for contacts & events. So a good alternative, is to still use this type, but perhaps change the database such that all lists live within the new tables (lists, image_lists). But have special tags. This would also make it easier on the AI I think. --- .../haystack/haystack/model/image_lists.go | 18 ++ backend/.gen/haystack/haystack/model/lists.go | 21 ++ .../haystack/haystack/table/image_lists.go | 81 ++++++++ backend/.gen/haystack/haystack/table/lists.go | 87 ++++++++ .../haystack/table/table_use_schema.go | 2 + backend/agents/list_agent.go | 189 ++++++++++++++++++ backend/events.go | 6 + backend/models/lists.go | 49 +++++ backend/schema.sql | 17 ++ 9 files changed, 470 insertions(+) create mode 100644 backend/.gen/haystack/haystack/model/image_lists.go create mode 100644 backend/.gen/haystack/haystack/model/lists.go create mode 100644 backend/.gen/haystack/haystack/table/image_lists.go create mode 100644 backend/.gen/haystack/haystack/table/lists.go create mode 100644 backend/agents/list_agent.go create mode 100644 backend/models/lists.go diff --git a/backend/.gen/haystack/haystack/model/image_lists.go b/backend/.gen/haystack/haystack/model/image_lists.go new file mode 100644 index 0000000..955858f --- /dev/null +++ b/backend/.gen/haystack/haystack/model/image_lists.go @@ -0,0 +1,18 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type ImageLists struct { + ID uuid.UUID `sql:"primary_key"` + ImageID uuid.UUID + ListID uuid.UUID +} diff --git a/backend/.gen/haystack/haystack/model/lists.go b/backend/.gen/haystack/haystack/model/lists.go new file mode 100644 index 0000000..ac24c1c --- /dev/null +++ b/backend/.gen/haystack/haystack/model/lists.go @@ -0,0 +1,21 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" + "time" +) + +type Lists struct { + ID uuid.UUID `sql:"primary_key"` + UserID uuid.UUID + Name string + Description string + CreatedAt *time.Time +} diff --git a/backend/.gen/haystack/haystack/table/image_lists.go b/backend/.gen/haystack/haystack/table/image_lists.go new file mode 100644 index 0000000..ec0f696 --- /dev/null +++ b/backend/.gen/haystack/haystack/table/image_lists.go @@ -0,0 +1,81 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var ImageLists = newImageListsTable("haystack", "image_lists", "") + +type imageListsTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + ImageID postgres.ColumnString + ListID postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type ImageListsTable struct { + imageListsTable + + EXCLUDED imageListsTable +} + +// AS creates new ImageListsTable with assigned alias +func (a ImageListsTable) AS(alias string) *ImageListsTable { + return newImageListsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new ImageListsTable with assigned schema name +func (a ImageListsTable) FromSchema(schemaName string) *ImageListsTable { + return newImageListsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new ImageListsTable with assigned table prefix +func (a ImageListsTable) WithPrefix(prefix string) *ImageListsTable { + return newImageListsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new ImageListsTable with assigned table suffix +func (a ImageListsTable) WithSuffix(suffix string) *ImageListsTable { + return newImageListsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newImageListsTable(schemaName, tableName, alias string) *ImageListsTable { + return &ImageListsTable{ + imageListsTable: newImageListsTableImpl(schemaName, tableName, alias), + EXCLUDED: newImageListsTableImpl("", "excluded", ""), + } +} + +func newImageListsTableImpl(schemaName, tableName, alias string) imageListsTable { + var ( + IDColumn = postgres.StringColumn("id") + ImageIDColumn = postgres.StringColumn("image_id") + ListIDColumn = postgres.StringColumn("list_id") + allColumns = postgres.ColumnList{IDColumn, ImageIDColumn, ListIDColumn} + mutableColumns = postgres.ColumnList{ImageIDColumn, ListIDColumn} + ) + + return imageListsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + ImageID: ImageIDColumn, + ListID: ListIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/lists.go b/backend/.gen/haystack/haystack/table/lists.go new file mode 100644 index 0000000..6c876c1 --- /dev/null +++ b/backend/.gen/haystack/haystack/table/lists.go @@ -0,0 +1,87 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var Lists = newListsTable("haystack", "lists", "") + +type listsTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + UserID postgres.ColumnString + Name postgres.ColumnString + Description postgres.ColumnString + CreatedAt postgres.ColumnTimestampz + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type ListsTable struct { + listsTable + + EXCLUDED listsTable +} + +// AS creates new ListsTable with assigned alias +func (a ListsTable) AS(alias string) *ListsTable { + return newListsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new ListsTable with assigned schema name +func (a ListsTable) FromSchema(schemaName string) *ListsTable { + return newListsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new ListsTable with assigned table prefix +func (a ListsTable) WithPrefix(prefix string) *ListsTable { + return newListsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new ListsTable with assigned table suffix +func (a ListsTable) WithSuffix(suffix string) *ListsTable { + return newListsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newListsTable(schemaName, tableName, alias string) *ListsTable { + return &ListsTable{ + listsTable: newListsTableImpl(schemaName, tableName, alias), + EXCLUDED: newListsTableImpl("", "excluded", ""), + } +} + +func newListsTableImpl(schemaName, tableName, alias string) listsTable { + var ( + IDColumn = postgres.StringColumn("id") + UserIDColumn = postgres.StringColumn("user_id") + NameColumn = postgres.StringColumn("name") + DescriptionColumn = postgres.StringColumn("description") + CreatedAtColumn = postgres.TimestampzColumn("created_at") + allColumns = postgres.ColumnList{IDColumn, UserIDColumn, NameColumn, DescriptionColumn, CreatedAtColumn} + mutableColumns = postgres.ColumnList{UserIDColumn, NameColumn, DescriptionColumn, CreatedAtColumn} + ) + + return listsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + UserID: UserIDColumn, + Name: NameColumn, + Description: DescriptionColumn, + CreatedAt: CreatedAtColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/table_use_schema.go b/backend/.gen/haystack/haystack/table/table_use_schema.go index f09be12..c838e15 100644 --- a/backend/.gen/haystack/haystack/table/table_use_schema.go +++ b/backend/.gen/haystack/haystack/table/table_use_schema.go @@ -16,10 +16,12 @@ func UseSchema(schema string) { ImageContacts = ImageContacts.FromSchema(schema) ImageEvents = ImageEvents.FromSchema(schema) ImageLinks = ImageLinks.FromSchema(schema) + ImageLists = ImageLists.FromSchema(schema) ImageLocations = ImageLocations.FromSchema(schema) ImageNotes = ImageNotes.FromSchema(schema) ImageTags = ImageTags.FromSchema(schema) ImageText = ImageText.FromSchema(schema) + Lists = Lists.FromSchema(schema) Locations = Locations.FromSchema(schema) Logs = Logs.FromSchema(schema) Notes = Notes.FromSchema(schema) diff --git a/backend/agents/list_agent.go b/backend/agents/list_agent.go new file mode 100644 index 0000000..3fc68d1 --- /dev/null +++ b/backend/agents/list_agent.go @@ -0,0 +1,189 @@ +package agents + +import ( + "context" + "encoding/json" + "screenmark/screenmark/.gen/haystack/haystack/model" + "screenmark/screenmark/agents/client" + "screenmark/screenmark/models" + + "github.com/charmbracelet/log" + "github.com/google/uuid" +) + +const listPrompt = ` +**You are an AI used to classify what list a certain image belongs in** + +You will need to decide using tool calls, if you must create a new list, or use an existing one. +You must be specific enough so it is useful, but not too specific such that all images belong on seperate lists. + +An example of lists are: +- Locations +- Events +- TV Shows +- Movies +- Books + +You must call "listLists" to see which available lists are already available. + +*Important* +You must not create lists with the names Locations, Events, Contacts or Notes. You can create lists adjacent to those, but +those lists are dealt with seperately. + +**Tools:** +* think: Internal reasoning/planning step. +* listLists: Get existing lists +* createList: Creates a new list with a name and description. +* addToList: Add to an existing list. +* stopAgent: Signal task completion. +` + +const listTools = ` +[ + { + "type": "function", + "function": { + "name": "think", + "description": "Use this tool to think through the image, evaluating the event and whether or not it exists in the users listEvents.", + "parameters": { + "type": "object", + "properties": { + "thought": { + "type": "string", + "description": "A singular thought about the image." + } + }, + "required": ["thought"] + } + } + }, + { + "type": "function", + "function": { + "name": "listLists", + "description": "Retrieves the list of the user's existing lists.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "createList", + "description": "Creates a new list", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of this new list." + }, + "description": { + "type": "string", + "description": "A simple description of this list." + } + }, + "required": ["name", "description"] + } + } + }, + { + "type": "function", + "function": { + "name": "addToList", + "description": "Adds an image to a list, this could be a new one you just created or not.", + "parameters": { + "type": "object", + "properties": { + "listId": { + "type": "string", + "description": "The UUID of the existing list" + } + }, + "required": ["listId"] + } + + } + }, + { + "type": "function", + "function": { + "name": "stopAgent", + "description": "Use this tool to signal that the contact processing for the current image is complete. Call this *only* when: 1) No contact info was found initially, OR 2) All found contacts were confirmed to already exist after calling listContacts, OR 3) All necessary createContact calls for new individuals have been completed.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + } +]` + +type listListsArguments struct{} +type createListArguments struct { + Name string `json:"name"` + Desription string `json:"description"` +} +type addToListArguments struct { + ListID string `json:"listId"` +} + +func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClient { + agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{ + SystemPrompt: listPrompt, + JsonTools: listTools, + Log: log, + EndToolCall: "stopAgent", + }) + + agentClient.ToolHandler.AddTool("think", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) { + return "Thought", nil + }) + + agentClient.ToolHandler.AddTool("listLists", func(info client.ToolHandlerInfo, args string, call client.ToolCall) (any, error) { + return listModel.List(context.Background(), info.UserId) + }) + + agentClient.ToolHandler.AddTool("createList", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) { + args := createListArguments{} + err := json.Unmarshal([]byte(_args), &args) + if err != nil { + return model.Events{}, err + } + + ctx := context.Background() + savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription) + + if err != nil { + return model.Lists{}, err + } + + return savedList, nil + }) + + agentClient.ToolHandler.AddTool("addToList", func(info client.ToolHandlerInfo, _args string, call client.ToolCall) (any, error) { + args := addToListArguments{} + err := json.Unmarshal([]byte(_args), &args) + if err != nil { + return "", err + } + + ctx := context.Background() + + listUuid, err := uuid.Parse(args.ListID) + if err != nil { + return "", err + } + + if err := listModel.SaveInto(ctx, listUuid, info.ImageId); err != nil { + return "", err + } + + return "Saved", nil + }) + + return agentClient +} diff --git a/backend/events.go b/backend/events.go index 52dada0..9ca56ce 100644 --- a/backend/events.go +++ b/backend/events.go @@ -37,6 +37,8 @@ func ListenNewImageEvents(db *sql.DB, notifier *Notifier[Notification]) { imageModel := models.NewImageModel(db) contactModel := models.NewContactModel(db) + listModel := models.NewListModel(db) + databaseEventLog := createLogger("Database Events 🤖", os.Stdout) databaseEventLog.SetLevel(log.DebugLevel) @@ -71,8 +73,12 @@ func ListenNewImageEvents(db *sql.DB, notifier *Notifier[Notification]) { return } + listAgent := agents.NewListAgent(createLogger("Lists 🖋️", splitWriter), listModel) + listAgent.RunAgent(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image) + orchestrator := agents.NewOrchestratorAgent(createLogger("Orchestrator 🎼", splitWriter), noteAgent, contactAgent, locationAgent, eventAgent, image.Image.ImageName, image.Image.Image) orchestrator.RunAgent(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image) + _, err = imageModel.FinishProcessing(ctx, image.ID) if err != nil { databaseEventLog.Error("Failed to finish processing", "ImageID", imageId, "error", err) diff --git a/backend/models/lists.go b/backend/models/lists.go new file mode 100644 index 0000000..324d472 --- /dev/null +++ b/backend/models/lists.go @@ -0,0 +1,49 @@ +package models + +import ( + "context" + "database/sql" + . "github.com/go-jet/jet/v2/postgres" + "screenmark/screenmark/.gen/haystack/haystack/model" + . "screenmark/screenmark/.gen/haystack/haystack/table" + + "github.com/google/uuid" +) + +type ListModel struct { + dbPool *sql.DB +} + +func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string) (model.Lists, error) { + stmt := Lists.INSERT(Lists.UserID, Lists.Name, Lists.Description). + VALUES(userId, name, description). + RETURNING(Lists.ID, Lists.Name, Lists.Description) + + newList := model.Lists{} + err := stmt.QueryContext(ctx, m.dbPool, &newList) + + return newList, err +} + +func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, error) { + stmt := Lists.SELECT(Lists.AllColumns). + WHERE(Lists.UserID.EQ(UUID(userId))) + + lists := []model.Lists{} + err := stmt.QueryContext(ctx, m.dbPool, &lists) + + return lists, err +} + +func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID) error { + stmt := ImageLists.INSERT(ImageLists.ListID, ImageLists.ImageID). + VALUES(listId, imageId) + + _, err := stmt.ExecContext(ctx, m.dbPool) + + return err +} + +func NewListModel(db *sql.DB) ListModel { + return ListModel{dbPool: db} +} diff --git a/backend/schema.sql b/backend/schema.sql index cf4d16f..9d719ee 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -183,6 +183,23 @@ CREATE TABLE haystack.logs ( created_at TIMESTAMP WITH TIME ZONE DEFAULT now() ); +CREATE TABLE haystack.lists ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES haystack.users (id), + + name TEXT NOT NULL, + description TEXT NOT NULL, + + created_at TIMESTAMP WITH TIME ZONE DEFAULT now() +); + +CREATE TABLE haystack.image_lists ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + image_id UUID NOT NULL REFERENCES haystack.image (id), + list_id UUID NOT NULL REFERENCES haystack.lists (id) +); + /* -----| Indexes |----- */ CREATE INDEX user_tags_index ON haystack.user_tags(tag);