From f72ee73020e030b68bd4ade757a11058437470b9 Mon Sep 17 00:00:00 2001 From: John Costa Date: Tue, 1 Apr 2025 20:45:43 +0000 Subject: [PATCH] feat(notes): saving the notes for any images for easy text searching --- .../haystack/haystack/model/image_notes.go | 18 ++++ backend/.gen/haystack/haystack/model/notes.go | 19 +++++ .../haystack/haystack/model/user_notes.go | 18 ++++ .../haystack/haystack/table/image_notes.go | 81 ++++++++++++++++++ backend/.gen/haystack/haystack/table/notes.go | 84 +++++++++++++++++++ .../haystack/table/table_use_schema.go | 3 + .../haystack/haystack/table/user_notes.go | 81 ++++++++++++++++++ backend/agents/event_location_agent.go | 1 + backend/agents/note_agent.go | 79 +++++++++++++++++ backend/main.go | 16 +++- backend/models/notes.go | 67 +++++++++++++++ backend/schema.sql | 22 +++++ 12 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 backend/.gen/haystack/haystack/model/image_notes.go create mode 100644 backend/.gen/haystack/haystack/model/notes.go create mode 100644 backend/.gen/haystack/haystack/model/user_notes.go create mode 100644 backend/.gen/haystack/haystack/table/image_notes.go create mode 100644 backend/.gen/haystack/haystack/table/notes.go create mode 100644 backend/.gen/haystack/haystack/table/user_notes.go create mode 100644 backend/agents/note_agent.go create mode 100644 backend/models/notes.go diff --git a/backend/.gen/haystack/haystack/model/image_notes.go b/backend/.gen/haystack/haystack/model/image_notes.go new file mode 100644 index 0000000..755a16c --- /dev/null +++ b/backend/.gen/haystack/haystack/model/image_notes.go @@ -0,0 +1,18 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type ImageNotes struct { + ID uuid.UUID `sql:"primary_key"` + ImageID uuid.UUID + NoteID uuid.UUID +} diff --git a/backend/.gen/haystack/haystack/model/notes.go b/backend/.gen/haystack/haystack/model/notes.go new file mode 100644 index 0000000..68ad546 --- /dev/null +++ b/backend/.gen/haystack/haystack/model/notes.go @@ -0,0 +1,19 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type Notes struct { + ID uuid.UUID `sql:"primary_key"` + Name string + Description *string + Content string +} diff --git a/backend/.gen/haystack/haystack/model/user_notes.go b/backend/.gen/haystack/haystack/model/user_notes.go new file mode 100644 index 0000000..e770bb3 --- /dev/null +++ b/backend/.gen/haystack/haystack/model/user_notes.go @@ -0,0 +1,18 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type UserNotes struct { + ID uuid.UUID `sql:"primary_key"` + UserID uuid.UUID + NoteID uuid.UUID +} diff --git a/backend/.gen/haystack/haystack/table/image_notes.go b/backend/.gen/haystack/haystack/table/image_notes.go new file mode 100644 index 0000000..c0e50c1 --- /dev/null +++ b/backend/.gen/haystack/haystack/table/image_notes.go @@ -0,0 +1,81 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var ImageNotes = newImageNotesTable("haystack", "image_notes", "") + +type imageNotesTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + ImageID postgres.ColumnString + NoteID postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type ImageNotesTable struct { + imageNotesTable + + EXCLUDED imageNotesTable +} + +// AS creates new ImageNotesTable with assigned alias +func (a ImageNotesTable) AS(alias string) *ImageNotesTable { + return newImageNotesTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new ImageNotesTable with assigned schema name +func (a ImageNotesTable) FromSchema(schemaName string) *ImageNotesTable { + return newImageNotesTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new ImageNotesTable with assigned table prefix +func (a ImageNotesTable) WithPrefix(prefix string) *ImageNotesTable { + return newImageNotesTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new ImageNotesTable with assigned table suffix +func (a ImageNotesTable) WithSuffix(suffix string) *ImageNotesTable { + return newImageNotesTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newImageNotesTable(schemaName, tableName, alias string) *ImageNotesTable { + return &ImageNotesTable{ + imageNotesTable: newImageNotesTableImpl(schemaName, tableName, alias), + EXCLUDED: newImageNotesTableImpl("", "excluded", ""), + } +} + +func newImageNotesTableImpl(schemaName, tableName, alias string) imageNotesTable { + var ( + IDColumn = postgres.StringColumn("id") + ImageIDColumn = postgres.StringColumn("image_id") + NoteIDColumn = postgres.StringColumn("note_id") + allColumns = postgres.ColumnList{IDColumn, ImageIDColumn, NoteIDColumn} + mutableColumns = postgres.ColumnList{ImageIDColumn, NoteIDColumn} + ) + + return imageNotesTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + ImageID: ImageIDColumn, + NoteID: NoteIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/notes.go b/backend/.gen/haystack/haystack/table/notes.go new file mode 100644 index 0000000..859d5c6 --- /dev/null +++ b/backend/.gen/haystack/haystack/table/notes.go @@ -0,0 +1,84 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var Notes = newNotesTable("haystack", "notes", "") + +type notesTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + Name postgres.ColumnString + Description postgres.ColumnString + Content postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type NotesTable struct { + notesTable + + EXCLUDED notesTable +} + +// AS creates new NotesTable with assigned alias +func (a NotesTable) AS(alias string) *NotesTable { + return newNotesTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new NotesTable with assigned schema name +func (a NotesTable) FromSchema(schemaName string) *NotesTable { + return newNotesTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new NotesTable with assigned table prefix +func (a NotesTable) WithPrefix(prefix string) *NotesTable { + return newNotesTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new NotesTable with assigned table suffix +func (a NotesTable) WithSuffix(suffix string) *NotesTable { + return newNotesTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newNotesTable(schemaName, tableName, alias string) *NotesTable { + return &NotesTable{ + notesTable: newNotesTableImpl(schemaName, tableName, alias), + EXCLUDED: newNotesTableImpl("", "excluded", ""), + } +} + +func newNotesTableImpl(schemaName, tableName, alias string) notesTable { + var ( + IDColumn = postgres.StringColumn("id") + NameColumn = postgres.StringColumn("name") + DescriptionColumn = postgres.StringColumn("description") + ContentColumn = postgres.StringColumn("content") + allColumns = postgres.ColumnList{IDColumn, NameColumn, DescriptionColumn, ContentColumn} + mutableColumns = postgres.ColumnList{NameColumn, DescriptionColumn, ContentColumn} + ) + + return notesTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + Name: NameColumn, + Description: DescriptionColumn, + Content: ContentColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/table_use_schema.go b/backend/.gen/haystack/haystack/table/table_use_schema.go index 424fcc4..94cb419 100644 --- a/backend/.gen/haystack/haystack/table/table_use_schema.go +++ b/backend/.gen/haystack/haystack/table/table_use_schema.go @@ -17,14 +17,17 @@ func UseSchema(schema string) { ImageEvents = ImageEvents.FromSchema(schema) ImageLinks = ImageLinks.FromSchema(schema) ImageLocations = ImageLocations.FromSchema(schema) + ImageNotes = ImageNotes.FromSchema(schema) ImageTags = ImageTags.FromSchema(schema) ImageText = ImageText.FromSchema(schema) Locations = Locations.FromSchema(schema) + Notes = Notes.FromSchema(schema) UserContacts = UserContacts.FromSchema(schema) UserEvents = UserEvents.FromSchema(schema) UserImages = UserImages.FromSchema(schema) UserImagesToProcess = UserImagesToProcess.FromSchema(schema) UserLocations = UserLocations.FromSchema(schema) + UserNotes = UserNotes.FromSchema(schema) UserTags = UserTags.FromSchema(schema) Users = Users.FromSchema(schema) } diff --git a/backend/.gen/haystack/haystack/table/user_notes.go b/backend/.gen/haystack/haystack/table/user_notes.go new file mode 100644 index 0000000..9241cc1 --- /dev/null +++ b/backend/.gen/haystack/haystack/table/user_notes.go @@ -0,0 +1,81 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var UserNotes = newUserNotesTable("haystack", "user_notes", "") + +type userNotesTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + UserID postgres.ColumnString + NoteID postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type UserNotesTable struct { + userNotesTable + + EXCLUDED userNotesTable +} + +// AS creates new UserNotesTable with assigned alias +func (a UserNotesTable) AS(alias string) *UserNotesTable { + return newUserNotesTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new UserNotesTable with assigned schema name +func (a UserNotesTable) FromSchema(schemaName string) *UserNotesTable { + return newUserNotesTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new UserNotesTable with assigned table prefix +func (a UserNotesTable) WithPrefix(prefix string) *UserNotesTable { + return newUserNotesTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new UserNotesTable with assigned table suffix +func (a UserNotesTable) WithSuffix(suffix string) *UserNotesTable { + return newUserNotesTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newUserNotesTable(schemaName, tableName, alias string) *UserNotesTable { + return &UserNotesTable{ + userNotesTable: newUserNotesTableImpl(schemaName, tableName, alias), + EXCLUDED: newUserNotesTableImpl("", "excluded", ""), + } +} + +func newUserNotesTableImpl(schemaName, tableName, alias string) userNotesTable { + var ( + IDColumn = postgres.StringColumn("id") + UserIDColumn = postgres.StringColumn("user_id") + NoteIDColumn = postgres.StringColumn("note_id") + allColumns = postgres.ColumnList{IDColumn, UserIDColumn, NoteIDColumn} + mutableColumns = postgres.ColumnList{UserIDColumn, NoteIDColumn} + ) + + return userNotesTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + UserID: UserIDColumn, + NoteID: NoteIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/agents/event_location_agent.go b/backend/agents/event_location_agent.go index fc51a9c..72757ad 100644 --- a/backend/agents/event_location_agent.go +++ b/backend/agents/event_location_agent.go @@ -208,6 +208,7 @@ func (agent EventLocationAgent) GetLocations(userId uuid.UUID, imageId uuid.UUID return err } +// TODO: extract this into a more general tool handler package. func (handler ToolsHandlers) Handle(info ToolHandlerInfo, request *AgentRequestBody) (string, error) { agentMessage := request.Messages[len(request.Messages)-1] diff --git a/backend/agents/note_agent.go b/backend/agents/note_agent.go new file mode 100644 index 0000000..1616ea6 --- /dev/null +++ b/backend/agents/note_agent.go @@ -0,0 +1,79 @@ +package agents + +import ( + "context" + "screenmark/screenmark/.gen/haystack/haystack/model" + "screenmark/screenmark/models" + + "github.com/google/uuid" +) + +const noteAgentPrompt = ` +You are a helpful agent, who's job is to extract notes from images. +Not all images contain notes, in such cases there's not need to create them. + +An image can have more than one note. + +You must return markdown, and adapt the text to best fit markdown. +Do not return anything except markdown. +` + +type NoteAgent struct { + client AgentClient + + noteModel models.NoteModel +} + +func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error { + request := AgentRequestBody{ + Model: "pixtral-12b-2409", + Temperature: 0.3, + ResponseFormat: ResponseFormat{ + Type: "text", + }, + } + + err := request.AddSystem(noteAgentPrompt) + if err != nil { + return err + } + + request.AddImage(imageName, imageData) + resp, err := agent.client.Request(&request) + if err != nil { + return err + } + + ctx := context.Background() + + markdown := resp.Choices[0].Message.Content + + note, err := agent.noteModel.Save(ctx, userId, model.Notes{ + Name: "the note", // TODO: add some json schema + Content: markdown, + }) + if err != nil { + return err + } + + _, err = agent.noteModel.SaveToImage(ctx, imageId, note.ID) + if err != nil { + return err + } + + return nil +} + +func NewNoteAgent(noteModel models.NoteModel) (NoteAgent, error) { + client, err := CreateAgentClient(noteAgentPrompt) + if err != nil { + return NoteAgent{}, err + } + + agent := NoteAgent{ + client: client, + noteModel: noteModel, + } + + return agent, nil +} diff --git a/backend/main.go b/backend/main.go index dd19f59..eebadef 100644 --- a/backend/main.go +++ b/backend/main.go @@ -81,6 +81,7 @@ func main() { eventModel := models.NewEventModel(db) userModel := models.NewUserModel(db) contactModel := models.NewContactModel(db) + noteModel := models.NewNoteModel(db) listener := pq.NewListener(os.Getenv("DB_CONNECTION"), time.Second, time.Second, func(event pq.ListenerEventType, err error) { if err != nil { @@ -109,7 +110,12 @@ func main() { panic(err) } - locationAgent, err := agents.NewLocationEventAgent(locationModel, eventModel, contactModel) + _, err = agents.NewLocationEventAgent(locationModel, eventModel, contactModel) + if err != nil { + panic(err) + } + + noteAgent, err := agents.NewNoteAgent(noteModel) if err != nil { panic(err) } @@ -128,8 +134,12 @@ func main() { return } - log.Println("Calling locationAgent!") - err = locationAgent.GetLocations(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image) + // log.Println("Calling locationAgent!") + // err = locationAgent.GetLocations(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image) + // log.Println(err) + + log.Println("Calling noteAgent!") + err = noteAgent.GetNotes(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image) log.Println(err) return diff --git a/backend/models/notes.go b/backend/models/notes.go new file mode 100644 index 0000000..4d6f68f --- /dev/null +++ b/backend/models/notes.go @@ -0,0 +1,67 @@ +package models + +import ( + "context" + "database/sql" + "screenmark/screenmark/.gen/haystack/haystack/model" + . "screenmark/screenmark/.gen/haystack/haystack/table" + + . "github.com/go-jet/jet/v2/postgres" + + "github.com/google/uuid" +) + +type NoteModel struct { + dbPool *sql.DB +} + +func (m NoteModel) List(ctx context.Context, userId uuid.UUID) ([]model.Notes, error) { + listNotesStmt := SELECT(Notes.AllColumns). + FROM( + Notes. + INNER_JOIN(UserNotes, UserNotes.NoteID.EQ(Notes.ID)), + ). + WHERE(UserNotes.UserID.EQ(UUID(userId))) + + locations := []model.Notes{} + + err := listNotesStmt.QueryContext(ctx, m.dbPool, &locations) + return locations, err +} + +func (m NoteModel) Save(ctx context.Context, userId uuid.UUID, note model.Notes) (model.Notes, error) { + insertNoteStmt := Notes. + INSERT(Notes.Name, Notes.Description, Notes.Content). + VALUES(note.Name, note.Description, note.Content). + RETURNING(Notes.AllColumns) + + insertedNote := model.Notes{} + err := insertNoteStmt.QueryContext(ctx, m.dbPool, &insertedNote) + if err != nil { + return model.Notes{}, err + } + + insertUserNoteStmt := UserNotes. + INSERT(UserNotes.UserID, UserNotes.NoteID). + VALUES(userId, insertedNote.ID) + + _, err = insertUserNoteStmt.ExecContext(ctx, m.dbPool) + + return insertedNote, err +} + +func (m NoteModel) SaveToImage(ctx context.Context, imageId uuid.UUID, noteId uuid.UUID) (model.ImageNotes, error) { + insertImageNoteStmt := ImageNotes. + INSERT(ImageNotes.ImageID, ImageNotes.NoteID). + VALUES(imageId, noteId). + RETURNING(ImageNotes.AllColumns) + + imageNote := model.ImageNotes{} + err := insertImageNoteStmt.QueryContext(ctx, m.dbPool, &imageNote) + + return imageNote, err +} + +func NewNoteModel(db *sql.DB) NoteModel { + return NoteModel{dbPool: db} +} diff --git a/backend/schema.sql b/backend/schema.sql index 9d6f7d7..7812025 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -118,6 +118,28 @@ CREATE TABLE haystack.user_events ( user_id UUID NOT NULL REFERENCES haystack.users (id) ); +CREATE TABLE haystack.notes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + + -- It seems name and description are frequent. We could use table inheritance. + name TEXT NOT NULL, + description TEXT, + + content TEXT NOT NULL +); + +CREATE TABLE haystack.image_notes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + image_id UUID NOT NULL REFERENCES haystack.image (id), + note_id UUID NOT NULL REFERENCES haystack.notes (id) +); + +CREATE TABLE haystack.user_notes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES haystack.users (id), + note_id UUID NOT NULL REFERENCES haystack.notes (id) +); + /* -----| Indexes |----- */ CREATE INDEX user_tags_index ON haystack.user_tags(tag);