feat(notes): saving the notes for any images for easy text searching

backend/.gen/haystack/haystack/model/image_notes.go

@@ -0,0 +1,18 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type ImageNotes struct {
+	ID      uuid.UUID `sql:"primary_key"`
+	ImageID uuid.UUID
+	NoteID  uuid.UUID
+}

backend/.gen/haystack/haystack/model/notes.go

@@ -0,0 +1,19 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type Notes struct {
+	ID          uuid.UUID `sql:"primary_key"`
+	Name        string
+	Description *string
+	Content     string
+}

backend/.gen/haystack/haystack/model/user_notes.go

@@ -0,0 +1,18 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type UserNotes struct {
+	ID     uuid.UUID `sql:"primary_key"`
+	UserID uuid.UUID
+	NoteID uuid.UUID
+}

backend/.gen/haystack/haystack/table/image_notes.go

@@ -0,0 +1,81 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var ImageNotes = newImageNotesTable("haystack", "image_notes", "")
+
+type imageNotesTable struct {
+	postgres.Table
+
+	// Columns
+	ID      postgres.ColumnString
+	ImageID postgres.ColumnString
+	NoteID  postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type ImageNotesTable struct {
+	imageNotesTable
+
+	EXCLUDED imageNotesTable
+}
+
+// AS creates new ImageNotesTable with assigned alias
+func (a ImageNotesTable) AS(alias string) *ImageNotesTable {
+	return newImageNotesTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new ImageNotesTable with assigned schema name
+func (a ImageNotesTable) FromSchema(schemaName string) *ImageNotesTable {
+	return newImageNotesTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new ImageNotesTable with assigned table prefix
+func (a ImageNotesTable) WithPrefix(prefix string) *ImageNotesTable {
+	return newImageNotesTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new ImageNotesTable with assigned table suffix
+func (a ImageNotesTable) WithSuffix(suffix string) *ImageNotesTable {
+	return newImageNotesTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newImageNotesTable(schemaName, tableName, alias string) *ImageNotesTable {
+	return &ImageNotesTable{
+		imageNotesTable: newImageNotesTableImpl(schemaName, tableName, alias),
+		EXCLUDED:        newImageNotesTableImpl("", "excluded", ""),
+	}
+}
+
+func newImageNotesTableImpl(schemaName, tableName, alias string) imageNotesTable {
+	var (
+		IDColumn       = postgres.StringColumn("id")
+		ImageIDColumn  = postgres.StringColumn("image_id")
+		NoteIDColumn   = postgres.StringColumn("note_id")
+		allColumns     = postgres.ColumnList{IDColumn, ImageIDColumn, NoteIDColumn}
+		mutableColumns = postgres.ColumnList{ImageIDColumn, NoteIDColumn}
+	)
+
+	return imageNotesTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		ID:      IDColumn,
+		ImageID: ImageIDColumn,
+		NoteID:  NoteIDColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/.gen/haystack/haystack/table/notes.go

@@ -0,0 +1,84 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var Notes = newNotesTable("haystack", "notes", "")
+
+type notesTable struct {
+	postgres.Table
+
+	// Columns
+	ID          postgres.ColumnString
+	Name        postgres.ColumnString
+	Description postgres.ColumnString
+	Content     postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type NotesTable struct {
+	notesTable
+
+	EXCLUDED notesTable
+}
+
+// AS creates new NotesTable with assigned alias
+func (a NotesTable) AS(alias string) *NotesTable {
+	return newNotesTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new NotesTable with assigned schema name
+func (a NotesTable) FromSchema(schemaName string) *NotesTable {
+	return newNotesTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new NotesTable with assigned table prefix
+func (a NotesTable) WithPrefix(prefix string) *NotesTable {
+	return newNotesTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new NotesTable with assigned table suffix
+func (a NotesTable) WithSuffix(suffix string) *NotesTable {
+	return newNotesTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newNotesTable(schemaName, tableName, alias string) *NotesTable {
+	return &NotesTable{
+		notesTable: newNotesTableImpl(schemaName, tableName, alias),
+		EXCLUDED:   newNotesTableImpl("", "excluded", ""),
+	}
+}
+
+func newNotesTableImpl(schemaName, tableName, alias string) notesTable {
+	var (
+		IDColumn          = postgres.StringColumn("id")
+		NameColumn        = postgres.StringColumn("name")
+		DescriptionColumn = postgres.StringColumn("description")
+		ContentColumn     = postgres.StringColumn("content")
+		allColumns        = postgres.ColumnList{IDColumn, NameColumn, DescriptionColumn, ContentColumn}
+		mutableColumns    = postgres.ColumnList{NameColumn, DescriptionColumn, ContentColumn}
+	)
+
+	return notesTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		ID:          IDColumn,
+		Name:        NameColumn,
+		Description: DescriptionColumn,
+		Content:     ContentColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/.gen/haystack/haystack/table/table_use_schema.go

@@ -17,14 +17,17 @@ func UseSchema(schema string) {
 	ImageEvents = ImageEvents.FromSchema(schema)
 	ImageLinks = ImageLinks.FromSchema(schema)
 	ImageLocations = ImageLocations.FromSchema(schema)
+	ImageNotes = ImageNotes.FromSchema(schema)
 	ImageTags = ImageTags.FromSchema(schema)
 	ImageText = ImageText.FromSchema(schema)
 	Locations = Locations.FromSchema(schema)
+	Notes = Notes.FromSchema(schema)
 	UserContacts = UserContacts.FromSchema(schema)
 	UserEvents = UserEvents.FromSchema(schema)
 	UserImages = UserImages.FromSchema(schema)
 	UserImagesToProcess = UserImagesToProcess.FromSchema(schema)
 	UserLocations = UserLocations.FromSchema(schema)
+	UserNotes = UserNotes.FromSchema(schema)
 	UserTags = UserTags.FromSchema(schema)
 	Users = Users.FromSchema(schema)
 }

backend/.gen/haystack/haystack/table/user_notes.go

@@ -0,0 +1,81 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var UserNotes = newUserNotesTable("haystack", "user_notes", "")
+
+type userNotesTable struct {
+	postgres.Table
+
+	// Columns
+	ID     postgres.ColumnString
+	UserID postgres.ColumnString
+	NoteID postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type UserNotesTable struct {
+	userNotesTable
+
+	EXCLUDED userNotesTable
+}
+
+// AS creates new UserNotesTable with assigned alias
+func (a UserNotesTable) AS(alias string) *UserNotesTable {
+	return newUserNotesTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new UserNotesTable with assigned schema name
+func (a UserNotesTable) FromSchema(schemaName string) *UserNotesTable {
+	return newUserNotesTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new UserNotesTable with assigned table prefix
+func (a UserNotesTable) WithPrefix(prefix string) *UserNotesTable {
+	return newUserNotesTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new UserNotesTable with assigned table suffix
+func (a UserNotesTable) WithSuffix(suffix string) *UserNotesTable {
+	return newUserNotesTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newUserNotesTable(schemaName, tableName, alias string) *UserNotesTable {
+	return &UserNotesTable{
+		userNotesTable: newUserNotesTableImpl(schemaName, tableName, alias),
+		EXCLUDED:       newUserNotesTableImpl("", "excluded", ""),
+	}
+}
+
+func newUserNotesTableImpl(schemaName, tableName, alias string) userNotesTable {
+	var (
+		IDColumn       = postgres.StringColumn("id")
+		UserIDColumn   = postgres.StringColumn("user_id")
+		NoteIDColumn   = postgres.StringColumn("note_id")
+		allColumns     = postgres.ColumnList{IDColumn, UserIDColumn, NoteIDColumn}
+		mutableColumns = postgres.ColumnList{UserIDColumn, NoteIDColumn}
+	)
+
+	return userNotesTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		ID:     IDColumn,
+		UserID: UserIDColumn,
+		NoteID: NoteIDColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/agents/event_location_agent.go

@@ -208,6 +208,7 @@ func (agent EventLocationAgent) GetLocations(userId uuid.UUID, imageId uuid.UUID
 	return err
 }
 
+// TODO: extract this into a more general tool handler package.
 func (handler ToolsHandlers) Handle(info ToolHandlerInfo, request *AgentRequestBody) (string, error) {
 	agentMessage := request.Messages[len(request.Messages)-1]
 

backend/agents/note_agent.go

@@ -0,0 +1,79 @@
+package agents
+
+import (
+	"context"
+	"screenmark/screenmark/.gen/haystack/haystack/model"
+	"screenmark/screenmark/models"
+
+	"github.com/google/uuid"
+)
+
+const noteAgentPrompt = `
+You are a helpful agent, who's job is to extract notes from images.
+Not all images contain notes, in such cases there's not need to create them.
+
+An image can have more than one note.
+
+You must return markdown, and adapt the text to best fit markdown.
+Do not return anything except markdown.
+`
+
+type NoteAgent struct {
+	client AgentClient
+
+	noteModel models.NoteModel
+}
+
+func (agent NoteAgent) GetNotes(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
+	request := AgentRequestBody{
+		Model:       "pixtral-12b-2409",
+		Temperature: 0.3,
+		ResponseFormat: ResponseFormat{
+			Type: "text",
+		},
+	}
+
+	err := request.AddSystem(noteAgentPrompt)
+	if err != nil {
+		return err
+	}
+
+	request.AddImage(imageName, imageData)
+	resp, err := agent.client.Request(&request)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+
+	markdown := resp.Choices[0].Message.Content
+
+	note, err := agent.noteModel.Save(ctx, userId, model.Notes{
+		Name:    "the note", // TODO: add some json schema
+		Content: markdown,
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = agent.noteModel.SaveToImage(ctx, imageId, note.ID)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewNoteAgent(noteModel models.NoteModel) (NoteAgent, error) {
+	client, err := CreateAgentClient(noteAgentPrompt)
+	if err != nil {
+		return NoteAgent{}, err
+	}
+
+	agent := NoteAgent{
+		client:    client,
+		noteModel: noteModel,
+	}
+
+	return agent, nil
+}

backend/main.go

@@ -81,6 +81,7 @@ func main() {
 	eventModel := models.NewEventModel(db)
 	userModel := models.NewUserModel(db)
 	contactModel := models.NewContactModel(db)
+	noteModel := models.NewNoteModel(db)
 
 	listener := pq.NewListener(os.Getenv("DB_CONNECTION"), time.Second, time.Second, func(event pq.ListenerEventType, err error) {
 		if err != nil {
@@ -109,7 +110,12 @@ func main() {
 						panic(err)
 					}
 
-					locationAgent, err := agents.NewLocationEventAgent(locationModel, eventModel, contactModel)
+					_, err = agents.NewLocationEventAgent(locationModel, eventModel, contactModel)
+					if err != nil {
+						panic(err)
+					}
+
+					noteAgent, err := agents.NewNoteAgent(noteModel)
 					if err != nil {
 						panic(err)
 					}
@@ -128,8 +134,12 @@ func main() {
 						return
 					}
 
-					log.Println("Calling locationAgent!")
-					err = locationAgent.GetLocations(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image)
+					// log.Println("Calling locationAgent!")
+					// err = locationAgent.GetLocations(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image)
+					// log.Println(err)
+
+					log.Println("Calling noteAgent!")
+					err = noteAgent.GetNotes(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image)
 					log.Println(err)
 
 					return

backend/models/notes.go

@@ -0,0 +1,67 @@
+package models
+
+import (
+	"context"
+	"database/sql"
+	"screenmark/screenmark/.gen/haystack/haystack/model"
+	. "screenmark/screenmark/.gen/haystack/haystack/table"
+
+	. "github.com/go-jet/jet/v2/postgres"
+
+	"github.com/google/uuid"
+)
+
+type NoteModel struct {
+	dbPool *sql.DB
+}
+
+func (m NoteModel) List(ctx context.Context, userId uuid.UUID) ([]model.Notes, error) {
+	listNotesStmt := SELECT(Notes.AllColumns).
+		FROM(
+			Notes.
+				INNER_JOIN(UserNotes, UserNotes.NoteID.EQ(Notes.ID)),
+		).
+		WHERE(UserNotes.UserID.EQ(UUID(userId)))
+
+	locations := []model.Notes{}
+
+	err := listNotesStmt.QueryContext(ctx, m.dbPool, &locations)
+	return locations, err
+}
+
+func (m NoteModel) Save(ctx context.Context, userId uuid.UUID, note model.Notes) (model.Notes, error) {
+	insertNoteStmt := Notes.
+		INSERT(Notes.Name, Notes.Description, Notes.Content).
+		VALUES(note.Name, note.Description, note.Content).
+		RETURNING(Notes.AllColumns)
+
+	insertedNote := model.Notes{}
+	err := insertNoteStmt.QueryContext(ctx, m.dbPool, &insertedNote)
+	if err != nil {
+		return model.Notes{}, err
+	}
+
+	insertUserNoteStmt := UserNotes.
+		INSERT(UserNotes.UserID, UserNotes.NoteID).
+		VALUES(userId, insertedNote.ID)
+
+	_, err = insertUserNoteStmt.ExecContext(ctx, m.dbPool)
+
+	return insertedNote, err
+}
+
+func (m NoteModel) SaveToImage(ctx context.Context, imageId uuid.UUID, noteId uuid.UUID) (model.ImageNotes, error) {
+	insertImageNoteStmt := ImageNotes.
+		INSERT(ImageNotes.ImageID, ImageNotes.NoteID).
+		VALUES(imageId, noteId).
+		RETURNING(ImageNotes.AllColumns)
+
+	imageNote := model.ImageNotes{}
+	err := insertImageNoteStmt.QueryContext(ctx, m.dbPool, &imageNote)
+
+	return imageNote, err
+}
+
+func NewNoteModel(db *sql.DB) NoteModel {
+	return NoteModel{dbPool: db}
+}

backend/schema.sql

@@ -118,6 +118,28 @@ CREATE TABLE haystack.user_events (
     user_id UUID NOT NULL REFERENCES haystack.users (id)
 );
 
+CREATE TABLE haystack.notes (
+	id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+
+    -- It seems name and description are frequent. We could use table inheritance.
+    name TEXT NOT NULL,
+    description TEXT,
+    
+    content TEXT NOT NULL
+);
+
+CREATE TABLE haystack.image_notes (
+	id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    image_id UUID NOT NULL REFERENCES haystack.image (id),
+    note_id UUID NOT NULL REFERENCES haystack.notes (id)
+);
+
+CREATE TABLE haystack.user_notes (
+	id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    user_id UUID NOT NULL REFERENCES haystack.users (id),
+    note_id UUID NOT NULL REFERENCES haystack.notes (id)
+);
+
 /* -----| Indexes |----- */
 
 CREATE INDEX user_tags_index ON haystack.user_tags(tag);