diff --git a/backend/.gen/haystack/haystack/model/image_schema_items.go b/backend/.gen/haystack/haystack/model/image_schema_items.go new file mode 100644 index 0000000..1c9983e --- /dev/null +++ b/backend/.gen/haystack/haystack/model/image_schema_items.go @@ -0,0 +1,19 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type ImageSchemaItems struct { + ID uuid.UUID `sql:"primary_key"` + Value *string + SchemaItemID uuid.UUID + ImageID uuid.UUID +} diff --git a/backend/.gen/haystack/haystack/model/schema_items.go b/backend/.gen/haystack/haystack/model/schema_items.go new file mode 100644 index 0000000..330ce55 --- /dev/null +++ b/backend/.gen/haystack/haystack/model/schema_items.go @@ -0,0 +1,20 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type SchemaItems struct { + ID uuid.UUID `sql:"primary_key"` + Item string + Value string + Description string + SchemaID uuid.UUID +} diff --git a/backend/.gen/haystack/haystack/model/schemas.go b/backend/.gen/haystack/haystack/model/schemas.go new file mode 100644 index 0000000..f4c04b1 --- /dev/null +++ b/backend/.gen/haystack/haystack/model/schemas.go @@ -0,0 +1,17 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package model + +import ( + "github.com/google/uuid" +) + +type Schemas struct { + ID uuid.UUID `sql:"primary_key"` + ListID uuid.UUID +} diff --git a/backend/.gen/haystack/haystack/table/image_schema_items.go b/backend/.gen/haystack/haystack/table/image_schema_items.go new file mode 100644 index 0000000..8d3b344 --- /dev/null +++ b/backend/.gen/haystack/haystack/table/image_schema_items.go @@ -0,0 +1,84 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var ImageSchemaItems = newImageSchemaItemsTable("haystack", "image_schema_items", "") + +type imageSchemaItemsTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + Value postgres.ColumnString + SchemaItemID postgres.ColumnString + ImageID postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type ImageSchemaItemsTable struct { + imageSchemaItemsTable + + EXCLUDED imageSchemaItemsTable +} + +// AS creates new ImageSchemaItemsTable with assigned alias +func (a ImageSchemaItemsTable) AS(alias string) *ImageSchemaItemsTable { + return newImageSchemaItemsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new ImageSchemaItemsTable with assigned schema name +func (a ImageSchemaItemsTable) FromSchema(schemaName string) *ImageSchemaItemsTable { + return newImageSchemaItemsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new ImageSchemaItemsTable with assigned table prefix +func (a ImageSchemaItemsTable) WithPrefix(prefix string) *ImageSchemaItemsTable { + return newImageSchemaItemsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new ImageSchemaItemsTable with assigned table suffix +func (a ImageSchemaItemsTable) WithSuffix(suffix string) *ImageSchemaItemsTable { + return newImageSchemaItemsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newImageSchemaItemsTable(schemaName, tableName, alias string) *ImageSchemaItemsTable { + return &ImageSchemaItemsTable{ + imageSchemaItemsTable: newImageSchemaItemsTableImpl(schemaName, tableName, alias), + EXCLUDED: newImageSchemaItemsTableImpl("", "excluded", ""), + } +} + +func newImageSchemaItemsTableImpl(schemaName, tableName, alias string) imageSchemaItemsTable { + var ( + IDColumn = postgres.StringColumn("id") + ValueColumn = postgres.StringColumn("value") + SchemaItemIDColumn = postgres.StringColumn("schema_item_id") + ImageIDColumn = postgres.StringColumn("image_id") + allColumns = postgres.ColumnList{IDColumn, ValueColumn, SchemaItemIDColumn, ImageIDColumn} + mutableColumns = postgres.ColumnList{ValueColumn, SchemaItemIDColumn, ImageIDColumn} + ) + + return imageSchemaItemsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + Value: ValueColumn, + SchemaItemID: SchemaItemIDColumn, + ImageID: ImageIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/schema_items.go b/backend/.gen/haystack/haystack/table/schema_items.go new file mode 100644 index 0000000..d075b4e --- /dev/null +++ b/backend/.gen/haystack/haystack/table/schema_items.go @@ -0,0 +1,87 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var SchemaItems = newSchemaItemsTable("haystack", "schema_items", "") + +type schemaItemsTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + Item postgres.ColumnString + Value postgres.ColumnString + Description postgres.ColumnString + SchemaID postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type SchemaItemsTable struct { + schemaItemsTable + + EXCLUDED schemaItemsTable +} + +// AS creates new SchemaItemsTable with assigned alias +func (a SchemaItemsTable) AS(alias string) *SchemaItemsTable { + return newSchemaItemsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new SchemaItemsTable with assigned schema name +func (a SchemaItemsTable) FromSchema(schemaName string) *SchemaItemsTable { + return newSchemaItemsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new SchemaItemsTable with assigned table prefix +func (a SchemaItemsTable) WithPrefix(prefix string) *SchemaItemsTable { + return newSchemaItemsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new SchemaItemsTable with assigned table suffix +func (a SchemaItemsTable) WithSuffix(suffix string) *SchemaItemsTable { + return newSchemaItemsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newSchemaItemsTable(schemaName, tableName, alias string) *SchemaItemsTable { + return &SchemaItemsTable{ + schemaItemsTable: newSchemaItemsTableImpl(schemaName, tableName, alias), + EXCLUDED: newSchemaItemsTableImpl("", "excluded", ""), + } +} + +func newSchemaItemsTableImpl(schemaName, tableName, alias string) schemaItemsTable { + var ( + IDColumn = postgres.StringColumn("id") + ItemColumn = postgres.StringColumn("item") + ValueColumn = postgres.StringColumn("value") + DescriptionColumn = postgres.StringColumn("description") + SchemaIDColumn = postgres.StringColumn("schema_id") + allColumns = postgres.ColumnList{IDColumn, ItemColumn, ValueColumn, DescriptionColumn, SchemaIDColumn} + mutableColumns = postgres.ColumnList{ItemColumn, ValueColumn, DescriptionColumn, SchemaIDColumn} + ) + + return schemaItemsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + Item: ItemColumn, + Value: ValueColumn, + Description: DescriptionColumn, + SchemaID: SchemaIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/schemas.go b/backend/.gen/haystack/haystack/table/schemas.go new file mode 100644 index 0000000..aef50dc --- /dev/null +++ b/backend/.gen/haystack/haystack/table/schemas.go @@ -0,0 +1,78 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var Schemas = newSchemasTable("haystack", "schemas", "") + +type schemasTable struct { + postgres.Table + + // Columns + ID postgres.ColumnString + ListID postgres.ColumnString + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList +} + +type SchemasTable struct { + schemasTable + + EXCLUDED schemasTable +} + +// AS creates new SchemasTable with assigned alias +func (a SchemasTable) AS(alias string) *SchemasTable { + return newSchemasTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new SchemasTable with assigned schema name +func (a SchemasTable) FromSchema(schemaName string) *SchemasTable { + return newSchemasTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new SchemasTable with assigned table prefix +func (a SchemasTable) WithPrefix(prefix string) *SchemasTable { + return newSchemasTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new SchemasTable with assigned table suffix +func (a SchemasTable) WithSuffix(suffix string) *SchemasTable { + return newSchemasTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newSchemasTable(schemaName, tableName, alias string) *SchemasTable { + return &SchemasTable{ + schemasTable: newSchemasTableImpl(schemaName, tableName, alias), + EXCLUDED: newSchemasTableImpl("", "excluded", ""), + } +} + +func newSchemasTableImpl(schemaName, tableName, alias string) schemasTable { + var ( + IDColumn = postgres.StringColumn("id") + ListIDColumn = postgres.StringColumn("list_id") + allColumns = postgres.ColumnList{IDColumn, ListIDColumn} + mutableColumns = postgres.ColumnList{ListIDColumn} + ) + + return schemasTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + ID: IDColumn, + ListID: ListIDColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + } +} diff --git a/backend/.gen/haystack/haystack/table/table_use_schema.go b/backend/.gen/haystack/haystack/table/table_use_schema.go index 4f9028f..2c9bad8 100644 --- a/backend/.gen/haystack/haystack/table/table_use_schema.go +++ b/backend/.gen/haystack/haystack/table/table_use_schema.go @@ -17,9 +17,12 @@ func UseSchema(schema string) { ImageEvents = ImageEvents.FromSchema(schema) ImageLists = ImageLists.FromSchema(schema) ImageLocations = ImageLocations.FromSchema(schema) + ImageSchemaItems = ImageSchemaItems.FromSchema(schema) Lists = Lists.FromSchema(schema) Locations = Locations.FromSchema(schema) Logs = Logs.FromSchema(schema) + SchemaItems = SchemaItems.FromSchema(schema) + Schemas = Schemas.FromSchema(schema) UserContacts = UserContacts.FromSchema(schema) UserEvents = UserEvents.FromSchema(schema) UserImages = UserImages.FromSchema(schema) diff --git a/backend/agents/description_agent.go b/backend/agents/description_agent.go index a745e10..826e5fb 100644 --- a/backend/agents/description_agent.go +++ b/backend/agents/description_agent.go @@ -41,15 +41,13 @@ func (agent DescriptionAgent) Describe(log *log.Logger, imageId uuid.UUID, image log.Debug("Sending description request") resp, err := agent.client.Request(&request) if err != nil { - return fmt.Errorf("Could not request", err) + return fmt.Errorf("Could not request. %s", err) } ctx := context.Background() markdown := resp.Choices[0].Message.Content - log.Debugf("Response %s", markdown) - err = agent.imageModel.AddDescription(ctx, imageId, markdown) if err != nil { return err diff --git a/backend/agents/list_agent.go b/backend/agents/list_agent.go index d3371d7..f04c7fd 100644 --- a/backend/agents/list_agent.go +++ b/backend/agents/list_agent.go @@ -27,13 +27,17 @@ An example of lists are: Another one of your tasks is to create a schema for this list. This should contain information that this, and following pictures contain. Be specific but also generic. You should use the parameters in "createList" to create this schema. +This schema should not be super specific. You must be able to understand the image, and if the content of the image doesnt seem relevant, try +and extract some meaning about what the image is. + You must call "listLists" to see which available lists are already available. +Use "createList" only once, don't create multiple lists for one image. **Tools:** * think: Internal reasoning/planning step. * listLists: Get existing lists -* createList: Creates a new list with a name and description. -* addToList: Add to an existing list. +* createList: Creates a new list with a name and description. Only use this once. +* addToList: Add to an existing list. This will also mean extracting information from this image, and inserting it, fitting the schema. * stopAgent: Signal task completion. ` @@ -89,19 +93,23 @@ const listTools = ` "items": { "type": "object", "properties": { - "key": { + "Item": { "type": "string", "description": "The name of the key for this specific field. Similar to a column in a database" }, - "value": { + "Value": { "type": "string", "enum": ["string", "number", "boolean"] + }, + "Description": { + "type": "string", + "description": "The description for this item" } } } } }, - "required": ["name", "description"] + "required": ["name", "description", "schema"] } } }, @@ -116,9 +124,26 @@ const listTools = ` "listId": { "type": "string", "description": "The UUID of the existing list" - } + }, + "schema": { + "type": "array", + "items": { + "type": "object", + "description": "A key-value of ID - value from this image to fit the schema. any of the values can be null", + "properties": { + "id": { + "type": "string", + "description": "The UUID of the schema item." + }, + "value": { + "type": "string", + "description": "the concrete value for this field" + } + } + } + } }, - "required": ["listId"] + "required": ["listId", "schema"] } } @@ -141,13 +166,11 @@ type listListsArguments struct{} type createListArguments struct { Name string `json:"name"` Desription string `json:"description"` - Schema []struct { - Key string `json:"key"` - Value string `json:"value"` - } + Schema []model.SchemaItems } type addToListArguments struct { ListID string `json:"listId"` + Schema []models.IDValue } func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClient { @@ -173,15 +196,16 @@ func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClien return model.Events{}, err } - log.Debug("Create list", "schema", args.Schema) - ctx := context.Background() - savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription) + savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription, args.Schema) if err != nil { - return model.Lists{}, err + log.Error(err) + return "", err } + log.Debug(savedList) + return savedList, nil }) @@ -199,7 +223,7 @@ func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClien return "", err } - if err := listModel.SaveInto(ctx, listUuid, info.ImageId); err != nil { + if err := listModel.SaveInto(ctx, listUuid, info.ImageId, args.Schema); err != nil { return "", err } diff --git a/backend/models/lists.go b/backend/models/lists.go index 324d472..ec5163f 100644 --- a/backend/models/lists.go +++ b/backend/models/lists.go @@ -3,10 +3,12 @@ package models import ( "context" "database/sql" - . "github.com/go-jet/jet/v2/postgres" + "fmt" "screenmark/screenmark/.gen/haystack/haystack/model" . "screenmark/screenmark/.gen/haystack/haystack/table" + . "github.com/go-jet/jet/v2/postgres" + "github.com/google/uuid" ) @@ -14,15 +16,74 @@ type ListModel struct { dbPool *sql.DB } -func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string) (model.Lists, error) { +type ListWithItems struct { + model.Lists + + Schema struct { + model.Schemas + + SchemaItems []model.SchemaItems + } +} + +func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string, schemaItems []model.SchemaItems) (ListWithItems, error) { + tx, err := m.dbPool.BeginTx(ctx, nil) + stmt := Lists.INSERT(Lists.UserID, Lists.Name, Lists.Description). VALUES(userId, name, description). RETURNING(Lists.ID, Lists.Name, Lists.Description) newList := model.Lists{} - err := stmt.QueryContext(ctx, m.dbPool, &newList) + err = stmt.QueryContext(ctx, tx, &newList) - return newList, err + if err != nil { + tx.Rollback() + return ListWithItems{}, fmt.Errorf("Could not save new list. %s", err) + } + + insertSchemaStmt := Schemas.INSERT(Schemas.ListID). + VALUES(newList.ID). + RETURNING(Schemas.ID) + + newSchema := model.Schemas{} + err = insertSchemaStmt.QueryContext(ctx, tx, &newSchema) + + if err != nil { + tx.Rollback() + return ListWithItems{}, fmt.Errorf("Could not save new schema. %s", err) + } + + // This is very interesting... + for i := range schemaItems { + schemaItems[i].SchemaID = newSchema.ID + } + + insertSchemaItemsStmt := SchemaItems.INSERT(SchemaItems.Item, SchemaItems.Value, SchemaItems.Description, SchemaItems.SchemaID). + MODELS(schemaItems) + _, err = insertSchemaItemsStmt.ExecContext(ctx, tx) + + if err != nil { + tx.Rollback() + return ListWithItems{}, fmt.Errorf("Could not save schema items. %s", err) + } + + err = tx.Commit() + if err != nil { + return ListWithItems{}, fmt.Errorf("Could not commit transaction. %s", err) + } + + getListAndItems := SELECT(Lists.AllColumns, Schemas.AllColumns, SchemaItems.AllColumns). + FROM( + Lists. + INNER_JOIN(Schemas, Schemas.ListID.EQ(Lists.ID)). + INNER_JOIN(SchemaItems, SchemaItems.SchemaID.EQ(Schemas.ID)), + ). + WHERE(Lists.ID.EQ(UUID(newList.ID))) + + listWithItems := ListWithItems{} + err = getListAndItems.QueryContext(ctx, m.dbPool, &listWithItems) + + return listWithItems, err } func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, error) { @@ -35,12 +96,50 @@ func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, e return lists, err } -func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID) error { +type IDValue struct { + ID string `json:"id"` + Value string `json:"value"` +} + +func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID, schemaValues []IDValue) error { + imageSchemaItems := make([]model.ImageSchemaItems, len(schemaValues)) + + for i, v := range schemaValues { + parsedId, err := uuid.Parse(v.ID) + if err != nil { + return err + } + + imageSchemaItems[i].SchemaItemID = parsedId + imageSchemaItems[i].ImageID = imageId + imageSchemaItems[i].Value = &v.Value + } + + tx, err := m.dbPool.BeginTx(ctx, nil) + if err != nil { + return err + } + stmt := ImageLists.INSERT(ImageLists.ListID, ImageLists.ImageID). VALUES(listId, imageId) - _, err := stmt.ExecContext(ctx, m.dbPool) + _, err = stmt.ExecContext(ctx, tx) + if err != nil { + tx.Rollback() + return fmt.Errorf("Could not insert new list. %s", err) + } + insertSchemaItemsStmt := ImageSchemaItems. + INSERT(ImageSchemaItems.Value, ImageSchemaItems.SchemaItemID, ImageSchemaItems.ImageID). + MODELS(imageSchemaItems) + + _, err = insertSchemaItemsStmt.ExecContext(ctx, tx) + if err != nil { + tx.Rollback() + return fmt.Errorf("Could not insert schema items. %s", err) + } + + err = tx.Commit() return err } diff --git a/backend/schema.sql b/backend/schema.sql index 675153a..e74976d 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -145,6 +145,31 @@ CREATE TABLE haystack.image_lists ( list_id UUID NOT NULL REFERENCES haystack.lists (id) ); +CREATE TABLE haystack.schemas ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + list_id UUID NOT NULL REFERENCES haystack.lists (id) +); + +CREATE TABLE haystack.schema_items ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + item TEXT NOT NULL, + value TEXT NOT NULL, + description TEXT NOT NULL, + + schema_id UUID NOT NULL REFERENCES haystack.schemas (id) +); + +CREATE TABLE haystack.image_schema_items ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + value TEXT, + + schema_item_id UUID NOT NULL REFERENCES haystack.schema_items (id), + image_id UUID NOT NULL REFERENCES haystack.image (id) +); + /* -----| Indexes |----- */ /* -----| Stored Procedures |----- */