feat: initial draft of generating a schema from one image

fix: error formatting

backend/.gen/haystack/haystack/model/image_schema_items.go

@@ -0,0 +1,19 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type ImageSchemaItems struct {
+	ID           uuid.UUID `sql:"primary_key"`
+	Value        *string
+	SchemaItemID uuid.UUID
+	ImageID      uuid.UUID
+}

backend/.gen/haystack/haystack/model/schema_items.go

@@ -0,0 +1,20 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type SchemaItems struct {
+	ID          uuid.UUID `sql:"primary_key"`
+	Item        string
+	Value       string
+	Description string
+	SchemaID    uuid.UUID
+}

backend/.gen/haystack/haystack/model/schemas.go

@@ -0,0 +1,17 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type Schemas struct {
+	ID     uuid.UUID `sql:"primary_key"`
+	ListID uuid.UUID
+}

backend/.gen/haystack/haystack/table/image_schema_items.go

@@ -0,0 +1,84 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var ImageSchemaItems = newImageSchemaItemsTable("haystack", "image_schema_items", "")
+
+type imageSchemaItemsTable struct {
+	postgres.Table
+
+	// Columns
+	ID           postgres.ColumnString
+	Value        postgres.ColumnString
+	SchemaItemID postgres.ColumnString
+	ImageID      postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type ImageSchemaItemsTable struct {
+	imageSchemaItemsTable
+
+	EXCLUDED imageSchemaItemsTable
+}
+
+// AS creates new ImageSchemaItemsTable with assigned alias
+func (a ImageSchemaItemsTable) AS(alias string) *ImageSchemaItemsTable {
+	return newImageSchemaItemsTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new ImageSchemaItemsTable with assigned schema name
+func (a ImageSchemaItemsTable) FromSchema(schemaName string) *ImageSchemaItemsTable {
+	return newImageSchemaItemsTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new ImageSchemaItemsTable with assigned table prefix
+func (a ImageSchemaItemsTable) WithPrefix(prefix string) *ImageSchemaItemsTable {
+	return newImageSchemaItemsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new ImageSchemaItemsTable with assigned table suffix
+func (a ImageSchemaItemsTable) WithSuffix(suffix string) *ImageSchemaItemsTable {
+	return newImageSchemaItemsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newImageSchemaItemsTable(schemaName, tableName, alias string) *ImageSchemaItemsTable {
+	return &ImageSchemaItemsTable{
+		imageSchemaItemsTable: newImageSchemaItemsTableImpl(schemaName, tableName, alias),
+		EXCLUDED:              newImageSchemaItemsTableImpl("", "excluded", ""),
+	}
+}
+
+func newImageSchemaItemsTableImpl(schemaName, tableName, alias string) imageSchemaItemsTable {
+	var (
+		IDColumn           = postgres.StringColumn("id")
+		ValueColumn        = postgres.StringColumn("value")
+		SchemaItemIDColumn = postgres.StringColumn("schema_item_id")
+		ImageIDColumn      = postgres.StringColumn("image_id")
+		allColumns         = postgres.ColumnList{IDColumn, ValueColumn, SchemaItemIDColumn, ImageIDColumn}
+		mutableColumns     = postgres.ColumnList{ValueColumn, SchemaItemIDColumn, ImageIDColumn}
+	)
+
+	return imageSchemaItemsTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		ID:           IDColumn,
+		Value:        ValueColumn,
+		SchemaItemID: SchemaItemIDColumn,
+		ImageID:      ImageIDColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/.gen/haystack/haystack/table/schema_items.go

@@ -0,0 +1,87 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var SchemaItems = newSchemaItemsTable("haystack", "schema_items", "")
+
+type schemaItemsTable struct {
+	postgres.Table
+
+	// Columns
+	ID          postgres.ColumnString
+	Item        postgres.ColumnString
+	Value       postgres.ColumnString
+	Description postgres.ColumnString
+	SchemaID    postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type SchemaItemsTable struct {
+	schemaItemsTable
+
+	EXCLUDED schemaItemsTable
+}
+
+// AS creates new SchemaItemsTable with assigned alias
+func (a SchemaItemsTable) AS(alias string) *SchemaItemsTable {
+	return newSchemaItemsTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new SchemaItemsTable with assigned schema name
+func (a SchemaItemsTable) FromSchema(schemaName string) *SchemaItemsTable {
+	return newSchemaItemsTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new SchemaItemsTable with assigned table prefix
+func (a SchemaItemsTable) WithPrefix(prefix string) *SchemaItemsTable {
+	return newSchemaItemsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new SchemaItemsTable with assigned table suffix
+func (a SchemaItemsTable) WithSuffix(suffix string) *SchemaItemsTable {
+	return newSchemaItemsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newSchemaItemsTable(schemaName, tableName, alias string) *SchemaItemsTable {
+	return &SchemaItemsTable{
+		schemaItemsTable: newSchemaItemsTableImpl(schemaName, tableName, alias),
+		EXCLUDED:         newSchemaItemsTableImpl("", "excluded", ""),
+	}
+}
+
+func newSchemaItemsTableImpl(schemaName, tableName, alias string) schemaItemsTable {
+	var (
+		IDColumn          = postgres.StringColumn("id")
+		ItemColumn        = postgres.StringColumn("item")
+		ValueColumn       = postgres.StringColumn("value")
+		DescriptionColumn = postgres.StringColumn("description")
+		SchemaIDColumn    = postgres.StringColumn("schema_id")
+		allColumns        = postgres.ColumnList{IDColumn, ItemColumn, ValueColumn, DescriptionColumn, SchemaIDColumn}
+		mutableColumns    = postgres.ColumnList{ItemColumn, ValueColumn, DescriptionColumn, SchemaIDColumn}
+	)
+
+	return schemaItemsTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		ID:          IDColumn,
+		Item:        ItemColumn,
+		Value:       ValueColumn,
+		Description: DescriptionColumn,
+		SchemaID:    SchemaIDColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/.gen/haystack/haystack/table/schemas.go

@@ -0,0 +1,78 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var Schemas = newSchemasTable("haystack", "schemas", "")
+
+type schemasTable struct {
+	postgres.Table
+
+	// Columns
+	ID     postgres.ColumnString
+	ListID postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type SchemasTable struct {
+	schemasTable
+
+	EXCLUDED schemasTable
+}
+
+// AS creates new SchemasTable with assigned alias
+func (a SchemasTable) AS(alias string) *SchemasTable {
+	return newSchemasTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new SchemasTable with assigned schema name
+func (a SchemasTable) FromSchema(schemaName string) *SchemasTable {
+	return newSchemasTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new SchemasTable with assigned table prefix
+func (a SchemasTable) WithPrefix(prefix string) *SchemasTable {
+	return newSchemasTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new SchemasTable with assigned table suffix
+func (a SchemasTable) WithSuffix(suffix string) *SchemasTable {
+	return newSchemasTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newSchemasTable(schemaName, tableName, alias string) *SchemasTable {
+	return &SchemasTable{
+		schemasTable: newSchemasTableImpl(schemaName, tableName, alias),
+		EXCLUDED:     newSchemasTableImpl("", "excluded", ""),
+	}
+}
+
+func newSchemasTableImpl(schemaName, tableName, alias string) schemasTable {
+	var (
+		IDColumn       = postgres.StringColumn("id")
+		ListIDColumn   = postgres.StringColumn("list_id")
+		allColumns     = postgres.ColumnList{IDColumn, ListIDColumn}
+		mutableColumns = postgres.ColumnList{ListIDColumn}
+	)
+
+	return schemasTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		ID:     IDColumn,
+		ListID: ListIDColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/.gen/haystack/haystack/table/table_use_schema.go

@@ -17,9 +17,12 @@ func UseSchema(schema string) {
 	ImageEvents = ImageEvents.FromSchema(schema)
 	ImageLists = ImageLists.FromSchema(schema)
 	ImageLocations = ImageLocations.FromSchema(schema)
+	ImageSchemaItems = ImageSchemaItems.FromSchema(schema)
 	Lists = Lists.FromSchema(schema)
 	Locations = Locations.FromSchema(schema)
 	Logs = Logs.FromSchema(schema)
+	SchemaItems = SchemaItems.FromSchema(schema)
+	Schemas = Schemas.FromSchema(schema)
 	UserContacts = UserContacts.FromSchema(schema)
 	UserEvents = UserEvents.FromSchema(schema)
 	UserImages = UserImages.FromSchema(schema)

backend/agents/description_agent.go

@@ -41,15 +41,13 @@ func (agent DescriptionAgent) Describe(log *log.Logger, imageId uuid.UUID, image
 	log.Debug("Sending description request")
 	resp, err := agent.client.Request(&request)
 	if err != nil {
-		return fmt.Errorf("Could not request", err)
+		return fmt.Errorf("Could not request. %s", err)
 	}
 
 	ctx := context.Background()
 
 	markdown := resp.Choices[0].Message.Content
 
-	log.Debugf("Response %s", markdown)
-
 	err = agent.imageModel.AddDescription(ctx, imageId, markdown)
 	if err != nil {
 		return err

backend/agents/list_agent.go

@@ -27,13 +27,17 @@ An example of lists are:
 Another one of your tasks is to create a schema for this list. This should contain information that this, and following
 pictures contain. Be specific but also generic. You should use the parameters in "createList" to create this schema.
 
+This schema should not be super specific. You must be able to understand the image, and if the content of the image doesnt seem relevant, try
+and extract some meaning about what the image is.
+
 You must call "listLists" to see which available lists are already available.
+Use "createList" only once, don't create multiple lists for one image.
 
 **Tools:**
 * think: Internal reasoning/planning step.
 * listLists: Get existing lists
-* createList: Creates a new list with a name and description.
+* createList: Creates a new list with a name and description. Only use this once.
-* addToList: Add to an existing list.
+* addToList: Add to an existing list. This will also mean extracting information from this image, and inserting it, fitting the schema.
 * stopAgent: Signal task completion.
 `
 
@@ -89,19 +93,23 @@ const listTools = `
 						"items": {
 							"type": "object",
 							"properties": {
-								"key": {
+								"Item": {
 									"type": "string",
 									"description": "The name of the key for this specific field. Similar to a column in a database"
 								},
-								"value": {
+								"Value": {
 									"type": "string",
 									"enum": ["string", "number", "boolean"]
+								},
+								"Description": {
+									"type": "string",
+									"description": "The description for this item"
 								}
 							}
 						}
 					}
                 },
-                "required": ["name", "description"]
+                "required": ["name", "description", "schema"]
             }
         }
     },
@@ -116,9 +124,26 @@ const listTools = `
                     "listId": {
                         "type": "string",
                         "description": "The UUID of the existing list"
-                    }
+                    },
+					"schema": {
+						"type": "array",
+						"items": {
+							"type": "object",
+							"description": "A key-value of ID - value from this image to fit the schema. any of the values can be null",
+							"properties": {
+								"id": {
+									"type": "string",
+									"description": "The UUID of the schema item."
+								},
+								"value": {
+									"type": "string",
+									"description": "the concrete value for this field"
+								}
+							}
+						}
+					}
                 },
-                "required": ["listId"]
+                "required": ["listId", "schema"]
             }
 
 		}
@@ -141,13 +166,11 @@ type listListsArguments struct{}
 type createListArguments struct {
 	Name       string `json:"name"`
 	Desription string `json:"description"`
-	Schema     []struct {
+	Schema     []model.SchemaItems
-		Key   string `json:"key"`
-		Value string `json:"value"`
-	}
 }
 type addToListArguments struct {
 	ListID string `json:"listId"`
+	Schema []models.IDValue
 }
 
 func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClient {
@@ -173,15 +196,16 @@ func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClien
 			return model.Events{}, err
 		}
 
-		log.Debug("Create list", "schema", args.Schema)
-
 		ctx := context.Background()
-		savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription)
+		savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription, args.Schema)
 
 		if err != nil {
-			return model.Lists{}, err
+			log.Error(err)
+			return "", err
 		}
 
+		log.Debug(savedList)
+
 		return savedList, nil
 	})
 
@@ -199,7 +223,7 @@ func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClien
 			return "", err
 		}
 
-		if err := listModel.SaveInto(ctx, listUuid, info.ImageId); err != nil {
+		if err := listModel.SaveInto(ctx, listUuid, info.ImageId, args.Schema); err != nil {
 			return "", err
 		}
 

backend/models/lists.go

@@ -3,10 +3,12 @@ package models
 import (
 	"context"
 	"database/sql"
-	. "github.com/go-jet/jet/v2/postgres"
+	"fmt"
 	"screenmark/screenmark/.gen/haystack/haystack/model"
 	. "screenmark/screenmark/.gen/haystack/haystack/table"
 
+	. "github.com/go-jet/jet/v2/postgres"
+
 	"github.com/google/uuid"
 )
 
@@ -14,15 +16,74 @@ type ListModel struct {
 	dbPool *sql.DB
 }
 
-func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string) (model.Lists, error) {
+type ListWithItems struct {
+	model.Lists
+
+	Schema struct {
+		model.Schemas
+
+		SchemaItems []model.SchemaItems
+	}
+}
+
+func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string, schemaItems []model.SchemaItems) (ListWithItems, error) {
+	tx, err := m.dbPool.BeginTx(ctx, nil)
+
 	stmt := Lists.INSERT(Lists.UserID, Lists.Name, Lists.Description).
 		VALUES(userId, name, description).
 		RETURNING(Lists.ID, Lists.Name, Lists.Description)
 
 	newList := model.Lists{}
-	err := stmt.QueryContext(ctx, m.dbPool, &newList)
+	err = stmt.QueryContext(ctx, tx, &newList)
 
-	return newList, err
+	if err != nil {
+		tx.Rollback()
+		return ListWithItems{}, fmt.Errorf("Could not save new list. %s", err)
+	}
+
+	insertSchemaStmt := Schemas.INSERT(Schemas.ListID).
+		VALUES(newList.ID).
+		RETURNING(Schemas.ID)
+
+	newSchema := model.Schemas{}
+	err = insertSchemaStmt.QueryContext(ctx, tx, &newSchema)
+
+	if err != nil {
+		tx.Rollback()
+		return ListWithItems{}, fmt.Errorf("Could not save new schema. %s", err)
+	}
+
+	// This is very interesting...
+	for i := range schemaItems {
+		schemaItems[i].SchemaID = newSchema.ID
+	}
+
+	insertSchemaItemsStmt := SchemaItems.INSERT(SchemaItems.Item, SchemaItems.Value, SchemaItems.Description, SchemaItems.SchemaID).
+		MODELS(schemaItems)
+	_, err = insertSchemaItemsStmt.ExecContext(ctx, tx)
+
+	if err != nil {
+		tx.Rollback()
+		return ListWithItems{}, fmt.Errorf("Could not save schema items. %s", err)
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return ListWithItems{}, fmt.Errorf("Could not commit transaction. %s", err)
+	}
+
+	getListAndItems := SELECT(Lists.AllColumns, Schemas.AllColumns, SchemaItems.AllColumns).
+		FROM(
+			Lists.
+				INNER_JOIN(Schemas, Schemas.ListID.EQ(Lists.ID)).
+				INNER_JOIN(SchemaItems, SchemaItems.SchemaID.EQ(Schemas.ID)),
+		).
+		WHERE(Lists.ID.EQ(UUID(newList.ID)))
+
+	listWithItems := ListWithItems{}
+	err = getListAndItems.QueryContext(ctx, m.dbPool, &listWithItems)
+
+	return listWithItems, err
 }
 
 func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, error) {
@@ -35,12 +96,50 @@ func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, e
 	return lists, err
 }
 
-func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID) error {
+type IDValue struct {
+	ID    string `json:"id"`
+	Value string `json:"value"`
+}
+
+func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID, schemaValues []IDValue) error {
+	imageSchemaItems := make([]model.ImageSchemaItems, len(schemaValues))
+
+	for i, v := range schemaValues {
+		parsedId, err := uuid.Parse(v.ID)
+		if err != nil {
+			return err
+		}
+
+		imageSchemaItems[i].SchemaItemID = parsedId
+		imageSchemaItems[i].ImageID = imageId
+		imageSchemaItems[i].Value = &v.Value
+	}
+
+	tx, err := m.dbPool.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
 	stmt := ImageLists.INSERT(ImageLists.ListID, ImageLists.ImageID).
 		VALUES(listId, imageId)
 
-	_, err := stmt.ExecContext(ctx, m.dbPool)
+	_, err = stmt.ExecContext(ctx, tx)
+	if err != nil {
+		tx.Rollback()
+		return fmt.Errorf("Could not insert new list. %s", err)
+	}
 
+	insertSchemaItemsStmt := ImageSchemaItems.
+		INSERT(ImageSchemaItems.Value, ImageSchemaItems.SchemaItemID, ImageSchemaItems.ImageID).
+		MODELS(imageSchemaItems)
+
+	_, err = insertSchemaItemsStmt.ExecContext(ctx, tx)
+	if err != nil {
+		tx.Rollback()
+		return fmt.Errorf("Could not insert schema items. %s", err)
+	}
+
+	err = tx.Commit()
 	return err
 }
 

backend/schema.sql

@@ -145,6 +145,31 @@ CREATE TABLE haystack.image_lists (
     list_id UUID NOT NULL REFERENCES haystack.lists (id)
 );
 
+CREATE TABLE haystack.schemas (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+
+    list_id UUID NOT NULL REFERENCES haystack.lists (id)
+);
+
+CREATE TABLE haystack.schema_items (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+
+    item TEXT NOT NULL,
+    value TEXT NOT NULL,
+    description TEXT NOT NULL,
+
+    schema_id UUID NOT NULL REFERENCES haystack.schemas (id)
+);
+
+CREATE TABLE haystack.image_schema_items (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+
+    value TEXT,
+
+    schema_item_id UUID NOT NULL REFERENCES haystack.schema_items (id),
+    image_id UUID NOT NULL REFERENCES haystack.image (id)
+);
+
 /* -----| Indexes |----- */
 
 /* -----| Stored Procedures |----- */