JohnCosta27/Haystack
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: initial draft of generating a schema from one image

Browse Source 
fix: error formatting
This commit is contained in:
John Costa
2025-07-29 11:35:56 +01:00
parent 88d033314e
commit 8597584cf0
11 changed files with 479 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
backend/.gen/haystack/haystack/model/image_schema_items.go Normal file
View File

@ -0,0 +1,19 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
)
type ImageSchemaItems struct {
ID uuid.UUID `sql:"primary_key"`
Value *string
SchemaItemID uuid.UUID
ImageID uuid.UUID
}

20
backend/.gen/haystack/haystack/model/schema_items.go Normal file
View File

@ -0,0 +1,20 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
)
type SchemaItems struct {
ID uuid.UUID `sql:"primary_key"`
Item string
Value string
Description string
SchemaID uuid.UUID
}

17
backend/.gen/haystack/haystack/model/schemas.go Normal file
View File

@ -0,0 +1,17 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
)
type Schemas struct {
ID uuid.UUID `sql:"primary_key"`
ListID uuid.UUID
}

84
backend/.gen/haystack/haystack/table/image_schema_items.go Normal file
View File

@ -0,0 +1,84 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var ImageSchemaItems = newImageSchemaItemsTable("haystack", "image_schema_items", "")
type imageSchemaItemsTable struct {
postgres.Table
// Columns
ID postgres.ColumnString
Value postgres.ColumnString
SchemaItemID postgres.ColumnString
ImageID postgres.ColumnString
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
}
type ImageSchemaItemsTable struct {
imageSchemaItemsTable
EXCLUDED imageSchemaItemsTable
}
// AS creates new ImageSchemaItemsTable with assigned alias
func (a ImageSchemaItemsTable) AS(alias string) *ImageSchemaItemsTable {
return newImageSchemaItemsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new ImageSchemaItemsTable with assigned schema name
func (a ImageSchemaItemsTable) FromSchema(schemaName string) *ImageSchemaItemsTable {
return newImageSchemaItemsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new ImageSchemaItemsTable with assigned table prefix
func (a ImageSchemaItemsTable) WithPrefix(prefix string) *ImageSchemaItemsTable {
return newImageSchemaItemsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new ImageSchemaItemsTable with assigned table suffix
func (a ImageSchemaItemsTable) WithSuffix(suffix string) *ImageSchemaItemsTable {
return newImageSchemaItemsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newImageSchemaItemsTable(schemaName, tableName, alias string) *ImageSchemaItemsTable {
return &ImageSchemaItemsTable{
imageSchemaItemsTable: newImageSchemaItemsTableImpl(schemaName, tableName, alias),
EXCLUDED: newImageSchemaItemsTableImpl("", "excluded", ""),
}
}
func newImageSchemaItemsTableImpl(schemaName, tableName, alias string) imageSchemaItemsTable {
var (
IDColumn = postgres.StringColumn("id")
ValueColumn = postgres.StringColumn("value")
SchemaItemIDColumn = postgres.StringColumn("schema_item_id")
ImageIDColumn = postgres.StringColumn("image_id")
allColumns = postgres.ColumnList{IDColumn, ValueColumn, SchemaItemIDColumn, ImageIDColumn}
mutableColumns = postgres.ColumnList{ValueColumn, SchemaItemIDColumn, ImageIDColumn}
)
return imageSchemaItemsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
ID: IDColumn,
Value: ValueColumn,
SchemaItemID: SchemaItemIDColumn,
ImageID: ImageIDColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
}
}

87
backend/.gen/haystack/haystack/table/schema_items.go Normal file
View File

@ -0,0 +1,87 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var SchemaItems = newSchemaItemsTable("haystack", "schema_items", "")
type schemaItemsTable struct {
postgres.Table
// Columns
ID postgres.ColumnString
Item postgres.ColumnString
Value postgres.ColumnString
Description postgres.ColumnString
SchemaID postgres.ColumnString
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
}
type SchemaItemsTable struct {
schemaItemsTable
EXCLUDED schemaItemsTable
}
// AS creates new SchemaItemsTable with assigned alias
func (a SchemaItemsTable) AS(alias string) *SchemaItemsTable {
return newSchemaItemsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new SchemaItemsTable with assigned schema name
func (a SchemaItemsTable) FromSchema(schemaName string) *SchemaItemsTable {
return newSchemaItemsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new SchemaItemsTable with assigned table prefix
func (a SchemaItemsTable) WithPrefix(prefix string) *SchemaItemsTable {
return newSchemaItemsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new SchemaItemsTable with assigned table suffix
func (a SchemaItemsTable) WithSuffix(suffix string) *SchemaItemsTable {
return newSchemaItemsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newSchemaItemsTable(schemaName, tableName, alias string) *SchemaItemsTable {
return &SchemaItemsTable{
schemaItemsTable: newSchemaItemsTableImpl(schemaName, tableName, alias),
EXCLUDED: newSchemaItemsTableImpl("", "excluded", ""),
}
}
func newSchemaItemsTableImpl(schemaName, tableName, alias string) schemaItemsTable {
var (
IDColumn = postgres.StringColumn("id")
ItemColumn = postgres.StringColumn("item")
ValueColumn = postgres.StringColumn("value")
DescriptionColumn = postgres.StringColumn("description")
SchemaIDColumn = postgres.StringColumn("schema_id")
allColumns = postgres.ColumnList{IDColumn, ItemColumn, ValueColumn, DescriptionColumn, SchemaIDColumn}
mutableColumns = postgres.ColumnList{ItemColumn, ValueColumn, DescriptionColumn, SchemaIDColumn}
)
return schemaItemsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
ID: IDColumn,
Item: ItemColumn,
Value: ValueColumn,
Description: DescriptionColumn,
SchemaID: SchemaIDColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
}
}

78
backend/.gen/haystack/haystack/table/schemas.go Normal file
View File

@ -0,0 +1,78 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var Schemas = newSchemasTable("haystack", "schemas", "")
type schemasTable struct {
postgres.Table
// Columns
ID postgres.ColumnString
ListID postgres.ColumnString
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
}
type SchemasTable struct {
schemasTable
EXCLUDED schemasTable
}
// AS creates new SchemasTable with assigned alias
func (a SchemasTable) AS(alias string) *SchemasTable {
return newSchemasTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new SchemasTable with assigned schema name
func (a SchemasTable) FromSchema(schemaName string) *SchemasTable {
return newSchemasTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new SchemasTable with assigned table prefix
func (a SchemasTable) WithPrefix(prefix string) *SchemasTable {
return newSchemasTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new SchemasTable with assigned table suffix
func (a SchemasTable) WithSuffix(suffix string) *SchemasTable {
return newSchemasTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newSchemasTable(schemaName, tableName, alias string) *SchemasTable {
return &SchemasTable{
schemasTable: newSchemasTableImpl(schemaName, tableName, alias),
EXCLUDED: newSchemasTableImpl("", "excluded", ""),
}
}
func newSchemasTableImpl(schemaName, tableName, alias string) schemasTable {
var (
IDColumn = postgres.StringColumn("id")
ListIDColumn = postgres.StringColumn("list_id")
allColumns = postgres.ColumnList{IDColumn, ListIDColumn}
mutableColumns = postgres.ColumnList{ListIDColumn}
)
return schemasTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
ID: IDColumn,
ListID: ListIDColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
}
}

3
backend/.gen/haystack/haystack/table/table_use_schema.go
View File

@ -17,9 +17,12 @@ func UseSchema(schema string) {
ImageEvents = ImageEvents.FromSchema(schema)
ImageLists = ImageLists.FromSchema(schema)
ImageLocations = ImageLocations.FromSchema(schema)
ImageSchemaItems = ImageSchemaItems.FromSchema(schema)
Lists = Lists.FromSchema(schema)
Locations = Locations.FromSchema(schema)
Logs = Logs.FromSchema(schema)
SchemaItems = SchemaItems.FromSchema(schema)
Schemas = Schemas.FromSchema(schema)
UserContacts = UserContacts.FromSchema(schema)
UserEvents = UserEvents.FromSchema(schema)
UserImages = UserImages.FromSchema(schema)

4
backend/agents/description_agent.go
View File

@ -41,15 +41,13 @@ func (agent DescriptionAgent) Describe(log *log.Logger, imageId uuid.UUID, image
log.Debug("Sending description request")
resp, err := agent.client.Request(&request)
if err != nil {
return fmt.Errorf("Could not request", err)
return fmt.Errorf("Could not request. %s", err)
}
ctx := context.Background()
markdown := resp.Choices[0].Message.Content
log.Debugf("Response %s", markdown)
err = agent.imageModel.AddDescription(ctx, imageId, markdown)
if err != nil {
return err

54
backend/agents/list_agent.go
View File

@ -27,13 +27,17 @@ An example of lists are:
Another one of your tasks is to create a schema for this list. This should contain information that this, and following
pictures contain. Be specific but also generic. You should use the parameters in "createList" to create this schema.
This schema should not be super specific. You must be able to understand the image, and if the content of the image doesnt seem relevant, try
and extract some meaning about what the image is.
You must call "listLists" to see which available lists are already available.
Use "createList" only once, don't create multiple lists for one image.
**Tools:**
* think: Internal reasoning/planning step.
* listLists: Get existing lists
* createList: Creates a new list with a name and description.
* addToList: Add to an existing list.
* createList: Creates a new list with a name and description. Only use this once.
* addToList: Add to an existing list. This will also mean extracting information from this image, and inserting it, fitting the schema.
* stopAgent: Signal task completion.
`
@ -89,19 +93,23 @@ const listTools = `
"items": {
"type": "object",
"properties": {
"key": {
"Item": {
"type": "string",
"description": "The name of the key for this specific field. Similar to a column in a database"
},
"value": {
"Value": {
"type": "string",
"enum": ["string", "number", "boolean"]
},
"Description": {
"type": "string",
"description": "The description for this item"
}
}
}
}
},
"required": ["name", "description"]
"required": ["name", "description", "schema"]
}
}
},
@ -116,9 +124,26 @@ const listTools = `
"listId": {
"type": "string",
"description": "The UUID of the existing list"
},
"schema": {
"type": "array",
"items": {
"type": "object",
"description": "A key-value of ID - value from this image to fit the schema. any of the values can be null",
"properties": {
"id": {
"type": "string",
"description": "The UUID of the schema item."
},
"value": {
"type": "string",
"description": "the concrete value for this field"
}
}
}
}
},
"required": ["listId"]
"required": ["listId", "schema"]
}
}
@ -141,13 +166,11 @@ type listListsArguments struct{}
type createListArguments struct {
Name string `json:"name"`
Desription string `json:"description"`
Schema []struct {
Key string `json:"key"`
Value string `json:"value"`
}
Schema []model.SchemaItems
}
type addToListArguments struct {
ListID string `json:"listId"`
Schema []models.IDValue
}
func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClient {
@ -173,15 +196,16 @@ func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClien
return model.Events{}, err
}
log.Debug("Create list", "schema", args.Schema)
ctx := context.Background()
savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription)
savedList, err := listModel.Save(ctx, info.UserId, args.Name, args.Desription, args.Schema)
if err != nil {
return model.Lists{}, err
log.Error(err)
return "", err
}
log.Debug(savedList)
return savedList, nil
})
@ -199,7 +223,7 @@ func NewListAgent(log *log.Logger, listModel models.ListModel) client.AgentClien
return "", err
}
if err := listModel.SaveInto(ctx, listUuid, info.ImageId); err != nil {
if err := listModel.SaveInto(ctx, listUuid, info.ImageId, args.Schema); err != nil {
return "", err
}

111
backend/models/lists.go
View File

@ -3,10 +3,12 @@ package models
import (
"context"
"database/sql"
. "github.com/go-jet/jet/v2/postgres"
"fmt"
"screenmark/screenmark/.gen/haystack/haystack/model"
. "screenmark/screenmark/.gen/haystack/haystack/table"
. "github.com/go-jet/jet/v2/postgres"
"github.com/google/uuid"
)
@ -14,15 +16,74 @@ type ListModel struct {
dbPool *sql.DB
}
func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string) (model.Lists, error) {
type ListWithItems struct {
model.Lists
Schema struct {
model.Schemas
SchemaItems []model.SchemaItems
}
}
func (m ListModel) Save(ctx context.Context, userId uuid.UUID, name string, description string, schemaItems []model.SchemaItems) (ListWithItems, error) {
tx, err := m.dbPool.BeginTx(ctx, nil)
stmt := Lists.INSERT(Lists.UserID, Lists.Name, Lists.Description).
VALUES(userId, name, description).
RETURNING(Lists.ID, Lists.Name, Lists.Description)
newList := model.Lists{}
err := stmt.QueryContext(ctx, m.dbPool, &newList)
err = stmt.QueryContext(ctx, tx, &newList)
return newList, err
if err != nil {
tx.Rollback()
return ListWithItems{}, fmt.Errorf("Could not save new list. %s", err)
}
insertSchemaStmt := Schemas.INSERT(Schemas.ListID).
VALUES(newList.ID).
RETURNING(Schemas.ID)
newSchema := model.Schemas{}
err = insertSchemaStmt.QueryContext(ctx, tx, &newSchema)
if err != nil {
tx.Rollback()
return ListWithItems{}, fmt.Errorf("Could not save new schema. %s", err)
}
// This is very interesting...
for i := range schemaItems {
schemaItems[i].SchemaID = newSchema.ID
}
insertSchemaItemsStmt := SchemaItems.INSERT(SchemaItems.Item, SchemaItems.Value, SchemaItems.Description, SchemaItems.SchemaID).
MODELS(schemaItems)
_, err = insertSchemaItemsStmt.ExecContext(ctx, tx)
if err != nil {
tx.Rollback()
return ListWithItems{}, fmt.Errorf("Could not save schema items. %s", err)
}
err = tx.Commit()
if err != nil {
return ListWithItems{}, fmt.Errorf("Could not commit transaction. %s", err)
}
getListAndItems := SELECT(Lists.AllColumns, Schemas.AllColumns, SchemaItems.AllColumns).
FROM(
Lists.
INNER_JOIN(Schemas, Schemas.ListID.EQ(Lists.ID)).
INNER_JOIN(SchemaItems, SchemaItems.SchemaID.EQ(Schemas.ID)),
).
WHERE(Lists.ID.EQ(UUID(newList.ID)))
listWithItems := ListWithItems{}
err = getListAndItems.QueryContext(ctx, m.dbPool, &listWithItems)
return listWithItems, err
}
func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, error) {
@ -35,12 +96,50 @@ func (m ListModel) List(ctx context.Context, userId uuid.UUID) ([]model.Lists, e
return lists, err
}
func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID) error {
type IDValue struct {
ID string `json:"id"`
Value string `json:"value"`
}
func (m ListModel) SaveInto(ctx context.Context, listId uuid.UUID, imageId uuid.UUID, schemaValues []IDValue) error {
imageSchemaItems := make([]model.ImageSchemaItems, len(schemaValues))
for i, v := range schemaValues {
parsedId, err := uuid.Parse(v.ID)
if err != nil {
return err
}
imageSchemaItems[i].SchemaItemID = parsedId
imageSchemaItems[i].ImageID = imageId
imageSchemaItems[i].Value = &v.Value
}
tx, err := m.dbPool.BeginTx(ctx, nil)
if err != nil {
return err
}
stmt := ImageLists.INSERT(ImageLists.ListID, ImageLists.ImageID).
VALUES(listId, imageId)
_, err := stmt.ExecContext(ctx, m.dbPool)
_, err = stmt.ExecContext(ctx, tx)
if err != nil {
tx.Rollback()
return fmt.Errorf("Could not insert new list. %s", err)
}
insertSchemaItemsStmt := ImageSchemaItems.
INSERT(ImageSchemaItems.Value, ImageSchemaItems.SchemaItemID, ImageSchemaItems.ImageID).
MODELS(imageSchemaItems)
_, err = insertSchemaItemsStmt.ExecContext(ctx, tx)
if err != nil {
tx.Rollback()
return fmt.Errorf("Could not insert schema items. %s", err)
}
err = tx.Commit()
return err
}

25
backend/schema.sql
View File

@ -145,6 +145,31 @@ CREATE TABLE haystack.image_lists (
list_id UUID NOT NULL REFERENCES haystack.lists (id)
);
CREATE TABLE haystack.schemas (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
list_id UUID NOT NULL REFERENCES haystack.lists (id)
);
CREATE TABLE haystack.schema_items (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
item TEXT NOT NULL,
value TEXT NOT NULL,
description TEXT NOT NULL,
schema_id UUID NOT NULL REFERENCES haystack.schemas (id)
);
CREATE TABLE haystack.image_schema_items (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
value TEXT,
schema_item_id UUID NOT NULL REFERENCES haystack.schema_items (id),
image_id UUID NOT NULL REFERENCES haystack.image (id)
);
/* -----| Indexes |----- */
/* -----| Stored Procedures |----- */