From 47dd025ae36646019c0697a7c00c691c66cce05b Mon Sep 17 00:00:00 2001 From: John Costa Date: Thu, 20 Mar 2025 17:59:00 +0000 Subject: [PATCH] feat: working e2e solution --- backend/ai_response.json | 186 +++++++++++++++++++--------------- backend/main.go | 5 +- backend/models/events.go | 4 + backend/models/links.go | 4 + backend/models/locations.go | 4 + backend/models/tags.go | 8 ++ backend/models/text.go | 4 + backend/openai.go | 192 +++++++++++++++++++++--------------- 8 files changed, 246 insertions(+), 161 deletions(-) diff --git a/backend/ai_response.json b/backend/ai_response.json index b862fad..986efa6 100644 --- a/backend/ai_response.json +++ b/backend/ai_response.json @@ -1,82 +1,108 @@ { - "name": "schema_description", - "schema": { - "type": "object", - "properties": { - "tags": { - "type": "array", - "description": "A list of tags you think the image is relevant to.", - "items": { - "type": "string" - } - }, - "text": { - "type": "array", - "description": "A list of sentences the image contains.", - "items": { - "type": "string" - } - }, - "links": { - "type": "array", - "description": "A list of all the links you can find in the image.", - "items": { - "type": "string" - } - }, - "events": { - "type": "array", - "description": "A list of events you see on the image, mostly there usually only 1 or 0 events", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "location": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "coordinates": { - "type": ["string", "null"] - }, - "address": { - "type": ["string", "null"] - }, - "description": { - "type": ["string", "null"] - } - } - } - } - } - }, - "locations": { - "type": "array", - "description": "A list of locations you see on the image, usually theres only 1 or 0 locations", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "coordinates": { - "type": ["string", "null"] - }, - "address": { - "type": ["string", "null"] - }, - "description": { - "type": ["string", "null"] - } - } - } - } - }, - "required": ["tags", "text", "links", "events", "locations"], - "additionalProperties": false - }, - "strict": true + "name": "image_info", + "strict": true, + "schema": { + "type": "object", + "title": "image", + "required": ["tags", "text", "links"], + "additionalProperties": false, + "properties": { + "tags": { + "type": "array", + "title": "tags", + "description": "A list of tags you think the image is relevant to.", + "items": { + "type": "string" + } + }, + "text": { + "type": "array", + "title": "text", + "description": "A list of sentences the image contains.", + "items": { + "type": "string" + } + }, + "links": { + "type": "array", + "title": "links", + "description": "A list of all the links you can find in the image.", + "items": { + "type": "string" + } + }, + "locations": { + "title": "locations", + "type": "array", + "description": "A list of locations you can find on the image, if any", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "title": "name", + "type": "string" + }, + "coordinates": { + "title": "coordinates", + "type": "string" + }, + "address": { + "title": "address", + "type": "string" + }, + "description": { + "title": "description", + "type": "string" + } + } + } + }, + "events": { + "title": "events", + "type": "array", + "description": "A list of events you find on the image, if any", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "title": "name" + }, + "locations": { + "title": "locations", + "type": "array", + "description": "A list of locations on this event, if any", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "title": "name", + "type": "string" + }, + "coordinates": { + "title": "coordinates", + "type": "string" + }, + "address": { + "title": "address", + "type": "string" + }, + "description": { + "title": "description", + "type": "string" + } + } + } + } + } + } + } + } + } } diff --git a/backend/main.go b/backend/main.go index 5d6bbab..4063404 100644 --- a/backend/main.go +++ b/backend/main.go @@ -116,11 +116,14 @@ func main() { imageInfo, err := openAiClient.GetImageInfo(image.Image.ImageName, image.Image.Image) if err != nil { - log.Println("Failed to GetToProcessWithData") + log.Println("Failed to GetImageInfo") log.Println(err) return } + log.Println("-----") + log.Println(imageInfo) + userImage, err := imageModel.FinishProcessing(ctx, image.ID) if err != nil { log.Println("Failed to FinishProcessing") diff --git a/backend/models/events.go b/backend/models/events.go index 8b15231..c4fb572 100644 --- a/backend/models/events.go +++ b/backend/models/events.go @@ -46,6 +46,10 @@ func (m EventModel) Save(ctx context.Context, events []model.Events) (model.Even } func (m EventModel) SaveToImage(ctx context.Context, imageId uuid.UUID, events []model.Events) error { + if len(events) == 0 { + return nil + } + event, err := m.Save(ctx, events) if err != nil { diff --git a/backend/models/links.go b/backend/models/links.go index e598b10..94f9dcc 100644 --- a/backend/models/links.go +++ b/backend/models/links.go @@ -13,6 +13,10 @@ type LinkModel struct { } func (m LinkModel) Save(ctx context.Context, imageId uuid.UUID, links []string) error { + if len(links) == 0 { + return nil + } + stmt := ImageLinks.INSERT(ImageLinks.ImageID, ImageLinks.Link) for _, link := range links { diff --git a/backend/models/locations.go b/backend/models/locations.go index 2cfcde4..60d786a 100644 --- a/backend/models/locations.go +++ b/backend/models/locations.go @@ -58,6 +58,10 @@ func (m LocationModel) Save(ctx context.Context, locations []model.Locations) (m } func (m LocationModel) SaveToImage(ctx context.Context, imageId uuid.UUID, locations []model.Locations) error { + if len(locations) == 0 { + return nil + } + location, err := m.Save(ctx, locations) if err != nil { diff --git a/backend/models/tags.go b/backend/models/tags.go index 4d32f95..949df90 100644 --- a/backend/models/tags.go +++ b/backend/models/tags.go @@ -28,6 +28,10 @@ type TagModel struct { // | -- -- // | ---- IQ ---- func (m TagModel) getNonExistantTags(ctx context.Context, userId uuid.UUID, tags []string) ([]string, error) { + if len(tags) == 0 { + return tags, nil + } + values := "" counter := 1 // big big SQL injection problem here? @@ -107,6 +111,10 @@ func (m TagModel) List(ctx context.Context, userId uuid.UUID) ([]model.UserTags, } func (m TagModel) SaveToImage(ctx context.Context, imageId uuid.UUID, tags []string) error { + if len(tags) == 0 { + return nil + } + userId, err := getUserIdFromImage(ctx, m.dbPool, imageId) if err != nil { return err diff --git a/backend/models/text.go b/backend/models/text.go index 031d01b..2f5c9ce 100644 --- a/backend/models/text.go +++ b/backend/models/text.go @@ -13,6 +13,10 @@ type TextModel struct { } func (m TextModel) Save(ctx context.Context, imageId uuid.UUID, texts []string) error { + if len(texts) == 0 { + return nil + } + saveImageTextStmt := ImageText.INSERT(ImageText.ImageID, ImageText.ImageText) for _, t := range texts { diff --git a/backend/openai.go b/backend/openai.go index b3481f8..0eab6af 100644 --- a/backend/openai.go +++ b/backend/openai.go @@ -145,90 +145,120 @@ You are an image information extractor. The user will provide you with screensho that the image might contain. You will also try your best to assign some tags to this image, avoid too many tags. Be sure to extract every link (URL) that you find. Use generic tags. + +You also want to extract events in the image, and the location/locations this event is hosted in. + +You need to extract locations in the image if any exist, and give the approximate coordinates for this location. ` const RESPONSE_FORMAT = ` { - "name": "schema_description", - "schema": { - "type": "object", - "properties": { - "tags": { - "type": "array", - "description": "A list of tags you think the image is relevant to.", - "items": { - "type": "string" - } - }, - "text": { - "type": "array", - "description": "A list of sentences the image contains.", - "items": { - "type": "string" - } - }, - "links": { - "type": "array", - "description": "A list of all the links you can find in the image.", - "items": { - "type": "string" - } - }, - "events": { - "type": "array", - "description": "A list of events you see on the image, mostly there usually only 1 or 0 events", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "location": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "coordinates": { - "type": ["string", "null"] - }, - "address": { - "type": ["string", "null"] - }, - "description": { - "type": ["string", "null"] - } - } - } - } - } - }, - "locations": { - "type": "array", - "description": "A list of locations you see on the image, usually theres only 1 or 0 locations", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "coordinates": { - "type": ["string", "null"] - }, - "address": { - "type": ["string", "null"] - }, - "description": { - "type": ["string", "null"] - } - } - } - } - }, - "required": ["tags", "text", "links", "events", "locations"], - "additionalProperties": false - }, - "strict": true + "name": "image_info", + "strict": true, + "schema": { + "type": "object", + "title": "image", + "required": ["tags", "text", "links"], + "additionalProperties": false, + "properties": { + "tags": { + "type": "array", + "title": "tags", + "description": "A list of tags you think the image is relevant to.", + "items": { + "type": "string" + } + }, + "text": { + "type": "array", + "title": "text", + "description": "A list of sentences the image contains.", + "items": { + "type": "string" + } + }, + "links": { + "type": "array", + "title": "links", + "description": "A list of all the links you can find in the image.", + "items": { + "type": "string" + } + }, + "locations": { + "title": "locations", + "type": "array", + "description": "A list of locations you can find on the image, if any", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "title": "name", + "type": "string" + }, + "coordinates": { + "title": "coordinates", + "type": "string" + }, + "address": { + "title": "address", + "type": "string" + }, + "description": { + "title": "description", + "type": "string" + } + } + } + }, + "events": { + "title": "events", + "type": "array", + "description": "A list of events you find on the image, if any", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "title": "name" + }, + "locations": { + "title": "locations", + "type": "array", + "description": "A list of locations on this event, if any", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "title": "name", + "type": "string" + }, + "coordinates": { + "title": "coordinates", + "type": "string" + }, + "address": { + "title": "address", + "type": "string" + }, + "description": { + "title": "description", + "type": "string" + } + } + } + } + } + } + } + } + } } ` @@ -340,6 +370,8 @@ func (client OpenAiClient) GetImageInfo(imageName string, imageData []byte) (Ima return ImageInfo{}, err } + log.Println(jsonSchema) + aiRequest.ResponseFormat = ResponseFormat{ Type: "json_schema", JsonSchema: jsonSchema,