JohnCosta27/Haystack
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: working e2e solution

Browse Source
This commit is contained in:
John Costa
2025-03-20 17:59:00 +00:00
parent f114ca06d8
commit 47dd025ae3
8 changed files with 246 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
backend/ai_response.json
View File

@ -1,82 +1,108 @@
{ {
"name": "schema_description", "name": "image_info",
"schema": { "strict": true,
"type": "object", "schema": {
"properties": { "type": "object",
"tags": { "title": "image",
"type": "array", "required": ["tags", "text", "links"],
"description": "A list of tags you think the image is relevant to.", "additionalProperties": false,
"items": { "properties": {
"type": "string" "tags": {
} "type": "array",
}, "title": "tags",
"text": { "description": "A list of tags you think the image is relevant to.",
"type": "array", "items": {
"description": "A list of sentences the image contains.", "type": "string"
"items": { }
"type": "string" },
} "text": {
}, "type": "array",
"links": { "title": "text",
"type": "array", "description": "A list of sentences the image contains.",
"description": "A list of all the links you can find in the image.", "items": {
"items": { "type": "string"
"type": "string" }
} },
}, "links": {
"events": { "type": "array",
"type": "array", "title": "links",
"description": "A list of events you see on the image, mostly there usually only 1 or 0 events", "description": "A list of all the links you can find in the image.",
"items": { "items": {
"type": "object", "type": "string"
"properties": { }
"name": { },
"type": "string" "locations": {
}, "title": "locations",
"location": { "type": "array",
"type": "object", "description": "A list of locations you can find on the image, if any",
"properties": { "items": {
"name": { "type": "object",
"type": "string" "required": ["name"],
}, "additionalProperties": false,
"coordinates": { "properties": {
"type": ["string", "null"] "name": {
}, "title": "name",
"address": { "type": "string"
"type": ["string", "null"] },
}, "coordinates": {
"description": { "title": "coordinates",
"type": ["string", "null"] "type": "string"
} },
} "address": {
} "title": "address",
} "type": "string"
} },
}, "description": {
"locations": { "title": "description",
"type": "array", "type": "string"
"description": "A list of locations you see on the image, usually theres only 1 or 0 locations", }
"items": { }
"type": "object", }
"properties": { },
"name": { "events": {
"type": "string" "title": "events",
}, "type": "array",
"coordinates": { "description": "A list of events you find on the image, if any",
"type": ["string", "null"] "items": {
}, "type": "object",
"address": { "required": ["name"],
"type": ["string", "null"] "additionalProperties": false,
}, "properties": {
"description": { "name": {
"type": ["string", "null"] "type": "string",
} "title": "name"
} },
} "locations": {
} "title": "locations",
}, "type": "array",
"required": ["tags", "text", "links", "events", "locations"], "description": "A list of locations on this event, if any",
"additionalProperties": false "items": {
}, "type": "object",
"strict": true "required": ["name"],
"additionalProperties": false,
"properties": {
"name": {
"title": "name",
"type": "string"
},
"coordinates": {
"title": "coordinates",
"type": "string"
},
"address": {
"title": "address",
"type": "string"
},
"description": {
"title": "description",
"type": "string"
}
}
}
}
}
}
}
}
}
} }

5
backend/main.go
View File

@ -116,11 +116,14 @@ func main() {
imageInfo, err := openAiClient.GetImageInfo(image.Image.ImageName, image.Image.Image) imageInfo, err := openAiClient.GetImageInfo(image.Image.ImageName, image.Image.Image)
if err != nil { if err != nil {
log.Println("Failed to GetToProcessWithData") log.Println("Failed to GetImageInfo")
log.Println(err) log.Println(err)
return return
} }
log.Println("-----")
log.Println(imageInfo)
userImage, err := imageModel.FinishProcessing(ctx, image.ID) userImage, err := imageModel.FinishProcessing(ctx, image.ID)
if err != nil { if err != nil {
log.Println("Failed to FinishProcessing") log.Println("Failed to FinishProcessing")

4
backend/models/events.go
View File

@ -46,6 +46,10 @@ func (m EventModel) Save(ctx context.Context, events []model.Events) (model.Even
} }
func (m EventModel) SaveToImage(ctx context.Context, imageId uuid.UUID, events []model.Events) error { func (m EventModel) SaveToImage(ctx context.Context, imageId uuid.UUID, events []model.Events) error {
if len(events) == 0 {
return nil
}
event, err := m.Save(ctx, events) event, err := m.Save(ctx, events)
if err != nil { if err != nil {

4
backend/models/links.go
View File

@ -13,6 +13,10 @@ type LinkModel struct {
} }
func (m LinkModel) Save(ctx context.Context, imageId uuid.UUID, links []string) error { func (m LinkModel) Save(ctx context.Context, imageId uuid.UUID, links []string) error {
if len(links) == 0 {
return nil
}
stmt := ImageLinks.INSERT(ImageLinks.ImageID, ImageLinks.Link) stmt := ImageLinks.INSERT(ImageLinks.ImageID, ImageLinks.Link)
for _, link := range links { for _, link := range links {

4
backend/models/locations.go
View File

@ -58,6 +58,10 @@ func (m LocationModel) Save(ctx context.Context, locations []model.Locations) (m
} }
func (m LocationModel) SaveToImage(ctx context.Context, imageId uuid.UUID, locations []model.Locations) error { func (m LocationModel) SaveToImage(ctx context.Context, imageId uuid.UUID, locations []model.Locations) error {
if len(locations) == 0 {
return nil
}
location, err := m.Save(ctx, locations) location, err := m.Save(ctx, locations)
if err != nil { if err != nil {

8
backend/models/tags.go
View File

@ -28,6 +28,10 @@ type TagModel struct {
// | -- -- // | -- --
// | ---- IQ ---- // | ---- IQ ----
func (m TagModel) getNonExistantTags(ctx context.Context, userId uuid.UUID, tags []string) ([]string, error) { func (m TagModel) getNonExistantTags(ctx context.Context, userId uuid.UUID, tags []string) ([]string, error) {
if len(tags) == 0 {
return tags, nil
}
values := "" values := ""
counter := 1 counter := 1
// big big SQL injection problem here? // big big SQL injection problem here?
@ -107,6 +111,10 @@ func (m TagModel) List(ctx context.Context, userId uuid.UUID) ([]model.UserTags,
} }
func (m TagModel) SaveToImage(ctx context.Context, imageId uuid.UUID, tags []string) error { func (m TagModel) SaveToImage(ctx context.Context, imageId uuid.UUID, tags []string) error {
if len(tags) == 0 {
return nil
}
userId, err := getUserIdFromImage(ctx, m.dbPool, imageId) userId, err := getUserIdFromImage(ctx, m.dbPool, imageId)
if err != nil { if err != nil {
return err return err

4
backend/models/text.go
View File

@ -13,6 +13,10 @@ type TextModel struct {
} }
func (m TextModel) Save(ctx context.Context, imageId uuid.UUID, texts []string) error { func (m TextModel) Save(ctx context.Context, imageId uuid.UUID, texts []string) error {
if len(texts) == 0 {
return nil
}
saveImageTextStmt := ImageText.INSERT(ImageText.ImageID, ImageText.ImageText) saveImageTextStmt := ImageText.INSERT(ImageText.ImageID, ImageText.ImageText)
for _, t := range texts { for _, t := range texts {

192
backend/openai.go
View File

@ -145,90 +145,120 @@ You are an image information extractor. The user will provide you with screensho
that the image might contain. You will also try your best to assign some tags to this image, avoid too many tags. that the image might contain. You will also try your best to assign some tags to this image, avoid too many tags.
Be sure to extract every link (URL) that you find. Be sure to extract every link (URL) that you find.
Use generic tags. Use generic tags.
You also want to extract events in the image, and the location/locations this event is hosted in.
You need to extract locations in the image if any exist, and give the approximate coordinates for this location.
` `
const RESPONSE_FORMAT = ` const RESPONSE_FORMAT = `
{ {
"name": "schema_description", "name": "image_info",
"schema": { "strict": true,
"type": "object", "schema": {
"properties": { "type": "object",
"tags": { "title": "image",
"type": "array", "required": ["tags", "text", "links"],
"description": "A list of tags you think the image is relevant to.", "additionalProperties": false,
"items": { "properties": {
"type": "string" "tags": {
} "type": "array",
}, "title": "tags",
"text": { "description": "A list of tags you think the image is relevant to.",
"type": "array", "items": {
"description": "A list of sentences the image contains.", "type": "string"
"items": { }
"type": "string" },
} "text": {
}, "type": "array",
"links": { "title": "text",
"type": "array", "description": "A list of sentences the image contains.",
"description": "A list of all the links you can find in the image.", "items": {
"items": { "type": "string"
"type": "string" }
} },
}, "links": {
"events": { "type": "array",
"type": "array", "title": "links",
"description": "A list of events you see on the image, mostly there usually only 1 or 0 events", "description": "A list of all the links you can find in the image.",
"items": { "items": {
"type": "object", "type": "string"
"properties": { }
"name": { },
"type": "string" "locations": {
}, "title": "locations",
"location": { "type": "array",
"type": "object", "description": "A list of locations you can find on the image, if any",
"properties": { "items": {
"name": { "type": "object",
"type": "string" "required": ["name"],
}, "additionalProperties": false,
"coordinates": { "properties": {
"type": ["string", "null"] "name": {
}, "title": "name",
"address": { "type": "string"
"type": ["string", "null"] },
}, "coordinates": {
"description": { "title": "coordinates",
"type": ["string", "null"] "type": "string"
} },
} "address": {
} "title": "address",
} "type": "string"
} },
}, "description": {
"locations": { "title": "description",
"type": "array", "type": "string"
"description": "A list of locations you see on the image, usually theres only 1 or 0 locations", }
"items": { }
"type": "object", }
"properties": { },
"name": { "events": {
"type": "string" "title": "events",
}, "type": "array",
"coordinates": { "description": "A list of events you find on the image, if any",
"type": ["string", "null"] "items": {
}, "type": "object",
"address": { "required": ["name"],
"type": ["string", "null"] "additionalProperties": false,
}, "properties": {
"description": { "name": {
"type": ["string", "null"] "type": "string",
} "title": "name"
} },
} "locations": {
} "title": "locations",
}, "type": "array",
"required": ["tags", "text", "links", "events", "locations"], "description": "A list of locations on this event, if any",
"additionalProperties": false "items": {
}, "type": "object",
"strict": true "required": ["name"],
"additionalProperties": false,
"properties": {
"name": {
"title": "name",
"type": "string"
},
"coordinates": {
"title": "coordinates",
"type": "string"
},
"address": {
"title": "address",
"type": "string"
},
"description": {
"title": "description",
"type": "string"
}
}
}
}
}
}
}
}
}
} }
` `
@ -340,6 +370,8 @@ func (client OpenAiClient) GetImageInfo(imageName string, imageData []byte) (Ima
return ImageInfo{}, err return ImageInfo{}, err
} }
log.Println(jsonSchema)
aiRequest.ResponseFormat = ResponseFormat{ aiRequest.ResponseFormat = ResponseFormat{
Type: "json_schema", Type: "json_schema",
JsonSchema: jsonSchema, JsonSchema: jsonSchema,