126 lines
2.7 KiB
Go
126 lines
2.7 KiB
Go
package agents
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"screenmark/screenmark/agents/client"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
const orchestratorPrompt = `
|
|
You are an Orchestrator for various AI agents.
|
|
|
|
The user will send you images and you have to determine which agents you have to call, in order to best help the user.
|
|
|
|
You might decide no agent needs to be called.
|
|
|
|
The agents are available as tool calls.
|
|
|
|
Agents available:
|
|
|
|
eventLocationAgent
|
|
|
|
Use it when you think the image contains an event or a location of any sort. This can be an event page, a map, an address or a date.
|
|
|
|
noteAgent
|
|
|
|
Use it when there is text on the screen. Any text, always use this. Use me!
|
|
|
|
defaultAgent
|
|
|
|
When none of the above apply.
|
|
|
|
Always call agents in parallel if you need to call more than 1.
|
|
`
|
|
|
|
const MY_TOOLS = `
|
|
[
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "eventLocationAgent",
|
|
"description": "Uses the event location agent",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {},
|
|
"required": []
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "noteAgent",
|
|
"description": "Uses the note agent",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {},
|
|
"required": []
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "defaultAgent",
|
|
"description": "Used when you dont think its a good idea to call other agents",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {},
|
|
"required": []
|
|
}
|
|
}
|
|
}
|
|
]`
|
|
|
|
type OrchestratorAgent struct {
|
|
client client.AgentClient
|
|
}
|
|
|
|
func (agent OrchestratorAgent) Orchestrate(userId uuid.UUID, imageId uuid.UUID, imageName string, imageData []byte) error {
|
|
toolChoice := "any"
|
|
|
|
var tools any
|
|
err := json.Unmarshal([]byte(MY_TOOLS), &tools)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
request := client.AgentRequestBody{
|
|
Model: "pixtral-12b-2409",
|
|
Temperature: 0.3,
|
|
ResponseFormat: client.ResponseFormat{
|
|
Type: "text",
|
|
},
|
|
ToolChoice: &toolChoice,
|
|
Tools: &tools,
|
|
}
|
|
|
|
err = request.AddSystem(orchestratorPrompt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
request.AddImage(imageName, imageData)
|
|
resp, err := agent.client.Request(&request)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println(resp)
|
|
|
|
return nil
|
|
}
|
|
|
|
func NewOrchestratorAgent() (OrchestratorAgent, error) {
|
|
agent, err := client.CreateAgentClient(orchestratorPrompt)
|
|
if err != nil {
|
|
return OrchestratorAgent{}, err
|
|
}
|
|
|
|
return OrchestratorAgent{
|
|
client: agent,
|
|
}, nil
|
|
}
|