feat: creating list processor

backend/agents/create_list_agent.go

@@ -21,6 +21,8 @@ and add a good description for each one.
 
 You can add fields if you think they make a lot of sense.
 You can remove fields if they are not correct, but be sure before you do this.
+
+You must respond in json format
 `
 
 const listJsonSchema = `
@@ -76,10 +78,10 @@ type createNewListArguments struct {
 type CreateListAgent struct {
 	client client.AgentClient
 
-	listModel models.StackModel
+	stackModel models.StackModel
 }
 
-func (agent *CreateListAgent) CreateList(log *log.Logger, userID uuid.UUID, userReq string) error {
+func (agent *CreateListAgent) CreateList(log *log.Logger, userID uuid.UUID, stackID uuid.UUID, title string, userReq string) error {
 	request := client.AgentRequestBody{
 		Model:       "policy/images",
 		Temperature: 0.3,
@@ -93,7 +95,10 @@ func (agent *CreateListAgent) CreateList(log *log.Logger, userID uuid.UUID, user
 	}
 
 	request.Chat.AddSystem(agent.client.Options.SystemPrompt)
-	request.Chat.AddUser(userReq)
+
+	req := fmt.Sprintf("List title: %s | Users list description: %s", title, userReq)
+
+	request.Chat.AddUser(req)
 
 	resp, err := agent.client.Request(&request)
 	if err != nil {
@@ -104,6 +109,8 @@ func (agent *CreateListAgent) CreateList(log *log.Logger, userID uuid.UUID, user
 
 	structuredOutput := resp.Choices[0].Message.Content
 
+	log.Info("", "res", structuredOutput)
+
 	var createListArgs createNewListArguments
 	err = json.Unmarshal([]byte(structuredOutput), &createListArgs)
 	if err != nil {
@@ -113,6 +120,8 @@ func (agent *CreateListAgent) CreateList(log *log.Logger, userID uuid.UUID, user
 	schemaItems := make([]model.SchemaItems, 0)
 	for _, field := range createListArgs.Fields {
 		schemaItems = append(schemaItems, model.SchemaItems{
+			StackID: stackID,
+
 			Item:        field.Name,
 			Description: field.Description,
 
@@ -120,12 +129,7 @@ func (agent *CreateListAgent) CreateList(log *log.Logger, userID uuid.UUID, user
 		})
 	}
 
-	_, err = agent.listModel.Save(ctx, userID, createListArgs.Title, createListArgs.Description, model.Progress_Complete)
-	if err != nil {
-		return fmt.Errorf("creating list agent, saving list: %w", err)
-	}
-
-	err = agent.listModel.SaveItems(ctx, schemaItems)
+	err = agent.stackModel.SaveItems(ctx, schemaItems)
 	if err != nil {
 		return fmt.Errorf("creating list agent, saving items: %w", err)
 	}

backend/agents/list_agent.go

@@ -176,7 +176,7 @@ type addToListArguments struct {
 	Schema []models.IDValue
 }
 
-func NewListAgent(log *log.Logger, stackModel models.StackModel, limitsMethods limits.LimitsManagerMethods) client.AgentClient {
+func NewStackAgent(log *log.Logger, stackModel models.StackModel, limitsMethods limits.LimitsManagerMethods) client.AgentClient {
 	agentClient := client.CreateAgentClient(client.CreateAgentClientOptions{
 		SystemPrompt: listPrompt,
 		JsonTools:    listTools,

backend/models/stacks.go

@@ -142,6 +142,20 @@ func (m StackModel) SaveSchemaItems(ctx context.Context, imageID uuid.UUID, item
 	return err
 }
 
+// ========================================
+// UPDATE methods
+// ========================================
+
+func (m StackModel) UpdateProcess(ctx context.Context, stackID uuid.UUID, process model.Progress) error {
+	updateStackProgressStmt := Stacks.UPDATE(Stacks.Status).
+		SET(process).
+		WHERE(Stacks.ID.EQ(UUID(stackID)))
+
+	_, err := updateStackProgressStmt.ExecContext(ctx, m.dbPool)
+
+	return err
+}
+
 // ========================================
 // DELETE methods
 // ========================================

backend/notifications/entities_notification.go

@@ -9,7 +9,7 @@ import (
 
 const (
 	IMAGE_TYPE = "image"
-	LIST_TYPE  = "list"
+	STACK_TYPE = "stack"
 )
 
 type ImageNotification struct {
@@ -21,10 +21,10 @@ type ImageNotification struct {
 	Status string
 }
 
-type ListNotification struct {
+type StackNotification struct {
 	Type string
 
-	ListID uuid.UUID
+	StackID uuid.UUID
 	Name    string
 
 	Status string
@@ -32,7 +32,7 @@ type ListNotification struct {
 
 type Notification struct {
 	image *ImageNotification
-	list  *ListNotification
+	stack *StackNotification
 }
 
 func GetImageNotification(image ImageNotification) Notification {
@@ -41,9 +41,9 @@ func GetImageNotification(image ImageNotification) Notification {
 	}
 }
 
-func GetListNotification(list ListNotification) Notification {
+func GetStackNotification(list StackNotification) Notification {
 	return Notification{
-		list: &list,
+		stack: &list,
 	}
 }
 
@@ -52,8 +52,8 @@ func (n Notification) MarshalJSON() ([]byte, error) {
 		return json.Marshal(n.image)
 	}
 
-	if n.list != nil {
-		return json.Marshal(n.list)
+	if n.stack != nil {
+		return json.Marshal(n.stack)
 	}
 
 	return nil, fmt.Errorf("no image or list present")

backend/processor/image.go

@@ -23,10 +23,7 @@ type ImageProcessor struct {
 	descriptionAgent agents.DescriptionAgent
 	stackAgent       client.AgentClient
 
-	// TODO: add the notifier here
-
 	Processor *Processor[model.Image]
-
 	notifier  *notifications.Notifier[notifications.Notification]
 }
 
@@ -128,7 +125,7 @@ func NewImageProcessor(
 	}
 
 	descriptionAgent := agents.NewDescriptionAgent(logger, imageModel)
-	stackAgent := agents.NewListAgent(logger, listModel, limitsManager)
+	stackAgent := agents.NewStackAgent(logger, listModel, limitsManager)
 
 	imageProcessor := ImageProcessor{
 		imageModel:       imageModel,

backend/processor/stack.go

@@ -0,0 +1,127 @@
+package processor
+
+import (
+	"context"
+	"fmt"
+	"screenmark/screenmark/.gen/haystack/haystack/model"
+	"screenmark/screenmark/agents"
+	"screenmark/screenmark/models"
+	"screenmark/screenmark/notifications"
+	"sync"
+
+	"github.com/charmbracelet/log"
+)
+
+const STACK_PROCESS_AT_A_TIME = 10
+
+// TODO:
+// This processor contains a lot of shared stuff.
+// If we ever want to do more generic stuff with "in-progress" and stuff
+// we can extract that into a common thing
+//
+// However, this will require a pretty big DB shuffle.
+
+type StackProcessor struct {
+	stackModel models.StackModel
+	logger     *log.Logger
+
+	stackAgent agents.CreateListAgent
+
+	Processor *Processor[model.Stacks]
+
+	notifier *notifications.Notifier[notifications.Notification]
+}
+
+func (p *StackProcessor) setStackToProcess(ctx context.Context, stack model.Stacks) {
+	err := p.stackModel.UpdateProcess(ctx, stack.ID, model.Progress_InProgress)
+	if err != nil {
+		// TODO: what can we actually do here for the errors?
+		// We can't stop the work for the others
+
+		p.logger.Error("failed to update stack", "err", err)
+
+		// TODO: we can use context here to actually pass some information through
+		return
+	}
+}
+
+func (p *StackProcessor) extractInfo(ctx context.Context, stack model.Stacks) {
+	err := p.stackAgent.CreateList(p.logger, stack.UserID, stack.ID, stack.Name, stack.Description)
+	if err != nil {
+		// Again, wtf do we do?
+		// Although i think the agent actually returns an error when it's finished
+		p.logger.Error("failed to process image", "err", err)
+		return
+	}
+}
+
+func (p *StackProcessor) processImage(stack model.Stacks) {
+	p.logger.Info("Processing image", "ID", stack.ID)
+
+	ctx := context.Background()
+
+	p.setStackToProcess(ctx, stack)
+
+	var wg sync.WaitGroup
+
+	// Future proofing!
+	wg.Add(1)
+
+	stackNotification := notifications.GetStackNotification(notifications.StackNotification{
+		Type:    notifications.STACK_TYPE,
+		Status:  string(model.Progress_InProgress),
+		StackID: stack.ID,
+		Name:    stack.Name,
+	})
+
+	err := p.notifier.SendAndCreate(stack.UserID.String(), stackNotification)
+	if err != nil {
+		p.logger.Error("sending in progress notification", "err", err)
+		return
+	}
+
+	go func() {
+		p.extractInfo(ctx, stack)
+		wg.Done()
+	}()
+
+	wg.Wait()
+
+	// TODO: there is some repeated code here. The ergonomicts of the notifications,
+	// isn't the best.
+	stackNotification = notifications.GetStackNotification(notifications.StackNotification{
+		Type:    notifications.STACK_TYPE,
+		Status:  string(model.Progress_Complete),
+		StackID: stack.ID,
+		Name:    stack.Name,
+	})
+
+	err = p.notifier.SendAndCreate(stack.UserID.String(), stackNotification)
+	if err != nil {
+		p.logger.Error("sending done notification", "err", err)
+		return
+	}
+}
+
+func NewStackProcessor(
+	logger *log.Logger,
+	stackModel models.StackModel,
+	notifier *notifications.Notifier[notifications.Notification],
+) (StackProcessor, error) {
+	if notifier == nil {
+		return StackProcessor{}, fmt.Errorf("notifier is nil")
+	}
+
+	stackAgent := agents.NewCreateListAgent(logger, stackModel)
+
+	imageProcessor := StackProcessor{
+		logger:     logger,
+		stackModel: stackModel,
+		stackAgent: stackAgent,
+		notifier:   notifier,
+	}
+
+	imageProcessor.Processor = NewProcessor(int(IMAGE_PROCESS_AT_A_TIME), imageProcessor.processImage)
+
+	return imageProcessor, nil
+}

backend/router.go

@@ -41,9 +41,16 @@ func setupRouter(db *sql.DB, jwtManager *ourmiddleware.JwtManager) (chi.Router,
 		return nil, fmt.Errorf("processor: %w", err)
 	}
 
-	go imageProcessor.Processor.Work()
+	stackProcessorLog := createLogger("Stack Processor", os.Stdout)
+	stackProcessor, err := processor.NewStackProcessor(stackProcessorLog, stackModel, &notifier)
+	if err != nil {
+		return nil, fmt.Errorf("processor: %w", err)
+	}
 
-	stackHandler := stacks.CreateStackHandler(db, limitsManager, jwtManager)
+	go imageProcessor.Processor.Work()
+	go stackProcessor.Processor.Work()
+
+	stackHandler := stacks.CreateStackHandler(db, limitsManager, jwtManager, stackProcessor.Processor)
 	authHandler := auth.CreateAuthHandler(db, jwtManager)
 	imageHandler := images.CreateImageHandler(db, limitsManager, jwtManager, imageProcessor.Processor)
 

backend/stacks/handler.go

@@ -8,6 +8,7 @@ import (
 	"screenmark/screenmark/limits"
 	"screenmark/screenmark/middleware"
 	"screenmark/screenmark/models"
+	"screenmark/screenmark/processor"
 
 	"github.com/charmbracelet/log"
 	"github.com/go-chi/chi/v5"
@@ -23,6 +24,8 @@ type StackHandler struct {
 	limitsManager limits.LimitsManagerMethods
 
 	jwtManager *middleware.JwtManager
+
+	processor *processor.Processor[model.Stacks]
 }
 
 func (h *StackHandler) getAllStacks(w http.ResponseWriter, r *http.Request) {
@@ -209,14 +212,16 @@ func (h *StackHandler) createStack(body CreateStackBody, w http.ResponseWriter,
 	}
 
 	// TODO: Add the stack processor here
-	_, err = h.stackModel.Save(ctx, userID, body.Title, body.Description, model.Progress_NotStarted)
+	stack, err := h.stackModel.Save(ctx, userID, body.Title, body.Description, model.Progress_NotStarted)
 	if err != nil {
 		h.logger.Warn("could not save stack", "err", err)
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
 
-	w.WriteHeader(http.StatusOK)
+	h.processor.Add(stack)
+
+	middleware.WriteJsonOrError(h.logger, stack, w)
 }
 
 func (h *StackHandler) CreateRoutes(r chi.Router) {
@@ -237,7 +242,12 @@ func (h *StackHandler) CreateRoutes(r chi.Router) {
 	})
 }
 
-func CreateStackHandler(db *sql.DB, limitsManager limits.LimitsManagerMethods, jwtManager *middleware.JwtManager) StackHandler {
+func CreateStackHandler(
+	db *sql.DB,
+	limitsManager limits.LimitsManagerMethods,
+	jwtManager *middleware.JwtManager,
+	processor *processor.Processor[model.Stacks],
+) StackHandler {
 	stackModel := models.NewStackModel(db)
 	imageModel := models.NewImageModel(db)
 	logger := log.New(os.Stdout).WithPrefix("Stacks")
@@ -248,5 +258,6 @@ func CreateStackHandler(db *sql.DB, limitsManager limits.LimitsManagerMethods, j
 		stackModel:    stackModel,
 		limitsManager: limitsManager,
 		jwtManager:    jwtManager,
+		processor:     processor,
 	}
 }

frontend/src/pages/list/index.tsx

@@ -57,59 +57,6 @@ const DeleteButton: Component<{ onDelete: () => void }> = (props) => {
     );
 };
 
-const DeleteSchemaItemButton: Component<{
-    onDelete: () => void;
-    itemName: string;
-}> = (props) => {
-    const [isOpen, setIsOpen] = createSignal(false);
-
-    return (
-        <>
-            <button
-                aria-label="Delete schema item"
-                class="text-red-600 hover:text-red-700 ml-2"
-                onClick={(e) => {
-                    e.stopPropagation();
-                    setIsOpen(true);
-                }}
-            >
-                ×
-            </button>
-
-            <Dialog.Root open={isOpen()} onOpenChange={setIsOpen}>
-                <Dialog.Portal>
-                    <Dialog.Overlay class="fixed inset-0 bg-black bg-opacity-50" />
-                    <div class="fixed inset-0 z-50 flex items-center justify-center p-4">
-                        <Dialog.Content class="bg-white rounded-lg shadow-xl max-w-md w-full p-6">
-                            <Dialog.Title class="text-lg font-bold mb-2">
-                                Delete Column
-                            </Dialog.Title>
-                            <Dialog.Description class="mb-4">
-                                Are you sure you want to delete the column "
-                                {props.itemName}"? This will remove this column
-                                and all its data from the list.
-                            </Dialog.Description>
-                            <div class="flex justify-end gap-2">
-                                <Dialog.CloseButton>
-                                    <button class="px-4 py-2 bg-gray-300 rounded hover:bg-gray-400">
-                                        Cancel
-                                    </button>
-                                </Dialog.CloseButton>
-                                <button
-                                    class="px-4 py-2 bg-red-600 text-white rounded hover:bg-red-700"
-                                    onClick={props.onDelete}
-                                >
-                                    Delete Column
-                                </button>
-                            </div>
-                        </Dialog.Content>
-                    </div>
-                </Dialog.Portal>
-            </Dialog.Root>
-        </>
-    );
-};
-
 const DeleteListButton: Component<{ onDelete: () => void }> = (props) => {
     const [isOpen, setIsOpen] = createSignal(false);
 
@@ -160,22 +107,17 @@ export const List: Component = () => {
     const { listId } = useParams();
     const nav = useNavigate();
 
-    const { stacks, onDeleteImageFromStack, onDeleteStackItem } =
-        useSearchImageContext();
+    const { stacks, onDeleteImageFromStack } = useSearchImageContext();
 
     const [accessToken] = createResource(getAccessToken);
 
     const list = () => stacks().find((l) => l.ID === listId);
 
     const handleDeleteList = async () => {
-        await deleteList(listId);
+        await deleteList(listId)
         nav("/");
     };
 
-    const handleDeleteSchemaItem = (schemaItemId: string) => {
-        onDeleteStackItem(listId, schemaItemId);
-    };
-
     return (
         <Suspense>
             <Show when={list()} fallback="List could not be found">
@@ -209,25 +151,15 @@ export const List: Component = () => {
                                         <For each={l().SchemaItems}>
                                             {(item, index) => (
                                                 <th
-                                                    class={`px-6 py-4 text-left text-sm font-semibold text-neutral-900 min-w-32 ${
-                                                        index() <
-                                                        l().SchemaItems.length -
+                                                    class={`px-6 py-4 text-left text-sm font-semibold text-neutral-900 min-w-32 ${index() <
+                                                        l().SchemaItems
+                                                            .length -
                                                         1
                                                         ? "border-r border-neutral-200"
                                                         : ""
                                                         }`}
                                                 >
-                                                    <div class="flex items-center">
                                                     {item.Item}
-                                                        <DeleteSchemaItemButton
-                                                            onDelete={() =>
-                                                                handleDeleteSchemaItem(
-                                                                    item.ID,
-                                                                )
-                                                            }
-                                                            itemName={item.Item}
-                                                        />
-                                                    </div>
                                                 </th>
                                             )}
                                         </For>
@@ -237,8 +169,7 @@ export const List: Component = () => {
                                     <For each={l().Images}>
                                         {(image, rowIndex) => (
                                             <tr
-                                                class={`hover:bg-neutral-50 transition-colors ${
-                                                    rowIndex() % 2 === 0
+                                                class={`hover:bg-neutral-50 transition-colors ${rowIndex() % 2 === 0
                                                     ? "bg-white"
                                                     : "bg-neutral-25"
                                                     }`}
@@ -268,8 +199,7 @@ export const List: Component = () => {
                                                 <For each={image.Items}>
                                                     {(item, colIndex) => (
                                                         <td
-                                                            class={`px-6 py-4 text-sm text-neutral-700 ${
-                                                                colIndex() <
+                                                            class={`px-6 py-4 text-sm text-neutral-700 ${colIndex() <
                                                                 image.Items
                                                                     .length -
                                                                 1