feat(logging): split logging to stdout & database to allow us to view it on webbrowser

backend/.gen/haystack/haystack/model/logs.go

@@ -0,0 +1,17 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package model
+
+import (
+	"github.com/google/uuid"
+)
+
+type Logs struct {
+	Log     string
+	ImageID uuid.UUID
+}

backend/.gen/haystack/haystack/table/logs.go

@@ -0,0 +1,78 @@
+//
+// Code generated by go-jet DO NOT EDIT.
+//
+// WARNING: Changes to this file may cause incorrect behavior
+// and will be lost if the code is regenerated
+//
+
+package table
+
+import (
+	"github.com/go-jet/jet/v2/postgres"
+)
+
+var Logs = newLogsTable("haystack", "logs", "")
+
+type logsTable struct {
+	postgres.Table
+
+	// Columns
+	Log     postgres.ColumnString
+	ImageID postgres.ColumnString
+
+	AllColumns     postgres.ColumnList
+	MutableColumns postgres.ColumnList
+}
+
+type LogsTable struct {
+	logsTable
+
+	EXCLUDED logsTable
+}
+
+// AS creates new LogsTable with assigned alias
+func (a LogsTable) AS(alias string) *LogsTable {
+	return newLogsTable(a.SchemaName(), a.TableName(), alias)
+}
+
+// Schema creates new LogsTable with assigned schema name
+func (a LogsTable) FromSchema(schemaName string) *LogsTable {
+	return newLogsTable(schemaName, a.TableName(), a.Alias())
+}
+
+// WithPrefix creates new LogsTable with assigned table prefix
+func (a LogsTable) WithPrefix(prefix string) *LogsTable {
+	return newLogsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
+}
+
+// WithSuffix creates new LogsTable with assigned table suffix
+func (a LogsTable) WithSuffix(suffix string) *LogsTable {
+	return newLogsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
+}
+
+func newLogsTable(schemaName, tableName, alias string) *LogsTable {
+	return &LogsTable{
+		logsTable: newLogsTableImpl(schemaName, tableName, alias),
+		EXCLUDED:  newLogsTableImpl("", "excluded", ""),
+	}
+}
+
+func newLogsTableImpl(schemaName, tableName, alias string) logsTable {
+	var (
+		LogColumn      = postgres.StringColumn("log")
+		ImageIDColumn  = postgres.StringColumn("image_id")
+		allColumns     = postgres.ColumnList{LogColumn, ImageIDColumn}
+		mutableColumns = postgres.ColumnList{LogColumn, ImageIDColumn}
+	)
+
+	return logsTable{
+		Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
+
+		//Columns
+		Log:     LogColumn,
+		ImageID: ImageIDColumn,
+
+		AllColumns:     allColumns,
+		MutableColumns: mutableColumns,
+	}
+}

backend/.gen/haystack/haystack/table/table_use_schema.go

@@ -21,6 +21,7 @@ func UseSchema(schema string) {
 	ImageTags = ImageTags.FromSchema(schema)
 	ImageText = ImageText.FromSchema(schema)
 	Locations = Locations.FromSchema(schema)
+	Logs = Logs.FromSchema(schema)
 	Notes = Notes.FromSchema(schema)
 	UserContacts = UserContacts.FromSchema(schema)
 	UserEvents = UserEvents.FromSchema(schema)

backend/events.go

@@ -13,18 +13,6 @@ import (
 	"github.com/lib/pq"
 )
 
-func createLogger(prefix string) *log.Logger {
-	logger := log.NewWithOptions(os.Stdout, log.Options{
-		ReportTimestamp: true,
-		TimeFormat:      time.Kitchen,
-		Prefix:          prefix,
-	})
-
-	logger.SetLevel(log.DebugLevel)
-
-	return logger
-}
-
 func ListenNewImageEvents(db *sql.DB, eventManager *EventManager) {
 	listener := pq.NewListener(os.Getenv("DB_CONNECTION"), time.Second, time.Second, func(event pq.ListenerEventType, err error) {
 		if err != nil {
@@ -39,7 +27,7 @@ func ListenNewImageEvents(db *sql.DB, eventManager *EventManager) {
 	imageModel := models.NewImageModel(db)
 	contactModel := models.NewContactModel(db)
 
-	databaseEventLog := createLogger("Database Events 🤖")
+	databaseEventLog := createLogger("Database Events 🤖", os.Stdout)
 	databaseEventLog.SetLevel(log.DebugLevel)
 
 	err := listener.Listen("new_image")
@@ -58,23 +46,25 @@ func ListenNewImageEvents(db *sql.DB, eventManager *EventManager) {
 			ctx := context.Background()
 
 			go func() {
-				noteAgent := agents.NewNoteAgent(createLogger("Notes 📝"), noteModel)
-				contactAgent := agents.NewContactAgent(createLogger("Contacts 👥"), contactModel)
-				locationAgent := agents.NewLocationAgent(createLogger("Locations 📍"), locationModel)
-				eventAgent := agents.NewEventAgent(createLogger("Events 📅"), eventModel, locationModel)
-
 				image, err := imageModel.GetToProcessWithData(ctx, imageId)
 				if err != nil {
 					databaseEventLog.Error("Failed to GetToProcessWithData", "error", err)
 					return
 				}
 
+				splitWriter := createDbStdoutWriter(db, image.ImageID)
+
+				noteAgent := agents.NewNoteAgent(createLogger("Notes 📝", splitWriter), noteModel)
+				contactAgent := agents.NewContactAgent(createLogger("Contacts 👥", splitWriter), contactModel)
+				locationAgent := agents.NewLocationAgent(createLogger("Locations 📍", splitWriter), locationModel)
+				eventAgent := agents.NewEventAgent(createLogger("Events 📅", splitWriter), eventModel, locationModel)
+
 				if err := imageModel.StartProcessing(ctx, image.ID); err != nil {
 					databaseEventLog.Error("Failed to FinishProcessing", "error", err)
 					return
 				}
 
-				orchestrator := agents.NewOrchestratorAgent(createLogger("Orchestrator 🎼"), noteAgent, contactAgent, locationAgent, eventAgent, image.Image.ImageName, image.Image.Image)
+				orchestrator := agents.NewOrchestratorAgent(createLogger("Orchestrator 🎼", splitWriter), noteAgent, contactAgent, locationAgent, eventAgent, image.Image.ImageName, image.Image.Image)
 				err = orchestrator.RunAgent(image.UserID, image.ImageID, image.Image.ImageName, image.Image.Image)
 				if err != nil {
 					databaseEventLog.Error("Orchestrator failed", "error", err)

backend/logs.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"database/sql"
+	"io"
+	"os"
+	"time"
+
+	. "screenmark/screenmark/.gen/haystack/haystack/table"
+
+	"github.com/charmbracelet/log"
+	"github.com/google/uuid"
+	"github.com/muesli/termenv"
+)
+
+type DatabaseWriter struct {
+	dbPool  *sql.DB
+	imageId uuid.UUID
+}
+
+func (w *DatabaseWriter) Write(p []byte) (n int, err error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+
+	insertLogStmt := Logs.
+		INSERT(Logs.Log, Logs.ImageID).
+		VALUES(string(p), w.imageId)
+
+	_, err = insertLogStmt.Exec(w.dbPool)
+
+	if err != nil {
+		return 0, err
+	} else {
+		return len(p), nil
+	}
+}
+
+func newDatabaseWriter(dbPool *sql.DB, imageId uuid.UUID) *DatabaseWriter {
+	io.MultiWriter()
+	return &DatabaseWriter{
+		dbPool:  dbPool,
+		imageId: imageId,
+	}
+}
+
+func createDbStdoutWriter(dbPool *sql.DB, imageId uuid.UUID) io.Writer {
+	dbWriter := newDatabaseWriter(dbPool, imageId)
+	return io.MultiWriter(os.Stdout, dbWriter)
+}
+
+func createLogger(prefix string, writer io.Writer) *log.Logger {
+	logger := log.NewWithOptions(writer, log.Options{
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          prefix,
+		Formatter:       log.TextFormatter,
+	})
+
+	logger.SetColorProfile(termenv.TrueColor)
+	logger.SetLevel(log.DebugLevel)
+
+	return logger
+}

backend/schema.sql

@@ -146,6 +146,11 @@ CREATE TABLE haystack.user_notes (
     note_id UUID NOT NULL REFERENCES haystack.notes (id)
 );
 
+CREATE TABLE haystack.logs (
+    log TEXT NOT NULL,
+    image_id UUID NOT NULL REFERENCES haystack.image (id)
+);
+
 /* -----| Indexes |----- */
 
 CREATE INDEX user_tags_index ON haystack.user_tags(tag);