Added better archive functionalty (backend side atm)

Code refactoring

server/archive/archive.go

@@ -0,0 +1,18 @@
+package archive
+
+import (
+	"database/sql"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/domain"
+)
+
+// alias type
+// TODO: remove after refactoring
+type Service = domain.Service
+type Entity = domain.ArchiveEntry
+
+func ApplyRouter(db *sql.DB) func(chi.Router) {
+	handler, _ := Container(db)
+	return handler.ApplyRouter()
+}

server/archive/container.go

@@ -0,0 +1,16 @@
+package archive
+
+import (
+	"database/sql"
+
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/domain"
+)
+
+func Container(db *sql.DB) (domain.RestHandler, domain.Service) {
+	var (
+		r = provideRepository(db)
+		s = provideService(r)
+		h = provideHandler(s)
+	)
+	return h, s
+}

server/archive/data/models.go

@@ -0,0 +1,13 @@
+package data
+
+import "time"
+
+type ArchiveEntry struct {
+	Id        string
+	Title     string
+	Path      string
+	Thumbnail string
+	Source    string
+	Metadata  string
+	CreatedAt time.Time
+}

server/archive/domain/archive.go

@@ -0,0 +1,51 @@
+package domain
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/data"
+)
+
+type ArchiveEntry struct {
+	Id        string    `json:"id"`
+	Title     string    `json:"title"`
+	Path      string    `json:"path"`
+	Thumbnail string    `json:"thumbnail"`
+	Source    string    `json:"source"`
+	Metadata  string    `json:"metadata"`
+	CreatedAt time.Time `json:"created_at"`
+}
+
+type PaginatedResponse[T any] struct {
+	First int64 `json:"first"`
+	Next  int64 `json:"next"`
+	Data  T     `json:"data"`
+}
+
+type Repository interface {
+	Archive(ctx context.Context, model *data.ArchiveEntry) error
+	SoftDelete(ctx context.Context, id string) (*data.ArchiveEntry, error)
+	HardDelete(ctx context.Context, id string) (*data.ArchiveEntry, error)
+	List(ctx context.Context, startRowId int, limit int) (*[]data.ArchiveEntry, error)
+	GetCursor(ctx context.Context, id string) (int64, error)
+}
+
+type Service interface {
+	Archive(ctx context.Context, entity *ArchiveEntry) error
+	SoftDelete(ctx context.Context, id string) (*ArchiveEntry, error)
+	HardDelete(ctx context.Context, id string) (*ArchiveEntry, error)
+	List(ctx context.Context, startRowId int, limit int) (*PaginatedResponse[[]ArchiveEntry], error)
+	GetCursor(ctx context.Context, id string) (int64, error)
+}
+
+type RestHandler interface {
+	List() http.HandlerFunc
+	Archive() http.HandlerFunc
+	SoftDelete() http.HandlerFunc
+	HardDelete() http.HandlerFunc
+	GetCursor() http.HandlerFunc
+	ApplyRouter() func(chi.Router)
+}

server/archive/provider.go

@@ -0,0 +1,42 @@
+package archive
+
+import (
+	"database/sql"
+	"sync"
+
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/domain"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/repository"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/rest"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/service"
+)
+
+var (
+	repo domain.Repository
+	svc  domain.Service
+	hand domain.RestHandler
+
+	repoOnce sync.Once
+	svcOnce  sync.Once
+	handOnce sync.Once
+)
+
+func provideRepository(db *sql.DB) domain.Repository {
+	repoOnce.Do(func() {
+		repo = repository.New(db)
+	})
+	return repo
+}
+
+func provideService(r domain.Repository) domain.Service {
+	svcOnce.Do(func() {
+		svc = service.New(r)
+	})
+	return svc
+}
+
+func provideHandler(s domain.Service) domain.RestHandler {
+	handOnce.Do(func() {
+		hand = rest.New(s)
+	})
+	return hand
+}

server/archive/repository/repository.go

@@ -0,0 +1,156 @@
+package repository
+
+import (
+	"context"
+	"database/sql"
+	"os"
+
+	"github.com/google/uuid"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/data"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/domain"
+)
+
+type Repository struct {
+	db *sql.DB
+}
+
+func New(db *sql.DB) domain.Repository {
+	return &Repository{
+		db: db,
+	}
+}
+
+func (r *Repository) Archive(ctx context.Context, entry *data.ArchiveEntry) error {
+	conn, err := r.db.Conn(ctx)
+	if err != nil {
+		return err
+	}
+
+	defer conn.Close()
+
+	_, err = conn.ExecContext(
+		ctx,
+		"INSERT INTO archive (id, title, path, thumbnail, source, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+		uuid.NewString(),
+		entry.Title,
+		entry.Path,
+		entry.Thumbnail,
+		entry.Source,
+		entry.Metadata,
+		entry.CreatedAt,
+	)
+
+	return err
+}
+
+func (r *Repository) SoftDelete(ctx context.Context, id string) (*data.ArchiveEntry, error) {
+	conn, err := r.db.Conn(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+
+	tx, err := conn.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	var model data.ArchiveEntry
+
+	row := tx.QueryRowContext(ctx, "SELECT * FROM archive WHERE id = ?", id)
+
+	if err := row.Scan(
+		&model.Id,
+		&model.Title,
+		&model.Path,
+		&model.Thumbnail,
+		&model.Source,
+		&model.Metadata,
+		&model.CreatedAt,
+	); err != nil {
+		return nil, err
+	}
+
+	_, err = tx.ExecContext(ctx, "DELETE FROM archive WHERE id = ?", id)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := tx.Commit(); err != nil {
+		return nil, err
+	}
+
+	return &model, nil
+}
+
+func (r *Repository) HardDelete(ctx context.Context, id string) (*data.ArchiveEntry, error) {
+	entry, err := r.SoftDelete(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := os.Remove(entry.Path); err != nil {
+		return nil, err
+	}
+
+	return entry, nil
+}
+
+func (r *Repository) List(ctx context.Context, startRowId int, limit int) (*[]data.ArchiveEntry, error) {
+	conn, err := r.db.Conn(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+
+	var entries []data.ArchiveEntry
+
+	// cursor based pagination
+	rows, err := conn.QueryContext(ctx, "SELECT rowid, * FROM archive WHERE rowid > ? LIMIT ?", startRowId, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	for rows.Next() {
+		var rowId int64
+		var entry data.ArchiveEntry
+
+		if err := rows.Scan(
+			&rowId,
+			&entry.Id,
+			&entry.Title,
+			&entry.Path,
+			&entry.Thumbnail,
+			&entry.Source,
+			&entry.Metadata,
+			&entry.CreatedAt,
+		); err != nil {
+			return &entries, err
+		}
+
+		entries = append(entries, entry)
+	}
+
+	return &entries, err
+}
+
+func (r *Repository) GetCursor(ctx context.Context, id string) (int64, error) {
+	conn, err := r.db.Conn(ctx)
+	if err != nil {
+		return -1, err
+	}
+	defer conn.Close()
+
+	row := conn.QueryRowContext(ctx, "SELECT rowid FROM archive WHERE id = ?", id)
+
+	var rowId int64
+
+	if err := row.Scan(&rowId); err != nil {
+		return -1, err
+	}
+
+	return rowId, nil
+}

server/archive/rest/handler.go

@@ -0,0 +1,162 @@
+package rest
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/domain"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/config"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/openid"
+
+	middlewares "github.com/marcopeocchi/yt-dlp-web-ui/v3/server/middleware"
+)
+
+type Handler struct {
+	service domain.Service
+}
+
+func New(service domain.Service) domain.RestHandler {
+	return &Handler{
+		service: service,
+	}
+}
+
+// List implements domain.RestHandler.
+func (h *Handler) List() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.Header().Set("Content-Type", "application/json")
+
+		var (
+			startRowIdParam = r.URL.Query().Get("id")
+			LimitParam      = r.URL.Query().Get("limit")
+		)
+
+		startRowId, err := strconv.Atoi(startRowIdParam)
+		if err != nil {
+			startRowId = 0
+		}
+
+		limit, err := strconv.Atoi(LimitParam)
+		if err != nil {
+			limit = 50
+		}
+
+		res, err := h.service.List(r.Context(), startRowId, limit)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// Archive implements domain.RestHandler.
+func (h *Handler) Archive() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.Header().Set("Content-Type", "application/json")
+
+		var req domain.ArchiveEntry
+
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		err := h.service.Archive(r.Context(), &req)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		json.NewEncoder(w).Encode("ok")
+	}
+}
+
+// HardDelete implements domain.RestHandler.
+func (h *Handler) HardDelete() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.Header().Set("Content-Type", "application/json")
+
+		id := chi.URLParam(r, "id")
+
+		res, err := h.service.HardDelete(r.Context(), id)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// SoftDelete implements domain.RestHandler.
+func (h *Handler) SoftDelete() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.Header().Set("Content-Type", "application/json")
+
+		id := chi.URLParam(r, "id")
+
+		res, err := h.service.SoftDelete(r.Context(), id)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// GetCursor implements domain.RestHandler.
+func (h *Handler) GetCursor() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		w.Header().Set("Content-Type", "application/json")
+
+		id := chi.URLParam(r, "id")
+
+		cursorId, err := h.service.GetCursor(r.Context(), id)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		if err := json.NewEncoder(w).Encode(cursorId); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// ApplyRouter implements domain.RestHandler.
+func (h *Handler) ApplyRouter() func(chi.Router) {
+	return func(r chi.Router) {
+		if config.Instance().RequireAuth {
+			r.Use(middlewares.Authenticated)
+		}
+		if config.Instance().UseOpenId {
+			r.Use(openid.Middleware)
+		}
+
+		r.Get("/", h.List())
+		r.Get("/cursor/{id}", h.GetCursor())
+		r.Post("/", h.Archive())
+		r.Delete("/soft/{id}", h.SoftDelete())
+		r.Delete("/hard/{id}", h.HardDelete())
+	}
+}

server/archive/service/service.go

@@ -0,0 +1,121 @@
+package service
+
+import (
+	"context"
+
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/data"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive/domain"
+)
+
+type Service struct {
+	repository domain.Repository
+}
+
+func New(repository domain.Repository) domain.Service {
+	return &Service{
+		repository: repository,
+	}
+}
+
+// Archive implements domain.Service.
+func (s *Service) Archive(ctx context.Context, entity *domain.ArchiveEntry) error {
+	return s.repository.Archive(ctx, &data.ArchiveEntry{
+		Id:        entity.Id,
+		Title:     entity.Title,
+		Path:      entity.Path,
+		Thumbnail: entity.Thumbnail,
+		Source:    entity.Source,
+		Metadata:  entity.Metadata,
+		CreatedAt: entity.CreatedAt,
+	})
+}
+
+// HardDelete implements domain.Service.
+func (s *Service) HardDelete(ctx context.Context, id string) (*domain.ArchiveEntry, error) {
+	res, err := s.repository.HardDelete(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return &domain.ArchiveEntry{
+		Id:        res.Id,
+		Title:     res.Title,
+		Path:      res.Path,
+		Thumbnail: res.Thumbnail,
+		Source:    res.Source,
+		Metadata:  res.Metadata,
+		CreatedAt: res.CreatedAt,
+	}, nil
+}
+
+// SoftDelete implements domain.Service.
+func (s *Service) SoftDelete(ctx context.Context, id string) (*domain.ArchiveEntry, error) {
+	res, err := s.repository.SoftDelete(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return &domain.ArchiveEntry{
+		Id:        res.Id,
+		Title:     res.Title,
+		Path:      res.Path,
+		Thumbnail: res.Thumbnail,
+		Source:    res.Source,
+		Metadata:  res.Metadata,
+		CreatedAt: res.CreatedAt,
+	}, nil
+}
+
+// List implements domain.Service.
+func (s *Service) List(
+	ctx context.Context,
+	startRowId int,
+	limit int,
+) (*domain.PaginatedResponse[[]domain.ArchiveEntry], error) {
+	res, err := s.repository.List(ctx, startRowId, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	entities := make([]domain.ArchiveEntry, len(*res))
+
+	for i, model := range *res {
+		entities[i] = domain.ArchiveEntry{
+			Id:        model.Id,
+			Title:     model.Title,
+			Path:      model.Path,
+			Thumbnail: model.Thumbnail,
+			Source:    model.Source,
+			Metadata:  model.Metadata,
+			CreatedAt: model.CreatedAt,
+		}
+	}
+
+	var (
+		first int64
+		next  int64
+	)
+
+	if len(entities) > 0 {
+		first, err = s.repository.GetCursor(ctx, entities[0].Id)
+		if err != nil {
+			return nil, err
+		}
+
+		next, err = s.repository.GetCursor(ctx, entities[len(entities)-1].Id)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &domain.PaginatedResponse[[]domain.ArchiveEntry]{
+		First: first,
+		Next:  next,
+		Data:  entities,
+	}, nil
+}
+
+// GetCursor implements domain.Service.
+func (s *Service) GetCursor(ctx context.Context, id string) (int64, error) {
+	return s.repository.GetCursor(ctx, id)
+}

server/archiver/archiver.go

@@ -0,0 +1,42 @@
+package archiver
+
+import (
+	"context"
+	"database/sql"
+	"log/slog"
+
+	evbus "github.com/asaskevich/EventBus"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/config"
+)
+
+const QueueName = "process:archive"
+
+var (
+	eventBus       = evbus.New()
+	archiveService archive.Service
+)
+
+type Message = archive.Entity
+
+func Register(db *sql.DB) {
+	_, s := archive.Container(db)
+	archiveService = s
+}
+
+func init() {
+	eventBus.Subscribe(QueueName, func(m *Message) {
+		slog.Info(
+			"archiving completed download",
+			slog.String("title", m.Title),
+			slog.String("source", m.Source),
+		)
+		archiveService.Archive(context.Background(), m)
+	})
+}
+
+func Publish(m *Message) {
+	if config.Instance().AutoArchive {
+		eventBus.Publish(QueueName, m)
+	}
+}

server/config/config.go

@@ -29,6 +29,7 @@ type Config struct {
 	OpenIdClientSecret string `yaml:"openid_client_secret"`
 	OpenIdRedirectURL  string `yaml:"openid_redirect_url"`
 	FrontendPath       string `yaml:"frontend_path"`
+	AutoArchive        bool   `yaml:"auto_archive"`
 }
 
 var (

server/dbutil/migrate.go

@@ -34,6 +34,21 @@ func Migrate(ctx context.Context, db *sql.DB) error {
 		return err
 	}
 
+	if _, err := db.ExecContext(
+		ctx,
+		`CREATE TABLE IF NOT EXISTS archive (
+			id CHAR(36) PRIMARY KEY,
+			title VARCHAR(255) NOT NULL,
+			path VARCHAR(255) NOT NULL,
+			thumbnail TEXT,
+			source VARCHAR(255),
+			metadata TEXT,
+			created_at DATETIME
+		)`,
+	); err != nil {
+		return err
+	}
+
 	if lockFileExists() {
 		return nil
 	}

server/internal/process.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archiver"
 	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/config"
 )
 
@@ -87,6 +88,7 @@ func (p *Process) Start() {
 	buildFilename(&p.Output)
 
 	templateReplacer := strings.NewReplacer("\n", "", "\t", "", " ", "")
+
 	baseParams := []string{
 		strings.Split(p.Url, "?list")[0], //no playlist
 		"--newline",
@@ -193,13 +195,12 @@ func (p *Process) parseLogEntry(entry []byte) {
 	if err := json.Unmarshal(entry, &postprocess); err == nil {
 		p.Output.SavedFilePath = postprocess.FilePath
 
-		slog.Info("postprocess",
-			slog.String("id", p.getShortId()),
-			slog.String("url", p.Url),
-			slog.String("filepath", postprocess.FilePath),
-		)
+		// slog.Info("postprocess",
+		// 	slog.String("id", p.getShortId()),
+		// 	slog.String("url", p.Url),
+		// 	slog.String("filepath", postprocess.FilePath),
+		// )
 	}
-
 }
 
 func (p *Process) detectYtDlpErrors(r io.Reader) {
@@ -218,6 +219,24 @@ func (p *Process) detectYtDlpErrors(r io.Reader) {
 // Convention: All completed processes has progress -1
 // and speed 0 bps.
 func (p *Process) Complete() {
+	// auto archive
+	// TODO: it's not that deterministic :/
+	if p.Progress.Percentage == "" && p.Progress.Speed == 0 {
+		var serializedMetadata bytes.Buffer
+
+		json.NewEncoder(&serializedMetadata).Encode(p.Info)
+
+		archiver.Publish(&archiver.Message{
+			Id:        p.Id,
+			Path:      p.Output.SavedFilePath,
+			Title:     p.Info.Title,
+			Thumbnail: p.Info.Thumbnail,
+			Source:    p.Url,
+			Metadata:  serializedMetadata.String(),
+			CreatedAt: p.Info.CreatedAt,
+		})
+	}
+
 	p.Progress = DownloadProgress{
 		Status:     StatusCompleted,
 		Percentage: "-1",

server/server.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/cors"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archive"
+	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/archiver"
 	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/config"
 	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/dbutil"
 	"github.com/marcopeocchi/yt-dlp-web-ui/v3/server/handlers"
@@ -145,6 +147,8 @@ func RunBlocking(rc *RunConfig) {
 }
 
 func newServer(c serverConfig) *http.Server {
+	archiver.Register(c.db)
+
 	service := ytdlpRPC.Container(c.mdb, c.mq, c.lm)
 	rpc.Register(service)
 
@@ -174,8 +178,8 @@ func newServer(c serverConfig) *http.Server {
 	// swagger
 	r.Mount("/openapi", http.FileServerFS(c.swagger))
 
-	// Archive routes
-	r.Route("/archive", func(r chi.Router) {
+	// Filebrowser routes
+	r.Route("/filebrowser", func(r chi.Router) {
 		if config.Instance().RequireAuth {
 			r.Use(middlewares.Authenticated)
 		}
@@ -189,6 +193,9 @@ func newServer(c serverConfig) *http.Server {
 		r.Get("/bulk", handlers.BulkDownload(c.mdb))
 	})
 
+	// Archive routes
+	r.Route("/archive", archive.ApplyRouter(c.db))
+
 	// Authentication routes
 	r.Route("/auth", func(r chi.Router) {
 		r.Post("/login", handlers.Login)