marco/yt-dlp-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactoring: config struct & pipelines

Browse Source
This commit is contained in:
marcopiovanello
2025-09-04 15:33:07 +02:00
parent 5dbe6d886f
commit 991bea1a7b
34 changed files with 517 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/archive/rest/handler.go
View File

@@ -146,10 +146,10 @@ func (h *Handler) GetCursor() http.HandlerFunc {
// ApplyRouter implements domain.RestHandler.
func (h *Handler) ApplyRouter() func(chi.Router) {
return func(r chi.Router) {
if config.Instance().RequireAuth {
if config.Instance().Authentication.RequireAuth {
r.Use(middlewares.Authenticated)
}
if config.Instance().UseOpenId {
if config.Instance().OpenId.UseOpenId {
r.Use(openid.Middleware)
}

2
server/archive/utils.go
View File

@@ -16,7 +16,7 @@ import (
func DownloadExists(ctx context.Context, url string) (bool, error) {
cmd := exec.CommandContext(
ctx,
config.Instance().DownloaderPath,
config.Instance().Paths.DownloaderPath,
"--print",
"%(extractor)s %(id)s",
url,

25
server/archiver/archiver.go
View File

@@ -5,15 +5,12 @@ import (
"database/sql"
"log/slog"
evbus "github.com/asaskevich/EventBus"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/archive"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
)
const QueueName = "process:archive"
var (
eventBus = evbus.New()
ch = make(chan *Message, 1)
archiveService archive.Service
)
@@ -25,18 +22,20 @@ func Register(db *sql.DB) {
}
func init() {
eventBus.Subscribe(QueueName, func(m *Message) {
slog.Info(
"archiving completed download",
slog.String("title", m.Title),
slog.String("source", m.Source),
)
archiveService.Archive(context.Background(), m)
})
go func() {
for m := range ch {
slog.Info(
"archiving completed download",
slog.String("title", m.Title),
slog.String("source", m.Source),
)
archiveService.Archive(context.Background(), m)
}
}()
}
func Publish(m *Message) {
if config.Instance().AutoArchive {
eventBus.Publish(QueueName, m)
ch <- m
}
}

98
server/config/config.go
View File

@@ -1,42 +1,64 @@
package config
import (
"os"
"path/filepath"
"sync"
"time"
"gopkg.in/yaml.v3"
)
type Config struct {
LogPath string `yaml:"log_path"`
EnableFileLogging bool `yaml:"enable_file_logging"`
BaseURL string `yaml:"base_url"`
Host string `yaml:"host"`
Port int `yaml:"port"`
DownloadPath string `yaml:"downloadPath"`
DownloaderPath string `yaml:"downloaderPath"`
RequireAuth bool `yaml:"require_auth"`
Username string `yaml:"username"`
Password string `yaml:"password"`
QueueSize int `yaml:"queue_size"`
LocalDatabasePath string `yaml:"local_database_path"`
SessionFilePath string `yaml:"session_file_path"`
path string // private
UseOpenId bool `yaml:"use_openid"`
OpenIdProviderURL string `yaml:"openid_provider_url"`
OpenIdClientId string `yaml:"openid_client_id"`
OpenIdClientSecret string `yaml:"openid_client_secret"`
OpenIdRedirectURL string `yaml:"openid_redirect_url"`
OpenIdEmailWhitelist []string `yaml:"openid_email_whitelist"`
FrontendPath string `yaml:"frontend_path"`
AutoArchive bool `yaml:"auto_archive"`
Twitch struct {
ClientId string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
CheckInterval time.Duration `yaml:"check_interval"`
} `yaml:"twitch"`
Server ServerConfig `yaml:"server"`
Logging LoggingConfig `yaml:"logging"`
Paths PathsConfig `yaml:"paths"`
Authentication AuthConfig `yaml:"authentication"`
OpenId OpenIdConfig `yaml:"openid"`
Frontend FrontendConfig `yaml:"frontend"`
AutoArchive bool `yaml:"auto_archive"`
Twitch TwitchConfig `yaml:"twitch"`
path string
}
type ServerConfig struct {
BaseURL string `yaml:"base_url"`
Host string `yaml:"host"`
Port int `yaml:"port"`
QueueSize int `yaml:"queue_size"`
}
type LoggingConfig struct {
LogPath string `yaml:"log_path"`
EnableFileLogging bool `yaml:"enable_file_logging"`
}
type PathsConfig struct {
DownloadPath string `yaml:"download_path"`
DownloaderPath string `yaml:"downloader_path"`
LocalDatabasePath string `yaml:"local_database_path"`
}
type AuthConfig struct {
RequireAuth bool `yaml:"require_auth"`
Username string `yaml:"username"`
PasswordHash string `yaml:"password"`
}
type OpenIdConfig struct {
UseOpenId bool `yaml:"use_openid"`
ProviderURL string `yaml:"openid_provider_url"`
ClientId string `yaml:"openid_client_id"`
ClientSecret string `yaml:"openid_client_secret"`
RedirectURL string `yaml:"openid_redirect_url"`
EmailWhitelist []string `yaml:"openid_email_whitelist"`
}
type FrontendConfig struct {
FrontendPath string `yaml:"frontend_path"`
}
type TwitchConfig struct {
ClientId string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
CheckInterval time.Duration `yaml:"check_interval"`
}
var (
@@ -54,22 +76,6 @@ func Instance() *Config {
return instance
}
// Initialises the Config struct given its config file
func (c *Config) LoadFile(filename string) error {
fd, err := os.Open(filename)
if err != nil {
return err
}
c.path = filename
if err := yaml.NewDecoder(fd).Decode(c); err != nil {
return err
}
return nil
}
// Path of the directory containing the config file
func (c *Config) Dir() string { return filepath.Dir(c.path) }

6
server/filebrowser/handlers.go
View File

@@ -89,7 +89,7 @@ type ListRequest struct {
}
func ListDownloaded(w http.ResponseWriter, r *http.Request) {
root := config.Instance().DownloadPath
root := config.Instance().Paths.DownloadPath
req := new(ListRequest)
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -157,7 +157,7 @@ func SendFile(w http.ResponseWriter, r *http.Request) {
filename := string(decoded)
root := config.Instance().DownloadPath
root := config.Instance().Paths.DownloadPath
if strings.Contains(filepath.Dir(filepath.Clean(filename)), filepath.Clean(root)) {
http.ServeFile(w, r, filename)
@@ -189,7 +189,7 @@ func DownloadFile(w http.ResponseWriter, r *http.Request) {
filename := string(decoded)
root := config.Instance().DownloadPath
root := config.Instance().Paths.DownloadPath
if strings.Contains(filepath.Dir(filepath.Clean(filename)), filepath.Clean(root)) {
w.Header().Add("Content-Disposition", "inline; filename=\""+filepath.Base(filename)+"\"")

2
server/formats/parser.go
View File

@@ -10,7 +10,7 @@ import (
)
func ParseURL(url string) (*Metadata, error) {
cmd := exec.Command(config.Instance().DownloaderPath, url, "-J")
cmd := exec.Command(config.Instance().Paths.DownloaderPath, url, "-J")
stdout, err := cmd.Output()
if err != nil {

4
server/internal/downloaders/generic.go
View File

@@ -63,7 +63,7 @@ func (g *GenericDownloader) Start() error {
g.Params = argsSanitizer(g.Params)
out := internal.DownloadOutput{
Path: config.Instance().DownloadPath,
Path: config.Instance().Paths.DownloadPath,
Filename: "%(title)s.%(ext)s",
}
@@ -101,7 +101,7 @@ func (g *GenericDownloader) Start() error {
slog.Info("requesting download", slog.String("url", g.URL), slog.Any("params", params))
cmd := exec.Command(config.Instance().DownloaderPath, params...)
cmd := exec.Command(config.Instance().Paths.DownloaderPath, params...)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := cmd.StdoutPipe()

6
server/internal/downloaders/livestream.go
View File

@@ -57,7 +57,7 @@ func (l *LiveStreamDownloader) Start() error {
params := append(baseParams, "-o", "-")
cmd := exec.Command(config.Instance().DownloaderPath, params...)
cmd := exec.Command(config.Instance().Paths.DownloaderPath, params...)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
// stdout = media stream
@@ -102,11 +102,11 @@ func (l *LiveStreamDownloader) Start() error {
if !l.hasFileWriter() {
go func() {
filepath.Join(
config.Instance().DownloadPath,
config.Instance().Paths.DownloadPath,
fmt.Sprintf("%s (live) %s.mp4", l.Id, time.Now().Format(time.ANSIC)),
)
defaultPath := filepath.Join(config.Instance().DownloadPath)
defaultPath := filepath.Join(config.Instance().Paths.DownloadPath)
f, err := os.Create(defaultPath)
if err != nil {
slog.Error("failed to create fallback file", slog.Any("err", err))

26
server/internal/kv/store.go
View File

@@ -1,17 +1,13 @@
package kv
import (
"encoding/gob"
"encoding/json"
"errors"
"log/slog"
"os"
"path/filepath"
"runtime"
"sync"
"time"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
@@ -111,28 +107,6 @@ func (m *Store) All() *[]internal.ProcessSnapshot {
return &running
}
// Persist the database in a single file named "session.dat"
func (m *Store) Persist() error {
running := m.All()
sf := filepath.Join(config.Instance().SessionFilePath, "session.dat")
fd, err := os.Create(sf)
if err != nil {
return errors.Join(errors.New("failed to persist session"), err)
}
m.mu.RLock()
defer m.mu.RUnlock()
session := Session{Processes: *running}
if err := gob.NewEncoder(fd).Encode(session); err != nil {
return errors.Join(errors.New("failed to persist session"), err)
}
return nil
}
// Restore a persisted state
func (m *Store) Restore(mq *queue.MessageQueue) {
m.mu.Lock()

4
server/internal/livestream/livestream.go
View File

@@ -54,13 +54,13 @@ func New(url string, done chan *LiveStream, mq *queue.MessageQueue, store *kv.St
// Start the livestream monitoring process, once completion signals on the done channel
func (l *LiveStream) Start() error {
cmd := exec.Command(
config.Instance().DownloaderPath,
config.Instance().Paths.DownloaderPath,
l.url,
"--wait-for-video", "30", // wait for the stream to be live and recheck every 10 secs
"--no-colors", // no ansi color fuzz
"--simulate",
"--newline",
"--paths", config.Instance().DownloadPath,
"--paths", config.Instance().Paths.DownloadPath,
)
stdout, err := cmd.StdoutPipe()

2
server/internal/livestream/livestream_test.go
View File

@@ -10,7 +10,7 @@ import (
)
func setupTest() {
config.Instance().DownloaderPath = "build/yt-dlp"
config.Instance().Paths.DownloaderPath = "build/yt-dlp"
}
const URL = "https://www.youtube.com/watch?v=pwoAyLGOysU"

5
server/internal/livestream/monitor.go
View File

@@ -17,6 +17,11 @@ type Monitor struct {
}
func NewMonitor(mq *queue.MessageQueue, store *kv.Store, db *bolt.DB) *Monitor {
db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucket)
return err
})
return &Monitor{
mq: mq,
db: db,

2
server/internal/metadata/fetchers.go
View File

@@ -15,7 +15,7 @@ import (
)
func DefaultFetcher(url string) (*common.DownloadMetadata, error) {
cmd := exec.Command(config.Instance().DownloaderPath, url, "-J")
cmd := exec.Command(config.Instance().Paths.DownloaderPath, url, "-J")
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := cmd.StdoutPipe()

92
server/internal/pipeline/rest.go Normal file
View File

@@ -0,0 +1,92 @@
package pipeline
import (
"encoding/json"
"net/http"
"github.com/go-chi/chi/v5"
bolt "go.etcd.io/bbolt"
)
type handler struct {
store *Store
}
func NewRestHandler(db *bolt.DB) *handler {
store, _ := NewStore(db)
return &handler{
store: store,
}
}
func (h *handler) GetPipeline(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
id := chi.URLParam(r, "id")
p, err := h.store.Get(id)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.NewEncoder(w).Encode(p); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func (h *handler) GetAllPipelines(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
p, err := h.store.List()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if err := json.NewEncoder(w).Encode(p); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func (h *handler) SavePipeline(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
defer r.Body.Close()
var req Pipeline
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
id, err := h.store.Save(req)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.NewEncoder(w).Encode(id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func (h *handler) DeletePipeline(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
id := chi.URLParam(r, "id")
err := h.store.Delete(id)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.NewEncoder(w).Encode("ok"); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}

32
server/internal/pipeline/store.go
View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"github.com/google/uuid"
bolt "go.etcd.io/bbolt"
)
@@ -13,6 +14,7 @@ type Step struct {
Type string `json:"type"` // es. "transcoder", "filewriter"
FFmpegArgs []string `json:"ffmpeg_args,omitempty"` // args da passare a ffmpeg
Path string `json:"path,omitempty"` // solo per filewriter
Extension string `json:"extension,omitempty"` // solo per filewriter
}
type Pipeline struct {
@@ -25,14 +27,9 @@ type Store struct {
db *bolt.DB
}
func NewStore(path string) (*Store, error) {
db, err := bolt.Open(path, 0600, nil)
if err != nil {
return nil, err
}
func NewStore(db *bolt.DB) (*Store, error) {
// init bucket
err = db.Update(func(tx *bolt.Tx) error {
err := db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(bucket)
return err
})
@@ -43,13 +40,17 @@ func NewStore(path string) (*Store, error) {
return &Store{db: db}, nil
}
func (s *Store) Save(p Pipeline) error {
data, err := json.Marshal(p)
if err != nil {
return err
func (s *Store) Save(p Pipeline) (string, error) {
if p.ID == "" {
p.ID = uuid.NewString()
}
return s.db.Update(func(tx *bolt.Tx) error {
data, err := json.Marshal(p)
if err != nil {
return "", err
}
return p.ID, s.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(bucket)
return b.Put([]byte(p.ID), data)
})
@@ -93,3 +94,10 @@ func (s *Store) List() ([]Pipeline, error) {
return result, nil
}
func (s *Store) Delete(id string) error {
return s.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(bucket)
return b.Delete([]byte(id))
})
}

162
server/internal/queue/message_queue.go
View File

@@ -5,101 +5,119 @@ import (
"errors"
"log/slog"
evbus "github.com/asaskevich/EventBus"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/metadata"
"golang.org/x/sync/semaphore"
)
const queueName = "process:pending"
type MessageQueue struct {
concurrency int
eventBus evbus.Bus
concurrency int
downloadQueue chan downloaders.Downloader
metadataQueue chan downloaders.Downloader
ctx context.Context
cancel context.CancelFunc
}
// Creates a new message queue.
// By default it will be created with a size equals to nthe number of logical
// CPU cores -1.
// The queue size can be set via the qs flag.
func NewMessageQueue() (*MessageQueue, error) {
qs := config.Instance().QueueSize
qs := config.Instance().Server.QueueSize
if qs <= 0 {
return nil, errors.New("invalid queue size")
}
ctx, cancel := context.WithCancel(context.Background())
return &MessageQueue{
concurrency: qs,
eventBus: evbus.New(),
concurrency: qs,
downloadQueue: make(chan downloaders.Downloader, qs*2),
metadataQueue: make(chan downloaders.Downloader, qs*4),
ctx: ctx,
cancel: cancel,
}, nil
}
// Publish a message to the queue and set the task to a peding state.
func (m *MessageQueue) Publish(p downloaders.Downloader) {
// needs to have an id set before
p.SetPending(true)
// Publish download job
func (m *MessageQueue) Publish(d downloaders.Downloader) {
d.SetPending(true)
m.eventBus.Publish(queueName, p)
select {
case m.downloadQueue <- d:
slog.Info("published download", slog.String("id", d.GetId()))
case <-m.ctx.Done():
slog.Warn("queue stopped, dropping download", slog.String("id", d.GetId()))
}
}
// Workers: download + metadata
func (m *MessageQueue) SetupConsumers() {
go m.downloadConsumer()
go m.metadataSubscriber()
// N parallel workers for downloadQueue
for i := 0; i < m.concurrency; i++ {
go m.downloadWorker(i)
}
// 1 serial worker for metadata
go m.metadataWorker()
}
// Setup the consumer listener which subscribes to the changes to the producer
// channel and triggers the "download" action.
func (m *MessageQueue) downloadConsumer() {
sem := semaphore.NewWeighted(int64(m.concurrency))
m.eventBus.SubscribeAsync(queueName, func(p downloaders.Downloader) {
sem.Acquire(context.Background(), 1)
defer sem.Release(1)
slog.Info("received process from event bus",
slog.String("bus", queueName),
slog.String("consumer", "downloadConsumer"),
slog.String("id", p.GetId()),
)
if !p.IsCompleted() {
slog.Info("started process",
slog.String("bus", queueName),
slog.String("id", p.GetId()),
)
p.Start()
}
}, false)
}
// Setup the metadata consumer listener which subscribes to the changes to the
// producer channel and adds metadata to each download.
func (m *MessageQueue) metadataSubscriber() {
// How many concurrent metadata fetcher jobs are spawned
// Since there's ongoing downloads, 1 job at time seems a good compromise
sem := semaphore.NewWeighted(1)
m.eventBus.SubscribeAsync(queueName, func(p downloaders.Downloader) {
sem.Acquire(context.Background(), 1)
defer sem.Release(1)
slog.Info("received process from event bus",
slog.String("bus", queueName),
slog.String("consumer", "metadataConsumer"),
slog.String("id", p.GetId()),
)
if p.IsCompleted() {
slog.Warn("proccess has an illegal state",
slog.String("id", p.GetId()),
slog.String("status", "completed"),
)
// Worker dei download
func (m *MessageQueue) downloadWorker(workerId int) {
for {
select {
case <-m.ctx.Done():
return
case p := <-m.downloadQueue:
if p == nil {
continue
}
if p.IsCompleted() {
continue
}
slog.Info("download worker started",
slog.Int("worker", workerId),
slog.String("id", p.GetId()),
)
p.Start()
// after the download starts succesfully we pass it to the metadata queue
select {
case m.metadataQueue <- p:
slog.Info("queued for metadata", slog.String("id", p.GetId()))
case <-m.ctx.Done():
return
}
}
p.SetMetadata(metadata.DefaultFetcher)
}, false)
}
}
func (m *MessageQueue) metadataWorker() {
for {
select {
case <-m.ctx.Done():
return
case p := <-m.metadataQueue:
if p == nil {
continue
}
slog.Info("metadata worker started",
slog.String("id", p.GetId()),
)
if p.IsCompleted() {
slog.Warn("metadata skipped, illegal state",
slog.String("id", p.GetId()),
)
continue
}
p.SetMetadata(metadata.DefaultFetcher)
}
}
}
func (m *MessageQueue) Stop() {
m.cancel()
close(m.downloadQueue)
close(m.metadataQueue)
}

4
server/logging/handler.go
View File

@@ -91,10 +91,10 @@ func sse(logger *ObservableLogger) http.HandlerFunc {
func ApplyRouter(logger *ObservableLogger) func(chi.Router) {
return func(r chi.Router) {
if config.Instance().RequireAuth {
if config.Instance().Authentication.RequireAuth {
r.Use(middlewares.Authenticated)
}
if config.Instance().UseOpenId {
if config.Instance().OpenId.UseOpenId {
r.Use(openid.Middleware)
}
r.Get("/ws", webSocket(logger))

18
server/middleware/utils.go
View File

@@ -8,14 +8,14 @@ import (
)
func ApplyAuthenticationByConfig(next http.Handler) http.Handler {
handler := next
handler := next
if config.Instance().RequireAuth {
handler = Authenticated(handler)
}
if config.Instance().UseOpenId {
handler = openid.Middleware(handler)
}
if config.Instance().Authentication.RequireAuth {
handler = Authenticated(handler)
}
if config.Instance().OpenId.UseOpenId {
handler = openid.Middleware(handler)
}
return handler
}
return handler
}

15
server/openid/config.go
View File

@@ -14,24 +14,27 @@ var (
)
func Configure() {
if !config.Instance().UseOpenId {
if !config.Instance().OpenId.UseOpenId {
return
}
provider, err := oidc.NewProvider(context.Background(), config.Instance().OpenIdProviderURL)
provider, err := oidc.NewProvider(
context.Background(),
config.Instance().OpenId.ProviderURL,
)
if err != nil {
panic(err)
}
oauth2Config = oauth2.Config{
ClientID: config.Instance().OpenIdClientId,
ClientSecret: config.Instance().OpenIdClientSecret,
RedirectURL: config.Instance().OpenIdRedirectURL,
ClientID: config.Instance().OpenId.ClientId,
ClientSecret: config.Instance().OpenId.ClientSecret,
RedirectURL: config.Instance().OpenId.RedirectURL,
Endpoint: provider.Endpoint(),
Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
}
verifier = provider.Verifier(&oidc.Config{
ClientID: config.Instance().OpenIdClientId,
ClientID: config.Instance().OpenId.ClientId,
})
}

2
server/openid/handler.go
View File

@@ -87,7 +87,7 @@ func doAuthentification(r *http.Request, setCookieCallback func(t *oauth2.Token)
return nil, err
}
whitelist := config.Instance().OpenIdEmailWhitelist
whitelist := config.Instance().OpenId.EmailWhitelist
if len(whitelist) > 0 && !slices.Contains(whitelist, claims.Email) {
return nil, errors.New("email address not found in ACL")

2
server/playlist/playlist.go
View File

@@ -22,7 +22,7 @@ func PlaylistDetect(req internal.DownloadRequest, mq *queue.MessageQueue, db *kv
urlWithParams := append([]string{req.URL}, params...)
var (
downloader = config.Instance().DownloaderPath
downloader = config.Instance().Paths.DownloaderPath
cmd = exec.Command(downloader, urlWithParams...)
)

4
server/rest/container.go
View File

@@ -19,10 +19,10 @@ func ApplyRouter(args *ContainerArgs) func(chi.Router) {
h := Container(args)
return func(r chi.Router) {
if config.Instance().RequireAuth {
if config.Instance().Authentication.RequireAuth {
r.Use(middlewares.Authenticated)
}
if config.Instance().UseOpenId {
if config.Instance().OpenId.UseOpenId {
r.Use(openid.Middleware)
}
r.Post("/exec", h.Exec())

2
server/rest/service.go
View File

@@ -179,7 +179,7 @@ func (s *Service) GetVersion(ctx context.Context) (string, string, error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
defer cancel()
cmd := exec.CommandContext(ctx, config.Instance().DownloaderPath, "--version")
cmd := exec.CommandContext(ctx, config.Instance().Paths.DownloaderPath, "--version")
go func() {
stdout, _ := cmd.Output()
result <- string(stdout)

4
server/rpc/container.go
View File

@@ -22,10 +22,10 @@ func Container(db *kv.Store, mq *queue.MessageQueue, lm *livestream.Monitor) *Se
// RPC service must be registered before applying this router!
func ApplyRouter() func(chi.Router) {
return func(r chi.Router) {
if config.Instance().RequireAuth {
if config.Instance().Authentication.RequireAuth {
r.Use(middlewares.Authenticated)
}
if config.Instance().UseOpenId {
if config.Instance().OpenId.UseOpenId {
r.Use(openid.Middleware)
}
r.Get("/ws", WebSocket)

113
server/server.go
View File

@@ -11,10 +11,8 @@ import (
"net/http"
"net/rpc"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/go-chi/chi/v5"
@@ -23,6 +21,7 @@ import (
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/filebrowser"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/livestream"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/pipeline"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/logging"
middlewares "github.com/marcopiovanello/yt-dlp-web-ui/v3/server/middleware"
@@ -44,29 +43,30 @@ type RunConfig struct {
}
type serverConfig struct {
frontend fs.FS
swagger fs.FS
mdb *kv.Store
db *bolt.DB
mq *queue.MessageQueue
lm *livestream.Monitor
tm *twitch.Monitor
frontend fs.FS
swagger fs.FS
mdb *kv.Store
db *bolt.DB
mq *queue.MessageQueue
lm *livestream.Monitor
taskRunner task.TaskRunner
twitchMonitor *twitch.Monitor
}
// TODO: change scope
var observableLogger = logging.NewObservableLogger()
func RunBlocking(rc *RunConfig) {
dbPath := filepath.Join(config.Instance().SessionFilePath, "bolt.db")
func Run(ctx context.Context, rc *RunConfig) error {
dbPath := filepath.Join(config.Instance().Paths.LocalDatabasePath, "bolt.db")
boltdb, err := bolt.Open(dbPath, 0600, nil)
if err != nil {
panic(err)
return err
}
mdb, err := kv.NewStore(boltdb, time.Second*15)
if err != nil {
panic(err)
return err
}
// ---- LOGGING ---------------------------------------------------
@@ -78,10 +78,10 @@ func RunBlocking(rc *RunConfig) {
conf := config.Instance()
// file based logging
if conf.EnableFileLogging {
logger, err := logging.NewRotableLogger(conf.LogPath)
if conf.Logging.EnableFileLogging {
logger, err := logging.NewRotableLogger(conf.Logging.LogPath)
if err != nil {
panic(err)
return err
}
defer logger.Rotate()
@@ -106,7 +106,7 @@ func RunBlocking(rc *RunConfig) {
mq, err := queue.NewMessageQueue()
if err != nil {
panic(err)
return err
}
mq.SetupConsumers()
go mdb.Restore(mq)
@@ -124,41 +124,45 @@ func RunBlocking(rc *RunConfig) {
boltdb,
)
go tm.Monitor(
context.TODO(),
ctx,
config.Instance().Twitch.CheckInterval,
twitch.DEFAULT_DOWNLOAD_HANDLER(mdb, mq),
)
go tm.Restore()
cronTaskRunner := task.NewCronTaskRunner(mq, mdb)
go cronTaskRunner.Spawner(ctx)
scfg := serverConfig{
frontend: rc.App,
swagger: rc.Swagger,
mdb: mdb,
db: boltdb,
mq: mq,
lm: lm,
tm: tm,
frontend: rc.App,
swagger: rc.Swagger,
mdb: mdb,
db: boltdb,
mq: mq,
lm: lm,
twitchMonitor: tm,
taskRunner: cronTaskRunner,
}
srv := newServer(scfg)
go gracefulShutdown(srv, &scfg)
go gracefulShutdown(ctx, srv, &scfg)
var (
network = "tcp"
address = fmt.Sprintf("%s:%d", conf.Host, conf.Port)
address = fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port)
)
// support unix sockets
if strings.HasPrefix(conf.Host, "/") {
if strings.HasPrefix(conf.Server.Host, "/") {
network = "unix"
address = conf.Host
address = conf.Server.Host
}
listener, err := net.Listen(network, address)
if err != nil {
slog.Error("failed to listen", slog.String("err", err.Error()))
return
return err
}
slog.Info("yt-dlp-webui started", slog.String("address", address))
@@ -166,14 +170,12 @@ func RunBlocking(rc *RunConfig) {
if err := srv.Serve(listener); err != nil {
slog.Warn("http server stopped", slog.String("err", err.Error()))
}
return nil
}
func newServer(c serverConfig) *http.Server {
// archiver.Register(c.db)
cronTaskRunner := task.NewCronTaskRunner(c.mq, c.mdb)
go cronTaskRunner.Spawner(context.TODO())
service := ytdlpRPC.Container(c.mdb, c.mq, c.lm)
rpc.Register(service)
@@ -197,7 +199,7 @@ func newServer(c serverConfig) *http.Server {
// use in dev
// r.Use(middleware.Logger)
baseUrl := config.Instance().BaseURL
baseUrl := config.Instance().Server.BaseURL
r.Mount(baseUrl+"/", http.StripPrefix(baseUrl, http.FileServerFS(c.frontend)))
// swagger
@@ -246,36 +248,35 @@ func newServer(c serverConfig) *http.Server {
r.Route("/status", status.ApplyRouter(c.mdb))
// Subscriptions
r.Route("/subscriptions", subscription.Container(c.db, cronTaskRunner).ApplyRouter())
r.Route("/subscriptions", subscription.Container(c.db, c.taskRunner).ApplyRouter())
// Twitch
r.Route("/twitch", func(r chi.Router) {
r.Use(middlewares.ApplyAuthenticationByConfig)
r.Get("/users", twitch.GetMonitoredUsers(c.tm))
r.Post("/user", twitch.MonitorUserHandler(c.tm))
r.Delete("/user/{user}", twitch.DeleteUser(c.tm))
r.Get("/users", twitch.GetMonitoredUsers(c.twitchMonitor))
r.Post("/user", twitch.MonitorUserHandler(c.twitchMonitor))
r.Delete("/user/{user}", twitch.DeleteUser(c.twitchMonitor))
})
// Pipelines
r.Route("/pipelines", func(r chi.Router) {
h := pipeline.NewRestHandler(c.db)
r.Use(middlewares.ApplyAuthenticationByConfig)
r.Get("/id/{id}", h.GetPipeline)
r.Get("/all", h.GetAllPipelines)
r.Post("/", h.SavePipeline)
r.Delete("/id/{id}", h.DeletePipeline)
})
return &http.Server{Handler: r}
}
func gracefulShutdown(srv *http.Server, cfg *serverConfig) {
ctx, stop := signal.NotifyContext(context.Background(),
os.Interrupt,
syscall.SIGTERM,
syscall.SIGQUIT,
)
func gracefulShutdown(ctx context.Context, srv *http.Server, cfg *serverConfig) {
<-ctx.Done()
slog.Info("shutdown signal received")
go func() {
<-ctx.Done()
slog.Info("shutdown signal received")
defer func() {
cfg.mdb.Persist()
cfg.db.Close()
stop()
srv.Shutdown(context.Background())
}()
defer func() {
cfg.db.Close()
srv.Shutdown(context.Background())
}()
}

4
server/subscription/rest/handler.go
View File

@@ -19,10 +19,10 @@ type RestHandler struct {
// ApplyRouter implements domain.RestHandler.
func (h *RestHandler) ApplyRouter() func(chi.Router) {
return func(r chi.Router) {
if config.Instance().RequireAuth {
if config.Instance().Authentication.RequireAuth {
r.Use(middlewares.Authenticated)
}
if config.Instance().UseOpenId {
if config.Instance().OpenId.UseOpenId {
r.Use(openid.Middleware)
}

2
server/subscription/task/runner.go
View File

@@ -129,7 +129,7 @@ func (t *CronTaskRunner) fetcher(ctx context.Context, req *monitorTask) time.Dur
cmd := exec.CommandContext(
ctx,
config.Instance().DownloaderPath,
config.Instance().Paths.DownloaderPath,
"-I1",
"--flat-playlist",
"--print", "webpage_url",

4
server/sys/fs.go
View File

@@ -14,7 +14,7 @@ import (
// FreeSpace gets the available Bytes writable to download directory
func FreeSpace() (uint64, error) {
var stat unix.Statfs_t
unix.Statfs(config.Instance().DownloadPath, &stat)
unix.Statfs(config.Instance().Paths.DownloadPath, &stat)
return (stat.Bavail * uint64(stat.Bsize)), nil
}
@@ -27,7 +27,7 @@ func DirectoryTree() (*[]string, error) {
}
var (
rootPath = config.Instance().DownloadPath
rootPath = config.Instance().Paths.DownloadPath
stack = internal.NewStack[Node]()
flattened = make([]string, 0)

2
server/twitch/monitor.go
View File

@@ -121,7 +121,7 @@ func DEFAULT_DOWNLOAD_HANDLER(db *kv.Store, mq *queue.MessageQueue) func(user st
var (
url = fmt.Sprintf("https://www.twitch.tv/%s", user)
filename = filepath.Join(
config.Instance().DownloadPath,
config.Instance().Paths.DownloadPath,
fmt.Sprintf("%s (live) %s", user, time.Now().Format(time.ANSIC)),
)
ext = ".webm"

2
server/updater/update.go
View File

@@ -8,7 +8,7 @@ import (
// Update using the builtin function of yt-dlp
func UpdateExecutable() error {
cmd := exec.Command(config.Instance().DownloaderPath, "-U")
cmd := exec.Command(config.Instance().Paths.DownloaderPath, "-U")
err := cmd.Start()
if err != nil {

13
server/user/handlers.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/golang-jwt/jwt/v5"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
"golang.org/x/crypto/bcrypt"
)
const TOKEN_COOKIE_NAME = "jwt-yt-dlp-webui"
@@ -26,11 +27,17 @@ func Login(w http.ResponseWriter, r *http.Request) {
}
var (
username = config.Instance().Username
password = config.Instance().Password
username = config.Instance().Authentication.Username
passwordHash = config.Instance().Authentication.PasswordHash
)
if username != req.Username || password != req.Password {
err := bcrypt.CompareHashAndPassword([]byte(passwordHash), []byte(req.Password))
if err != nil {
http.Error(w, "invalid username or password", http.StatusBadRequest)
return
}
if username != req.Username {
http.Error(w, "invalid username or password", http.StatusBadRequest)
return
}