refactoring-1
introduced pipelines and abstracted download process.go in Downloader interface
This commit is contained in:
42
server/internal/downloaders/common.go
Normal file
42
server/internal/downloaders/common.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package downloaders
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
|
||||
)
|
||||
|
||||
type DownloaderBase struct {
|
||||
Id string
|
||||
URL string
|
||||
Metadata common.DownloadMetadata
|
||||
Pending bool
|
||||
Completed bool
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
func (d *DownloaderBase) FetchMetadata(fetcher func(url string) (*common.DownloadMetadata, error)) {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
meta, err := fetcher(d.URL)
|
||||
if err != nil {
|
||||
slog.Error("failed to retrieve metadata", slog.Any("err", err))
|
||||
return
|
||||
}
|
||||
|
||||
d.Metadata = *meta
|
||||
}
|
||||
|
||||
func (d *DownloaderBase) SetPending(p bool) {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
d.Pending = p
|
||||
}
|
||||
|
||||
func (d *DownloaderBase) Complete() {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
d.Completed = true
|
||||
}
|
||||
26
server/internal/downloaders/downloader.go
Normal file
26
server/internal/downloaders/downloader.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package downloaders
|
||||
|
||||
import (
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
|
||||
)
|
||||
|
||||
type Downloader interface {
|
||||
Start() error
|
||||
Stop() error
|
||||
Status() *internal.ProcessSnapshot
|
||||
|
||||
SetOutput(output internal.DownloadOutput)
|
||||
SetProgress(progress internal.DownloadProgress)
|
||||
SetMetadata(fetcher func(url string) (*common.DownloadMetadata, error))
|
||||
SetPending(p bool)
|
||||
|
||||
IsCompleted() bool
|
||||
|
||||
UpdateSavedFilePath(path string)
|
||||
|
||||
RestoreFromSnapshot(*internal.ProcessSnapshot) error
|
||||
|
||||
GetId() string
|
||||
GetUrl() string
|
||||
}
|
||||
211
server/internal/downloaders/generic.go
Normal file
211
server/internal/downloaders/generic.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package downloaders
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
|
||||
)
|
||||
|
||||
const downloadTemplate = `download:
|
||||
{
|
||||
"eta":%(progress.eta)s,
|
||||
"percentage":"%(progress._percent_str)s",
|
||||
"speed":%(progress.speed)s
|
||||
}`
|
||||
|
||||
// filename not returning the correct extension after postprocess
|
||||
const postprocessTemplate = `postprocess:
|
||||
{
|
||||
"filepath":"%(info.filepath)s"
|
||||
}
|
||||
`
|
||||
|
||||
type GenericDownloader struct {
|
||||
Params []string
|
||||
|
||||
AutoRemove bool
|
||||
|
||||
progress internal.DownloadProgress
|
||||
output internal.DownloadOutput
|
||||
|
||||
proc *os.Process
|
||||
|
||||
logConsumer LogConsumer
|
||||
|
||||
// embedded
|
||||
DownloaderBase
|
||||
}
|
||||
|
||||
func NewGenericDownload(url string, params []string) Downloader {
|
||||
g := &GenericDownloader{
|
||||
logConsumer: NewJSONLogConsumer(),
|
||||
}
|
||||
// in base
|
||||
g.Id = uuid.NewString()
|
||||
g.URL = url
|
||||
return g
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) Start() error {
|
||||
g.SetPending(true)
|
||||
|
||||
g.Params = argsSanitizer(g.Params)
|
||||
|
||||
out := internal.DownloadOutput{
|
||||
Path: config.Instance().DownloadPath,
|
||||
Filename: "%(title)s.%(ext)s",
|
||||
}
|
||||
|
||||
if g.output.Path != "" {
|
||||
out.Path = g.output.Path
|
||||
}
|
||||
|
||||
if g.output.Filename != "" {
|
||||
out.Filename = g.output.Filename
|
||||
}
|
||||
|
||||
buildFilename(&g.output)
|
||||
|
||||
templateReplacer := strings.NewReplacer("\n", "", "\t", "", " ", "")
|
||||
|
||||
baseParams := []string{
|
||||
strings.Split(g.URL, "?list")[0], //no playlist
|
||||
"--newline",
|
||||
"--no-colors",
|
||||
"--no-playlist",
|
||||
"--progress-template",
|
||||
templateReplacer.Replace(downloadTemplate),
|
||||
"--progress-template",
|
||||
templateReplacer.Replace(postprocessTemplate),
|
||||
"--no-exec",
|
||||
}
|
||||
|
||||
// if user asked to manually override the output path...
|
||||
if !(slices.Contains(g.Params, "-P") || slices.Contains(g.Params, "--paths")) {
|
||||
g.Params = append(g.Params, "-o")
|
||||
g.Params = append(g.Params, fmt.Sprintf("%s/%s", out.Path, out.Filename))
|
||||
}
|
||||
|
||||
params := append(baseParams, g.Params...)
|
||||
|
||||
slog.Info("requesting download", slog.String("url", g.URL), slog.Any("params", params))
|
||||
|
||||
cmd := exec.Command(config.Instance().DownloaderPath, params...)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
slog.Error("failed to get a stdout pipe", slog.Any("err", err))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
slog.Error("failed to get a stderr pipe", slog.Any("err", err))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
slog.Error("failed to start yt-dlp process", slog.Any("err", err))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
g.proc = cmd.Process
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer func() {
|
||||
stdout.Close()
|
||||
g.Complete()
|
||||
cancel()
|
||||
}()
|
||||
|
||||
logs := make(chan []byte)
|
||||
go produceLogs(stdout, logs)
|
||||
go consumeLogs(ctx, logs, g.logConsumer, g)
|
||||
|
||||
go printYtDlpErrors(stderr, g.Id, g.URL)
|
||||
|
||||
g.SetPending(false)
|
||||
return cmd.Wait()
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) Stop() error {
|
||||
defer func() {
|
||||
g.progress.Status = internal.StatusCompleted
|
||||
g.Complete()
|
||||
}()
|
||||
// yt-dlp uses multiple child process the parent process
|
||||
// has been spawned with setPgid = true. To properly kill
|
||||
// all subprocesses a SIGTERM need to be sent to the correct
|
||||
// process group
|
||||
if g.proc == nil {
|
||||
return errors.New("*os.Process not set")
|
||||
}
|
||||
|
||||
pgid, err := syscall.Getpgid(g.proc.Pid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syscall.Kill(-pgid, syscall.SIGTERM); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) Status() *internal.ProcessSnapshot {
|
||||
return &internal.ProcessSnapshot{
|
||||
Id: g.Id,
|
||||
Info: g.Metadata,
|
||||
Progress: g.progress,
|
||||
Output: g.output,
|
||||
Params: g.Params,
|
||||
DownloaderName: "generic",
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) UpdateSavedFilePath(p string) { g.output.SavedFilePath = p }
|
||||
|
||||
func (g *GenericDownloader) SetOutput(o internal.DownloadOutput) { g.output = o }
|
||||
func (g *GenericDownloader) SetProgress(p internal.DownloadProgress) { g.progress = p }
|
||||
|
||||
func (g *GenericDownloader) SetMetadata(fetcher func(url string) (*common.DownloadMetadata, error)) {
|
||||
g.FetchMetadata(fetcher)
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) SetPending(p bool) {
|
||||
g.Pending = p
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) GetId() string { return g.Id }
|
||||
func (g *GenericDownloader) GetUrl() string { return g.URL }
|
||||
|
||||
func (g *GenericDownloader) RestoreFromSnapshot(snap *internal.ProcessSnapshot) error {
|
||||
if snap == nil {
|
||||
return errors.New("cannot restore nil snapshot")
|
||||
}
|
||||
|
||||
s := *snap
|
||||
|
||||
g.Id = s.Id
|
||||
g.URL = s.Info.URL
|
||||
g.Metadata = s.Info
|
||||
g.progress = s.Progress
|
||||
g.output = s.Output
|
||||
g.Params = s.Params
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GenericDownloader) IsCompleted() bool { return g.Completed }
|
||||
205
server/internal/downloaders/livestream.go
Normal file
205
server/internal/downloaders/livestream.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package downloaders
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/pipes"
|
||||
)
|
||||
|
||||
type LiveStreamDownloader struct {
|
||||
progress internal.DownloadProgress
|
||||
|
||||
proc *os.Process
|
||||
|
||||
logConsumer LogConsumer
|
||||
|
||||
pipes []pipes.Pipe
|
||||
|
||||
// embedded
|
||||
DownloaderBase
|
||||
}
|
||||
|
||||
func NewLiveStreamDownloader(url string, pipes []pipes.Pipe) Downloader {
|
||||
l := &LiveStreamDownloader{
|
||||
logConsumer: NewFFMpegLogConsumer(),
|
||||
pipes: pipes,
|
||||
}
|
||||
// in base
|
||||
l.Id = uuid.NewString()
|
||||
l.URL = url
|
||||
return l
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) Start() error {
|
||||
l.SetPending(true)
|
||||
|
||||
baseParams := []string{
|
||||
l.URL,
|
||||
"--newline",
|
||||
"--no-colors",
|
||||
"--no-playlist",
|
||||
"--no-exec",
|
||||
}
|
||||
|
||||
params := append(baseParams, "-o", "-")
|
||||
|
||||
cmd := exec.Command(config.Instance().DownloaderPath, params...)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
|
||||
// stdout = media stream
|
||||
media, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
slog.Error("failed to get media stdout", slog.Any("err", err))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// stderr = log/progress
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
slog.Error("failed to get stderr pipe", slog.Any("err", err))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
slog.Error("failed to start yt-dlp process", slog.Any("err", err))
|
||||
panic(err)
|
||||
}
|
||||
|
||||
l.proc = cmd.Process
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer func() {
|
||||
l.Complete()
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// --- costruisci pipeline ---
|
||||
reader := io.Reader(media)
|
||||
for _, pipe := range l.pipes {
|
||||
nr, err := pipe.Connect(reader)
|
||||
if err != nil {
|
||||
slog.Error("pipe failed", slog.String("pipe", pipe.Name()), slog.Any("err", err))
|
||||
return err
|
||||
}
|
||||
reader = nr
|
||||
}
|
||||
|
||||
// --- fallback: se nessun FileWriter, scrivi su file ---
|
||||
if !l.hasFileWriter() {
|
||||
go func() {
|
||||
filepath.Join(
|
||||
config.Instance().DownloadPath,
|
||||
fmt.Sprintf("%s (live) %s.mp4", l.Id, time.Now().Format(time.ANSIC)),
|
||||
)
|
||||
|
||||
defaultPath := filepath.Join(config.Instance().DownloadPath)
|
||||
f, err := os.Create(defaultPath)
|
||||
if err != nil {
|
||||
slog.Error("failed to create fallback file", slog.Any("err", err))
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = io.Copy(f, reader)
|
||||
if err != nil {
|
||||
slog.Error("copy error", slog.Any("err", err))
|
||||
}
|
||||
slog.Info("download saved", slog.String("path", defaultPath))
|
||||
}()
|
||||
}
|
||||
|
||||
// --- logs consumer ---
|
||||
logs := make(chan []byte)
|
||||
go produceLogs(stderr, logs)
|
||||
go consumeLogs(ctx, logs, l.logConsumer, l)
|
||||
|
||||
l.progress.Status = internal.StatusLiveStream
|
||||
|
||||
return cmd.Wait()
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) Stop() error {
|
||||
defer func() {
|
||||
l.progress.Status = internal.StatusCompleted
|
||||
l.Complete()
|
||||
}()
|
||||
// yt-dlp uses multiple child process the parent process
|
||||
// has been spawned with setPgid = true. To properly kill
|
||||
// all subprocesses a SIGTERM need to be sent to the correct
|
||||
// process group
|
||||
if l.proc == nil {
|
||||
return errors.New("*os.Process not set")
|
||||
}
|
||||
|
||||
pgid, err := syscall.Getpgid(l.proc.Pid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syscall.Kill(-pgid, syscall.SIGTERM); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) Status() *internal.ProcessSnapshot {
|
||||
return &internal.ProcessSnapshot{
|
||||
Id: l.Id,
|
||||
Info: l.Metadata,
|
||||
Progress: l.progress,
|
||||
DownloaderName: "livestream",
|
||||
}
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) UpdateSavedFilePath(p string) {}
|
||||
|
||||
func (l *LiveStreamDownloader) SetOutput(o internal.DownloadOutput) {}
|
||||
func (l *LiveStreamDownloader) SetProgress(p internal.DownloadProgress) { l.progress = p }
|
||||
|
||||
func (l *LiveStreamDownloader) SetMetadata(fetcher func(url string) (*common.DownloadMetadata, error)) {
|
||||
l.FetchMetadata(fetcher)
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) SetPending(p bool) {
|
||||
l.Pending = p
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) GetId() string { return l.Id }
|
||||
func (l *LiveStreamDownloader) GetUrl() string { return l.URL }
|
||||
|
||||
func (l *LiveStreamDownloader) RestoreFromSnapshot(snap *internal.ProcessSnapshot) error {
|
||||
if snap == nil {
|
||||
return errors.New("cannot restore nil snapshot")
|
||||
}
|
||||
|
||||
s := *snap
|
||||
|
||||
l.Id = s.Id
|
||||
l.URL = s.Info.URL
|
||||
l.Metadata = s.Info
|
||||
l.progress = s.Progress
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LiveStreamDownloader) IsCompleted() bool { return l.Completed }
|
||||
|
||||
func (l *LiveStreamDownloader) hasFileWriter() bool {
|
||||
return slices.ContainsFunc(l.pipes, func(p pipes.Pipe) bool {
|
||||
return p.Name() == "file-writer"
|
||||
})
|
||||
}
|
||||
68
server/internal/downloaders/logging.go
Normal file
68
server/internal/downloaders/logging.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package downloaders
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
|
||||
)
|
||||
|
||||
type LogConsumer interface {
|
||||
GetName() string
|
||||
ParseLogEntry(entry []byte, downloader Downloader)
|
||||
}
|
||||
|
||||
type JSONLogConsumer struct{}
|
||||
|
||||
func NewJSONLogConsumer() LogConsumer {
|
||||
return &JSONLogConsumer{}
|
||||
}
|
||||
|
||||
func (j *JSONLogConsumer) GetName() string { return "json-log-consumer" }
|
||||
|
||||
func (j *JSONLogConsumer) ParseLogEntry(entry []byte, d Downloader) {
|
||||
var progress internal.ProgressTemplate
|
||||
var postprocess internal.PostprocessTemplate
|
||||
|
||||
if err := json.Unmarshal(entry, &progress); err == nil {
|
||||
d.SetProgress(internal.DownloadProgress{
|
||||
Status: internal.StatusDownloading,
|
||||
Percentage: progress.Percentage,
|
||||
Speed: progress.Speed,
|
||||
ETA: progress.Eta,
|
||||
})
|
||||
|
||||
slog.Info("progress",
|
||||
slog.String("id", j.GetShortId(d.GetId())),
|
||||
slog.String("url", d.GetUrl()),
|
||||
slog.String("percentage", progress.Percentage),
|
||||
)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(entry, &postprocess); err == nil {
|
||||
d.UpdateSavedFilePath(postprocess.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func (j *JSONLogConsumer) GetShortId(id string) string {
|
||||
return strings.Split(id, "-")[0]
|
||||
}
|
||||
|
||||
//TODO: split in different files
|
||||
|
||||
type FFMpegLogConsumer struct{}
|
||||
|
||||
func NewFFMpegLogConsumer() LogConsumer {
|
||||
return &JSONLogConsumer{}
|
||||
}
|
||||
|
||||
func (f *FFMpegLogConsumer) GetName() string { return "ffmpeg-log-consumer" }
|
||||
|
||||
func (f *FFMpegLogConsumer) ParseLogEntry(entry []byte, d Downloader) {
|
||||
slog.Info("ffmpeg output",
|
||||
slog.String("id", d.GetId()),
|
||||
slog.String("url", d.GetUrl()),
|
||||
slog.String("output", string(entry)),
|
||||
)
|
||||
}
|
||||
1
server/internal/downloaders/playlist.go
Normal file
1
server/internal/downloaders/playlist.go
Normal file
@@ -0,0 +1 @@
|
||||
package downloaders
|
||||
76
server/internal/downloaders/utils.go
Normal file
76
server/internal/downloaders/utils.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package downloaders
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
|
||||
)
|
||||
|
||||
func argsSanitizer(params []string) []string {
|
||||
params = slices.DeleteFunc(params, func(e string) bool {
|
||||
match, _ := regexp.MatchString(`(\$\{)|(\&\&)`, e)
|
||||
return match
|
||||
})
|
||||
|
||||
params = slices.DeleteFunc(params, func(e string) bool {
|
||||
return e == ""
|
||||
})
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
func buildFilename(o *internal.DownloadOutput) {
|
||||
if o.Filename != "" && strings.Contains(o.Filename, ".%(ext)s") {
|
||||
o.Filename += ".%(ext)s"
|
||||
}
|
||||
|
||||
o.Filename = strings.Replace(
|
||||
o.Filename,
|
||||
".%(ext)s.%(ext)s",
|
||||
".%(ext)s",
|
||||
1,
|
||||
)
|
||||
}
|
||||
|
||||
func produceLogs(r io.Reader, logs chan<- []byte) {
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(r)
|
||||
|
||||
for scanner.Scan() {
|
||||
logs <- scanner.Bytes()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func consumeLogs(ctx context.Context, logs <-chan []byte, c LogConsumer, d Downloader) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("detaching logs",
|
||||
slog.String("url", d.GetUrl()),
|
||||
slog.String("id", c.GetName()),
|
||||
)
|
||||
return
|
||||
case entry := <-logs:
|
||||
c.ParseLogEntry(entry, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printYtDlpErrors(stdout io.Reader, shortId, url string) {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
|
||||
for scanner.Scan() {
|
||||
slog.Error("yt-dlp process error",
|
||||
slog.String("id", shortId),
|
||||
slog.String("url", url),
|
||||
slog.String("err", scanner.Text()),
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user