refactoring: config struct & pipelines

introduced pipelines and abstracted download process.go in Downloader interface

.deploystack/docker-run.txt

@@ -0,0 +1 @@
+docker run -d -p 3033:3033 -v /downloads:/downloads marcobaobao/yt-dlp-webui

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/go
+{
+	"name": "Go",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/go:1-1.23-bookworm",
+	"features": {
+		"ghcr.io/devcontainers-extra/features/pnpm:2": {},
+		"ghcr.io/devcontainers-extra/features/ffmpeg-apt-get:1": {},
+		"ghcr.io/devcontainers-extra/features/yt-dlp:2": {}
+	}
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "go version"
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.dockerignore

@@ -3,6 +3,7 @@
 result/
 result
 dist
+.pnpm-store/
 .pnpm-debug.log
 node_modules
 .env
@@ -20,9 +21,11 @@ cookies.txt
 __debug*
 ui/
 .idea
+.idea/
 frontend/.pnp.cjs
 frontend/.pnp.loader.mjs
 frontend/.yarn/install-state.gz
 .db.lock
 livestreams.dat
-.git
+.vite/deps
+archive.txt

.gitignore

@@ -3,6 +3,7 @@
 result/
 result
 dist
+.pnpm-store/
 .pnpm-debug.log
 node_modules
 .env
@@ -26,3 +27,6 @@ frontend/.pnp.loader.mjs
 frontend/.yarn/install-state.gz
 .db.lock
 livestreams.dat
+.vite/deps
+archive.txt
+twitch-monitor.dat

Dockerfile

@@ -1,8 +1,8 @@
 # Node (pnpm) ------------------------------------------------------------------
-FROM node:20-slim AS ui
+FROM node:22-slim AS ui
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
+RUN corepack prepare pnpm@10.0.0 --activate && corepack enable
 COPY . /usr/src/yt-dlp-webui
 
 WORKDIR /usr/src/yt-dlp-webui/frontend
@@ -24,11 +24,12 @@ COPY --from=ui /usr/src/yt-dlp-webui/frontend /usr/src/yt-dlp-webui/frontend
 RUN CGO_ENABLED=0 GOOS=linux go build -o yt-dlp-webui
 # -----------------------------------------------------------------------------
 
-# dependencies ----------------------------------------------------------------
-FROM alpine:edge
+# Runtime ---------------------------------------------------------------------
+FROM python:3.13.2-alpine3.21
 
 RUN apk update && \
-apk add ffmpeg yt-dlp ca-certificates curl wget psmisc
+apk add ffmpeg ca-certificates curl wget gnutls --no-cache && \
+pip install "yt-dlp[default,curl-cffi,mutagen,pycryptodomex,phantomjs,secretstorage]"
 
 VOLUME /downloads /config
 
@@ -39,4 +40,4 @@ COPY --from=build /usr/src/yt-dlp-webui/yt-dlp-webui /app
 ENV JWT_SECRET=secret
 
 EXPOSE 3033
-ENTRYPOINT [ "./yt-dlp-webui" , "--out", "/downloads", "--conf", "/config/config.yml", "--db", "/config/local.db" ]
+ENTRYPOINT [ "./yt-dlp-webui" , "--out", "/downloads", "--conf", "/config/config.yml", "--db", "/config/local.db" ]

README.md

@@ -25,6 +25,11 @@ docker pull ghcr.io/marcopiovanello/yt-dlp-web-ui:latest
 
 *Keeps the project alive!* 😃
 
+## Community stuff
+Feel free to join :)
+
+[Discord](https://discord.gg/GZAX5FfGzE)
+
 ## Some screeshots
 ![image](https://github.com/user-attachments/assets/fc43a3fb-ecf9-449d-b5cb-5d5635020c00)
 ![image](https://github.com/user-attachments/assets/3210f6ac-0dd8-403c-b839-3c24ff7d7d00)
@@ -110,6 +115,16 @@ services:
     restart: unless-stopped
 ```
 
+### ⚡ One-Click Deploy
+
+| Cloud Provider | Deploy Button |
+|----------------|---------------|
+| AWS | <a href="https://deploystack.io/deploy/marcopiovanello-yt-dlp-web-ui?provider=aws&language=cfn"><img src="https://raw.githubusercontent.com/deploystackio/deploy-templates/refs/heads/main/.assets/img/aws.svg" height="38"></a> |
+| DigitalOcean | <a href="https://deploystack.io/deploy/marcopiovanello-yt-dlp-web-ui?provider=do&language=dop"><img src="https://raw.githubusercontent.com/deploystackio/deploy-templates/refs/heads/main/.assets/img/do.svg" height="38"></a> |
+| Render | <a href="https://deploystack.io/deploy/marcopiovanello-yt-dlp-web-ui?provider=rnd&language=rnd"><img src="https://raw.githubusercontent.com/deploystackio/deploy-templates/refs/heads/main/.assets/img/rnd.svg" height="38"></a> |
+
+<sub>Generated by <a href="https://deploystack.io/c/marcopiovanello-yt-dlp-web-ui" target="_blank">DeployStack.io</a></sub>
+
 ## [Prebuilt binaries](https://github.com/marcopiovanello/yt-dlp-web-ui/releases) installation
 
 ```sh

frontend/package.json

@@ -1,11 +1,12 @@
 {
   "name": "yt-dlp-webui",
-  "version": "3.2.3",
+  "version": "3.2.6",
   "description": "Frontend compontent of yt-dlp-webui",
   "scripts": {
-    "dev": "vite",
+    "dev": "vite --host 0.0.0.0",
     "build": "vite build"
   },
+  "type": "module",
   "author": "marcopiovanello",
   "license": "GPL-3.0-only",
   "private": true,
@@ -17,11 +18,11 @@
     "@mui/icons-material": "^6.2.0",
     "@mui/material": "^6.2.0",
     "fp-ts": "^2.16.5",
+    "jotai": "^2.10.3",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "react-router-dom": "^6.23.1",
     "react-virtuoso": "^4.7.11",
-    "jotai": "^2.10.3",
     "rxjs": "^7.8.1"
   },
   "devDependencies": {

frontend/src/Layout.tsx

@@ -6,6 +6,7 @@ import LiveTvIcon from '@mui/icons-material/LiveTv'
 import Menu from '@mui/icons-material/Menu'
 import SettingsIcon from '@mui/icons-material/Settings'
 import TerminalIcon from '@mui/icons-material/Terminal'
+import UpdateIcon from '@mui/icons-material/Update'
 import { Box, createTheme } from '@mui/material'
 import CssBaseline from '@mui/material/CssBaseline'
 import Divider from '@mui/material/Divider'
@@ -27,6 +28,7 @@ import Footer from './components/Footer'
 import Logout from './components/Logout'
 import SocketSubscriber from './components/SocketSubscriber'
 import ThemeToggler from './components/ThemeToggler'
+import TwitchIcon from './components/TwitchIcon'
 import { useI18n } from './hooks/useI18n'
 import Toaster from './providers/ToasterProvider'
 import { getAccentValue } from './utils'
@@ -140,6 +142,32 @@ export default function Layout() {
                 <ListItemText primary={i18n.t('archiveButtonLabel')} />
               </ListItemButton>
             </Link>
+            <Link to={'/subscriptions'} style={
+              {
+                textDecoration: 'none',
+                color: mode === 'dark' ? '#ffffff' : '#000000DE'
+              }
+            }>
+              <ListItemButton>
+                <ListItemIcon>
+                  <UpdateIcon />
+                </ListItemIcon>
+                <ListItemText primary={i18n.t('subscriptionsButtonLabel')} />
+              </ListItemButton>
+            </Link>
+            <Link to={'/twitch'} style={
+              {
+                textDecoration: 'none',
+                color: mode === 'dark' ? '#ffffff' : '#000000DE'
+              }
+            }>
+              <ListItemButton>
+                <ListItemIcon>
+                  <TwitchIcon />
+                </ListItemIcon>
+                <ListItemText primary={"Twitch"} />
+              </ListItemButton>
+            </Link>
             <Link to={'/monitor'} style={
               {
                 textDecoration: 'none',

frontend/src/assets/i18n.yaml

@@ -18,4 +18,5 @@ languages:
   russian: ru.yaml
   swedish: sv.yaml
   ukrainian: uk.yaml
-  chinese: zh_CN.yaml
+  chinese: zh_CN.yaml
+  hungarian: hu.yaml

frontend/src/assets/i18n/ca.yaml

@@ -21,6 +21,7 @@ keys:
   overridesAnchor: Anul·lacions
   pathOverrideOption: Sobreescriure en la ruta de sortida
   filenameOverrideOption: Sobreescriure el nom del fitxer
+  autoFileExtensionOption: Afegeix l'extensió de fitxer automàticament
   customFilename: Nom d'arxiu personalitzat (en blanc per utilitzar el predeterminat)
   customPath: Ruta personalitzada
   customArgs: Habilitar els arguments yt-dlp personalitzats (un gran poder comporta una gran responsabilitat)
@@ -68,3 +69,12 @@ keys:
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
   deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/de.yaml

@@ -22,6 +22,7 @@ keys:
   overridesAnchor: Überschreibungen
   pathOverrideOption: Ausgabe-Pfad Überschreibung aktivieren
   filenameOverrideOption: Ausgabe-Dateiname Überschreibung aktivieren
+  autoFileExtensionOption: Dateierweiterung automatisch hinzufügen
   customFilename: Benutzerdefinierter Dateiname (Leer lassen um Standardwert zu nutzen)
   customPath: Benutzerdefinierter Pfad
   customArgs: Benutzerdefinierte yt-dlp Argumente aktivieren (Auf viel Macht folgt große Verantwortung)
@@ -70,3 +71,12 @@ keys:
   noFilesFound: 'Keine Dateien gefunden'
   tableView: 'Tabellenansicht'
   deleteSelected: 'Ausgewählte löschen'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/en_US.yaml

@@ -22,15 +22,16 @@ keys:
   overridesAnchor: Overrides
   pathOverrideOption: Enable output path overriding
   filenameOverrideOption: Enable output file name overriding
+  autoFileExtensionOption: Automatically add file extension
   customFilename: Custom filename (leave blank to use default)
   customPath: Custom path
   customArgs: Enable custom yt-dlp args (great power = great responsibilities)
   customArgsInput: Custom yt-dlp arguments
-  rpcConnErr: Error while conencting to RPC server
+  rpcConnErr: Error while connecting to RPC server
   splashText: No active downloads
   archiveTitle: Archive
   clipboardAction: Copied URL to clipboard
-  playlistCheckbox: Download playlist (it will take time, after submitting you may close this window)
+  playlistCheckbox: Download playlist
   restartAppMessage: Needs a page reload to take effect
   servedFromReverseProxyCheckbox: Is behind a reverse proxy
   urlBase: URL base, for reverse proxy support (subdir), defaults to empty
@@ -69,4 +70,17 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription
+  clearCompletedButton: 'Clear completed'
+  twitchIntegrationInfo: |
+    To enable monitoring Twitch streams follow this wiki page.
+    https://github.com/marcopiovanello/yt-dlp-web-ui/wiki/Twitch-integration

frontend/src/assets/i18n/es.yaml

@@ -21,6 +21,7 @@ keys:
   overridesAnchor: Anulaciones
   pathOverrideOption: Sobreescribir en la ruta de salida
   filenameOverrideOption: Sobreescribir el nombre del fichero
+  autoFileExtensionOption: Agregar extensión de archivo automáticamente
   customFilename: Nombre de archivo personalizado (en blanco para usar el predeterminado)
   customPath: Ruta personalizada
   customArgs: Habilitar los argumentos yt-dlp personalizados (un gran poder conlleva una gran responsabilidad)
@@ -67,4 +68,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/fr.yaml

@@ -22,6 +22,7 @@ keys:
   overridesAnchor: Remplacer
   pathOverrideOption: Activer le remplacement du chemin de sortie
   filenameOverrideOption: Activer le remplacement du nom du fichier de sortie
+  autoFileExtensionOption: Ajouter automatiquement l'extension de fichier
   customFilename: Nom de fichier personnalisé (laisser vide pour utiliser le nom par défaut)
   customPath: Chemin personnalisé
   customArgs: Activer les args personnalisés yt-dlp (grand pouvoir = grandes responsabilités)
@@ -71,4 +72,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/hu.yaml

@@ -0,0 +1,82 @@
+keys:
+  urlInput: Video URL (soronként egy)
+  statusTitle: Állapot
+  statusReady: Előkészítve
+  selectFormatButton: Válassz formátumot
+  startButton: Indítás
+  abortAllButton: Összes megszakítása
+  updateBinButton: yt-dlp bináris frissítése
+  darkThemeButton: Sötét téma
+  lightThemeButton: Világos téma
+  settingsAnchor: Beállítások
+  serverAddressTitle: Szerver címe
+  serverPortTitle: Port
+  extractAudioCheckbox: Audio konvertálása
+  noMTimeCheckbox: Fájl módosítás időpontja ne legyen beállítva
+  bgReminder: Miután a lap bezárásra kerül, a letöltés folytatódni fog a háttérben.
+  toastConnected: 'Kapcsolódva: '
+  toastUpdated: yt-dlp bináris frissítése sikeres volt!
+  formatSelectionEnabler: Video/audio formátum manuális kiválasztásának engedélyezése
+  themeSelect: 'Téma'
+  languageSelect: 'Nyelv'
+  overridesAnchor: Felülbírálások
+  pathOverrideOption: Letöltési útvonal felülbírálása
+  filenameOverrideOption: Letöltési fájlnév felülbírálása
+  autoFileExtensionOption: Automatikus fájlkiterjesztés
+  customFilename: Egyedi fájlnév (hagyd üresen, hogy a fájlnév automatikusan generálódjon)
+  customPath: Egyedi útvonal
+  customArgs: Egyedi yt-dlp argumentumok (Nagy hatalommal nagy felelősség jár.)
+  customArgsInput: Egyedi yt-dlp argumentumok
+  rpcConnErr: Hiba történt az RPC szerver történő kapcsolódáskor
+  splashText: Nincs aktív letöltés
+  archiveTitle: Archívum
+  clipboardAction: URL a vágólapra másolva.
+  playlistCheckbox: Lejátszási lista letöltése (Több időt vehet igénybe. A letöltés a háttérben történik, a böngészőablak szabadon bezárható.)
+  restartAppMessage: Az oldal újratöltése lehet szükséges a változtatások megjelenítéséhez.
+  servedFromReverseProxyCheckbox: Reverse proxy mögötti működés
+  urlBase: URL base, reverse proxy támogatásához (subdir), alapból üres
+  newDownloadButton: Új letöltés
+  homeButtonLabel: Kezdőlap
+  archiveButtonLabel: Archívum
+  settingsButtonLabel: Beállítások
+  rpcAuthenticationLabel: RPC bejelentkezés
+  themeTogglerLabel: Témaválasztó
+  loadingLabel: Betöltés...
+  appTitle: Alkalmazás címe
+  savedTemplates: Mentett sablonok
+  templatesEditor: Sablonszerkesztő
+  templatesEditorNameLabel: Sablon neve
+  templatesEditorContentLabel: Sablon tartalma
+  logsTitle: 'Naplók'
+  awaitingLogs: 'Napló letöltése...'
+  bulkDownload: 'Fájlok letöltése ZIP archívumként'
+  rpcPollingTimeTitle: RPC lekérdezési időköz
+  rpcPollingTimeDescription: Rövidebb időköz nagyobb processzor terheléssel járhat (mind szerver és böngésző oldalon is)
+  templatesReloadInfo: Az új sablon megjelenéséhez újra kell tölteni az oldalt.
+  livestreamURLInput: Élő stream URL
+  livestreamStatusWaiting: Várakozás a kezdésre
+  livestreamStatusDownloading: Letöltés
+  livestreamStatusCompleted: Letöltve
+  livestreamStatusErrored: Hiba
+  livestreamStatusUnknown: Ismeretlen
+  livestreamNoMonitoring: Nincsenek figyelt élő adások
+  livestreamDownloadInfo: |
+    Ez figyelni fog egy még el nem indított élő közvetítést. Minden folyamat a --wait-for-video 10 paraméterrel lesz végrehajtva.
+    Ha egy már elindított élő közvetítés van megadva, az továbbra is letöltésre kerül, de a folyamatát nem követi nyomon.
+    Amint elindul, az élő közvetítés átkerül a letöltések oldalra..
+  livestreamExperimentalWarning: Ez a funkció még kísérleti. Nem garantált a hibamentes működés.
+  accentSelect: 'Kiemelt szín'
+  generalDownloadSettings: 'Általános letöltési beállítások'
+  deleteCookies: Sütik törlése
+  noFilesFound: 'Nem található fájlok'
+  tableView: 'Táblázatos Nézet'
+  deleteSelected: 'Kiválasztottak törlése'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/it_IT.yaml

@@ -1,8 +1,9 @@
 keys:
   urlInput: URL Video (uno per linea)
   statusTitle: Stato
-  startButton: Inizia
   statusReady: Pronto
+  selectFormatButton: Seziona formato
+  startButton: Inizia
   abortAllButton: Termina tutto
   updateBinButton: Aggiorna yt-dlp
   darkThemeButton: Tema scuro
@@ -21,51 +22,61 @@ keys:
   overridesAnchor: Sovrascritture
   pathOverrideOption: Abilita sovrascrittura percorso di output
   filenameOverrideOption: Abilita sovrascrittura del nome del file di output
-  customFilename: Custom filename (leave blank to use default)
-  customPath: Custom path
-  customArgs: Enable custom yt-dlp args (great power = great responsabilities)
-  customArgsInput: Custom yt-dlp arguments
-  rpcConnErr: Error nella connessione al server RPC
+  autoFileExtensionOption: Aggiungi estensione automaticamente
+  customFilename: Nome file personalizzato (lascia vuoto per utilizzare quello predefinito)
+  customPath: Percorso personalizzato
+  customArgs: Abilita argomenti yt-dlp personalizzati (grande potere = grandi responsabilità)
+  customArgsInput: Argomenti yt-dlp personalizzati
+  rpcConnErr: Errore nella connessione al server RPC
   splashText: Nessun download attivo
   archiveTitle: Archivio
   clipboardAction: URL copiato negli appunti
   playlistCheckbox: Download playlist (richiederà tempo, puoi chiudere la finestra dopo l'inoltro)
-  restartAppMessage: La finestra deve essere ricaricata perché abbia effetto
-  servedFromReverseProxyCheckbox: Is behind a reverse proxy
+  restartAppMessage: La finestra deve essere ricaricata affinché abbia effetto
+  servedFromReverseProxyCheckbox: È dietro un reverse proxy
+  urlBase: base URL, per supporto a reverse proxy (subdir), default vuoto
   newDownloadButton: Nuovo download
   homeButtonLabel: Home
-  archiveButtonLabel: Archive
-  settingsButtonLabel: Settings
-  rpcAuthenticationLabel: RPC authentication
-  themeTogglerLabel: Theme toggler
-  loadingLabel: Loading...
+  archiveButtonLabel: Archivio
+  settingsButtonLabel: Impostazioni
+  rpcAuthenticationLabel: Autenticazione RPC
+  themeTogglerLabel: Selettore Tema
+  loadingLabel: Caricamento...
   appTitle: Titolo applicazione
-  savedTemplates: Template salvati
-  templatesEditor: Editor template
-  templatesEditorNameLabel: Nome template
-  templatesEditorContentLabel: Contentunto template
+  savedTemplates: Modelli salvati
+  templatesEditor: Editor modelli
+  templatesEditorNameLabel: Nome modello
+  templatesEditorContentLabel: Contenuto del modello
   logsTitle: 'Logs'
   awaitingLogs: 'Awaiting logs...'
-  bulkDownload: 'Download files in a zip archive'
-  templatesReloadInfo: To register a new template it might need a page reload.
-  livestreamURLInput: Livestream URL
-  livestreamStatusWaiting: Waiting/Wait start
-  livestreamStatusDownloading: Downloading
-  livestreamStatusCompleted: Completed
-  livestreamStatusErrored: Errored
-  livestreamStatusUnknown: Unknown
-  livestreamNoMonitoring: No livestreams monitored
-  livestreamDownloadInfo: |
-    This will monitor yet to start livestream. Each process will be executed with --wait-for-video 10.
-    If an already started livestream is provided it will be still downloaded but its progress will not be tracked.
-    Once started the livestream will be migrated to the downloads page.
-  livestreamExperimentalWarning: This feature is still experimental. Something might break!
-  accentSelect: 'Accent'
-  urlBase: base URL, per supporto a reverse proxy (subdir), default vuoto
+  bulkDownload: 'Scaricare i file in un archivio zip'
   rpcPollingTimeTitle: Intervallo di polling RPC
   rpcPollingTimeDescription: Un intervallo più corto implica un maggior utilizzo di CPU (lato client e server)
-  generalDownloadSettings: 'General Download Settings'
-  deleteCookies: Delete Cookies
-  noFilesFound: 'No Files Found'
-  tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  templatesReloadInfo: Per registrare un nuovo modello potrebbe essere necessario ricaricare la pagina.
+  livestreamURLInput: Livestream URL
+  livestreamStatusWaiting: Attesa inizio
+  livestreamStatusDownloading: Downloading
+  livestreamStatusCompleted: Completato
+  livestreamStatusErrored: Errore
+  livestreamStatusUnknown: Sconosciuto
+  livestreamNoMonitoring: Nessun livestream monitorato
+  livestreamDownloadInfo: |
+    Questo monitorerà il livestream ancora da avviare. Ogni processo verrà eseguito con --wait-for-video 10.
+    Se viene fornito un livestream già avviato, questo verrà comunque scaricato, ma il suo progresso non verrà monitorato.
+    Una volta avviato, il livestream verrà migrato nella pagina dei download.
+  livestreamExperimentalWarning: Questa funzione è ancora sperimentale. Qualcosa potrebbe rompersi!
+  accentSelect: 'Accent'
+  generalDownloadSettings: 'Impostazioni generali di download'
+  deleteCookies: Elimina Cookies
+  noFilesFound: 'Nessun file trovato'
+  tableView: 'Vista Tabella'
+  deleteSelected: 'Elimina selezionati'
+  subscriptionsButtonLabel: 'Abbonamenti'
+  subscriptionsEmptyLabel: 'Nessuna iscrizione'
+  subscriptionsURLInput: 'URL Canale'
+  subscriptionsInfo: |
+    Iscrive a un canale definito. Verrà scaricato solo l'ultimo video.
+    Il lavoro di monitoraggio sarà programmato/attivato da un'espressione cron definita (se lasciata vuota, l'impostazione predefinita è ogni 5 minuti).
+  cronExpressionLabel: 'Espressione Cron'
+  editButtonLabel: 'Modifica'
+  newSubscriptionButton: Nuova iscrizione

frontend/src/assets/i18n/ja.yaml

@@ -22,6 +22,7 @@ keys:
   overridesAnchor: 上書き
   pathOverrideOption: 保存するディレクトリ
   filenameOverrideOption: ファイル名の上書き
+  autoFileExtensionOption: 自動ファイル拡張子
   customFilename:  (空白の場合は元のファイル名)
   customPath: 保存先
   customArgs: yt-dlpのオプションの有効化 (最適設定にする場合)
@@ -68,4 +69,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/ko.yaml

@@ -21,6 +21,7 @@ keys:
   overridesAnchor: Overrides
   pathOverrideOption: Enable output path overriding
   filenameOverrideOption: Enable output file name overriding
+  autoFileExtensionOption: 자동으로 파일 확장자 추가
   customFilename: Custom filename (leave blank to use default)
   customPath: Custom path
   customArgs: Enable custom yt-dlp args (great power = great responsabilities)
@@ -67,4 +68,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/pl.yaml

@@ -21,6 +21,7 @@ keys:
   overridesAnchor: Przedefiniuj
   pathOverrideOption: Aktywuj zastąpienie ścieżki źródłowej
   filenameOverrideOption: Aktywuj zastępowanie nazwy pliku źródłowego
+  autoFileExtensionOption: Automatyczne rozszerzenie pliku
   customFilename: Wprowadź nazwę pliku (pozostaw puste, aby użyć nazwy domyślnej)
   customPath: Ustaw ścieżkę
   customArgs: Uwzględnij konfigurowalne argumenty yt-dlp (wielka moc = wielka odpowiedzialność)
@@ -67,4 +68,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/pt_BR.yaml

@@ -22,6 +22,7 @@ keys:
   overridesAnchor: Substituições
   pathOverrideOption: Habilitar substituição do caminho de saída
   filenameOverrideOption: Habilitar substituição do nome do arquivo de saída
+  autoFileExtensionOption: Adicionar extensão de arquivo automaticamente
   customFilename: Nome de arquivo personalizado (deixe em branco para usar o padrão)
   customPath: Caminho personalizado
   customArgs: Habilitar argumentos personalizados do yt-dlp (grandes poderes = grandes responsabilidades)
@@ -69,4 +70,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/ru.yaml

@@ -21,6 +21,7 @@ keys:
   overridesAnchor: Переопределить
   pathOverrideOption: Активировать переопределение выходного пути
   filenameOverrideOption: Активировать переопределение имени выходного файла
+  autoFileExtensionOption: Автоматическое расширение файла
   customFilename: Задать имя файла (оставьте пустым, чтобы использовать значение по умолчанию)
   customPath: Задать путь
   customArgs: Включить настраиваемые аргументы yt-dlp (большая сила = большая ответственность)
@@ -67,4 +68,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/sv.yaml

@@ -22,6 +22,7 @@ keys:
   overridesAnchor: Överskrivningar
   pathOverrideOption: Tillåt överskrivning av filsökvägen
   filenameOverrideOption: Tillåt överskrivning av filnamn
+  autoFileExtensionOption: Lägg till filändelse automatiskt
   customFilename: Eget filnamn (lämna blankt för standardnamn)
   customPath: Egen filsökväg
   customArgs: Tillåt egna yt-dlp-argument (frihet under ansvar!)
@@ -69,4 +70,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/uk.yaml

@@ -21,6 +21,7 @@ keys:
   overridesAnchor: Перевизначити
   pathOverrideOption: Активувати перевизначення вихідного шляху
   filenameOverrideOption: Активувати перевизначення імені вихідного файлу
+  autoFileExtensionOption: Автоматичне додавання розширення файлу
   customFilename: Введіть ім'я файлу (залишіть порожнім, щоб використовувати значення за замовчуванням)
   customPath: Задати шлях
   customArgs: Включити аргументи, що настроюються yt-dlp (велика сила = велика відповідальність)
@@ -67,4 +68,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/assets/i18n/zh_CN.yaml

@@ -22,6 +22,7 @@ keys:
   overridesAnchor: 覆盖
   pathOverrideOption: 启用输出路径覆盖
   filenameOverrideOption: 启用输出文件名覆盖
+  autoFileExtensionOption: 自动文件扩展名
   customFilename: 自定义文件名（留空使用默认值）
   customPath: 自定义路径
   customArgs: 启用自定义 yt-dlp 参数（能力越大 = 责任越大）
@@ -69,4 +70,13 @@ keys:
   deleteCookies: Delete Cookies
   noFilesFound: 'No Files Found'
   tableView: 'Table View'
-  deleteSelected: 'Delete selected'
+  deleteSelected: 'Delete selected'
+  subscriptionsButtonLabel: 'Subscriptions'
+  subscriptionsEmptyLabel: 'No subscriptions'
+  subscriptionsURLInput: 'Channel URL'
+  subscriptionsInfo: |
+    Subscribes to a defined channel. Only the last video will be downloaded.
+    The monitor job will be scheduled/triggered by a defined cron expression (defaults to every 5 minutes if left blank).
+  cronExpressionLabel: 'Cron expression'
+  editButtonLabel: 'Edit'
+  newSubscriptionButton: New subscription

frontend/src/atoms/downloadTemplate.ts

@@ -1,10 +1,10 @@
 import { getOrElse } from 'fp-ts/lib/Either'
 import { pipe } from 'fp-ts/lib/function'
+import { atom } from 'jotai'
+import { atomWithStorage } from 'jotai/utils'
 import { ffetch } from '../lib/httpClient'
 import { CustomTemplate } from '../types'
 import { serverSideCookiesState, serverURL } from './settings'
-import { atom } from 'jotai'
-import { atomWithStorage } from 'jotai/utils'
 
 export const cookiesTemplateState = atom<Promise<string>>(async (get) =>
   await get(serverSideCookiesState)
@@ -22,12 +22,6 @@ export const filenameTemplateState = atomWithStorage(
   localStorage.getItem('lastFilenameTemplate') ?? ''
 )
 
-export const downloadTemplateState = atom<Promise<string>>(async (get) =>
-  `${get(customArgsState)} ${await get(cookiesTemplateState)}`
-    .replace(/  +/g, ' ')
-    .trim()
-)
-
 export const savedTemplatesState = atom<Promise<CustomTemplate[]>>(async (get) => {
   const task = ffetch<CustomTemplate[]>(`${get(serverURL)}/api/v1/template/all`)
   const either = await task()

frontend/src/atoms/settings.ts

@@ -19,7 +19,8 @@ export const languages = [
   'russian',
   'spanish',
   'swedish',
-  'ukrainian'
+  'ukrainian',
+  'hungarian'
 ] as const
 
 export type Language = (typeof languages)[number]
@@ -39,6 +40,7 @@ export interface SettingsState {
   cliArgs: string
   formatSelection: boolean
   fileRenaming: boolean
+  autoFileExtension: boolean
   pathOverriding: boolean
   enableCustomArgs: boolean
   listView: boolean
@@ -81,6 +83,11 @@ export const fileRenamingState = atomWithStorage(
   localStorage.getItem('file-renaming') === 'true'
 )
 
+export const autoFileExtensionState = atomWithStorage(
+  'auto-file-extension',
+  localStorage.getItem('auto-file-extension') === 'true'
+)
+
 export const pathOverridingState = atomWithStorage(
   'path-overriding',
   localStorage.getItem('path-overriding') === 'true'
@@ -114,11 +121,18 @@ export const appTitleState = atomWithStorage(
 export const serverAddressAndPortState = atom((get) => {
   if (get(servedFromReverseProxySubDirState)) {
     return `${get(serverAddressState)}/${get(servedFromReverseProxySubDirState)}/`
+      .replaceAll('"', '')    // XXX: atomWithStorage uses JSON.stringify to serialize
+      .replaceAll('//', '/')  // which puts extra double quotes.
   }
   if (get(servedFromReverseProxyState)) {
     return `${get(serverAddressState)}`
+      .replaceAll('"', '')
   }
-  return `${get(serverAddressState)}:${get(serverPortState)}`
+
+  const sap = `${get(serverAddressState)}:${get(serverPortState)}`
+    .replaceAll('"', '')
+
+  return sap.endsWith('/') ? sap.slice(0, -1) : sap
 })
 
 export const serverURL = atom((get) =>
@@ -127,15 +141,17 @@ export const serverURL = atom((get) =>
 
 export const rpcWebSocketEndpoint = atom((get) => {
   const proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
-  return `${proto}//${get(serverAddressAndPortState)}/rpc/ws`
-}
-)
+  const sap = get(serverAddressAndPortState)
+
+  return `${proto}//${sap.endsWith('/') ? sap.slice(0, -1) : sap}/rpc/ws`
+})
 
 export const rpcHTTPEndpoint = atom((get) => {
   const proto = window.location.protocol
-  return `${proto}//${get(serverAddressAndPortState)}/rpc/http`
-}
-)
+  const sap = get(serverAddressAndPortState)
+
+  return `${proto}//${sap.endsWith('/') ? sap.slice(0, -1) : sap}/rpc/http`
+})
 
 export const serverSideCookiesState = atom<Promise<string>>(async (get) => await pipe(
   ffetch<Readonly<{ cookies: string }>>(`${get(serverURL)}/api/v1/cookies`),
@@ -167,10 +183,10 @@ export const settingsState = atom<SettingsState>((get) => ({
   cliArgs: get(latestCliArgumentsState),
   formatSelection: get(formatSelectionState),
   fileRenaming: get(fileRenamingState),
+  autoFileExtension: get(autoFileExtensionState),
   pathOverriding: get(pathOverridingState),
   enableCustomArgs: get(enableCustomArgsState),
   listView: get(listViewState),
   servedFromReverseProxy: get(servedFromReverseProxyState),
   appTitle: get(appTitleState)
-})
-)
+}))

frontend/src/components/CustomArgsTextField.tsx

@@ -0,0 +1,35 @@
+import { TextField } from '@mui/material'
+import { useAtom, useAtomValue } from 'jotai'
+import { customArgsState } from '../atoms/downloadTemplate'
+import { settingsState } from '../atoms/settings'
+import { useI18n } from '../hooks/useI18n'
+import { useEffect } from 'react'
+
+const CustomArgsTextField: React.FC = () => {
+  const { i18n } = useI18n()
+
+  const settings = useAtomValue(settingsState)
+
+  const [customArgs, setCustomArgs] = useAtom(customArgsState)
+
+  useEffect(() => {
+    setCustomArgs('')
+  }, [])
+
+  const handleCustomArgsChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    setCustomArgs(e.target.value)
+  }
+
+  return (
+    <TextField
+      fullWidth
+      label={i18n.t('customArgsInput')}
+      variant="outlined"
+      onChange={handleCustomArgsChange}
+      value={customArgs}
+      disabled={settings.formatSelection}
+    />
+  )
+}
+
+export default CustomArgsTextField

frontend/src/components/DownloadDialog.tsx

@@ -11,7 +11,10 @@ import {
   Grid,
   IconButton,
   InputAdornment,
+  MenuItem,
   Paper,
+  Select,
+  SelectChangeEvent,
   TextField
 } from '@mui/material'
 import AppBar from '@mui/material/AppBar'
@@ -25,14 +28,13 @@ import {
   FC,
   Suspense,
   forwardRef,
-  useEffect,
   useRef,
   useState,
   useTransition
 } from 'react'
 import {
+  cookiesTemplateState,
   customArgsState,
-  downloadTemplateState,
   filenameTemplateState,
   savedTemplatesState
 } from '../atoms/downloadTemplate'
@@ -44,6 +46,7 @@ import { useI18n } from '../hooks/useI18n'
 import { useRPC } from '../hooks/useRPC'
 import type { DLMetadata } from '../types'
 import { toFormatArgs } from '../utils'
+import CustomArgsTextField from './CustomArgsTextField'
 import ExtraDownloadOptions from './ExtraDownloadOptions'
 import LoadingBackdrop from './LoadingBackdrop'
 
@@ -66,8 +69,9 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
   const settings = useAtomValue(settingsState)
   const isConnected = useAtomValue(connectedState)
   const availableDownloadPaths = useAtomValue(availableDownloadPathsState)
-  const downloadTemplate = useAtomValue(downloadTemplateState)
   const savedTemplates = useAtomValue(savedTemplatesState)
+  const customArgs = useAtomValue(customArgsState)
+  const cookies = useAtomValue(cookiesTemplateState)
 
   const [downloadFormats, setDownloadFormats] = useState<DLMetadata>()
   const [pickedVideoFormat, setPickedVideoFormat] = useState('')
@@ -75,14 +79,14 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
   const [pickedBestFormat, setPickedBestFormat] = useState('')
   const [isFormatsLoading, setIsFormatsLoading] = useState(false)
 
-  const [customArgs, setCustomArgs] = useAtom(customArgsState)
-
   const [downloadPath, setDownloadPath] = useState('')
 
   const [filenameTemplate, setFilenameTemplate] = useAtom(
     filenameTemplateState
   )
 
+  const [fileExtension, setFileExtension] = useState('.%(ext)s')
+
   const [url, setUrl] = useState('')
 
   const [isPlaylist, setIsPlaylist] = useState(false)
@@ -96,10 +100,6 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
 
   const [isPending, startTransition] = useTransition()
 
-  useEffect(() => {
-    setCustomArgs('')
-  }, [open])
-
   /**
     * Retrive url from input, cli-arguments from checkboxes and emits via WebSocket
   */
@@ -110,12 +110,16 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
       if (pickedAudioFormat !== '') codes.push(pickedAudioFormat)
       if (pickedBestFormat !== '') codes.push(pickedBestFormat)
 
+      const downloadTemplate = `${customArgs} ${cookies}`
+        .replace(/  +/g, ' ')
+        .trim()
+
       await new Promise(r => setTimeout(r, 10))
       client.download({
         url: immediate || line,
         args: `${toFormatArgs(codes)} ${downloadTemplate}`,
         pathOverride: downloadPath ?? '',
-        renameTo: settings.fileRenaming ? filenameTemplate : '',
+        renameTo: settings.fileRenaming ? filenameTemplate + (settings.autoFileExtension ? fileExtension : '') : '',
         playlist: isPlaylist,
       })
 
@@ -169,8 +173,8 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
     setFilenameTemplate(e.target.value)
   }
 
-  const handleCustomArgsChange = (e: React.ChangeEvent<HTMLInputElement>) => {
-    setCustomArgs(e.target.value)
+  const handleFileExtensionChange = (e: SelectChangeEvent<string>) => {
+    setFileExtension(e.target.value)
   }
 
   const parseUrlListFile = async (e: React.ChangeEvent<HTMLInputElement>) => {
@@ -268,25 +272,22 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
                   />
                 </Grid>
                 <Grid container spacing={1} sx={{ mt: 1 }}>
-                  {
-                    settings.enableCustomArgs &&
+                  {settings.enableCustomArgs &&
                     <Grid item xs={12}>
-                      <TextField
-                        fullWidth
-                        label={i18n.t('customArgsInput')}
-                        variant="outlined"
-                        onChange={handleCustomArgsChange}
-                        value={customArgs}
-                        disabled={
-                          !isConnected ||
-                          (settings.formatSelection && downloadFormats != null)
-                        }
-                      />
+                      <CustomArgsTextField />
                     </Grid>
                   }
                   {
                     settings.fileRenaming &&
-                    <Grid item xs={settings.pathOverriding ? 8 : 12}>
+                    <Grid item xs={
+                      !settings.autoFileExtension && !settings.pathOverriding
+                        ? 12
+                        : !settings.autoFileExtension && settings.pathOverriding
+                          ? 8
+                          : settings.autoFileExtension && !settings.pathOverriding
+                            ? 10
+                            : 6
+                    }>
                       <TextField
                         sx={{ mt: 1 }}
                         ref={customFilenameInputRef}
@@ -302,6 +303,22 @@ const DownloadDialog: FC<Props> = ({ open, onClose, onDownloadStart }) => {
                       />
                     </Grid>
                   }
+                  {
+                    settings.autoFileExtension &&
+                    <Grid item xs={2}>
+                      <Select
+                        sx={{ mt: 1 }}
+                        fullWidth
+                        label={i18n.t('autoFileExtension')}
+                        value={fileExtension}
+                        onChange={handleFileExtensionChange}
+                        variant="outlined">
+                        <MenuItem value=".%(ext)s">Auto</MenuItem>
+                        <MenuItem value=".mp4">mp4</MenuItem>
+                        <MenuItem value=".mkv">mkv</MenuItem>
+                      </Select>
+                    </Grid>
+                  }
                   {
                     settings.pathOverriding &&
                     <Grid item xs={4}>

frontend/src/components/HomeSpeedDial.tsx

@@ -1,5 +1,6 @@
 import AddCircleIcon from '@mui/icons-material/AddCircle'
 import BuildCircleIcon from '@mui/icons-material/BuildCircle'
+import ClearAllIcon from '@mui/icons-material/ClearAll'
 import DeleteForeverIcon from '@mui/icons-material/DeleteForever'
 import FolderZipIcon from '@mui/icons-material/FolderZip'
 import FormatListBulleted from '@mui/icons-material/FormatListBulleted'
@@ -42,6 +43,11 @@ const HomeSpeedDial: React.FC<Props> = ({ onDownloadOpen, onEditorOpen }) => {
         tooltipTitle={i18n.t('bulkDownload')}
         onClick={() => window.open(`${serverAddr}/archive/bulk?token=${localStorage.getItem('token')}`)}
       />
+      <SpeedDialAction
+        icon={<ClearAllIcon />}
+        tooltipTitle={i18n.t('clearCompletedButton')}
+        onClick={() => client.clearCompleted()}
+      />
       <SpeedDialAction
         icon={<DeleteForeverIcon />}
         tooltipTitle={i18n.t('abortAllButton')}

frontend/src/components/TemplatesEditor.tsx

@@ -1,7 +1,5 @@
 import AddIcon from '@mui/icons-material/Add'
 import CloseIcon from '@mui/icons-material/Close'
-import DeleteIcon from '@mui/icons-material/Delete'
-import EditIcon from '@mui/icons-material/Edit'
 import {
   Alert,
   AppBar,
@@ -20,13 +18,13 @@ import {
 import { TransitionProps } from '@mui/material/transitions'
 import { matchW } from 'fp-ts/lib/Either'
 import { pipe } from 'fp-ts/lib/function'
+import { useAtomValue } from 'jotai'
 import { forwardRef, useEffect, useState, useTransition } from 'react'
 import { serverURL } from '../atoms/settings'
 import { useToast } from '../hooks/toast'
 import { useI18n } from '../hooks/useI18n'
 import { ffetch } from '../lib/httpClient'
 import { CustomTemplate } from '../types'
-import { useAtomValue } from 'jotai'
 import TemplateTextField from './TemplateTextField'
 
 const Transition = forwardRef(function Transition(

frontend/src/components/TwitchIcon.tsx

@@ -0,0 +1,22 @@
+import { useAtomValue } from 'jotai'
+import { settingsState } from '../atoms/settings'
+
+const TwitchIcon: React.FC = () => {
+  const { theme } = useAtomValue(settingsState)
+
+  return (
+    <svg
+      role="img"
+      viewBox="0 0 24 24"
+      width={24}
+      height={24}
+      xmlns="http://www.w3.org/2000/svg"
+      style={{ fill: theme === 'dark' ? '#fff' : '#757575' }}
+    >
+      <title>Twitch</title>
+      <path d="M11.571 4.714h1.715v5.143H11.57zm4.715 0H18v5.143h-1.714zM6 0L1.714 4.286v15.428h5.143V24l4.286-4.286h3.428L22.286 12V0zm14.571 11.143l-3.428 3.428h-3.429l-3 3v-3H6.857V1.714h13.714Z" />
+    </svg>
+  )
+}
+
+export default TwitchIcon

frontend/src/components/VersionIndicator.tsx

@@ -7,6 +7,7 @@ const VersionIndicator: React.FC = () => {
 
   return (
     <div style={{ display: 'flex', gap: 4, alignItems: 'center' }}>
+      <Chip label={`UI v3.2.5`} variant="outlined" size="small" />
       <Chip label={`RPC v${version.rpcVersion}`} variant="outlined" size="small" />
       <Chip label={`yt-dlp v${version.ytdlpVersion}`} variant="outlined" size="small" />
     </div>

frontend/src/components/subscriptions/NoSubscriptions.tsx

@@ -0,0 +1,38 @@
+import UpdateIcon from '@mui/icons-material/Update'
+import { Container, SvgIcon, Typography, styled } from '@mui/material'
+import { useI18n } from '../../hooks/useI18n'
+
+const FlexContainer = styled(Container)({
+  display: 'flex',
+  minWidth: '100%',
+  minHeight: '80vh',
+  alignItems: 'center',
+  justifyContent: 'center',
+  flexDirection: 'column'
+})
+
+const Title = styled(Typography)({
+  display: 'flex',
+  width: '100%',
+  alignItems: 'center',
+  justifyContent: 'center',
+  paddingBottom: '0.5rem'
+})
+
+
+export default function NoSubscriptions() {
+  const { i18n } = useI18n()
+
+  return (
+    <FlexContainer>
+      <Title fontWeight={'500'} fontSize={72} color={'gray'}>
+        <SvgIcon sx={{ fontSize: '200px' }}>
+          <UpdateIcon />
+        </SvgIcon>
+      </Title>
+      <Title fontWeight={'500'} fontSize={36} color={'gray'}>
+        {i18n.t('subscriptionsEmptyLabel')}
+      </Title>
+    </FlexContainer>
+  )
+}

frontend/src/components/subscriptions/SubscriptionsDialog.tsx

@@ -0,0 +1,164 @@
+import CloseIcon from '@mui/icons-material/Close'
+import {
+  Alert,
+  AppBar,
+  Box,
+  Button,
+  Container,
+  Dialog,
+  Grid,
+  IconButton,
+  Paper,
+  Slide,
+  TextField,
+  Toolbar,
+  Typography
+} from '@mui/material'
+import { TransitionProps } from '@mui/material/transitions'
+import { matchW } from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import { useAtomValue } from 'jotai'
+import { forwardRef, startTransition, useState } from 'react'
+import { customArgsState } from '../../atoms/downloadTemplate'
+import { serverURL } from '../../atoms/settings'
+import { useToast } from '../../hooks/toast'
+import { useI18n } from '../../hooks/useI18n'
+import { ffetch } from '../../lib/httpClient'
+import { Subscription } from '../../services/subscriptions'
+import ExtraDownloadOptions from '../ExtraDownloadOptions'
+
+type Props = {
+  open: boolean
+  onClose: () => void
+}
+
+const Transition = forwardRef(function Transition(
+  props: TransitionProps & {
+    children: React.ReactElement
+  },
+  ref: React.Ref<unknown>,
+) {
+  return <Slide direction="up" ref={ref} {...props} />
+})
+
+const SubscriptionsDialog: React.FC<Props> = ({ open, onClose }) => {
+  const [subscriptionURL, setSubscriptionURL] = useState('')
+  const [subscriptionCron, setSubscriptionCron] = useState('')
+
+  const customArgs = useAtomValue(customArgsState)
+
+  const { i18n } = useI18n()
+  const { pushMessage } = useToast()
+
+  const baseURL = useAtomValue(serverURL)
+
+  const submit = async (sub: Omit<Subscription, 'id'>) => {
+    const task = ffetch<void>(`${baseURL}/subscriptions`, {
+      method: 'POST',
+      body: JSON.stringify(sub)
+    })
+    const either = await task()
+
+    pipe(
+      either,
+      matchW(
+        (l) => pushMessage(l, 'error'),
+        (_) => onClose()
+      )
+    )
+  }
+
+  return (
+    <Dialog
+      fullScreen
+      open={open}
+      onClose={onClose}
+      TransitionComponent={Transition}
+    >
+      <AppBar sx={{ position: 'relative' }}>
+        <Toolbar>
+          <IconButton
+            edge="start"
+            color="inherit"
+            onClick={onClose}
+            aria-label="close"
+          >
+            <CloseIcon />
+          </IconButton>
+          <Typography sx={{ ml: 2, flex: 1 }} variant="h6" component="div">
+            {i18n.t('subscriptionsButtonLabel')}
+          </Typography>
+        </Toolbar>
+      </AppBar>
+      <Box sx={{
+        backgroundColor: (theme) => theme.palette.background.default,
+        minHeight: (theme) => `calc(99vh - ${theme.mixins.toolbar.minHeight}px)`
+      }}>
+        <Container sx={{ my: 4 }}>
+          <Grid container spacing={2}>
+            <Grid item xs={12}>
+              <Paper
+                elevation={4}
+                sx={{
+                  p: 2,
+                  display: 'flex',
+                  flexDirection: 'column',
+                }}
+              >
+                <Grid container gap={1.5}>
+                  <Grid item xs={12}>
+                    <Alert severity="info">
+                      {i18n.t('subscriptionsInfo')}
+                    </Alert>
+                    <Alert severity="warning" sx={{ mt: 1 }}>
+                      {i18n.t('livestreamExperimentalWarning')}
+                    </Alert>
+                  </Grid>
+                  <Grid item xs={12} mt={1}>
+                    <TextField
+                      multiline
+                      fullWidth
+                      label={i18n.t('subscriptionsURLInput')}
+                      variant="outlined"
+                      placeholder="https://www.youtube.com/@SomeChannelThatExists/videos"
+                      onChange={(e) => setSubscriptionURL(e.target.value)}
+                    />
+                  </Grid>
+                  <Grid item xs={8} mt={-2}>
+                    <ExtraDownloadOptions />
+                  </Grid>
+                  <Grid item xs={3.871}>
+                    <TextField
+                      multiline
+                      fullWidth
+                      label={i18n.t('cronExpressionLabel')}
+                      variant="outlined"
+                      placeholder="*/5 * * * *"
+                      onChange={(e) => setSubscriptionCron(e.target.value)}
+                    />
+                  </Grid>
+                  <Grid item xs={12}>
+                    <Button
+                      sx={{ mt: 2 }}
+                      variant="contained"
+                      disabled={subscriptionURL === ''}
+                      onClick={() => startTransition(() => submit({
+                        url: subscriptionURL,
+                        params: customArgs,
+                        cron_expression: subscriptionCron
+                      }))}
+                    >
+                      {i18n.t('startButton')}
+                    </Button>
+                  </Grid>
+                </Grid>
+              </Paper>
+            </Grid>
+          </Grid>
+        </Container>
+      </Box>
+    </Dialog>
+  )
+}
+
+export default SubscriptionsDialog

frontend/src/components/subscriptions/SubscriptionsEditDialog.tsx

@@ -0,0 +1,162 @@
+import CloseIcon from '@mui/icons-material/Close'
+import {
+  Alert,
+  AppBar,
+  Box,
+  Button,
+  Container,
+  Dialog,
+  Grid,
+  IconButton,
+  Paper,
+  Slide,
+  TextField,
+  Toolbar,
+  Typography
+} from '@mui/material'
+import { TransitionProps } from '@mui/material/transitions'
+import { matchW } from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import { useAtomValue } from 'jotai'
+import { forwardRef, startTransition, useState } from 'react'
+import { customArgsState } from '../../atoms/downloadTemplate'
+import { serverURL } from '../../atoms/settings'
+import { useToast } from '../../hooks/toast'
+import { useI18n } from '../../hooks/useI18n'
+import { ffetch } from '../../lib/httpClient'
+import { Subscription } from '../../services/subscriptions'
+import ExtraDownloadOptions from '../ExtraDownloadOptions'
+
+type Props = {
+  subscription: Subscription | undefined
+  onClose: () => void
+}
+
+const Transition = forwardRef(function Transition(
+  props: TransitionProps & {
+    children: React.ReactElement
+  },
+  ref: React.Ref<unknown>,
+) {
+  return <Slide direction="up" ref={ref} {...props} />
+})
+
+const SubscriptionsEditDialog: React.FC<Props> = ({ subscription, onClose }) => {
+  const [subscriptionURL, setSubscriptionURL] = useState('')
+  const [subscriptionCron, setSubscriptionCron] = useState('')
+
+  const customArgs = useAtomValue(customArgsState)
+
+  const { i18n } = useI18n()
+  const { pushMessage } = useToast()
+
+  const baseURL = useAtomValue(serverURL)
+
+  const editSubscription = async (sub: Subscription) => {
+    const task = ffetch<void>(`${baseURL}/subscriptions`, {
+      method: 'PATCH',
+      body: JSON.stringify(sub)
+    })
+    const either = await task()
+
+    pipe(
+      either,
+      matchW(
+        (l) => pushMessage(l, 'error'),
+        (_) => onClose()
+      )
+    )
+  }
+
+  return (
+    <Dialog
+      fullScreen
+      open={!!subscription}
+      TransitionComponent={Transition}
+    >
+      <AppBar sx={{ position: 'relative' }}>
+        <Toolbar>
+          <IconButton
+            edge="start"
+            color="inherit"
+            onClick={() => onClose()}
+            aria-label="close"
+          >
+            <CloseIcon />
+          </IconButton>
+          <Typography sx={{ ml: 2, flex: 1 }} variant="h6" component="div">
+            {i18n.t('subscriptionsButtonLabel')}
+          </Typography>
+        </Toolbar>
+      </AppBar>
+      <Box sx={{
+        backgroundColor: (theme) => theme.palette.background.default,
+        minHeight: (theme) => `calc(99vh - ${theme.mixins.toolbar.minHeight}px)`
+      }}>
+        <Container sx={{ my: 4 }}>
+          <Grid container spacing={2}>
+            <Grid item xs={12}>
+              <Paper
+                elevation={4}
+                sx={{
+                  p: 2,
+                  display: 'flex',
+                  flexDirection: 'column',
+                }}
+              >
+                <Grid container gap={1.5}>
+                  <Grid item xs={12}>
+                    <Alert severity="info">
+                      Editing {subscription?.url}
+                    </Alert>
+                  </Grid>
+                  <Grid item xs={12} mt={1}>
+                    <TextField
+                      multiline
+                      fullWidth
+                      label={i18n.t('subscriptionsURLInput')}
+                      variant="outlined"
+                      defaultValue={subscription?.url}
+                      placeholder="https://www.youtube.com/@SomeChannelThatExists/videos"
+                      onChange={(e) => setSubscriptionURL(e.target.value)}
+                    />
+                  </Grid>
+                  <Grid item xs={8} mt={-2}>
+                    <ExtraDownloadOptions />
+                  </Grid>
+                  <Grid item xs={3.871}>
+                    <TextField
+                      multiline
+                      fullWidth
+                      label={i18n.t('cronExpressionLabel')}
+                      variant="outlined"
+                      placeholder="*/5 * * * *"
+                      defaultValue={subscription?.cron_expression}
+                      onChange={(e) => setSubscriptionCron(e.target.value)}
+                    />
+                  </Grid>
+                  <Grid item xs={12}>
+                    <Button
+                      sx={{ mt: 2 }}
+                      variant="contained"
+                      onClick={() => startTransition(async () => await editSubscription({
+                        id: subscription?.id ?? '',
+                        url: subscriptionURL || subscription?.url!,
+                        params: customArgs || subscription?.params!,
+                        cron_expression: subscriptionCron || subscription?.cron_expression!
+                      }))}
+                    >
+                      {i18n.t('editButtonLabel')}
+                    </Button>
+                  </Grid>
+                </Grid>
+              </Paper>
+            </Grid>
+          </Grid>
+        </Container>
+      </Box>
+    </Dialog>
+  )
+}
+
+export default SubscriptionsEditDialog

frontend/src/components/subscriptions/SubscriptionsSpeedDial.tsx

@@ -0,0 +1,27 @@
+import AddCircleIcon from '@mui/icons-material/AddCircle'
+import { SpeedDial, SpeedDialAction, SpeedDialIcon } from '@mui/material'
+import { useI18n } from '../../hooks/useI18n'
+
+type Props = {
+  onOpen: () => void
+}
+
+const SubscriptionsSpeedDial: React.FC<Props> = ({ onOpen }) => {
+  const { i18n } = useI18n()
+
+  return (
+    <SpeedDial
+      ariaLabel="Subscriptions speed dial"
+      sx={{ position: 'absolute', bottom: 64, right: 24 }}
+      icon={<SpeedDialIcon />}
+    >
+      <SpeedDialAction
+        icon={<AddCircleIcon />}
+        tooltipTitle={i18n.t('newSubscriptionButton')}
+        onClick={onOpen}
+      />
+    </SpeedDial>
+  )
+}
+
+export default SubscriptionsSpeedDial

frontend/src/components/twitch/TwitchDialog.tsx

@@ -0,0 +1,140 @@
+import CloseIcon from '@mui/icons-material/Close'
+import {
+  Alert,
+  AppBar,
+  Box,
+  Button,
+  Container,
+  Dialog,
+  Grid,
+  IconButton,
+  Paper,
+  Slide,
+  TextField,
+  Toolbar,
+  Typography
+} from '@mui/material'
+import { TransitionProps } from '@mui/material/transitions'
+import { matchW } from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import { useAtomValue } from 'jotai'
+import { forwardRef, startTransition, useState } from 'react'
+import { serverURL } from '../../atoms/settings'
+import { useToast } from '../../hooks/toast'
+import { useI18n } from '../../hooks/useI18n'
+import { ffetch } from '../../lib/httpClient'
+
+type Props = {
+  open: boolean
+  onClose: () => void
+}
+
+const Transition = forwardRef(function Transition(
+  props: TransitionProps & {
+    children: React.ReactElement
+  },
+  ref: React.Ref<unknown>,
+) {
+  return <Slide direction="up" ref={ref} {...props} />
+})
+
+const TwitchDialog: React.FC<Props> = ({ open, onClose }) => {
+  const [channelURL, setChannelURL] = useState('')
+
+  const { i18n } = useI18n()
+  const { pushMessage } = useToast()
+
+  const baseURL = useAtomValue(serverURL)
+
+  const submit = async (channelURL: string) => {
+    const task = ffetch<void>(`${baseURL}/twitch/user`, {
+      method: 'POST',
+      body: JSON.stringify({
+        user: channelURL.split('/').at(-1)
+      })
+    })
+    const either = await task()
+
+    pipe(
+      either,
+      matchW(
+        (l) => pushMessage(l, 'error'),
+        (_) => onClose()
+      )
+    )
+  }
+
+  return (
+    <Dialog
+      fullScreen
+      open={open}
+      onClose={onClose}
+      TransitionComponent={Transition}
+    >
+      <AppBar sx={{ position: 'relative' }}>
+        <Toolbar>
+          <IconButton
+            edge="start"
+            color="inherit"
+            onClick={onClose}
+            aria-label="close"
+          >
+            <CloseIcon />
+          </IconButton>
+          <Typography sx={{ ml: 2, flex: 1 }} variant="h6" component="div">
+            {i18n.t('subscriptionsButtonLabel')}
+          </Typography>
+        </Toolbar>
+      </AppBar>
+      <Box sx={{
+        backgroundColor: (theme) => theme.palette.background.default,
+        minHeight: (theme) => `calc(99vh - ${theme.mixins.toolbar.minHeight}px)`
+      }}>
+        <Container sx={{ my: 4 }}>
+          <Grid container spacing={2}>
+            <Grid item xs={12}>
+              <Paper
+                elevation={4}
+                sx={{
+                  p: 2,
+                  display: 'flex',
+                  flexDirection: 'column',
+                }}
+              >
+                <Grid container gap={1.5}>
+                  <Grid item xs={12}>
+                    <Alert severity="info">
+                      {i18n.t('twitchIntegrationInfo')}
+                    </Alert>
+                  </Grid>
+                  <Grid item xs={12} mt={1}>
+                    <TextField
+                      multiline
+                      fullWidth
+                      label={i18n.t('subscriptionsURLInput')}
+                      variant="outlined"
+                      placeholder="https://www.twitch.tv/a_twitch_user_that_exists"
+                      onChange={(e) => setChannelURL(e.target.value)}
+                    />
+                  </Grid>
+                  <Grid item xs={12}>
+                    <Button
+                      sx={{ mt: 2 }}
+                      variant="contained"
+                      disabled={channelURL === ''}
+                      onClick={() => startTransition(() => submit(channelURL))}
+                    >
+                      {i18n.t('startButton')}
+                    </Button>
+                  </Grid>
+                </Grid>
+              </Paper>
+            </Grid>
+          </Grid>
+        </Container>
+      </Box>
+    </Dialog>
+  )
+}
+
+export default TwitchDialog

frontend/src/hooks/useFetch.ts

@@ -0,0 +1,35 @@
+import { pipe } from 'fp-ts/lib/function'
+import { matchW } from 'fp-ts/lib/TaskEither'
+import { useAtomValue } from 'jotai'
+import { useEffect, useState } from 'react'
+import { serverURL } from '../atoms/settings'
+import { ffetch } from '../lib/httpClient'
+import { useToast } from './toast'
+
+const useFetch = <R>(resource: string) => {
+  const base = useAtomValue(serverURL)
+
+  const { pushMessage } = useToast()
+
+  const [data, setData] = useState<R>()
+  const [error, setError] = useState<string>()
+
+  const fetcher = () => pipe(
+    ffetch<R>(`${base}${resource}`),
+    matchW(
+      (l) => {
+        setError(l)
+        pushMessage(l, 'error')
+      },
+      (r) => setData(r)
+    )
+  )()
+
+  useEffect(() => {
+    fetcher()
+  }, [])
+
+  return { data, error, fetcher }
+}
+
+export default useFetch

frontend/src/lib/httpClient.ts

@@ -1,12 +1,9 @@
 import { tryCatch } from 'fp-ts/TaskEither'
+import * as J from 'fp-ts/Json'
+import * as E from 'fp-ts/Either'
+import { pipe } from 'fp-ts/lib/function'
 
-export const ffetch = <T>(url: string, opt?: RequestInit) => tryCatch(
-  () => fetcher<T>(url, opt),
-  (e) => `error while fetching: ${e}`
-)
-
-
-const fetcher = async <T>(url: string, opt?: RequestInit) => {
+async function fetcher(url: string, opt?: RequestInit, controller?: AbortController): Promise<string> {
   const jwt = localStorage.getItem('token')
 
   if (opt && !opt.headers) {
@@ -20,11 +17,27 @@ const fetcher = async <T>(url: string, opt?: RequestInit) => {
     headers: {
       ...opt?.headers,
       'X-Authentication': jwt ?? ''
-    }
+    },
+    signal: controller?.signal
   })
 
   if (!res.ok) {
     throw await res.text()
   }
-  return res.json() as T
-}
+
+
+
+  return res.text()
+}
+
+export const ffetch = <T>(url: string, opt?: RequestInit, controller?: AbortController) => tryCatch(
+  async () => pipe(
+    await fetcher(url, opt, controller),
+    J.parse,
+    E.match(
+      (l) => l as T,
+      (r) => r as T
+    )
+  ),
+  (e) => `error while fetching: ${e}`
+)

frontend/src/lib/i18n.ts

@@ -38,7 +38,7 @@ export default class Translator {
   t(key: string): string {
     if (this.current) {
       //@ts-ignore
-      return this.current[key] ?? fallback.keys[key]
+      return this.current[key] ?? fallback.keys[key] ?? 'caption not defined'
     }
     return 'caption not defined'
   }

frontend/src/lib/rpcClient.ts

@@ -41,11 +41,13 @@ export class RPCClient {
     })
   }
 
-  private argsSanitizer(args: string) {
+  private argsSanitizer(args: string): string[] {
+    const splitOnlyWhitespaces = /[^\s"']+|"([^"]*)"|'([^']*)'/gm
+
     return args
-      .split(' ')
-      .map(a => a.trim().replaceAll("'", '').replaceAll('"', ''))
-      .filter(Boolean)
+      .match(splitOnlyWhitespaces)
+      ?.map(a => a.trim())
+      .filter(Boolean) ?? []
   }
 
   private async sendHTTP<T>(req: RPCRequest) {
@@ -198,4 +200,11 @@ export class RPCClient {
       params: []
     })
   }
+
+  public clearCompleted() {
+    return this.sendHTTP({
+      method: 'Service.ClearCompleted',
+      params: []
+    })
+  }
 }

frontend/src/router.tsx

@@ -6,10 +6,12 @@ import Terminal from './views/Terminal'
 
 const Home = lazy(() => import('./views/Home'))
 const Login = lazy(() => import('./views/Login'))
+const Twitch = lazy(() => import('./views/Twitch'))
 const Archive = lazy(() => import('./views/Archive'))
 const Settings = lazy(() => import('./views/Settings'))
 const LiveStream = lazy(() => import('./views/Livestream'))
 const Filebrowser = lazy(() => import('./views/Filebrowser'))
+const Subscriptions = lazy(() => import('./views/Subscriptions'))
 
 const ErrorBoundary = lazy(() => import('./components/ErrorBoundary'))
 
@@ -73,6 +75,19 @@ export const router = createHashRouter([
           </Suspense >
         )
       },
+      {
+        path: '/subscriptions',
+        element: (
+          <Suspense fallback={<CircularProgress />}>
+            <Subscriptions />
+          </Suspense >
+        ),
+        errorElement: (
+          <Suspense fallback={<CircularProgress />}>
+            <ErrorBoundary />
+          </Suspense >
+        )
+      },
       {
         path: '/login',
         element: (
@@ -97,6 +112,14 @@ export const router = createHashRouter([
           </Suspense >
         )
       },
+      {
+        path: '/twitch',
+        element: (
+          <Suspense fallback={<CircularProgress />}>
+            <Twitch />
+          </Suspense >
+        )
+      },
     ]
   },
 ])

frontend/src/services/subscriptions.ts

@@ -0,0 +1,37 @@
+// import { PaginatedResponse } from '../types'
+
+export type Subscription = {
+  id: string
+  url: string
+  params: string
+  cron_expression: string
+}
+
+// class SubscriptionService {
+//   private _baseURL: string = ''
+
+//   public set baseURL(v: string) {
+//     this._baseURL = v
+//   }
+
+//   public async delete(id: string): Promise<void> {
+
+//   }
+
+//   public async listPaginated(start: number, limit: number = 50): Promise<PaginatedResponse<Subscription[]>> {
+//     const res = await fetch(`${this._baseURL}/subscriptions?id=${start}&limit=${limit}`)
+//     const data: PaginatedResponse<Subscription[]> = await res.json()
+
+//     return data
+//   }
+
+//   public async submit(sub: Subscription): Promise<void> {
+
+//   }
+
+//   public async edit(sub: Subscription): Promise<void> {
+
+//   }
+// }
+
+// export default SubscriptionService

frontend/src/types/index.ts

@@ -13,6 +13,7 @@ export type RPCMethods =
   | "Service.ProgressLivestream"
   | "Service.KillLivestream"
   | "Service.KillAllLivestream"
+  | "Service.ClearCompleted"
 
 export type RPCRequest = {
   method: RPCMethods

frontend/src/views/Settings.tsx

@@ -36,6 +36,7 @@ import {
   appTitleState,
   enableCustomArgsState,
   fileRenamingState,
+  autoFileExtensionState,
   formatSelectionState,
   languageState,
   languages,
@@ -61,6 +62,7 @@ export default function Settings() {
   const [formatSelection, setFormatSelection] = useAtom(formatSelectionState)
   const [pathOverriding, setPathOverriding] = useAtom(pathOverridingState)
   const [fileRenaming, setFileRenaming] = useAtom(fileRenamingState)
+  const [autoFileExtension, setAutoFileExtension] = useAtom(autoFileExtensionState)
   const [enableArgs, setEnableArgs] = useAtom(enableCustomArgsState)
 
   const [serverAddr, setServerAddr] = useAtom(serverAddressState)
@@ -343,12 +345,30 @@ export default function Settings() {
                 <Switch
                   defaultChecked={fileRenaming}
                   onChange={() => {
+                    if (fileRenaming) {
+                      setAutoFileExtension(false)
+                    }
                     setFileRenaming(state => !state)
                   }}
                 />
               }
               label={i18n.t('filenameOverrideOption')}
             />
+            {
+              <FormControlLabel
+                control={
+                  <Switch
+                    disabled={!fileRenaming}
+                    checked={fileRenaming ? autoFileExtension : false}
+                    defaultChecked={autoFileExtension}
+                    onChange={() => {
+                      setAutoFileExtension(state => !state)
+                    }}
+                  />
+                }
+                label={i18n.t('autoFileExtensionOption')}
+              /> 
+            }
             <FormControlLabel
               control={
                 <Switch

frontend/src/views/Subscriptions.tsx

@@ -0,0 +1,157 @@
+import DeleteIcon from '@mui/icons-material/Delete'
+import EditIcon from '@mui/icons-material/Edit'
+import {
+  Box,
+  Button,
+  Container,
+  Paper,
+  Table, TableBody, TableCell, TableContainer,
+  TableHead, TablePagination, TableRow
+} from '@mui/material'
+import { matchW } from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import { useAtomValue } from 'jotai'
+import { useState, useTransition } from 'react'
+import { serverURL } from '../atoms/settings'
+import LoadingBackdrop from '../components/LoadingBackdrop'
+import NoSubscriptions from '../components/subscriptions/NoSubscriptions'
+import SubscriptionsDialog from '../components/subscriptions/SubscriptionsDialog'
+import SubscriptionsEditDialog from '../components/subscriptions/SubscriptionsEditDialog'
+import SubscriptionsSpeedDial from '../components/subscriptions/SubscriptionsSpeedDial'
+import { useToast } from '../hooks/toast'
+import useFetch from '../hooks/useFetch'
+import { useI18n } from '../hooks/useI18n'
+import { ffetch } from '../lib/httpClient'
+import { Subscription } from '../services/subscriptions'
+import { PaginatedResponse } from '../types'
+
+const SubscriptionsView: React.FC = () => {
+  const { i18n } = useI18n()
+  const { pushMessage } = useToast()
+
+  const baseURL = useAtomValue(serverURL)
+
+  const [selectedSubscription, setSelectedSubscription] = useState<Subscription>()
+  const [openDialog, setOpenDialog] = useState(false)
+
+  const [startId, setStartId] = useState(0)
+  const [limit, setLimit] = useState(9)
+  const [page, setPage] = useState(0)
+
+  const { data: subs, fetcher: refecth } = useFetch<PaginatedResponse<Subscription[]>>(
+    `/subscriptions?id=${startId}&limit=${limit}`
+  )
+
+  const [isPending, startTransition] = useTransition()
+
+  const deleteSubscription = async (id: string) => {
+    const task = ffetch<void>(`${baseURL}/subscriptions/${id}`, {
+      method: 'DELETE',
+    })
+    const either = await task()
+
+    pipe(
+      either,
+      matchW(
+        (l) => pushMessage(l, 'error'),
+        () => refecth()
+      )
+    )
+  }
+
+  return (
+    <>
+      <LoadingBackdrop isLoading={!subs || isPending} />
+
+      <SubscriptionsSpeedDial onOpen={() => setOpenDialog(s => !s)} />
+
+      <SubscriptionsEditDialog
+        subscription={selectedSubscription}
+        onClose={() => {
+          setSelectedSubscription(undefined)
+          refecth()
+        }}
+      />
+      <SubscriptionsDialog open={openDialog} onClose={() => {
+        setOpenDialog(s => !s)
+        refecth()
+      }} />
+
+      {!subs || subs.data.length === 0 ?
+        <NoSubscriptions /> :
+        <Container maxWidth="xl" sx={{ mt: 4, mb: 8 }}>
+          <Paper sx={{
+            p: 2.5,
+            display: 'flex',
+            flexDirection: 'column',
+            minHeight: '80vh',
+          }}>
+            <TableContainer component={Box}>
+              <Table sx={{ minWidth: '100%' }}>
+                <TableHead>
+                  <TableRow>
+                    <TableCell align="left">URL</TableCell>
+                    <TableCell align="right">Params</TableCell>
+                    <TableCell align="right">{i18n.t('cronExpressionLabel')}</TableCell>
+                    <TableCell align="center">Actions</TableCell>
+                  </TableRow>
+                </TableHead>
+                <TableBody sx={{ mb: 'auto' }}>
+                  {subs.data.map(x => (
+                    <TableRow
+                      key={x.id}
+                      sx={{ '&:last-child td, &:last-child th': { border: 0 } }}
+                    >
+                      <TableCell>{x.url}</TableCell>
+                      <TableCell align='right'>
+                        {x.params}
+                      </TableCell>
+                      <TableCell align='right'>
+                        {x.cron_expression}
+                      </TableCell>
+                      <TableCell align='center'>
+                        <Button
+                          variant='contained'
+                          size='small'
+                          sx={{ mr: 0.5 }}
+                          onClick={() => setSelectedSubscription(x)}
+                        >
+                          <EditIcon />
+                        </Button>
+                        <Button
+                          variant='contained'
+                          size='small'
+                          onClick={() => startTransition(async () => await deleteSubscription(x.id))}
+                        >
+                          <DeleteIcon />
+                        </Button>
+                      </TableCell>
+                    </TableRow>
+                  ))}
+                </TableBody>
+              </Table>
+            </TableContainer>
+            <TablePagination
+              component="div"
+              count={-1}
+              page={page}
+              onPageChange={(_, p) => {
+                if (p < page) {
+                  setPage(s => (s - 1 <= 0 ? 0 : s - 1))
+                  setStartId(subs.first)
+                  return
+                }
+                setPage(s => s + 1)
+                setStartId(subs.next)
+              }}
+              rowsPerPage={limit}
+              rowsPerPageOptions={[9, 10, 25, 50, 100]}
+              onRowsPerPageChange={(e) => { setLimit(parseInt(e.target.value)) }}
+            />
+          </Paper>
+        </Container>}
+    </>
+  )
+}
+
+export default SubscriptionsView

frontend/src/views/Twitch.tsx

@@ -0,0 +1,77 @@
+import {
+  Chip,
+  Container,
+  Paper
+} from '@mui/material'
+import { matchW } from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import { useAtomValue } from 'jotai'
+import { useState, useTransition } from 'react'
+import { serverURL } from '../atoms/settings'
+import LoadingBackdrop from '../components/LoadingBackdrop'
+import NoSubscriptions from '../components/subscriptions/NoSubscriptions'
+import SubscriptionsSpeedDial from '../components/subscriptions/SubscriptionsSpeedDial'
+import TwitchDialog from '../components/twitch/TwitchDialog'
+import { useToast } from '../hooks/toast'
+import useFetch from '../hooks/useFetch'
+import { ffetch } from '../lib/httpClient'
+
+const TwitchView: React.FC = () => {
+  const { pushMessage } = useToast()
+
+  const baseURL = useAtomValue(serverURL)
+
+  const [openDialog, setOpenDialog] = useState(false)
+
+  const { data: users, fetcher: refetch } = useFetch<Array<string>>('/twitch/users')
+
+  const [isPending, startTransition] = useTransition()
+
+  const deleteUser = async (user: string) => {
+    const task = ffetch<void>(`${baseURL}/twitch/user/${user}`, {
+      method: 'DELETE',
+    })
+    const either = await task()
+
+    pipe(
+      either,
+      matchW(
+        (l) => pushMessage(l, 'error'),
+        () => refetch()
+      )
+    )
+  }
+
+  return (
+    <>
+      <LoadingBackdrop isLoading={!users || isPending} />
+
+      <SubscriptionsSpeedDial onOpen={() => setOpenDialog(s => !s)} />
+
+      <TwitchDialog open={openDialog} onClose={() => {
+        setOpenDialog(s => !s)
+        refetch()
+      }} />
+
+      {
+        !users || users.length === 0 ?
+          <NoSubscriptions /> :
+          <Container maxWidth="xl" sx={{ mt: 4, mb: 8 }}>
+            <Paper sx={{
+              p: 2.5,
+              minHeight: '80vh',
+            }}>
+              {users.map(user => (
+                <Chip
+                  label={user}
+                  onDelete={() => startTransition(async () => await deleteUser(user))}
+                />
+              ))}
+            </Paper>
+          </Container>
+      }
+    </>
+  )
+}
+
+export default TwitchView

go.mod

@@ -1,37 +1,35 @@
 module github.com/marcopiovanello/yt-dlp-web-ui/v3
 
-go 1.23
+go 1.24
 
 require (
-	github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef
-	github.com/coreos/go-oidc/v3 v3.11.0
-	github.com/go-chi/chi/v5 v5.1.0
-	github.com/go-chi/cors v1.2.1
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/coreos/go-oidc/v3 v3.15.0
+	github.com/go-chi/chi/v5 v5.2.3
+	github.com/go-chi/cors v1.2.2
+	github.com/golang-jwt/jwt/v5 v5.3.0
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.3
-	golang.org/x/oauth2 v0.23.0
-	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.25.0
-	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.33.1
+	github.com/robfig/cron/v3 v3.0.1
+	github.com/spf13/viper v1.20.1
+	go.etcd.io/bbolt v1.4.3
+	golang.org/x/crypto v0.41.0
+	golang.org/x/oauth2 v0.30.0
+	golang.org/x/sys v0.35.0
 )
 
 require (
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	golang.org/x/crypto v0.27.0 // indirect
-	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a // indirect
-	modernc.org/libc v1.61.0 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.8.0 // indirect
-	modernc.org/strutil v1.2.0 // indirect
-	modernc.org/token v1.1.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,87 +1,76 @@
-github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
-github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
-github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI=
-github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
+github.com/coreos/go-oidc/v3 v3.15.0 h1:R6Oz8Z4bqWR7VFQ+sPSvZPQv4x8M+sJkDO5ojgwlyAg=
+github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
-github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
-github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
-github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
-github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
-github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
+github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
+github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
+github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
+github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
+github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
-github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
-github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
-github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
-golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
-golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
-golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
-golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
-golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
-golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
-golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE=
-golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
+github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
+github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
+github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
+github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
+github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
+go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
+golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
+golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
+golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
-modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
-modernc.org/ccgo/v4 v4.21.0 h1:kKPI3dF7RIag8YcToh5ZwDcVMIv6VGa0ED5cvh0LMW4=
-modernc.org/ccgo/v4 v4.21.0/go.mod h1:h6kt6H/A2+ew/3MW/p6KEoQmrq/i3pr0J/SiwiaF/g0=
-modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
-modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
-modernc.org/gc/v2 v2.5.0 h1:bJ9ChznK1L1mUtAQtxi0wi5AtAs5jQuw4PrPHO5pb6M=
-modernc.org/gc/v2 v2.5.0/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
-modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a h1:CfbpOLEo2IwNzJdMvE8aiRbPMxoTpgAJeyePh0SmO8M=
-modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.61.0 h1:eGFcvWpqlnoGwzZeZe3PWJkkKbM/3SUGyk1DVZQ0TpE=
-modernc.org/libc v1.61.0/go.mod h1:DvxVX89wtGTu+r72MLGhygpfi3aUGgZRdAYGCAVVud0=
-modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
-modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
-modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
-modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
-modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
-modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
-modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
-modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
-modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
-modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

main.go

@@ -1,117 +1,102 @@
 package main
 
 import (
+	"context"
 	"embed"
 	"flag"
 	"io/fs"
-	"log"
+	"log/slog"
 	"os"
+	"os/signal"
 	"runtime"
+	"syscall"
 
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/cli"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/openid"
+
+	"github.com/spf13/viper"
 )
 
-var (
-	host              string
-	port              int
-	queueSize         int
-	configFile        string
-	downloadPath      string
-	downloaderPath    string
-	sessionFilePath   string
-	localDatabasePath string
-	frontendPath      string
+//go:embed frontend/dist/index.html
+//go:embed frontend/dist/assets/*
+var frontend embed.FS
 
-	requireAuth bool
-	username    string
-	password    string
-
-	userFromEnv = os.Getenv("USERNAME")
-	passFromEnv = os.Getenv("PASSWORD")
-
-	logFile           string
-	enableFileLogging bool
-
-	//go:embed frontend/dist/index.html
-	//go:embed frontend/dist/assets/*
-	frontend embed.FS
-
-	//go:embed openapi/*
-	swagger embed.FS
-)
-
-func init() {
-	flag.StringVar(&host, "host", "0.0.0.0", "Host where server will listen at")
-	flag.IntVar(&port, "port", 3033, "Port where server will listen at")
-	flag.IntVar(&queueSize, "qs", 2, "Queue size (concurrent downloads)")
-
-	flag.StringVar(&configFile, "conf", "./config.yml", "Config file path")
-	flag.StringVar(&downloadPath, "out", ".", "Where files will be saved")
-	flag.StringVar(&downloaderPath, "driver", "yt-dlp", "yt-dlp executable path")
-	flag.StringVar(&sessionFilePath, "session", ".", "session file path")
-	flag.StringVar(&localDatabasePath, "db", "local.db", "local database path")
-	flag.StringVar(&frontendPath, "web", "", "frontend web resources path")
-
-	flag.BoolVar(&enableFileLogging, "fl", false, "enable outputting logs to a file")
-	flag.StringVar(&logFile, "lf", "yt-dlp-webui.log", "set log file location")
-
-	flag.BoolVar(&requireAuth, "auth", false, "Enable RPC authentication")
-	flag.StringVar(&username, "user", userFromEnv, "Username required for auth")
-	flag.StringVar(&password, "pass", passFromEnv, "Password required for auth")
-
-	flag.Parse()
-}
+//go:embed openapi/*
+var swagger embed.FS
 
 func main() {
-	frontend, err := fs.Sub(frontend, "frontend/dist")
-	if err != nil {
-		log.Fatalln(err)
+	// Parse optional config path from flag
+	var configFile string
+	flag.StringVar(&configFile, "conf", "./config.yml", "Config file path")
+	flag.Parse()
+
+	v := viper.New()
+	v.SetConfigFile(configFile)
+	v.SetConfigType("yaml")
+
+	// Defaults
+	v.SetDefault("server.host", "0.0.0.0")
+	v.SetDefault("server.port", 3033)
+	v.SetDefault("server.queue_size", 2)
+	v.SetDefault("paths.download_path", ".")
+	v.SetDefault("paths.downloader_path", "yt-dlp")
+	v.SetDefault("paths.local_database_path", ".")
+	v.SetDefault("logging.log_path", "yt-dlp-webui.log")
+	v.SetDefault("logging.enable_file_logging", false)
+	v.SetDefault("authentication.require_auth", false)
+
+	// Env binding
+	v.SetEnvPrefix("APP")
+	v.AutomaticEnv()
+
+	// Load YAML file if exists
+	if err := v.ReadInConfig(); err != nil {
+		slog.Debug("using defaults")
 	}
 
-	if frontendPath != "" {
-		frontend = os.DirFS(frontendPath)
+	cfg := config.Instance()
+	if err := v.Unmarshal(&cfg); err != nil {
+		slog.Error("failed to load config", "error", err)
 	}
 
-	c := config.Instance()
-
-	{
-		// init the config struct with the values from flags
-		// TODO: find an alternative way to populate the config struct from flags or config file
-		c.Host = host
-		c.Port = port
-
-		c.QueueSize = queueSize
-
-		c.DownloadPath = downloadPath
-		c.DownloaderPath = downloaderPath
-		c.SessionFilePath = sessionFilePath
-		c.LocalDatabasePath = localDatabasePath
-
-		c.LogPath = logFile
-		c.EnableFileLogging = enableFileLogging
-
-		c.RequireAuth = requireAuth
-		c.Username = username
-		c.Password = password
+	if cfg.Server.QueueSize <= 0 || runtime.NumCPU() <= 2 {
+		cfg.Server.QueueSize = 2
 	}
 
-	// limit concurrent downloads for systems with 2 or less logical cores
-	if runtime.NumCPU() <= 2 {
-		c.QueueSize = 1
-	}
-
-	// if config file is found it will be merged with the current config struct
-	if err := c.LoadFile(configFile); err != nil {
-		log.Println(cli.BgRed, "config", cli.Reset, err)
+	// 6. Frontend FS
+	var appFS fs.FS
+	if fp := v.GetString("frontend_path"); fp != "" {
+		appFS = os.DirFS(fp)
+	} else {
+		sub, err := fs.Sub(frontend, "frontend/dist")
+		if err != nil {
+			slog.Error("failed to load embedded frontend", "error", err)
+			os.Exit(1)
+		}
+		appFS = sub
 	}
 
+	// Configure OpenID if needed
 	openid.Configure()
 
-	server.RunBlocking(&server.RunConfig{
-		App:     frontend,
+	// Graceful shutdown
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	slog.Info("starting server",
+		"host", cfg.Server.Host,
+		"port", cfg.Server.Port,
+		"queue_size", cfg.Server.QueueSize,
+	)
+
+	if err := server.Run(ctx, &server.RunConfig{
+		App:     appFS,
 		Swagger: swagger,
-	})
+	}); err != nil {
+		slog.Error("server stopped with error", "error", err)
+		os.Exit(1)
+	}
+
+	slog.Info("server exited cleanly")
 }

server/archive/rest/handler.go

@@ -146,10 +146,10 @@ func (h *Handler) GetCursor() http.HandlerFunc {
 // ApplyRouter implements domain.RestHandler.
 func (h *Handler) ApplyRouter() func(chi.Router) {
 	return func(r chi.Router) {
-		if config.Instance().RequireAuth {
+		if config.Instance().Authentication.RequireAuth {
 			r.Use(middlewares.Authenticated)
 		}
-		if config.Instance().UseOpenId {
+		if config.Instance().OpenId.UseOpenId {
 			r.Use(openid.Middleware)
 		}
 

server/archive/utils.go

@@ -0,0 +1,58 @@
+package archive
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+)
+
+// Perform a search on the archive.txt file an determines if a download
+// has already be done.
+func DownloadExists(ctx context.Context, url string) (bool, error) {
+	cmd := exec.CommandContext(
+		ctx,
+		config.Instance().Paths.DownloaderPath,
+		"--print",
+		"%(extractor)s %(id)s",
+		url,
+	)
+	stdout, err := cmd.Output()
+	if err != nil {
+		return false, err
+	}
+
+	extractorAndURL := bytes.Trim(stdout, "\n")
+
+	fd, err := os.Open(filepath.Join(config.Instance().Dir(), "archive.txt"))
+	if err != nil {
+		return false, err
+	}
+	defer fd.Close()
+
+	scanner := bufio.NewScanner(fd)
+
+	// search linearly for lower memory usage...
+	// the a pre-sorted with hashed values version of the archive.txt file can be loaded in memory
+	// and perform a binary search on it.
+	for scanner.Scan() {
+		if bytes.Equal(scanner.Bytes(), extractorAndURL) {
+			return true, nil
+		}
+	}
+
+	// data, err := io.ReadAll(fd)
+	// if err != nil {
+	// 	return false, err
+	// }
+
+	// slices.BinarySearchFunc(data, extractorAndURL, func(a []byte, b []byte) int {
+	// 	return hash(a).Compare(hash(b))
+	// })
+
+	return false, nil
+}

server/archiver/archiver.go

@@ -5,15 +5,12 @@ import (
 	"database/sql"
 	"log/slog"
 
-	evbus "github.com/asaskevich/EventBus"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/archive"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
 )
 
-const QueueName = "process:archive"
-
 var (
-	eventBus       = evbus.New()
+	ch             = make(chan *Message, 1)
 	archiveService archive.Service
 )
 
@@ -25,18 +22,20 @@ func Register(db *sql.DB) {
 }
 
 func init() {
-	eventBus.Subscribe(QueueName, func(m *Message) {
-		slog.Info(
-			"archiving completed download",
-			slog.String("title", m.Title),
-			slog.String("source", m.Source),
-		)
-		archiveService.Archive(context.Background(), m)
-	})
+	go func() {
+		for m := range ch {
+			slog.Info(
+				"archiving completed download",
+				slog.String("title", m.Title),
+				slog.String("source", m.Source),
+			)
+			archiveService.Archive(context.Background(), m)
+		}
+	}()
 }
 
 func Publish(m *Message) {
 	if config.Instance().AutoArchive {
-		eventBus.Publish(QueueName, m)
+		ch <- m
 	}
 }

server/common/types.go

@@ -0,0 +1,18 @@
+package common
+
+import "time"
+
+// Used to deser the yt-dlp -J output
+type DownloadMetadata struct {
+	URL         string    `json:"url"`
+	Title       string    `json:"title"`
+	Thumbnail   string    `json:"thumbnail"`
+	Resolution  string    `json:"resolution"`
+	Size        int32     `json:"filesize_approx"`
+	VCodec      string    `json:"vcodec"`
+	ACodec      string    `json:"acodec"`
+	Extension   string    `json:"ext"`
+	OriginalURL string    `json:"original_url"`
+	FileName    string    `json:"filename"`
+	CreatedAt   time.Time `json:"created_at"`
+}

server/config/config.go

@@ -1,35 +1,64 @@
 package config
 
 import (
-	"os"
 	"path/filepath"
 	"sync"
-
-	"gopkg.in/yaml.v3"
+	"time"
 )
 
 type Config struct {
-	LogPath            string `yaml:"log_path"`
-	EnableFileLogging  bool   `yaml:"enable_file_logging"`
-	BaseURL            string `yaml:"base_url"`
-	Host               string `yaml:"host"`
-	Port               int    `yaml:"port"`
-	DownloadPath       string `yaml:"downloadPath"`
-	DownloaderPath     string `yaml:"downloaderPath"`
-	RequireAuth        bool   `yaml:"require_auth"`
-	Username           string `yaml:"username"`
-	Password           string `yaml:"password"`
-	QueueSize          int    `yaml:"queue_size"`
-	LocalDatabasePath  string `yaml:"local_database_path"`
-	SessionFilePath    string `yaml:"session_file_path"`
-	path               string // private
-	UseOpenId          bool   `yaml:"use_openid"`
-	OpenIdProviderURL  string `yaml:"openid_provider_url"`
-	OpenIdClientId     string `yaml:"openid_client_id"`
-	OpenIdClientSecret string `yaml:"openid_client_secret"`
-	OpenIdRedirectURL  string `yaml:"openid_redirect_url"`
-	FrontendPath       string `yaml:"frontend_path"`
-	AutoArchive        bool   `yaml:"auto_archive"`
+	Server         ServerConfig   `yaml:"server"`
+	Logging        LoggingConfig  `yaml:"logging"`
+	Paths          PathsConfig    `yaml:"paths"`
+	Authentication AuthConfig     `yaml:"authentication"`
+	OpenId         OpenIdConfig   `yaml:"openid"`
+	Frontend       FrontendConfig `yaml:"frontend"`
+	AutoArchive    bool           `yaml:"auto_archive"`
+	Twitch         TwitchConfig   `yaml:"twitch"`
+	path           string
+}
+
+type ServerConfig struct {
+	BaseURL   string `yaml:"base_url"`
+	Host      string `yaml:"host"`
+	Port      int    `yaml:"port"`
+	QueueSize int    `yaml:"queue_size"`
+}
+
+type LoggingConfig struct {
+	LogPath           string `yaml:"log_path"`
+	EnableFileLogging bool   `yaml:"enable_file_logging"`
+}
+
+type PathsConfig struct {
+	DownloadPath      string `yaml:"download_path"`
+	DownloaderPath    string `yaml:"downloader_path"`
+	LocalDatabasePath string `yaml:"local_database_path"`
+}
+
+type AuthConfig struct {
+	RequireAuth  bool   `yaml:"require_auth"`
+	Username     string `yaml:"username"`
+	PasswordHash string `yaml:"password"`
+}
+
+type OpenIdConfig struct {
+	UseOpenId      bool     `yaml:"use_openid"`
+	ProviderURL    string   `yaml:"openid_provider_url"`
+	ClientId       string   `yaml:"openid_client_id"`
+	ClientSecret   string   `yaml:"openid_client_secret"`
+	RedirectURL    string   `yaml:"openid_redirect_url"`
+	EmailWhitelist []string `yaml:"openid_email_whitelist"`
+}
+
+type FrontendConfig struct {
+	FrontendPath string `yaml:"frontend_path"`
+}
+
+type TwitchConfig struct {
+	ClientId      string        `yaml:"client_id"`
+	ClientSecret  string        `yaml:"client_secret"`
+	CheckInterval time.Duration `yaml:"check_interval"`
 }
 
 var (
@@ -41,27 +70,12 @@ func Instance() *Config {
 	if instance == nil {
 		instanceOnce.Do(func() {
 			instance = &Config{}
+			instance.Twitch.CheckInterval = time.Minute * 5
 		})
 	}
 	return instance
 }
 
-// Initialises the Config struct given its config file
-func (c *Config) LoadFile(filename string) error {
-	fd, err := os.Open(filename)
-	if err != nil {
-		return err
-	}
-
-	c.path = filename
-
-	if err := yaml.NewDecoder(fd).Decode(c); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // Path of the directory containing the config file
 func (c *Config) Dir() string { return filepath.Dir(c.path) }
 

server/dbutil/migrate.go

@@ -49,6 +49,18 @@ func Migrate(ctx context.Context, db *sql.DB) error {
 		return err
 	}
 
+	if _, err := db.ExecContext(
+		ctx,
+		`CREATE TABLE IF NOT EXISTS subscriptions (
+			id CHAR(36) PRIMARY KEY,
+			url VARCHAR(2048) UNIQUE NOT NULL,
+			params TEXT NOT NULL,
+			cron TEXT
+		)`,
+	); err != nil {
+		return err
+	}
+
 	if lockFileExists() {
 		return nil
 	}

server/filebrowser/handlers.go

@@ -1,4 +1,4 @@
-package handlers
+package filebrowser
 
 import (
 	"archive/zip"
@@ -19,6 +19,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
 )
 
 /*
@@ -88,7 +89,7 @@ type ListRequest struct {
 }
 
 func ListDownloaded(w http.ResponseWriter, r *http.Request) {
-	root := config.Instance().DownloadPath
+	root := config.Instance().Paths.DownloadPath
 	req := new(ListRequest)
 
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -156,7 +157,7 @@ func SendFile(w http.ResponseWriter, r *http.Request) {
 
 	filename := string(decoded)
 
-	root := config.Instance().DownloadPath
+	root := config.Instance().Paths.DownloadPath
 
 	if strings.Contains(filepath.Dir(filepath.Clean(filename)), filepath.Clean(root)) {
 		http.ServeFile(w, r, filename)
@@ -188,7 +189,7 @@ func DownloadFile(w http.ResponseWriter, r *http.Request) {
 
 	filename := string(decoded)
 
-	root := config.Instance().DownloadPath
+	root := config.Instance().Paths.DownloadPath
 
 	if strings.Contains(filepath.Dir(filepath.Clean(filename)), filepath.Clean(root)) {
 		w.Header().Add("Content-Disposition", "inline; filename=\""+filepath.Base(filename)+"\"")
@@ -207,9 +208,9 @@ func DownloadFile(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusUnauthorized)
 }
 
-func BulkDownload(mdb *internal.MemoryDB) http.HandlerFunc {
+func BulkDownload(mdb *kv.Store) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		ps := slices.DeleteFunc(*mdb.All(), func(e internal.ProcessResponse) bool {
+		ps := slices.DeleteFunc(*mdb.All(), func(e internal.ProcessSnapshot) bool {
 			return e.Progress.Status != internal.StatusCompleted
 		})
 

server/formats/parser.go

@@ -10,7 +10,7 @@ import (
 )
 
 func ParseURL(url string) (*Metadata, error) {
-	cmd := exec.Command(config.Instance().DownloaderPath, url, "-J")
+	cmd := exec.Command(config.Instance().Paths.DownloaderPath, url, "-J")
 
 	stdout, err := cmd.Output()
 	if err != nil {

server/internal/balancer.go

@@ -1,34 +0,0 @@
-package internal
-
-import (
-	"container/heap"
-)
-
-type LoadBalancer struct {
-	pool Pool
-	done chan *Worker
-}
-
-func (b *LoadBalancer) Balance(work chan Process) {
-	for {
-		select {
-		case req := <-work:
-			b.dispatch(req)
-		case w := <-b.done:
-			b.completed(w)
-		}
-	}
-}
-
-func (b *LoadBalancer) dispatch(req Process) {
-	w := heap.Pop(&b.pool).(*Worker)
-	w.requests <- req
-	w.pending++
-	heap.Push(&b.pool, w)
-}
-
-func (b *LoadBalancer) completed(w *Worker) {
-	w.pending--
-	heap.Remove(&b.pool, w.index)
-	heap.Push(&b.pool, w)
-}

server/internal/common_types.go

@@ -1,6 +1,8 @@
 package internal
 
-import "time"
+import (
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+)
 
 // Used to unmarshall yt-dlp progress
 type ProgressTemplate struct {
@@ -29,35 +31,21 @@ type DownloadProgress struct {
 	ETA        float64 `json:"eta"`
 }
 
-// Used to deser the yt-dlp -J output
-type DownloadInfo struct {
-	URL         string    `json:"url"`
-	Title       string    `json:"title"`
-	Thumbnail   string    `json:"thumbnail"`
-	Resolution  string    `json:"resolution"`
-	Size        int32     `json:"filesize_approx"`
-	VCodec      string    `json:"vcodec"`
-	ACodec      string    `json:"acodec"`
-	Extension   string    `json:"ext"`
-	OriginalURL string    `json:"original_url"`
-	FileName    string    `json:"filename"`
-	CreatedAt   time.Time `json:"created_at"`
-}
-
 // struct representing the response sent to the client
 // as JSON-RPC result field
-type ProcessResponse struct {
-	Id       string           `json:"id"`
-	Progress DownloadProgress `json:"progress"`
-	Info     DownloadInfo     `json:"info"`
-	Output   DownloadOutput   `json:"output"`
-	Params   []string         `json:"params"`
+type ProcessSnapshot struct {
+	Id             string                  `json:"id"`
+	Progress       DownloadProgress        `json:"progress"`
+	Info           common.DownloadMetadata `json:"info"`
+	Output         DownloadOutput          `json:"output"`
+	Params         []string                `json:"params"`
+	DownloaderName string                  `json:"downloader_name"`
 }
 
 // struct representing the current status of the memoryDB
 // used for serializaton/persistence reasons
 type Session struct {
-	Processes []ProcessResponse `json:"processes"`
+	Snapshots []ProcessSnapshot `json:"processes"`
 }
 
 // struct representing the intent to stop a specific process
@@ -85,3 +73,11 @@ type CustomTemplate struct {
 	Name    string `json:"name"`
 	Content string `json:"content"`
 }
+
+const (
+	StatusPending = iota
+	StatusDownloading
+	StatusCompleted
+	StatusErrored
+	StatusLiveStream
+)

server/internal/downloaders/common.go

@@ -0,0 +1,42 @@
+package downloaders
+
+import (
+	"log/slog"
+	"sync"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+)
+
+type DownloaderBase struct {
+	Id        string
+	URL       string
+	Metadata  common.DownloadMetadata
+	Pending   bool
+	Completed bool
+	mutex     sync.Mutex
+}
+
+func (d *DownloaderBase) FetchMetadata(fetcher func(url string) (*common.DownloadMetadata, error)) {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	meta, err := fetcher(d.URL)
+	if err != nil {
+		slog.Error("failed to retrieve metadata", slog.Any("err", err))
+		return
+	}
+
+	d.Metadata = *meta
+}
+
+func (d *DownloaderBase) SetPending(p bool) {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	d.Pending = p
+}
+
+func (d *DownloaderBase) Complete() {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+	d.Completed = true
+}

server/internal/downloaders/downloader.go

@@ -0,0 +1,26 @@
+package downloaders
+
+import (
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+)
+
+type Downloader interface {
+	Start() error
+	Stop() error
+	Status() *internal.ProcessSnapshot
+
+	SetOutput(output internal.DownloadOutput)
+	SetProgress(progress internal.DownloadProgress)
+	SetMetadata(fetcher func(url string) (*common.DownloadMetadata, error))
+	SetPending(p bool)
+
+	IsCompleted() bool
+
+	UpdateSavedFilePath(path string)
+
+	RestoreFromSnapshot(*internal.ProcessSnapshot) error
+
+	GetId() string
+	GetUrl() string
+}

server/internal/downloaders/generic.go

@@ -0,0 +1,211 @@
+package downloaders
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"slices"
+	"strings"
+	"syscall"
+
+	"github.com/google/uuid"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+)
+
+const downloadTemplate = `download:
+{
+	"eta":%(progress.eta)s,
+	"percentage":"%(progress._percent_str)s",
+	"speed":%(progress.speed)s
+}`
+
+// filename not returning the correct extension after postprocess
+const postprocessTemplate = `postprocess:
+{
+	"filepath":"%(info.filepath)s"
+}
+`
+
+type GenericDownloader struct {
+	Params []string
+
+	AutoRemove bool
+
+	progress internal.DownloadProgress
+	output   internal.DownloadOutput
+
+	proc *os.Process
+
+	logConsumer LogConsumer
+
+	// embedded
+	DownloaderBase
+}
+
+func NewGenericDownload(url string, params []string) Downloader {
+	g := &GenericDownloader{
+		logConsumer: NewJSONLogConsumer(),
+	}
+	// in base
+	g.Id = uuid.NewString()
+	g.URL = url
+	return g
+}
+
+func (g *GenericDownloader) Start() error {
+	g.SetPending(true)
+
+	g.Params = argsSanitizer(g.Params)
+
+	out := internal.DownloadOutput{
+		Path:     config.Instance().Paths.DownloadPath,
+		Filename: "%(title)s.%(ext)s",
+	}
+
+	if g.output.Path != "" {
+		out.Path = g.output.Path
+	}
+
+	if g.output.Filename != "" {
+		out.Filename = g.output.Filename
+	}
+
+	buildFilename(&g.output)
+
+	templateReplacer := strings.NewReplacer("\n", "", "\t", "", " ", "")
+
+	baseParams := []string{
+		strings.Split(g.URL, "?list")[0], //no playlist
+		"--newline",
+		"--no-colors",
+		"--no-playlist",
+		"--progress-template",
+		templateReplacer.Replace(downloadTemplate),
+		"--progress-template",
+		templateReplacer.Replace(postprocessTemplate),
+		"--no-exec",
+	}
+
+	// if user asked to manually override the output path...
+	if !(slices.Contains(g.Params, "-P") || slices.Contains(g.Params, "--paths")) {
+		g.Params = append(g.Params, "-o")
+		g.Params = append(g.Params, fmt.Sprintf("%s/%s", out.Path, out.Filename))
+	}
+
+	params := append(baseParams, g.Params...)
+
+	slog.Info("requesting download", slog.String("url", g.URL), slog.Any("params", params))
+
+	cmd := exec.Command(config.Instance().Paths.DownloaderPath, params...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		slog.Error("failed to get a stdout pipe", slog.Any("err", err))
+		panic(err)
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		slog.Error("failed to get a stderr pipe", slog.Any("err", err))
+		panic(err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		slog.Error("failed to start yt-dlp process", slog.Any("err", err))
+		panic(err)
+	}
+
+	g.proc = cmd.Process
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer func() {
+		stdout.Close()
+		g.Complete()
+		cancel()
+	}()
+
+	logs := make(chan []byte)
+	go produceLogs(stdout, logs)
+	go consumeLogs(ctx, logs, g.logConsumer, g)
+
+	go printYtDlpErrors(stderr, g.Id, g.URL)
+
+	g.SetPending(false)
+	return cmd.Wait()
+}
+
+func (g *GenericDownloader) Stop() error {
+	defer func() {
+		g.progress.Status = internal.StatusCompleted
+		g.Complete()
+	}()
+	// yt-dlp uses multiple child process the parent process
+	// has been spawned with setPgid = true. To properly kill
+	// all subprocesses a SIGTERM need to be sent to the correct
+	// process group
+	if g.proc == nil {
+		return errors.New("*os.Process not set")
+	}
+
+	pgid, err := syscall.Getpgid(g.proc.Pid)
+	if err != nil {
+		return err
+	}
+	if err := syscall.Kill(-pgid, syscall.SIGTERM); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (g *GenericDownloader) Status() *internal.ProcessSnapshot {
+	return &internal.ProcessSnapshot{
+		Id:             g.Id,
+		Info:           g.Metadata,
+		Progress:       g.progress,
+		Output:         g.output,
+		Params:         g.Params,
+		DownloaderName: "generic",
+	}
+}
+
+func (g *GenericDownloader) UpdateSavedFilePath(p string) { g.output.SavedFilePath = p }
+
+func (g *GenericDownloader) SetOutput(o internal.DownloadOutput)     { g.output = o }
+func (g *GenericDownloader) SetProgress(p internal.DownloadProgress) { g.progress = p }
+
+func (g *GenericDownloader) SetMetadata(fetcher func(url string) (*common.DownloadMetadata, error)) {
+	g.FetchMetadata(fetcher)
+}
+
+func (g *GenericDownloader) SetPending(p bool) {
+	g.Pending = p
+}
+
+func (g *GenericDownloader) GetId() string  { return g.Id }
+func (g *GenericDownloader) GetUrl() string { return g.URL }
+
+func (g *GenericDownloader) RestoreFromSnapshot(snap *internal.ProcessSnapshot) error {
+	if snap == nil {
+		return errors.New("cannot restore nil snapshot")
+	}
+
+	s := *snap
+
+	g.Id = s.Id
+	g.URL = s.Info.URL
+	g.Metadata = s.Info
+	g.progress = s.Progress
+	g.output = s.Output
+	g.Params = s.Params
+
+	return nil
+}
+
+func (g *GenericDownloader) IsCompleted() bool { return g.Completed }

server/internal/downloaders/livestream.go

@@ -0,0 +1,205 @@
+package downloaders
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"syscall"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/pipes"
+)
+
+type LiveStreamDownloader struct {
+	progress internal.DownloadProgress
+
+	proc *os.Process
+
+	logConsumer LogConsumer
+
+	pipes []pipes.Pipe
+
+	// embedded
+	DownloaderBase
+}
+
+func NewLiveStreamDownloader(url string, pipes []pipes.Pipe) Downloader {
+	l := &LiveStreamDownloader{
+		logConsumer: NewFFMpegLogConsumer(),
+		pipes:       pipes,
+	}
+	// in base
+	l.Id = uuid.NewString()
+	l.URL = url
+	return l
+}
+
+func (l *LiveStreamDownloader) Start() error {
+	l.SetPending(true)
+
+	baseParams := []string{
+		l.URL,
+		"--newline",
+		"--no-colors",
+		"--no-playlist",
+		"--no-exec",
+	}
+
+	params := append(baseParams, "-o", "-")
+
+	cmd := exec.Command(config.Instance().Paths.DownloaderPath, params...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+
+	// stdout = media stream
+	media, err := cmd.StdoutPipe()
+	if err != nil {
+		slog.Error("failed to get media stdout", slog.Any("err", err))
+		panic(err)
+	}
+
+	// stderr = log/progress
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		slog.Error("failed to get stderr pipe", slog.Any("err", err))
+		panic(err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		slog.Error("failed to start yt-dlp process", slog.Any("err", err))
+		panic(err)
+	}
+
+	l.proc = cmd.Process
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer func() {
+		l.Complete()
+		cancel()
+	}()
+
+	// --- costruisci pipeline ---
+	reader := io.Reader(media)
+	for _, pipe := range l.pipes {
+		nr, err := pipe.Connect(reader)
+		if err != nil {
+			slog.Error("pipe failed", slog.String("pipe", pipe.Name()), slog.Any("err", err))
+			return err
+		}
+		reader = nr
+	}
+
+	// --- fallback: se nessun FileWriter, scrivi su file ---
+	if !l.hasFileWriter() {
+		go func() {
+			filepath.Join(
+				config.Instance().Paths.DownloadPath,
+				fmt.Sprintf("%s (live) %s.mp4", l.Id, time.Now().Format(time.ANSIC)),
+			)
+
+			defaultPath := filepath.Join(config.Instance().Paths.DownloadPath)
+			f, err := os.Create(defaultPath)
+			if err != nil {
+				slog.Error("failed to create fallback file", slog.Any("err", err))
+				return
+			}
+			defer f.Close()
+
+			_, err = io.Copy(f, reader)
+			if err != nil {
+				slog.Error("copy error", slog.Any("err", err))
+			}
+			slog.Info("download saved", slog.String("path", defaultPath))
+		}()
+	}
+
+	// --- logs consumer ---
+	logs := make(chan []byte)
+	go produceLogs(stderr, logs)
+	go consumeLogs(ctx, logs, l.logConsumer, l)
+
+	l.progress.Status = internal.StatusLiveStream
+
+	return cmd.Wait()
+}
+
+func (l *LiveStreamDownloader) Stop() error {
+	defer func() {
+		l.progress.Status = internal.StatusCompleted
+		l.Complete()
+	}()
+	// yt-dlp uses multiple child process the parent process
+	// has been spawned with setPgid = true. To properly kill
+	// all subprocesses a SIGTERM need to be sent to the correct
+	// process group
+	if l.proc == nil {
+		return errors.New("*os.Process not set")
+	}
+
+	pgid, err := syscall.Getpgid(l.proc.Pid)
+	if err != nil {
+		return err
+	}
+	if err := syscall.Kill(-pgid, syscall.SIGTERM); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *LiveStreamDownloader) Status() *internal.ProcessSnapshot {
+	return &internal.ProcessSnapshot{
+		Id:             l.Id,
+		Info:           l.Metadata,
+		Progress:       l.progress,
+		DownloaderName: "livestream",
+	}
+}
+
+func (l *LiveStreamDownloader) UpdateSavedFilePath(p string) {}
+
+func (l *LiveStreamDownloader) SetOutput(o internal.DownloadOutput)     {}
+func (l *LiveStreamDownloader) SetProgress(p internal.DownloadProgress) { l.progress = p }
+
+func (l *LiveStreamDownloader) SetMetadata(fetcher func(url string) (*common.DownloadMetadata, error)) {
+	l.FetchMetadata(fetcher)
+}
+
+func (l *LiveStreamDownloader) SetPending(p bool) {
+	l.Pending = p
+}
+
+func (l *LiveStreamDownloader) GetId() string  { return l.Id }
+func (l *LiveStreamDownloader) GetUrl() string { return l.URL }
+
+func (l *LiveStreamDownloader) RestoreFromSnapshot(snap *internal.ProcessSnapshot) error {
+	if snap == nil {
+		return errors.New("cannot restore nil snapshot")
+	}
+
+	s := *snap
+
+	l.Id = s.Id
+	l.URL = s.Info.URL
+	l.Metadata = s.Info
+	l.progress = s.Progress
+
+	return nil
+}
+
+func (l *LiveStreamDownloader) IsCompleted() bool { return l.Completed }
+
+func (l *LiveStreamDownloader) hasFileWriter() bool {
+	return slices.ContainsFunc(l.pipes, func(p pipes.Pipe) bool {
+		return p.Name() == "file-writer"
+	})
+}

server/internal/downloaders/logging.go

@@ -0,0 +1,68 @@
+package downloaders
+
+import (
+	"encoding/json"
+	"log/slog"
+	"strings"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+)
+
+type LogConsumer interface {
+	GetName() string
+	ParseLogEntry(entry []byte, downloader Downloader)
+}
+
+type JSONLogConsumer struct{}
+
+func NewJSONLogConsumer() LogConsumer {
+	return &JSONLogConsumer{}
+}
+
+func (j *JSONLogConsumer) GetName() string { return "json-log-consumer" }
+
+func (j *JSONLogConsumer) ParseLogEntry(entry []byte, d Downloader) {
+	var progress internal.ProgressTemplate
+	var postprocess internal.PostprocessTemplate
+
+	if err := json.Unmarshal(entry, &progress); err == nil {
+		d.SetProgress(internal.DownloadProgress{
+			Status:     internal.StatusDownloading,
+			Percentage: progress.Percentage,
+			Speed:      progress.Speed,
+			ETA:        progress.Eta,
+		})
+
+		slog.Info("progress",
+			slog.String("id", j.GetShortId(d.GetId())),
+			slog.String("url", d.GetUrl()),
+			slog.String("percentage", progress.Percentage),
+		)
+	}
+
+	if err := json.Unmarshal(entry, &postprocess); err == nil {
+		d.UpdateSavedFilePath(postprocess.FilePath)
+	}
+}
+
+func (j *JSONLogConsumer) GetShortId(id string) string {
+	return strings.Split(id, "-")[0]
+}
+
+//TODO: split in different files
+
+type FFMpegLogConsumer struct{}
+
+func NewFFMpegLogConsumer() LogConsumer {
+	return &JSONLogConsumer{}
+}
+
+func (f *FFMpegLogConsumer) GetName() string { return "ffmpeg-log-consumer" }
+
+func (f *FFMpegLogConsumer) ParseLogEntry(entry []byte, d Downloader) {
+	slog.Info("ffmpeg output",
+		slog.String("id", d.GetId()),
+		slog.String("url", d.GetUrl()),
+		slog.String("output", string(entry)),
+	)
+}

server/internal/downloaders/playlist.go

@@ -0,0 +1 @@
+package downloaders

server/internal/downloaders/utils.go

@@ -0,0 +1,76 @@
+package downloaders
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"log/slog"
+	"regexp"
+	"slices"
+	"strings"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+)
+
+func argsSanitizer(params []string) []string {
+	params = slices.DeleteFunc(params, func(e string) bool {
+		match, _ := regexp.MatchString(`(\$\{)|(\&\&)`, e)
+		return match
+	})
+
+	params = slices.DeleteFunc(params, func(e string) bool {
+		return e == ""
+	})
+
+	return params
+}
+
+func buildFilename(o *internal.DownloadOutput) {
+	if o.Filename != "" && strings.Contains(o.Filename, ".%(ext)s") {
+		o.Filename += ".%(ext)s"
+	}
+
+	o.Filename = strings.Replace(
+		o.Filename,
+		".%(ext)s.%(ext)s",
+		".%(ext)s",
+		1,
+	)
+}
+
+func produceLogs(r io.Reader, logs chan<- []byte) {
+	go func() {
+		scanner := bufio.NewScanner(r)
+
+		for scanner.Scan() {
+			logs <- scanner.Bytes()
+		}
+	}()
+}
+
+func consumeLogs(ctx context.Context, logs <-chan []byte, c LogConsumer, d Downloader) {
+	for {
+		select {
+		case <-ctx.Done():
+			slog.Info("detaching logs",
+				slog.String("url", d.GetUrl()),
+				slog.String("id", c.GetName()),
+			)
+			return
+		case entry := <-logs:
+			c.ParseLogEntry(entry, d)
+		}
+	}
+}
+
+func printYtDlpErrors(stdout io.Reader, shortId, url string) {
+	scanner := bufio.NewScanner(stdout)
+
+	for scanner.Scan() {
+		slog.Error("yt-dlp process error",
+			slog.String("id", shortId),
+			slog.String("url", url),
+			slog.String("err", scanner.Text()),
+		)
+	}
+}

server/internal/kv/store.go

@@ -0,0 +1,173 @@
+package kv
+
+import (
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+var (
+	bucket      = []byte("downloads")
+	memDbEvents = make(chan downloaders.Downloader, runtime.NumCPU())
+)
+
+// In-Memory Thread-Safe Key-Value Storage with optional persistence
+type Store struct {
+	db    *bolt.DB
+	table map[string]downloaders.Downloader
+	mu    sync.RWMutex
+}
+
+func NewStore(db *bolt.DB, snaptshotInteval time.Duration) (*Store, error) {
+	// init bucket
+	err := db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucket)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Store{
+		db:    db,
+		table: make(map[string]downloaders.Downloader),
+	}
+
+	go func() {
+		ticker := time.NewTicker(snaptshotInteval)
+		for range ticker.C {
+			s.Snapshot()
+		}
+	}()
+
+	return s, err
+}
+
+// Get a process pointer given its id
+func (m *Store) Get(id string) (downloaders.Downloader, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	entry, ok := m.table[id]
+	if !ok {
+		return nil, errors.New("no process found for the given key")
+	}
+
+	return entry, nil
+}
+
+// Store a pointer of a process and return its id
+func (m *Store) Set(d downloaders.Downloader) string {
+	m.mu.Lock()
+	m.table[d.GetId()] = d
+	m.mu.Unlock()
+
+	return d.GetId()
+}
+
+// Removes a process progress, given the process id
+func (m *Store) Delete(id string) {
+	m.mu.Lock()
+	delete(m.table, id)
+	m.mu.Unlock()
+}
+
+func (m *Store) Keys() *[]string {
+	var running []string
+
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	for id := range m.table {
+		running = append(running, id)
+	}
+
+	return &running
+}
+
+// Returns a slice of all currently stored processes progess
+func (m *Store) All() *[]internal.ProcessSnapshot {
+	running := []internal.ProcessSnapshot{}
+
+	m.mu.RLock()
+	for _, v := range m.table {
+		running = append(running, *(v.Status()))
+	}
+	m.mu.RUnlock()
+
+	return &running
+}
+
+// Restore a persisted state
+func (m *Store) Restore(mq *queue.MessageQueue) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	var snapshot []internal.ProcessSnapshot
+
+	m.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		return b.ForEach(func(k, v []byte) error {
+			var snap internal.ProcessSnapshot
+			if err := json.Unmarshal(v, &snap); err != nil {
+				return err
+			}
+			snapshot = append(snapshot, snap)
+			return nil
+		})
+	})
+
+	for _, snap := range snapshot {
+		var restored downloaders.Downloader
+		if snap.DownloaderName == "generic" {
+			d := downloaders.NewGenericDownload("", []string{})
+			err := d.RestoreFromSnapshot(&snap)
+			if err != nil {
+				continue
+			}
+			restored = d
+			m.table[snap.Id] = restored
+			if !restored.(*downloaders.GenericDownloader).DownloaderBase.Completed {
+				mq.Publish(restored)
+			}
+		}
+	}
+}
+
+func (m *Store) EventListener() {
+	for p := range memDbEvents {
+		if p.Status().DownloaderName == "livestream" {
+			slog.Info("compacting Store", slog.String("id", p.GetId()))
+			m.Delete(p.GetId())
+		}
+	}
+}
+
+func (m *Store) Snapshot() error {
+	slog.Debug("snapshotting downloads state")
+
+	running := m.All()
+
+	return m.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		for _, v := range *running {
+			data, err := json.Marshal(v)
+			if err != nil {
+				return err
+			}
+			if err := b.Put([]byte(v.Id), data); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

server/internal/kv/types.go

@@ -0,0 +1,9 @@
+package kv
+
+import "github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+
+// struct representing the current status of the memoryDB
+// used for serializaton/persistence reasons
+type Session struct {
+	Processes []internal.ProcessSnapshot `json:"processes"`
+}

server/internal/livestream/livestream.go

@@ -11,7 +11,10 @@ import (
 	"time"
 
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/pipes"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
 )
 
 const (
@@ -32,11 +35,11 @@ type LiveStream struct {
 	waitTime     time.Duration
 	liveDate     time.Time
 
-	mq *internal.MessageQueue
-	db *internal.MemoryDB
+	mq    *queue.MessageQueue
+	store *kv.Store
 }
 
-func New(url string, done chan *LiveStream, mq *internal.MessageQueue, db *internal.MemoryDB) *LiveStream {
+func New(url string, done chan *LiveStream, mq *queue.MessageQueue, store *kv.Store) *LiveStream {
 	return &LiveStream{
 		url:          url,
 		done:         done,
@@ -44,20 +47,20 @@ func New(url string, done chan *LiveStream, mq *internal.MessageQueue, db *inter
 		waitTime:     time.Second * 0,
 		waitTimeChan: make(chan time.Duration),
 		mq:           mq,
-		db:           db,
+		store:        store,
 	}
 }
 
 // Start the livestream monitoring process, once completion signals on the done channel
 func (l *LiveStream) Start() error {
 	cmd := exec.Command(
-		config.Instance().DownloaderPath,
+		config.Instance().Paths.DownloaderPath,
 		l.url,
 		"--wait-for-video", "30", // wait for the stream to be live and recheck every 10 secs
 		"--no-colors", // no ansi color fuzz
 		"--simulate",
 		"--newline",
-		"--paths", config.Instance().DownloadPath,
+		"--paths", config.Instance().Paths.DownloadPath,
 	)
 
 	stdout, err := cmd.StdoutPipe()
@@ -87,13 +90,12 @@ func (l *LiveStream) Start() error {
 	l.done <- l
 
 	// Send the started livestream to the message queue! :D
-	p := &internal.Process{
-		Url:        l.url,
-		Livestream: true,
-		Params:     []string{"--downloader", "ffmpeg", "--no-part"},
-	}
-	l.db.Set(p)
-	l.mq.Publish(p)
+
+	//TODO: add pipes
+	d := downloaders.NewLiveStreamDownloader(l.url, []pipes.Pipe{})
+
+	l.store.Set(d)
+	l.mq.Publish(d)
 
 	return nil
 }
@@ -141,26 +143,13 @@ func (l *LiveStream) monitorStartTime(r io.Reader) {
 		}
 	}
 
-	const TRIES = 5
-	/*
-		if it's waiting a livestream the 5th line will indicate the time to live
-		its a dumb and not robust method.
+	scanner.Scan()
 
-		example:
-			[youtube] Extracting URL: https://www.youtube.com/watch?v=IQVbGfVVjgY
-			[youtube] IQVbGfVVjgY: Downloading webpage
-			[youtube] IQVbGfVVjgY: Downloading ios player API JSON
-			[youtube] IQVbGfVVjgY: Downloading web creator player API JSON
-			WARNING: [youtube] This live event will begin in 27 minutes.       <- STDERR, ignore
-			[wait] Waiting for 00:27:15 - Press Ctrl+C to try now              <- 5th line
-	*/
-	for range TRIES {
+	for !strings.Contains(scanner.Text(), "Waiting for") {
 		scanner.Scan()
-
-		if strings.Contains(scanner.Text(), "Waiting for") {
-			waitTimeScanner()
-		}
 	}
+
+	waitTimeScanner()
 }
 
 func (l *LiveStream) WaitTime() <-chan time.Duration {

server/internal/livestream/livestream_test.go

@@ -5,19 +5,22 @@ import (
 	"time"
 
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
 )
 
 func setupTest() {
-	config.Instance().DownloaderPath = "yt-dlp"
+	config.Instance().Paths.DownloaderPath = "build/yt-dlp"
 }
 
+const URL = "https://www.youtube.com/watch?v=pwoAyLGOysU"
+
 func TestLivestream(t *testing.T) {
 	setupTest()
 
 	done := make(chan *LiveStream)
 
-	ls := New("https://www.youtube.com/watch?v=LSm1daKezcE", done, &internal.MessageQueue{}, &internal.MemoryDB{})
+	ls := New(URL, done, &queue.MessageQueue{}, &kv.Store{})
 	go ls.Start()
 
 	time.AfterFunc(time.Second*20, func() {

server/internal/livestream/monitor.go

@@ -1,27 +1,31 @@
 package livestream
 
 import (
-	"encoding/gob"
-	"log/slog"
-	"maps"
-	"os"
-	"path/filepath"
-
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
+	bolt "go.etcd.io/bbolt"
 )
 
+var bucket = []byte("livestreams")
+
 type Monitor struct {
-	db      *internal.MemoryDB     // where the just started livestream will be published
-	mq      *internal.MessageQueue // where the just started livestream will be published
+	db      *bolt.DB
+	store   *kv.Store              // where the just started livestream will be published
+	mq      *queue.MessageQueue    // where the just started livestream will be published
 	streams map[string]*LiveStream // keeps track of the livestreams
 	done    chan *LiveStream       // to signal individual processes completition
 }
 
-func NewMonitor(mq *internal.MessageQueue, db *internal.MemoryDB) *Monitor {
+func NewMonitor(mq *queue.MessageQueue, store *kv.Store, db *bolt.DB) *Monitor {
+	db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucket)
+		return err
+	})
+
 	return &Monitor{
 		mq:      mq,
 		db:      db,
+		store:   store,
 		streams: make(map[string]*LiveStream),
 		done:    make(chan *LiveStream),
 	}
@@ -31,14 +35,24 @@ func NewMonitor(mq *internal.MessageQueue, db *internal.MemoryDB) *Monitor {
 func (m *Monitor) Schedule() {
 	for l := range m.done {
 		delete(m.streams, l.url)
+
+		m.db.Update(func(tx *bolt.Tx) error {
+			b := tx.Bucket(bucket)
+			return b.Delete([]byte(l.url))
+		})
 	}
 }
 
 func (m *Monitor) Add(url string) {
-	ls := New(url, m.done, m.mq, m.db)
+	ls := New(url, m.done, m.mq, m.store)
 
 	go ls.Start()
 	m.streams[url] = ls
+
+	m.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		return b.Put([]byte(url), []byte{})
+	})
 }
 
 func (m *Monitor) Remove(url string) error {
@@ -58,11 +72,6 @@ func (m *Monitor) Status() LiveStreamStatus {
 	status := make(LiveStreamStatus)
 
 	for k, v := range m.streams {
-		// wt, ok := <-v.WaitTime()
-		// if !ok {
-		// 	continue
-		// }
-
 		status[k] = Status{
 			Status:   v.status,
 			WaitTime: v.waitTime,
@@ -73,46 +82,13 @@ func (m *Monitor) Status() LiveStreamStatus {
 	return status
 }
 
-// Persist the monitor current state to a file.
-// The file is located in the configured config directory
-func (m *Monitor) Persist() error {
-	fd, err := os.Create(filepath.Join(config.Instance().Dir(), "livestreams.dat"))
-	if err != nil {
-		return err
-	}
-
-	defer fd.Close()
-
-	slog.Debug("persisting livestream monitor state")
-
-	var toPersist []string
-	for url := range maps.Keys(m.streams) {
-		toPersist = append(toPersist, url)
-	}
-
-	return gob.NewEncoder(fd).Encode(toPersist)
-}
-
 // Restore a saved state and resume the monitored livestreams
 func (m *Monitor) Restore() error {
-	fd, err := os.Open(filepath.Join(config.Instance().Dir(), "livestreams.dat"))
-	if err != nil {
-		return err
-	}
-
-	defer fd.Close()
-
-	var toRestore []string
-
-	if err := gob.NewDecoder(fd).Decode(&toRestore); err != nil {
-		return err
-	}
-
-	for _, url := range toRestore {
-		m.Add(url)
-	}
-
-	slog.Debug("restored livestream monitor state")
-
-	return nil
+	return m.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		return b.ForEach(func(k, v []byte) error {
+			m.Add(string(k))
+			return nil
+		})
+	})
 }

server/internal/memory_db.go

@@ -1,146 +0,0 @@
-package internal
-
-import (
-	"encoding/gob"
-	"errors"
-	"os"
-	"path/filepath"
-	"sync"
-
-	"github.com/google/uuid"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-)
-
-// In-Memory Thread-Safe Key-Value Storage with optional persistence
-type MemoryDB struct {
-	table map[string]*Process
-	mu    sync.RWMutex
-}
-
-func NewMemoryDB() *MemoryDB {
-	return &MemoryDB{
-		table: make(map[string]*Process),
-	}
-}
-
-// Get a process pointer given its id
-func (m *MemoryDB) Get(id string) (*Process, error) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	entry, ok := m.table[id]
-	if !ok {
-		return nil, errors.New("no process found for the given key")
-	}
-
-	return entry, nil
-}
-
-// Store a pointer of a process and return its id
-func (m *MemoryDB) Set(process *Process) string {
-	id := uuid.NewString()
-
-	m.mu.Lock()
-	process.Id = id
-	m.table[id] = process
-	m.mu.Unlock()
-
-	return id
-}
-
-// Removes a process progress, given the process id
-func (m *MemoryDB) Delete(id string) {
-	m.mu.Lock()
-	delete(m.table, id)
-	m.mu.Unlock()
-}
-
-func (m *MemoryDB) Keys() *[]string {
-	var running []string
-
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	for id := range m.table {
-		running = append(running, id)
-	}
-
-	return &running
-}
-
-// Returns a slice of all currently stored processes progess
-func (m *MemoryDB) All() *[]ProcessResponse {
-	running := []ProcessResponse{}
-
-	m.mu.RLock()
-	for k, v := range m.table {
-		running = append(running, ProcessResponse{
-			Id:       k,
-			Info:     v.Info,
-			Progress: v.Progress,
-			Output:   v.Output,
-			Params:   v.Params,
-		})
-	}
-	m.mu.RUnlock()
-
-	return &running
-}
-
-// Persist the database in a single file named "session.dat"
-func (m *MemoryDB) Persist() error {
-	running := m.All()
-
-	sf := filepath.Join(config.Instance().SessionFilePath, "session.dat")
-
-	fd, err := os.Create(sf)
-	if err != nil {
-		return errors.Join(errors.New("failed to persist session"), err)
-	}
-
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	session := Session{Processes: *running}
-
-	if err := gob.NewEncoder(fd).Encode(session); err != nil {
-		return errors.Join(errors.New("failed to persist session"), err)
-	}
-
-	return nil
-}
-
-// Restore a persisted state
-func (m *MemoryDB) Restore(mq *MessageQueue) {
-	sf := filepath.Join(config.Instance().SessionFilePath, "session.dat")
-
-	fd, err := os.Open(sf)
-	if err != nil {
-		return
-	}
-
-	var session Session
-
-	if err := gob.NewDecoder(fd).Decode(&session); err != nil {
-		return
-	}
-
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	for _, proc := range session.Processes {
-		restored := &Process{
-			Id:       proc.Id,
-			Url:      proc.Info.URL,
-			Info:     proc.Info,
-			Progress: proc.Progress,
-			Output:   proc.Output,
-			Params:   proc.Params,
-		}
-
-		m.table[proc.Id] = restored
-
-		if restored.Progress.Status != StatusCompleted {
-			mq.Publish(restored)
-		}
-	}
-}

server/internal/message_queue.go

@@ -1,112 +0,0 @@
-package internal
-
-import (
-	"context"
-	"errors"
-	"log/slog"
-
-	evbus "github.com/asaskevich/EventBus"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-	"golang.org/x/sync/semaphore"
-)
-
-const queueName = "process:pending"
-
-type MessageQueue struct {
-	concurrency int
-	eventBus    evbus.Bus
-}
-
-// Creates a new message queue.
-// By default it will be created with a size equals to nthe number of logical
-// CPU cores -1.
-// The queue size can be set via the qs flag.
-func NewMessageQueue() (*MessageQueue, error) {
-	qs := config.Instance().QueueSize
-
-	if qs <= 0 {
-		return nil, errors.New("invalid queue size")
-	}
-
-	return &MessageQueue{
-		concurrency: qs,
-		eventBus:    evbus.New(),
-	}, nil
-}
-
-// Publish a message to the queue and set the task to a peding state.
-func (m *MessageQueue) Publish(p *Process) {
-	// needs to have an id set before
-	p.SetPending()
-
-	m.eventBus.Publish(queueName, p)
-}
-
-func (m *MessageQueue) SetupConsumers() {
-	go m.downloadConsumer()
-	go m.metadataSubscriber()
-}
-
-// Setup the consumer listener which subscribes to the changes to the producer
-// channel and triggers the "download" action.
-func (m *MessageQueue) downloadConsumer() {
-	sem := semaphore.NewWeighted(int64(m.concurrency))
-
-	m.eventBus.SubscribeAsync(queueName, func(p *Process) {
-		sem.Acquire(context.Background(), 1)
-		defer sem.Release(1)
-
-		slog.Info("received process from event bus",
-			slog.String("bus", queueName),
-			slog.String("consumer", "downloadConsumer"),
-			slog.String("id", p.getShortId()),
-		)
-
-		if p.Progress.Status != StatusCompleted {
-			slog.Info("started process",
-				slog.String("bus", queueName),
-				slog.String("id", p.getShortId()),
-			)
-			if p.Livestream {
-				// livestreams have higher priorty and they ignore the semaphore
-				go p.Start()
-			} else {
-				p.Start()
-			}
-		}
-	}, false)
-}
-
-// Setup the metadata consumer listener which subscribes to the changes to the
-// producer channel and adds metadata to each download.
-func (m *MessageQueue) metadataSubscriber() {
-	// How many concurrent metadata fetcher jobs are spawned
-	// Since there's ongoing downloads, 1 job at time seems a good compromise
-	sem := semaphore.NewWeighted(1)
-
-	m.eventBus.SubscribeAsync(queueName, func(p *Process) {
-		sem.Acquire(context.Background(), 1)
-		defer sem.Release(1)
-
-		slog.Info("received process from event bus",
-			slog.String("bus", queueName),
-			slog.String("consumer", "metadataConsumer"),
-			slog.String("id", p.getShortId()),
-		)
-
-		if p.Progress.Status == StatusCompleted {
-			slog.Warn("proccess has an illegal state",
-				slog.String("id", p.getShortId()),
-				slog.Int("status", p.Progress.Status),
-			)
-			return
-		}
-
-		if err := p.SetMetadata(); err != nil {
-			slog.Error("failed to retrieve metadata",
-				slog.String("id", p.getShortId()),
-				slog.String("err", err.Error()),
-			)
-		}
-	}, false)
-}

server/internal/metadata/fetchers.go

@@ -0,0 +1,57 @@
+package metadata
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+)
+
+func DefaultFetcher(url string) (*common.DownloadMetadata, error) {
+	cmd := exec.Command(config.Instance().Paths.DownloaderPath, url, "-J")
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	meta := common.DownloadMetadata{
+		URL:       url,
+		CreatedAt: time.Now(),
+	}
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	var bufferedStderr bytes.Buffer
+
+	go func() {
+		io.Copy(&bufferedStderr, stderr)
+	}()
+
+	slog.Info("retrieving metadata", slog.String("url", url))
+
+	if err := json.NewDecoder(stdout).Decode(&meta); err != nil {
+		return nil, err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return nil, errors.New(bufferedStderr.String())
+	}
+
+	return &meta, nil
+}

server/internal/pipeline/rest.go

@@ -0,0 +1,92 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	bolt "go.etcd.io/bbolt"
+)
+
+type handler struct {
+	store *Store
+}
+
+func NewRestHandler(db *bolt.DB) *handler {
+	store, _ := NewStore(db)
+	return &handler{
+		store: store,
+	}
+}
+
+func (h *handler) GetPipeline(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	id := chi.URLParam(r, "id")
+
+	p, err := h.store.Get(id)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(p); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func (h *handler) GetAllPipelines(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	p, err := h.store.List()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(p); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func (h *handler) SavePipeline(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	defer r.Body.Close()
+	var req Pipeline
+
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	id, err := h.store.Save(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(id); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func (h *handler) DeletePipeline(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	id := chi.URLParam(r, "id")
+
+	err := h.store.Delete(id)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode("ok"); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}

server/internal/pipeline/store.go

@@ -0,0 +1,103 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/uuid"
+	bolt "go.etcd.io/bbolt"
+)
+
+var bucket = []byte("pipelines")
+
+type Step struct {
+	Type       string   `json:"type"`                  // es. "transcoder", "filewriter"
+	FFmpegArgs []string `json:"ffmpeg_args,omitempty"` // args da passare a ffmpeg
+	Path       string   `json:"path,omitempty"`        // solo per filewriter
+	Extension  string   `json:"extension,omitempty"`   // solo per filewriter
+}
+
+type Pipeline struct {
+	ID    string `json:"id"`
+	Name  string `json:"name"`
+	Steps []Step `json:"steps"`
+}
+
+type Store struct {
+	db *bolt.DB
+}
+
+func NewStore(db *bolt.DB) (*Store, error) {
+	// init bucket
+	err := db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucket)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Store{db: db}, nil
+}
+
+func (s *Store) Save(p Pipeline) (string, error) {
+	if p.ID == "" {
+		p.ID = uuid.NewString()
+	}
+
+	data, err := json.Marshal(p)
+	if err != nil {
+		return "", err
+	}
+
+	return p.ID, s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		return b.Put([]byte(p.ID), data)
+	})
+}
+
+func (s *Store) Get(id string) (*Pipeline, error) {
+	var p Pipeline
+
+	err := s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		v := b.Get([]byte(id))
+		if v == nil {
+			return fmt.Errorf("pipeline %s not found", id)
+		}
+		return json.Unmarshal(v, &p)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &p, nil
+}
+
+func (s *Store) List() ([]Pipeline, error) {
+	var result []Pipeline
+
+	err := s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		return b.ForEach(func(k, v []byte) error {
+			var p Pipeline
+			if err := json.Unmarshal(v, &p); err != nil {
+				return err
+			}
+			result = append(result, p)
+			return nil
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (s *Store) Delete(id string) error {
+	return s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(bucket)
+		return b.Delete([]byte(id))
+	})
+}

server/internal/pipes/FileWriter.go

@@ -0,0 +1,45 @@
+package pipes
+
+import (
+	"io"
+	"log/slog"
+	"os"
+)
+
+type FileWriter struct {
+	Path    string
+	IsFinal bool
+}
+
+func (f *FileWriter) Name() string { return "file-writer" }
+
+func (f *FileWriter) Connect(r io.Reader) (io.Reader, error) {
+	file, err := os.Create(f.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	if f.IsFinal {
+		go func() {
+			defer file.Close()
+			if _, err := io.Copy(file, r); err != nil {
+				slog.Error("FileWriter (final) error", slog.Any("err", err))
+			}
+		}()
+		return r, nil
+	}
+
+	pr, pw := io.Pipe()
+
+	go func() {
+		defer file.Close()
+		defer pw.Close()
+
+		writer := io.MultiWriter(file, pw)
+		if _, err := io.Copy(writer, r); err != nil {
+			slog.Error("FileWriter (pipeline) error", slog.Any("err", err))
+		}
+	}()
+
+	return pr, nil
+}

server/internal/pipes/Transcoder.go

@@ -0,0 +1,66 @@
+package pipes
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"log/slog"
+	"os/exec"
+	"strings"
+)
+
+type Transcoder struct {
+	Args []string
+}
+
+func (t *Transcoder) Name() string { return "ffmpeg-transcoder" }
+
+func (t *Transcoder) Connect(r io.Reader) (io.Reader, error) {
+	cmd := exec.Command("ffmpeg",
+		append([]string{"-i", "pipe:0"}, append(t.Args, "-f", "webm", "pipe:1")...)...,
+	)
+
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	go func() {
+		reader := bufio.NewReader(stderr)
+		var line string
+
+		for {
+			part, err := reader.ReadString('\r')
+			line += part
+			if err != nil {
+				break
+			}
+
+			line = strings.TrimRight(line, "\r\n")
+			slog.Info("ffmpeg transcoder", slog.String("log", line))
+			line = ""
+		}
+	}()
+
+	go func() {
+		defer stdin.Close()
+		_, err := io.Copy(stdin, r)
+		if err != nil && !errors.Is(err, io.EOF) {
+			slog.Error("transcoder stdin error", slog.Any("err", err))
+		}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	return stdout, nil
+}

server/internal/pipes/pipe.go

@@ -0,0 +1,8 @@
+package pipes
+
+import "io"
+
+type Pipe interface {
+	Name() string
+	Connect(r io.Reader) (io.Reader, error)
+}

server/internal/pool.go

@@ -1,16 +0,0 @@
-package internal
-
-type Pool []*Worker
-
-func (h Pool) Len() int           { return len(h) }
-func (h Pool) Less(i, j int) bool { return h[i].index < h[j].index }
-func (h Pool) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
-func (h *Pool) Push(x any)        { *h = append(*h, x.(*Worker)) }
-
-func (h *Pool) Pop() any {
-	old := *h
-	n := len(old)
-	x := old[n-1]
-	*h = old[0 : n-1]
-	return x
-}

server/internal/process.go

@@ -1,381 +0,0 @@
-package internal
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"regexp"
-	"slices"
-	"syscall"
-
-	"os"
-	"os/exec"
-	"strings"
-	"time"
-
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/archiver"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-)
-
-const downloadTemplate = `download:
-{
-	"eta":%(progress.eta)s,
-	"percentage":"%(progress._percent_str)s",
-	"speed":%(progress.speed)s
-}`
-
-// filename not returning the correct extension after postprocess
-const postprocessTemplate = `postprocess:
-{
-	"filepath":"%(info.filepath)s"
-}
-`
-
-const (
-	StatusPending = iota
-	StatusDownloading
-	StatusCompleted
-	StatusErrored
-)
-
-// Process descriptor
-type Process struct {
-	Id         string
-	Url        string
-	Livestream bool
-	Params     []string
-	Info       DownloadInfo
-	Progress   DownloadProgress
-	Output     DownloadOutput
-	proc       *os.Process
-}
-
-// Starts spawns/forks a new yt-dlp process and parse its stdout.
-// The process is spawned to outputting a custom progress text that
-// Resembles a JSON Object in order to Unmarshal it later.
-// This approach is anyhow not perfect: quotes are not escaped properly.
-// Each process is not identified by its PID but by a UUIDv4
-func (p *Process) Start() {
-	// escape bash variable escaping and command piping, you'll never know
-	// what they might come with...
-	p.Params = slices.DeleteFunc(p.Params, func(e string) bool {
-		match, _ := regexp.MatchString(`(\$\{)|(\&\&)`, e)
-		return match
-	})
-
-	p.Params = slices.DeleteFunc(p.Params, func(e string) bool {
-		return e == ""
-	})
-
-	out := DownloadOutput{
-		Path:     config.Instance().DownloadPath,
-		Filename: "%(title)s.%(ext)s",
-	}
-
-	if p.Output.Path != "" {
-		out.Path = p.Output.Path
-	}
-
-	if p.Output.Filename != "" {
-		out.Filename = p.Output.Filename
-	}
-
-	buildFilename(&p.Output)
-
-	templateReplacer := strings.NewReplacer("\n", "", "\t", "", " ", "")
-
-	baseParams := []string{
-		strings.Split(p.Url, "?list")[0], //no playlist
-		"--newline",
-		"--no-colors",
-		"--no-playlist",
-		"--progress-template",
-		templateReplacer.Replace(downloadTemplate),
-		"--progress-template",
-		templateReplacer.Replace(postprocessTemplate),
-	}
-
-	// if user asked to manually override the output path...
-	if !(slices.Contains(p.Params, "-P") || slices.Contains(p.Params, "--paths")) {
-		p.Params = append(p.Params, "-o")
-		p.Params = append(p.Params, fmt.Sprintf("%s/%s", out.Path, out.Filename))
-	}
-
-	params := append(baseParams, p.Params...)
-
-	slog.Info("requesting download", slog.String("url", p.Url), slog.Any("params", params))
-
-	cmd := exec.Command(config.Instance().DownloaderPath, params...)
-	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		slog.Error("failed to get a stdout pipe", slog.Any("err", err))
-		panic(err)
-	}
-
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		slog.Error("failed to get a stderr pipe", slog.Any("err", err))
-		panic(err)
-	}
-
-	if err := cmd.Start(); err != nil {
-		slog.Error("failed to start yt-dlp process", slog.Any("err", err))
-		panic(err)
-	}
-
-	p.proc = cmd.Process
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer func() {
-		stdout.Close()
-		p.Complete()
-		cancel()
-	}()
-
-	logs := make(chan []byte)
-	go produceLogs(stdout, logs)
-	go p.consumeLogs(ctx, logs)
-
-	go p.detectYtDlpErrors(stderr)
-
-	cmd.Wait()
-}
-
-func produceLogs(r io.Reader, logs chan<- []byte) {
-	go func() {
-		scanner := bufio.NewScanner(r)
-
-		for scanner.Scan() {
-			logs <- scanner.Bytes()
-		}
-	}()
-}
-
-func (p *Process) consumeLogs(ctx context.Context, logs <-chan []byte) {
-	for {
-		select {
-		case <-ctx.Done():
-			slog.Info("detaching from yt-dlp stdout",
-				slog.String("id", p.getShortId()),
-				slog.String("url", p.Url),
-			)
-			return
-		case entry := <-logs:
-			p.parseLogEntry(entry)
-		}
-	}
-}
-
-func (p *Process) parseLogEntry(entry []byte) {
-	var progress ProgressTemplate
-	var postprocess PostprocessTemplate
-
-	if err := json.Unmarshal(entry, &progress); err == nil {
-		p.Progress = DownloadProgress{
-			Status:     StatusDownloading,
-			Percentage: progress.Percentage,
-			Speed:      progress.Speed,
-			ETA:        progress.Eta,
-		}
-
-		slog.Info("progress",
-			slog.String("id", p.getShortId()),
-			slog.String("url", p.Url),
-			slog.String("percentage", progress.Percentage),
-		)
-	}
-
-	if err := json.Unmarshal(entry, &postprocess); err == nil {
-		p.Output.SavedFilePath = postprocess.FilePath
-
-		// slog.Info("postprocess",
-		// 	slog.String("id", p.getShortId()),
-		// 	slog.String("url", p.Url),
-		// 	slog.String("filepath", postprocess.FilePath),
-		// )
-	}
-}
-
-func (p *Process) detectYtDlpErrors(r io.Reader) {
-	scanner := bufio.NewScanner(r)
-
-	for scanner.Scan() {
-		slog.Error("yt-dlp process error",
-			slog.String("id", p.getShortId()),
-			slog.String("url", p.Url),
-			slog.String("err", scanner.Text()),
-		)
-	}
-}
-
-// Keep process in the memoryDB but marks it as complete
-// Convention: All completed processes has progress -1
-// and speed 0 bps.
-func (p *Process) Complete() {
-	// auto archive
-	// TODO: it's not that deterministic :/
-	if p.Progress.Percentage == "" && p.Progress.Speed == 0 {
-		var serializedMetadata bytes.Buffer
-
-		json.NewEncoder(&serializedMetadata).Encode(p.Info)
-
-		archiver.Publish(&archiver.Message{
-			Id:        p.Id,
-			Path:      p.Output.SavedFilePath,
-			Title:     p.Info.Title,
-			Thumbnail: p.Info.Thumbnail,
-			Source:    p.Url,
-			Metadata:  serializedMetadata.String(),
-			CreatedAt: p.Info.CreatedAt,
-		})
-	}
-
-	p.Progress = DownloadProgress{
-		Status:     StatusCompleted,
-		Percentage: "-1",
-		Speed:      0,
-		ETA:        0,
-	}
-
-	// for safety, if the filename is not set, set it with original function
-	if p.Output.SavedFilePath == "" {
-		p.GetFileName(&p.Output)
-	}
-
-	slog.Info("finished",
-		slog.String("id", p.getShortId()),
-		slog.String("url", p.Url),
-	)
-}
-
-// Kill a process and remove it from the memory
-func (p *Process) Kill() error {
-	defer func() {
-		p.Progress.Status = StatusCompleted
-	}()
-	// yt-dlp uses multiple child process the parent process
-	// has been spawned with setPgid = true. To properly kill
-	// all subprocesses a SIGTERM need to be sent to the correct
-	// process group
-	if p.proc == nil {
-		return errors.New("*os.Process not set")
-	}
-
-	pgid, err := syscall.Getpgid(p.proc.Pid)
-	if err != nil {
-		return err
-	}
-	if err := syscall.Kill(-pgid, syscall.SIGTERM); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (p *Process) GetFileName(o *DownloadOutput) error {
-	cmd := exec.Command(
-		config.Instance().DownloaderPath,
-		"--print", "filename",
-		"-o", fmt.Sprintf("%s/%s", o.Path, o.Filename),
-		p.Url,
-	)
-	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-
-	out, err := cmd.Output()
-	if err != nil {
-		return err
-	}
-
-	p.Output.SavedFilePath = strings.Trim(string(out), "\n")
-	return nil
-}
-
-func (p *Process) SetPending() {
-	// Since video's title isn't available yet, fill in with the URL.
-	p.Info = DownloadInfo{
-		URL:       p.Url,
-		Title:     p.Url,
-		CreatedAt: time.Now(),
-	}
-	p.Progress.Status = StatusPending
-}
-
-func (p *Process) SetMetadata() error {
-	cmd := exec.Command(config.Instance().DownloaderPath, p.Url, "-J")
-	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		slog.Error("failed to connect to stdout",
-			slog.String("id", p.getShortId()),
-			slog.String("url", p.Url),
-			slog.String("err", err.Error()),
-		)
-		return err
-	}
-
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		slog.Error("failed to connect to stderr",
-			slog.String("id", p.getShortId()),
-			slog.String("url", p.Url),
-			slog.String("err", err.Error()),
-		)
-		return err
-	}
-
-	info := DownloadInfo{
-		URL:       p.Url,
-		CreatedAt: time.Now(),
-	}
-
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-
-	var bufferedStderr bytes.Buffer
-
-	go func() {
-		io.Copy(&bufferedStderr, stderr)
-	}()
-
-	slog.Info("retrieving metadata",
-		slog.String("id", p.getShortId()),
-		slog.String("url", p.Url),
-	)
-
-	if err := json.NewDecoder(stdout).Decode(&info); err != nil {
-		return err
-	}
-
-	p.Info = info
-	p.Progress.Status = StatusPending
-
-	if err := cmd.Wait(); err != nil {
-		return errors.New(bufferedStderr.String())
-	}
-
-	return nil
-}
-
-func (p *Process) getShortId() string { return strings.Split(p.Id, "-")[0] }
-
-func buildFilename(o *DownloadOutput) {
-	if o.Filename != "" && strings.Contains(o.Filename, ".%(ext)s") {
-		o.Filename += ".%(ext)s"
-	}
-
-	o.Filename = strings.Replace(
-		o.Filename,
-		".%(ext)s.%(ext)s",
-		".%(ext)s",
-		1,
-	)
-}

server/internal/queue/message_queue.go

@@ -0,0 +1,123 @@
+package queue
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/metadata"
+)
+
+type MessageQueue struct {
+	concurrency   int
+	downloadQueue chan downloaders.Downloader
+	metadataQueue chan downloaders.Downloader
+	ctx           context.Context
+	cancel        context.CancelFunc
+}
+
+func NewMessageQueue() (*MessageQueue, error) {
+	qs := config.Instance().Server.QueueSize
+	if qs <= 0 {
+		return nil, errors.New("invalid queue size")
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	return &MessageQueue{
+		concurrency:   qs,
+		downloadQueue: make(chan downloaders.Downloader, qs*2),
+		metadataQueue: make(chan downloaders.Downloader, qs*4),
+		ctx:           ctx,
+		cancel:        cancel,
+	}, nil
+}
+
+// Publish download job
+func (m *MessageQueue) Publish(d downloaders.Downloader) {
+	d.SetPending(true)
+
+	select {
+	case m.downloadQueue <- d:
+		slog.Info("published download", slog.String("id", d.GetId()))
+	case <-m.ctx.Done():
+		slog.Warn("queue stopped, dropping download", slog.String("id", d.GetId()))
+	}
+}
+
+// Workers: download + metadata
+func (m *MessageQueue) SetupConsumers() {
+	// N parallel workers for downloadQueue
+	for i := 0; i < m.concurrency; i++ {
+		go m.downloadWorker(i)
+	}
+
+	// 1 serial worker for metadata
+	go m.metadataWorker()
+}
+
+// Worker dei download
+func (m *MessageQueue) downloadWorker(workerId int) {
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case p := <-m.downloadQueue:
+			if p == nil {
+				continue
+			}
+			if p.IsCompleted() {
+				continue
+			}
+
+			slog.Info("download worker started",
+				slog.Int("worker", workerId),
+				slog.String("id", p.GetId()),
+			)
+
+			p.Start()
+
+			// after the download starts succesfully we pass it to the metadata queue
+			select {
+			case m.metadataQueue <- p:
+				slog.Info("queued for metadata", slog.String("id", p.GetId()))
+			case <-m.ctx.Done():
+				return
+			}
+		}
+	}
+}
+
+func (m *MessageQueue) metadataWorker() {
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case p := <-m.metadataQueue:
+			if p == nil {
+				continue
+			}
+
+			slog.Info("metadata worker started",
+				slog.String("id", p.GetId()),
+			)
+
+			if p.IsCompleted() {
+				slog.Warn("metadata skipped, illegal state",
+					slog.String("id", p.GetId()),
+				)
+				continue
+			}
+
+			p.SetMetadata(metadata.DefaultFetcher)
+		}
+	}
+}
+
+func (m *MessageQueue) Stop() {
+	m.cancel()
+	close(m.downloadQueue)
+	close(m.metadataQueue)
+}

server/internal/worker.go

@@ -1,15 +0,0 @@
-package internal
-
-type Worker struct {
-	requests chan Process // downloads to do
-	pending  int          // downloads pending
-	index    int          // index in the heap
-}
-
-func (w *Worker) Work(done chan *Worker) {
-	for {
-		req := <-w.requests
-		req.Start()
-		done <- w
-	}
-}

server/logging/handler.go

@@ -91,10 +91,10 @@ func sse(logger *ObservableLogger) http.HandlerFunc {
 
 func ApplyRouter(logger *ObservableLogger) func(chi.Router) {
 	return func(r chi.Router) {
-		if config.Instance().RequireAuth {
+		if config.Instance().Authentication.RequireAuth {
 			r.Use(middlewares.Authenticated)
 		}
-		if config.Instance().UseOpenId {
+		if config.Instance().OpenId.UseOpenId {
 			r.Use(openid.Middleware)
 		}
 		r.Get("/ws", webSocket(logger))

server/middleware/utils.go

@@ -0,0 +1,21 @@
+package middlewares
+
+import (
+	"net/http"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/openid"
+)
+
+func ApplyAuthenticationByConfig(next http.Handler) http.Handler {
+	handler := next
+
+	if config.Instance().Authentication.RequireAuth {
+		handler = Authenticated(handler)
+	}
+	if config.Instance().OpenId.UseOpenId {
+		handler = openid.Middleware(handler)
+	}
+
+	return handler
+}

server/openid/config.go

@@ -14,24 +14,27 @@ var (
 )
 
 func Configure() {
-	if !config.Instance().UseOpenId {
+	if !config.Instance().OpenId.UseOpenId {
 		return
 	}
 
-	provider, err := oidc.NewProvider(context.Background(), config.Instance().OpenIdProviderURL)
+	provider, err := oidc.NewProvider(
+		context.Background(),
+		config.Instance().OpenId.ProviderURL,
+	)
 	if err != nil {
 		panic(err)
 	}
 
 	oauth2Config = oauth2.Config{
-		ClientID:     config.Instance().OpenIdClientId,
-		ClientSecret: config.Instance().OpenIdClientSecret,
-		RedirectURL:  config.Instance().OpenIdRedirectURL,
+		ClientID:     config.Instance().OpenId.ClientId,
+		ClientSecret: config.Instance().OpenId.ClientSecret,
+		RedirectURL:  config.Instance().OpenId.RedirectURL,
 		Endpoint:     provider.Endpoint(),
 		Scopes:       []string{oidc.ScopeOpenID, "profile", "email"},
 	}
 
 	verifier = provider.Verifier(&oidc.Config{
-		ClientID: config.Instance().OpenIdClientId,
+		ClientID: config.Instance().OpenId.ClientId,
 	})
 }

server/openid/handler.go

@@ -6,10 +6,12 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"slices"
 	"time"
 
 	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/google/uuid"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
 	"golang.org/x/oauth2"
 )
 
@@ -76,6 +78,21 @@ func doAuthentification(r *http.Request, setCookieCallback func(t *oauth2.Token)
 		return nil, err
 	}
 
+	var claims struct {
+		Email    string `json:"email"`
+		Verified bool   `json:"email_verified"`
+	}
+
+	if err := idToken.Claims(&claims); err != nil {
+		return nil, err
+	}
+
+	whitelist := config.Instance().OpenId.EmailWhitelist
+
+	if len(whitelist) > 0 && !slices.Contains(whitelist, claims.Email) {
+		return nil, errors.New("email address not found in ACL")
+	}
+
 	nonce, err := r.Cookie("nonce")
 	if err != nil {
 		return nil, err

server/playlist/modifiers.go

@@ -0,0 +1,86 @@
+package playlist
+
+import (
+	"slices"
+	"strconv"
+
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+)
+
+/*
+	Applicable modifiers
+
+				full									|		   short		  |					description
+	---------------------------------------------------------------------------------
+	--playlist-start NUMBER     |    -I NUMBER:	  |	  discard first N entries
+	--playlist-end NUMBER       |    -I :NUMBER   |   discard last N entries
+	--playlist-reverse          |    -I ::-1			|   self explanatory
+	--max-downloads NUMBER      |                 |   stops after N completed downloads
+*/
+
+func ApplyModifiers(entries *[]common.DownloadMetadata, args []string) error {
+	for i, modifier := range args {
+		switch modifier {
+		case "--playlist-start":
+			return playlistStart(i, modifier, args, entries)
+
+		case "--playlist-end":
+			return playlistEnd(i, modifier, args, entries)
+
+		case "--max-downloads":
+			return maxDownloads(i, modifier, args, entries)
+
+		case "--playlist-reverse":
+			slices.Reverse(*entries)
+			return nil
+		}
+	}
+	return nil
+}
+
+func playlistStart(i int, modifier string, args []string, entries *[]common.DownloadMetadata) error {
+	if !guard(i, len(modifier)) {
+		return nil
+	}
+
+	n, err := strconv.Atoi(args[i+1])
+	if err != nil {
+		return err
+	}
+
+	*entries = (*entries)[n:]
+
+	return nil
+}
+
+func playlistEnd(i int, modifier string, args []string, entries *[]common.DownloadMetadata) error {
+	if !guard(i, len(modifier)) {
+		return nil
+	}
+
+	n, err := strconv.Atoi(args[i+1])
+	if err != nil {
+		return err
+	}
+
+	*entries = (*entries)[:n]
+
+	return nil
+}
+
+func maxDownloads(i int, modifier string, args []string, entries *[]common.DownloadMetadata) error {
+	if !guard(i, len(modifier)) {
+		return nil
+	}
+
+	n, err := strconv.Atoi(args[i+1])
+	if err != nil {
+		return err
+	}
+
+	*entries = (*entries)[0:n]
+
+	return nil
+}
+
+func guard(i, len int) bool { return i+1 < len-1 }

server/playlist/playlist.go

@@ -1,4 +1,4 @@
-package internal
+package playlist
 
 import (
 	"encoding/json"
@@ -9,20 +9,21 @@ import (
 	"strings"
 	"time"
 
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
 )
 
-type metadata struct {
-	Entries       []DownloadInfo `json:"entries"`
-	Count         int            `json:"playlist_count"`
-	PlaylistTitle string         `json:"title"`
-	Type          string         `json:"_type"`
-}
+func PlaylistDetect(req internal.DownloadRequest, mq *queue.MessageQueue, db *kv.Store) error {
+	params := append(req.Params, "--flat-playlist", "-J")
+	urlWithParams := append([]string{req.URL}, params...)
 
-func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error {
 	var (
-		downloader = config.Instance().DownloaderPath
-		cmd        = exec.Command(downloader, req.URL, "--flat-playlist", "-J")
+		downloader = config.Instance().Paths.DownloaderPath
+		cmd        = exec.Command(downloader, urlWithParams...)
 	)
 
 	stdout, err := cmd.StdoutPipe()
@@ -30,7 +31,7 @@ func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error {
 		return err
 	}
 
-	var m metadata
+	var m Metadata
 
 	if err := cmd.Start(); err != nil {
 		return err
@@ -52,13 +53,21 @@ func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error {
 		return errors.New("probably not a valid URL")
 	}
 
-	if m.Type == "playlist" {
-		entries := slices.CompactFunc(slices.Compact(m.Entries), func(a DownloadInfo, b DownloadInfo) bool {
+	if m.IsPlaylist() {
+		entries := slices.CompactFunc(slices.Compact(m.Entries), func(a common.DownloadMetadata, b common.DownloadMetadata) bool {
 			return a.URL == b.URL
 		})
 
+		entries = slices.DeleteFunc(entries, func(e common.DownloadMetadata) bool {
+			return strings.Contains(e.URL, "list=")
+		})
+
 		slog.Info("playlist detected", slog.String("url", req.URL), slog.Int("count", len(entries)))
 
+		if err := ApplyModifiers(&entries, req.Params); err != nil {
+			return err
+		}
+
 		for i, meta := range entries {
 			// detect playlist title from metadata since each playlist entry will be
 			// treated as an individual download
@@ -72,31 +81,22 @@ func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error {
 			//XXX: it's idiotic but it works: virtually delay the creation time
 			meta.CreatedAt = time.Now().Add(time.Millisecond * time.Duration(i*10))
 
-			proc := &Process{
-				Url:      meta.URL,
-				Progress: DownloadProgress{},
-				Output:   DownloadOutput{Filename: req.Rename},
-				Info:     meta,
-				Params:   req.Params,
-			}
+			downloader := downloaders.NewGenericDownload(meta.URL, req.Params)
+			downloader.SetOutput(internal.DownloadOutput{Filename: req.Rename})
+			// downloader.SetMetadata(meta)
 
-			proc.Info.URL = meta.URL
-
-			time.Sleep(time.Millisecond)
-
-			db.Set(proc)
-			mq.Publish(proc)
+			db.Set(downloader)
+			mq.Publish(downloader)
 		}
+
+		return nil
 	}
 
-	proc := &Process{
-		Url:    req.URL,
-		Params: req.Params,
-	}
+	d := downloaders.NewGenericDownload(req.URL, req.Params)
 
-	db.Set(proc)
-	mq.Publish(proc)
-	slog.Info("sending new process to message queue", slog.String("url", proc.Url))
+	db.Set(d)
+	mq.Publish(d)
+	slog.Info("sending new process to message queue", slog.String("url", d.GetUrl()))
 
 	return cmd.Wait()
 }

server/playlist/types.go

@@ -0,0 +1,12 @@
+package playlist
+
+import "github.com/marcopiovanello/yt-dlp-web-ui/v3/server/common"
+
+type Metadata struct {
+	Entries       []common.DownloadMetadata `json:"entries"`
+	Count         int                       `json:"playlist_count"`
+	PlaylistTitle string                    `json:"title"`
+	Type          string                    `json:"_type"`
+}
+
+func (m *Metadata) IsPlaylist() bool { return m.Type == "playlist" }

server/rest/common.go

@@ -1,13 +1,16 @@
 package rest
 
 import (
-	"database/sql"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/livestream"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
 
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	bolt "go.etcd.io/bbolt"
 )
 
 type ContainerArgs struct {
-	DB  *sql.DB
-	MDB *internal.MemoryDB
-	MQ  *internal.MessageQueue
+	DB  *bolt.DB
+	MDB *kv.Store
+	MQ  *queue.MessageQueue
+	LM  *livestream.Monitor
 }

server/rest/container.go

@@ -19,10 +19,10 @@ func ApplyRouter(args *ContainerArgs) func(chi.Router) {
 	h := Container(args)
 
 	return func(r chi.Router) {
-		if config.Instance().RequireAuth {
+		if config.Instance().Authentication.RequireAuth {
 			r.Use(middlewares.Authenticated)
 		}
-		if config.Instance().UseOpenId {
+		if config.Instance().OpenId.UseOpenId {
 			r.Use(openid.Middleware)
 		}
 		r.Post("/exec", h.Exec())

server/rest/provider.go

@@ -14,11 +14,7 @@ var (
 
 func ProvideService(args *ContainerArgs) *Service {
 	serviceOnce.Do(func() {
-		service = &Service{
-			mdb: args.MDB,
-			db:  args.DB,
-			mq:  args.MQ,
-		}
+		service = NewService(args.MDB, args.DB, args.MQ, args.LM)
 	})
 	return service
 }

server/rest/service.go

@@ -2,8 +2,9 @@ package rest
 
 import (
 	"context"
-	"database/sql"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -12,41 +13,62 @@ import (
 	"github.com/google/uuid"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/livestream"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/playlist"
+
+	bolt "go.etcd.io/bbolt"
 )
 
 type Service struct {
-	mdb *internal.MemoryDB
-	db  *sql.DB
-	mq  *internal.MessageQueue
+	mdb *kv.Store
+	db  *bolt.DB
+	mq  *queue.MessageQueue
 	lm  *livestream.Monitor
 }
 
-func (s *Service) Exec(req internal.DownloadRequest) (string, error) {
-	p := &internal.Process{
-		Url:    req.URL,
-		Params: req.Params,
-		Output: internal.DownloadOutput{
-			Path:     req.Path,
-			Filename: req.Rename,
-		},
+func NewService(
+	mdb *kv.Store,
+	db *bolt.DB,
+	mq *queue.MessageQueue,
+	lm *livestream.Monitor,
+) *Service {
+	db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte("templates"))
+		return err
+	})
+	return &Service{
+		mdb: mdb,
+		db:  db,
+		mq:  mq,
+		lm:  lm,
 	}
+}
 
-	id := s.mdb.Set(p)
-	s.mq.Publish(p)
+func (s *Service) Exec(req internal.DownloadRequest) (string, error) {
+	d := downloaders.NewGenericDownload(req.URL, req.Params)
+	d.SetOutput(internal.DownloadOutput{
+		Path:     req.Path,
+		Filename: req.Rename,
+	})
+
+	id := s.mdb.Set(d)
+	s.mq.Publish(d)
 
 	return id, nil
 }
 
 func (s *Service) ExecPlaylist(req internal.DownloadRequest) error {
-	return internal.PlaylistDetect(req, s.mq, s.mdb)
+	return playlist.PlaylistDetect(req, s.mq, s.mdb)
 }
 
 func (s *Service) ExecLivestream(req internal.DownloadRequest) {
 	s.lm.Add(req.URL)
 }
 
-func (s *Service) Running(ctx context.Context) (*[]internal.ProcessResponse, error) {
+func (s *Service) Running(ctx context.Context) (*[]internal.ProcessSnapshot, error) {
 	select {
 	case <-ctx.Done():
 		return nil, context.Canceled
@@ -84,64 +106,56 @@ func (s *Service) SetCookies(ctx context.Context, cookies string) error {
 }
 
 func (s *Service) SaveTemplate(ctx context.Context, template *internal.CustomTemplate) error {
-	conn, err := s.db.Conn(ctx)
-	if err != nil {
-		return err
-	}
-
-	defer conn.Close()
-
-	_, err = conn.ExecContext(
-		ctx,
-		"INSERT INTO templates (id, name, content) VALUES (?, ?, ?)",
-		uuid.NewString(),
-		template.Name,
-		template.Content,
-	)
-
-	return err
+	return s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte("templates"))
+		v, err := json.Marshal(template)
+		if err != nil {
+			return err
+		}
+		return b.Put([]byte(uuid.NewString()), v)
+	})
 }
 
 func (s *Service) GetTemplates(ctx context.Context) (*[]internal.CustomTemplate, error) {
-	conn, err := s.db.Conn(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	defer conn.Close()
-
-	rows, err := conn.QueryContext(ctx, "SELECT * FROM templates")
-	if err != nil {
-		return nil, err
-	}
-
-	defer rows.Close()
-
 	templates := make([]internal.CustomTemplate, 0)
 
-	for rows.Next() {
-		t := internal.CustomTemplate{}
-
-		err := rows.Scan(&t.Id, &t.Name, &t.Content)
-		if err != nil {
-			return nil, err
+	err := s.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte("templates"))
+		if b == nil {
+			return nil // bucket vuoto, restituisco lista vuota
 		}
 
-		templates = append(templates, t)
+		return b.ForEach(func(k, v []byte) error {
+			var t internal.CustomTemplate
+			if err := json.Unmarshal(v, &t); err != nil {
+				return err
+			}
+			templates = append(templates, t)
+			return nil
+		})
+	})
+
+	if err != nil {
+		return nil, err
 	}
 
 	return &templates, nil
 }
 
 func (s *Service) UpdateTemplate(ctx context.Context, t *internal.CustomTemplate) (*internal.CustomTemplate, error) {
-	conn, err := s.db.Conn(ctx)
+	data, err := json.Marshal(t)
 	if err != nil {
 		return nil, err
 	}
 
-	defer conn.Close()
+	err = s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte("templates"))
+		if b == nil {
+			return fmt.Errorf("bucket templates not found")
+		}
+		return b.Put([]byte(t.Id), data)
+	})
 
-	_, err = conn.ExecContext(ctx, "UPDATE templates SET name = ?, content = ? WHERE id = ?", t.Name, t.Content, t.Id)
 	if err != nil {
 		return nil, err
 	}
@@ -150,28 +164,22 @@ func (s *Service) UpdateTemplate(ctx context.Context, t *internal.CustomTemplate
 }
 
 func (s *Service) DeleteTemplate(ctx context.Context, id string) error {
-	conn, err := s.db.Conn(ctx)
-	if err != nil {
-		return err
-	}
-
-	defer conn.Close()
-
-	_, err = conn.ExecContext(ctx, "DELETE FROM templates WHERE id = ?", id)
-
-	return err
+	return s.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte("templates"))
+		return b.Delete([]byte(id))
+	})
 }
 
 func (s *Service) GetVersion(ctx context.Context) (string, string, error) {
 	//TODO: load from realease properties file, or anything else outside code
-	const CURRENT_RPC_VERSION = "3.2.3"
+	const CURRENT_RPC_VERSION = "3.2.6"
 
 	result := make(chan string, 1)
 
 	ctx, cancel := context.WithTimeout(ctx, time.Second*10)
 	defer cancel()
 
-	cmd := exec.CommandContext(ctx, config.Instance().DownloaderPath, "--version")
+	cmd := exec.CommandContext(ctx, config.Instance().Paths.DownloaderPath, "--version")
 	go func() {
 		stdout, _ := cmd.Output()
 		result <- string(stdout)

server/rpc/container.go

@@ -3,14 +3,15 @@ package rpc
 import (
 	"github.com/go-chi/chi/v5"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/livestream"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
 	middlewares "github.com/marcopiovanello/yt-dlp-web-ui/v3/server/middleware"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/openid"
 )
 
 // Dependency injection container.
-func Container(db *internal.MemoryDB, mq *internal.MessageQueue, lm *livestream.Monitor) *Service {
+func Container(db *kv.Store, mq *queue.MessageQueue, lm *livestream.Monitor) *Service {
 	return &Service{
 		db: db,
 		mq: mq,
@@ -21,10 +22,10 @@ func Container(db *internal.MemoryDB, mq *internal.MessageQueue, lm *livestream.
 // RPC service must be registered before applying this router!
 func ApplyRouter() func(chi.Router) {
 	return func(r chi.Router) {
-		if config.Instance().RequireAuth {
+		if config.Instance().Authentication.RequireAuth {
 			r.Use(middlewares.Authenticated)
 		}
-		if config.Instance().UseOpenId {
+		if config.Instance().OpenId.UseOpenId {
 			r.Use(openid.Middleware)
 		}
 		r.Get("/ws", WebSocket)

server/rpc/service.go

@@ -6,18 +6,22 @@ import (
 
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/formats"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/downloaders"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/livestream"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/playlist"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/sys"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/updater"
 )
 
 type Service struct {
-	db *internal.MemoryDB
-	mq *internal.MessageQueue
+	db *kv.Store
+	mq *queue.MessageQueue
 	lm *livestream.Monitor
 }
 
-type Running []internal.ProcessResponse
+type Running []internal.ProcessSnapshot
 type Pending []string
 
 type NoArgs struct{}
@@ -25,26 +29,23 @@ type NoArgs struct{}
 // Exec spawns a Process.
 // The result of the execution is the newly spawned process Id.
 func (s *Service) Exec(args internal.DownloadRequest, result *string) error {
-	p := &internal.Process{
-		Url:    args.URL,
-		Params: args.Params,
-		Output: internal.DownloadOutput{
-			Path:     args.Path,
-			Filename: args.Rename,
-		},
-	}
+	d := downloaders.NewGenericDownload(args.URL, args.Params)
+	d.SetOutput(internal.DownloadOutput{
+		Path:     args.Path,
+		Filename: args.Rename,
+	})
 
-	s.db.Set(p)
-	s.mq.Publish(p)
+	s.db.Set(d)
+	s.mq.Publish(d)
 
-	*result = p.Id
+	*result = d.GetId()
 	return nil
 }
 
 // Exec spawns a Process.
 // The result of the execution is the newly spawned process Id.
 func (s *Service) ExecPlaylist(args internal.DownloadRequest, result *string) error {
-	err := internal.PlaylistDetect(args, s.mq, s.db)
+	err := playlist.PlaylistDetect(args, s.mq, s.db)
 	if err != nil {
 		return err
 	}
@@ -87,12 +88,12 @@ func (s *Service) KillAllLivestream(args NoArgs, result *struct{}) error {
 
 // Progess retrieves the Progress of a specific Process given its Id
 func (s *Service) Progess(args internal.DownloadRequest, progress *internal.DownloadProgress) error {
-	proc, err := s.db.Get(args.Id)
+	dl, err := s.db.Get(args.Id)
 	if err != nil {
 		return err
 	}
 
-	*progress = proc.Progress
+	*progress = dl.Status().Progress
 	return nil
 }
 
@@ -106,7 +107,7 @@ func (s *Service) Formats(args internal.DownloadRequest, meta *formats.Metadata)
 	}
 
 	if metadata.IsPlaylist() {
-		go internal.PlaylistDetect(args, s.mq, s.db)
+		go playlist.PlaylistDetect(args, s.mq, s.db)
 	}
 
 	*meta = *metadata
@@ -129,22 +130,22 @@ func (s *Service) Running(args NoArgs, running *Running) error {
 func (s *Service) Kill(args string, killed *string) error {
 	slog.Info("Trying killing process with id", slog.String("id", args))
 
-	proc, err := s.db.Get(args)
+	download, err := s.db.Get(args)
 	if err != nil {
 		return err
 	}
 
-	if proc == nil {
+	if download == nil {
 		return errors.New("nil process")
 	}
 
-	if err := proc.Kill(); err != nil {
-		slog.Info("failed killing process", slog.String("id", proc.Id), slog.Any("err", err))
+	if err := download.Stop(); err != nil {
+		slog.Info("failed killing process", slog.String("id", download.GetId()), slog.Any("err", err))
 		return err
 	}
 
-	s.db.Delete(proc.Id)
-	slog.Info("succesfully killed process", slog.String("id", proc.Id))
+	s.db.Delete(download.GetId())
+	slog.Info("succesfully killed process", slog.String("id", download.GetId()))
 
 	return nil
 }
@@ -156,33 +157,33 @@ func (s *Service) KillAll(args NoArgs, killed *string) error {
 
 	var (
 		keys       = s.db.Keys()
-		removeFunc = func(p *internal.Process) error {
-			defer s.db.Delete(p.Id)
-			return p.Kill()
+		removeFunc = func(d downloaders.Downloader) error {
+			defer s.db.Delete(d.GetId())
+			return d.Stop()
 		}
 	)
 
 	for _, key := range *keys {
-		proc, err := s.db.Get(key)
+		dl, err := s.db.Get(key)
 		if err != nil {
 			return err
 		}
 
-		if proc == nil {
+		if dl == nil {
 			s.db.Delete(key)
 			continue
 		}
 
-		if err := removeFunc(proc); err != nil {
+		if err := removeFunc(dl); err != nil {
 			slog.Info(
 				"failed killing process",
-				slog.String("id", proc.Id),
+				slog.String("id", dl.GetId()),
 				slog.Any("err", err),
 			)
 			continue
 		}
 
-		slog.Info("succesfully killed process", slog.String("id", proc.Id))
+		slog.Info("succesfully killed process", slog.String("id", dl.GetId()))
 	}
 
 	return nil
@@ -195,6 +196,35 @@ func (s *Service) Clear(args string, killed *string) error {
 	return nil
 }
 
+// Removes completed processes
+func (s *Service) ClearCompleted(cleared *string) error {
+	var (
+		keys       = s.db.Keys()
+		removeFunc = func(d downloaders.Downloader) error {
+			defer s.db.Delete(d.GetId())
+
+			if !d.IsCompleted() {
+				return nil
+			}
+
+			return d.Stop()
+		}
+	)
+
+	for _, key := range *keys {
+		proc, err := s.db.Get(key)
+		if err != nil {
+			return err
+		}
+
+		if err := removeFunc(proc); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // FreeSpace gets the available from package sys util
 func (s *Service) FreeSpace(args NoArgs, free *uint64) error {
 	freeSpace, err := sys.FreeSpace()

server/server.go

@@ -3,7 +3,6 @@ package server
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"io"
 	"io/fs"
@@ -12,28 +11,30 @@ import (
 	"net/http"
 	"net/rpc"
 	"os"
-	"os/signal"
+	"path/filepath"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/cors"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/archive"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/archiver"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/dbutil"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/handlers"
-	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/filebrowser"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/livestream"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/pipeline"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/queue"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/logging"
 	middlewares "github.com/marcopiovanello/yt-dlp-web-ui/v3/server/middleware"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/openid"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/rest"
 	ytdlpRPC "github.com/marcopiovanello/yt-dlp-web-ui/v3/server/rpc"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/status"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/subscription"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/subscription/task"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/twitch"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/user"
 
-	_ "modernc.org/sqlite"
+	bolt "go.etcd.io/bbolt"
 )
 
 type RunConfig struct {
@@ -42,19 +43,31 @@ type RunConfig struct {
 }
 
 type serverConfig struct {
-	frontend fs.FS
-	swagger  fs.FS
-	mdb      *internal.MemoryDB
-	db       *sql.DB
-	mq       *internal.MessageQueue
-	lm       *livestream.Monitor
+	frontend      fs.FS
+	swagger       fs.FS
+	mdb           *kv.Store
+	db            *bolt.DB
+	mq            *queue.MessageQueue
+	lm            *livestream.Monitor
+	taskRunner    task.TaskRunner
+	twitchMonitor *twitch.Monitor
 }
 
 // TODO: change scope
 var observableLogger = logging.NewObservableLogger()
 
-func RunBlocking(rc *RunConfig) {
-	mdb := internal.NewMemoryDB()
+func Run(ctx context.Context, rc *RunConfig) error {
+	dbPath := filepath.Join(config.Instance().Paths.LocalDatabasePath, "bolt.db")
+
+	boltdb, err := bolt.Open(dbPath, 0600, nil)
+	if err != nil {
+		return err
+	}
+
+	mdb, err := kv.NewStore(boltdb, time.Second*15)
+	if err != nil {
+		return err
+	}
 
 	// ---- LOGGING ---------------------------------------------------
 	logWriters := []io.Writer{
@@ -65,10 +78,10 @@ func RunBlocking(rc *RunConfig) {
 	conf := config.Instance()
 
 	// file based logging
-	if conf.EnableFileLogging {
-		logger, err := logging.NewRotableLogger(conf.LogPath)
+	if conf.Logging.EnableFileLogging {
+		logger, err := logging.NewRotableLogger(conf.Logging.LogPath)
 		if err != nil {
-			panic(err)
+			return err
 		}
 
 		defer logger.Rotate()
@@ -91,53 +104,65 @@ func RunBlocking(rc *RunConfig) {
 	slog.SetDefault(logger)
 	// ----------------------------------------------------------------
 
-	db, err := sql.Open("sqlite", conf.LocalDatabasePath)
+	mq, err := queue.NewMessageQueue()
 	if err != nil {
-		slog.Error("failed to open database", slog.String("err", err.Error()))
-	}
-
-	if err := dbutil.Migrate(context.Background(), db); err != nil {
-		slog.Error("failed to init database", slog.String("err", err.Error()))
-	}
-
-	mq, err := internal.NewMessageQueue()
-	if err != nil {
-		panic(err)
+		return err
 	}
 	mq.SetupConsumers()
 	go mdb.Restore(mq)
+	go mdb.EventListener()
 
-	lm := livestream.NewMonitor(mq, mdb)
+	lm := livestream.NewMonitor(mq, mdb, boltdb)
 	go lm.Schedule()
 	go lm.Restore()
 
-	srv := newServer(serverConfig{
-		frontend: rc.App,
-		swagger:  rc.Swagger,
-		mdb:      mdb,
-		mq:       mq,
-		db:       db,
-		lm:       lm,
-	})
+	tm := twitch.NewMonitor(
+		twitch.NewAuthenticationManager(
+			config.Instance().Twitch.ClientId,
+			config.Instance().Twitch.ClientSecret,
+		),
+		boltdb,
+	)
+	go tm.Monitor(
+		ctx,
+		config.Instance().Twitch.CheckInterval,
+		twitch.DEFAULT_DOWNLOAD_HANDLER(mdb, mq),
+	)
+	go tm.Restore()
 
-	go gracefulShutdown(srv, mdb)
-	go autoPersist(time.Minute*5, mdb, lm)
+	cronTaskRunner := task.NewCronTaskRunner(mq, mdb)
+	go cronTaskRunner.Spawner(ctx)
+
+	scfg := serverConfig{
+		frontend:      rc.App,
+		swagger:       rc.Swagger,
+		mdb:           mdb,
+		db:            boltdb,
+		mq:            mq,
+		lm:            lm,
+		twitchMonitor: tm,
+		taskRunner:    cronTaskRunner,
+	}
+
+	srv := newServer(scfg)
+
+	go gracefulShutdown(ctx, srv, &scfg)
 
 	var (
 		network = "tcp"
-		address = fmt.Sprintf("%s:%d", conf.Host, conf.Port)
+		address = fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port)
 	)
 
 	// support unix sockets
-	if strings.HasPrefix(conf.Host, "/") {
+	if strings.HasPrefix(conf.Server.Host, "/") {
 		network = "unix"
-		address = conf.Host
+		address = conf.Server.Host
 	}
 
 	listener, err := net.Listen(network, address)
 	if err != nil {
 		slog.Error("failed to listen", slog.String("err", err.Error()))
-		return
+		return err
 	}
 
 	slog.Info("yt-dlp-webui started", slog.String("address", address))
@@ -145,11 +170,12 @@ func RunBlocking(rc *RunConfig) {
 	if err := srv.Serve(listener); err != nil {
 		slog.Warn("http server stopped", slog.String("err", err.Error()))
 	}
+
+	return nil
 }
 
 func newServer(c serverConfig) *http.Server {
-	archiver.Register(c.db)
-
+	// archiver.Register(c.db)
 	service := ytdlpRPC.Container(c.mdb, c.mq, c.lm)
 	rpc.Register(service)
 
@@ -173,7 +199,7 @@ func newServer(c serverConfig) *http.Server {
 	// use in dev
 	// r.Use(middleware.Logger)
 
-	baseUrl := config.Instance().BaseURL
+	baseUrl := config.Instance().Server.BaseURL
 	r.Mount(baseUrl+"/", http.StripPrefix(baseUrl, http.FileServerFS(c.frontend)))
 
 	// swagger
@@ -181,26 +207,21 @@ func newServer(c serverConfig) *http.Server {
 
 	// Filebrowser routes
 	r.Route("/filebrowser", func(r chi.Router) {
-		if config.Instance().RequireAuth {
-			r.Use(middlewares.Authenticated)
-		}
-		if config.Instance().UseOpenId {
-			r.Use(openid.Middleware)
-		}
-		r.Post("/downloaded", handlers.ListDownloaded)
-		r.Post("/delete", handlers.DeleteFile)
-		r.Get("/d/{id}", handlers.DownloadFile)
-		r.Get("/v/{id}", handlers.SendFile)
-		r.Get("/bulk", handlers.BulkDownload(c.mdb))
+		r.Use(middlewares.ApplyAuthenticationByConfig)
+		r.Post("/downloaded", filebrowser.ListDownloaded)
+		r.Post("/delete", filebrowser.DeleteFile)
+		r.Get("/d/{id}", filebrowser.DownloadFile)
+		r.Get("/v/{id}", filebrowser.SendFile)
+		r.Get("/bulk", filebrowser.BulkDownload(c.mdb))
 	})
 
 	// Archive routes
-	r.Route("/archive", archive.ApplyRouter(c.db))
+	// r.Route("/archive", archive.ApplyRouter(c.db))
 
 	// Authentication routes
 	r.Route("/auth", func(r chi.Router) {
-		r.Post("/login", handlers.Login)
-		r.Get("/logout", handlers.Logout)
+		r.Post("/login", user.Login)
+		r.Get("/logout", user.Logout)
 
 		r.Route("/openid", func(r chi.Router) {
 			r.Get("/login", openid.Login)
@@ -217,6 +238,7 @@ func newServer(c serverConfig) *http.Server {
 		DB:  c.db,
 		MDB: c.mdb,
 		MQ:  c.mq,
+		LM:  c.lm,
 	}))
 
 	// Logging
@@ -225,39 +247,36 @@ func newServer(c serverConfig) *http.Server {
 	// Status
 	r.Route("/status", status.ApplyRouter(c.mdb))
 
+	// Subscriptions
+	r.Route("/subscriptions", subscription.Container(c.db, c.taskRunner).ApplyRouter())
+
+	// Twitch
+	r.Route("/twitch", func(r chi.Router) {
+		r.Use(middlewares.ApplyAuthenticationByConfig)
+		r.Get("/users", twitch.GetMonitoredUsers(c.twitchMonitor))
+		r.Post("/user", twitch.MonitorUserHandler(c.twitchMonitor))
+		r.Delete("/user/{user}", twitch.DeleteUser(c.twitchMonitor))
+	})
+
+	// Pipelines
+	r.Route("/pipelines", func(r chi.Router) {
+		h := pipeline.NewRestHandler(c.db)
+		r.Use(middlewares.ApplyAuthenticationByConfig)
+		r.Get("/id/{id}", h.GetPipeline)
+		r.Get("/all", h.GetAllPipelines)
+		r.Post("/", h.SavePipeline)
+		r.Delete("/id/{id}", h.DeletePipeline)
+	})
+
 	return &http.Server{Handler: r}
 }
 
-func gracefulShutdown(srv *http.Server, db *internal.MemoryDB) {
-	ctx, stop := signal.NotifyContext(context.Background(),
-		os.Interrupt,
-		syscall.SIGTERM,
-		syscall.SIGQUIT,
-	)
+func gracefulShutdown(ctx context.Context, srv *http.Server, cfg *serverConfig) {
+	<-ctx.Done()
+	slog.Info("shutdown signal received")
 
-	go func() {
-		<-ctx.Done()
-		slog.Info("shutdown signal received")
-
-		defer func() {
-			db.Persist()
-
-			stop()
-			srv.Shutdown(context.Background())
-		}()
+	defer func() {
+		cfg.db.Close()
+		srv.Shutdown(context.Background())
 	}()
 }
-
-func autoPersist(d time.Duration, db *internal.MemoryDB, lm *livestream.Monitor) {
-	for {
-		if err := db.Persist(); err != nil {
-			slog.Warn("failed to persisted session", slog.Any("err", err))
-		}
-		if err := lm.Persist(); err != nil {
-			slog.Warn(
-				"failed to persisted livestreams monitor session", slog.Any("err", err.Error()))
-		}
-		slog.Debug("sucessfully persisted session")
-		time.Sleep(d)
-	}
-}

server/status/repository/repository.go

@@ -5,11 +5,12 @@ import (
 	"slices"
 
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
+	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal/kv"
 	"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/status/domain"
 )
 
 type Repository struct {
-	mdb *internal.MemoryDB
+	mdb *kv.Store
 }
 
 // DownloadSpeed implements domain.Repository.
@@ -29,7 +30,7 @@ func (r *Repository) DownloadSpeed(ctx context.Context) int64 {
 func (r *Repository) Completed(ctx context.Context) int {
 	processes := r.mdb.All()
 
-	completed := slices.DeleteFunc(*processes, func(p internal.ProcessResponse) bool {
+	completed := slices.DeleteFunc(*processes, func(p internal.ProcessSnapshot) bool {
 		return p.Progress.Status != internal.StatusCompleted
 	})
 
@@ -40,7 +41,7 @@ func (r *Repository) Completed(ctx context.Context) int {
 func (r *Repository) Downloading(ctx context.Context) int {
 	processes := r.mdb.All()
 
-	downloading := slices.DeleteFunc(*processes, func(p internal.ProcessResponse) bool {
+	downloading := slices.DeleteFunc(*processes, func(p internal.ProcessSnapshot) bool {
 		return p.Progress.Status != internal.StatusDownloading
 	})
 
@@ -51,14 +52,14 @@ func (r *Repository) Downloading(ctx context.Context) int {
 func (r *Repository) Pending(ctx context.Context) int {
 	processes := r.mdb.All()
 
-	pending := slices.DeleteFunc(*processes, func(p internal.ProcessResponse) bool {
+	pending := slices.DeleteFunc(*processes, func(p internal.ProcessSnapshot) bool {
 		return p.Progress.Status != internal.StatusPending
 	})
 
 	return len(pending)
 }
 
-func New(mdb *internal.MemoryDB) domain.Repository {
+func New(mdb *kv.Store) domain.Repository {
 	return &Repository{
 		mdb: mdb,
 	}