Compare commits

...

12 Commits

Author SHA1 Message Date
Marco Piovanello
f4a0f688af Feat twitch livestreams (#334)
* backend code

* fixed twitch authentication
2025-08-25 12:54:16 +02:00
Marco Piovanello
14a03d6a77 Prevent RCEs with crafted inputs 2025-07-23 10:21:34 +02:00
Marco Piovanello
8a73079fad Update Dockerfile 2025-04-13 20:13:59 +02:00
f578f44cfd refactor: prevent multiple slashes 2025-03-30 10:29:13 +02:00
cbe16c5c6c refactoring: readded abort controller to httpClient.ts 2025-03-30 10:21:19 +02:00
3cebaf7f61 refactor: extra slashes prevention 2025-03-30 10:17:30 +02:00
Marco Piovanello
2d2cb1dc3a Update README.md 2025-03-30 09:54:27 +02:00
Marco Piovanello
43bcc40907 293 tiny gui improvement (#296)
* clicking on the speed dial will open download dialog

* refactor: prevent multiple slashes
2025-03-29 21:27:28 +01:00
Marco Piovanello
2af27e51be Chore dockerfile refactor (#287)
* removed yt-dlp alpine package

* use python3-alpine base image
2025-03-22 16:17:25 +01:00
Marco Piovanello
8c18242aaf removed yt-dlp alpine package (#286) 2025-03-22 15:27:48 +01:00
Marco Piovanello
66bebb2529 Update README.md 2025-03-17 11:23:29 +01:00
Marco Piovanello
e223e030ac restrict user with a whitelist (#282) 2025-03-17 11:13:20 +01:00
16 changed files with 517 additions and 49 deletions

1
.gitignore vendored
View File

@@ -29,3 +29,4 @@ frontend/.yarn/install-state.gz
livestreams.dat
.vite/deps
archive.txt
twitch-monitor.dat

View File

@@ -24,11 +24,12 @@ COPY --from=ui /usr/src/yt-dlp-webui/frontend /usr/src/yt-dlp-webui/frontend
RUN CGO_ENABLED=0 GOOS=linux go build -o yt-dlp-webui
# -----------------------------------------------------------------------------
# dependencies ----------------------------------------------------------------
FROM alpine:edge
# Runtime ---------------------------------------------------------------------
FROM python:3.13.2-alpine3.21
RUN apk update && \
apk add ffmpeg yt-dlp ca-certificates curl wget psmisc
apk add ffmpeg ca-certificates curl wget gnutls --no-cache && \
pip install "yt-dlp[default,curl-cffi,mutagen,pycryptodomex,phantomjs,secretstorage]"
VOLUME /downloads /config
@@ -39,4 +40,4 @@ COPY --from=build /usr/src/yt-dlp-webui/yt-dlp-webui /app
ENV JWT_SECRET=secret
EXPOSE 3033
ENTRYPOINT [ "./yt-dlp-webui" , "--out", "/downloads", "--conf", "/config/config.yml", "--db", "/config/local.db" ]
ENTRYPOINT [ "./yt-dlp-webui" , "--out", "/downloads", "--conf", "/config/config.yml", "--db", "/config/local.db" ]

View File

@@ -28,7 +28,7 @@ docker pull ghcr.io/marcopiovanello/yt-dlp-web-ui:latest
## Community stuff
Feel free to join :)
[![Discord Banner](https://api.weblutions.com/discord/invite/3Sj9ZZHv/)](https://discord.gg/WRnVWr4y)
[Discord](https://discord.gg/GZAX5FfGzE)
## Some screeshots
![image](https://github.com/user-attachments/assets/fc43a3fb-ecf9-449d-b5cb-5d5635020c00)

View File

@@ -121,14 +121,18 @@ export const appTitleState = atomWithStorage(
export const serverAddressAndPortState = atom((get) => {
if (get(servedFromReverseProxySubDirState)) {
return `${get(serverAddressState)}/${get(servedFromReverseProxySubDirState)}/`
.replaceAll('"', '') // TODO: atomWithStorage put extra double quotes on strings
.replaceAll('"', '') // XXX: atomWithStorage uses JSON.stringify to serialize
.replaceAll('//', '/') // which puts extra double quotes.
}
if (get(servedFromReverseProxyState)) {
return `${get(serverAddressState)}`
.replaceAll('"', '')
}
return `${get(serverAddressState)}:${get(serverPortState)}`
const sap = `${get(serverAddressState)}:${get(serverPortState)}`
.replaceAll('"', '')
return sap.endsWith('/') ? sap.slice(0, -1) : sap
})
export const serverURL = atom((get) =>
@@ -137,12 +141,16 @@ export const serverURL = atom((get) =>
export const rpcWebSocketEndpoint = atom((get) => {
const proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
return `${proto}//${get(serverAddressAndPortState)}/rpc/ws`
const sap = get(serverAddressAndPortState)
return `${proto}//${sap.endsWith('/') ? sap.slice(0, -1) : sap}/rpc/ws`
})
export const rpcHTTPEndpoint = atom((get) => {
const proto = window.location.protocol
return `${proto}//${get(serverAddressAndPortState)}/rpc/http`
const sap = get(serverAddressAndPortState)
return `${proto}//${sap.endsWith('/') ? sap.slice(0, -1) : sap}/rpc/http`
})
export const serverSideCookiesState = atom<Promise<string>>(async (get) => await pipe(

View File

@@ -32,6 +32,7 @@ const HomeSpeedDial: React.FC<Props> = ({ onDownloadOpen, onEditorOpen }) => {
ariaLabel="Home speed dial"
sx={{ position: 'absolute', bottom: 64, right: 24 }}
icon={<SpeedDialIcon />}
onClick={onDownloadOpen}
>
<SpeedDialAction
icon={listView ? <ViewAgendaIcon /> : <FormatListBulleted />}

View File

@@ -1,6 +1,9 @@
import { tryCatch } from 'fp-ts/TaskEither'
import * as J from 'fp-ts/Json'
import * as E from 'fp-ts/Either'
import { pipe } from 'fp-ts/lib/function'
async function fetcher<T>(url: string, opt?: RequestInit): Promise<T> {
async function fetcher(url: string, opt?: RequestInit, controller?: AbortController): Promise<string> {
const jwt = localStorage.getItem('token')
if (opt && !opt.headers) {
@@ -14,17 +17,27 @@ async function fetcher<T>(url: string, opt?: RequestInit): Promise<T> {
headers: {
...opt?.headers,
'X-Authentication': jwt ?? ''
}
},
signal: controller?.signal
})
if (!res.ok) {
throw await res.text()
}
return res.json() as T
return res.text()
}
export const ffetch = <T>(url: string, opt?: RequestInit) => tryCatch(
() => fetcher<T>(url, opt),
export const ffetch = <T>(url: string, opt?: RequestInit, controller?: AbortController) => tryCatch(
async () => pipe(
await fetcher(url, opt, controller),
J.parse,
E.match(
(l) => l as T,
(r) => r as T
)
),
(e) => `error while fetching: ${e}`
)

View File

@@ -4,32 +4,39 @@ import (
"os"
"path/filepath"
"sync"
"time"
"gopkg.in/yaml.v3"
)
type Config struct {
LogPath string `yaml:"log_path"`
EnableFileLogging bool `yaml:"enable_file_logging"`
BaseURL string `yaml:"base_url"`
Host string `yaml:"host"`
Port int `yaml:"port"`
DownloadPath string `yaml:"downloadPath"`
DownloaderPath string `yaml:"downloaderPath"`
RequireAuth bool `yaml:"require_auth"`
Username string `yaml:"username"`
Password string `yaml:"password"`
QueueSize int `yaml:"queue_size"`
LocalDatabasePath string `yaml:"local_database_path"`
SessionFilePath string `yaml:"session_file_path"`
path string // private
UseOpenId bool `yaml:"use_openid"`
OpenIdProviderURL string `yaml:"openid_provider_url"`
OpenIdClientId string `yaml:"openid_client_id"`
OpenIdClientSecret string `yaml:"openid_client_secret"`
OpenIdRedirectURL string `yaml:"openid_redirect_url"`
FrontendPath string `yaml:"frontend_path"`
AutoArchive bool `yaml:"auto_archive"`
LogPath string `yaml:"log_path"`
EnableFileLogging bool `yaml:"enable_file_logging"`
BaseURL string `yaml:"base_url"`
Host string `yaml:"host"`
Port int `yaml:"port"`
DownloadPath string `yaml:"downloadPath"`
DownloaderPath string `yaml:"downloaderPath"`
RequireAuth bool `yaml:"require_auth"`
Username string `yaml:"username"`
Password string `yaml:"password"`
QueueSize int `yaml:"queue_size"`
LocalDatabasePath string `yaml:"local_database_path"`
SessionFilePath string `yaml:"session_file_path"`
path string // private
UseOpenId bool `yaml:"use_openid"`
OpenIdProviderURL string `yaml:"openid_provider_url"`
OpenIdClientId string `yaml:"openid_client_id"`
OpenIdClientSecret string `yaml:"openid_client_secret"`
OpenIdRedirectURL string `yaml:"openid_redirect_url"`
OpenIdEmailWhitelist []string `yaml:"openid_email_whitelist"`
FrontendPath string `yaml:"frontend_path"`
AutoArchive bool `yaml:"auto_archive"`
Twitch struct {
ClientId string `yaml:"client_id"`
ClientSecret string `yaml:"client_secret"`
CheckInterval time.Duration `yaml:"check_interval"`
} `yaml:"twitch"`
}
var (

View File

@@ -100,6 +100,7 @@ func (p *Process) Start() {
templateReplacer.Replace(downloadTemplate),
"--progress-template",
templateReplacer.Replace(postprocessTemplate),
"--no-exec",
}
// if user asked to manually override the output path...

View File

@@ -0,0 +1,20 @@
package middlewares
import (
"net/http"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/openid"
)
func ApplyAuthenticationByConfig(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if config.Instance().RequireAuth {
Authenticated(next)
}
if config.Instance().UseOpenId {
openid.Middleware(next)
}
next.ServeHTTP(w, r)
})
}

View File

@@ -6,10 +6,12 @@ import (
"encoding/json"
"errors"
"net/http"
"slices"
"time"
"github.com/coreos/go-oidc/v3/oidc"
"github.com/google/uuid"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
"golang.org/x/oauth2"
)
@@ -76,6 +78,21 @@ func doAuthentification(r *http.Request, setCookieCallback func(t *oauth2.Token)
return nil, err
}
var claims struct {
Email string `json:"email"`
Verified bool `json:"email_verified"`
}
if err := idToken.Claims(&claims); err != nil {
return nil, err
}
whitelist := config.Instance().OpenIdEmailWhitelist
if len(whitelist) > 0 && !slices.Contains(whitelist, claims.Email) {
return nil, errors.New("email address not found in ACL")
}
nonce, err := r.Cookie("nonce")
if err != nil {
return nil, err

View File

@@ -34,6 +34,7 @@ import (
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/status"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/subscription"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/subscription/task"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/twitch"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/user"
_ "modernc.org/sqlite"
@@ -51,6 +52,7 @@ type serverConfig struct {
db *sql.DB
mq *internal.MessageQueue
lm *livestream.Monitor
tm *twitch.Monitor
}
// TODO: change scope
@@ -115,17 +117,33 @@ func RunBlocking(rc *RunConfig) {
go lm.Schedule()
go lm.Restore()
srv := newServer(serverConfig{
tm := twitch.NewMonitor(
twitch.NewAuthenticationManager(
config.Instance().Twitch.ClientId,
config.Instance().Twitch.ClientSecret,
),
)
go tm.Monitor(
context.TODO(),
config.Instance().Twitch.CheckInterval,
twitch.DEFAULT_DOWNLOAD_HANDLER(mdb, mq),
)
go tm.Restore()
scfg := serverConfig{
frontend: rc.App,
swagger: rc.Swagger,
mdb: mdb,
mq: mq,
db: db,
lm: lm,
})
tm: tm,
}
go gracefulShutdown(srv, mdb)
go autoPersist(time.Minute*5, mdb, lm)
srv := newServer(scfg)
go gracefulShutdown(srv, &scfg)
go autoPersist(time.Minute*5, mdb, lm, tm)
var (
network = "tcp"
@@ -188,12 +206,7 @@ func newServer(c serverConfig) *http.Server {
// Filebrowser routes
r.Route("/filebrowser", func(r chi.Router) {
if config.Instance().RequireAuth {
r.Use(middlewares.Authenticated)
}
if config.Instance().UseOpenId {
r.Use(openid.Middleware)
}
r.Use(middlewares.ApplyAuthenticationByConfig)
r.Post("/downloaded", filebrowser.ListDownloaded)
r.Post("/delete", filebrowser.DeleteFile)
r.Get("/d/{id}", filebrowser.DownloadFile)
@@ -235,10 +248,17 @@ func newServer(c serverConfig) *http.Server {
// Subscriptions
r.Route("/subscriptions", subscription.Container(c.db, cronTaskRunner).ApplyRouter())
// Twitch
r.Route("/twitch", func(r chi.Router) {
r.Use(middlewares.ApplyAuthenticationByConfig)
r.Get("/all", twitch.GetMonitoredUsers(c.tm))
r.Post("/add", twitch.MonitorUserHandler(c.tm))
})
return &http.Server{Handler: r}
}
func gracefulShutdown(srv *http.Server, db *internal.MemoryDB) {
func gracefulShutdown(srv *http.Server, cfg *serverConfig) {
ctx, stop := signal.NotifyContext(context.Background(),
os.Interrupt,
syscall.SIGTERM,
@@ -250,7 +270,9 @@ func gracefulShutdown(srv *http.Server, db *internal.MemoryDB) {
slog.Info("shutdown signal received")
defer func() {
db.Persist()
cfg.mdb.Persist()
cfg.lm.Persist()
cfg.tm.Persist()
stop()
srv.Shutdown(context.Background())
@@ -258,8 +280,14 @@ func gracefulShutdown(srv *http.Server, db *internal.MemoryDB) {
}()
}
func autoPersist(d time.Duration, db *internal.MemoryDB, lm *livestream.Monitor) {
func autoPersist(
d time.Duration,
db *internal.MemoryDB,
lm *livestream.Monitor,
tm *twitch.Monitor,
) {
for {
time.Sleep(d)
if err := db.Persist(); err != nil {
slog.Warn("failed to persisted session", slog.Any("err", err))
}
@@ -267,7 +295,10 @@ func autoPersist(d time.Duration, db *internal.MemoryDB, lm *livestream.Monitor)
slog.Warn(
"failed to persisted livestreams monitor session", slog.Any("err", err.Error()))
}
if err := tm.Persist(); err != nil {
slog.Warn(
"failed to persisted twitch monitor session", slog.Any("err", err.Error()))
}
slog.Debug("sucessfully persisted session")
time.Sleep(d)
}
}

75
server/twitch/auth.go Normal file
View File

@@ -0,0 +1,75 @@
package twitch
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"time"
)
const authURL = "https://id.twitch.tv/oauth2/token"
type AuthResponse struct {
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`
TokenType string `json:"token_type"`
}
type AccessToken struct {
Token string
Expiry time.Time
}
type AuthenticationManager struct {
clientId string
clientSecret string
accesToken *AccessToken
}
func NewAuthenticationManager(clientId, clientSecret string) *AuthenticationManager {
return &AuthenticationManager{
clientId: clientId,
clientSecret: clientSecret,
accesToken: &AccessToken{},
}
}
func (a *AuthenticationManager) GetAccessToken() (*AccessToken, error) {
if a.accesToken != nil && a.accesToken.Token != "" && a.accesToken.Expiry.After(time.Now()) {
return a.accesToken, nil
}
data := url.Values{}
data.Set("client_id", a.clientId)
data.Set("client_secret", a.clientSecret)
data.Set("grant_type", "client_credentials")
resp, err := http.PostForm(authURL, data)
if err != nil {
return nil, fmt.Errorf("errore richiesta token: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status non OK: %s", resp.Status)
}
var auth AuthResponse
if err := json.NewDecoder(resp.Body).Decode(&auth); err != nil {
return nil, fmt.Errorf("errore decoding JSON: %w", err)
}
token := &AccessToken{
Token: auth.AccessToken,
Expiry: time.Now().Add(time.Duration(auth.ExpiresIn) * time.Second),
}
a.accesToken = token
return token, nil
}
func (a *AuthenticationManager) GetClientId() string {
return a.clientId
}

91
server/twitch/client.go Normal file
View File

@@ -0,0 +1,91 @@
package twitch
import (
"encoding/json"
"io"
"net/http"
"time"
)
const twitchAPIURL = "https://api.twitch.tv/helix"
type Client struct {
authenticationManager AuthenticationManager
}
func NewTwitchClient(am *AuthenticationManager) *Client {
return &Client{
authenticationManager: *am,
}
}
type streamResp struct {
Data []struct {
ID string `json:"id"`
UserName string `json:"user_name"`
Title string `json:"title"`
GameName string `json:"game_name"`
StartedAt string `json:"started_at"`
} `json:"data"`
}
func (c *Client) doRequest(endpoint string, params map[string]string) ([]byte, error) {
token, err := c.authenticationManager.GetAccessToken()
if err != nil {
return nil, err
}
reqURL := twitchAPIURL + endpoint
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
return nil, err
}
q := req.URL.Query()
for k, v := range params {
q.Set(k, v)
}
req.URL.RawQuery = q.Encode()
req.Header.Set("Client-Id", c.authenticationManager.GetClientId())
req.Header.Set("Authorization", "Bearer "+token.Token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
func (c *Client) PollStream(channel string, liveChannel chan<- *StreamInfo) error {
body, err := c.doRequest("/streams", map[string]string{"user_login": channel})
if err != nil {
return err
}
var sr streamResp
if err := json.Unmarshal(body, &sr); err != nil {
return err
}
if len(sr.Data) == 0 {
liveChannel <- &StreamInfo{UserName: channel, IsLive: false}
return nil
}
s := sr.Data[0]
started, _ := time.Parse(time.RFC3339, s.StartedAt)
liveChannel <- &StreamInfo{
ID: s.ID,
UserName: s.UserName,
Title: s.Title,
GameName: s.GameName,
StartedAt: started,
IsLive: true,
}
return nil
}

142
server/twitch/monitor.go Normal file
View File

@@ -0,0 +1,142 @@
package twitch
import (
"context"
"encoding/gob"
"fmt"
"iter"
"log/slog"
"maps"
"os"
"path/filepath"
"sync"
"time"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/config"
"github.com/marcopiovanello/yt-dlp-web-ui/v3/server/internal"
)
type Monitor struct {
liveChannel chan *StreamInfo
monitored map[string]*Client
lastState map[string]bool
mu sync.RWMutex
authenticationManager *AuthenticationManager
}
func NewMonitor(authenticationManager *AuthenticationManager) *Monitor {
return &Monitor{
liveChannel: make(chan *StreamInfo, 16),
monitored: make(map[string]*Client),
lastState: make(map[string]bool),
authenticationManager: authenticationManager,
}
}
func (m *Monitor) Add(user string) {
m.mu.Lock()
defer m.mu.Unlock()
m.monitored[user] = NewTwitchClient(m.authenticationManager)
slog.Info("added user to twitch monitor", slog.String("user", user))
}
func (m *Monitor) Monitor(ctx context.Context, interval time.Duration, handler func(url string) error) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
m.mu.RLock()
for user, client := range m.monitored {
u := user
c := client
go func() {
if err := c.PollStream(u, m.liveChannel); err != nil {
slog.Error("polling failed", slog.String("user", u), slog.Any("err", err))
}
}()
}
m.mu.RUnlock()
case stream := <-m.liveChannel:
wasLive := m.lastState[stream.UserName]
if stream.IsLive && !wasLive {
slog.Info("stream went live", slog.String("user", stream.UserName))
if err := handler(fmt.Sprintf("https://www.twitch.tv/%s", stream.UserName)); err != nil {
slog.Error("handler failed", slog.String("user", stream.UserName), slog.Any("err", err))
}
}
m.lastState[stream.UserName] = stream.IsLive
case <-ctx.Done():
slog.Info("stopping twitch monitor")
return
}
}
}
func (m *Monitor) GetMonitoredUsers() iter.Seq[string] {
m.mu.RLock()
defer m.mu.RUnlock()
return maps.Keys(m.monitored)
}
func DEFAULT_DOWNLOAD_HANDLER(db *internal.MemoryDB, mq *internal.MessageQueue) func(url string) error {
return func(url string) error {
p := &internal.Process{
Url: url,
Livestream: true,
Params: []string{"--downloader", "ffmpeg", "--no-part"},
}
db.Set(p)
mq.Publish(p)
return nil
}
}
func (m *Monitor) Persist() error {
filename := filepath.Join(config.Instance().SessionFilePath, "twitch-monitor.dat")
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
enc := gob.NewEncoder(f)
users := make([]string, 0, len(m.monitored))
for user := range m.monitored {
users = append(users, user)
}
return enc.Encode(users)
}
func (m *Monitor) Restore() error {
filename := filepath.Join(config.Instance().SessionFilePath, "twitch-monitor.dat")
f, err := os.Open(filename)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
defer f.Close()
dec := gob.NewDecoder(f)
var users []string
if err := dec.Decode(&users); err != nil {
return err
}
m.monitored = make(map[string]*Client)
for _, user := range users {
m.monitored[user] = NewTwitchClient(m.authenticationManager)
}
return nil
}

40
server/twitch/rest.go Normal file
View File

@@ -0,0 +1,40 @@
package twitch
import (
"encoding/json"
"net/http"
"slices"
)
type addUserReq struct {
User string `json:"user"`
}
func MonitorUserHandler(m *Monitor) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req addUserReq
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
m.Add(req.User)
if err := json.NewEncoder(w).Encode("ok"); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}
func GetMonitoredUsers(m *Monitor) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
it := m.GetMonitoredUsers()
if err := json.NewEncoder(w).Encode(slices.Collect(it)); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}

20
server/twitch/types.go Normal file
View File

@@ -0,0 +1,20 @@
package twitch
import "time"
type StreamInfo struct {
ID string
UserName string
Title string
GameName string
StartedAt time.Time
IsLive bool
}
type VodInfo struct {
ID string
Title string
URL string
Duration string
CreatedAt time.Time
}