Use cookies saved server side (#188)

* retrieve cookies stored server side

fixed netscape cookies validation pipeline

* code refactoring

frontend/src/atoms/downloadTemplate.ts

@@ -1,16 +1,15 @@
-import { atom, selector } from 'recoil'
-import { CustomTemplate } from '../types'
-import { ffetch } from '../lib/httpClient'
-import { serverURL } from './settings'
-import { pipe } from 'fp-ts/lib/function'
 import { getOrElse } from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import { atom, selector } from 'recoil'
+import { ffetch } from '../lib/httpClient'
+import { CustomTemplate } from '../types'
+import { serverSideCookiesState, serverURL } from './settings'
 
-export const cookiesTemplateState = atom({
+export const cookiesTemplateState = selector({
   key: 'cookiesTemplateState',
-  default: localStorage.getItem('cookiesTemplate') ?? '',
+  get: ({ get }) => get(serverSideCookiesState)
-  effects: [
+    ? '--cookies=cookies.txt'
-    ({ onSet }) => onSet(e => localStorage.setItem('cookiesTemplate', e))
+    : ''
-  ]
 })
 
 export const customArgsState = atom({

frontend/src/atoms/settings.ts

@@ -1,4 +1,7 @@
+import { pipe } from 'fp-ts/lib/function'
+import { matchW } from 'fp-ts/lib/TaskEither'
 import { atom, selector } from 'recoil'
+import { ffetch } from '../lib/httpClient'
 import { prefersDarkMode } from '../utils'
 
 export const languages = [
@@ -187,13 +190,15 @@ export const rpcHTTPEndpoint = selector({
   }
 })
 
-export const cookiesState = atom({
+export const serverSideCookiesState = selector<string>({
-  key: 'cookiesState',
+  key: 'serverSideCookiesState',
-  default: localStorage.getItem('yt-dlp-cookies') ?? '',
+  get: async ({ get }) => await pipe(
-  effects: [
+    ffetch<Readonly<{ cookies: string }>>(`${get(serverURL)}/api/v1/cookies`),
-    ({ onSet }) =>
+    matchW(
-      onSet(c => localStorage.setItem('yt-dlp-cookies', c))
+      () => '',
-  ]
+      (r) => r.cookies
+    )
+  )()
 })
 
 const themeSelector = selector<ThemeNarrowed>({

frontend/src/atoms/status.ts

@@ -1,5 +1,10 @@
+import { pipe } from 'fp-ts/lib/function'
+import { of } from 'fp-ts/lib/Task'
+import { getOrElse } from 'fp-ts/lib/TaskEither'
 import { atom, selector } from 'recoil'
+import { ffetch } from '../lib/httpClient'
 import { rpcClientState } from './rpc'
+import { serverURL } from './settings'
 
 export const connectedState = atom({
   key: 'connectedState',
@@ -23,3 +28,14 @@ export const availableDownloadPathsState = selector({
     return res.result
   }
 })
+
+export const ytdlpVersionState = selector<string>({
+  key: 'ytdlpVersionState',
+  get: async ({ get }) => await pipe(
+    ffetch<string>(`${get(serverURL)}/api/v1/version`),
+    getOrElse(() => pipe(
+      'unknown version',
+      of
+    )),
+  )()
+})

frontend/src/components/CookiesTextField.tsx

@@ -1,22 +1,20 @@
-import { TextField } from '@mui/material'
+import { Button, TextField } from '@mui/material'
 import * as A from 'fp-ts/Array'
 import * as E from 'fp-ts/Either'
 import * as O from 'fp-ts/Option'
+import { matchW } from 'fp-ts/lib/TaskEither'
 import { pipe } from 'fp-ts/lib/function'
 import { useMemo } from 'react'
-import { useRecoilState, useRecoilValue } from 'recoil'
+import { useRecoilValue } from 'recoil'
 import { Subject, debounceTime, distinctUntilChanged } from 'rxjs'
-import { cookiesTemplateState } from '../atoms/downloadTemplate'
+import { serverSideCookiesState, serverURL } from '../atoms/settings'
-import { cookiesState, serverURL } from '../atoms/settings'
 import { useSubscription } from '../hooks/observable'
 import { useToast } from '../hooks/toast'
 import { ffetch } from '../lib/httpClient'
 
 const validateCookie = (cookie: string) => pipe(
   cookie,
-  cookie => cookie.replace(/\s\s+/g, ' '),
+  cookie => cookie.split('\t'),
-  cookie => cookie.replaceAll('\t', ' '),
-  cookie => cookie.split(' '),
   E.of,
   E.flatMap(
     E.fromPredicate(
@@ -68,13 +66,19 @@ const validateCookie = (cookie: string) => pipe(
   ),
 )
 
+const noopValidator = (s: string): E.Either<string, string[]> => pipe(
+  s,
+  s => s.split('\t'),
+  E.of
+)
+
+const isCommentOrNewLine = (s: string) => s === '' || s.startsWith('\n') || s.startsWith('#')
+
 const CookiesTextField: React.FC = () => {
   const serverAddr = useRecoilValue(serverURL)
-  const [, setCookies] = useRecoilState(cookiesTemplateState)
+  const savedCookies = useRecoilValue(serverSideCookiesState)
-  const [savedCookies, setSavedCookies] = useRecoilState(cookiesState)
 
   const { pushMessage } = useToast()
-  const flag = '--cookies=cookies.txt'
 
   const cookies$ = useMemo(() => new Subject<string>(), [])
 
@@ -86,28 +90,41 @@ const CookiesTextField: React.FC = () => {
       })
     })()
 
+  const deleteCookies = () => pipe(
+    ffetch(`${serverAddr}/api/v1/cookies`, {
+      method: 'DELETE',
+    }),
+    matchW(
+      (l) => pushMessage(l, 'error'),
+      (_) => {
+        pushMessage('Deleted cookies', 'success')
+        pushMessage(`Reload the page to apply the changes`, 'info')
+      }
+    )
+  )()
+
   const validateNetscapeCookies = (cookies: string) => pipe(
     cookies,
     cookies => cookies.split('\n'),
-    cookies => cookies.filter(f => !f.startsWith('\n')), // empty lines
+    A.map(c => isCommentOrNewLine(c) ? noopValidator(c) : validateCookie(c)), // validate line
-    cookies => cookies.filter(f => !f.startsWith('# ')), // comments
+    A.mapWithIndex((i, either) => pipe(                  // detect errors and return the either
-    cookies => cookies.filter(Boolean), // empty lines
-    A.map(validateCookie),
-    A.mapWithIndex((i, either) => pipe(
       either,
-      E.matchW(
+      E.match(
-        (l) => pushMessage(`Error in line ${i + 1}: ${l}`, 'warning'),
+        (l) => {
-        () => E.isRight(either)
+          pushMessage(`Error in line ${i + 1}: ${l}`, 'warning')
+          return either
+        },
+        (_) => either
       ),
     )),
-    A.filter(Boolean),
+    A.filter(c => E.isRight(c)),                         // filter the line who didn't pass the validation
-    A.match(
+    A.map(E.getOrElse(() => new Array<string>())),       // cast the array of eithers to an array of tokens
-      () => false,
+    A.filter(f => f.length > 0),                         // filter the empty tokens
-      (c) => {
+    A.map(f => f.join('\t')),                            // join the tokens in a TAB separated string
-        pushMessage(`Valid ${c.length} Netscape cookies`, 'info')
+    A.reduce('', (c, n) => `${c}${n}\n`),                // reduce all to a single string separated by \n
-        return true
+    parsed => parsed.length > 0                          // if nothing has passed the validation return none
-      }
+      ? O.some(parsed)
-    )
+      : O.none
   )
 
   useSubscription(
@@ -117,22 +134,17 @@ const CookiesTextField: React.FC = () => {
     ),
     (cookies) => pipe(
       cookies,
-      cookies => {
-        setSavedCookies(cookies)
-        return cookies
-      },
       validateNetscapeCookies,
-      O.fromPredicate(f => f === true),
       O.match(
-        () => setCookies(''),
+        () => pushMessage('No valid cookies', 'warning'),
-        async () => {
+        async (some) => {
           pipe(
-            await submitCookies(cookies),
+            await submitCookies(some.trimEnd()),
             E.match(
               (l) => pushMessage(`${l}`, 'error'),
               () => {
-                pushMessage(`Saved Netscape cookies`, 'success')
+                pushMessage(`Saved ${some.split('\n').length} Netscape cookies`, 'success')
-                setCookies(flag)
+                pushMessage('Reload the page to apply the changes', 'info')
               }
             )
           )
@@ -142,15 +154,18 @@ const CookiesTextField: React.FC = () => {
   )
 
   return (
-    <TextField
+    <>
-      label="Netscape Cookies"
+      <TextField
-      multiline
+        label="Netscape Cookies"
-      maxRows={20}
+        multiline
-      minRows={4}
+        maxRows={20}
-      fullWidth
+        minRows={4}
-      defaultValue={savedCookies}
+        fullWidth
-      onChange={(e) => cookies$.next(e.currentTarget.value)}
+        defaultValue={savedCookies}
-    />
+        onChange={(e) => cookies$.next(e.currentTarget.value)}
+      />
+      <Button onClick={deleteCookies}>Delete cookies</Button>
+    </>
   )
 }
 

frontend/src/components/Footer.tsx

@@ -37,7 +37,9 @@ const Footer: React.FC = () => {
         <div style={{ display: 'flex', gap: 4, alignItems: 'center' }}>
           {/* TODO: make it dynamic */}
           <Chip label="RPC v3.2.0" variant="outlined" size="small" />
-          <VersionIndicator />
+          <Suspense>
+            <VersionIndicator />
+          </Suspense>
         </div>
         <div style={{ display: 'flex', gap: 4, 'alignItems': 'center' }}>
           <div style={{

frontend/src/components/VersionIndicator.tsx

@@ -1,32 +1,9 @@
 import { Chip, CircularProgress } from '@mui/material'
-import { useEffect, useState } from 'react'
 import { useRecoilValue } from 'recoil'
-import { serverURL } from '../atoms/settings'
+import { ytdlpVersionState } from '../atoms/status'
-import { useToast } from '../hooks/toast'
 
 const VersionIndicator: React.FC = () => {
-  const serverAddr = useRecoilValue(serverURL)
+  const version = useRecoilValue(ytdlpVersionState)
-
-  const [version, setVersion] = useState('')
-  const { pushMessage } = useToast()
-
-  const fetchVersion = async () => {
-    const res = await fetch(`${serverAddr}/api/v1/version`, {
-      headers: {
-        'X-Authentication': localStorage.getItem('token') ?? ''
-      }
-    })
-
-    if (!res.ok) {
-      return pushMessage(await res.text(), 'error')
-    }
-
-    setVersion(await res.json())
-  }
-
-  useEffect(() => {
-    fetchVersion()
-  }, [])
 
   return (
     version

frontend/src/lib/rpcClient.ts

@@ -82,7 +82,9 @@ export class RPCClient {
       : ''
 
     const sanitizedArgs = this.argsSanitizer(
-      req.args.replace('-o', '').replace(rename, '')
+      req.args
+        .replace('-o', '')
+        .replace(rename, '')
     )
 
     if (req.playlist) {
@@ -177,14 +179,14 @@ export class RPCClient {
   }
 
   public killLivestream(url: string) {
-    return this.sendHTTP<LiveStreamProgress>({
+    return this.sendHTTP({
       method: 'Service.KillLivestream',
       params: [url]
     })
   }
 
   public killAllLivestream() {
-    return this.sendHTTP<LiveStreamProgress>({
+    return this.sendHTTP({
       method: 'Service.KillAllLivestream',
       params: []
     })

frontend/src/views/Settings.tsx

@@ -18,7 +18,7 @@ import {
   Typography,
   capitalize
 } from '@mui/material'
-import { useEffect, useMemo, useState } from 'react'
+import { Suspense, useEffect, useMemo, useState } from 'react'
 import { useRecoilState } from 'recoil'
 import {
   Subject,
@@ -347,7 +347,9 @@ export default function Settings() {
           <Typography variant="h6" color="primary" sx={{ mb: 2 }}>
             Cookies
           </Typography>
-          <CookiesTextField />
+          <Suspense>
+            <CookiesTextField />
+          </Suspense>
         </Grid>
         <Grid>
           <Stack direction="row">

server/internal/message_queue.go

@@ -63,6 +63,10 @@ func (m *MessageQueue) downloadConsumer() {
 		)
 
 		if p.Progress.Status != StatusCompleted {
+			slog.Info("started process",
+				slog.String("bus", queueName),
+				slog.String("id", p.getShortId()),
+			)
 			if p.Livestream {
 				// livestreams have higher priorty and they ignore the semaphore
 				go p.Start()
@@ -70,11 +74,6 @@ func (m *MessageQueue) downloadConsumer() {
 				p.Start()
 			}
 		}
-
-		slog.Info("started process",
-			slog.String("bus", queueName),
-			slog.String("id", p.getShortId()),
-		)
 	}, false)
 }
 

server/internal/process.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -21,7 +22,6 @@ import (
 
 	"github.com/marcopeocchi/yt-dlp-web-ui/server/cli"
 	"github.com/marcopeocchi/yt-dlp-web-ui/server/config"
-	"github.com/marcopeocchi/yt-dlp-web-ui/server/rx"
 )
 
 const template = `download:
@@ -36,7 +36,6 @@ const (
 	StatusDownloading
 	StatusCompleted
 	StatusErrored
-	StatusLivestream
 )
 
 // Process descriptor
@@ -103,81 +102,102 @@ func (p *Process) Start() {
 
 	params := append(baseParams, p.Params...)
 
-	// ----------------- main block ----------------- //
+	slog.Info("requesting download", slog.String("url", p.Url), slog.Any("params", params))
+
 	cmd := exec.Command(config.Instance().DownloaderPath, params...)
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 
-	r, err := cmd.StdoutPipe()
+	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		slog.Error(
+		slog.Error("failed to connect to stdout", slog.Any("err", err))
-			"failed to connect to stdout",
+		panic(err)
-			slog.String("err", err.Error()),
+	}
-		)
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		slog.Error("failed to connect to stdout", slog.Any("err", err))
 		panic(err)
 	}
 
 	if err := cmd.Start(); err != nil {
-		slog.Error(
+		slog.Error("failed to start yt-dlp process", slog.Any("err", err))
-			"failed to start yt-dlp process",
-			slog.String("err", err.Error()),
-		)
 		panic(err)
 	}
 
 	p.proc = cmd.Process
 
-	// --------------- progress block --------------- //
+	ctx, cancel := context.WithCancel(context.Background())
-	var (
+	defer func() {
-		sourceChan = make(chan []byte)
+		stdout.Close()
-		doneChan   = make(chan struct{})
+		p.Complete()
-	)
+		cancel()
+	}()
 
-	// spawn a goroutine that does the dirty job of parsing the stdout
+	logs := make(chan []byte)
-	// filling the channel with as many stdout line as yt-dlp produces (producer)
+	go produceLogs(stdout, logs)
+	go p.consumeLogs(ctx, logs)
+
+	go p.detectYtDlpErrors(stderr)
+
+	cmd.Wait()
+}
+
+func produceLogs(r io.Reader, logs chan<- []byte) {
 	go func() {
-		scan := bufio.NewScanner(r)
+		scanner := bufio.NewScanner(r)
 
-		defer func() {
+		for scanner.Scan() {
-			r.Close()
+			logs <- scanner.Bytes()
-			p.Complete()
-
-			doneChan <- struct{}{}
-
-			close(sourceChan)
-			close(doneChan)
-		}()
-
-		for scan.Scan() {
-			sourceChan <- scan.Bytes()
 		}
 	}()
+}
 
-	// Slows down the unmarshal operation to every 500ms
+func (p *Process) consumeLogs(ctx context.Context, logs <-chan []byte) {
-	go func() {
+	for {
-		rx.Sample(time.Millisecond*500, sourceChan, doneChan, func(event []byte) {
+		select {
-			var progress ProgressTemplate
+		case <-ctx.Done():
-
+			slog.Info("detaching from yt-dlp stdout",
-			if err := json.Unmarshal(event, &progress); err != nil {
-				return
-			}
-
-			p.Progress = DownloadProgress{
-				Status:     StatusDownloading,
-				Percentage: progress.Percentage,
-				Speed:      progress.Speed,
-				ETA:        progress.Eta,
-			}
-
-			slog.Info("progress",
 				slog.String("id", p.getShortId()),
 				slog.String("url", p.Url),
-				slog.String("percentage", progress.Percentage),
 			)
-		})
+			return
-	}()
+		case entry := <-logs:
+			p.parseLogEntry(entry)
+		}
+	}
+}
 
-	// ------------- end progress block ------------- //
+func (p *Process) parseLogEntry(entry []byte) {
-	cmd.Wait()
+	var progress ProgressTemplate
+
+	if err := json.Unmarshal(entry, &progress); err != nil {
+		return
+	}
+
+	p.Progress = DownloadProgress{
+		Status:     StatusDownloading,
+		Percentage: progress.Percentage,
+		Speed:      progress.Speed,
+		ETA:        progress.Eta,
+	}
+
+	slog.Info("progress",
+		slog.String("id", p.getShortId()),
+		slog.String("url", p.Url),
+		slog.String("percentage", progress.Percentage),
+	)
+}
+
+func (p *Process) detectYtDlpErrors(r io.Reader) {
+	scanner := bufio.NewScanner(r)
+
+	for scanner.Scan() {
+		slog.Error("yt-dlp process error",
+			slog.String("id", p.getShortId()),
+			slog.String("url", p.Url),
+			slog.String("err", scanner.Text()),
+		)
+	}
 }
 
 // Keep process in the memoryDB but marks it as complete
@@ -222,6 +242,7 @@ func (p *Process) Kill() error {
 }
 
 // Returns the available format for this URL
+//
 // TODO: Move out from process.go
 func (p *Process) GetFormats() (DownloadFormats, error) {
 	cmd := exec.Command(config.Instance().DownloaderPath, p.Url, "-J")

server/rest/container.go

@@ -28,7 +28,9 @@ func ApplyRouter(args *ContainerArgs) func(chi.Router) {
 		r.Post("/exec", h.Exec())
 		r.Get("/running", h.Running())
 		r.Get("/version", h.GetVersion())
+		r.Get("/cookies", h.GetCookies())
 		r.Post("/cookies", h.SetCookies())
+		r.Delete("/cookies", h.DeleteCookies())
 		r.Post("/template", h.AddTemplate())
 		r.Get("/template/all", h.GetTemplates())
 		r.Delete("/template/{id}", h.DeleteTemplate())

server/rest/handlers.go

@@ -60,6 +60,27 @@ func (h *Handler) Running() http.HandlerFunc {
 	}
 }
 
+func (h *Handler) GetCookies() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+
+		cookies, err := h.service.GetCookies(r.Context())
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		res := &internal.SetCookiesRequest{
+			Cookies: string(cookies),
+		}
+
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
 func (h *Handler) SetCookies() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
@@ -87,6 +108,23 @@ func (h *Handler) SetCookies() http.HandlerFunc {
 	}
 }
 
+func (h *Handler) DeleteCookies() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+
+		err := h.service.SetCookies(r.Context(), "")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		err = json.NewEncoder(w).Encode("ok")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+	}
+}
+
 func (h *Handler) AddTemplate() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()

server/rest/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"io"
 	"os"
 	"os/exec"
 	"time"
@@ -44,6 +45,22 @@ func (s *Service) Running(ctx context.Context) (*[]internal.ProcessResponse, err
 	}
 }
 
+func (s *Service) GetCookies(ctx context.Context) ([]byte, error) {
+	fd, err := os.Open("cookies.txt")
+	if err != nil {
+		return nil, err
+	}
+
+	defer fd.Close()
+
+	cookies, err := io.ReadAll(fd)
+	if err != nil {
+		return nil, err
+	}
+
+	return cookies, nil
+}
+
 func (s *Service) SetCookies(ctx context.Context, cookies string) error {
 	fd, err := os.Create("cookies.txt")
 	if err != nil {