Refactoring

.github/workflows/docker-publish.yml

@@ -0,0 +1,93 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+#  schedule:
+#    - cron: '39 13 * * *'
+  push:
+    branches: [ master ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422
+        with:
+          cosign-release: 'v1.4.0'
+
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          COSIGN_EXPERIMENTAL: "true"
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: cosign sign ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }}

README.md

@@ -4,9 +4,14 @@ A not so terrible web ui for yt-dlp.
 Created for the only purpose of *consume* videos from my server/nas.  
 I will eventually make this better as soon as I can. Not in the immediate.  
 
-**Background jobs now are retrieved. It's still rudimentary but it leverages**
+Changelog:
-**on yt-dlp resume feature**  
+```
+26/01/22: Multiple downloads are being implemented. Maybe by next release they will be there.
+Refactoring and JSDoc.
 
+04/01/22: Background jobs now are retrieved!! It's still rudimentary but it leverages on yt-dlp resume feature
+
+```
 <img src="https://i.ibb.co/7VBK1PY/1.png">
 
 ## Now with dark mode
@@ -18,11 +23,11 @@ I will eventually make this better as soon as I can. Not in the immediate.
 The avaible settings are currently only:
 -   Server address
 -   Switch theme
--   Retrieve background jobs
+-   Extract audio
 
 Future releases will have:
--   Multi download
+-   Multi download *on its way*
--   ~~Exctract audio~~
+-   ~~Exctract audio~~ *done*
 -   Format selection
 
 ## Docker installation
@@ -50,6 +55,11 @@ mkdir downloads
 node server.js
 ```
 
+
+## Regarding multiple downloads
+There's a way to circumvent the single download restriction **BUT IT LEADS TO UNDEFINED BEHAVIOUR**.  
+Fire up multiple tabs and make a download for each tab. I know that's horrible but it's gonna be fixed by next release.
+
 ## Todo list
 - ~~retrieve background tasks~~
 - better ui/ux

frontend/src/App.tsx

@@ -43,9 +43,9 @@ export function App() {
 
     useEffect(() => {
         socket.on('pending-jobs', (jobs: Array<any>) => {
-            if (jobs.length > 0) {
+            //if (jobs.length > 0) {
             socket.emit('retrieve-jobs')
-            }
+            //}
         })
     }, [])
 

lib/Process.js

@@ -0,0 +1,122 @@
+const { spawn } = require('child_process');
+const { deleteDownloadByPID, insertDownload } = require('./db');
+const { logger } = require('./logger');
+
+/**
+ * Represents a download process that spawns yt-dlp.
+ * @constructor
+ * @param {string} url - The downlaod url.
+ * @param {string} params - The cli arguments passed by the frontend.
+ * @param {object} settings - The download settings passed by the frontend.
+ */
+
+class Process {
+    constructor(url, params, settings) {
+        this.url = url;
+        this.params = params || ' ';
+        this.settings = settings
+        this.stdout = undefined;
+        this.pid = undefined;
+        this.info = null;
+    }
+
+    /**
+     * function that launch the download process, sets the stdout property and the pid
+     * @param {Function} callback not yet implemented
+     * @returns {Promise<this>} the process instance
+     */
+    async start(callback) {
+        await this.__internalGetInfo();
+
+        const ytldp = spawn('./lib/yt-dlp',
+            [
+                '-o', `${this.settings?.download_path || 'downloads/'}%(title)s.%(ext)s`,
+                this.params,
+                this.url
+            ]
+        );
+
+        this.pid = ytldp.pid;
+        this.stdout = ytldp.stdout;
+
+        logger('proc', `Spawned a new process, pid: ${this.pid}`)
+
+        await insertDownload(this.url, null, null, null, this.pid);
+
+        return this;
+    }
+
+    /**
+     * @private
+     * function used internally by the download process to fetch information, usually thumbnail and title
+     * @returns Promise to the lock
+     */
+    async __internalGetInfo() {
+        let lock = true;
+        let stdoutChunks = [];
+        const ytdlpInfo = spawn('./lib/yt-dlp', ['-s', '-j', this.url]);
+
+        ytdlpInfo.stdout.on('data', (data) => {
+            stdoutChunks.push(data);
+        });
+
+        ytdlpInfo.on('exit', () => {
+            try {
+                const buffer = Buffer.concat(stdoutChunks);
+                const json = JSON.parse(buffer.toString());
+                this.info = json;
+                this.lock = false;
+
+            } catch (e) {
+                this.info = {
+                    title: "",
+                    thumbnail: "",
+                };
+            }
+        });
+
+        if (!lock) {
+            return true;
+        }
+    }
+
+    /**
+     * function that kills the current process
+     */
+    async kill() {
+        spawn('kill', [this.pid]).on('exit', () => {
+            deleteDownloadByPID(this.pid).then(() => {
+                logger('db', `Deleted ${this.pid} because SIGKILL`)
+            })
+        });
+    }
+
+    /**
+     * pid getter function
+     * @returns {number} pid
+     */
+    getPid() {
+        if (!this.pid) {
+            throw "Process isn't started"
+        }
+        return this.pid;
+    }
+
+    /**
+     * stdout getter function
+     * @returns {ReadableStream} stdout as stream
+     */
+    getStdout() {
+        return this.stdout
+    }
+
+    /**
+     * download info getter function
+     * @returns {object}
+     */
+    getInfo() {
+        return this.info
+    }
+}
+
+module.exports = Process;

lib/ProcessPool.js

@@ -0,0 +1,62 @@
+/**
+ * @class
+ * Represents a download process that spawns yt-dlp.
+ */
+
+class ProcessPool {
+    constructor() {
+        this._pool = new Map();
+        this._size = 0;
+    }
+
+    /**
+     * Pool size getter
+     * @returns {number} pool's size
+     */
+    size() {
+        return this._size;
+    }
+
+    /**
+     * Add a process to the pool
+     * @param {Process} process 
+     */
+    add(process) {
+        this._pool.set(process.getPid(), process)
+    }
+
+    /**
+     * Delete a process from the pool
+     * @param {Process} process 
+     */
+    remove(process) {
+        this._pool.delete(process.getPid())
+    }
+
+    /**
+     * Delete a process from the pool by its pid
+     * @param {number} pid 
+     */
+    removeByPid(pid) {
+        this._pool.delete(pid)
+    }
+
+    /**
+     * get an iterator for the pool
+     * @returns {IterableIterator} iterator 
+     */
+    iterator() {
+        return this._pool.entries()
+    }
+
+    /**
+     * get a process by its pid
+     * @param {number} pid 
+     * @returns {Process}
+     */
+    getByPid(pid) {
+        return this._pool.get(pid)
+    }
+}
+
+module.exports = ProcessPool;

lib/db.js

@@ -4,6 +4,9 @@ const { existsInProc } = require('./procUtils')
 
 const db = require('better-sqlite3')('downloads.db')
 
+/**
+ * Inits the repository, the tables.
+ */
 async function init() {
     try {
         db.exec(`CREATE TABLE downloads (
@@ -21,10 +24,23 @@ async function init() {
     }
 }
 
+/**
+ * Get an instance of the db.
+ * @returns {BetterSqlite3.Database} Current database instance
+ */
 async function get_db() {
     return db
 }
 
+/**
+ * Insert an new download to the database
+ * @param {string} url the video url
+ * @param {string} title the title fetched by the info process
+ * @param {string} thumbnail the thumbnail url fetched by the info process
+ * @param {string} size optional - the download size
+ * @param {number} PID the pid of the downloader
+ * @returns {Promise<string>} the download UUID
+ */
 async function insertDownload(url, title, thumbnail, size, PID) {
     const uid = uuid.v1()
     try {
@@ -42,28 +58,43 @@ async function insertDownload(url, title, thumbnail, size, PID) {
     return uid
 }
 
+/**
+ * Retrieve all downloads from the database
+ * @returns {ArrayLike} a collection of results
+ */
 async function retrieveAll() {
     return db
         .prepare('SELECT * FROM downloads')
         .all()
 }
 
+/**
+ * Delete a download by its uuid
+ * @param {string} uid the to-be-deleted download uuid
+ */
 async function deleteDownloadById(uid) {
     db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run()
 }
 
+/**
+ * Delete a download by its pid
+ * @param {string} pid the to-be-deleted download pid
+ */
 async function deleteDownloadByPID(PID) {
     db.prepare(`DELETE FROM downloads WHERE process_pid=${PID}`).run()
 }
 
+/**
+ * Deletes the downloads that aren't active anymore
+ * @returns {Promise<ArrayLike>}
+ */
 async function pruneDownloads() {
     const all = await retrieveAll()
     return all.map(job => {
         if (existsInProc(job.process_pid)) {
             return job
-        } else {
-            deleteDownloadByPID(job.process_pid)
         }
+        deleteDownloadByPID(job.process_pid)
     })
 }
 

lib/downloader.js

@@ -1,10 +1,18 @@
 const { spawn } = require('child_process');
 const { from, interval } = require('rxjs');
 const { throttle } = require('rxjs/operators');
-const { insertDownload, deleteDownloadByPID, pruneDownloads } = require('./db');
+const { Socket } = require('socket.io');
+const { pruneDownloads } = require('./db');
 const { logger } = require('./logger');
-const { retriveStdoutFromProcFd, killProcess } = require('./procUtils');
+const Process = require('./Process');
+const ProcessPool = require('./ProcessPool');
+const { killProcess } = require('./procUtils');
+
+// settings read from settings.json
 let settings;
+let coldRestart = true;
+
+const pool = new ProcessPool();
 
 try {
     settings = require('../settings.json');
@@ -13,10 +21,14 @@ catch (e) {
     console.warn("settings.json not found");
 }
 
-const isWindows = process.platform === 'win32';
+/**
-
+ * Invoke a new download.  
+ * Called by the websocket messages listener.
+ * @param {Socket} socket current connection socket
+ * @param {object} payload frontend download payload
+ * @returns 
+ */
 async function download(socket, payload) {
-
     if (!payload || payload.url === '' || payload.url === null) {
         socket.emit('progress', { status: 'Done!' });
         return;
@@ -25,82 +37,124 @@ async function download(socket, payload) {
     const url = payload.url
     const params = payload.params?.xa ? '-x' : '';
 
-    await getDownloadInfo(socket, url);
+    const p = new Process(url, params, settings);
 
-    const ytldp = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`,
+    p.start().then(downloader => {
-        [
+        pool.add(p)
-            '-o', `${settings.download_path || 'downloads/'}%(title)s.%(ext)s`,
+        let infoLock = true;
-            params,
+        let pid = downloader.getPid();
-            url
-        ]
-    );
 
-    await insertDownload(url, null, null, null, ytldp.pid);
+        from(downloader.getStdout())              // stdout as observable
-
-    from(ytldp.stdout)  // stdout as observable
             .pipe(throttle(() => interval(500)))  // discard events closer than 500ms
             .subscribe({
                 next: (stdout) => {
-                //let _stdout = String(stdout)
+                    if (infoLock) {
-                socket.emit('progress', formatter(String(stdout))) // finally, emit
+                        if (downloader.getInfo() === null) {
-                //logger('download', `Fetching ${_stdout}`)
+                            return;
+                        }
+                        socket.emit('info', downloader.getInfo());
+                        infoLock = false;
+                    }
+                    socket.emit('progress', formatter(String(stdout), pid)) // finally, emit
                 },
                 complete: () => {
-                socket.emit('progress', { status: 'Done!' })
+                    downloader.kill().then(() => {
+                        socket.emit('progress', {
+                            status: 'Done!',
+                            process: pid,
+                        })
+                        pool.remove(downloader);
+                    })
+                },
+                error: () => {
+                    socket.emit('progress', { status: 'Done!' });
                 }
             });
-
-    ytldp.on('exit', () => {
-        socket.emit('progress', { status: 'Done!' })
-        logger('download', 'Done!')
-
-        deleteDownloadByPID(ytldp.pid).then(() => {
-            logger('db', `Deleted ${ytldp.pid} because SIGKILL`)
-        })
     })
 }
 
+/**
+ * Retrieve all downloads.  
+ * If the server has just been launched retrieve the ones saved to the database.  
+ * If the server is running fetches them from the process pool.
+ * @param {Socket} socket current connection socket
+ * @returns 
+ */
 async function retriveDownload(socket) {
-    const downloads = await pruneDownloads();
+    // it's a cold restart: the server has just been started with pending
-    if (downloads.length > 0) {
+    // downloads, so fetch them from the database and resume.
-        for (const _download of downloads) {
+    if (coldRestart) {
-            await killProcess(_download.process_pid);
+        coldRestart = false;
-            await download(socket, _download);
+        let downloads = await pruneDownloads();
+        downloads = [... new Set(downloads)];
+        logger('dl', `Cold restart, retrieving ${downloads.length} jobs`)
+        for (const entry of downloads) {
+            if (entry) {
+                await download(socket, entry);
             }
         }
-}
+        return;
+    }
 
-async function getDownloadInfo(socket, url) {
+    // it's an hot-reload the server it's running and the frontend ask for
-    let stdoutChunks = [];
+    // the pending job: retrieve them from the "in-memory database" (ProcessPool)
-    const ytdlpInfo = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`, ['-s', '-j', url]);
+    logger('dl', `Retrieving jobs from pool`)
+    const it = pool.iterator();
 
-    ytdlpInfo.stdout.on('data', (data) => {
+    for (const entry of it) {
-        stdoutChunks.push(data);
+        const [pid, process] = entry;
+        await killProcess(pid);
+        await download(socket, {
+            url: process.url,
+            params: process.params
         });
-
-    ytdlpInfo.on('exit', () => {
-        try {
-            const buffer = Buffer.concat(stdoutChunks);
-            const json = JSON.parse(buffer.toString());
-            socket.emit('info', json);
-        } catch (e) {
-            socket.emit('progress', { status: 'Aborted' });
-            logger('download', 'Done!');
     }
-    })
 }
 
-function abortDownload(socket) {
+/**
-    const res = process.platform === 'win32' ?
+ * Abort a specific download if pid is provided, in the other case
-        spawn('taskkill', ['/IM', 'yt-dlp.exe', '/F', '/T']) :
+ * calls the abortAllDownloads function
-        spawn('killall', ['yt-dlp']);
+ * @see abortAllDownloads
-    res.on('exit', () => {
+ * @param {Socket} socket currenct connection socket
-        socket.emit('progress', { status: 'Aborted' });
+ * @param {*} args args sent by the frontend. MUST contain the PID.
-        logger('download', 'Aborting downloads');
+ * @returns 
+ */
+function abortDownload(socket, args) {
+    if (!args) {
+        abortAllDownloads(socket);
+        return;
+    }
+    const { pid } = args;
+
+    spawn('kill', [pid])
+        .on('exit', () => {
+            socket.emit('progress', {
+                status: 'Aborted',
+                process: pid,
+            });
+            logger('dl', `Aborting download ${pid}`);
         });
 }
 
-const formatter = (stdout) => {
+/**
+ * Unconditionally kills all yt-dlp process.
+ * @param {Socket} socket currenct connection socket
+ */
+function abortAllDownloads(socket) {
+    spawn('killall', ['yt-dlp'])
+        .on('exit', () => {
+            socket.emit('progress', { status: 'Aborted' });
+            logger('dl', 'Aborting downloads');
+        });
+}
+
+/**
+ * @private Formats the yt-dlp stdout to a frontend-readable format
+ * @param {string} stdout stdout as string
+ * @param {number} pid current process id relative to stdout
+ * @returns 
+ */
+const formatter = (stdout, pid) => {
     const cleanStdout = stdout
         .replace(/\s\s+/g, ' ')
         .split(' ');
@@ -112,6 +166,7 @@ const formatter = (stdout) => {
                 progress: cleanStdout[1],
                 size: cleanStdout[3],
                 dlSpeed: cleanStdout[5],
+                pid: pid,
             }
         case 'merge':
             return {
@@ -126,5 +181,6 @@ const formatter = (stdout) => {
 module.exports = {
     download: download,
     abortDownload: abortDownload,
+    abortAllDownloads: abortAllDownloads,
     retriveDownload: retriveDownload,
 }

lib/logger.js

@@ -1,7 +1,16 @@
+/**
+ * Simplest logger function, takes two argument: first one put between
+ * square brackets (the protocol), the second one it's the effective message
+ * @param {string} proto protocol
+ * @param {string} args message
+ */
 const logger = (proto, args) => {
     console.log(`[${proto}]\t${args}`)
 }
 
+/**
+ * CLI splash
+ */
 const splash = () => {
     console.log("-------------------------------------------------")
     console.log("yt-dlp-webUI - A web-ui for yt-dlp, simply enough")

lib/procUtils.js

@@ -3,6 +3,11 @@ const fs = require('fs');
 const net = require('net');
 const { logger } = require('./logger');
 
+/**
+ * Browse /proc in order to find the specific pid
+ * @param {number} pid 
+ * @returns {*} process stats if any
+ */
 function existsInProc(pid) {
     try {
         return fs.statSync(`/proc/${pid}`)
@@ -24,6 +29,10 @@ function retriveStdoutFromProcFd(pid) {
 }
 */
 
+/**
+ * Kills a process with a sys-call
+ * @param {number} pid the killed process pid
+ */
 async function killProcess(pid) {
     const res = spawn('kill', [pid])
     res.on('exit', () => {

lib/updater.js

@@ -1,6 +1,7 @@
 const https = require('https');
 const fs = require('fs');
 const path = require('path');
+const { Socket } = require('socket.io');
 
 // endpoint to github API
 const options = {
@@ -13,7 +14,11 @@ const options = {
     port: 443,
 }
 
-// build the binary url based on the release tag
+/**
+ * Build the binary url based on the release tag
+ * @param {string} release yt-dlp GitHub release tag
+ * @returns {*} the fetch options with the correct tag and headers
+ */
 function buildDonwloadOptions(release) {
     return {
         hostname: 'github.com',
@@ -26,7 +31,9 @@ function buildDonwloadOptions(release) {
     }
 }
 
-// main
+/**
+ * gets the yt-dlp latest binary URL from GitHub API
+ */
 async function update() {
     // ensure that the binary has been removed
     try {
@@ -53,7 +60,10 @@ async function update() {
         })
     })
 }
-
+/**
+ * Utility that Pipes the latest binary to a file
+ * @param {string} url yt-dlp GitHub release url
+ */
 function downloadBinary(url) {
     https.get(url, res => {
         // if it is a redirect follow the url
@@ -70,7 +80,10 @@ function downloadBinary(url) {
         })
     })
 }
-
+/**
+ * Invoke the yt-dlp update procedure
+ * @param {Socket} socket the current connection socket
+ */
 function updateFromFrontend(socket) {
     update().then(() => {
         socket.emit('updated')

server.js

@@ -5,8 +5,13 @@ const Koa = require('koa'),
     { join } = require('path'),
     { Server } = require('socket.io'),
     { createServer } = require('http'),
-    { download, abortDownload, retriveDownload } = require('./lib/downloader'),
     { ytdlpUpdater } = require('./lib/updater'),
+    {
+        download,
+        abortDownload,
+        retriveDownload,
+        abortAllDownloads
+    } = require('./lib/downloader'),
     db = require('./lib/db');
 
 const app = new Koa()
@@ -25,8 +30,11 @@ io.on('connection', socket => {
         logger('ws', args?.url)
         download(socket, args)
     })
-    socket.on('abort', () => {
+    socket.on('abort', (args) => {
-        abortDownload(socket)
+        abortDownload(socket, args)
+    })
+    socket.on('abort-all', () => {
+        abortAllDownloads(socket)
     })
     socket.on('update-bin', () => {
         ytdlpUpdater(socket)