Refactoring

lib/Process.js

@@ -0,0 +1,122 @@
+const { spawn } = require('child_process');
+const { deleteDownloadByPID, insertDownload } = require('./db');
+const { logger } = require('./logger');
+
+/**
+ * Represents a download process that spawns yt-dlp.
+ * @constructor
+ * @param {string} url - The downlaod url.
+ * @param {string} params - The cli arguments passed by the frontend.
+ * @param {object} settings - The download settings passed by the frontend.
+ */
+
+class Process {
+    constructor(url, params, settings) {
+        this.url = url;
+        this.params = params || ' ';
+        this.settings = settings
+        this.stdout = undefined;
+        this.pid = undefined;
+        this.info = null;
+    }
+
+    /**
+     * function that launch the download process, sets the stdout property and the pid
+     * @param {Function} callback not yet implemented
+     * @returns {Promise<this>} the process instance
+     */
+    async start(callback) {
+        await this.__internalGetInfo();
+
+        const ytldp = spawn('./lib/yt-dlp',
+            [
+                '-o', `${this.settings?.download_path || 'downloads/'}%(title)s.%(ext)s`,
+                this.params,
+                this.url
+            ]
+        );
+
+        this.pid = ytldp.pid;
+        this.stdout = ytldp.stdout;
+
+        logger('proc', `Spawned a new process, pid: ${this.pid}`)
+
+        await insertDownload(this.url, null, null, null, this.pid);
+
+        return this;
+    }
+
+    /**
+     * @private
+     * function used internally by the download process to fetch information, usually thumbnail and title
+     * @returns Promise to the lock
+     */
+    async __internalGetInfo() {
+        let lock = true;
+        let stdoutChunks = [];
+        const ytdlpInfo = spawn('./lib/yt-dlp', ['-s', '-j', this.url]);
+
+        ytdlpInfo.stdout.on('data', (data) => {
+            stdoutChunks.push(data);
+        });
+
+        ytdlpInfo.on('exit', () => {
+            try {
+                const buffer = Buffer.concat(stdoutChunks);
+                const json = JSON.parse(buffer.toString());
+                this.info = json;
+                this.lock = false;
+
+            } catch (e) {
+                this.info = {
+                    title: "",
+                    thumbnail: "",
+                };
+            }
+        });
+
+        if (!lock) {
+            return true;
+        }
+    }
+
+    /**
+     * function that kills the current process
+     */
+    async kill() {
+        spawn('kill', [this.pid]).on('exit', () => {
+            deleteDownloadByPID(this.pid).then(() => {
+                logger('db', `Deleted ${this.pid} because SIGKILL`)
+            })
+        });
+    }
+
+    /**
+     * pid getter function
+     * @returns {number} pid
+     */
+    getPid() {
+        if (!this.pid) {
+            throw "Process isn't started"
+        }
+        return this.pid;
+    }
+
+    /**
+     * stdout getter function
+     * @returns {ReadableStream} stdout as stream
+     */
+    getStdout() {
+        return this.stdout
+    }
+
+    /**
+     * download info getter function
+     * @returns {object}
+     */
+    getInfo() {
+        return this.info
+    }
+}
+
+module.exports = Process;

lib/ProcessPool.js

@@ -0,0 +1,62 @@
+/**
+ * @class
+ * Represents a download process that spawns yt-dlp.
+ */
+
+class ProcessPool {
+    constructor() {
+        this._pool = new Map();
+        this._size = 0;
+    }
+
+    /**
+     * Pool size getter
+     * @returns {number} pool's size
+     */
+    size() {
+        return this._size;
+    }
+
+    /**
+     * Add a process to the pool
+     * @param {Process} process 
+     */
+    add(process) {
+        this._pool.set(process.getPid(), process)
+    }
+
+    /**
+     * Delete a process from the pool
+     * @param {Process} process 
+     */
+    remove(process) {
+        this._pool.delete(process.getPid())
+    }
+
+    /**
+     * Delete a process from the pool by its pid
+     * @param {number} pid 
+     */
+    removeByPid(pid) {
+        this._pool.delete(pid)
+    }
+
+    /**
+     * get an iterator for the pool
+     * @returns {IterableIterator} iterator 
+     */
+    iterator() {
+        return this._pool.entries()
+    }
+
+    /**
+     * get a process by its pid
+     * @param {number} pid 
+     * @returns {Process}
+     */
+    getByPid(pid) {
+        return this._pool.get(pid)
+    }
+}
+
+module.exports = ProcessPool;

lib/db.js

@@ -4,6 +4,9 @@ const { existsInProc } = require('./procUtils')
 
 const db = require('better-sqlite3')('downloads.db')
 
+/**
+ * Inits the repository, the tables.
+ */
 async function init() {
     try {
         db.exec(`CREATE TABLE downloads (
@@ -21,10 +24,23 @@ async function init() {
     }
 }
 
+/**
+ * Get an instance of the db.
+ * @returns {BetterSqlite3.Database} Current database instance
+ */
 async function get_db() {
     return db
 }
 
+/**
+ * Insert an new download to the database
+ * @param {string} url the video url
+ * @param {string} title the title fetched by the info process
+ * @param {string} thumbnail the thumbnail url fetched by the info process
+ * @param {string} size optional - the download size
+ * @param {number} PID the pid of the downloader
+ * @returns {Promise<string>} the download UUID
+ */
 async function insertDownload(url, title, thumbnail, size, PID) {
     const uid = uuid.v1()
     try {
@@ -42,28 +58,43 @@ async function insertDownload(url, title, thumbnail, size, PID) {
     return uid
 }
 
+/**
+ * Retrieve all downloads from the database
+ * @returns {ArrayLike} a collection of results
+ */
 async function retrieveAll() {
     return db
         .prepare('SELECT * FROM downloads')
         .all()
 }
 
+/**
+ * Delete a download by its uuid
+ * @param {string} uid the to-be-deleted download uuid
+ */
 async function deleteDownloadById(uid) {
     db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run()
 }
 
+/**
+ * Delete a download by its pid
+ * @param {string} pid the to-be-deleted download pid
+ */
 async function deleteDownloadByPID(PID) {
     db.prepare(`DELETE FROM downloads WHERE process_pid=${PID}`).run()
 }
 
+/**
+ * Deletes the downloads that aren't active anymore
+ * @returns {Promise<ArrayLike>}
+ */
 async function pruneDownloads() {
     const all = await retrieveAll()
     return all.map(job => {
         if (existsInProc(job.process_pid)) {
             return job
-        } else {
-            deleteDownloadByPID(job.process_pid)
         }
+        deleteDownloadByPID(job.process_pid)
     })
 }
 

lib/downloader.js

@@ -1,10 +1,18 @@
 const { spawn } = require('child_process');
 const { from, interval } = require('rxjs');
 const { throttle } = require('rxjs/operators');
-const { insertDownload, deleteDownloadByPID, pruneDownloads } = require('./db');
+const { Socket } = require('socket.io');
+const { pruneDownloads } = require('./db');
 const { logger } = require('./logger');
-const { retriveStdoutFromProcFd, killProcess } = require('./procUtils');
+const Process = require('./Process');
+const ProcessPool = require('./ProcessPool');
+const { killProcess } = require('./procUtils');
+
+// settings read from settings.json
 let settings;
+let coldRestart = true;
+
+const pool = new ProcessPool();
 
 try {
     settings = require('../settings.json');
@@ -13,10 +21,14 @@ catch (e) {
     console.warn("settings.json not found");
 }
 
-const isWindows = process.platform === 'win32';
-
+/**
+ * Invoke a new download.  
+ * Called by the websocket messages listener.
+ * @param {Socket} socket current connection socket
+ * @param {object} payload frontend download payload
+ * @returns 
+ */
 async function download(socket, payload) {
-
     if (!payload || payload.url === '' || payload.url === null) {
         socket.emit('progress', { status: 'Done!' });
         return;
@@ -25,82 +37,124 @@ async function download(socket, payload) {
     const url = payload.url
     const params = payload.params?.xa ? '-x' : '';
 
-    await getDownloadInfo(socket, url);
+    const p = new Process(url, params, settings);
 
-    const ytldp = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`,
-        [
-            '-o', `${settings.download_path || 'downloads/'}%(title)s.%(ext)s`,
-            params,
-            url
-        ]
-    );
+    p.start().then(downloader => {
+        pool.add(p)
+        let infoLock = true;
+        let pid = downloader.getPid();
 
-    await insertDownload(url, null, null, null, ytldp.pid);
-
-    from(ytldp.stdout)  // stdout as observable
-        .pipe(throttle(() => interval(500)))  // discard events closer than 500ms
-        .subscribe({
-            next: (stdout) => {
-                //let _stdout = String(stdout)
-                socket.emit('progress', formatter(String(stdout))) // finally, emit
-                //logger('download', `Fetching ${_stdout}`)
-            },
-            complete: () => {
-                socket.emit('progress', { status: 'Done!' })
-            }
-        });
-
-    ytldp.on('exit', () => {
-        socket.emit('progress', { status: 'Done!' })
-        logger('download', 'Done!')
-
-        deleteDownloadByPID(ytldp.pid).then(() => {
-            logger('db', `Deleted ${ytldp.pid} because SIGKILL`)
-        })
+        from(downloader.getStdout())              // stdout as observable
+            .pipe(throttle(() => interval(500)))  // discard events closer than 500ms
+            .subscribe({
+                next: (stdout) => {
+                    if (infoLock) {
+                        if (downloader.getInfo() === null) {
+                            return;
+                        }
+                        socket.emit('info', downloader.getInfo());
+                        infoLock = false;
+                    }
+                    socket.emit('progress', formatter(String(stdout), pid)) // finally, emit
+                },
+                complete: () => {
+                    downloader.kill().then(() => {
+                        socket.emit('progress', {
+                            status: 'Done!',
+                            process: pid,
+                        })
+                        pool.remove(downloader);
+                    })
+                },
+                error: () => {
+                    socket.emit('progress', { status: 'Done!' });
+                }
+            });
     })
 }
 
+/**
+ * Retrieve all downloads.  
+ * If the server has just been launched retrieve the ones saved to the database.  
+ * If the server is running fetches them from the process pool.
+ * @param {Socket} socket current connection socket
+ * @returns 
+ */
 async function retriveDownload(socket) {
-    const downloads = await pruneDownloads();
-    if (downloads.length > 0) {
-        for (const _download of downloads) {
-            await killProcess(_download.process_pid);
-            await download(socket, _download);
+    // it's a cold restart: the server has just been started with pending
+    // downloads, so fetch them from the database and resume.
+    if (coldRestart) {
+        coldRestart = false;
+        let downloads = await pruneDownloads();
+        downloads = [... new Set(downloads)];
+        logger('dl', `Cold restart, retrieving ${downloads.length} jobs`)
+        for (const entry of downloads) {
+            if (entry) {
+                await download(socket, entry);
+            }
         }
+        return;
+    }
+
+    // it's an hot-reload the server it's running and the frontend ask for
+    // the pending job: retrieve them from the "in-memory database" (ProcessPool)
+    logger('dl', `Retrieving jobs from pool`)
+    const it = pool.iterator();
+
+    for (const entry of it) {
+        const [pid, process] = entry;
+        await killProcess(pid);
+        await download(socket, {
+            url: process.url,
+            params: process.params
+        });
     }
 }
 
-async function getDownloadInfo(socket, url) {
-    let stdoutChunks = [];
-    const ytdlpInfo = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`, ['-s', '-j', url]);
+/**
+ * Abort a specific download if pid is provided, in the other case
+ * calls the abortAllDownloads function
+ * @see abortAllDownloads
+ * @param {Socket} socket currenct connection socket
+ * @param {*} args args sent by the frontend. MUST contain the PID.
+ * @returns 
+ */
+function abortDownload(socket, args) {
+    if (!args) {
+        abortAllDownloads(socket);
+        return;
+    }
+    const { pid } = args;
 
-    ytdlpInfo.stdout.on('data', (data) => {
-        stdoutChunks.push(data);
-    });
+    spawn('kill', [pid])
+        .on('exit', () => {
+            socket.emit('progress', {
+                status: 'Aborted',
+                process: pid,
+            });
+            logger('dl', `Aborting download ${pid}`);
+        });
+}
 
-    ytdlpInfo.on('exit', () => {
-        try {
-            const buffer = Buffer.concat(stdoutChunks);
-            const json = JSON.parse(buffer.toString());
-            socket.emit('info', json);
-        } catch (e) {
+/**
+ * Unconditionally kills all yt-dlp process.
+ * @param {Socket} socket currenct connection socket
+ */
+function abortAllDownloads(socket) {
+    spawn('killall', ['yt-dlp'])
+        .on('exit', () => {
             socket.emit('progress', { status: 'Aborted' });
-            logger('download', 'Done!');
-        }
-    })
+            logger('dl', 'Aborting downloads');
+        });
 }
 
-function abortDownload(socket) {
-    const res = process.platform === 'win32' ?
-        spawn('taskkill', ['/IM', 'yt-dlp.exe', '/F', '/T']) :
-        spawn('killall', ['yt-dlp']);
-    res.on('exit', () => {
-        socket.emit('progress', { status: 'Aborted' });
-        logger('download', 'Aborting downloads');
-    });
-}
-
-const formatter = (stdout) => {
+/**
+ * @private Formats the yt-dlp stdout to a frontend-readable format
+ * @param {string} stdout stdout as string
+ * @param {number} pid current process id relative to stdout
+ * @returns 
+ */
+const formatter = (stdout, pid) => {
     const cleanStdout = stdout
         .replace(/\s\s+/g, ' ')
         .split(' ');
@@ -112,6 +166,7 @@ const formatter = (stdout) => {
                 progress: cleanStdout[1],
                 size: cleanStdout[3],
                 dlSpeed: cleanStdout[5],
+                pid: pid,
             }
         case 'merge':
             return {
@@ -126,5 +181,6 @@ const formatter = (stdout) => {
 module.exports = {
     download: download,
     abortDownload: abortDownload,
+    abortAllDownloads: abortAllDownloads,
     retriveDownload: retriveDownload,
 }

lib/logger.js

@@ -1,7 +1,16 @@
+/**
+ * Simplest logger function, takes two argument: first one put between
+ * square brackets (the protocol), the second one it's the effective message
+ * @param {string} proto protocol
+ * @param {string} args message
+ */
 const logger = (proto, args) => {
     console.log(`[${proto}]\t${args}`)
 }
 
+/**
+ * CLI splash
+ */
 const splash = () => {
     console.log("-------------------------------------------------")
     console.log("yt-dlp-webUI - A web-ui for yt-dlp, simply enough")

lib/procUtils.js

@@ -3,6 +3,11 @@ const fs = require('fs');
 const net = require('net');
 const { logger } = require('./logger');
 
+/**
+ * Browse /proc in order to find the specific pid
+ * @param {number} pid 
+ * @returns {*} process stats if any
+ */
 function existsInProc(pid) {
     try {
         return fs.statSync(`/proc/${pid}`)
@@ -24,6 +29,10 @@ function retriveStdoutFromProcFd(pid) {
 }
 */
 
+/**
+ * Kills a process with a sys-call
+ * @param {number} pid the killed process pid
+ */
 async function killProcess(pid) {
     const res = spawn('kill', [pid])
     res.on('exit', () => {

lib/updater.js

@@ -1,6 +1,7 @@
 const https = require('https');
 const fs = require('fs');
 const path = require('path');
+const { Socket } = require('socket.io');
 
 // endpoint to github API
 const options = {
@@ -13,7 +14,11 @@ const options = {
     port: 443,
 }
 
-// build the binary url based on the release tag
+/**
+ * Build the binary url based on the release tag
+ * @param {string} release yt-dlp GitHub release tag
+ * @returns {*} the fetch options with the correct tag and headers
+ */
 function buildDonwloadOptions(release) {
     return {
         hostname: 'github.com',
@@ -26,7 +31,9 @@ function buildDonwloadOptions(release) {
     }
 }
 
-// main
+/**
+ * gets the yt-dlp latest binary URL from GitHub API
+ */
 async function update() {
     // ensure that the binary has been removed
     try {
@@ -53,7 +60,10 @@ async function update() {
         })
     })
 }
-
+/**
+ * Utility that Pipes the latest binary to a file
+ * @param {string} url yt-dlp GitHub release url
+ */
 function downloadBinary(url) {
     https.get(url, res => {
         // if it is a redirect follow the url
@@ -70,7 +80,10 @@ function downloadBinary(url) {
         })
     })
 }
-
+/**
+ * Invoke the yt-dlp update procedure
+ * @param {Socket} socket the current connection socket
+ */
 function updateFromFrontend(socket) {
     update().then(() => {
         socket.emit('updated')