experimental multidownload implemented

README.md

@@ -6,6 +6,8 @@ I will eventually make this better as soon as I can. Not in the immediate.
 
 Changelog:
 ```
+27/01/22: Multidownload implemented!
+
 26/01/22: Multiple downloads are being implemented. Maybe by next release they will be there.
 Refactoring and JSDoc.
 
@@ -26,7 +28,7 @@ The avaible settings are currently only:
 -   Extract audio
 
 Future releases will have:
--   Multi download *on its way*
+-   ~~Multi download~~ *experimental*
 -   ~~Exctract audio~~ *done*
 -   Format selection
 
@@ -56,10 +58,6 @@ node server.js
 ```
 
 
-## Regarding multiple downloads
-There's a way to circumvent the single download restriction **BUT IT LEADS TO UNDEFINED BEHAVIOUR**.  
-Fire up multiple tabs and make a download for each tab. I know that's horrible but it's gonna be fixed by next release.
-
 ## Todo list
 - ~~retrieve background tasks~~
 - better ui/ux

frontend/src/App.tsx

@@ -1,5 +1,5 @@
 import { io } from "socket.io-client";
-import React, { useState, useEffect } from "react";
+import React, { useState, useEffect, Fragment } from "react";
 import {
     Container,
     Row,
@@ -9,27 +9,37 @@ import {
     FormControl,
     Button,
     ButtonGroup,
-    Toast,
 } from "react-bootstrap";
 import { validateDomain, validateIP } from "./utils";
-import { IDLInfo, IMessage } from "./interfaces";
+import { IDLInfo, IDLInfoBase, IMessage } from "./interfaces";
+import { MessageToast } from "./components/MessageToast";
 import './App.css';
 
 const socket = io(`http://${localStorage.getItem('server-addr') || 'localhost'}:3022`)
 
 export function App() {
 
-    const [progress, setProgress] = useState(0)
+    const [progressMap, setProgressMap] = useState(new Map<number, number>());
-    const [message, setMessage] = useState('')
+    const [messageMap, setMessageMap] = useState(new Map<number, string>());
-    const [halt, setHalt] = useState(false)
+    const [downloadInfoMap, setDownloadInfoMap] = useState(new Map<number, IDLInfoBase>());
-    const [url, setUrl] = useState('')
+
-    const [showToast, setShowToast] = useState(false)
+    const [halt, setHalt] = useState(false);
-    const [invalidIP, setInvalidIP] = useState(false)
+    const [url, setUrl] = useState('');
-    const [updatedBin, setUpdatedBin] = useState(false)
+    const [showToast, setShowToast] = useState(false);
-    const [showSettings, setShowSettings] = useState(false)
+    const [invalidIP, setInvalidIP] = useState(false);
-    const [darkMode, setDarkMode] = useState(localStorage.getItem('theme') === 'dark')
+    const [updatedBin, setUpdatedBin] = useState(false);
-    const [extractAudio, setExtractAudio] = useState(localStorage.getItem('-x') === 'true')
+    const [showSettings, setShowSettings] = useState(false);
-    const [downloadInfo, setDownloadInfo] = useState<IDLInfo>()
+    const [darkMode, setDarkMode] = useState(localStorage.getItem('theme') === 'dark');
+    const [extractAudio, setExtractAudio] = useState(localStorage.getItem('-x') === 'true');
+
+    const updateInStateMap = (k: number, v: any, target: Map<number, any>, callback: Function, remove: boolean = false) => {
+        if (remove) {
+            target.delete(k)
+            callback(new Map(target))
+            return;
+        }
+        callback(new Map(target.set(k, v)));
+    }
 
     useEffect(() => {
         socket.on('connect', () => {
@@ -42,10 +52,8 @@ export function App() {
     }, [])
 
     useEffect(() => {
-        socket.on('pending-jobs', (jobs: Array<any>) => {
+        socket.on('pending-jobs', () => {
-            //if (jobs.length > 0) {
             socket.emit('retrieve-jobs')
-            //}
         })
     }, [])
 
@@ -57,21 +65,32 @@ export function App() {
 
     useEffect(() => {
         socket.on('info', (data: IDLInfo) => {
-            setDownloadInfo(data)
+            updateInStateMap(data.pid, data.info, downloadInfoMap, setDownloadInfoMap)
         })
     }, [])
 
+
     useEffect(() => {
         socket.on('progress', (data: IMessage) => {
-            setMessage(`operation: ${data.status || '...'} \nprogress: ${data.progress || '?'} \nsize: ${data.size || '?'} \nspeed: ${data.dlSpeed || '?'}`)
             if (data.status === 'Done!' || data.status === 'Aborted') {
                 setHalt(false)
-                setMessage('Done!')
+                updateInStateMap(
-                setProgress(0)
+                    data.pid,
+                    'Done!',
+                    messageMap,
+                    setMessageMap
+                )
+                updateInStateMap(data.pid, 0, progressMap, setProgressMap)
                 return
             }
+            updateInStateMap(
+                data.pid,
+                `operation: ${data.status || '...'} \nprogress: ${data.progress || '?'} \nsize: ${data.size || '?'} \nspeed: ${data.dlSpeed || '?'}`,
+                messageMap,
+                setMessageMap
+            )
             if (data.progress) {
-                setProgress(Math.ceil(Number(data.progress.replace('%', ''))))
+                updateInStateMap(data.pid, Math.ceil(Number(data.progress.replace('%', ''))), progressMap, setProgressMap)
             }
             // if (data.dlSpeed) {
             //     const event = new CustomEvent<number>("dlSpeed", { "detail": detectSpeed(data.dlSpeed) });
@@ -95,6 +114,9 @@ export function App() {
                 xa: extractAudio
             },
         })
+        setUrl('')
+        const input: HTMLInputElement = document.getElementById('urlInput') as HTMLInputElement;
+        input.value = '';
     }
 
     const handleUrlChange = (e: React.ChangeEvent<HTMLInputElement>) => {
@@ -114,12 +136,13 @@ export function App() {
         }
     }
 
-    const abort = () => {
+    const abort = (id?: number) => {
-        setDownloadInfo({
+        if (id) {
-            title: '',
+            updateInStateMap(id, null, downloadInfoMap, setDownloadInfoMap, true)
-            thumbnail: ''
+            socket.emit('abort', { pid: id })
-        })
+            return
-        socket.emit('abort')
+        }
+        socket.emit('abort-all')
         setHalt(false)
     }
 
@@ -149,7 +172,7 @@ export function App() {
     }
 
     return (
-        <React.Fragment>
+        <main>
             <Container className="pb-5">
                 <Row>
                     <Col lg={7} xs={12}>
@@ -160,39 +183,85 @@ export function App() {
                         <div className="p-3 stack-box shadow">
                             <InputGroup>
                                 <FormControl
+                                    id="urlInput"
                                     className="url-input"
                                     placeholder="YouTube or other supported service video url"
                                     onChange={handleUrlChange}
                                 />
                             </InputGroup>
-
+                            {
-                            <div className="mt-2 status-box">
+                                Array.from(messageMap).length === 0 ?
-                                <Row>
+                                    <div className="mt-2 status-box">
-                                    {downloadInfo ? <p>{downloadInfo.title}</p> : null}
+                                        <Row>
-                                    <Col sm={9}>
+                                            <Col sm={9}>
-                                        <h6>Status</h6>
+                                                <h6>Status</h6>
-                                        {!message ? <pre>Ready</pre> : null}
+                                                <pre>Ready</pre>
-                                        <pre id='status'>{message}</pre>
+                                            </Col>
-                                    </Col>
+                                        </Row>
-                                    <Col sm={3}>
+                                    </div> : null
-                                        <br />
+                            }
-                                        <img className="img-fluid rounded" src={downloadInfo?.thumbnail} />
+                            { /*Super big brain flatMap moment*/
-                                    </Col>
+                                Array.from(messageMap).flatMap(message => (
-                                </Row>
+                                    <Fragment key={message[0]}>
-                                {/* <Col>
+                                        {
-                                <Statistics></Statistics>
+                                            /*
-                            </Col> */}
+                                                Message[0] => key, the pid which is shared with the progress Map
-                            </div>
+                                                Message[1] => value, the actual formatted message sent from server
+                                             */
+                                        }
+                                        {message[1] && message[1] !== 'Done!' ?
+                                            <div className="mt-2 status-box">
+                                                <Row>
+                                                    {
+                                                        downloadInfoMap.get(message[0]) ?
+                                                            <p>{downloadInfoMap.get(message[0]).title}</p> :
+                                                            null
+                                                    }
+                                                    <Col sm={9}>
+                                                        <h6>Status</h6>
+                                                        {!message[1] ? <pre>Ready</pre> : null}
+                                                        <pre id='status'>{message[1]}</pre>
+                                                    </Col>
+                                                    <Col sm={3}>
+                                                        <br />
+                                                        <img className="img-fluid rounded" src={
+                                                            downloadInfoMap.get(message[0]) ?
+                                                                downloadInfoMap.get(message[0]).thumbnail :
+                                                                ''
+                                                        } />
+                                                    </Col>
+                                                </Row>
+                                            </div> : null
+                                        }
+                                        {
+                                            /*
+                                                Gets the progress by indexing the map with the pid
+                                             */
+                                        }
+                                        {progressMap.get(message[0]) ?
+                                            <ProgressBar className="container-padding mt-2" now={progressMap.get(message[0])} variant="primary" /> :
+                                            null
+                                        }
+                                        {message[1] && message[1] !== 'Done!' ?
+                                            <Row>
+                                                <Col>
+                                                    <div className="px-2">
+                                                        <Button variant="" className="float-end" active size="sm" onClick={() => abort(message[0])}>x</Button>
+                                                    </div>
+                                                </Col>
+                                            </Row> : null
+                                        }
+                                    </Fragment>
+                                ))
+                            }
 
                             <ButtonGroup className="mt-2">
-                                <Button onClick={() => sendUrl()} disabled={halt}>Start</Button>
+                                <Button onClick={() => sendUrl()} disabled={false}>Start</Button>
-                                <Button active onClick={() => abort()}>Abort</Button>
+                                <Button active onClick={() => abort()}>Abort all</Button>
                             </ButtonGroup>
 
-                            {progress ? <ProgressBar className="container-padding mt-2" now={progress} variant="primary" /> : null}
                         </div>
 
-
                         <div className="my-4">
                             <span className="settings" onClick={() => setShowSettings(!showSettings)}>Settings</span>
                         </div>
@@ -237,36 +306,18 @@ export function App() {
                         </div>
                     </Col>
                     <Col>
-                        <Toast
+                        <MessageToast flag={showToast} callback={setShowToast}>
-                            show={showToast}
+                            {`Connected to ${localStorage.getItem('server-addr') || 'localhost'}`}
-                            onClose={() => setShowToast(false)}
+                        </MessageToast>
-                            bg={'primary'}
+                        <MessageToast flag={updatedBin} callback={setUpdatedBin}>
-                            delay={1500}
+                            Updated yt-dlp binary!
-                            autohide
+                        </MessageToast>
-                            className="mt-5"
-                        >
-                            <Toast.Body className="text-light">
-                                {`Connected to ${localStorage.getItem('server-addr') || 'localhost'}`}
-                            </Toast.Body>
-                        </Toast>
-                        <Toast
-                            show={updatedBin}
-                            onClose={() => setUpdatedBin(false)}
-                            bg={'primary'}
-                            delay={1500}
-                            autohide
-                            className="mt-5"
-                        >
-                            <Toast.Body className="text-light">
-                                Updated yt-dlp binary!
-                            </Toast.Body>
-                        </Toast>
                     </Col>
                 </Row>
             </Container>
             <div className="container pb-5">
                 <small>Made with ❤️ by Marcobaobao</small>
             </div>
-        </React.Fragment>
+        </main>
     )
 }

frontend/src/components/MessageToast.tsx

@@ -0,0 +1,29 @@
+import React from "react";
+import {
+    Toast,
+} from "react-bootstrap";
+
+type Props = {
+    flag: boolean,
+    callback: Function,
+    children:
+    | JSX.Element
+    | string
+}
+
+export function MessageToast({ flag, callback, children }: Props) {
+    return (
+        <Toast
+            show={flag}
+            onClose={() => callback(false)}
+            bg={'primary'}
+            delay={1500}
+            autohide
+            className="mt-5"
+        >
+            <Toast.Body className="text-light">
+                {children}
+            </Toast.Body>
+        </Toast>
+    );
+}

frontend/src/interfaces.ts

@@ -3,13 +3,20 @@ export interface IMessage {
     progress?: string,
     size?: string,
     dlSpeed?: string
+    pid: number
 }
 
-export interface IDLInfo {
+export interface IDLInfoBase {
     title: string,
     thumbnail: string,
     upload_date?: string | Date,
     duration?: number
+    resolution?: string
+}
+
+export interface IDLInfo {
+    pid: number,
+    info: IDLInfoBase
 }
 
 export interface IDLSpeed {

lib/Process.js

@@ -7,7 +7,7 @@ const { logger } = require('./logger');
  * @constructor
  * @param {string} url - The downlaod url.
  * @param {string} params - The cli arguments passed by the frontend.
- * @param {object} settings - The download settings passed by the frontend.
+ * @param {*} settings - The download settings passed by the frontend.
  */
 
 class Process {
@@ -26,7 +26,7 @@ class Process {
      * @returns {Promise<this>} the process instance
      */
     async start(callback) {
-        await this.__internalGetInfo();
+        await this.#__internalGetInfo();
 
         const ytldp = spawn('./lib/yt-dlp',
             [
@@ -51,7 +51,7 @@ class Process {
      * function used internally by the download process to fetch information, usually thumbnail and title
      * @returns Promise to the lock
      */
-    async __internalGetInfo() {
+    async #__internalGetInfo() {
         let lock = true;
         let stdoutChunks = [];
         const ytdlpInfo = spawn('./lib/yt-dlp', ['-s', '-j', this.url]);

lib/downloader.js

@@ -40,6 +40,7 @@ async function download(socket, payload) {
     const p = new Process(url, params, settings);
 
     p.start().then(downloader => {
+
         pool.add(p)
         let infoLock = true;
         let pid = downloader.getPid();
@@ -52,7 +53,9 @@ async function download(socket, payload) {
                         if (downloader.getInfo() === null) {
                             return;
                         }
-                        socket.emit('info', downloader.getInfo());
+                        socket.emit('info', {
+                            pid: pid, info: downloader.getInfo()
+                        });
                         infoLock = false;
                     }
                     socket.emit('progress', formatter(String(stdout), pid)) // finally, emit
@@ -61,13 +64,15 @@ async function download(socket, payload) {
                     downloader.kill().then(() => {
                         socket.emit('progress', {
                             status: 'Done!',
-                            process: pid,
+                            pid: pid,
                         })
                         pool.remove(downloader);
                     })
                 },
                 error: () => {
-                    socket.emit('progress', { status: 'Done!' });
+                    socket.emit('progress', {
+                        status: 'Done!', pid: pid
+                    });
                 }
             });
     })
@@ -99,14 +104,23 @@ async function retriveDownload(socket) {
     // it's an hot-reload the server it's running and the frontend ask for
     // the pending job: retrieve them from the "in-memory database" (ProcessPool)
     logger('dl', `Retrieving jobs from pool`)
-    const it = pool.iterator();
 
+    const it = pool.iterator();
+    tempWorkQueue = new Array();
+
+    // sanitize
     for (const entry of it) {
         const [pid, process] = entry;
+        pool.removeByPid(pid);
         await killProcess(pid);
+        tempWorkQueue.push(process);
+    }
+
+    // resume the jobs
+    for (const entry of tempWorkQueue) {
         await download(socket, {
-            url: process.url,
+            url: entry.url,
-            params: process.params
+            params: entry.params,
         });
     }
 }

server.js

@@ -10,7 +10,7 @@ const Koa = require('koa'),
         download,
         abortDownload,
         retriveDownload,
-        abortAllDownloads
+        abortAllDownloads,
     } = require('./lib/downloader'),
     db = require('./lib/db');
 
@@ -23,10 +23,13 @@ const io = new Server(server, {
     }
 })
 
+/*
+    WebSocket listeners
+*/
 io.on('connection', socket => {
     logger('ws', `${socket.handshake.address} connected!`)
-    // message listeners
+
-    socket.on('send-url', args => {
+    socket.on('send-url', (args) => {
         logger('ws', args?.url)
         download(socket, args)
     })
@@ -39,10 +42,10 @@ io.on('connection', socket => {
     socket.on('update-bin', () => {
         ytdlpUpdater(socket)
     })
-    socket.on('fetch-jobs', async () => {
+    socket.on('fetch-jobs', () => {
-        socket.emit('pending-jobs', await db.retrieveAll())
+        socket.emit('pending-jobs', db.retrieveAll())
     })
-    socket.on('retrieve-jobs', async () => {
+    socket.on('retrieve-jobs', () => {
         retriveDownload(socket)
     })
 })