From aa79d8a0d0c4c69cf0b8d93ed5e171c944954186 Mon Sep 17 00:00:00 2001 From: marcobaobao Date: Tue, 4 Jan 2022 12:55:36 +0100 Subject: [PATCH] monthly update --- .dockerignore | 3 +- .github/workflows/docker-image.yml | 18 ++ .gitignore | 4 +- Dockerfile | 3 +- README.md | 11 +- downloads.db | Bin 0 -> 12288 bytes frontend/src/App.tsx | 251 ++++++++++-------- frontend/src/components/StackableInput.tsx | 34 +++ frontend/src/components/Statistics.tsx | 50 +++- frontend/src/events.ts | 3 + .../src/{interfaces.tsx => interfaces.ts} | 4 +- frontend/src/utils.ts | 13 + lib/db.js | 74 ++++++ lib/downloader.js | 87 ++++-- lib/logger.js | 13 +- lib/procUtils.js | 38 +++ package.json | 15 +- server.js | 31 ++- 18 files changed, 491 insertions(+), 161 deletions(-) create mode 100644 .github/workflows/docker-image.yml create mode 100644 downloads.db create mode 100644 frontend/src/components/StackableInput.tsx create mode 100644 frontend/src/events.ts rename frontend/src/{interfaces.tsx => interfaces.ts} (89%) create mode 100644 lib/db.js create mode 100644 lib/procUtils.js diff --git a/.dockerignore b/.dockerignore index 457610c..a4b9131 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,5 @@ package-lock.json lib/*.exe lib/yt-dlp .env -*.mp4 \ No newline at end of file +*.mp4 +*.ytdl \ No newline at end of file diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..e57d42a --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,18 @@ +name: Docker Image CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image + run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) diff --git a/.gitignore b/.gitignore index aa9ad5a..2d7069d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ node_modules lib/*.exe lib/yt-dlp .env -*.mp4 \ No newline at end of file +*.mp4 +*.ytdl +*.part \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7de5453..33c7e01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /usr/src/yt-dlp-webui COPY package*.json ./ RUN apt-get update RUN apt-get install curl ffmpeg -y +RUN apt-get install psmisc RUN npm install COPY . . RUN npm run build @@ -12,4 +13,4 @@ RUN chmod +x ./lib/fetch-yt-dlp.sh RUN ./lib/fetch-yt-dlp.sh && mv yt-dlp ./lib RUN rm -rf .parcel-cache EXPOSE 3022 -CMD [ "node" , "./server.js" ] \ No newline at end of file +CMD [ "node" , "./server.js" ] diff --git a/README.md b/README.md index c5473ab..797f40f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ # yt-dlp Web UI -A terrible web ui for yt-dlp. +A not so terrible web ui for yt-dlp. Created for the only purpose of *consume* videos from my server/nas. I will eventually make this better as soon as I can. Not in the immediate. +**Background jobs now are retrieved. It's still rudimentary but it leverages** +**on yt-dlp resume feature** + ## Now with dark mode @@ -15,9 +18,11 @@ I will eventually make this better as soon as I can. Not in the immediate. The avaible settings are currently only: - Server address - Switch theme +- Retrieve background jobs Future releases will have: -- Exctract audio +- Multi download +- ~~Exctract audio~~ - Format selection ## Docker installation @@ -46,5 +51,5 @@ node server.js ``` ## Todo list -- retrieve background tasks +- ~~retrieve background tasks~~ - better ui/ux diff --git a/downloads.db b/downloads.db new file mode 100644 index 0000000000000000000000000000000000000000..a8b42bd3ac541053804b8a45ccc3ecec959c97ee GIT binary patch literal 12288 zcmeI%!EVzq7yw|WAciJw9j6^Q)Y2r>N+hk5HftwMqo5oR-Nw3dm^c(Sc1T2;WQo&k zyMYi7zXUHHg2ezR8nfhy~lX#KMK?Ya(74+0sM%)S9{nnZDMzzRi*#5^-g@oM$EVvpi&I2ZYZ&nHk&;re^CU0aaD{B&R69=?6BwW=Pg*8ktDpHvY7 y1V8`;KmY_l00ck)1V8`;KmY_l;6Dmnc)u3=|JqtmzwM81vIkL-4eK}AEcgk4$;M^? literal 0 HcmV?d00001 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b695b14..8ce2a90 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -28,17 +28,27 @@ export function App() { const [updatedBin, setUpdatedBin] = useState(false) const [showSettings, setShowSettings] = useState(false) const [darkMode, setDarkMode] = useState(localStorage.getItem('theme') === 'dark') + const [extractAudio, setExtractAudio] = useState(localStorage.getItem('-x') === 'true') const [downloadInfo, setDownloadInfo] = useState() useEffect(() => { socket.on('connect', () => { setShowToast(true) + socket.emit('fetch-jobs') }) return () => { socket.disconnect() } }, []) + useEffect(() => { + socket.on('pending-jobs', (jobs: Array) => { + if (jobs.length > 0) { + socket.emit('retrieve-jobs') + } + }) + }, []) + useEffect(() => { darkMode ? document.body.classList.add('dark') : @@ -54,15 +64,19 @@ export function App() { useEffect(() => { socket.on('progress', (data: IMessage) => { setMessage(`operation: ${data.status || '...'} \nprogress: ${data.progress || '?'} \nsize: ${data.size || '?'} \nspeed: ${data.dlSpeed || '?'}`) - if (data.status === 'Done!') { + if (data.status === 'Done!' || data.status === 'Aborted') { setHalt(false) setMessage('Done!') setProgress(0) return } - setProgress( - Math.ceil(Number(data.progress.replace('%', ''))) - ) + if (data.progress) { + setProgress(Math.ceil(Number(data.progress.replace('%', '')))) + } + // if (data.dlSpeed) { + // const event = new CustomEvent("dlSpeed", { "detail": detectSpeed(data.dlSpeed) }); + // document.dispatchEvent(event); + // } }) }, []) @@ -75,7 +89,12 @@ export function App() { const sendUrl = () => { setHalt(true) - socket.emit('send-url', url) + socket.emit('send-url', { + url: url, + params: { + xa: extractAudio + }, + }) } const handleUrlChange = (e: React.ChangeEvent) => { @@ -119,111 +138,135 @@ export function App() { } } + const toggleExtractAudio = () => { + if (extractAudio) { + localStorage.setItem('-x', 'false') + setExtractAudio(false) + } else { + localStorage.setItem('-x', 'true') + setExtractAudio(true) + } + } + return ( - - - -
-

yt-dlp WebUI

-
+ + + + +
+

yt-dlp WebUI

+
-
- - - +
+ + + -
- - {downloadInfo ?

{downloadInfo.title}

: null} - -
Status
- {!message ? 
Ready
: null} - 
{message}
- - -
- - - +
+ + {downloadInfo ?

{downloadInfo.title}

: null} + +
Status
+ {!message ? 
Ready
: null} + 
{message}
+ + +
+ + + + {/* + + */} +
+ + + + + + + {progress ? : null}
- - - - - {progress ? : null} -
+
+ setShowSettings(!showSettings)}>Settings +
- -
- setShowSettings(!showSettings)}>Settings -
- - {showSettings ? -
-
Server address
- - ws:// - - :3022 - - {' '} - -
: - null - } - -
-
Once you close this page the download will continue in the background.
-
It won't be possible retriving the progress though.
-
- Made with ❤️ by Marcobaobao - - - setShowToast(false)} - bg={'primary'} - delay={1500} - autohide - className="mt-5" - > - - {`Connected to ${localStorage.getItem('server-addr') || 'localhost'}`} - - - setUpdatedBin(false)} - bg={'primary'} - delay={1500} - autohide - className="mt-5" - > - - Updated yt-dlp binary! - - - - - + {showSettings ? +
+
Server address
+ + ws:// + + :3022 + + {' '} + +
+ toggleExtractAudio()} checked={extractAudio} /> + +
+
: + null + } +
+
+ + Once you close this page the download will continue in the background. + +
+ + + setShowToast(false)} + bg={'primary'} + delay={1500} + autohide + className="mt-5" + > + + {`Connected to ${localStorage.getItem('server-addr') || 'localhost'}`} + + + setUpdatedBin(false)} + bg={'primary'} + delay={1500} + autohide + className="mt-5" + > + + Updated yt-dlp binary! + + + + + +
+ Made with ❤️ by Marcobaobao +
+ ) } \ No newline at end of file diff --git a/frontend/src/components/StackableInput.tsx b/frontend/src/components/StackableInput.tsx new file mode 100644 index 0000000..2c53443 --- /dev/null +++ b/frontend/src/components/StackableInput.tsx @@ -0,0 +1,34 @@ +import React from "react"; +import { + InputGroup, + FormControl, + Button, + ProgressBar +} from "react-bootstrap"; + +export function StackableInput(props: any) { + return ( + + + + + +
+
Status
+ 
{props.message}
+
+ + {props.progress ? + : + null + } + + {/* {' '} */} + {/* {' '} */} + + ) +} \ No newline at end of file diff --git a/frontend/src/components/Statistics.tsx b/frontend/src/components/Statistics.tsx index 8dc998c..ff6b522 100644 --- a/frontend/src/components/Statistics.tsx +++ b/frontend/src/components/Statistics.tsx @@ -1,12 +1,52 @@ -import React, { useState } from "react"; -import { IDLSpeed } from "../interfaces"; +import React, { useEffect, useRef, useState } from "react"; +import { Line } from "react-chartjs-2"; +import { + Chart as ChartJS, + CategoryScale, + LinearScale, + PointElement, + LineElement, + Title, + Tooltip, + Legend, +} from 'chart.js'; +import { on } from "../events"; -export function Statistics(props: any) { - const [dataset, setDataset] = useState>() +ChartJS.register( + CategoryScale, + LinearScale, + PointElement, + LineElement, + Title, + Tooltip, + Legend +); + +export function Statistics() { + const dataset = new Array(); + const chartRef = useRef(null) + + useEffect(() => { + on('dlSpeed', (data: CustomEvent) => { + dataset.push(data.detail) + chartRef.current.update() + }) + }, []) + + const data = { + labels: dataset.map(() => ''), + datasets: [ + { + data: dataset, + label: 'download speed', + borderColor: 'rgb(53, 162, 235)', + } + ] + } return (
- +
) } \ No newline at end of file diff --git a/frontend/src/events.ts b/frontend/src/events.ts new file mode 100644 index 0000000..41da9c6 --- /dev/null +++ b/frontend/src/events.ts @@ -0,0 +1,3 @@ +export function on(eventType: string, listener: any) { + document.addEventListener(eventType, listener) +} \ No newline at end of file diff --git a/frontend/src/interfaces.tsx b/frontend/src/interfaces.ts similarity index 89% rename from frontend/src/interfaces.tsx rename to frontend/src/interfaces.ts index 9b28602..5f0ef57 100644 --- a/frontend/src/interfaces.tsx +++ b/frontend/src/interfaces.ts @@ -2,7 +2,7 @@ export interface IMessage { status: string, progress?: string, size?: string, - dlSpeed?: string | IDLSpeed + dlSpeed?: string } export interface IDLInfo { @@ -15,4 +15,4 @@ export interface IDLInfo { export interface IDLSpeed { effective: number, unit: string, -} \ No newline at end of file +} diff --git a/frontend/src/utils.ts b/frontend/src/utils.ts index dc368b6..a0e17c7 100644 --- a/frontend/src/utils.ts +++ b/frontend/src/utils.ts @@ -13,4 +13,17 @@ export function ellipsis(str: string, lim: number): string { return str.length > lim ? `${str.substr(0, lim)}...` : str } return '' +} + +export function detectSpeed(str: string): number { + let effective = str.match(/[\d,]+(\.\d+)?/)[0] + const unit = str.replace(effective, '') + switch (unit) { + case 'MiB/s': + return Number(effective) * 1000 + case 'KiB/s': + return Number(effective) + default: + return 0 + } } \ No newline at end of file diff --git a/lib/db.js b/lib/db.js new file mode 100644 index 0000000..b9b9456 --- /dev/null +++ b/lib/db.js @@ -0,0 +1,74 @@ +const uuid = require('uuid') +const { logger } = require('./logger') +const { existsInProc } = require('./procUtils') + +const db = require('better-sqlite3')('downloads.db') + +async function init() { + try { + db.exec(`CREATE TABLE downloads ( + uid varchar(36) NOT NULL, + url text NOT NULL, + title text, + thumbnail text, + created date, + size text, + process_pid int NOT NULL, + PRIMARY KEY (uid) + )`) + } catch (e) { + logger('db', 'Table already created, ignoring') + } +} + +async function get_db() { + return db +} + +async function insertDownload(url, title, thumbnail, size, PID) { + const uid = uuid.v1() + db + .prepare(` + INSERT INTO downloads + (uid, url, title, thumbnail, size, process_pid) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run(uid, url, title, thumbnail, size, PID) + + return uid +} + +async function retrieveAll() { + return db + .prepare('SELECT * FROM downloads') + .all() +} + +async function deleteDownloadById(uid) { + db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run() +} + +async function deleteDownloadByPID(PID) { + db.prepare(`DELETE FROM downloads WHERE process_pid=${PID}`).run() +} + +async function pruneDownloads() { + const all = await retrieveAll() + return all.map(job => { + if (existsInProc(job.process_pid)) { + return job + } else { + deleteDownloadByPID(job.process_pid) + } + }) +} + +module.exports = { + init: init, + getDB: get_db, + insertDownload: insertDownload, + retrieveAll: retrieveAll, + deleteDownloadById: deleteDownloadById, + deleteDownloadByPID: deleteDownloadByPID, + pruneDownloads: pruneDownloads, +} \ No newline at end of file diff --git a/lib/downloader.js b/lib/downloader.js index 0b47392..e4b3be3 100644 --- a/lib/downloader.js +++ b/lib/downloader.js @@ -1,29 +1,41 @@ const { spawn } = require('child_process'); -const logger = require('./logger'); const { from, interval } = require('rxjs'); const { throttle } = require('rxjs/operators'); +const { insertDownload, deleteDownloadByPID, pruneDownloads } = require('./db'); +const { logger } = require('./logger'); +const { retriveStdoutFromProcFd, killProcess } = require('./procUtils'); let settings; try { - settings = require('../settings.json') + settings = require('../settings.json'); } catch (e) { - console.warn("settings.json not found") + console.warn("settings.json not found"); } -const isWindows = process.platform === 'win32' +const isWindows = process.platform === 'win32'; -const download = (socket, url) => { - if (url === '' || url === null) { - socket.emit('progress', { status: 'Done!' }) - return +async function download(socket, payload) { + + if (!payload || payload.url === '' || payload.url === null) { + socket.emit('progress', { status: 'Done!' }); + return; } - getDownloadInfo(socket, url) + const url = payload.url + const params = payload.params?.xa ? '-x' : ''; + + await getDownloadInfo(socket, url); const ytldp = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`, - ['-o', `${settings.download_path || 'downloads/'}%(title)s.%(ext)s`, url] - ) + [ + '-o', `${settings.download_path || 'downloads/'}%(title)s.%(ext)s`, + params, + url + ] + ); + + await insertDownload(url, null, null, null, ytldp.pid); from(ytldp.stdout) // stdout as observable .pipe(throttle(() => interval(500))) // discard events closer than 500ms @@ -31,49 +43,69 @@ const download = (socket, url) => { next: (stdout) => { //let _stdout = String(stdout) socket.emit('progress', formatter(String(stdout))) // finally, emit - //logger('download', `Fetching ${stdout}`) + //logger('download', `Fetching ${_stdout}`) }, complete: () => { socket.emit('progress', { status: 'Done!' }) } - }) + }); ytldp.on('exit', () => { socket.emit('progress', { status: 'Done!' }) logger('download', 'Done!') + + deleteDownloadByPID(ytldp.pid).then(() => { + logger('db', `Deleted ${ytldp.pid} because SIGKILL`) + }) }) } -const getDownloadInfo = (socket, url) => { +async function retriveDownload(socket) { + const downloads = await pruneDownloads(); + + if (downloads.length > 0) { + for (const _download of downloads) { + await killProcess(_download.process_pid); + await download(socket, _download.url); + } + } +} + +async function getDownloadInfo(socket, url) { let stdoutChunks = []; const ytdlpInfo = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`, ['-s', '-j', url]); ytdlpInfo.stdout.on('data', (data) => { - stdoutChunks.push(data) - }) + stdoutChunks.push(data); + }); ytdlpInfo.on('exit', () => { - const buffer = Buffer.concat(stdoutChunks) - const json = JSON.parse(buffer.toString()) - socket.emit('info', json) + try { + const buffer = Buffer.concat(stdoutChunks); + const json = JSON.parse(buffer.toString()); + socket.emit('info', json); + } catch (e) { + socket.emit('progress', { status: 'Aborted' }); + logger('download', 'Done!'); + } }) } -const abortDownload = (socket) => { +function abortDownload(socket) { const res = process.platform === 'win32' ? spawn('taskkill', ['/IM', 'yt-dlp.exe', '/F', '/T']) : - spawn('killall', ['yt-dlp']) + spawn('killall', ['yt-dlp']); res.on('exit', () => { - socket.emit('progress', 'Aborted!') - logger('download', 'Aborting downloads') - }) + socket.emit('progress', { status: 'Aborted' }); + logger('download', 'Aborting downloads'); + }); } const formatter = (stdout) => { const cleanStdout = stdout .replace(/\s\s+/g, ' ') - .split(' ') - const status = cleanStdout[0].replace(/\[|\]|\r/g, '') + .split(' '); + const status = cleanStdout[0].replace(/\[|\]|\r/g, ''); switch (status) { case 'download': return { @@ -82,7 +114,7 @@ const formatter = (stdout) => { size: cleanStdout[3], dlSpeed: cleanStdout[5], } - case 'merger': + case 'merge': return { status: 'merging', progress: '100', @@ -95,4 +127,5 @@ const formatter = (stdout) => { module.exports = { download: download, abortDownload: abortDownload, + retriveDownload: retriveDownload, } diff --git a/lib/logger.js b/lib/logger.js index 6d37787..8c4084b 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,5 +1,14 @@ const logger = (proto, args) => { - console.log(`[${proto}] ${args}`) + console.log(`[${proto}]\t${args}`) } -module.exports = logger \ No newline at end of file +const splash = () => { + console.log("-------------------------------------------------") + console.log("yt-dlp-webUI - A web-ui for yt-dlp, simply enough") + console.log("-------------------------------------------------") +} + +module.exports = { + logger: logger, + splash: splash, +} \ No newline at end of file diff --git a/lib/procUtils.js b/lib/procUtils.js new file mode 100644 index 0000000..303f16c --- /dev/null +++ b/lib/procUtils.js @@ -0,0 +1,38 @@ +const { spawn } = require('child_process'); +const fs = require('fs'); +const net = require('net'); +const { logger } = require('./logger'); + +function existsInProc(pid) { + try { + return fs.statSync(`/proc/${pid}`) + } catch (e) { + logger('proc', `pid ${pid} not found in procfs`) + } +} + +/* +function retriveStdoutFromProcFd(pid) { + if (existsInProc(pid)) { + const unixSocket = fs.readlinkSync(`/proc/${pid}/fd/1`).replace('socket:[', '127.0.0.1:').replace(']', '') + if (unixSocket) { + console.log(unixSocket) + logger('proc', `found pending job on pid: ${pid} attached to UNIX socket: ${unixSocket}`) + return net.createConnection(unixSocket) + } + } +} +*/ + +async function killProcess(pid) { + const res = spawn('kill', [pid]) + res.on('exit', () => { + logger('proc', `Successfully killed yt-dlp process, pid: ${pid}`) + }) +} + +module.exports = { + existsInProc: existsInProc, + //retriveStdoutFromProcFd: retriveStdoutFromProcFd, + killProcess: killProcess, +} \ No newline at end of file diff --git a/package.json b/package.json index 6251779..c6d6c02 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "A terrible webUI for yt-dlp, all-in-one solution.", "scripts": { - "start": "node --harmony server.js", + "start": "node server.js", "dev": "nodemon app.js", "build": "parcel build ./frontend/index.html", "fe": "parcel ./frontend/index.html --open", @@ -13,16 +13,23 @@ "license": "ISC", "dependencies": { "@koa/cors": "^3.1.0", + "better-sqlite3": "^7.4.5", + "chart.js": "^3.6.0", "koa": "^2.13.4", "koa-static": "^5.0.0", + "ordered-binary": "^1.2.1", "react": "^17.0.2", - "react-bootstrap": "^2.0.2", + "react-bootstrap": "2.0.2", + "react-chartjs-2": "^4.0.0", "react-dom": "^17.0.2", "rxjs": "^7.4.0", "socket.io": "^4.3.2", - "socket.io-client": "^4.3.2" + "socket.io-client": "^4.3.2", + "sqlite": "^4.0.23", + "uuid": "^8.3.2" }, "devDependencies": { - "parcel": "^2.0.1" + "parcel": "^2.0.1", + "typescript": "^4.5.2" } } \ No newline at end of file diff --git a/server.js b/server.js index 54a0f61..7331729 100644 --- a/server.js +++ b/server.js @@ -1,12 +1,13 @@ -const Koa = require('koa'); -const serve = require('koa-static'); -const path = require('path'); -const { Server } = require('socket.io'); -const { createServer } = require('http'); -const cors = require('@koa/cors'); -const logger = require('./lib/logger'); -const { download, abortDownload } = require('./lib/downloader'); -const { ytdlpUpdater } = require('./lib/updater'); +const Koa = require('koa'), + serve = require('koa-static'), + cors = require('@koa/cors'), + { logger, splash } = require('./lib/logger'), + { join } = require('path'), + { Server } = require('socket.io'), + { createServer } = require('http'), + { download, abortDownload, retriveDownload } = require('./lib/downloader'), + { ytdlpUpdater } = require('./lib/updater'), + db = require('./lib/db'); const app = new Koa() const server = createServer(app.callback()) @@ -30,6 +31,12 @@ io.on('connection', socket => { socket.on('update-bin', () => { ytdlpUpdater(socket) }) + socket.on('fetch-jobs', async () => { + socket.emit('pending-jobs', await db.pruneDownloads()) + }) + socket.on('retrieve-jobs', async () => { + retriveDownload(socket) + }) }) io.on('disconnect', (socket) => { @@ -38,8 +45,10 @@ io.on('disconnect', (socket) => { app .use(cors()) - .use(serve(path.join(__dirname, 'dist'))) + .use(serve(join(__dirname, 'dist'))) +splash() logger('koa', `Server started on port ${process.env.PORT || 3022}`) -server.listen(process.env.PORT || 3022) \ No newline at end of file +db.init() + .then(() => server.listen(process.env.PORT || 3022)) \ No newline at end of file