## Now with dark mode
@@ -15,9 +18,11 @@ I will eventually make this better as soon as I can. Not in the immediate.
The avaible settings are currently only:
- Server address
- Switch theme
+- Retrieve background jobs
Future releases will have:
-- Exctract audio
+- Multi download
+- ~~Exctract audio~~
- Format selection
## Docker installation
@@ -46,5 +51,5 @@ node server.js
```
## Todo list
-- retrieve background tasks
+- ~~retrieve background tasks~~
- better ui/ux
diff --git a/downloads.db b/downloads.db
new file mode 100644
index 0000000..a8b42bd
Binary files /dev/null and b/downloads.db differ
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index b695b14..8ce2a90 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -28,17 +28,27 @@ export function App() {
const [updatedBin, setUpdatedBin] = useState(false)
const [showSettings, setShowSettings] = useState(false)
const [darkMode, setDarkMode] = useState(localStorage.getItem('theme') === 'dark')
+ const [extractAudio, setExtractAudio] = useState(localStorage.getItem('-x') === 'true')
const [downloadInfo, setDownloadInfo] = useStateyt-dlp WebUI
- +yt-dlp WebUI
+ -
-
-
+ setShowToast(false)}
- bg={'primary'}
- delay={1500}
- autohide
- className="mt-5"
- >
-
- {`Connected to ${localStorage.getItem('server-addr') || 'localhost'}`}
-
-
- setUpdatedBin(false)}
- bg={'primary'}
- delay={1500}
- autohide
- className="mt-5"
- >
-
- Updated yt-dlp binary!
-
-
-
-
-
+ {showSettings ?
+ setShowToast(false)}
+ bg={'primary'}
+ delay={1500}
+ autohide
+ className="mt-5"
+ >
+
+ {`Connected to ${localStorage.getItem('server-addr') || 'localhost'}`}
+
+
+ setUpdatedBin(false)}
+ bg={'primary'}
+ delay={1500}
+ autohide
+ className="mt-5"
+ >
+
+ Updated yt-dlp binary!
+
+
+
+
+
+
+
+
+
+
+ )
+}
\ No newline at end of file
diff --git a/frontend/src/components/Statistics.tsx b/frontend/src/components/Statistics.tsx
index 8dc998c..ff6b522 100644
--- a/frontend/src/components/Statistics.tsx
+++ b/frontend/src/components/Statistics.tsx
@@ -1,12 +1,52 @@
-import React, { useState } from "react";
-import { IDLSpeed } from "../interfaces";
+import React, { useEffect, useRef, useState } from "react";
+import { Line } from "react-chartjs-2";
+import {
+ Chart as ChartJS,
+ CategoryScale,
+ LinearScale,
+ PointElement,
+ LineElement,
+ Title,
+ Tooltip,
+ Legend,
+} from 'chart.js';
+import { on } from "../events";
-export function Statistics(props: any) {
- const [dataset, setDataset] = useState>()
+ChartJS.register(
+ CategoryScale,
+ LinearScale,
+ PointElement,
+ LineElement,
+ Title,
+ Tooltip,
+ Legend
+);
+
+export function Statistics() {
+ const dataset = new Array();
+ const chartRef = useRef(null)
+
+ useEffect(() => {
+ on('dlSpeed', (data: CustomEvent) => {
+ dataset.push(data.detail)
+ chartRef.current.update()
+ })
+ }, [])
+
+ const data = {
+ labels: dataset.map(() => ''),
+ datasets: [
+ {
+ data: dataset,
+ label: 'download speed',
+ borderColor: 'rgb(53, 162, 235)',
+ }
+ ]
+ }
return (
+
+
-
-
-
-
- {progress ?
+
-
- {downloadInfo ?
-
-
-
+
+
+
+
+
+ {progress ?
- {downloadInfo.title}
: null} -Status
- {!message ?Ready: null} -
{message}
-
- -
+
+ {downloadInfo ?
+
+
+
+ {/*
+
+ {downloadInfo.title}
: null} +Status
+ {!message ?Ready: null} +
{message}
+
+ +
+ setShowSettings(!showSettings)}>Settings
+
-
-
- setShowSettings(!showSettings)}>Settings
-
-
- {showSettings ?
-
-
- ws://
- :3022
-
- {' '}
-
-
:
- null
- }
-
-
- Server address
-Once you close this page the download will continue in the background.
- It won't be possible retriving the progress though.
-
- Made with ❤️ by Marcobaobao
-
-
+
+ ws://
+ :3022
+
+ {' '}
+
+
:
+ null
+ }
+
+ Server address
+
+ toggleExtractAudio()} checked={extractAudio} />
+
+
+
+
+ Once you close this page the download will continue in the background.
+
+
+
+
+ Made with ❤️ by Marcobaobao
+
+
)
}
\ No newline at end of file
diff --git a/frontend/src/components/StackableInput.tsx b/frontend/src/components/StackableInput.tsx
new file mode 100644
index 0000000..2c53443
--- /dev/null
+++ b/frontend/src/components/StackableInput.tsx
@@ -0,0 +1,34 @@
+import React from "react";
+import {
+ InputGroup,
+ FormControl,
+ Button,
+ ProgressBar
+} from "react-bootstrap";
+
+export function StackableInput(props: any) {
+ return (
+
+
+
+ {props.progress ?
+ Status
+{props.message}
+
-
+
)
}
\ No newline at end of file
diff --git a/frontend/src/events.ts b/frontend/src/events.ts
new file mode 100644
index 0000000..41da9c6
--- /dev/null
+++ b/frontend/src/events.ts
@@ -0,0 +1,3 @@
+export function on(eventType: string, listener: any) {
+ document.addEventListener(eventType, listener)
+}
\ No newline at end of file
diff --git a/frontend/src/interfaces.tsx b/frontend/src/interfaces.ts
similarity index 89%
rename from frontend/src/interfaces.tsx
rename to frontend/src/interfaces.ts
index 9b28602..5f0ef57 100644
--- a/frontend/src/interfaces.tsx
+++ b/frontend/src/interfaces.ts
@@ -2,7 +2,7 @@ export interface IMessage {
status: string,
progress?: string,
size?: string,
- dlSpeed?: string | IDLSpeed
+ dlSpeed?: string
}
export interface IDLInfo {
@@ -15,4 +15,4 @@ export interface IDLInfo {
export interface IDLSpeed {
effective: number,
unit: string,
-}
\ No newline at end of file
+}
diff --git a/frontend/src/utils.ts b/frontend/src/utils.ts
index dc368b6..a0e17c7 100644
--- a/frontend/src/utils.ts
+++ b/frontend/src/utils.ts
@@ -13,4 +13,17 @@ export function ellipsis(str: string, lim: number): string {
return str.length > lim ? `${str.substr(0, lim)}...` : str
}
return ''
+}
+
+export function detectSpeed(str: string): number {
+ let effective = str.match(/[\d,]+(\.\d+)?/)[0]
+ const unit = str.replace(effective, '')
+ switch (unit) {
+ case 'MiB/s':
+ return Number(effective) * 1000
+ case 'KiB/s':
+ return Number(effective)
+ default:
+ return 0
+ }
}
\ No newline at end of file
diff --git a/lib/db.js b/lib/db.js
new file mode 100644
index 0000000..b9b9456
--- /dev/null
+++ b/lib/db.js
@@ -0,0 +1,74 @@
+const uuid = require('uuid')
+const { logger } = require('./logger')
+const { existsInProc } = require('./procUtils')
+
+const db = require('better-sqlite3')('downloads.db')
+
+async function init() {
+ try {
+ db.exec(`CREATE TABLE downloads (
+ uid varchar(36) NOT NULL,
+ url text NOT NULL,
+ title text,
+ thumbnail text,
+ created date,
+ size text,
+ process_pid int NOT NULL,
+ PRIMARY KEY (uid)
+ )`)
+ } catch (e) {
+ logger('db', 'Table already created, ignoring')
+ }
+}
+
+async function get_db() {
+ return db
+}
+
+async function insertDownload(url, title, thumbnail, size, PID) {
+ const uid = uuid.v1()
+ db
+ .prepare(`
+ INSERT INTO downloads
+ (uid, url, title, thumbnail, size, process_pid)
+ VALUES (?, ?, ?, ?, ?, ?)`
+ )
+ .run(uid, url, title, thumbnail, size, PID)
+
+ return uid
+}
+
+async function retrieveAll() {
+ return db
+ .prepare('SELECT * FROM downloads')
+ .all()
+}
+
+async function deleteDownloadById(uid) {
+ db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run()
+}
+
+async function deleteDownloadByPID(PID) {
+ db.prepare(`DELETE FROM downloads WHERE process_pid=${PID}`).run()
+}
+
+async function pruneDownloads() {
+ const all = await retrieveAll()
+ return all.map(job => {
+ if (existsInProc(job.process_pid)) {
+ return job
+ } else {
+ deleteDownloadByPID(job.process_pid)
+ }
+ })
+}
+
+module.exports = {
+ init: init,
+ getDB: get_db,
+ insertDownload: insertDownload,
+ retrieveAll: retrieveAll,
+ deleteDownloadById: deleteDownloadById,
+ deleteDownloadByPID: deleteDownloadByPID,
+ pruneDownloads: pruneDownloads,
+}
\ No newline at end of file
diff --git a/lib/downloader.js b/lib/downloader.js
index 0b47392..e4b3be3 100644
--- a/lib/downloader.js
+++ b/lib/downloader.js
@@ -1,29 +1,41 @@
const { spawn } = require('child_process');
-const logger = require('./logger');
const { from, interval } = require('rxjs');
const { throttle } = require('rxjs/operators');
+const { insertDownload, deleteDownloadByPID, pruneDownloads } = require('./db');
+const { logger } = require('./logger');
+const { retriveStdoutFromProcFd, killProcess } = require('./procUtils');
let settings;
try {
- settings = require('../settings.json')
+ settings = require('../settings.json');
}
catch (e) {
- console.warn("settings.json not found")
+ console.warn("settings.json not found");
}
-const isWindows = process.platform === 'win32'
+const isWindows = process.platform === 'win32';
-const download = (socket, url) => {
- if (url === '' || url === null) {
- socket.emit('progress', { status: 'Done!' })
- return
+async function download(socket, payload) {
+
+ if (!payload || payload.url === '' || payload.url === null) {
+ socket.emit('progress', { status: 'Done!' });
+ return;
}
- getDownloadInfo(socket, url)
+ const url = payload.url
+ const params = payload.params?.xa ? '-x' : '';
+
+ await getDownloadInfo(socket, url);
const ytldp = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`,
- ['-o', `${settings.download_path || 'downloads/'}%(title)s.%(ext)s`, url]
- )
+ [
+ '-o', `${settings.download_path || 'downloads/'}%(title)s.%(ext)s`,
+ params,
+ url
+ ]
+ );
+
+ await insertDownload(url, null, null, null, ytldp.pid);
from(ytldp.stdout) // stdout as observable
.pipe(throttle(() => interval(500))) // discard events closer than 500ms
@@ -31,49 +43,69 @@ const download = (socket, url) => {
next: (stdout) => {
//let _stdout = String(stdout)
socket.emit('progress', formatter(String(stdout))) // finally, emit
- //logger('download', `Fetching ${stdout}`)
+ //logger('download', `Fetching ${_stdout}`)
},
complete: () => {
socket.emit('progress', { status: 'Done!' })
}
- })
+ });
ytldp.on('exit', () => {
socket.emit('progress', { status: 'Done!' })
logger('download', 'Done!')
+
+ deleteDownloadByPID(ytldp.pid).then(() => {
+ logger('db', `Deleted ${ytldp.pid} because SIGKILL`)
+ })
})
}
-const getDownloadInfo = (socket, url) => {
+async function retriveDownload(socket) {
+ const downloads = await pruneDownloads();
+
+ if (downloads.length > 0) {
+ for (const _download of downloads) {
+ await killProcess(_download.process_pid);
+ await download(socket, _download.url);
+ }
+ }
+}
+
+async function getDownloadInfo(socket, url) {
let stdoutChunks = [];
const ytdlpInfo = spawn(`./lib/yt-dlp${isWindows ? '.exe' : ''}`, ['-s', '-j', url]);
ytdlpInfo.stdout.on('data', (data) => {
- stdoutChunks.push(data)
- })
+ stdoutChunks.push(data);
+ });
ytdlpInfo.on('exit', () => {
- const buffer = Buffer.concat(stdoutChunks)
- const json = JSON.parse(buffer.toString())
- socket.emit('info', json)
+ try {
+ const buffer = Buffer.concat(stdoutChunks);
+ const json = JSON.parse(buffer.toString());
+ socket.emit('info', json);
+ } catch (e) {
+ socket.emit('progress', { status: 'Aborted' });
+ logger('download', 'Done!');
+ }
})
}
-const abortDownload = (socket) => {
+function abortDownload(socket) {
const res = process.platform === 'win32' ?
spawn('taskkill', ['/IM', 'yt-dlp.exe', '/F', '/T']) :
- spawn('killall', ['yt-dlp'])
+ spawn('killall', ['yt-dlp']);
res.on('exit', () => {
- socket.emit('progress', 'Aborted!')
- logger('download', 'Aborting downloads')
- })
+ socket.emit('progress', { status: 'Aborted' });
+ logger('download', 'Aborting downloads');
+ });
}
const formatter = (stdout) => {
const cleanStdout = stdout
.replace(/\s\s+/g, ' ')
- .split(' ')
- const status = cleanStdout[0].replace(/\[|\]|\r/g, '')
+ .split(' ');
+ const status = cleanStdout[0].replace(/\[|\]|\r/g, '');
switch (status) {
case 'download':
return {
@@ -82,7 +114,7 @@ const formatter = (stdout) => {
size: cleanStdout[3],
dlSpeed: cleanStdout[5],
}
- case 'merger':
+ case 'merge':
return {
status: 'merging',
progress: '100',
@@ -95,4 +127,5 @@ const formatter = (stdout) => {
module.exports = {
download: download,
abortDownload: abortDownload,
+ retriveDownload: retriveDownload,
}
diff --git a/lib/logger.js b/lib/logger.js
index 6d37787..8c4084b 100644
--- a/lib/logger.js
+++ b/lib/logger.js
@@ -1,5 +1,14 @@
const logger = (proto, args) => {
- console.log(`[${proto}] ${args}`)
+ console.log(`[${proto}]\t${args}`)
}
-module.exports = logger
\ No newline at end of file
+const splash = () => {
+ console.log("-------------------------------------------------")
+ console.log("yt-dlp-webUI - A web-ui for yt-dlp, simply enough")
+ console.log("-------------------------------------------------")
+}
+
+module.exports = {
+ logger: logger,
+ splash: splash,
+}
\ No newline at end of file
diff --git a/lib/procUtils.js b/lib/procUtils.js
new file mode 100644
index 0000000..303f16c
--- /dev/null
+++ b/lib/procUtils.js
@@ -0,0 +1,38 @@
+const { spawn } = require('child_process');
+const fs = require('fs');
+const net = require('net');
+const { logger } = require('./logger');
+
+function existsInProc(pid) {
+ try {
+ return fs.statSync(`/proc/${pid}`)
+ } catch (e) {
+ logger('proc', `pid ${pid} not found in procfs`)
+ }
+}
+
+/*
+function retriveStdoutFromProcFd(pid) {
+ if (existsInProc(pid)) {
+ const unixSocket = fs.readlinkSync(`/proc/${pid}/fd/1`).replace('socket:[', '127.0.0.1:').replace(']', '')
+ if (unixSocket) {
+ console.log(unixSocket)
+ logger('proc', `found pending job on pid: ${pid} attached to UNIX socket: ${unixSocket}`)
+ return net.createConnection(unixSocket)
+ }
+ }
+}
+*/
+
+async function killProcess(pid) {
+ const res = spawn('kill', [pid])
+ res.on('exit', () => {
+ logger('proc', `Successfully killed yt-dlp process, pid: ${pid}`)
+ })
+}
+
+module.exports = {
+ existsInProc: existsInProc,
+ //retriveStdoutFromProcFd: retriveStdoutFromProcFd,
+ killProcess: killProcess,
+}
\ No newline at end of file
diff --git a/package.json b/package.json
index 6251779..c6d6c02 100644
--- a/package.json
+++ b/package.json
@@ -3,7 +3,7 @@
"version": "1.0.0",
"description": "A terrible webUI for yt-dlp, all-in-one solution.",
"scripts": {
- "start": "node --harmony server.js",
+ "start": "node server.js",
"dev": "nodemon app.js",
"build": "parcel build ./frontend/index.html",
"fe": "parcel ./frontend/index.html --open",
@@ -13,16 +13,23 @@
"license": "ISC",
"dependencies": {
"@koa/cors": "^3.1.0",
+ "better-sqlite3": "^7.4.5",
+ "chart.js": "^3.6.0",
"koa": "^2.13.4",
"koa-static": "^5.0.0",
+ "ordered-binary": "^1.2.1",
"react": "^17.0.2",
- "react-bootstrap": "^2.0.2",
+ "react-bootstrap": "2.0.2",
+ "react-chartjs-2": "^4.0.0",
"react-dom": "^17.0.2",
"rxjs": "^7.4.0",
"socket.io": "^4.3.2",
- "socket.io-client": "^4.3.2"
+ "socket.io-client": "^4.3.2",
+ "sqlite": "^4.0.23",
+ "uuid": "^8.3.2"
},
"devDependencies": {
- "parcel": "^2.0.1"
+ "parcel": "^2.0.1",
+ "typescript": "^4.5.2"
}
}
\ No newline at end of file
diff --git a/server.js b/server.js
index 54a0f61..7331729 100644
--- a/server.js
+++ b/server.js
@@ -1,12 +1,13 @@
-const Koa = require('koa');
-const serve = require('koa-static');
-const path = require('path');
-const { Server } = require('socket.io');
-const { createServer } = require('http');
-const cors = require('@koa/cors');
-const logger = require('./lib/logger');
-const { download, abortDownload } = require('./lib/downloader');
-const { ytdlpUpdater } = require('./lib/updater');
+const Koa = require('koa'),
+ serve = require('koa-static'),
+ cors = require('@koa/cors'),
+ { logger, splash } = require('./lib/logger'),
+ { join } = require('path'),
+ { Server } = require('socket.io'),
+ { createServer } = require('http'),
+ { download, abortDownload, retriveDownload } = require('./lib/downloader'),
+ { ytdlpUpdater } = require('./lib/updater'),
+ db = require('./lib/db');
const app = new Koa()
const server = createServer(app.callback())
@@ -30,6 +31,12 @@ io.on('connection', socket => {
socket.on('update-bin', () => {
ytdlpUpdater(socket)
})
+ socket.on('fetch-jobs', async () => {
+ socket.emit('pending-jobs', await db.pruneDownloads())
+ })
+ socket.on('retrieve-jobs', async () => {
+ retriveDownload(socket)
+ })
})
io.on('disconnect', (socket) => {
@@ -38,8 +45,10 @@ io.on('disconnect', (socket) => {
app
.use(cors())
- .use(serve(path.join(__dirname, 'dist')))
+ .use(serve(join(__dirname, 'dist')))
+splash()
logger('koa', `Server started on port ${process.env.PORT || 3022}`)
-server.listen(process.env.PORT || 3022)
\ No newline at end of file
+db.init()
+ .then(() => server.listen(process.env.PORT || 3022))
\ No newline at end of file