marco/yt-dlp-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

server migration to TypeScript

Browse Source
This commit is contained in:
marcobaobao
2022-01-30 00:53:08 +01:00
parent 7d745ec4cc
commit 9dcfade3fd
18 changed files with 346 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
server/fetch-yt-dlp.sh
View File

@@ -1,16 +0,0 @@
#!/bin/bash
echo "Downloading latest yt-dlp build..."
rm -f yt-dlp
RELEASE=$(curl --silent "https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest" |
grep '"tag_name":' |
sed -E 's/.*"([^"]+)".*/\1/'
)
wget "https://github.com/yt-dlp/yt-dlp/releases/download/$RELEASE/yt-dlp"
chmod +x yt-dlp
echo "Done!"

47
server/Process.js → server/src/core/Process.ts
View File

@@ -1,6 +1,10 @@
const { spawn } = require('child_process');
const { deleteDownloadByPID, insertDownload } = require('./db');
const { logger } = require('./logger');
import { spawn } from 'child_process';
import { join } from 'path';
import { Readable } from 'stream';
import { deleteDownloadByPID, insertDownload } from '../db/db';
import Logger from '../utils/BetterLogger';
const log = new Logger();
/**
* Represents a download process that spawns yt-dlp.
@@ -11,9 +15,18 @@ const { logger } = require('./logger');
*/
class Process {
constructor(url, params, settings) {
private url: string;
private params: Array<string>;
private settings: any;
private stdout: Readable;
private pid: number;
private info: any;
private lock: boolean;
private exePath = join(__dirname, 'yt-dlp');
constructor(url: string, params: Array<string>, settings: any) {
this.url = url;
this.params = params || ' ';
this.params = params || [];
this.settings = settings
this.stdout = undefined;
this.pid = undefined;
@@ -25,10 +38,10 @@ class Process {
* @param {Function} callback not yet implemented
* @returns {Promise<this>} the process instance
*/
async start(callback) {
async start(callback?: Function): Promise<this> {
await this.#__internalGetInfo();
const ytldp = spawn('./server/yt-dlp',
const ytldp = spawn(this.exePath,
['-o', `${this.settings?.download_path || 'downloads/'}%(title)s.%(ext)s`]
.concat(this.params)
.concat([this.url])
@@ -37,9 +50,9 @@ class Process {
this.pid = ytldp.pid;
this.stdout = ytldp.stdout;
logger('proc', `Spawned a new process, pid: ${this.pid}`)
log.info('proc', `Spawned a new process, pid: ${this.pid}`)
await insertDownload(this.url, null, null, null, this.pid);
await insertDownload(this.url, this.info?.title, this.info?.thumbnail, null, this.pid);
return this;
}
@@ -52,7 +65,7 @@ class Process {
async #__internalGetInfo() {
let lock = true;
let stdoutChunks = [];
const ytdlpInfo = spawn('./server/yt-dlp', ['-s', '-j', this.url]);
const ytdlpInfo = spawn(this.exePath, ['-s', '-j', this.url]);
ytdlpInfo.stdout.on('data', (data) => {
stdoutChunks.push(data);
@@ -82,9 +95,9 @@ class Process {
* function that kills the current process
*/
async kill() {
spawn('kill', [this.pid]).on('exit', () => {
spawn('kill', [String(this.pid)]).on('exit', () => {
deleteDownloadByPID(this.pid).then(() => {
logger('db', `Deleted ${this.pid} because SIGKILL`)
log.info('db', `Deleted ${this.pid} because SIGKILL`)
})
});
}
@@ -93,7 +106,7 @@ class Process {
* pid getter function
* @returns {number} pid
*/
getPid() {
getPid(): number {
if (!this.pid) {
throw "Process isn't started"
}
@@ -102,9 +115,9 @@ class Process {
/**
* stdout getter function
* @returns {ReadableStream} stdout as stream
* @returns {Readable} stdout as stream
*/
getStdout() {
getStdout(): Readable {
return this.stdout
}
@@ -112,9 +125,9 @@ class Process {
* download info getter function
* @returns {object}
*/
getInfo() {
getInfo(): object {
return this.info
}
}
module.exports = Process;
export default Process;

19
server/ProcessPool.js → server/src/core/ProcessPool.ts
View File

@@ -3,7 +3,12 @@
* Represents a download process that spawns yt-dlp.
*/
import Process from "./Process";
class ProcessPool {
private _pool: Map<number, Process>;
private _size: number;
constructor() {
this._pool = new Map();
this._size = 0;
@@ -13,7 +18,7 @@ class ProcessPool {
* Pool size getter
* @returns {number} pool's size
*/
size() {
size(): number {
return this._size;
}
@@ -21,7 +26,7 @@ class ProcessPool {
* Add a process to the pool
* @param {Process} process
*/
add(process) {
add(process: Process) {
this._pool.set(process.getPid(), process)
}
@@ -29,7 +34,7 @@ class ProcessPool {
* Delete a process from the pool
* @param {Process} process
*/
remove(process) {
remove(process: Process) {
this._pool.delete(process.getPid())
}
@@ -37,7 +42,7 @@ class ProcessPool {
* Delete a process from the pool by its pid
* @param {number} pid
*/
removeByPid(pid) {
removeByPid(pid: number) {
this._pool.delete(pid)
}
@@ -45,7 +50,7 @@ class ProcessPool {
* get an iterator for the pool
* @returns {IterableIterator} iterator
*/
iterator() {
iterator(): IterableIterator<[number, Process]> {
return this._pool.entries()
}
@@ -54,9 +59,9 @@ class ProcessPool {
* @param {number} pid
* @returns {Process}
*/
getByPid(pid) {
getByPid(pid: number): Process {
return this._pool.get(pid)
}
}
module.exports = ProcessPool;
export default ProcessPool;

47
server/downloader.js → server/src/core/downloader.ts
View File

@@ -1,16 +1,18 @@
const { spawn } = require('child_process');
const { from, interval } = require('rxjs');
const { throttle } = require('rxjs/operators');
const { Socket } = require('socket.io');
const { pruneDownloads } = require('./db');
const { logger } = require('./logger');
const Process = require('./Process');
const ProcessPool = require('./ProcessPool');
const { killProcess } = require('./procUtils');
import { spawn } from 'child_process';
import { from, interval } from 'rxjs';
import { throttle } from 'rxjs/operators';
import { pruneDownloads } from '../db/db';
import { killProcess } from '../utils/procUtils';
import Logger from '../utils/BetterLogger';
import Process from './Process';
import ProcessPool from './ProcessPool';
import { Socket } from 'socket.io';
import { IPayload } from '../interfaces/IPayload';
// settings read from settings.json
let settings;
let coldRestart = true;
const log = new Logger();
const pool = new ProcessPool();
@@ -28,7 +30,7 @@ catch (e) {
* @param {object} payload frontend download payload
* @returns
*/
async function download(socket, payload) {
export async function download(socket: Socket, payload: IPayload) {
if (!payload || payload.url === '' || payload.url === null) {
socket.emit('progress', { status: 'Done!' });
return;
@@ -87,14 +89,14 @@ async function download(socket, payload) {
* @param {Socket} socket current connection socket
* @returns
*/
async function retriveDownload(socket) {
export async function retriveDownload(socket: Socket) {
// it's a cold restart: the server has just been started with pending
// downloads, so fetch them from the database and resume.
if (coldRestart) {
coldRestart = false;
let downloads = await pruneDownloads();
downloads = [... new Set(downloads)];
logger('dl', `Cold restart, retrieving ${downloads.length} jobs`)
log.info('dl', `Cold restart, retrieving ${downloads.length} jobs`)
for (const entry of downloads) {
if (entry) {
await download(socket, entry);
@@ -105,10 +107,10 @@ async function retriveDownload(socket) {
// it's an hot-reload the server it's running and the frontend ask for
// the pending job: retrieve them from the "in-memory database" (ProcessPool)
logger('dl', `Retrieving jobs ${pool.size()} from pool`)
log.info('dl', `Retrieving ${pool.size()} jobs from pool`)
const it = pool.iterator();
tempWorkQueue = new Array();
const tempWorkQueue = new Array();
// sanitize
for (const entry of it) {
@@ -135,7 +137,7 @@ async function retriveDownload(socket) {
* @param {*} args args sent by the frontend. MUST contain the PID.
* @returns
*/
function abortDownload(socket, args) {
export function abortDownload(socket: Socket, args: any) {
if (!args) {
abortAllDownloads(socket);
return;
@@ -148,7 +150,7 @@ function abortDownload(socket, args) {
status: 'Aborted',
process: pid,
});
logger('dl', `Aborting download ${pid}`);
log.warn('dl', `Aborting download ${pid}`);
});
}
@@ -156,11 +158,11 @@ function abortDownload(socket, args) {
* Unconditionally kills all yt-dlp process.
* @param {Socket} socket currenct connection socket
*/
function abortAllDownloads(socket) {
export function abortAllDownloads(socket: Socket) {
spawn('killall', ['yt-dlp'])
.on('exit', () => {
socket.emit('progress', { status: 'Aborted' });
logger('dl', 'Aborting downloads');
log.info('dl', 'Aborting downloads');
});
}
@@ -170,7 +172,7 @@ function abortAllDownloads(socket) {
* @param {number} pid current process id relative to stdout
* @returns
*/
const formatter = (stdout, pid) => {
const formatter = (stdout: string, pid: number) => {
const cleanStdout = stdout
.replace(/\s\s+/g, ' ')
.split(' ');
@@ -193,10 +195,3 @@ const formatter = (stdout, pid) => {
return { progress: '0' }
}
}
module.exports = {
download: download,
abortDownload: abortDownload,
abortAllDownloads: abortAllDownloads,
retriveDownload: retriveDownload,
}

39
server/db.js → server/src/db/db.ts
View File

@@ -1,13 +1,14 @@
const uuid = require('uuid')
const { logger } = require('./logger')
const { existsInProc } = require('./procUtils')
import { v1 } from 'uuid';
import { existsInProc } from '../utils/procUtils';
import Logger from '../utils/BetterLogger';
const db = require('better-sqlite3')('downloads.db');
const db = require('better-sqlite3')('downloads.db')
const log = new Logger();
/**
* Inits the repository, the tables.
*/
async function init() {
export async function init() {
try {
db.exec(`CREATE TABLE downloads (
uid varchar(36) NOT NULL,
@@ -20,7 +21,7 @@ async function init() {
PRIMARY KEY (uid)
)`)
} catch (e) {
logger('db', 'Table already created, ignoring')
log.warn('db', 'Table already created, ignoring')
}
}
@@ -28,7 +29,7 @@ async function init() {
* Get an instance of the db.
* @returns {BetterSqlite3.Database} Current database instance
*/
async function get_db() {
export async function get_db(): Promise<any> {
return db
}
@@ -41,8 +42,8 @@ async function get_db() {
* @param {number} PID the pid of the downloader
* @returns {Promise<string>} the download UUID
*/
async function insertDownload(url, title, thumbnail, size, PID) {
const uid = uuid.v1()
export async function insertDownload(url: string, title: string, thumbnail: string, size: string, PID: number): Promise<string> {
const uid = v1()
try {
db
.prepare(`
@@ -52,7 +53,7 @@ async function insertDownload(url, title, thumbnail, size, PID) {
)
.run(uid, url, title, thumbnail, size, PID)
} catch (error) {
logger('db', 'some error occourred')
log.err('db', 'some error occourred')
}
return uid
@@ -62,7 +63,7 @@ async function insertDownload(url, title, thumbnail, size, PID) {
* Retrieve all downloads from the database
* @returns {ArrayLike} a collection of results
*/
async function retrieveAll() {
export async function retrieveAll(): Promise<any> {
return db
.prepare('SELECT * FROM downloads')
.all()
@@ -72,7 +73,7 @@ async function retrieveAll() {
* Delete a download by its uuid
* @param {string} uid the to-be-deleted download uuid
*/
async function deleteDownloadById(uid) {
export async function deleteDownloadById(uid: string) {
db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run()
}
@@ -80,7 +81,7 @@ async function deleteDownloadById(uid) {
* Delete a download by its pid
* @param {string} pid the to-be-deleted download pid
*/
async function deleteDownloadByPID(PID) {
export async function deleteDownloadByPID(PID) {
db.prepare(`DELETE FROM downloads WHERE process_pid=${PID}`).run()
}
@@ -88,7 +89,7 @@ async function deleteDownloadByPID(PID) {
* Deletes the downloads that aren't active anymore
* @returns {Promise<ArrayLike>}
*/
async function pruneDownloads() {
export async function pruneDownloads(): Promise<any> {
const all = await retrieveAll()
return all.map(job => {
if (existsInProc(job.process_pid)) {
@@ -96,14 +97,4 @@ async function pruneDownloads() {
}
deleteDownloadByPID(job.process_pid)
})
}
module.exports = {
init: init,
getDB: get_db,
insertDownload: insertDownload,
retrieveAll: retrieveAll,
deleteDownloadById: deleteDownloadById,
deleteDownloadByPID: deleteDownloadByPID,
pruneDownloads: pruneDownloads,
}

7
server/src/interfaces/IPayload.ts Normal file
View File

@@ -0,0 +1,7 @@
export interface IPayload {
url: string
params: Array<string> | string,
title?: string,
thumbnail?: string,
size?: string,
}

0
server/src/interfaces/IRecord.ts Normal file
View File

63
server/src/main.ts Normal file
View File

@@ -0,0 +1,63 @@
import Koa from 'koa';
import serve from 'koa-static';
import cors from '@koa/cors';
import { logger, splash } from './utils/logger';
import { join } from 'path';
import { Server } from 'socket.io';
import { createServer } from 'http';
import { ytdlpUpdater } from './utils/updater';
import { download, abortDownload, retriveDownload, abortAllDownloads } from './core/downloader';
import Logger from './utils/BetterLogger';
import { retrieveAll, init } from './db/db';
const app = new Koa()
const log = new Logger()
const server = createServer(app.callback())
const io = new Server(server, {
cors: {
origin: "*",
methods: ["GET", "POST"]
}
})
/*
WebSocket listeners
*/
io.on('connection', socket => {
logger('ws', `${socket.handshake.address} connected!`)
socket.on('send-url', (args) => {
logger('ws', args?.url)
download(socket, args)
})
socket.on('abort', (args) => {
abortDownload(socket, args)
})
socket.on('abort-all', () => {
abortAllDownloads(socket)
})
socket.on('update-bin', () => {
ytdlpUpdater(socket)
})
socket.on('fetch-jobs', () => {
socket.emit('pending-jobs', retrieveAll())
})
socket.on('retrieve-jobs', () => {
retriveDownload(socket)
})
})
io.on('disconnect', (socket) => {
logger('ws', `${socket.handshake.address} disconnected`)
})
app
.use(cors())
.use(serve(join(__dirname, 'frontend')))
splash()
log.info('koa', `Server started on port ${process.env.PORT || 3022}`)
init()
.then(() => server.listen(process.env.PORT || 3022))
.catch(err => log.err('db', err))

46
server/src/utils/BetterLogger.ts Normal file
View File

@@ -0,0 +1,46 @@
const ansi = {
reset: '\u001b[0m',
red: '\u001b[31m',
cyan: '\u001b[36m',
green: '\u001b[32m',
yellow: '\u001b[93m',
}
class Logger {
/**
* Print a standard info message
* @param {string} proto the context/protocol/section outputting the message
* @param {string} args the acutal message
*/
info(proto: string, args: string) {
process.stdout.write(
this.#__formatter(proto, args)
)
}
/**
* Print a warn message
* @param {string} proto the context/protocol/section outputting the message
* @param {string} args the acutal message
*/
warn(proto: string, args: string) {
process.stdout.write(
`${ansi.yellow}${this.#__formatter(proto, args)}${ansi.reset}`
)
}
/**
* Print an error message
* @param {string} proto the context/protocol/section outputting the message
* @param {string} args the acutal message
*/
err(proto: string, args: string) {
process.stdout.write(
`${ansi.red}${this.#__formatter(proto, args)}${ansi.reset}`
)
}
#__formatter(proto: any, args: any) {
return `[${proto}]\t${args}\n`
}
}
export default Logger;

9
server/logger.js → server/src/utils/logger.ts
View File

@@ -4,20 +4,15 @@
* @param {string} proto protocol
* @param {string} args message
*/
const logger = (proto, args) => {
export const logger = (proto: string, args: string) => {
console.log(`[${proto}]\t${args}`)
}
/**
* CLI splash
*/
const splash = () => {
export const splash = () => {
console.log("-------------------------------------------------")
console.log("yt-dlp-webUI - A web-ui for yt-dlp, simply enough")
console.log("-------------------------------------------------")
}
module.exports = {
logger: logger,
splash: splash,
}

20
server/procUtils.js → server/src/utils/procUtils.ts
View File

@@ -1,14 +1,14 @@
const { spawn } = require('child_process');
const fs = require('fs');
const net = require('net');
const { logger } = require('./logger');
import { spawn } from 'child_process';
import fs = require('fs');
import net = require('net');
import { logger } from './logger';
/**
* Browse /proc in order to find the specific pid
* @param {number} pid
* @returns {*} process stats if any
*/
function existsInProc(pid) {
export function existsInProc(pid: number): any {
try {
return fs.statSync(`/proc/${pid}`)
} catch (e) {
@@ -33,15 +33,9 @@ function retriveStdoutFromProcFd(pid) {
* Kills a process with a sys-call
* @param {number} pid the killed process pid
*/
async function killProcess(pid) {
const res = spawn('kill', [pid])
export async function killProcess(pid: number) {
const res = spawn('kill', [String(pid)])
res.on('exit', () => {
logger('proc', `Successfully killed yt-dlp process, pid: ${pid}`)
})
}
module.exports = {
existsInProc: existsInProc,
//retriveStdoutFromProcFd: retriveStdoutFromProcFd,
killProcess: killProcess,
}

6
server/updater.js → server/src/utils/updater.js
View File

@@ -37,7 +37,7 @@ function buildDonwloadOptions(release) {
async function update() {
// ensure that the binary has been removed
try {
fs.rmSync(path.join(__dirname, 'yt-dlp'))
fs.rmSync(path.join(__dirname, '..', 'core', 'yt-dlp'))
}
catch (e) {
console.log('file not found!')
@@ -70,11 +70,11 @@ function downloadBinary(url) {
if (res.statusCode === 301 || res.statusCode === 302) {
return downloadBinary(res.headers.location)
}
let bin = fs.createWriteStream(path.join(__dirname, 'yt-dlp'))
let bin = fs.createWriteStream(path.join(__dirname, '..', 'core', 'yt-dlp'))
res.pipe(bin)
// once the connection has ended make the file executable
res.on('end', () => {
fs.chmod(path.join(__dirname, 'yt-dlp'), 0o775, err => {
fs.chmod(path.join(__dirname, '..', 'core', 'yt-dlp'), 0o775, err => {
err ? console.error('failed updating!') : console.log('done!')
})
})