From 9dcfade3fd889a7aa0b284e5593fc4740b5682b5 Mon Sep 17 00:00:00 2001 From: marcobaobao Date: Sun, 30 Jan 2022 00:53:08 +0100 Subject: [PATCH] server migration to TypeScript --- .dockerignore | 5 +- .gitignore | 5 +- Dockerfile | 7 +- server/fetch-yt-dlp.sh => fetch-yt-dlp.sh | 0 package.json | 19 ++- pnpm-lock.yaml | 148 +++++++++++++++++- server/{Process.js => src/core/Process.ts} | 47 ++++-- .../core/ProcessPool.ts} | 19 ++- .../{downloader.js => src/core/downloader.ts} | 47 +++--- server/{db.js => src/db/db.ts} | 39 ++--- server/src/interfaces/IPayload.ts | 7 + server/src/interfaces/IRecord.ts | 0 server.js => server/src/main.ts | 38 +++-- server/src/utils/BetterLogger.ts | 46 ++++++ server/{logger.js => src/utils/logger.ts} | 9 +- .../{procUtils.js => src/utils/procUtils.ts} | 20 +-- server/{ => src/utils}/updater.js | 6 +- tsconfig.json | 17 ++ 18 files changed, 346 insertions(+), 133 deletions(-) rename server/fetch-yt-dlp.sh => fetch-yt-dlp.sh (100%) rename server/{Process.js => src/core/Process.ts} (66%) rename server/{ProcessPool.js => src/core/ProcessPool.ts} (74%) rename server/{downloader.js => src/core/downloader.ts} (83%) rename server/{db.js => src/db/db.ts} (71%) create mode 100644 server/src/interfaces/IPayload.ts create mode 100644 server/src/interfaces/IRecord.ts rename server.js => server/src/main.ts (53%) create mode 100644 server/src/utils/BetterLogger.ts rename server/{logger.js => src/utils/logger.ts} (80%) rename server/{procUtils.js => src/utils/procUtils.ts} (70%) rename server/{ => src/utils}/updater.js (91%) create mode 100644 tsconfig.json diff --git a/.dockerignore b/.dockerignore index bfae780..2e8188d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,11 @@ node_modules dist package-lock.json +pnpm-lock.yaml .parcel-cache .git -server/*.exe -server/yt-dlp +src/server/core/*.exe +src/server/core/yt-dlp .env *.mp4 *.ytdl diff --git a/.gitignore b/.gitignore index 56fae11..ca249d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ .parcel-cache dist package-lock.json +pnpm-lock.yaml node_modules -server/*.exe -server/yt-dlp +src/server/core/*.exe +src/server/core/yt-dlp .env *.mp4 *.ytdl diff --git a/Dockerfile b/Dockerfile index 9349bcd..0944fcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,8 @@ RUN apt-get install curl ffmpeg -y RUN apt-get install psmisc RUN npm install COPY . . -RUN npm run build -RUN chmod +x ./server/fetch-yt-dlp.sh -RUN ./server/fetch-yt-dlp.sh && mv yt-dlp ./server +RUN chmod +x ./fetch-yt-dlp.sh +RUN npm run build-all RUN rm -rf .parcel-cache EXPOSE 3022 -CMD [ "node" , "./server.js" ] +CMD [ "node" , "./dist/main.js" ] diff --git a/server/fetch-yt-dlp.sh b/fetch-yt-dlp.sh similarity index 100% rename from server/fetch-yt-dlp.sh rename to fetch-yt-dlp.sh diff --git a/package.json b/package.json index 9b91cdc..ee2d031 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,29 @@ { - "name": "youtube-dlp-web", - "version": "1.0.0", + "name": "yt-dlp-webui", + "version": "1.1.0", "description": "A terrible webUI for yt-dlp, all-in-one solution.", "scripts": { "start": "node server.js", "dev": "nodemon app.js", - "build": "parcel build ./frontend/index.html", + "build": "parcel build ./frontend/index.html --dist-dir ./dist/frontend", + "build-server": "tsc --build", + "build-all": "tsc --build && npm run build && npm run fetch", + "clean": "tsc --build --clean", + "clean-all": "rm -r dist", "fe": "parcel ./frontend/index.html --open", - "fetch": "./server/fetch-yt-dlp.sh && mv yt-dlp ./server" + "fetch-dev": "./fetch-yt-dlp.sh && mv yt-dlp ./server/core", + "fetch": "./fetch-yt-dlp.sh && mv yt-dlp ./dist/core" }, "author": "marcobaobao", "license": "ISC", "dependencies": { "@koa/cors": "^3.1.0", + "@types/better-sqlite3": "^7.4.2", + "@types/koa": "^2.13.4", + "@types/koa-static": "^4.0.2", + "@types/koa__cors": "^3.1.1", + "@types/node": "^17.0.13", + "@types/uuid": "^8.3.4", "better-sqlite3": "^7.4.5", "chart.js": "^3.6.0", "koa": "^2.13.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d8acfe7..8dcb2c4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2,6 +2,12 @@ lockfileVersion: 5.3 specifiers: '@koa/cors': ^3.1.0 + '@types/better-sqlite3': ^7.4.2 + '@types/koa': ^2.13.4 + '@types/koa-static': ^4.0.2 + '@types/koa__cors': ^3.1.1 + '@types/node': ^17.0.13 + '@types/uuid': ^8.3.4 better-sqlite3: ^7.4.5 chart.js: ^3.6.0 koa: ^2.13.4 @@ -22,6 +28,12 @@ specifiers: dependencies: '@koa/cors': 3.1.0 + '@types/better-sqlite3': 7.4.2 + '@types/koa': 2.13.4 + '@types/koa-static': 4.0.2 + '@types/koa__cors': 3.1.1 + '@types/node': 17.0.13 + '@types/uuid': 8.3.4 better-sqlite3: 7.5.0 chart.js: 3.7.0 koa: 2.13.4 @@ -1162,30 +1174,139 @@ packages: engines: {node: '>=10.13.0'} dev: true + /@types/accepts/1.3.5: + resolution: {integrity: sha512-jOdnI/3qTpHABjM5cx1Hc0sKsPoYCp+DP/GJRGtDlPd7fiV9oXGGIcjW/ZOxLIvjGz8MA+uMZI9metHlgqbgwQ==} + dependencies: + '@types/node': 17.0.13 + dev: false + + /@types/better-sqlite3/7.4.2: + resolution: {integrity: sha512-HUXWMOmRgOrXJ0SKt6kxqUaZtGkr0HCuaEt/76LojT6bkTu0lb0uhr3K1su9T09mskDKyQwNMvT7WithFN10PQ==} + dependencies: + '@types/node': 17.0.13 + dev: false + + /@types/body-parser/1.19.2: + resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} + dependencies: + '@types/connect': 3.4.35 + '@types/node': 17.0.13 + dev: false + /@types/component-emitter/1.2.11: resolution: {integrity: sha512-SRXjM+tfsSlA9VuG8hGO2nft2p8zjXCK1VcC6N4NXbBbYbSia9kzCChYQajIjzIqOOOuh5Ock6MmV2oux4jDZQ==} dev: false + /@types/connect/3.4.35: + resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} + dependencies: + '@types/node': 17.0.13 + dev: false + + /@types/content-disposition/0.5.4: + resolution: {integrity: sha512-0mPF08jn9zYI0n0Q/Pnz7C4kThdSt+6LD4amsrYDDpgBfrVWa3TcCOxKX1zkGgYniGagRv8heN2cbh+CAn+uuQ==} + dev: false + /@types/cookie/0.4.1: resolution: {integrity: sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==} dev: false + /@types/cookies/0.7.7: + resolution: {integrity: sha512-h7BcvPUogWbKCzBR2lY4oqaZbO3jXZksexYJVFvkrFeLgbZjQkU4x8pRq6eg2MHXQhY0McQdqmmsxRWlVAHooA==} + dependencies: + '@types/connect': 3.4.35 + '@types/express': 4.17.13 + '@types/keygrip': 1.0.2 + '@types/node': 17.0.13 + dev: false + /@types/cors/2.8.12: resolution: {integrity: sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==} dev: false + /@types/express-serve-static-core/4.17.28: + resolution: {integrity: sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==} + dependencies: + '@types/node': 17.0.13 + '@types/qs': 6.9.7 + '@types/range-parser': 1.2.4 + dev: false + + /@types/express/4.17.13: + resolution: {integrity: sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==} + dependencies: + '@types/body-parser': 1.19.2 + '@types/express-serve-static-core': 4.17.28 + '@types/qs': 6.9.7 + '@types/serve-static': 1.13.10 + dev: false + + /@types/http-assert/1.5.3: + resolution: {integrity: sha512-FyAOrDuQmBi8/or3ns4rwPno7/9tJTijVW6aQQjK02+kOQ8zmoNg2XJtAuQhvQcy1ASJq38wirX5//9J1EqoUA==} + dev: false + + /@types/http-errors/1.8.2: + resolution: {integrity: sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==} + dev: false + /@types/http-proxy/1.17.8: resolution: {integrity: sha512-5kPLG5BKpWYkw/LVOGWpiq3nEVqxiN32rTgI53Sk12/xHFQ2rG3ehI9IO+O3W2QoKeyB92dJkoka8SUm6BX1pA==} dependencies: - '@types/node': 17.0.12 + '@types/node': 17.0.13 dev: true /@types/invariant/2.2.35: resolution: {integrity: sha512-DxX1V9P8zdJPYQat1gHyY0xj3efl8gnMVjiM9iCY6y27lj+PoQWkgjt8jDqmovPqULkKVpKRg8J36iQiA+EtEg==} dev: false - /@types/node/17.0.12: - resolution: {integrity: sha512-4YpbAsnJXWYK/fpTVFlMIcUIho2AYCi4wg5aNPrG1ng7fn/1/RZfCIpRCiBX+12RVa34RluilnvCqD+g3KiSiA==} + /@types/keygrip/1.0.2: + resolution: {integrity: sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw==} + dev: false + + /@types/koa-compose/3.2.5: + resolution: {integrity: sha512-B8nG/OoE1ORZqCkBVsup/AKcvjdgoHnfi4pZMn5UwAPCbhk/96xyv284eBYW8JlQbQ7zDmnpFr68I/40mFoIBQ==} + dependencies: + '@types/koa': 2.13.4 + dev: false + + /@types/koa-send/4.1.3: + resolution: {integrity: sha512-daaTqPZlgjIJycSTNjKpHYuKhXYP30atFc1pBcy6HHqB9+vcymDgYTguPdx9tO4HMOqNyz6bz/zqpxt5eLR+VA==} + dependencies: + '@types/koa': 2.13.4 + dev: false + + /@types/koa-static/4.0.2: + resolution: {integrity: sha512-ns/zHg+K6XVPMuohjpOlpkR1WLa4VJ9czgUP9bxkCDn0JZBtUWbD/wKDZzPGDclkQK1bpAEScufCHOy8cbfL0w==} + dependencies: + '@types/koa': 2.13.4 + '@types/koa-send': 4.1.3 + dev: false + + /@types/koa/2.13.4: + resolution: {integrity: sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw==} + dependencies: + '@types/accepts': 1.3.5 + '@types/content-disposition': 0.5.4 + '@types/cookies': 0.7.7 + '@types/http-assert': 1.5.3 + '@types/http-errors': 1.8.2 + '@types/keygrip': 1.0.2 + '@types/koa-compose': 3.2.5 + '@types/node': 17.0.13 + dev: false + + /@types/koa__cors/3.1.1: + resolution: {integrity: sha512-O7MBkCocnLrpEvkMrYAp17arUDS+KuS5bXMG/Z4aPSbrO7vrYB6YrqcsTD3Dp2OnAL3j4WME2k/x2kOcyzwNUw==} + dependencies: + '@types/koa': 2.13.4 + dev: false + + /@types/mime/1.3.2: + resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} + dev: false + + /@types/node/17.0.13: + resolution: {integrity: sha512-Y86MAxASe25hNzlDbsviXl8jQHb0RDvKt4c40ZJQ1Don0AAL0STLZSs4N+6gLEO55pedy7r2cLwS+ZDxPm/2Bw==} /@types/parse-json/4.0.0: resolution: {integrity: sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==} @@ -1195,6 +1316,14 @@ packages: resolution: {integrity: sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ==} dev: false + /@types/qs/6.9.7: + resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==} + dev: false + + /@types/range-parser/1.2.4: + resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==} + dev: false + /@types/react-transition-group/4.4.4: resolution: {integrity: sha512-7gAPz7anVK5xzbeQW9wFBDg7G++aPLAFY0QaSMOou9rJZpbuI58WAuJrgu+qR92l61grlnCUe7AFX8KGahAgug==} dependencies: @@ -1213,6 +1342,17 @@ packages: resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==} dev: false + /@types/serve-static/1.13.10: + resolution: {integrity: sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==} + dependencies: + '@types/mime': 1.3.2 + '@types/node': 17.0.13 + dev: false + + /@types/uuid/8.3.4: + resolution: {integrity: sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==} + dev: false + /@types/warning/3.0.0: resolution: {integrity: sha1-DSUBJorY+ZYrdA04fEZU9fjiPlI=} dev: false @@ -2101,7 +2241,7 @@ packages: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.12 - '@types/node': 17.0.12 + '@types/node': 17.0.13 accepts: 1.3.7 base64id: 2.0.0 cookie: 0.4.1 diff --git a/server/Process.js b/server/src/core/Process.ts similarity index 66% rename from server/Process.js rename to server/src/core/Process.ts index c949052..27b1130 100644 --- a/server/Process.js +++ b/server/src/core/Process.ts @@ -1,6 +1,10 @@ -const { spawn } = require('child_process'); -const { deleteDownloadByPID, insertDownload } = require('./db'); -const { logger } = require('./logger'); +import { spawn } from 'child_process'; +import { join } from 'path'; +import { Readable } from 'stream'; +import { deleteDownloadByPID, insertDownload } from '../db/db'; +import Logger from '../utils/BetterLogger'; + +const log = new Logger(); /** * Represents a download process that spawns yt-dlp. @@ -11,9 +15,18 @@ const { logger } = require('./logger'); */ class Process { - constructor(url, params, settings) { + private url: string; + private params: Array; + private settings: any; + private stdout: Readable; + private pid: number; + private info: any; + private lock: boolean; + private exePath = join(__dirname, 'yt-dlp'); + + constructor(url: string, params: Array, settings: any) { this.url = url; - this.params = params || ' '; + this.params = params || []; this.settings = settings this.stdout = undefined; this.pid = undefined; @@ -25,10 +38,10 @@ class Process { * @param {Function} callback not yet implemented * @returns {Promise} the process instance */ - async start(callback) { + async start(callback?: Function): Promise { await this.#__internalGetInfo(); - const ytldp = spawn('./server/yt-dlp', + const ytldp = spawn(this.exePath, ['-o', `${this.settings?.download_path || 'downloads/'}%(title)s.%(ext)s`] .concat(this.params) .concat([this.url]) @@ -37,9 +50,9 @@ class Process { this.pid = ytldp.pid; this.stdout = ytldp.stdout; - logger('proc', `Spawned a new process, pid: ${this.pid}`) + log.info('proc', `Spawned a new process, pid: ${this.pid}`) - await insertDownload(this.url, null, null, null, this.pid); + await insertDownload(this.url, this.info?.title, this.info?.thumbnail, null, this.pid); return this; } @@ -52,7 +65,7 @@ class Process { async #__internalGetInfo() { let lock = true; let stdoutChunks = []; - const ytdlpInfo = spawn('./server/yt-dlp', ['-s', '-j', this.url]); + const ytdlpInfo = spawn(this.exePath, ['-s', '-j', this.url]); ytdlpInfo.stdout.on('data', (data) => { stdoutChunks.push(data); @@ -82,9 +95,9 @@ class Process { * function that kills the current process */ async kill() { - spawn('kill', [this.pid]).on('exit', () => { + spawn('kill', [String(this.pid)]).on('exit', () => { deleteDownloadByPID(this.pid).then(() => { - logger('db', `Deleted ${this.pid} because SIGKILL`) + log.info('db', `Deleted ${this.pid} because SIGKILL`) }) }); } @@ -93,7 +106,7 @@ class Process { * pid getter function * @returns {number} pid */ - getPid() { + getPid(): number { if (!this.pid) { throw "Process isn't started" } @@ -102,9 +115,9 @@ class Process { /** * stdout getter function - * @returns {ReadableStream} stdout as stream + * @returns {Readable} stdout as stream */ - getStdout() { + getStdout(): Readable { return this.stdout } @@ -112,9 +125,9 @@ class Process { * download info getter function * @returns {object} */ - getInfo() { + getInfo(): object { return this.info } } -module.exports = Process; \ No newline at end of file +export default Process; \ No newline at end of file diff --git a/server/ProcessPool.js b/server/src/core/ProcessPool.ts similarity index 74% rename from server/ProcessPool.js rename to server/src/core/ProcessPool.ts index 22e2401..e19f93c 100644 --- a/server/ProcessPool.js +++ b/server/src/core/ProcessPool.ts @@ -3,7 +3,12 @@ * Represents a download process that spawns yt-dlp. */ +import Process from "./Process"; + class ProcessPool { + private _pool: Map; + private _size: number; + constructor() { this._pool = new Map(); this._size = 0; @@ -13,7 +18,7 @@ class ProcessPool { * Pool size getter * @returns {number} pool's size */ - size() { + size(): number { return this._size; } @@ -21,7 +26,7 @@ class ProcessPool { * Add a process to the pool * @param {Process} process */ - add(process) { + add(process: Process) { this._pool.set(process.getPid(), process) } @@ -29,7 +34,7 @@ class ProcessPool { * Delete a process from the pool * @param {Process} process */ - remove(process) { + remove(process: Process) { this._pool.delete(process.getPid()) } @@ -37,7 +42,7 @@ class ProcessPool { * Delete a process from the pool by its pid * @param {number} pid */ - removeByPid(pid) { + removeByPid(pid: number) { this._pool.delete(pid) } @@ -45,7 +50,7 @@ class ProcessPool { * get an iterator for the pool * @returns {IterableIterator} iterator */ - iterator() { + iterator(): IterableIterator<[number, Process]> { return this._pool.entries() } @@ -54,9 +59,9 @@ class ProcessPool { * @param {number} pid * @returns {Process} */ - getByPid(pid) { + getByPid(pid: number): Process { return this._pool.get(pid) } } -module.exports = ProcessPool; \ No newline at end of file +export default ProcessPool; \ No newline at end of file diff --git a/server/downloader.js b/server/src/core/downloader.ts similarity index 83% rename from server/downloader.js rename to server/src/core/downloader.ts index 36153b3..13c6268 100644 --- a/server/downloader.js +++ b/server/src/core/downloader.ts @@ -1,16 +1,18 @@ -const { spawn } = require('child_process'); -const { from, interval } = require('rxjs'); -const { throttle } = require('rxjs/operators'); -const { Socket } = require('socket.io'); -const { pruneDownloads } = require('./db'); -const { logger } = require('./logger'); -const Process = require('./Process'); -const ProcessPool = require('./ProcessPool'); -const { killProcess } = require('./procUtils'); +import { spawn } from 'child_process'; +import { from, interval } from 'rxjs'; +import { throttle } from 'rxjs/operators'; +import { pruneDownloads } from '../db/db'; +import { killProcess } from '../utils/procUtils'; +import Logger from '../utils/BetterLogger'; +import Process from './Process'; +import ProcessPool from './ProcessPool'; +import { Socket } from 'socket.io'; +import { IPayload } from '../interfaces/IPayload'; // settings read from settings.json let settings; let coldRestart = true; +const log = new Logger(); const pool = new ProcessPool(); @@ -28,7 +30,7 @@ catch (e) { * @param {object} payload frontend download payload * @returns */ -async function download(socket, payload) { +export async function download(socket: Socket, payload: IPayload) { if (!payload || payload.url === '' || payload.url === null) { socket.emit('progress', { status: 'Done!' }); return; @@ -87,14 +89,14 @@ async function download(socket, payload) { * @param {Socket} socket current connection socket * @returns */ -async function retriveDownload(socket) { +export async function retriveDownload(socket: Socket) { // it's a cold restart: the server has just been started with pending // downloads, so fetch them from the database and resume. if (coldRestart) { coldRestart = false; let downloads = await pruneDownloads(); downloads = [... new Set(downloads)]; - logger('dl', `Cold restart, retrieving ${downloads.length} jobs`) + log.info('dl', `Cold restart, retrieving ${downloads.length} jobs`) for (const entry of downloads) { if (entry) { await download(socket, entry); @@ -105,10 +107,10 @@ async function retriveDownload(socket) { // it's an hot-reload the server it's running and the frontend ask for // the pending job: retrieve them from the "in-memory database" (ProcessPool) - logger('dl', `Retrieving jobs ${pool.size()} from pool`) + log.info('dl', `Retrieving ${pool.size()} jobs from pool`) const it = pool.iterator(); - tempWorkQueue = new Array(); + const tempWorkQueue = new Array(); // sanitize for (const entry of it) { @@ -135,7 +137,7 @@ async function retriveDownload(socket) { * @param {*} args args sent by the frontend. MUST contain the PID. * @returns */ -function abortDownload(socket, args) { +export function abortDownload(socket: Socket, args: any) { if (!args) { abortAllDownloads(socket); return; @@ -148,7 +150,7 @@ function abortDownload(socket, args) { status: 'Aborted', process: pid, }); - logger('dl', `Aborting download ${pid}`); + log.warn('dl', `Aborting download ${pid}`); }); } @@ -156,11 +158,11 @@ function abortDownload(socket, args) { * Unconditionally kills all yt-dlp process. * @param {Socket} socket currenct connection socket */ -function abortAllDownloads(socket) { +export function abortAllDownloads(socket: Socket) { spawn('killall', ['yt-dlp']) .on('exit', () => { socket.emit('progress', { status: 'Aborted' }); - logger('dl', 'Aborting downloads'); + log.info('dl', 'Aborting downloads'); }); } @@ -170,7 +172,7 @@ function abortAllDownloads(socket) { * @param {number} pid current process id relative to stdout * @returns */ -const formatter = (stdout, pid) => { +const formatter = (stdout: string, pid: number) => { const cleanStdout = stdout .replace(/\s\s+/g, ' ') .split(' '); @@ -193,10 +195,3 @@ const formatter = (stdout, pid) => { return { progress: '0' } } } - -module.exports = { - download: download, - abortDownload: abortDownload, - abortAllDownloads: abortAllDownloads, - retriveDownload: retriveDownload, -} diff --git a/server/db.js b/server/src/db/db.ts similarity index 71% rename from server/db.js rename to server/src/db/db.ts index c35f4f4..9c33f2d 100644 --- a/server/db.js +++ b/server/src/db/db.ts @@ -1,13 +1,14 @@ -const uuid = require('uuid') -const { logger } = require('./logger') -const { existsInProc } = require('./procUtils') +import { v1 } from 'uuid'; +import { existsInProc } from '../utils/procUtils'; +import Logger from '../utils/BetterLogger'; +const db = require('better-sqlite3')('downloads.db'); -const db = require('better-sqlite3')('downloads.db') +const log = new Logger(); /** * Inits the repository, the tables. */ -async function init() { +export async function init() { try { db.exec(`CREATE TABLE downloads ( uid varchar(36) NOT NULL, @@ -20,7 +21,7 @@ async function init() { PRIMARY KEY (uid) )`) } catch (e) { - logger('db', 'Table already created, ignoring') + log.warn('db', 'Table already created, ignoring') } } @@ -28,7 +29,7 @@ async function init() { * Get an instance of the db. * @returns {BetterSqlite3.Database} Current database instance */ -async function get_db() { +export async function get_db(): Promise { return db } @@ -41,8 +42,8 @@ async function get_db() { * @param {number} PID the pid of the downloader * @returns {Promise} the download UUID */ -async function insertDownload(url, title, thumbnail, size, PID) { - const uid = uuid.v1() +export async function insertDownload(url: string, title: string, thumbnail: string, size: string, PID: number): Promise { + const uid = v1() try { db .prepare(` @@ -52,7 +53,7 @@ async function insertDownload(url, title, thumbnail, size, PID) { ) .run(uid, url, title, thumbnail, size, PID) } catch (error) { - logger('db', 'some error occourred') + log.err('db', 'some error occourred') } return uid @@ -62,7 +63,7 @@ async function insertDownload(url, title, thumbnail, size, PID) { * Retrieve all downloads from the database * @returns {ArrayLike} a collection of results */ -async function retrieveAll() { +export async function retrieveAll(): Promise { return db .prepare('SELECT * FROM downloads') .all() @@ -72,7 +73,7 @@ async function retrieveAll() { * Delete a download by its uuid * @param {string} uid the to-be-deleted download uuid */ -async function deleteDownloadById(uid) { +export async function deleteDownloadById(uid: string) { db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run() } @@ -80,7 +81,7 @@ async function deleteDownloadById(uid) { * Delete a download by its pid * @param {string} pid the to-be-deleted download pid */ -async function deleteDownloadByPID(PID) { +export async function deleteDownloadByPID(PID) { db.prepare(`DELETE FROM downloads WHERE process_pid=${PID}`).run() } @@ -88,7 +89,7 @@ async function deleteDownloadByPID(PID) { * Deletes the downloads that aren't active anymore * @returns {Promise} */ -async function pruneDownloads() { +export async function pruneDownloads(): Promise { const all = await retrieveAll() return all.map(job => { if (existsInProc(job.process_pid)) { @@ -96,14 +97,4 @@ async function pruneDownloads() { } deleteDownloadByPID(job.process_pid) }) -} - -module.exports = { - init: init, - getDB: get_db, - insertDownload: insertDownload, - retrieveAll: retrieveAll, - deleteDownloadById: deleteDownloadById, - deleteDownloadByPID: deleteDownloadByPID, - pruneDownloads: pruneDownloads, } \ No newline at end of file diff --git a/server/src/interfaces/IPayload.ts b/server/src/interfaces/IPayload.ts new file mode 100644 index 0000000..25f1e04 --- /dev/null +++ b/server/src/interfaces/IPayload.ts @@ -0,0 +1,7 @@ +export interface IPayload { + url: string + params: Array | string, + title?: string, + thumbnail?: string, + size?: string, +} \ No newline at end of file diff --git a/server/src/interfaces/IRecord.ts b/server/src/interfaces/IRecord.ts new file mode 100644 index 0000000..e69de29 diff --git a/server.js b/server/src/main.ts similarity index 53% rename from server.js rename to server/src/main.ts index c8cd7ba..511f8c2 100644 --- a/server.js +++ b/server/src/main.ts @@ -1,20 +1,17 @@ -const Koa = require('koa'), - serve = require('koa-static'), - cors = require('@koa/cors'), - { logger, splash } = require('./server/logger'), - { join } = require('path'), - { Server } = require('socket.io'), - { createServer } = require('http'), - { ytdlpUpdater } = require('./server/updater'), - { - download, - abortDownload, - retriveDownload, - abortAllDownloads, - } = require('./server/downloader'), - db = require('./server/db'); +import Koa from 'koa'; +import serve from 'koa-static'; +import cors from '@koa/cors'; +import { logger, splash } from './utils/logger'; +import { join } from 'path'; +import { Server } from 'socket.io'; +import { createServer } from 'http'; +import { ytdlpUpdater } from './utils/updater'; +import { download, abortDownload, retriveDownload, abortAllDownloads } from './core/downloader'; +import Logger from './utils/BetterLogger'; +import { retrieveAll, init } from './db/db'; const app = new Koa() +const log = new Logger() const server = createServer(app.callback()) const io = new Server(server, { cors: { @@ -43,7 +40,7 @@ io.on('connection', socket => { ytdlpUpdater(socket) }) socket.on('fetch-jobs', () => { - socket.emit('pending-jobs', db.retrieveAll()) + socket.emit('pending-jobs', retrieveAll()) }) socket.on('retrieve-jobs', () => { retriveDownload(socket) @@ -56,10 +53,11 @@ io.on('disconnect', (socket) => { app .use(cors()) - .use(serve(join(__dirname, 'dist'))) + .use(serve(join(__dirname, 'frontend'))) splash() -logger('koa', `Server started on port ${process.env.PORT || 3022}`) +log.info('koa', `Server started on port ${process.env.PORT || 3022}`) -db.init() - .then(() => server.listen(process.env.PORT || 3022)) \ No newline at end of file +init() + .then(() => server.listen(process.env.PORT || 3022)) + .catch(err => log.err('db', err)) diff --git a/server/src/utils/BetterLogger.ts b/server/src/utils/BetterLogger.ts new file mode 100644 index 0000000..02902d9 --- /dev/null +++ b/server/src/utils/BetterLogger.ts @@ -0,0 +1,46 @@ +const ansi = { + reset: '\u001b[0m', + red: '\u001b[31m', + cyan: '\u001b[36m', + green: '\u001b[32m', + yellow: '\u001b[93m', +} + +class Logger { + /** + * Print a standard info message + * @param {string} proto the context/protocol/section outputting the message + * @param {string} args the acutal message + */ + info(proto: string, args: string) { + process.stdout.write( + this.#__formatter(proto, args) + ) + } + /** + * Print a warn message + * @param {string} proto the context/protocol/section outputting the message + * @param {string} args the acutal message + */ + warn(proto: string, args: string) { + process.stdout.write( + `${ansi.yellow}${this.#__formatter(proto, args)}${ansi.reset}` + ) + } + /** + * Print an error message + * @param {string} proto the context/protocol/section outputting the message + * @param {string} args the acutal message + */ + err(proto: string, args: string) { + process.stdout.write( + `${ansi.red}${this.#__formatter(proto, args)}${ansi.reset}` + ) + } + + #__formatter(proto: any, args: any) { + return `[${proto}]\t${args}\n` + } +} + +export default Logger; \ No newline at end of file diff --git a/server/logger.js b/server/src/utils/logger.ts similarity index 80% rename from server/logger.js rename to server/src/utils/logger.ts index bd5b8a1..0762b85 100644 --- a/server/logger.js +++ b/server/src/utils/logger.ts @@ -4,20 +4,15 @@ * @param {string} proto protocol * @param {string} args message */ -const logger = (proto, args) => { +export const logger = (proto: string, args: string) => { console.log(`[${proto}]\t${args}`) } /** * CLI splash */ -const splash = () => { +export const splash = () => { console.log("-------------------------------------------------") console.log("yt-dlp-webUI - A web-ui for yt-dlp, simply enough") console.log("-------------------------------------------------") -} - -module.exports = { - logger: logger, - splash: splash, } \ No newline at end of file diff --git a/server/procUtils.js b/server/src/utils/procUtils.ts similarity index 70% rename from server/procUtils.js rename to server/src/utils/procUtils.ts index 753e0f0..4a77582 100644 --- a/server/procUtils.js +++ b/server/src/utils/procUtils.ts @@ -1,14 +1,14 @@ -const { spawn } = require('child_process'); -const fs = require('fs'); -const net = require('net'); -const { logger } = require('./logger'); +import { spawn } from 'child_process'; +import fs = require('fs'); +import net = require('net'); +import { logger } from './logger'; /** * Browse /proc in order to find the specific pid * @param {number} pid * @returns {*} process stats if any */ -function existsInProc(pid) { +export function existsInProc(pid: number): any { try { return fs.statSync(`/proc/${pid}`) } catch (e) { @@ -33,15 +33,9 @@ function retriveStdoutFromProcFd(pid) { * Kills a process with a sys-call * @param {number} pid the killed process pid */ -async function killProcess(pid) { - const res = spawn('kill', [pid]) +export async function killProcess(pid: number) { + const res = spawn('kill', [String(pid)]) res.on('exit', () => { logger('proc', `Successfully killed yt-dlp process, pid: ${pid}`) }) } - -module.exports = { - existsInProc: existsInProc, - //retriveStdoutFromProcFd: retriveStdoutFromProcFd, - killProcess: killProcess, -} \ No newline at end of file diff --git a/server/updater.js b/server/src/utils/updater.js similarity index 91% rename from server/updater.js rename to server/src/utils/updater.js index af8da7d..b028483 100644 --- a/server/updater.js +++ b/server/src/utils/updater.js @@ -37,7 +37,7 @@ function buildDonwloadOptions(release) { async function update() { // ensure that the binary has been removed try { - fs.rmSync(path.join(__dirname, 'yt-dlp')) + fs.rmSync(path.join(__dirname, '..', 'core', 'yt-dlp')) } catch (e) { console.log('file not found!') @@ -70,11 +70,11 @@ function downloadBinary(url) { if (res.statusCode === 301 || res.statusCode === 302) { return downloadBinary(res.headers.location) } - let bin = fs.createWriteStream(path.join(__dirname, 'yt-dlp')) + let bin = fs.createWriteStream(path.join(__dirname, '..', 'core', 'yt-dlp')) res.pipe(bin) // once the connection has ended make the file executable res.on('end', () => { - fs.chmod(path.join(__dirname, 'yt-dlp'), 0o775, err => { + fs.chmod(path.join(__dirname, '..', 'core', 'yt-dlp'), 0o775, err => { err ? console.error('failed updating!') : console.log('done!') }) }) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8f84243 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "outDir": "./dist", + "allowJs": true, + "target": "ES2018", + "module": "commonjs", + "esModuleInterop": true, + "strict": false, + "noEmit": false + }, + "exclude": [ + "node_modules" + ], + "include": [ + "./server/src/**/*" + ] +} \ No newline at end of file