marco/yt-dlp-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deprecated sqlitedb in favor of in-memory db

Browse Source
This commit is contained in:
marcobaobao
2022-05-06 11:28:01 +02:00
parent fd304605d9
commit 1e5be1ea11
8 changed files with 953 additions and 3845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM node:18-alpine3.14
FROM node:16-alpine3.15
RUN mkdir -p /usr/src/yt-dlp-webui/download
VOLUME /usr/src/yt-dlp-webui/downloads
WORKDIR /usr/src/yt-dlp-webui

26
package.json
View File

@@ -24,23 +24,33 @@
"@mui/material": "^5.6.4",
"@reduxjs/toolkit": "^1.8.1",
"babel-plugin-dev-expression": "^0.2.3",
"better-sqlite3": "^7.4.5",
"react": "^17.0.2",
"react-dom": "^17.0.2",
"react-redux": "^8.0.1",
"react-router-dom": "^6.3.0",
"rxjs": "^7.4.0",
"socket.io": "^4.3.2",
"socket.io-client": "^4.3.2",
"rxjs": "^7.5.5",
"socket.io": "^4.5.0",
"socket.io-client": "^4.5.0",
"uuid": "^8.3.2"
},
"devDependencies": {
"@types/react-router-dom": "^5.3.3",
"@parcel/transformer-yaml": "^2.5.0",
"@types/better-sqlite3": "^7.4.2",
"@types/node": "^17.0.13",
"@types/node": "^17.0.31",
"@types/react-router-dom": "^5.3.3",
"@types/uuid": "^8.3.4",
"parcel": "^2.5.0",
"typescript": "^4.5.2"
"typescript": "^4.6.4"
},
"pnpm": {
"peerDependencyRules": {
"ignoreMissing": [
"@babel/core",
"@parcel/core",
"@parcel/core@^2.5.0",
"@babel/core@^7.0.0-0",
"@babel/core@^7.0.0",
"@babel/plugin-syntax-jsx"
]
}
}
}

4539
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

14
server/src/core/Process.ts
View File

@@ -1,7 +1,6 @@
import { spawn } from 'child_process';
import { join } from 'path';
import { Readable } from 'stream';
import { deleteDownloadByPID, insertDownload } from '../db/db';
import { ISettings } from '../interfaces/ISettings';
import Logger from '../utils/BetterLogger';
@@ -53,15 +52,6 @@ class Process {
log.info('proc', `Spawned a new process, pid: ${this.pid}`)
await insertDownload(
this.url,
this.info?.title,
this.info?.thumbnail,
null,
this.params.reduce((prev, next) => `${prev} ${next}`),
this.pid
);
return this;
}
@@ -104,9 +94,7 @@ class Process {
*/
async kill() {
spawn('kill', [String(this.pid)]).on('exit', () => {
deleteDownloadByPID(this.pid).then(() => {
log.info('db', `Deleted ${this.pid} because SIGKILL`)
})
log.info('db', `Deleted ${this.pid} because SIGKILL`)
});
}

4
server/src/core/downloader.ts
View File

@@ -1,7 +1,6 @@
import { spawn } from 'child_process';
import { from, interval } from 'rxjs';
import { throttle } from 'rxjs/operators';
import { pruneDownloads } from '../db/db';
import { killProcess } from '../utils/procUtils';
import { Socket } from 'socket.io';
import { IPayload } from '../interfaces/IPayload';
@@ -84,6 +83,7 @@ export async function download(socket: Socket, payload: IPayload) {
}
/**
* @deprecated
* Retrieve all downloads.
* If the server has just been launched retrieve the ones saved to the database.
* If the server is running fetches them from the process pool.
@@ -95,7 +95,7 @@ export async function retrieveDownload(socket: Socket) {
// downloads, so fetch them from the database and resume.
if (coldRestart) {
coldRestart = false;
let downloads = await pruneDownloads();
let downloads = [];
// sanitize
downloads = [... new Set(downloads.filter(el => el !== undefined))];
log.info('dl', `Cold restart, retrieving ${downloads.length} jobs`)

103
server/src/db/db.ts
View File

@@ -1,103 +0,0 @@
import { v1 } from 'uuid';
import { existsInProc } from '../utils/procUtils';
import { IRecord } from '../interfaces/IRecord';
import Logger from '../utils/BetterLogger';
const db = require('better-sqlite3')('downloads.db');
const log = new Logger();
/**
* Inits the repository, the tables.
*/
export async function init() {
try {
db.exec(`CREATE TABLE downloads (
uid varchar(36) NOT NULL,
url text NOT NULL,
title text,
thumbnail text,
created date,
size text,
params text,
pid int NOT NULL,
PRIMARY KEY (uid)
)`)
} catch (e) {
log.warn('db', 'Table already created, ignoring')
}
}
/**
* Get an instance of the db.
* @returns {BetterSqlite3.Database} Current database instance
*/
export async function get_db(): Promise<any> {
return db
}
/**
* Insert an new download to the database
* @param {string} url the video url
* @param {string} title the title fetched by the info process
* @param {string} thumbnail the thumbnail url fetched by the info process
* @param {string} size optional - the download size
* @param {string} params optional - the download parameters, cli arguments
* @param {number} PID the pid of the downloader
* @returns {Promise<string>} the download UUID
*/
export async function insertDownload(url: string, title: string, thumbnail: string, size: string, params: string, PID: number): Promise<string> {
const uid = v1()
try {
db
.prepare(`
INSERT INTO downloads
(uid, url, title, thumbnail, size, params, pid)
VALUES (?, ?, ?, ?, ?, ?, ?)`
)
.run(uid, url, title, thumbnail, size, params, PID)
} catch (error) {
log.err('db', error)
}
return uid
}
/**
* Retrieve all downloads from the database
* @returns {ArrayLike} a collection of results
*/
export async function retrieveAll(): Promise<Array<IRecord>> {
return db
.prepare('SELECT * FROM downloads')
.all()
}
/**
* Delete a download by its uuid
* @param {string} uid the to-be-deleted download uuid
*/
export async function deleteDownloadById(uid: string) {
db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run()
}
/**
* Delete a download by its pid
* @param {string} pid the to-be-deleted download pid
*/
export async function deleteDownloadByPID(pid: number) {
db.prepare(`DELETE FROM downloads WHERE pid=${pid}`).run()
}
/**
* Deletes the downloads that aren't active anymore
* @returns {Promise<ArrayLike>}
*/
export async function pruneDownloads(): Promise<Array<IRecord>> {
const all = await retrieveAll()
return all.map(job => {
if (existsInProc(job.pid)) {
return job
}
deleteDownloadByPID(job.pid)
})
}

103
server/src/db_deprecated/db.ts Normal file
View File

@@ -0,0 +1,103 @@
// import { v1 } from 'uuid';
// import { existsInProc } from '../utils/procUtils';
// import { IRecord } from '../interfaces/IRecord';
// import Logger from '../utils/BetterLogger';
// const db = require('better-sqlite3')('downloads.db');
// const log = new Logger();
// /**
// * Inits the repository, the tables.
// */
// export async function init() {
// try {
// db.exec(`CREATE TABLE downloads (
// uid varchar(36) NOT NULL,
// url text NOT NULL,
// title text,
// thumbnail text,
// created date,
// size text,
// params text,
// pid int NOT NULL,
// PRIMARY KEY (uid)
// )`)
// } catch (e) {
// log.warn('db', 'Table already created, ignoring')
// }
// }
// /**
// * Get an instance of the db.
// * @returns {BetterSqlite3.Database} Current database instance
// */
// export async function get_db(): Promise<any> {
// return db
// }
// /**
// * Insert an new download to the database
// * @param {string} url the video url
// * @param {string} title the title fetched by the info process
// * @param {string} thumbnail the thumbnail url fetched by the info process
// * @param {string} size optional - the download size
// * @param {string} params optional - the download parameters, cli arguments
// * @param {number} PID the pid of the downloader
// * @returns {Promise<string>} the download UUID
// */
// export async function insertDownload(url: string, title: string, thumbnail: string, size: string, params: string, PID: number): Promise<string> {
// const uid = v1()
// try {
// db
// .prepare(`
// INSERT INTO downloads
// (uid, url, title, thumbnail, size, params, pid)
// VALUES (?, ?, ?, ?, ?, ?, ?)`
// )
// .run(uid, url, title, thumbnail, size, params, PID)
// } catch (error) {
// log.err('db', error)
// }
// return uid
// }
// /**
// * Retrieve all downloads from the database
// * @returns {ArrayLike} a collection of results
// */
// export async function retrieveAll(): Promise<Array<IRecord>> {
// return db
// .prepare('SELECT * FROM downloads')
// .all()
// }
// /**
// * Delete a download by its uuid
// * @param {string} uid the to-be-deleted download uuid
// */
// export async function deleteDownloadById(uid: string) {
// db.prepare(`DELETE FROM downloads WHERE uid=${uid}`).run()
// }
// /**
// * Delete a download by its pid
// * @param {string} pid the to-be-deleted download pid
// */
// export async function deleteDownloadByPID(pid: number) {
// db.prepare(`DELETE FROM downloads WHERE pid=${pid}`).run()
// }
// /**
// * Deletes the downloads that aren't active anymore
// * @returns {Promise<ArrayLike>}
// */
// export async function pruneDownloads(): Promise<Array<IRecord>> {
// const all = await retrieveAll()
// return all.map(job => {
// if (existsInProc(job.pid)) {
// return job
// }
// deleteDownloadByPID(job.pid)
// })
// }

7
server/src/main.ts
View File

@@ -3,7 +3,6 @@ import { join } from 'path';
import { Server } from 'socket.io';
import { ytdlpUpdater } from './utils/updater';
import { download, abortDownload, retrieveDownload, abortAllDownloads } from './core/downloader';
import { init } from './db/db';
import { getFreeDiskSpace } from './utils/procUtils';
import Logger from './utils/BetterLogger';
import Jean from './core/HTTPServer';
@@ -48,15 +47,11 @@ io.on('disconnect', (socket) => {
logger('ws', `${socket.handshake.address} disconnected`)
})
server.listen(process.env.PORT || 3022)
splash()
log.info('http', `Server started on port ${process.env.PORT || 3022}`)
init()
.then(() => server.listen(process.env.PORT || 3022))
.catch(err => log.err('db', err))
/**
* Cleanup handler
*/