diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8d7b524..03f4362 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -78,7 +78,6 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64/v8,linux/arm/v7 # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker diff --git a/.gitignore b/.gitignore index 6091f8e..28d2c1c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ src/server/core/yt-dlp *.mp4 *.ytdl *.part -*.db \ No newline at end of file +*.db +*.DS_Store \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 122a175..f9d05b4 100755 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -16,6 +16,7 @@ import MuiAppBar, { AppBarProps as MuiAppBarProps } from '@mui/material/AppBar'; import { ChevronLeft, Dashboard, + Download, Menu, Settings as SettingsIcon, SettingsEthernet, Storage, @@ -32,6 +33,7 @@ import Settings from "./Settings"; import { io } from "socket.io-client"; import { RootState, store } from './stores/store'; import { Provider, useSelector } from "react-redux"; +import ArchivedDownloads from "./Archived"; const drawerWidth: number = 240; @@ -166,10 +168,6 @@ function AppContent() {  {settings.serverAddr} - - - - @@ -200,6 +198,20 @@ function AppContent() { + {/* Next release: list downloaded files */} + {/* + + + + + + + */} }> }> + }> diff --git a/frontend/src/Archived.tsx b/frontend/src/Archived.tsx new file mode 100644 index 0000000..424d3e3 --- /dev/null +++ b/frontend/src/Archived.tsx @@ -0,0 +1,46 @@ +import React, { useEffect, useState } from "react"; +import { Backdrop, CircularProgress, Container, Grid } from "@mui/material"; +import { ArchiveResult } from "./components/ArchiveResult"; +import { useSelector } from "react-redux"; +import { RootState } from "./stores/store"; + +export default function archivedDownloads() { + const [loading, setLoading] = useState(true); + const [archived, setArchived] = useState([]); + + const settings = useSelector((state: RootState) => state.settings) + + useEffect(() => { + fetch(`http://${settings.serverAddr}:3022/getAllDownloaded`) + .then(res => res.json()) + .then(data => setArchived(data)) + .then(() => setLoading(false)) + }, []); + + return ( + + theme.zIndex.drawer + 1 }} + open={loading} + > + + + { + archived.length > 0 ? + + { + archived.map((el, idx) => + + + + ) + } + + : null + } + + ); +} \ No newline at end of file diff --git a/frontend/src/Home.tsx b/frontend/src/Home.tsx index bc52674..cdeeca1 100644 --- a/frontend/src/Home.tsx +++ b/frontend/src/Home.tsx @@ -4,7 +4,7 @@ import { useDispatch, useSelector } from "react-redux"; import { Socket } from "socket.io-client"; import { StackableResult } from "./components/StackableResult"; import { connected, disconnected, downloading, finished } from "./features/status/statusSlice"; -import { IDLInfo, IDLInfoBase, IMessage } from "./interfaces"; +import { IDLInfo, IDLInfoBase, IDownloadInfo, IMessage } from "./interfaces"; import { RootState } from "./stores/store"; import { updateInStateMap, } from "./utils"; @@ -48,6 +48,14 @@ export default function Home({ socket }: Props) { }) }, []) + /* Handle download information sent by server */ + useEffect(() => { + socket.on('available-formats', (data: IDownloadInfo) => { + setShowBackdrop(false) + console.log(data) + }) + }, []) + /* Handle download information sent by server */ useEffect(() => { socket.on('info', (data: IDLInfo) => { @@ -60,13 +68,6 @@ export default function Home({ socket }: Props) { /* Handle per-download progress */ useEffect(() => { socket.on('progress', (data: IMessage) => { - if (showBackdrop) { - setShowBackdrop(false) - } - if (!status.downloading) { - setShowBackdrop(false) - dispatch(downloading()) - } if (data.status === 'Done!' || data.status === 'Aborted') { setShowBackdrop(false) updateInStateMap(data.pid, 'Done!', messageMap, setMessageMap); @@ -147,11 +148,21 @@ export default function Home({ socket }: Props) { /> - + - - + diff --git a/frontend/src/components/ArchiveResult.tsx b/frontend/src/components/ArchiveResult.tsx new file mode 100644 index 0000000..2d575de --- /dev/null +++ b/frontend/src/components/ArchiveResult.tsx @@ -0,0 +1,30 @@ +import { Card, CardActionArea, CardContent, CardMedia, Skeleton, Typography } from "@mui/material"; +import { ellipsis } from "../utils"; + +type Props = { + title: string, + thumbnail: string, + url: string, +} + +export function ArchiveResult({ title, thumbnail, url }: Props) { + return ( + + window.open(url)}> + {thumbnail ? + : + + } + + + {ellipsis(title, 72)} + + + + + ) +} \ No newline at end of file diff --git a/frontend/src/components/MessageToast.tsx b/frontend/src/components/MessageToast.tsx deleted file mode 100644 index c2c57fb..0000000 --- a/frontend/src/components/MessageToast.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import React from "react"; -import { - Toast, -} from "react-bootstrap"; - -type Props = { - flag: boolean, - callback: Function, - children: - | JSX.Element - | string -} - -export function MessageToast({ flag, callback, children }: Props) { - return ( - callback(false)} - bg={'primary'} - delay={1500} - autohide - className="mt-5" - > - - {children} - - - ); -} \ No newline at end of file diff --git a/frontend/src/components/StackableResult.tsx b/frontend/src/components/StackableResult.tsx index 8fbe57b..c5cdddf 100644 --- a/frontend/src/components/StackableResult.tsx +++ b/frontend/src/components/StackableResult.tsx @@ -1,4 +1,3 @@ -import { Fragment } from "react"; import { EightK, FourK, Hd, Sd } from "@mui/icons-material"; import { Button, Card, CardActionArea, CardActions, CardContent, CardMedia, Chip, Grid, LinearProgress, Skeleton, Stack, Typography } from "@mui/material"; import { IMessage } from "../interfaces"; @@ -14,7 +13,6 @@ type Props = { } export function StackableResult({ formattedLog, title, thumbnail, resolution, progress, stopCallback }: Props) { - const guessResolution = (xByY: string): JSX.Element => { if (!xByY) return null; if (xByY.includes('4320')) return (); diff --git a/frontend/src/interfaces.ts b/frontend/src/interfaces.ts index 544651d..d3b9d03 100644 --- a/frontend/src/interfaces.ts +++ b/frontend/src/interfaces.ts @@ -14,6 +14,21 @@ export interface IDLInfoBase { resolution?: string } +export interface IDownloadInfo { + formats: Array, + best: IDownloadInfoSection, + thumbnail: string, + title: string, +} + +export interface IDownloadInfoSection { + format_id: string, + format_note: string, + fps: number, + resolution: string, + vcodec: string, +} + export interface IDLInfo { pid: number, info: IDLInfoBase diff --git a/package.json b/package.json index 707c100..f806613 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,14 @@ "dependencies": { "@emotion/react": "^11.9.0", "@emotion/styled": "^11.8.1", + "@koa/cors": "^3.3.0", "@mui/icons-material": "^5.6.2", "@mui/material": "^5.6.4", "@reduxjs/toolkit": "^1.8.1", + "koa": "^2.13.4", + "koa-router": "^10.1.1", + "koa-static": "^5.0.0", + "mime-types": "^2.1.35", "react": "^17.0.2", "react-dom": "^17.0.2", "react-redux": "^8.0.1", @@ -34,6 +39,9 @@ }, "devDependencies": { "@parcel/transformer-yaml": "^2.5.0", + "@types/koa": "^2.13.4", + "@types/koa-router": "^7.4.4", + "@types/mime-types": "^2.1.1", "@types/node": "^17.0.31", "@types/react-router-dom": "^5.3.3", "@types/uuid": "^8.3.4", diff --git a/server/src/core/Process.ts b/server/src/core/Process.ts index a8ad9dd..51d809d 100644 --- a/server/src/core/Process.ts +++ b/server/src/core/Process.ts @@ -39,9 +39,7 @@ class Process { * @param callback not yet implemented * @returns the process instance */ - async start(callback?: Function): Promise { - await this.internalGetInfo(); - + public async start(callback?: Function): Promise { const sanitizedParams = this.params.filter((param: string) => availableParams.includes(param)); const ytldp = spawn(this.exePath, @@ -59,37 +57,50 @@ class Process { } /** - * @private * function used internally by the download process to fetch information, usually thumbnail and title * @returns Promise to the lock */ - private async internalGetInfo() { - this.lock = true; + public getInfo(): Promise { let stdoutChunks = []; - const ytdlpInfo = spawn(this.exePath, ['-s', '-j', this.url]); + const ytdlpInfo = spawn(this.exePath, ['-j', this.url]); ytdlpInfo.stdout.on('data', (data) => { stdoutChunks.push(data); }); - ytdlpInfo.on('exit', () => { - try { - const buffer = Buffer.concat(stdoutChunks); - const json = JSON.parse(buffer.toString()); - this.info = json; - this.lock = false; + return new Promise((resolve, reject) => { + ytdlpInfo.on('exit', () => { + try { + const buffer = Buffer.concat(stdoutChunks); + const json = JSON.parse(buffer.toString()); + this.info = json; + this.lock = false; + resolve({ + formats: json.formats.map((format: IDownloadInfoSection) => { + return { + format_id: format.format_id ?? '', + format_note: format.format_note ?? '', + fps: format.fps ?? '', + resolution: format.resolution ?? '', + vcodec: format.vcodec ?? '', + } + }), + best: { + format_id: json.format_id ?? '', + format_note: json.format_note ?? '', + fps: json.fps ?? '', + resolution: json.resolution ?? '', + vcodec: json.vcodec ?? '', + }, + thumbnail: json.thumbnail, + title: json.title, + }); - } catch (e) { - this.info = { - title: "", - thumbnail: "", - }; - } - }); - - if (!this.lock) { - return true; - } + } catch (e) { + reject('failed fetching formats, downloading best available'); + } + }); + }) } /** @@ -119,14 +130,6 @@ class Process { getStdout(): Readable { return this.stdout } - - /** - * download info getter function - * @returns {*} - */ - getInfo(): any { - return this.info - } } export default Process; \ No newline at end of file diff --git a/server/src/core/downloadArchive.ts b/server/src/core/downloadArchive.ts new file mode 100644 index 0000000..58354c7 --- /dev/null +++ b/server/src/core/downloadArchive.ts @@ -0,0 +1,15 @@ +import { resolve } from "path"; + +const archived = [ + { + id: 1, + title: 'AleXa (알렉사) – Voting Open in American Song Contest Grand Final!', + path: resolve('downloads/AleXa (알렉사) – Voting Open in American Song Contest Grand Final!.webm'), + img: 'https://i.ytimg.com/vi/WbBUz7pjUnM/hq720.jpg?sqp=-oaymwEcCNAFEJQDSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLAi5MNtvpgnY9aRpdFlhAfhdV7Zlg', + }, +] + +export function listDownloaded(ctx: any, next: any) { + ctx.body = archived + next() +} diff --git a/server/src/core/downloader.ts b/server/src/core/downloader.ts index 982058e..5dbaff7 100644 --- a/server/src/core/downloader.ts +++ b/server/src/core/downloader.ts @@ -23,6 +23,18 @@ catch (e) { new Promise(resolve => setTimeout(resolve, 500)) .then(() => log.warn('dl', 'settings.json not found, ignore if using Docker')); } +/** + * Get download info such as thumbnail, title, resolution and list all formats + * @param socket + * @param url + */ +export async function getFormatsAndInfo(socket: Socket, url: string) { + let p = new Process(url, [], settings); + const formats = await p.getInfo(); + console.log(formats) + socket.emit('available-formats', formats) + p = null; +} /** * Invoke a new download. @@ -42,27 +54,21 @@ export async function download(socket: Socket, payload: IPayload) { payload.params.split(' ') : payload.params; - const p = new Process(url, params, settings); + let p = new Process(url, params, settings); p.start().then(downloader => { pool.add(p) - let infoLock = true; let pid = downloader.getPid(); + p.getInfo().then(info => { + socket.emit('info', { pid: pid, info: info }); + }); + from(downloader.getStdout()) // stdout as observable .pipe(throttle(() => interval(500))) // discard events closer than 500ms .subscribe({ next: (stdout) => { - if (infoLock) { - if (downloader.getInfo() === null) { - return; - } - socket.emit('info', { - pid: pid, info: downloader.getInfo() - }); - infoLock = false; - } - socket.emit('progress', formatter(String(stdout), pid)) // finally, emit + socket.emit('progress', formatter(String(stdout), pid)) }, complete: () => { downloader.kill().then(() => { @@ -79,11 +85,10 @@ export async function download(socket: Socket, payload: IPayload) { }); } }); - }) + }); } /** - * @deprecated * Retrieve all downloads. * If the server has just been launched retrieve the ones saved to the database. * If the server is running fetches them from the process pool. diff --git a/server/src/core/streamer.ts b/server/src/core/streamer.ts new file mode 100644 index 0000000..6ac70a2 --- /dev/null +++ b/server/src/core/streamer.ts @@ -0,0 +1,39 @@ +import { stat, createReadStream } from 'fs'; +import { lookup } from 'mime-types'; + +export function streamer(ctx: any, next: any) { + const filepath = '/Users/marco/dev/homebrew/yt-dlp-web-ui/downloads/AleXa (알렉사) – Voting Open in American Song Contest Grand Final!.webm' + stat(filepath, (err, stat) => { + if (err) { + ctx.response.status = 404; + ctx.body = { err: 'resource not found' }; + next(); + } + const fileSize = stat.size; + const range = ctx.headers.range; + if (range) { + const parts = range.replace(/bytes=/, '').split('-'); + const start = parseInt(parts[0], 10); + const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1; + const chunksize = end - start + 1; + const file = createReadStream(filepath, { start, end }); + const head = { + 'Content-Range': `bytes ${start}-${end}/${fileSize}`, + 'Accept-Ranges': 'bytes', + 'Content-Length': chunksize, + 'Content-Type': lookup(filepath) + }; + ctx.res.writeHead(206, head); + file.pipe(ctx.res); + next(); + } else { + const head = { + 'Content-Length': fileSize, + 'Content-Type': 'video/mp4' + }; + ctx.res.writeHead(200, head); + createReadStream(ctx.params.filepath).pipe(ctx.res); + next(); + } + }); +} \ No newline at end of file diff --git a/server/src/db/db.ts b/server/src/db/db.ts new file mode 100644 index 0000000..e69de29 diff --git a/server/src/db_deprecated/db.ts b/server/src/db/db_deprecated.ts similarity index 100% rename from server/src/db_deprecated/db.ts rename to server/src/db/db_deprecated.ts diff --git a/server/src/interfaces/IDownloadInfo.ts b/server/src/interfaces/IDownloadInfo.ts new file mode 100644 index 0000000..036a476 --- /dev/null +++ b/server/src/interfaces/IDownloadInfo.ts @@ -0,0 +1,14 @@ +interface IDownloadInfo { + formats: Array, + best: IDownloadInfoSection, + thumbnail: string, + title: string, +} + +interface IDownloadInfoSection { + format_id: string, + format_note: string, + fps: number, + resolution: string, + vcodec: string, +} \ No newline at end of file diff --git a/server/src/main.ts b/server/src/main.ts index 52d6951..7f426da 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -2,28 +2,53 @@ import { logger, splash } from './utils/logger'; import { join } from 'path'; import { Server } from 'socket.io'; import { ytdlpUpdater } from './utils/updater'; -import { download, abortDownload, retrieveDownload, abortAllDownloads } from './core/downloader'; +import { download, abortDownload, retrieveDownload, abortAllDownloads, getFormatsAndInfo } from './core/downloader'; import { getFreeDiskSpace } from './utils/procUtils'; import Logger from './utils/BetterLogger'; -import Jean from './core/HTTPServer'; +import { listDownloaded } from './core/downloadArchive'; +import { createServer } from 'http'; +import * as Koa from 'koa'; +import * as Router from 'koa-router'; +import * as serve from 'koa-static'; +import * as cors from '@koa/cors'; +import { streamer } from './core/streamer'; -const server = new Jean(join(__dirname, 'frontend')).createServer(); +const app = new Koa(); +const server = createServer(app.callback()); +const router = new Router(); const log = new Logger(); const io = new Server(server, { cors: { origin: "*", methods: ["GET", "POST"] } +}); + +// Koa routing + +router.get('/settings', (ctx, next) => { + ctx.redirect('/') + next() +}) +router.get('/downloaded', (ctx, next) => { + ctx.redirect('/') + next() +}) +router.get('/getAllDownloaded', (ctx, next) => { + listDownloaded(ctx, next) +}) +router.get('/stream/:filepath', (ctx, next) => { + streamer(ctx, next) }) -/* - WebSocket listeners -*/ +// WebSocket listeners + io.on('connection', socket => { logger('ws', `${socket.handshake.address} connected!`) socket.on('send-url', (args) => { logger('ws', args?.url) + //if (args.url) getFormatsAndInfo(socket, args?.url) download(socket, args) }) socket.on('abort', (args) => { @@ -47,6 +72,10 @@ io.on('disconnect', (socket) => { logger('ws', `${socket.handshake.address} disconnected`) }) +app.use(serve(join(__dirname, 'frontend'))) +app.use(router.routes()) +app.use(cors()) + server.listen(process.env.PORT || 3022) splash() @@ -63,7 +92,7 @@ const gracefullyStop = () => { process.exit(0) } -/* Intercepts singnals and perform cleanups before shutting down. */ +// Intercepts singnals and perform cleanups before shutting down. process .on('SIGTERM', () => gracefullyStop()) .on('SIGUSR1', () => gracefullyStop())