Code refactoring and set up format selection

Set up format selection and archive download for next releases

.github/workflows/docker-publish.yml

@@ -78,7 +78,6 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker

.gitignore

@@ -11,3 +11,4 @@ src/server/core/yt-dlp
 *.ytdl
 *.part
 *.db
+*.DS_Store

frontend/src/App.tsx

@@ -16,6 +16,7 @@ import MuiAppBar, { AppBarProps as MuiAppBarProps } from '@mui/material/AppBar';
 import {
     ChevronLeft,
     Dashboard,
+    Download,
     Menu, Settings as SettingsIcon,
     SettingsEthernet,
     Storage,
@@ -32,6 +33,7 @@ import Settings from "./Settings";
 import { io } from "socket.io-client";
 import { RootState, store } from './stores/store';
 import { Provider, useSelector } from "react-redux";
+import ArchivedDownloads from "./Archived";
 
 
 const drawerWidth: number = 240;
@@ -166,10 +168,6 @@ function AppContent() {
                                 <SettingsEthernet></SettingsEthernet>
                                 <span>&nbsp;{settings.serverAddr}</span>
                             </div>
-                            <IconButton color="inherit">
-                                <Badge badgeContent={0} color="secondary">
-                                </Badge>
-                            </IconButton>
                         </Toolbar>
                     </AppBar>
                     <Drawer variant="permanent" open={open}>
@@ -200,6 +198,20 @@ function AppContent() {
                                     <ListItemText primary="Home" />
                                 </ListItemButton>
                             </Link>
+                            {/* Next release: list downloaded files */}
+                            {/* <Link to={'/downloaded'} style={
+                                {
+                                    textDecoration: 'none',
+                                    color: mode === 'dark' ? '#ffffff' : '#000000DE'
+                                }
+                            }>
+                                <ListItemButton disabled={status.downloading}>
+                                    <ListItemIcon>
+                                        <Download />
+                                    </ListItemIcon>
+                                    <ListItemText primary="Downloaded" />
+                                </ListItemButton>
+                            </Link> */}
                             <Link to={'/settings'} style={
                                 {
                                     textDecoration: 'none',
@@ -227,6 +239,7 @@ function AppContent() {
                         <Routes>
                             <Route path="/" element={<Home socket={socket}></Home>}></Route>
                             <Route path="/settings" element={<Settings socket={socket}></Settings>}></Route>
+                            <Route path="/downloaded" element={<ArchivedDownloads></ArchivedDownloads>}></Route>
                         </Routes>
                     </Box>
                 </Box>

frontend/src/Archived.tsx

@@ -0,0 +1,46 @@
+import React, { useEffect, useState } from "react";
+import { Backdrop, CircularProgress, Container, Grid } from "@mui/material";
+import { ArchiveResult } from "./components/ArchiveResult";
+import { useSelector } from "react-redux";
+import { RootState } from "./stores/store";
+
+export default function archivedDownloads() {
+    const [loading, setLoading] = useState(true);
+    const [archived, setArchived] = useState([]);
+
+    const settings = useSelector((state: RootState) => state.settings)
+
+    useEffect(() => {
+        fetch(`http://${settings.serverAddr}:3022/getAllDownloaded`)
+            .then(res => res.json())
+            .then(data => setArchived(data))
+            .then(() => setLoading(false))
+    }, []);
+
+    return (
+        <Container maxWidth="lg" sx={{ mt: 4, mb: 4 }}>
+            <Backdrop
+                sx={{ color: '#fff', zIndex: (theme) => theme.zIndex.drawer + 1 }}
+                open={loading}
+            >
+                <CircularProgress color="primary" />
+            </Backdrop>
+            {
+                archived.length > 0 ?
+                    <Grid container spacing={{ xs: 2, md: 2 }} columns={{ xs: 4, sm: 8, md: 12 }} pt={2}>
+                        {
+                            archived.map((el, idx) =>
+                                <Grid key={`${idx}-${el.id}`} item xs={4} sm={4} md={4}>
+                                    <ArchiveResult
+                                        url={`http://${settings.serverAddr}:3022/stream/${el.id}`}
+                                        thumbnail={el.img} title={el.title}
+                                    />
+                                </Grid>
+                            )
+                        }
+                    </Grid>
+                    : null
+            }
+        </Container>
+    );
+}

frontend/src/Home.tsx

@@ -4,7 +4,7 @@ import { useDispatch, useSelector } from "react-redux";
 import { Socket } from "socket.io-client";
 import { StackableResult } from "./components/StackableResult";
 import { connected, disconnected, downloading, finished } from "./features/status/statusSlice";
-import { IDLInfo, IDLInfoBase, IMessage } from "./interfaces";
+import { IDLInfo, IDLInfoBase, IDownloadInfo, IMessage } from "./interfaces";
 import { RootState } from "./stores/store";
 import { updateInStateMap, } from "./utils";
 
@@ -48,6 +48,14 @@ export default function Home({ socket }: Props) {
         })
     }, [])
 
+    /* Handle download information sent by server */
+    useEffect(() => {
+        socket.on('available-formats', (data: IDownloadInfo) => {
+            setShowBackdrop(false)
+            console.log(data)
+        })
+    }, [])
+
     /* Handle download information sent by server */
     useEffect(() => {
         socket.on('info', (data: IDLInfo) => {
@@ -60,13 +68,6 @@ export default function Home({ socket }: Props) {
     /* Handle per-download progress */
     useEffect(() => {
         socket.on('progress', (data: IMessage) => {
-            if (showBackdrop) {
-                setShowBackdrop(false)
-            }
-            if (!status.downloading) {
-                setShowBackdrop(false)
-                dispatch(downloading())
-            }
             if (data.status === 'Done!' || data.status === 'Aborted') {
                 setShowBackdrop(false)
                 updateInStateMap(data.pid, 'Done!', messageMap, setMessageMap);
@@ -147,11 +148,21 @@ export default function Home({ socket }: Props) {
                         />
                         <Grid container spacing={1} pt={2}>
                             <Grid item>
-                                <Button variant="contained" onClick={() => sendUrl()} disabled={false}>{settings.i18n.t('startButton')}</Button>
+                                <Button
+                                    variant="contained"
+                                    disabled={url === ''}
+                                    onClick={() => sendUrl()}
+                                >
+                                    {settings.i18n.t('startButton')}
+                                </Button>
                             </Grid>
                             <Grid item>
-                                <Button variant="contained" onClick={() => abort()}>{settings.i18n.t('abortAllButton')}</Button>
+                                <Button
-
+                                    variant="contained"
+                                    onClick={() => abort()}
+                                >
+                                    {settings.i18n.t('abortAllButton')}
+                                </Button>
                             </Grid>
                         </Grid>
                     </Paper>

frontend/src/components/ArchiveResult.tsx

@@ -0,0 +1,30 @@
+import { Card, CardActionArea, CardContent, CardMedia, Skeleton, Typography } from "@mui/material";
+import { ellipsis } from "../utils";
+
+type Props = {
+    title: string,
+    thumbnail: string,
+    url: string,
+}
+
+export function ArchiveResult({ title, thumbnail, url }: Props) {
+    return (
+        <Card>
+            <CardActionArea onClick={() => window.open(url)}>
+                {thumbnail ?
+                    <CardMedia
+                        component="img"
+                        height={180}
+                        image={thumbnail}
+                    /> :
+                    <Skeleton variant="rectangular" height={180} />
+                }
+                <CardContent>
+                    <Typography gutterBottom variant="body2" component="div">
+                        {ellipsis(title, 72)}
+                    </Typography>
+                </CardContent>
+            </CardActionArea>
+        </Card>
+    )
+}

frontend/src/components/MessageToast.tsx

@@ -1,29 +0,0 @@
-import React from "react";
-import {
-    Toast,
-} from "react-bootstrap";
-
-type Props = {
-    flag: boolean,
-    callback: Function,
-    children:
-    | JSX.Element
-    | string
-}
-
-export function MessageToast({ flag, callback, children }: Props) {
-    return (
-        <Toast
-            show={flag}
-            onClose={() => callback(false)}
-            bg={'primary'}
-            delay={1500}
-            autohide
-            className="mt-5"
-        >
-            <Toast.Body className="text-light">
-                {children}
-            </Toast.Body>
-        </Toast>
-    );
-}

frontend/src/components/StackableResult.tsx

@@ -1,4 +1,3 @@
-import { Fragment } from "react";
 import { EightK, FourK, Hd, Sd } from "@mui/icons-material";
 import { Button, Card, CardActionArea, CardActions, CardContent, CardMedia, Chip, Grid, LinearProgress, Skeleton, Stack, Typography } from "@mui/material";
 import { IMessage } from "../interfaces";
@@ -14,7 +13,6 @@ type Props = {
 }
 
 export function StackableResult({ formattedLog, title, thumbnail, resolution, progress, stopCallback }: Props) {
-
     const guessResolution = (xByY: string): JSX.Element => {
         if (!xByY) return null;
         if (xByY.includes('4320')) return (<EightK color="primary" />);

frontend/src/interfaces.ts

@@ -14,6 +14,21 @@ export interface IDLInfoBase {
     resolution?: string
 }
 
+export interface IDownloadInfo {
+    formats: Array<IDownloadInfoSection>,
+    best: IDownloadInfoSection,
+    thumbnail: string,
+    title: string,
+}
+
+export interface IDownloadInfoSection {
+    format_id: string,
+    format_note: string,
+    fps: number,
+    resolution: string,
+    vcodec: string,
+}
+
 export interface IDLInfo {
     pid: number,
     info: IDLInfoBase

package.json

@@ -20,9 +20,14 @@
   "dependencies": {
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
+    "@koa/cors": "^3.3.0",
     "@mui/icons-material": "^5.6.2",
     "@mui/material": "^5.6.4",
     "@reduxjs/toolkit": "^1.8.1",
+    "koa": "^2.13.4",
+    "koa-router": "^10.1.1",
+    "koa-static": "^5.0.0",
+    "mime-types": "^2.1.35",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-redux": "^8.0.1",
@@ -34,6 +39,9 @@
   },
   "devDependencies": {
     "@parcel/transformer-yaml": "^2.5.0",
+    "@types/koa": "^2.13.4",
+    "@types/koa-router": "^7.4.4",
+    "@types/mime-types": "^2.1.1",
     "@types/node": "^17.0.31",
     "@types/react-router-dom": "^5.3.3",
     "@types/uuid": "^8.3.4",

server/src/core/Process.ts

@@ -39,9 +39,7 @@ class Process {
      * @param callback not yet implemented
      * @returns the process instance
      */
-    async start(callback?: Function): Promise<this> {
+    public async start(callback?: Function): Promise<this> {
-        await this.internalGetInfo();
-
         const sanitizedParams = this.params.filter((param: string) => availableParams.includes(param));
 
         const ytldp = spawn(this.exePath,
@@ -59,37 +57,50 @@ class Process {
     }
 
     /**
-     * @private
      * function used internally by the download process to fetch information, usually thumbnail and title
      * @returns Promise to the lock
      */
-    private async internalGetInfo() {
+    public getInfo(): Promise<IDownloadInfo> {
-        this.lock = true;
         let stdoutChunks = [];
-        const ytdlpInfo = spawn(this.exePath, ['-s', '-j', this.url]);
+        const ytdlpInfo = spawn(this.exePath, ['-j', this.url]);
 
         ytdlpInfo.stdout.on('data', (data) => {
             stdoutChunks.push(data);
         });
 
+        return new Promise((resolve, reject) => {
             ytdlpInfo.on('exit', () => {
                 try {
                     const buffer = Buffer.concat(stdoutChunks);
                     const json = JSON.parse(buffer.toString());
                     this.info = json;
                     this.lock = false;
-
+                    resolve({
-            } catch (e) {
+                        formats: json.formats.map((format: IDownloadInfoSection) => {
-                this.info = {
+                            return {
-                    title: "",
+                                format_id: format.format_id ?? '',
-                    thumbnail: "",
+                                format_note: format.format_note ?? '',
-                };
+                                fps: format.fps ?? '',
+                                resolution: format.resolution ?? '',
+                                vcodec: format.vcodec ?? '',
                             }
+                        }),
+                        best: {
+                            format_id: json.format_id ?? '',
+                            format_note: json.format_note ?? '',
+                            fps: json.fps ?? '',
+                            resolution: json.resolution ?? '',
+                            vcodec: json.vcodec ?? '',
+                        },
+                        thumbnail: json.thumbnail,
+                        title: json.title,
                     });
 
-        if (!this.lock) {
+                } catch (e) {
-            return true;
+                    reject('failed fetching formats, downloading best available');
                 }
+            });
+        })
     }
 
     /**
@@ -119,14 +130,6 @@ class Process {
     getStdout(): Readable {
         return this.stdout
     }
-
-    /**
-     * download info getter function
-     * @returns {*}
-     */
-    getInfo(): any {
-        return this.info
-    }
 }
 
 export default Process;

server/src/core/downloadArchive.ts

@@ -0,0 +1,15 @@
+import { resolve } from "path";
+
+const archived = [
+    {
+        id: 1,
+        title: 'AleXa (알렉사) – Voting Open in American Song Contest Grand Final!',
+        path: resolve('downloads/AleXa (알렉사) – Voting Open in American Song Contest Grand Final!.webm'),
+        img: 'https://i.ytimg.com/vi/WbBUz7pjUnM/hq720.jpg?sqp=-oaymwEcCNAFEJQDSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLAi5MNtvpgnY9aRpdFlhAfhdV7Zlg',
+    },
+]
+
+export function listDownloaded(ctx: any, next: any) {
+    ctx.body = archived
+    next()
+}

server/src/core/downloader.ts

@@ -23,6 +23,18 @@ catch (e) {
     new Promise(resolve => setTimeout(resolve, 500))
         .then(() => log.warn('dl', 'settings.json not found, ignore if using Docker'));
 }
+/**
+ * Get download info such as thumbnail, title, resolution and list all formats
+ * @param socket 
+ * @param url 
+ */
+export async function getFormatsAndInfo(socket: Socket, url: string) {
+    let p = new Process(url, [], settings);
+    const formats = await p.getInfo();
+    console.log(formats)
+    socket.emit('available-formats', formats)
+    p = null;
+}
 
 /**
  * Invoke a new download.  
@@ -42,27 +54,21 @@ export async function download(socket: Socket, payload: IPayload) {
         payload.params.split(' ') :
         payload.params;
 
-    const p = new Process(url, params, settings);
+    let p = new Process(url, params, settings);
 
     p.start().then(downloader => {
         pool.add(p)
-        let infoLock = true;
         let pid = downloader.getPid();
 
+        p.getInfo().then(info => {
+            socket.emit('info', { pid: pid, info: info });
+        });
+
         from(downloader.getStdout())              // stdout as observable
             .pipe(throttle(() => interval(500)))  // discard events closer than 500ms
             .subscribe({
                 next: (stdout) => {
-                    if (infoLock) {
+                    socket.emit('progress', formatter(String(stdout), pid))
-                        if (downloader.getInfo() === null) {
-                            return;
-                        }
-                        socket.emit('info', {
-                            pid: pid, info: downloader.getInfo()
-                        });
-                        infoLock = false;
-                    }
-                    socket.emit('progress', formatter(String(stdout), pid)) // finally, emit
                 },
                 complete: () => {
                     downloader.kill().then(() => {
@@ -79,11 +85,10 @@ export async function download(socket: Socket, payload: IPayload) {
                     });
                 }
             });
-    })
+    });
 }
 
 /**
- * @deprecated
  * Retrieve all downloads.  
  * If the server has just been launched retrieve the ones saved to the database.  
  * If the server is running fetches them from the process pool.

server/src/core/streamer.ts

@@ -0,0 +1,39 @@
+import { stat, createReadStream } from 'fs';
+import { lookup } from 'mime-types';
+
+export function streamer(ctx: any, next: any) {
+    const filepath = '/Users/marco/dev/homebrew/yt-dlp-web-ui/downloads/AleXa (알렉사) – Voting Open in American Song Contest Grand Final!.webm'
+    stat(filepath, (err, stat) => {
+        if (err) {
+            ctx.response.status = 404;
+            ctx.body = { err: 'resource not found' };
+            next();
+        }
+        const fileSize = stat.size;
+        const range = ctx.headers.range;
+        if (range) {
+            const parts = range.replace(/bytes=/, '').split('-');
+            const start = parseInt(parts[0], 10);
+            const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
+            const chunksize = end - start + 1;
+            const file = createReadStream(filepath, { start, end });
+            const head = {
+                'Content-Range': `bytes ${start}-${end}/${fileSize}`,
+                'Accept-Ranges': 'bytes',
+                'Content-Length': chunksize,
+                'Content-Type': lookup(filepath)
+            };
+            ctx.res.writeHead(206, head);
+            file.pipe(ctx.res);
+            next();
+        } else {
+            const head = {
+                'Content-Length': fileSize,
+                'Content-Type': 'video/mp4'
+            };
+            ctx.res.writeHead(200, head);
+            createReadStream(ctx.params.filepath).pipe(ctx.res);
+            next();
+        }
+    });
+}

server/src/interfaces/IDownloadInfo.ts

@@ -0,0 +1,14 @@
+interface IDownloadInfo {
+    formats: Array<IDownloadInfoSection>,
+    best: IDownloadInfoSection,
+    thumbnail: string,
+    title: string,
+}
+
+interface IDownloadInfoSection {
+    format_id: string,
+    format_note: string,
+    fps: number,
+    resolution: string,
+    vcodec: string,
+}

server/src/main.ts

@@ -2,28 +2,53 @@ import { logger, splash } from './utils/logger';
 import { join } from 'path';
 import { Server } from 'socket.io';
 import { ytdlpUpdater } from './utils/updater';
-import { download, abortDownload, retrieveDownload, abortAllDownloads } from './core/downloader';
+import { download, abortDownload, retrieveDownload, abortAllDownloads, getFormatsAndInfo } from './core/downloader';
 import { getFreeDiskSpace } from './utils/procUtils';
 import Logger from './utils/BetterLogger';
-import Jean from './core/HTTPServer';
+import { listDownloaded } from './core/downloadArchive';
+import { createServer } from 'http';
+import * as Koa from 'koa';
+import * as Router from 'koa-router';
+import * as serve from 'koa-static';
+import * as cors from '@koa/cors';
+import { streamer } from './core/streamer';
 
-const server = new Jean(join(__dirname, 'frontend')).createServer();
+const app = new Koa();
+const server = createServer(app.callback());
+const router = new Router();
 const log = new Logger();
 const io = new Server(server, {
     cors: {
         origin: "*",
         methods: ["GET", "POST"]
     }
+});
+
+// Koa routing
+
+router.get('/settings', (ctx, next) => {
+    ctx.redirect('/')
+    next()
+})
+router.get('/downloaded', (ctx, next) => {
+    ctx.redirect('/')
+    next()
+})
+router.get('/getAllDownloaded', (ctx, next) => {
+    listDownloaded(ctx, next)
+})
+router.get('/stream/:filepath', (ctx, next) => {
+    streamer(ctx, next)
 })
 
-/*
+// WebSocket listeners
-    WebSocket listeners
+
-*/
 io.on('connection', socket => {
     logger('ws', `${socket.handshake.address} connected!`)
 
     socket.on('send-url', (args) => {
         logger('ws', args?.url)
+        //if (args.url) getFormatsAndInfo(socket, args?.url)
         download(socket, args)
     })
     socket.on('abort', (args) => {
@@ -47,6 +72,10 @@ io.on('disconnect', (socket) => {
     logger('ws', `${socket.handshake.address} disconnected`)
 })
 
+app.use(serve(join(__dirname, 'frontend')))
+app.use(router.routes())
+app.use(cors())
+
 server.listen(process.env.PORT || 3022)
 
 splash()
@@ -63,7 +92,7 @@ const gracefullyStop = () => {
     process.exit(0)
 }
 
-/* Intercepts singnals and perform cleanups before shutting down. */
+// Intercepts singnals and perform cleanups before shutting down.
 process
     .on('SIGTERM', () => gracefullyStop())
     .on('SIGUSR1', () => gracefullyStop())