marco/yt-dlp-webui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code refactoring and set up format selection

Browse Source 
Set up format selection and archive download for next releases
This commit is contained in:
marcobaobao
2022-06-02 17:01:26 +02:00
parent 413b89166b
commit 975784ed72
18 changed files with 298 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/docker-publish.yml vendored
View File

@@ -78,7 +78,6 @@ jobs:
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
# Sign the resulting Docker image digest except on PRs.
# This will only write to the public Rekor transparency log when the Docker

3
.gitignore vendored
View File

@@ -10,4 +10,5 @@ src/server/core/yt-dlp
*.mp4
*.ytdl
*.part
*.db
*.db
*.DS_Store

21
frontend/src/App.tsx
View File

@@ -16,6 +16,7 @@ import MuiAppBar, { AppBarProps as MuiAppBarProps } from '@mui/material/AppBar';
import {
ChevronLeft,
Dashboard,
Download,
Menu, Settings as SettingsIcon,
SettingsEthernet,
Storage,
@@ -32,6 +33,7 @@ import Settings from "./Settings";
import { io } from "socket.io-client";
import { RootState, store } from './stores/store';
import { Provider, useSelector } from "react-redux";
import ArchivedDownloads from "./Archived";
const drawerWidth: number = 240;
@@ -166,10 +168,6 @@ function AppContent() {
<SettingsEthernet></SettingsEthernet>
<span>&nbsp;{settings.serverAddr}</span>
</div>
<IconButton color="inherit">
<Badge badgeContent={0} color="secondary">
</Badge>
</IconButton>
</Toolbar>
</AppBar>
<Drawer variant="permanent" open={open}>
@@ -200,6 +198,20 @@ function AppContent() {
<ListItemText primary="Home" />
</ListItemButton>
</Link>
{/* Next release: list downloaded files */}
{/* <Link to={'/downloaded'} style={
{
textDecoration: 'none',
color: mode === 'dark' ? '#ffffff' : '#000000DE'
}
}>
<ListItemButton disabled={status.downloading}>
<ListItemIcon>
<Download />
</ListItemIcon>
<ListItemText primary="Downloaded" />
</ListItemButton>
</Link> */}
<Link to={'/settings'} style={
{
textDecoration: 'none',
@@ -227,6 +239,7 @@ function AppContent() {
<Routes>
<Route path="/" element={<Home socket={socket}></Home>}></Route>
<Route path="/settings" element={<Settings socket={socket}></Settings>}></Route>
<Route path="/downloaded" element={<ArchivedDownloads></ArchivedDownloads>}></Route>
</Routes>
</Box>
</Box>

46
frontend/src/Archived.tsx Normal file
View File

@@ -0,0 +1,46 @@
import React, { useEffect, useState } from "react";
import { Backdrop, CircularProgress, Container, Grid } from "@mui/material";
import { ArchiveResult } from "./components/ArchiveResult";
import { useSelector } from "react-redux";
import { RootState } from "./stores/store";
export default function archivedDownloads() {
const [loading, setLoading] = useState(true);
const [archived, setArchived] = useState([]);
const settings = useSelector((state: RootState) => state.settings)
useEffect(() => {
fetch(`http://${settings.serverAddr}:3022/getAllDownloaded`)
.then(res => res.json())
.then(data => setArchived(data))
.then(() => setLoading(false))
}, []);
return (
<Container maxWidth="lg" sx={{ mt: 4, mb: 4 }}>
<Backdrop
sx={{ color: '#fff', zIndex: (theme) => theme.zIndex.drawer + 1 }}
open={loading}
>
<CircularProgress color="primary" />
</Backdrop>
{
archived.length > 0 ?
<Grid container spacing={{ xs: 2, md: 2 }} columns={{ xs: 4, sm: 8, md: 12 }} pt={2}>
{
archived.map((el, idx) =>
<Grid key={`${idx}-${el.id}`} item xs={4} sm={4} md={4}>
<ArchiveResult
url={`http://${settings.serverAddr}:3022/stream/${el.id}`}
thumbnail={el.img} title={el.title}
/>
</Grid>
)
}
</Grid>
: null
}
</Container>
);
}

33
frontend/src/Home.tsx
View File

@@ -4,7 +4,7 @@ import { useDispatch, useSelector } from "react-redux";
import { Socket } from "socket.io-client";
import { StackableResult } from "./components/StackableResult";
import { connected, disconnected, downloading, finished } from "./features/status/statusSlice";
import { IDLInfo, IDLInfoBase, IMessage } from "./interfaces";
import { IDLInfo, IDLInfoBase, IDownloadInfo, IMessage } from "./interfaces";
import { RootState } from "./stores/store";
import { updateInStateMap, } from "./utils";
@@ -48,6 +48,14 @@ export default function Home({ socket }: Props) {
})
}, [])
/* Handle download information sent by server */
useEffect(() => {
socket.on('available-formats', (data: IDownloadInfo) => {
setShowBackdrop(false)
console.log(data)
})
}, [])
/* Handle download information sent by server */
useEffect(() => {
socket.on('info', (data: IDLInfo) => {
@@ -60,13 +68,6 @@ export default function Home({ socket }: Props) {
/* Handle per-download progress */
useEffect(() => {
socket.on('progress', (data: IMessage) => {
if (showBackdrop) {
setShowBackdrop(false)
}
if (!status.downloading) {
setShowBackdrop(false)
dispatch(downloading())
}
if (data.status === 'Done!' || data.status === 'Aborted') {
setShowBackdrop(false)
updateInStateMap(data.pid, 'Done!', messageMap, setMessageMap);
@@ -147,11 +148,21 @@ export default function Home({ socket }: Props) {
/>
<Grid container spacing={1} pt={2}>
<Grid item>
<Button variant="contained" onClick={() => sendUrl()} disabled={false}>{settings.i18n.t('startButton')}</Button>
<Button
variant="contained"
disabled={url === ''}
onClick={() => sendUrl()}
>
{settings.i18n.t('startButton')}
</Button>
</Grid>
<Grid item>
<Button variant="contained" onClick={() => abort()}>{settings.i18n.t('abortAllButton')}</Button>
<Button
variant="contained"
onClick={() => abort()}
>
{settings.i18n.t('abortAllButton')}
</Button>
</Grid>
</Grid>
</Paper>

30
frontend/src/components/ArchiveResult.tsx Normal file
View File

@@ -0,0 +1,30 @@
import { Card, CardActionArea, CardContent, CardMedia, Skeleton, Typography } from "@mui/material";
import { ellipsis } from "../utils";
type Props = {
title: string,
thumbnail: string,
url: string,
}
export function ArchiveResult({ title, thumbnail, url }: Props) {
return (
<Card>
<CardActionArea onClick={() => window.open(url)}>
{thumbnail ?
<CardMedia
component="img"
height={180}
image={thumbnail}
/> :
<Skeleton variant="rectangular" height={180} />
}
<CardContent>
<Typography gutterBottom variant="body2" component="div">
{ellipsis(title, 72)}
</Typography>
</CardContent>
</CardActionArea>
</Card>
)
}

29
frontend/src/components/MessageToast.tsx
View File

@@ -1,29 +0,0 @@
import React from "react";
import {
Toast,
} from "react-bootstrap";
type Props = {
flag: boolean,
callback: Function,
children:
| JSX.Element
| string
}
export function MessageToast({ flag, callback, children }: Props) {
return (
<Toast
show={flag}
onClose={() => callback(false)}
bg={'primary'}
delay={1500}
autohide
className="mt-5"
>
<Toast.Body className="text-light">
{children}
</Toast.Body>
</Toast>
);
}

2
frontend/src/components/StackableResult.tsx
View File

@@ -1,4 +1,3 @@
import { Fragment } from "react";
import { EightK, FourK, Hd, Sd } from "@mui/icons-material";
import { Button, Card, CardActionArea, CardActions, CardContent, CardMedia, Chip, Grid, LinearProgress, Skeleton, Stack, Typography } from "@mui/material";
import { IMessage } from "../interfaces";
@@ -14,7 +13,6 @@ type Props = {
}
export function StackableResult({ formattedLog, title, thumbnail, resolution, progress, stopCallback }: Props) {
const guessResolution = (xByY: string): JSX.Element => {
if (!xByY) return null;
if (xByY.includes('4320')) return (<EightK color="primary" />);

15
frontend/src/interfaces.ts
View File

@@ -14,6 +14,21 @@ export interface IDLInfoBase {
resolution?: string
}
export interface IDownloadInfo {
formats: Array<IDownloadInfoSection>,
best: IDownloadInfoSection,
thumbnail: string,
title: string,
}
export interface IDownloadInfoSection {
format_id: string,
format_note: string,
fps: number,
resolution: string,
vcodec: string,
}
export interface IDLInfo {
pid: number,
info: IDLInfoBase

8
package.json
View File

@@ -20,9 +20,14 @@
"dependencies": {
"@emotion/react": "^11.9.0",
"@emotion/styled": "^11.8.1",
"@koa/cors": "^3.3.0",
"@mui/icons-material": "^5.6.2",
"@mui/material": "^5.6.4",
"@reduxjs/toolkit": "^1.8.1",
"koa": "^2.13.4",
"koa-router": "^10.1.1",
"koa-static": "^5.0.0",
"mime-types": "^2.1.35",
"react": "^17.0.2",
"react-dom": "^17.0.2",
"react-redux": "^8.0.1",
@@ -34,6 +39,9 @@
},
"devDependencies": {
"@parcel/transformer-yaml": "^2.5.0",
"@types/koa": "^2.13.4",
"@types/koa-router": "^7.4.4",
"@types/mime-types": "^2.1.1",
"@types/node": "^17.0.31",
"@types/react-router-dom": "^5.3.3",
"@types/uuid": "^8.3.4",

67
server/src/core/Process.ts
View File

@@ -39,9 +39,7 @@ class Process {
* @param callback not yet implemented
* @returns the process instance
*/
async start(callback?: Function): Promise<this> {
await this.internalGetInfo();
public async start(callback?: Function): Promise<this> {
const sanitizedParams = this.params.filter((param: string) => availableParams.includes(param));
const ytldp = spawn(this.exePath,
@@ -59,37 +57,50 @@ class Process {
}
/**
* @private
* function used internally by the download process to fetch information, usually thumbnail and title
* @returns Promise to the lock
*/
private async internalGetInfo() {
this.lock = true;
public getInfo(): Promise<IDownloadInfo> {
let stdoutChunks = [];
const ytdlpInfo = spawn(this.exePath, ['-s', '-j', this.url]);
const ytdlpInfo = spawn(this.exePath, ['-j', this.url]);
ytdlpInfo.stdout.on('data', (data) => {
stdoutChunks.push(data);
});
ytdlpInfo.on('exit', () => {
try {
const buffer = Buffer.concat(stdoutChunks);
const json = JSON.parse(buffer.toString());
this.info = json;
this.lock = false;
return new Promise((resolve, reject) => {
ytdlpInfo.on('exit', () => {
try {
const buffer = Buffer.concat(stdoutChunks);
const json = JSON.parse(buffer.toString());
this.info = json;
this.lock = false;
resolve({
formats: json.formats.map((format: IDownloadInfoSection) => {
return {
format_id: format.format_id ?? '',
format_note: format.format_note ?? '',
fps: format.fps ?? '',
resolution: format.resolution ?? '',
vcodec: format.vcodec ?? '',
}
}),
best: {
format_id: json.format_id ?? '',
format_note: json.format_note ?? '',
fps: json.fps ?? '',
resolution: json.resolution ?? '',
vcodec: json.vcodec ?? '',
},
thumbnail: json.thumbnail,
title: json.title,
});
} catch (e) {
this.info = {
title: "",
thumbnail: "",
};
}
});
if (!this.lock) {
return true;
}
} catch (e) {
reject('failed fetching formats, downloading best available');
}
});
})
}
/**
@@ -119,14 +130,6 @@ class Process {
getStdout(): Readable {
return this.stdout
}
/**
* download info getter function
* @returns {*}
*/
getInfo(): any {
return this.info
}
}
export default Process;

15
server/src/core/downloadArchive.ts Normal file
View File

@@ -0,0 +1,15 @@
import { resolve } from "path";
const archived = [
{
id: 1,
title: 'AleXa (알렉사) – Voting Open in American Song Contest Grand Final!',
path: resolve('downloads/AleXa (알렉사) – Voting Open in American Song Contest Grand Final!.webm'),
img: 'https://i.ytimg.com/vi/WbBUz7pjUnM/hq720.jpg?sqp=-oaymwEcCNAFEJQDSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLAi5MNtvpgnY9aRpdFlhAfhdV7Zlg',
},
]
export function listDownloaded(ctx: any, next: any) {
ctx.body = archived
next()
}

33
server/src/core/downloader.ts
View File

@@ -23,6 +23,18 @@ catch (e) {
new Promise(resolve => setTimeout(resolve, 500))
.then(() => log.warn('dl', 'settings.json not found, ignore if using Docker'));
}
/**
* Get download info such as thumbnail, title, resolution and list all formats
* @param socket
* @param url
*/
export async function getFormatsAndInfo(socket: Socket, url: string) {
let p = new Process(url, [], settings);
const formats = await p.getInfo();
console.log(formats)
socket.emit('available-formats', formats)
p = null;
}
/**
* Invoke a new download.
@@ -42,27 +54,21 @@ export async function download(socket: Socket, payload: IPayload) {
payload.params.split(' ') :
payload.params;
const p = new Process(url, params, settings);
let p = new Process(url, params, settings);
p.start().then(downloader => {
pool.add(p)
let infoLock = true;
let pid = downloader.getPid();
p.getInfo().then(info => {
socket.emit('info', { pid: pid, info: info });
});
from(downloader.getStdout()) // stdout as observable
.pipe(throttle(() => interval(500))) // discard events closer than 500ms
.subscribe({
next: (stdout) => {
if (infoLock) {
if (downloader.getInfo() === null) {
return;
}
socket.emit('info', {
pid: pid, info: downloader.getInfo()
});
infoLock = false;
}
socket.emit('progress', formatter(String(stdout), pid)) // finally, emit
socket.emit('progress', formatter(String(stdout), pid))
},
complete: () => {
downloader.kill().then(() => {
@@ -79,11 +85,10 @@ export async function download(socket: Socket, payload: IPayload) {
});
}
});
})
});
}
/**
* @deprecated
* Retrieve all downloads.
* If the server has just been launched retrieve the ones saved to the database.
* If the server is running fetches them from the process pool.

39
server/src/core/streamer.ts Normal file
View File

@@ -0,0 +1,39 @@
import { stat, createReadStream } from 'fs';
import { lookup } from 'mime-types';
export function streamer(ctx: any, next: any) {
const filepath = '/Users/marco/dev/homebrew/yt-dlp-web-ui/downloads/AleXa (알렉사) – Voting Open in American Song Contest Grand Final!.webm'
stat(filepath, (err, stat) => {
if (err) {
ctx.response.status = 404;
ctx.body = { err: 'resource not found' };
next();
}
const fileSize = stat.size;
const range = ctx.headers.range;
if (range) {
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
const chunksize = end - start + 1;
const file = createReadStream(filepath, { start, end });
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': lookup(filepath)
};
ctx.res.writeHead(206, head);
file.pipe(ctx.res);
next();
} else {
const head = {
'Content-Length': fileSize,
'Content-Type': 'video/mp4'
};
ctx.res.writeHead(200, head);
createReadStream(ctx.params.filepath).pipe(ctx.res);
next();
}
});
}

0
server/src/db/db.ts Normal file
View File

0
server/src/db_deprecated/db.ts → server/src/db/db_deprecated.ts
View File

14
server/src/interfaces/IDownloadInfo.ts Normal file
View File

@@ -0,0 +1,14 @@
interface IDownloadInfo {
formats: Array<IDownloadInfoSection>,
best: IDownloadInfoSection,
thumbnail: string,
title: string,
}
interface IDownloadInfoSection {
format_id: string,
format_note: string,
fps: number,
resolution: string,
vcodec: string,
}

43
server/src/main.ts
View File

@@ -2,28 +2,53 @@ import { logger, splash } from './utils/logger';
import { join } from 'path';
import { Server } from 'socket.io';
import { ytdlpUpdater } from './utils/updater';
import { download, abortDownload, retrieveDownload, abortAllDownloads } from './core/downloader';
import { download, abortDownload, retrieveDownload, abortAllDownloads, getFormatsAndInfo } from './core/downloader';
import { getFreeDiskSpace } from './utils/procUtils';
import Logger from './utils/BetterLogger';
import Jean from './core/HTTPServer';
import { listDownloaded } from './core/downloadArchive';
import { createServer } from 'http';
import * as Koa from 'koa';
import * as Router from 'koa-router';
import * as serve from 'koa-static';
import * as cors from '@koa/cors';
import { streamer } from './core/streamer';
const server = new Jean(join(__dirname, 'frontend')).createServer();
const app = new Koa();
const server = createServer(app.callback());
const router = new Router();
const log = new Logger();
const io = new Server(server, {
cors: {
origin: "*",
methods: ["GET", "POST"]
}
});
// Koa routing
router.get('/settings', (ctx, next) => {
ctx.redirect('/')
next()
})
router.get('/downloaded', (ctx, next) => {
ctx.redirect('/')
next()
})
router.get('/getAllDownloaded', (ctx, next) => {
listDownloaded(ctx, next)
})
router.get('/stream/:filepath', (ctx, next) => {
streamer(ctx, next)
})
/*
WebSocket listeners
*/
// WebSocket listeners
io.on('connection', socket => {
logger('ws', `${socket.handshake.address} connected!`)
socket.on('send-url', (args) => {
logger('ws', args?.url)
//if (args.url) getFormatsAndInfo(socket, args?.url)
download(socket, args)
})
socket.on('abort', (args) => {
@@ -47,6 +72,10 @@ io.on('disconnect', (socket) => {
logger('ws', `${socket.handshake.address} disconnected`)
})
app.use(serve(join(__dirname, 'frontend')))
app.use(router.routes())
app.use(cors())
server.listen(process.env.PORT || 3022)
splash()
@@ -63,7 +92,7 @@ const gracefullyStop = () => {
process.exit(0)
}
/* Intercepts singnals and perform cleanups before shutting down. */
// Intercepts singnals and perform cleanups before shutting down.
process
.on('SIGTERM', () => gracefullyStop())
.on('SIGUSR1', () => gracefullyStop())