From 6aa2d4198828d00421b628fbd3ef41e449c5326c Mon Sep 17 00:00:00 2001 From: Marco <35533749+marcopeocchi@users.noreply.github.com> Date: Tue, 9 Jan 2024 14:29:18 +0100 Subject: [PATCH] Logging in webUI, Archive view refactor (#127) * test logging * test impl for logging * implemented "live logging", restyle templates dropdown * moved extract audio to downloadDialog, fixed labels * code refactoring * buffering logs --- .gitignore | 3 +- frontend/package.json | 1 + frontend/pnpm-lock.yaml | 7 + frontend/src/Layout.tsx | 14 ++ frontend/src/assets/i18n.yaml | 24 +++ frontend/src/components/DownloadDialog.tsx | 42 ++++-- .../src/components/ExtraDownloadOptions.tsx | 22 ++- frontend/src/components/LogTerminal.tsx | 91 ++++++++++++ frontend/src/index.tsx | 3 + frontend/src/router.tsx | 9 ++ frontend/src/views/Archive.tsx | 139 ++++++++++++++---- frontend/src/views/Terminal.tsx | 9 ++ go.mod | 9 ++ go.sum | 40 ++++- server/config/parser.go | 2 + server/internal/memory_db.go | 12 +- server/internal/message_queue.go | 4 +- server/internal/playlist.go | 20 +-- server/internal/process.go | 52 ++++--- server/logging/handler.go | 80 ++++++++++ server/logging/observable_logger.go | 29 ++++ server/rpc/container.go | 13 +- server/rpc/service.go | 18 ++- server/server.go | 44 ++++-- server/utils/logrotate.go | 55 +++++++ 25 files changed, 630 insertions(+), 112 deletions(-) create mode 100644 frontend/src/components/LogTerminal.tsx create mode 100644 frontend/src/views/Terminal.tsx create mode 100644 server/logging/handler.go create mode 100644 server/logging/observable_logger.go create mode 100644 server/utils/logrotate.go diff --git a/.gitignore b/.gitignore index ba66166..ba67957 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ yt-dlp-webui session.dat config.yml cookies.txt -__debug* \ No newline at end of file +__debug* +ui/ \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json index 55ec490..81a38e2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,6 +13,7 @@ "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", "@fontsource/roboto": "^5.0.6", + "@fontsource/roboto-mono": "^5.0.16", "@mui/icons-material": "^5.11.16", "@mui/material": "^5.13.5", "fp-ts": "^2.16.1", diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 6aba87b..e2471e1 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -14,6 +14,9 @@ dependencies: '@fontsource/roboto': specifier: ^5.0.6 version: 5.0.6 + '@fontsource/roboto-mono': + specifier: ^5.0.16 + version: 5.0.16 '@mui/icons-material': specifier: ^5.11.16 version: 5.11.16(@mui/material@5.13.5)(@types/react@18.2.29)(react@18.2.0) @@ -433,6 +436,10 @@ packages: dev: true optional: true + /@fontsource/roboto-mono@5.0.16: + resolution: {integrity: sha512-unZYfjXts55DQyODz0I9DzbSrS5DRKPNq9crJpNJe/Vy818bLnijprcJv3fvqwdDqTT0dRm2Fhk09QEIdtAc+Q==} + dev: false + /@fontsource/roboto@5.0.6: resolution: {integrity: sha512-SksyUGbdqY7a71l/ywdX5fm37fepCakgHZEZ3H4uP5A9wDJewrcSACtIzw6yi9QjJIYoj1xArhDSOz4XJxyWow==} dev: false diff --git a/frontend/src/Layout.tsx b/frontend/src/Layout.tsx index 2801122..ee21040 100644 --- a/frontend/src/Layout.tsx +++ b/frontend/src/Layout.tsx @@ -29,6 +29,7 @@ import SocketSubscriber from './components/SocketSubscriber' import ThemeToggler from './components/ThemeToggler' import { useI18n } from './hooks/useI18n' import Toaster from './providers/ToasterProvider' +import TerminalIcon from '@mui/icons-material/Terminal' export default function Layout() { const [open, setOpen] = useState(false) @@ -138,6 +139,19 @@ export default function Layout() { + + + + + + + + = ({ open, onClose, onDownloadStart }) => { const isConnected = useRecoilValue(connectedState) const availableDownloadPaths = useRecoilValue(availableDownloadPathsState) const downloadTemplate = useRecoilValue(downloadTemplateState) + const savedTemplates = useRecoilValue(savedTemplatesState) const [downloadFormats, setDownloadFormats] = useState() const [pickedVideoFormat, setPickedVideoFormat] = useState('') @@ -70,6 +71,8 @@ const DownloadDialog: FC = ({ open, onClose, onDownloadStart }) => { const [pickedBestFormat, setPickedBestFormat] = useState('') const [customArgs, setCustomArgs] = useRecoilState(customArgsState) + const [, setCliArgs] = useRecoilState(latestCliArgumentsState) + const [downloadPath, setDownloadPath] = useState('') const [filenameTemplate, setFilenameTemplate] = useRecoilState( @@ -81,7 +84,7 @@ const DownloadDialog: FC = ({ open, onClose, onDownloadStart }) => { const [isPlaylist, setIsPlaylist] = useState(false) - const cliArgs = useMemo(() => + const argsBuilder = useMemo(() => new CliArguments().fromString(settings.cliArgs), [settings.cliArgs] ) @@ -108,7 +111,7 @@ const DownloadDialog: FC = ({ open, onClose, onDownloadStart }) => { client.download({ url: immediate || url || workingUrl, - args: `${cliArgs.toString()} ${toFormatArgs(codes)} ${downloadTemplate}`, + args: `${argsBuilder.toString()} ${toFormatArgs(codes)} ${downloadTemplate}`, pathOverride: downloadPath ?? '', renameTo: settings.fileRenaming ? filenameTemplate : '', playlist: isPlaylist, @@ -313,9 +316,31 @@ const DownloadDialog: FC = ({ open, onClose, onDownloadStart }) => { } - + {savedTemplates.length > 0 && } + + + setIsPlaylist(state => !state)} />} + checked={isPlaylist} + label={i18n.t('playlistCheckbox')} + /> + + + setCliArgs(argsBuilder.toggleExtractAudio().toString())} + /> + } + checked={argsBuilder.extractAudio} + onChange={() => setCliArgs(argsBuilder.toggleExtractAudio().toString())} + disabled={settings.formatSelection} + label={i18n.t('extractAudioCheckbox')} + /> + + - - setIsPlaylist(state => !state)} />} - checked={isPlaylist} - label={i18n.t('playlistCheckbox')} - /> - diff --git a/frontend/src/components/ExtraDownloadOptions.tsx b/frontend/src/components/ExtraDownloadOptions.tsx index 697cdfd..d531a06 100644 --- a/frontend/src/components/ExtraDownloadOptions.tsx +++ b/frontend/src/components/ExtraDownloadOptions.tsx @@ -1,4 +1,4 @@ -import { Autocomplete, Box, TextField } from '@mui/material' +import { Autocomplete, Box, TextField, Typography } from '@mui/material' import { useRecoilState, useRecoilValue } from 'recoil' import { customArgsState, savedTemplatesState } from '../atoms/downloadTemplate' import { useI18n } from '../hooks/useI18n' @@ -22,9 +22,23 @@ const ExtraDownloadOptions: React.FC = () => { renderOption={(props, option) => ( - {option.label} + {...props} + > + + + {option.label} + + + {option.content} + + )} sx={{ width: '100%', mt: 2 }} diff --git a/frontend/src/components/LogTerminal.tsx b/frontend/src/components/LogTerminal.tsx new file mode 100644 index 0000000..3ba2b62 --- /dev/null +++ b/frontend/src/components/LogTerminal.tsx @@ -0,0 +1,91 @@ +import { Box, CircularProgress, Container, Paper, Typography } from '@mui/material' +import { useEffect, useMemo, useRef, useState } from 'react' +import { useRecoilValue } from 'recoil' +import { serverURL } from '../atoms/settings' +import { useI18n } from '../hooks/useI18n' + +const token = localStorage.getItem('token') + +const LogTerminal: React.FC = () => { + const serverAddr = useRecoilValue(serverURL) + + const { i18n } = useI18n() + + const [logBuffer, setLogBuffer] = useState([]) + + const boxRef = useRef(null) + + const eventSource = useMemo( + () => new EventSource(`${serverAddr}/log/sse?token=${token}`), + [serverAddr] + ) + + useEffect(() => { + eventSource.addEventListener('log', event => { + const msg: string[] = JSON.parse(event.data) + setLogBuffer(buff => [...buff, ...msg].slice(-100)) + + boxRef.current?.scrollTo(0, boxRef.current.scrollHeight) + }) + + // TODO: in dev mode it breaks sse + return () => eventSource.close() + }, [eventSource]) + + const logEntryStyle = (data: string) => { + if (data.includes("level=ERROR")) { + return { color: 'red' } + } + if (data.includes("level=WARN")) { + return { color: 'orange' } + } + return {} + } + + return ( + + + + {i18n.t('logsTitle')} + + {(logBuffer.length === 0) && + + + {i18n.t('awaitingLogs')} + + + } + + {logBuffer.map((log, idx) => ( + + {log} + + ))} + + + + ) +} + +export default LogTerminal \ No newline at end of file diff --git a/frontend/src/index.tsx b/frontend/src/index.tsx index e2c6117..935033d 100644 --- a/frontend/src/index.tsx +++ b/frontend/src/index.tsx @@ -6,6 +6,9 @@ import '@fontsource/roboto/300.css' import '@fontsource/roboto/400.css' import '@fontsource/roboto/500.css' import '@fontsource/roboto/700.css' +import '@fontsource/roboto/700.css' + +import '@fontsource/roboto-mono' const root = createRoot(document.getElementById('root')!) diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index 4c8147c..0856472 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -2,6 +2,7 @@ import { CircularProgress } from '@mui/material' import { Suspense, lazy } from 'react' import { createHashRouter } from 'react-router-dom' import Layout from './Layout' +import Terminal from './views/Terminal' const Home = lazy(() => import('./views/Home')) const Login = lazy(() => import('./views/Login')) @@ -36,6 +37,14 @@ export const router = createHashRouter([ ) }, + { + path: '/log', + element: ( + }> + + + ) + }, { path: '/archive', element: ( diff --git a/frontend/src/views/Archive.tsx b/frontend/src/views/Archive.tsx index cabdd3c..1d62fff 100644 --- a/frontend/src/views/Archive.tsx +++ b/frontend/src/views/Archive.tsx @@ -9,12 +9,13 @@ import { DialogContent, DialogContentText, DialogTitle, - IconButton, List, ListItem, ListItemButton, ListItemIcon, ListItemText, + MenuItem, + MenuList, Paper, SpeedDial, SpeedDialAction, @@ -27,6 +28,7 @@ import FolderIcon from '@mui/icons-material/Folder' import InsertDriveFileIcon from '@mui/icons-material/InsertDriveFile' import VideoFileIcon from '@mui/icons-material/VideoFile' +import DownloadIcon from '@mui/icons-material/Download' import { matchW } from 'fp-ts/lib/TaskEither' import { pipe } from 'fp-ts/lib/function' import { useEffect, useMemo, useState, useTransition } from 'react' @@ -38,12 +40,14 @@ import { useObservable } from '../hooks/observable' import { useToast } from '../hooks/toast' import { useI18n } from '../hooks/useI18n' import { ffetch } from '../lib/httpClient' -import { DeleteRequest, DirectoryEntry } from '../types' +import { DirectoryEntry } from '../types' import { base64URLEncode, roundMiB } from '../utils' -import DownloadIcon from '@mui/icons-material/Download' - export default function Downloaded() { + const [menuPos, setMenuPos] = useState({ x: 0, y: 0 }) + const [showMenu, setShowMenu] = useState(false) + const [currentFile, setCurrentFile] = useState() + const serverAddr = useRecoilValue(serverURL) const navigate = useNavigate() @@ -135,19 +139,24 @@ export default function Downloaded() { : selected$.next([...selected$.value, name]) } + const deleteFile = (entry: DirectoryEntry) => pipe( + ffetch(`${serverAddr}/archive/delete`, { + method: 'POST', + body: JSON.stringify({ + path: entry.path, + shaSum: entry.shaSum, + }) + }), + matchW( + (l) => pushMessage(l, 'error'), + (_) => fetcher() + ) + )() + const deleteSelected = () => { Promise.all(selectable .filter(entry => entry.selected) - .map(entry => fetch(`${serverAddr}/archive/delete`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - path: entry.path, - shaSum: entry.shaSum, - } as DeleteRequest) - })) + .map(deleteFile) ).then(fetcher) } @@ -172,18 +181,42 @@ export default function Downloaded() { }) return ( - + setShowMenu(false)} + > + { + if (currentFile) { + downloadFile(currentFile?.path) + setCurrentFile(undefined) + } + }} + onDelete={() => { + if (currentFile) { + deleteFile(currentFile) + setCurrentFile(undefined) + } + }} + /> theme.zIndex.drawer + 1 }} open={!(files$.observed) || isPending} > - + setShowMenu(false)} + > {i18n.t('archiveTitle')} @@ -191,6 +224,12 @@ export default function Downloaded() { {selectable.length === 0 && 'No files found'} {selectable.map((file, idx) => ( { + e.preventDefault() + setCurrentFile(file) + setMenuPos({ x: e.clientX, y: e.clientY }) + setShowMenu(true) + }} key={idx} secondaryAction={
@@ -202,13 +241,6 @@ export default function Downloaded() { } {!file.isDirectory && <> - downloadFile(file.path)} - sx={{ marginLeft: 1.5 }} - > - - - - + @@ -287,4 +323,43 @@ export default function Downloaded() { ) +} + +const IconMenu: React.FC<{ + posX: number + posY: number + hide: boolean + onDownload: () => void + onDelete: () => void +}> = ({ posX, posY, hide, onDelete, onDownload }) => { + return ( + theme.zIndex.drawer + 1, + }}> + + + + + + + Download + + + + + + + + Delete + + + + + ) } \ No newline at end of file diff --git a/frontend/src/views/Terminal.tsx b/frontend/src/views/Terminal.tsx new file mode 100644 index 0000000..b93acc9 --- /dev/null +++ b/frontend/src/views/Terminal.tsx @@ -0,0 +1,9 @@ +import LogTerminal from '../components/LogTerminal' + +const Terminal: React.FC = () => { + return ( + + ) +} + +export default Terminal \ No newline at end of file diff --git a/go.mod b/go.mod index 80d681d..24fcf59 100644 --- a/go.mod +++ b/go.mod @@ -9,19 +9,28 @@ require ( github.com/google/uuid v1.5.0 github.com/gorilla/websocket v1.5.1 github.com/marcopeocchi/fazzoletti v0.0.0-20230308161120-c545580f79fa + github.com/reactivex/rxgo/v2 v2.5.0 golang.org/x/sys v0.15.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.28.0 ) require ( + github.com/cenkalti/backoff/v4 v4.0.0 // indirect + github.com/davecgh/go-spew v1.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emirpasic/gods v1.12.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/stretchr/objx v0.1.0 // indirect + github.com/stretchr/testify v1.4.0 // indirect + github.com/teivah/onecontext v0.0.0-20200513185103-40f981bfd775 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/tools v0.16.1 // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.41.0 // indirect modernc.org/ccgo/v3 v3.16.15 // indirect diff --git a/go.sum b/go.sum index e9133f2..ed7131f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ +github.com/cenkalti/backoff/v4 v4.0.0 h1:6VeaLF9aI+MAUQ95106HwWzYZgJJpZ4stumjj6RFYAU= +github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= @@ -14,26 +20,58 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/marcopeocchi/fazzoletti v0.0.0-20230308161120-c545580f79fa h1:uaAQLGhN4SesB9inOQ1Q6EH+BwTWHQOvwhR0TIJvnYc= github.com/marcopeocchi/fazzoletti v0.0.0-20230308161120-c545580f79fa/go.mod h1:RvfVo/6Sbnfra9kkvIxDW8NYOOaYsHjF0DdtMCs9cdo= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/reactivex/rxgo/v2 v2.5.0 h1:FhPgHwX9vKdNQB2gq9EPt+EKk9QrrzoeztGbEEnZam4= +github.com/reactivex/rxgo/v2 v2.5.0/go.mod h1:bs4fVZxcb5ZckLIOeIeVH942yunJLWDABWGbrHAW+qU= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/teivah/onecontext v0.0.0-20200513185103-40f981bfd775 h1:BLNsFR8l/hj/oGjnJXkd4Vi3s4kQD3/3x8HSAE4bzN0= +github.com/teivah/onecontext v0.0.0-20200513185103-40f981bfd775/go.mod h1:XUZ4x3oGhWfiOnUvTslnKKs39AWUct3g3yJvXTQSJOQ= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= diff --git a/server/config/parser.go b/server/config/parser.go index 10bd6aa..bedff3b 100644 --- a/server/config/parser.go +++ b/server/config/parser.go @@ -8,6 +8,8 @@ import ( ) type Config struct { + CurrentLogFile string + LogPath string `yaml:"log_path"` Host string `yaml:"host"` Port int `yaml:"port"` DownloadPath string `yaml:"downloadPath"` diff --git a/server/internal/memory_db.go b/server/internal/memory_db.go index ab9d71c..195384b 100644 --- a/server/internal/memory_db.go +++ b/server/internal/memory_db.go @@ -4,13 +4,11 @@ import ( "encoding/gob" "errors" "fmt" - "log" "os" "path/filepath" "sync" "github.com/google/uuid" - "github.com/marcopeocchi/yt-dlp-web-ui/server/cli" "github.com/marcopeocchi/yt-dlp-web-ui/server/config" ) @@ -94,14 +92,14 @@ func (m *MemoryDB) All() *[]ProcessResponse { } // WIP: Persist the database in a single file named "session.dat" -func (m *MemoryDB) Persist() { +func (m *MemoryDB) Persist() error { running := m.All() sf := filepath.Join(config.Instance().SessionFilePath, "session.dat") fd, err := os.Create(sf) if err != nil { - log.Println(cli.Red, "Failed to persist session", cli.Reset) + return errors.Join(errors.New("failed to persist session"), err) } session := Session{ @@ -110,10 +108,10 @@ func (m *MemoryDB) Persist() { err = gob.NewEncoder(fd).Encode(session) if err != nil { - log.Println(cli.Red, "Failed to persist session", cli.Reset) + return errors.Join(errors.New("failed to persist session"), err) } - log.Println(cli.BgBlue, "Successfully serialized session", cli.Reset) + return nil } // WIP: Restore a persisted state @@ -146,6 +144,4 @@ func (m *MemoryDB) Restore() { go restored.Start() } } - - log.Println(cli.BgGreen, "Successfully restored session", cli.Reset) } diff --git a/server/internal/message_queue.go b/server/internal/message_queue.go index a319802..d6dcb57 100644 --- a/server/internal/message_queue.go +++ b/server/internal/message_queue.go @@ -1,8 +1,6 @@ package internal import ( - "log" - "github.com/marcopeocchi/yt-dlp-web-ui/server/config" ) @@ -19,7 +17,7 @@ func NewMessageQueue() *MessageQueue { size := config.Instance().QueueSize if size <= 0 { - log.Fatalln("invalid queue size") + panic("invalid queue size") } return &MessageQueue{ diff --git a/server/internal/playlist.go b/server/internal/playlist.go index 0a4d843..422e8e9 100644 --- a/server/internal/playlist.go +++ b/server/internal/playlist.go @@ -3,12 +3,11 @@ package internal import ( "encoding/json" "errors" - "log" + "log/slog" "os/exec" "strings" "time" - "github.com/marcopeocchi/yt-dlp-web-ui/server/cli" "github.com/marcopeocchi/yt-dlp-web-ui/server/config" ) @@ -19,7 +18,7 @@ type metadata struct { Type string `json:"_type"` } -func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error { +func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB, logger *slog.Logger) error { var ( downloader = config.Instance().DownloaderPath cmd = exec.Command(downloader, req.URL, "-J") @@ -37,14 +36,14 @@ func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error { return err } - log.Println(cli.BgRed, "Decoding metadata", cli.Reset, req.URL) + logger.Info("decoding metadata", slog.String("url", req.URL)) err = json.NewDecoder(stdout).Decode(&m) if err != nil { return err } - log.Println(cli.BgGreen, "Decoded metadata", cli.Reset, req.URL) + logger.Info("decoded metadata", slog.String("url", req.URL)) if m.Type == "" { cmd.Wait() @@ -52,8 +51,10 @@ func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error { } if m.Type == "playlist" { - log.Println( - cli.BgGreen, "Playlist detected", cli.Reset, m.Count, "entries", + logger.Info( + "playlist detected", + slog.String("url", req.URL), + slog.Int("count", m.Count), ) for i, meta := range m.Entries { @@ -93,8 +94,7 @@ func PlaylistDetect(req DownloadRequest, mq *MessageQueue, db *MemoryDB) error { proc := &Process{Url: req.URL, Params: req.Params} mq.Publish(proc) - log.Println("Sending new process to message queue", proc.Url) + logger.Info("sending new process to message queue", slog.String("url", proc.Url)) - err = cmd.Wait() - return err + return cmd.Wait() } diff --git a/server/internal/process.go b/server/internal/process.go index dd5b20d..8d46a59 100644 --- a/server/internal/process.go +++ b/server/internal/process.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/json" "fmt" + "log/slog" "regexp" "sync" "syscall" @@ -50,6 +51,7 @@ type Process struct { Progress DownloadProgress Output DownloadOutput proc *os.Process + Logger *slog.Logger } type DownloadOutput struct { @@ -106,13 +108,21 @@ func (p *Process) Start() { r, err := cmd.StdoutPipe() if err != nil { - log.Panicln(err) + p.Logger.Error( + "failed to connect to stdout", + slog.String("err", err.Error()), + ) + panic(err) } scan := bufio.NewScanner(r) err = cmd.Start() if err != nil { - log.Panicln(err) + p.Logger.Error( + "failed to start yt-dlp process", + slog.String("err", err.Error()), + ) + panic(err) } p.proc = cmd.Process @@ -151,10 +161,10 @@ func (p *Process) Start() { Speed: stdout.Speed, ETA: stdout.Eta, } - log.Println( - cli.BgGreen, "DL", cli.Reset, - cli.BgBlue, p.getShortId(), cli.Reset, - p.Url, stdout.Percentage, + p.Logger.Info("progress", + slog.String("id", p.getShortId()), + slog.String("url", p.Url), + slog.String("percentege", stdout.Percentage), ) } }) @@ -175,12 +185,9 @@ func (p *Process) Complete() { ETA: 0, } - shortId := p.getShortId() - - log.Println( - cli.BgMagenta, "FINISH", cli.Reset, - cli.BgBlue, shortId, cli.Reset, - p.Url, + p.Logger.Info("finished", + slog.String("id", p.getShortId()), + slog.String("url", p.Url), ) } @@ -197,7 +204,7 @@ func (p *Process) Kill() error { } err = syscall.Kill(-pgid, syscall.SIGTERM) - log.Println("Killed process", p.Id) + p.Logger.Info("killed process", slog.String("id", p.Id)) return err } @@ -233,6 +240,12 @@ func (p *Process) GetFormatsSync() (DownloadFormats, error) { p.Url, ) + p.Logger.Info( + "retrieving metadata", + slog.String("caller", "getFormats"), + slog.String("url", p.Url), + ) + go func() { decodingError = json.Unmarshal(stdout, &info) wg.Done() @@ -264,7 +277,11 @@ func (p *Process) SetMetadata() error { stdout, err := cmd.StdoutPipe() if err != nil { - log.Println("Cannot retrieve info for", p.Url) + p.Logger.Error("failed retrieving info", + slog.String("id", p.getShortId()), + slog.String("url", p.Url), + slog.String("err", err.Error()), + ) return err } @@ -278,10 +295,9 @@ func (p *Process) SetMetadata() error { return err } - log.Println( - cli.BgRed, "Metadata", cli.Reset, - cli.BgBlue, p.getShortId(), cli.Reset, - p.Url, + p.Logger.Info("retrieving metadata", + slog.String("id", p.getShortId()), + slog.String("url", p.Url), ) err = json.NewDecoder(stdout).Decode(&info) diff --git a/server/logging/handler.go b/server/logging/handler.go new file mode 100644 index 0000000..a21116e --- /dev/null +++ b/server/logging/handler.go @@ -0,0 +1,80 @@ +package logging + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" + "github.com/gorilla/websocket" + "github.com/marcopeocchi/yt-dlp-web-ui/server/config" + middlewares "github.com/marcopeocchi/yt-dlp-web-ui/server/middleware" +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, + ReadBufferSize: 1000, + WriteBufferSize: 1000, +} + +func webSocket(w http.ResponseWriter, r *http.Request) { + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + for msg := range logsObservable.Observe() { + c.WriteJSON(msg.V) + } +} + +func sse(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "SSE not supported", http.StatusInternalServerError) + return + } + + for msg := range logsObservable.Observe() { + if msg.E != nil { + http.Error(w, msg.E.Error(), http.StatusInternalServerError) + return + } + + var ( + b bytes.Buffer + sb strings.Builder + ) + + if err := json.NewEncoder(&b).Encode(msg.V); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + sb.WriteString("event: log\n") + sb.WriteString("data: " + b.String() + "\n\n") + + fmt.Fprint(w, sb.String()) + + flusher.Flush() + } +} + +func ApplyRouter() func(chi.Router) { + return func(r chi.Router) { + if config.Instance().RequireAuth { + r.Use(middlewares.Authenticated) + } + r.Get("/ws", webSocket) + r.Get("/sse", sse) + } +} diff --git a/server/logging/observable_logger.go b/server/logging/observable_logger.go new file mode 100644 index 0000000..e1c4359 --- /dev/null +++ b/server/logging/observable_logger.go @@ -0,0 +1,29 @@ +package logging + +import ( + "time" + + "github.com/reactivex/rxgo/v2" +) + +var ( + logsChan = make(chan rxgo.Item, 100) + logsObservable = rxgo. + FromChannel(logsChan, rxgo.WithBackPressureStrategy(rxgo.Drop)). + BufferWithTime(rxgo.WithDuration(time.Millisecond * 500)) +) + +type ObservableLogger struct{} + +func NewObservableLogger() *ObservableLogger { + return &ObservableLogger{} +} + +func (o *ObservableLogger) Write(p []byte) (n int, err error) { + logsChan <- rxgo.Of(string(p)) + + n = len(p) + err = nil + + return +} diff --git a/server/rpc/container.go b/server/rpc/container.go index 1528c8d..7328fc3 100644 --- a/server/rpc/container.go +++ b/server/rpc/container.go @@ -1,6 +1,8 @@ package rpc import ( + "log/slog" + "github.com/go-chi/chi/v5" "github.com/marcopeocchi/yt-dlp-web-ui/server/config" "github.com/marcopeocchi/yt-dlp-web-ui/server/internal" @@ -8,10 +10,15 @@ import ( ) // Dependency injection container. -func Container(db *internal.MemoryDB, mq *internal.MessageQueue) *Service { +func Container( + db *internal.MemoryDB, + mq *internal.MessageQueue, + logger *slog.Logger, +) *Service { return &Service{ - db: db, - mq: mq, + db: db, + mq: mq, + logger: logger, } } diff --git a/server/rpc/service.go b/server/rpc/service.go index bd7f2d8..2df2c32 100644 --- a/server/rpc/service.go +++ b/server/rpc/service.go @@ -1,7 +1,7 @@ package rpc import ( - "log" + "log/slog" "github.com/marcopeocchi/yt-dlp-web-ui/server/internal" "github.com/marcopeocchi/yt-dlp-web-ui/server/sys" @@ -9,8 +9,9 @@ import ( ) type Service struct { - db *internal.MemoryDB - mq *internal.MessageQueue + db *internal.MemoryDB + mq *internal.MessageQueue + logger *slog.Logger } type Running []internal.ProcessResponse @@ -34,6 +35,7 @@ func (s *Service) Exec(args internal.DownloadRequest, result *string) error { Path: args.Path, Filename: args.Rename, }, + Logger: s.logger, } s.db.Set(p) @@ -46,7 +48,7 @@ func (s *Service) Exec(args internal.DownloadRequest, result *string) error { // Exec spawns a Process. // The result of the execution is the newly spawned process Id. func (s *Service) ExecPlaylist(args internal.DownloadRequest, result *string) error { - err := internal.PlaylistDetect(args, s.mq, s.db) + err := internal.PlaylistDetect(args, s.mq, s.db, s.logger) if err != nil { return err } @@ -88,7 +90,7 @@ func (s *Service) Running(args NoArgs, running *Running) error { // Kill kills a process given its id and remove it from the memoryDB func (s *Service) Kill(args string, killed *string) error { - log.Println("Trying killing process with id", args) + s.logger.Info("Trying killing process with id", slog.String("id", args)) proc, err := s.db.Get(args) if err != nil { @@ -106,7 +108,7 @@ func (s *Service) Kill(args string, killed *string) error { // KillAll kills all process unconditionally and removes them from // the memory db func (s *Service) KillAll(args NoArgs, killed *string) error { - log.Println("Killing all spawned processes", args) + s.logger.Info("Killing all spawned processes") keys := s.db.Keys() var err error for _, key := range *keys { @@ -125,7 +127,7 @@ func (s *Service) KillAll(args NoArgs, killed *string) error { // Remove a process from the db rendering it unusable if active func (s *Service) Clear(args string, killed *string) error { - log.Println("Clearing process with id", args) + s.logger.Info("Clearing process with id", slog.String("id", args)) s.db.Delete(args) return nil } @@ -148,7 +150,7 @@ func (s *Service) DirectoryTree(args NoArgs, tree *[]string) error { // Updates the yt-dlp binary using its builtin function func (s *Service) UpdateExecutable(args NoArgs, updated *bool) error { - log.Println("Updating yt-dlp executable to the latest release") + s.logger.Info("Updating yt-dlp executable to the latest release") err := updater.UpdateExecutable() if err != nil { *updated = true diff --git a/server/server.go b/server/server.go index 216fe13..e57086b 100644 --- a/server/server.go +++ b/server/server.go @@ -4,8 +4,9 @@ import ( "context" "database/sql" "fmt" + "io" "io/fs" - "log" + "log/slog" "net/http" "net/rpc" "os" @@ -20,6 +21,7 @@ import ( "github.com/marcopeocchi/yt-dlp-web-ui/server/dbutils" "github.com/marcopeocchi/yt-dlp-web-ui/server/handlers" "github.com/marcopeocchi/yt-dlp-web-ui/server/internal" + "github.com/marcopeocchi/yt-dlp-web-ui/server/logging" middlewares "github.com/marcopeocchi/yt-dlp-web-ui/server/middleware" "github.com/marcopeocchi/yt-dlp-web-ui/server/rest" ytdlpRPC "github.com/marcopeocchi/yt-dlp-web-ui/server/rpc" @@ -29,6 +31,7 @@ import ( type serverConfig struct { frontend fs.FS + logger *slog.Logger host string port int mdb *internal.MemoryDB @@ -40,14 +43,21 @@ func RunBlocking(host string, port int, frontend fs.FS, dbPath string) { var mdb internal.MemoryDB mdb.Restore() + logger := slog.New( + slog.NewTextHandler( + io.MultiWriter(os.Stdout, logging.NewObservableLogger()), + nil, + ), + ) + db, err := sql.Open("sqlite", dbPath) if err != nil { - log.Fatalln(err) + logger.Error("failed to open database", slog.String("err", err.Error())) } err = dbutils.AutoMigrate(context.Background(), db) if err != nil { - log.Fatalln(err) + logger.Error("failed to init database", slog.String("err", err.Error())) } mq := internal.NewMessageQueue() @@ -55,6 +65,7 @@ func RunBlocking(host string, port int, frontend fs.FS, dbPath string) { srv := newServer(serverConfig{ frontend: frontend, + logger: logger, host: host, port: port, mdb: &mdb, @@ -63,13 +74,15 @@ func RunBlocking(host string, port int, frontend fs.FS, dbPath string) { }) go gracefulShutdown(srv, &mdb) - go autoPersist(time.Minute*5, &mdb) + go autoPersist(time.Minute*5, &mdb, logger) - log.Fatal(srv.ListenAndServe()) + if err := srv.ListenAndServe(); err != nil { + logger.Warn("http server stopped", slog.String("err", err.Error())) + } } func newServer(c serverConfig) *http.Server { - service := ytdlpRPC.Container(c.mdb, c.mq) + service := ytdlpRPC.Container(c.mdb, c.mq, c.logger) rpc.Register(service) r := chi.NewRouter() @@ -91,9 +104,7 @@ func newServer(c serverConfig) *http.Server { r.Use(corsMiddleware.Handler) r.Use(middleware.Logger) - app := http.FileServer(http.FS(c.frontend)) - - r.Mount("/", app) + r.Mount("/", http.FileServer(http.FS(c.frontend))) // Archive routes r.Route("/archive", func(r chi.Router) { @@ -118,6 +129,9 @@ func newServer(c serverConfig) *http.Server { // REST API handlers r.Route("/api/v1", rest.ApplyRouter(c.db, c.mdb, c.mq)) + // Logging + r.Route("/log", logging.ApplyRouter()) + return &http.Server{ Addr: fmt.Sprintf("%s:%d", c.host, c.port), Handler: r, @@ -133,7 +147,7 @@ func gracefulShutdown(srv *http.Server, db *internal.MemoryDB) { go func() { <-ctx.Done() - log.Println("shutdown signal received") + slog.Info("shutdown signal received") defer func() { db.Persist() @@ -143,9 +157,15 @@ func gracefulShutdown(srv *http.Server, db *internal.MemoryDB) { }() } -func autoPersist(d time.Duration, db *internal.MemoryDB) { +func autoPersist(d time.Duration, db *internal.MemoryDB, logger *slog.Logger) { for { - db.Persist() + if err := db.Persist(); err != nil { + logger.Info( + "failed to persisted session", + slog.String("err", err.Error()), + ) + } + logger.Info("sucessfully persisted session") time.Sleep(d) } } diff --git a/server/utils/logrotate.go b/server/utils/logrotate.go new file mode 100644 index 0000000..684f1a9 --- /dev/null +++ b/server/utils/logrotate.go @@ -0,0 +1,55 @@ +package utils + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/marcopeocchi/yt-dlp-web-ui/server/config" +) + +func LogRotate() (*os.File, error) { + logs := findLogs() + + for _, log := range logs { + logfd, err := os.Open(log) + if err != nil { + return nil, err + } + + gzWriter, err := os.Create(log + ".gz") + if err != nil { + return nil, err + } + + _, err = io.Copy(gzWriter, logfd) + if err != nil { + return nil, err + } + } + + logfile := time.Now().String() + ".log" + config.Instance().CurrentLogFile = logfile + + return os.Create(logfile) +} + +func findLogs() []string { + var ( + logfiles []string + root = config.Instance().LogPath + ) + + filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if filepath.Ext(d.Name()) == ".log" { + logfiles = append(logfiles, path) + } + return nil + }) + return logfiles +}