updated memory_db.go

Closes #202
reduced log rotation memory usage
2024-09-18 21:25:06 +02:00 · 2024-09-18 17:50:14 +02:00 · 2024-09-18 17:49:25 +02:00 · 2024-09-18 13:08:57 +02:00 · 2024-09-18 13:05:06 +02:00 · 2024-09-18 11:40:49 +02:00
12 changed files with 222 additions and 221 deletions
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -10,4 +10,4 @@ liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
 lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
+custom: ['https://paypal.me/marcofw']
--- a/README.md
+++ b/README.md
@@ -1,7 +1,10 @@
 # yt-dlp Web UI

-A not so terrible web ui for yt-dlp.  
-Created for the only purpose of *fetching* videos from my server/nas. 
+A not so terrible web ui for yt-dlp. 
+
+High performance extendeable web ui and RPC server for yt-dlp with low impact on resources.
+
+Created for the only purpose of *fetching* videos from my server/nas and monitor upcoming livestreams. 

 **Docker images are available on [Docker Hub](https://hub.docker.com/r/marcobaobao/yt-dlp-webui) or [ghcr.io](https://github.com/marcopeocchi/yt-dlp-web-ui/pkgs/container/yt-dlp-web-ui)**.

@@ -13,6 +16,16 @@ docker pull marcobaobao/yt-dlp-webui
 docker pull ghcr.io/marcopeocchi/yt-dlp-web-ui:latest
 ```

+## Donate to yt-dlp-webui development
+[PayPal](https://paypal.me/marcofw)
+
+*Keeps the project alive!* 😃
+
+## Some screeshots
+![image](https://github.com/user-attachments/assets/fc43a3fb-ecf9-449d-b5cb-5d5635020c00)
+![image](https://github.com/user-attachments/assets/3210f6ac-0dd8-403c-b839-3c24ff7d7d00)
+![image](https://github.com/user-attachments/assets/16450a40-cda6-4c8b-9d20-8ec36282f6ed)
+
 ## Video showcase
 [app.webm](https://github.com/marcopeocchi/yt-dlp-web-ui/assets/35533749/91545bc4-233d-4dde-8504-27422cb26964)

@@ -115,21 +128,29 @@ Usage yt-dlp-webui:
  -auth
        Enable RPC authentication
  -conf string
-        Config file path
+        Config file path (default "./config.yml")
+  -db string
+        local database path (default "local.db")
  -driver string
        yt-dlp executable path (default "yt-dlp")
-  -out string
-        Where files will be saved (default ".")
+  -fl
+        enable file based logging
  -host string
        Host where server will listen at (default "0.0.0.0")
+  -lf string
+        set log file location (default "yt-dlp-webui.log")
+  -out string
+        Where files will be saved (default ".")
+  -pass string
+        Password required for auth
  -port int
        Port where server will listen at (default 3033)
  -qs int
-        Download queue size (defaults to the number of logical CPU. A min of 2 is recomended.)
+        Queue size (concurrent downloads) (default 2)
+  -session string
+        session file path (default ".")
  -user string
        Username required for auth
-  -pass string
-        Password required for auth
 ```

 ### Config file
@@ -155,17 +176,23 @@ require_auth: true
 username: my_username
 password: my_random_secret

-# [optional] The download queue size (default: 8)
-queue_size: 4
+# [optional] The download queue size (default: logical cpu cores)
+queue_size: 4 # min. 2

 # [optional] Full path to the yt-dlp (default: "yt-dlp")
 downloaderPath: /usr/local/bin/yt-dlp

+# [optional] Enable file based logging with rotation (default: false)
+#enable_file_logging: false
+
 # [optional] Directory where the log file will be stored (default: ".")
 #log_path: .

 # [optional] Directory where the session database file will be stored (default: ".")
 #session_file_path: .
+
+# [optional] Path where the sqlite database will be created/opened (default: "./local.db")
+#local_database_path
 ```

 ### Systemd integration
@@ -214,13 +241,9 @@ ExecStart=/usr/local/bin/yt-dlp-webui --conf /home/your_user/yt-dlp-webui-workin

 ## Manual installation
 ```sh
-# the dependencies are: python3, ffmpeg, nodejs, psmisc, go.
+# the dependencies are: yt-dlp, ffmpeg, nodejs, go, make.

-cd frontend
-npm i
-npm run build
-
-go build -o yt-dlp-webui main.go
+make all
 ```
 ## Open-API
 Navigate to `/openapi` to see the related swagger.
@@ -243,7 +266,7 @@ For more info, please refer to the [official documentation](https://nixos.org/le
 `yt-dlp-webui` isn't your ordinary website where to download stuff from the internet, so don't try asking for links of where this is hosted. It's a self hosted platform for a Linux NAS.

 ## Troubleshooting
-   **It says that it isn't connected/ip in the header is not defined.**
-    - You must set the server ip address in the settings section (gear icon).
+-   **It says that it isn't connected.**
+    - In some circumstances, you must set the server ip address or hostname in the settings section (gear icon).
 -   **The download  doesn't start.**
-    - As before server address is not specified or simply yt-dlp process takes a lot of time to fire up. (Forking yt-dlp isn't fast especially if you have a lower-end/low-power NAS/server/desktop where the server is running)
+    - Simply, yt-dlp process takes a lot of time to fire up. (yt-dlp isn't fast especially if you have a lower-end/low-power NAS/server/desktop. Furthermore some yt-dlp builds are slower than others)
--- a/frontend/src/assets/i18n.yaml
+++ b/frontend/src/assets/i18n.yaml
@@ -515,38 +515,38 @@ languages:
    customPath: 保存先
    customArgs: yt-dlpのオプションの有効化 (最適設定にする場合)
    customArgsInput: yt-dlpのオプション
-    rpcConnErr: Error while conencting to RPC server
-    splashText: No active downloads
-    archiveTitle: Archive
-    clipboardAction: Copied URL to clipboard
-    playlistCheckbox: Download playlist (it will take time, after submitting you may even close this window)
-    servedFromReverseProxyCheckbox: Is behind a reverse proxy subfolder
-    newDownloadButton: New download
-    homeButtonLabel: Home
-    archiveButtonLabel: Archive
-    settingsButtonLabel: Settings
-    rpcAuthenticationLabel: RPC authentication
-    themeTogglerLabel: Theme toggler
-    loadingLabel: Loading...
-    appTitle: App title
-    savedTemplates: Saved templates
-    templatesEditor: Templates editor
-    templatesEditorNameLabel: Template name
-    templatesEditorContentLabel: Template content
-    logsTitle: 'Logs'
-    awaitingLogs: 'Awaiting logs...'
-    bulkDownload: 'Download files in a zip archive'
-    livestreamURLInput: Livestream URL
-    livestreamStatusWaiting: Waiting/Wait start
-    livestreamStatusDownloading: Downloading
-    livestreamStatusCompleted: Completed
-    livestreamStatusErrored: Errored
-    livestreamStatusUnknown: Unknown
+    rpcConnErr: RPCサーバーへの接続中にエラーが発生しました
+    splashText: アクティブなダウンロードはありません
+    archiveTitle: アーカイブ
+    clipboardAction: URLをクリップボードにコピーしました
+    playlistCheckbox: プレイリストをダウンロード (これには時間がかかりますが、処理中はウィンドウを閉じることができます)
+    servedFromReverseProxyCheckbox: リバースプロキシのサブフォルダにあります
+    newDownloadButton: 新しくダウンロード
+    homeButtonLabel: ホーム
+    archiveButtonLabel: アーカイブ
+    settingsButtonLabel: 設定
+    rpcAuthenticationLabel: RPC認証
+    themeTogglerLabel: テーマ切り替え
+    loadingLabel: 読み込み中...
+    appTitle: アプリタイトル
+    savedTemplates: 保存したテンプレート
+    templatesEditor: テンプレートエディター
+    templatesEditorNameLabel: テンプレート名
+    templatesEditorContentLabel: テンプレート内容
+    logsTitle: 'ログ'
+    awaitingLogs: 'ログを待機中...'
+    bulkDownload: 'ダウンロードしたファイルをZIPで保存'
+    livestreamURLInput: ライブストリームURL
+    livestreamStatusWaiting: 開始を待っています
+    livestreamStatusDownloading: ダウンロード中
+    livestreamStatusCompleted: 完了
+    livestreamStatusErrored: エラー
+    livestreamStatusUnknown: 不明
    livestreamDownloadInfo: |
-      This will monitor yet to start livestream. Each process will be executed with --wait-for-video 10.
-      If an already started livestream is provided it will be still downloaded but its progress will not be tracked.
-      Once started the livestream will be migrated to the downloads page.
-    livestreamExperimentalWarning: This feature is still experimental. Something might break!
+      まだ開始されていないライブストリームを監視します。各プロセスは、--wait-for-video 10 で実行されます。
+      すでに開始されているライブストリームが提供された場合、ダウンロードは継続されますが進行状況は追跡されません。
+      ライブストリームが開始されると、ダウンロードページに移動されます。
+    livestreamExperimentalWarning: この機能は実験的なものです。何かが壊れるかもしれません！
  catalan:
    urlInput: URL de YouTube o d'un altre servei compatible
    statusTitle: Estat
--- a/frontend/src/atoms/settings.ts
+++ b/frontend/src/atoms/settings.ts
@@ -8,12 +8,14 @@ export const languages = [
  'english',
  'chinese',
  'russian',
+  'french',
  'italian',
  'spanish',
  'korean',
  'japanese',
  'catalan',
  'ukrainian',
+  'swedish',
  'polish',
  'german'
 ] as const
--- a/frontend/src/components/LogTerminal.tsx
+++ b/frontend/src/components/LogTerminal.tsx
@@ -22,8 +22,8 @@ const LogTerminal: React.FC = () => {

  useEffect(() => {
    eventSource.addEventListener('log', event => {
-      const msg: string[] = JSON.parse(event.data)
-      setLogBuffer(buff => [...buff, ...msg].slice(-500))
+      const msg: string = JSON.parse(event.data)
+      setLogBuffer(buff => [...buff, msg].slice(-500))

      boxRef.current?.scrollTo(0, boxRef.current.scrollHeight)
    })
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,6 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.3
-	github.com/reactivex/rxgo/v2 v2.5.0
 	golang.org/x/oauth2 v0.23.0
 	golang.org/x/sync v0.8.0
 	golang.org/x/sys v0.25.0
@@ -19,21 +18,16 @@ require (
 )

 require (
-	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
-	github.com/teivah/onecontext v1.3.0 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a // indirect
 	modernc.org/libc v1.61.0 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
--- a/go.sum
+++ b/go.sum
@@ -1,18 +1,11 @@
 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
-github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
-github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
-github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI=
 github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
-github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
 github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
@@ -42,78 +35,40 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/reactivex/rxgo/v2 v2.5.0 h1:FhPgHwX9vKdNQB2gq9EPt+EKk9QrrzoeztGbEEnZam4=
-github.com/reactivex/rxgo/v2 v2.5.0/go.mod h1:bs4fVZxcb5ZckLIOeIeVH942yunJLWDABWGbrHAW+qU=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/teivah/onecontext v0.0.0-20200513185103-40f981bfd775/go.mod h1:XUZ4x3oGhWfiOnUvTslnKKs39AWUct3g3yJvXTQSJOQ=
-github.com/teivah/onecontext v1.3.0 h1:tbikMhAlo6VhAuEGCvhc8HlTnpX4xTNPTOseWuhO1J0=
-github.com/teivah/onecontext v1.3.0/go.mod h1:hoW1nmdPVK/0jrvGtcx8sCKYs2PiS4z0zzfdeuEVyb0=
-go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
-go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
 golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
-golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
-golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
 golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
 golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
-golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
 golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
 modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
-modernc.org/ccgo/v4 v4.20.7 h1:skrinQsjxWfvj6nbC3ztZPJy+NuwmB3hV9zX/pthNYQ=
-modernc.org/ccgo/v4 v4.20.7/go.mod h1:UOkI3JSG2zT4E2ioHlncSOZsXbuDCZLvPi3uMlZT5GY=
 modernc.org/ccgo/v4 v4.21.0 h1:kKPI3dF7RIag8YcToh5ZwDcVMIv6VGa0ED5cvh0LMW4=
+modernc.org/ccgo/v4 v4.21.0/go.mod h1:h6kt6H/A2+ew/3MW/p6KEoQmrq/i3pr0J/SiwiaF/g0=
 modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
 modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
 modernc.org/gc/v2 v2.5.0 h1:bJ9ChznK1L1mUtAQtxi0wi5AtAs5jQuw4PrPHO5pb6M=
 modernc.org/gc/v2 v2.5.0/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
 modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a h1:CfbpOLEo2IwNzJdMvE8aiRbPMxoTpgAJeyePh0SmO8M=
 modernc.org/gc/v3 v3.0.0-20240801135723-a856999a2e4a/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.59.9 h1:k+nNDDakwipimgmJ1D9H466LhFeSkaPPycAs1OZiDmY=
-modernc.org/libc v1.59.9/go.mod h1:EY/egGEU7Ju66eU6SBqCNYaFUDuc4npICkMWnU5EE3A=
-modernc.org/libc v1.60.1 h1:at373l8IFRTkJIkAU85BIuUoBM4T1b51ds0E1ovPG2s=
-modernc.org/libc v1.60.1/go.mod h1:xJuobKuNxKH3RUatS7GjR+suWj+5c2K7bi4m/S5arOY=
 modernc.org/libc v1.61.0 h1:eGFcvWpqlnoGwzZeZe3PWJkkKbM/3SUGyk1DVZQ0TpE=
 modernc.org/libc v1.61.0/go.mod h1:DvxVX89wtGTu+r72MLGhygpfi3aUGgZRdAYGCAVVud0=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
@@ -124,8 +79,6 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
 modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
-modernc.org/sqlite v1.32.0 h1:6BM4uGza7bWypsw4fdLRsLxut6bHe4c58VeqjRgST8s=
-modernc.org/sqlite v1.32.0/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA=
 modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
 modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
 modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
--- a/server/internal/memory_db.go
+++ b/server/internal/memory_db.go
@@ -111,7 +111,9 @@ func (m *MemoryDB) Persist() error {

 // Restore a persisted state
 func (m *MemoryDB) Restore(mq *MessageQueue) {
-	fd, err := os.Open("session.dat")
+	sf := filepath.Join(config.Instance().SessionFilePath, "session.dat")
+
+	fd, err := os.Open(sf)
 	if err != nil {
 		return
 	}
--- a/server/logging/file_logger.go
+++ b/server/logging/file_logger.go
@@ -3,21 +3,23 @@ package logging
 import (
 	"compress/gzip"
 	"io"
+	"log/slog"
 	"os"
+	"strings"
 	"sync"
 	"time"
 )

 /*
-	File base logger with log-rotate capabilities.
-	The rotate process must be initiated from an external goroutine.
+implements io.Writer interface

-	After rotation the previous logs file are compressed with gzip algorithm.
+File base logger with log-rotate capabilities.
+The rotate process must be initiated from an external goroutine.

-	The rotated log follows this naming: [filename].UTC time.gz
+After rotation the previous logs file are compressed with gzip algorithm.
+
+The rotated log follows this naming: [filename].UTC time.gz
 */
-
-// implements io.Writer interface
 type LogRotateWriter struct {
 	mu       sync.Mutex
 	fd       *os.File
@@ -40,50 +42,41 @@ func (w *LogRotateWriter) Write(b []byte) (int, error) {
 }

 func (w *LogRotateWriter) Rotate() error {
-	var err error
+	slog.Info("started log rotation")
+
 	w.mu.Lock()

-	gzFile, err := os.Create(w.filename + "." + time.Now().Format(time.RFC3339) + ".gz")
+	gzFile, err := os.Create(strings.TrimSuffix(w.filename, ".log") + "-" + time.Now().Format(time.RFC3339) + ".log.gz")
 	if err != nil {
 		return err
 	}

-	data, err := io.ReadAll(w.fd)
-	if err != nil {
-		return err
-	}
+	zw := gzip.NewWriter(gzFile)

 	defer func() {
-		w.mu.Unlock()
-		w.gzipLog(gzFile, &data)
+		zw.Close()
+		zw.Flush()
+		gzFile.Close()
 	}()

-	_, err = os.Stat(w.filename)
-	if err != nil {
+	if _, err := os.Stat(w.filename); err != nil {
 		return err
 	}

-	if w.fd != nil {
-		err = w.fd.Close()
-		w.fd = nil
-		if err != nil {
-			return err
-		}
-	}
+	fd, _ := os.Open(w.filename)
+	io.Copy(zw, fd)
+	fd.Close()

-	err = os.Remove(w.filename)
-	if err != nil {
+	w.fd.Close()
+
+	if err := os.Remove(w.filename); err != nil {
 		return err
 	}

-	w.fd, err = os.Create(w.filename)
+	w.fd, _ = os.Create(w.filename)
+
+	w.mu.Unlock()
+	slog.Info("ended log rotation")
+
 	return err
 }
-
-func (w *LogRotateWriter) gzipLog(wr io.Writer, data *[]byte) error {
-	if _, err := gzip.NewWriter(wr).Write(*data); err != nil {
-		return err
-	}
-
-	return nil
-}
--- a/server/logging/handler.go
+++ b/server/logging/handler.go
@@ -3,9 +3,9 @@ package logging
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
+	"io"
+	"log/slog"
 	"net/http"
-	"strings"

 	"github.com/go-chi/chi/v5"
 	"github.com/gorilla/websocket"
@@ -22,58 +22,74 @@ var upgrader = websocket.Upgrader{
 	WriteBufferSize: 1000,
 }

-func webSocket(w http.ResponseWriter, r *http.Request) {
-	c, err := upgrader.Upgrade(w, r, nil)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-
-	for msg := range logsObservable.Observe() {
-		c.WriteJSON(msg.V)
-	}
-}
-
-func sse(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/event-stream")
-	w.Header().Set("Cache-Control", "no-cache")
-	w.Header().Set("Connection", "keep-alive")
-
-	flusher, ok := w.(http.Flusher)
-	if !ok {
-		http.Error(w, "SSE not supported", http.StatusInternalServerError)
-		return
-	}
-
-	for msg := range logsObservable.Observe() {
-		if msg.E != nil {
-			http.Error(w, msg.E.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		var (
-			b  bytes.Buffer
-			sb strings.Builder
-		)
-
-		if err := json.NewEncoder(&b).Encode(msg.V); err != nil {
+func webSocket(logger *ObservableLogger) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		c, err := upgrader.Upgrade(w, r, nil)
+		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}

-		sb.WriteString("event: log\n")
-		sb.WriteString("data: ")
-		sb.WriteString(b.String())
-		sb.WriteRune('\n')
-		sb.WriteRune('\n')
+		logs := logger.Observe(r.Context())

-		fmt.Fprint(w, sb.String())
-
-		flusher.Flush()
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case msg := <-logs:
+				c.WriteJSON(msg)
+			}
+		}
 	}
 }

-func ApplyRouter() func(chi.Router) {
+func sse(logger *ObservableLogger) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		w.Header().Set("Cache-Control", "no-cache")
+		w.Header().Set("Connection", "keep-alive")
+
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			http.Error(w, "SSE not supported", http.StatusInternalServerError)
+			return
+		}
+
+		logs := logger.Observe(r.Context())
+
+		for {
+			select {
+			case <-r.Context().Done():
+				slog.Info("detaching from logger")
+				return
+			case msg, ok := <-logs:
+				if !ok {
+					http.Error(w, "closed logs channel", http.StatusInternalServerError)
+					return
+				}
+
+				var b bytes.Buffer
+
+				b.WriteString("event: log\n")
+				b.WriteString("data: ")
+
+				if err := json.NewEncoder(&b).Encode(msg); err != nil {
+					http.Error(w, err.Error(), http.StatusInternalServerError)
+					return
+				}
+
+				b.WriteRune('\n')
+				b.WriteRune('\n')
+
+				io.Copy(w, &b)
+
+				flusher.Flush()
+			}
+		}
+	}
+}
+
+func ApplyRouter(logger *ObservableLogger) func(chi.Router) {
 	return func(r chi.Router) {
 		if config.Instance().RequireAuth {
 			r.Use(middlewares.Authenticated)
@@ -81,7 +97,7 @@ func ApplyRouter() func(chi.Router) {
 		if config.Instance().UseOpenId {
 			r.Use(openid.Middleware)
 		}
-		r.Get("/ws", webSocket)
-		r.Get("/sse", sse)
+		r.Get("/ws", webSocket(logger))
+		r.Get("/sse", sse(logger))
 	}
 }
--- a/server/logging/observable_logger.go
+++ b/server/logging/observable_logger.go
@@ -1,40 +1,53 @@
 package logging

 import (
-	"time"
-
-	"github.com/reactivex/rxgo/v2"
+	"context"
 )

 /*
-	Logger implementation using the observable pattern.
-	Implements io.Writer interface.
+Logger implementation using the observable pattern.
+Implements io.Writer interface.

-	The observable is an event source which drops everythigng unless there's
-	a subscriber connected.
+The observable is an event source which drops everythigng unless there's
+a subscriber connected.

-	The observer implementatios are a http ServerSentEvents handler and a
-	websocket one in handler.go
+The observer implementatios are a http ServerSentEvents handler and a
+websocket one in handler.go
 */
-
-var (
-	logsChan       = make(chan rxgo.Item, 100)
-	logsObservable = rxgo.
-			FromEventSource(logsChan, rxgo.WithBackPressureStrategy(rxgo.Drop)).
-			BufferWithTime(rxgo.WithDuration(time.Millisecond * 500))
-)
-
-type ObservableLogger struct{}
+type ObservableLogger struct {
+	logsChan chan []byte
+}

 func NewObservableLogger() *ObservableLogger {
-	return &ObservableLogger{}
+	return &ObservableLogger{
+		logsChan: make(chan []byte, 100),
+	}
 }

 func (o *ObservableLogger) Write(p []byte) (n int, err error) {
-	logsChan <- rxgo.Of(string(p))
-
-	n = len(p)
-	err = nil
-
-	return
+	select {
+	case o.logsChan <- p:
+		n = len(p)
+		err = nil
+		return
+	default:
+		return
+	}
+}
+
+func (o *ObservableLogger) Observe(ctx context.Context) <-chan string {
+	logs := make(chan string)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case logLine := <-o.logsChan:
+				logs <- string(logLine)
+			}
+		}
+	}()
+
+	return logs
 }
--- a/server/server.go
+++ b/server/server.go
@@ -47,13 +47,16 @@ type serverConfig struct {
 	lm       *livestream.Monitor
 }

+// TODO: change scope
+var observableLogger = logging.NewObservableLogger()
+
 func RunBlocking(rc *RunConfig) {
 	mdb := internal.NewMemoryDB()

 	// ---- LOGGING ---------------------------------------------------
 	logWriters := []io.Writer{
 		os.Stdout,
-		logging.NewObservableLogger(), // for web-ui
+		observableLogger, // for web-ui
 	}

 	conf := config.Instance()
@@ -65,6 +68,8 @@ func RunBlocking(rc *RunConfig) {
 			panic(err)
 		}

+		defer logger.Rotate()
+
 		go func() {
 			for {
 				time.Sleep(time.Hour * 24)
@@ -207,7 +212,7 @@ func newServer(c serverConfig) *http.Server {
 	}))

 	// Logging
-	r.Route("/log", logging.ApplyRouter())
+	r.Route("/log", logging.ApplyRouter(observableLogger))

 	return &http.Server{Handler: r}
 }
Author	SHA1	Message	Date
Marco Piovanello	8c0e7b3cb8	updated memory_db.go Closes #202	2024-09-18 21:25:06 +02:00
marcobaobao	38da65a848	reduced log rotation memory usage	2024-09-18 17:50:14 +02:00
Marco Piovanello	64fbdbbbdf	Dropping rxgo (#201 ) * rxgo event source to channel with drop strategy * code optimizations	2024-09-18 17:49:25 +02:00
Marco Piovanello	a00059ca88	enabled french and swedish (#199 )	2024-09-18 13:08:57 +02:00
Oさん	7abccaec71	Update Japanese translation (#198 )	2024-09-18 13:05:06 +02:00
Marco Piovanello	f075d3d531	Update README.md	2024-09-18 11:40:49 +02:00
Marco Piovanello	190d891578	Update README.md	2024-09-18 11:40:34 +02:00
marcobaobao	645cc6f9a1	readme update	2024-09-18 11:30:09 +02:00