Fixhancement: truncate large logs, improve auto-scroll (#11239)

src-ui/src/app/components/admin/logs/logs.component.html

@@ -3,9 +3,23 @@
   i18n-title
   info="Review the log files for the application and for email checking."
   i18n-info>
-  <div class="form-check form-switch">
-    <input class="form-check-input" type="checkbox" role="switch" [(ngModel)]="autoRefreshEnabled">
-    <label class="form-check-label" for="autoRefreshSwitch" i18n>Auto refresh</label>
+  <div class="input-group input-group-sm align-items-center">
+    <div class="input-group input-group-sm me-3">
+      <span class="input-group-text text-muted" i18n>Show</span>
+      <input
+        class="form-control"
+        type="number"
+        min="100"
+        step="100"
+        [(ngModel)]="limit"
+        (ngModelChange)="onLimitChange($event)"
+        style="width: 100px;">
+      <span class="input-group-text text-muted" i18n>lines</span>
+    </div>
+    <div class="form-check form-switch mt-1">
+      <input class="form-check-input" type="checkbox" role="switch" [(ngModel)]="autoRefreshEnabled">
+      <label class="form-check-label" for="autoRefreshSwitch" i18n>Auto refresh</label>
+    </div>
   </div>
 </pngx-page-header>
 

src-ui/src/app/components/admin/logs/logs.component.spec.ts

@@ -67,7 +67,7 @@ describe('LogsComponent', () => {
   })
 
   it('should display logs with first log initially', () => {
-    expect(logSpy).toHaveBeenCalledWith('paperless')
+    expect(logSpy).toHaveBeenCalledWith('paperless', 5000)
     fixture.detectChanges()
     expect(fixture.debugElement.nativeElement.textContent).toContain(
       paperless_logs[0]
@@ -78,7 +78,7 @@ describe('LogsComponent', () => {
     fixture.debugElement
       .queryAll(By.directive(NgbNavLink))[1]
       .nativeElement.dispatchEvent(new MouseEvent('click'))
-    expect(logSpy).toHaveBeenCalledWith('mail')
+    expect(logSpy).toHaveBeenCalledWith('mail', 5000)
   })
 
   it('should handle error with no logs', () => {
@@ -101,4 +101,13 @@ describe('LogsComponent', () => {
     jest.advanceTimersByTime(6000)
     expect(reloadSpy).toHaveBeenCalledTimes(2)
   })
+
+  it('should debounce limit changes before reloading logs', () => {
+    const initialCalls = reloadSpy.mock.calls.length
+    component.onLimitChange(6000)
+    jest.advanceTimersByTime(299)
+    expect(reloadSpy).toHaveBeenCalledTimes(initialCalls)
+    jest.advanceTimersByTime(1)
+    expect(reloadSpy).toHaveBeenCalledTimes(initialCalls + 1)
+  })
 })

src-ui/src/app/components/admin/logs/logs.component.ts

@@ -13,7 +13,7 @@ import {
 } from '@angular/core'
 import { FormsModule, ReactiveFormsModule } from '@angular/forms'
 import { NgbNavModule } from '@ng-bootstrap/ng-bootstrap'
-import { filter, takeUntil, timer } from 'rxjs'
+import { Subject, debounceTime, filter, takeUntil, timer } from 'rxjs'
 import { LogService } from 'src/app/services/rest/log.service'
 import { PageHeaderComponent } from '../../common/page-header/page-header.component'
 import { LoadingComponentWithPermissions } from '../../loading-component/loading.component'
@@ -47,9 +47,17 @@ export class LogsComponent
 
   public autoRefreshEnabled: boolean = true
 
+  public limit: number = 5000
+
+  private readonly limitChange$ = new Subject<number>()
+
   @ViewChild('logContainer') logContainer: CdkVirtualScrollViewport
 
   ngOnInit(): void {
+    this.limitChange$
+      .pipe(debounceTime(300), takeUntil(this.unsubscribeNotifier))
+      .subscribe(() => this.reloadLogs())
+
     this.logService
       .list()
       .pipe(takeUntil(this.unsubscribeNotifier))
@@ -75,16 +83,33 @@ export class LogsComponent
     super.ngOnDestroy()
   }
 
+  onLimitChange(limit: number): void {
+    this.limitChange$.next(limit)
+  }
+
   reloadLogs() {
     this.loading = true
     this.logService
-      .get(this.activeLog)
+      .get(this.activeLog, this.limit)
       .pipe(takeUntil(this.unsubscribeNotifier))
       .subscribe({
         next: (result) => {
-          this.logs = this.parseLogsWithLevel(result)
           this.loading = false
-          this.scrollToBottom()
+          const parsed = this.parseLogsWithLevel(result)
+          const hasChanges =
+            parsed.length !== this.logs.length ||
+            parsed.some((log, idx) => {
+              const current = this.logs[idx]
+              return (
+                !current ||
+                current.message !== log.message ||
+                current.level !== log.level
+              )
+            })
+          if (hasChanges) {
+            this.logs = parsed
+            this.scrollToBottom()
+          }
         },
         error: () => {
           this.logs = []

src-ui/src/app/services/rest/log.service.spec.ts

@@ -49,4 +49,14 @@ describe('LogService', () => {
     )
     expect(req.request.method).toEqual('GET')
   })
+
+  it('should pass limit param on logs get when provided', () => {
+    const id: string = 'mail'
+    const limit: number = 100
+    subscription = service.get(id, limit).subscribe()
+    const req = httpTestingController.expectOne(
+      `${environment.apiBaseUrl}${endpoint}/${id}/?limit=${limit}`
+    )
+    expect(req.request.method).toEqual('GET')
+  })
 })

src-ui/src/app/services/rest/log.service.ts

@@ -1,4 +1,4 @@
-import { HttpClient } from '@angular/common/http'
+import { HttpClient, HttpParams } from '@angular/common/http'
 import { Injectable, inject } from '@angular/core'
 import { Observable } from 'rxjs'
 import { environment } from 'src/environments/environment'
@@ -13,7 +13,13 @@ export class LogService {
     return this.http.get<string[]>(`${environment.apiBaseUrl}logs/`)
   }
 
-  get(id: string): Observable<string[]> {
-    return this.http.get<string[]>(`${environment.apiBaseUrl}logs/${id}/`)
+  get(id: string, limit?: number): Observable<string[]> {
+    let params = new HttpParams()
+    if (limit !== undefined) {
+      params = params.set('limit', limit.toString())
+    }
+    return this.http.get<string[]>(`${environment.apiBaseUrl}logs/${id}/`, {
+      params,
+    })
   }
 }

src/documents/tests/test_api_documents.py

@@ -2250,6 +2250,23 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertListEqual(response.data, ["test", "test2"])
 
+    def test_get_log_with_limit(self):
+        log_data = "test1\ntest2\ntest3\n"
+        with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:
+            f.write(log_data)
+        response = self.client.get("/api/logs/paperless/", {"limit": 2})
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertListEqual(response.data, ["test2", "test3"])
+
+    def test_get_log_with_invalid_limit(self):
+        log_data = "test1\ntest2\n"
+        with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:
+            f.write(log_data)
+        response = self.client.get("/api/logs/paperless/", {"limit": "abc"})
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        response = self.client.get("/api/logs/paperless/", {"limit": -5})
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
     def test_invalid_regex_other_algorithm(self):
         for endpoint in ["correspondents", "tags", "document_types"]:
             response = self.client.post(

src/documents/views.py

@@ -6,6 +6,7 @@ import re
 import tempfile
 import zipfile
 from collections import defaultdict
+from collections import deque
 from datetime import datetime
 from pathlib import Path
 from time import mktime
@@ -70,6 +71,7 @@ from rest_framework import parsers
 from rest_framework import serializers
 from rest_framework.decorators import action
 from rest_framework.exceptions import NotFound
+from rest_framework.exceptions import ValidationError
 from rest_framework.filters import OrderingFilter
 from rest_framework.filters import SearchFilter
 from rest_framework.generics import GenericAPIView
@@ -1363,6 +1365,13 @@ class UnifiedSearchViewSet(DocumentViewSet):
                 type=OpenApiTypes.STR,
                 location=OpenApiParameter.PATH,
             ),
+            OpenApiParameter(
+                name="limit",
+                type=OpenApiTypes.INT,
+                location=OpenApiParameter.QUERY,
+                description="Return only the last N entries from the log file",
+                required=False,
+            ),
         ],
         responses={
             (200, "application/json"): serializers.ListSerializer(
@@ -1394,8 +1403,22 @@ class LogViewSet(ViewSet):
         if not log_file.is_file():
             raise Http404
 
+        limit_param = request.query_params.get("limit")
+        if limit_param is not None:
+            try:
+                limit = int(limit_param)
+            except (TypeError, ValueError):
+                raise ValidationError({"limit": "Must be a positive integer"})
+            if limit < 1:
+                raise ValidationError({"limit": "Must be a positive integer"})
+        else:
+            limit = None
+
         with log_file.open() as f:
-            lines = [line.rstrip() for line in f.readlines()]
+            if limit is None:
+                lines = [line.rstrip() for line in f.readlines()]
+            else:
+                lines = [line.rstrip() for line in deque(f, maxlen=limit)]
 
         return Response(lines)