Merge branch 'dev' into ui-perms-tweaks

docs/advanced_usage.md

@@ -589,6 +589,12 @@ case, Paperless will remove the staging copy as well as the scan, and give you a
 message asking you to restart the process from scratch, by scanning the odd pages again,
 followed by the even pages.
 
+It's important that the scan files get consumed in the correct order, and one at a time.
+You therefore need to make sure that Paperless is running while you upload the files into
+the directory; and if you're using [polling](/configuration#polling), make sure that
+`CONSUMER_POLLING` is set to a value lower than it takes for the second scan to appear,
+like 5-10 or even lower.
+
 Another thing that might happen is that you start a double sided scan, but then forget
 to upload the second file. To avoid collating the wrong documents if you then come back
 a day later to scan a new double-sided document, Paperless will only keep an "odd numbered
@@ -597,11 +603,11 @@ scan a completely new "odd numbered pages" one. The old staging file will get di
 
 ### Interaction with "subdirs as tags"
 
-The collation feature can be used together with the "subdirs as tags" feature (but this is not
+The collation feature can be used together with the [subdirs as tags](/configuration#consume_config)
-a requirement). Just create a correctly named double-sided subdir in the hierachy and upload
+feature (but this is not a requirement). Just create a correctly named double-sided subdir
-your scans there. For example, both `double-sided/foo/bar` as well as `foo/bar/double-sided` will
+in the hierachy and upload your scans there. For example, both `double-sided/foo/bar` as
-cause the collated document to be treated as if it were uploaded into `foo/bar` and receive both
+well as `foo/bar/double-sided` will cause the collated document to be treated as if it
-`foo` and `bar` tags, but not `double-sided`.
+were uploaded into `foo/bar` and receive both `foo` and `bar` tags, but not `double-sided`.
 
 ### Interaction with document splitting
 

docs/configuration.md

@@ -501,6 +501,19 @@ HTTP header/value expected by Django, eg `'["HTTP_X_FORWARDED_PROTO", "https"]'`
     Settings this value has security implications.  Read the Django documentation
     and be sure you understand its usage before setting it.
 
+`PAPERLESS_EMAIL_CERTIFICATE_FILE=<path>`
+
+: Configures an additional SSL certificate file containing a [certificate](https://docs.python.org/3/library/ssl.html#certificates)
+or certificate chain which should be trusted for validating SSL connections against mail providers.
+This is for use with self-signed certificates against local IMAP servers.
+
+    Defaults to None.
+
+!!! warning
+
+    Settings this value has security implications for the security of your email.
+    Understand what it does and be sure you need to before setting.
+
 ## OCR settings {#ocr}
 
 Paperless uses [OCRmyPDF](https://ocrmypdf.readthedocs.io/en/latest/)

src-ui/src/app/components/common/input/tags/tags.component.html

@@ -2,7 +2,7 @@
   <label class="form-label" for="tags" i18n>Tags</label>
 
   <div class="input-group flex-nowrap">
-    <ng-select name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="value"
+    <ng-select #tagSelect name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="value"
       [disabled]="disabled"
       [multiple]="true"
       [closeOnSelect]="false"
@@ -11,11 +11,7 @@
       [addTag]="allowCreate ? createTagRef : false"
       addTagText="Add tag"
       i18n-addTagText
-      (change)="onChange(value)"
+      (change)="onChange(value)">
-      (search)="onSearch($event)"
-      (focus)="clearLastSearchTerm()"
-      (clear)="clearLastSearchTerm()"
-      (blur)="onBlur()">
 
       <ng-template ng-label-tmp let-item="item">
         <span class="tag-wrap tag-wrap-delete" (mousedown)="removeTag($event, item.id)">

src-ui/src/app/components/common/input/tags/tags.component.spec.ts

@@ -15,16 +15,28 @@ import {
   DEFAULT_MATCHING_ALGORITHM,
   MATCH_ALL,
 } from 'src/app/data/matching-model'
-import { NgSelectModule } from '@ng-select/ng-select'
+import { NgSelectComponent, NgSelectModule } from '@ng-select/ng-select'
 import { RouterTestingModule } from '@angular/router/testing'
 import { HttpClientTestingModule } from '@angular/common/http/testing'
 import { of } from 'rxjs'
 import { TagService } from 'src/app/services/rest/tag.service'
 import {
+  NgbAccordionModule,
   NgbModal,
   NgbModalModule,
   NgbModalRef,
+  NgbPopoverModule,
 } from '@ng-bootstrap/ng-bootstrap'
+import { TagEditDialogComponent } from '../../edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
+import { CheckComponent } from '../check/check.component'
+import { IfOwnerDirective } from 'src/app/directives/if-owner.directive'
+import { TextComponent } from '../text/text.component'
+import { ColorComponent } from '../color/color.component'
+import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
+import { PermissionsFormComponent } from '../permissions/permissions-form/permissions-form.component'
+import { SelectComponent } from '../select/select.component'
+import { ColorSliderModule } from 'ngx-color/slider'
+import { By } from '@angular/platform-browser'
 
 const tags: PaperlessTag[] = [
   {
@@ -56,12 +68,32 @@ describe('TagsComponent', () => {
 
   beforeEach(async () => {
     TestBed.configureTestingModule({
-      declarations: [TagsComponent],
+      declarations: [
+        TagsComponent,
+        TagEditDialogComponent,
+        TextComponent,
+        ColorComponent,
+        IfOwnerDirective,
+        SelectComponent,
+        TextComponent,
+        PermissionsFormComponent,
+        ColorComponent,
+        CheckComponent,
+      ],
       providers: [
         {
           provide: TagService,
           useValue: {
-            listAll: () => of(tags),
+            listAll: () =>
+              of({
+                results: tags,
+              }),
+            create: () =>
+              of({
+                name: 'bar',
+                id: 99,
+                color: '#fff000',
+              }),
           },
         },
       ],
@@ -72,6 +104,8 @@ describe('TagsComponent', () => {
         RouterTestingModule,
         HttpClientTestingModule,
         NgbModalModule,
+        NgbAccordionModule,
+        NgbPopoverModule,
       ],
     }).compileComponents()
 
@@ -85,7 +119,7 @@ describe('TagsComponent', () => {
   })
 
   it('should support suggestions', () => {
-    expect(component.value).toBeUndefined()
+    expect(component.value).toHaveLength(0)
     component.value = []
     component.tags = tags
     component.suggestions = [1, 2]
@@ -107,18 +141,18 @@ describe('TagsComponent', () => {
   it('should support create new using last search term and open a modal', () => {
     let activeInstances: NgbModalRef[]
     modalService.activeInstances.subscribe((v) => (activeInstances = v))
-    component.onSearch({ term: 'bar' })
+    component.select.searchTerm = 'foobar'
     component.createTag()
     expect(modalService.hasOpenModals()).toBeTruthy()
-    expect(activeInstances[0].componentInstance.object.name).toEqual('bar')
+    expect(activeInstances[0].componentInstance.object.name).toEqual('foobar')
+    const editDialog = activeInstances[0]
+      .componentInstance as TagEditDialogComponent
+    editDialog.save() // create is mocked
+    fixture.detectChanges()
+    fixture.whenStable().then(() => {
+      expect(fixture.debugElement.nativeElement.textContent).toContain('foobar')
+    })
   })
-
-  it('should clear search term on blur after delay', fakeAsync(() => {
-    const clearSpy = jest.spyOn(component, 'clearLastSearchTerm')
-    component.onBlur()
-    tick(3000)
-    expect(clearSpy).toHaveBeenCalled()
-  }))
 
   it('support remove tags', () => {
     component.tags = tags
@@ -132,6 +166,7 @@ describe('TagsComponent', () => {
   })
 
   it('should get tags', () => {
+    component.tags = null
     expect(component.getTag(2)).toBeNull()
     component.tags = tags
     expect(component.getTag(2)).toEqual(tags[1])

src-ui/src/app/components/common/input/tags/tags.component.ts

@@ -5,6 +5,7 @@ import {
   Input,
   OnInit,
   Output,
+  ViewChild,
 } from '@angular/core'
 import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms'
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
@@ -12,6 +13,8 @@ import { PaperlessTag } from 'src/app/data/paperless-tag'
 import { TagEditDialogComponent } from '../../edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
 import { TagService } from 'src/app/services/rest/tag.service'
 import { EditDialogMode } from '../../edit-dialog/edit-dialog.component'
+import { first, firstValueFrom, tap } from 'rxjs'
+import { NgSelectComponent } from '@ng-select/ng-select'
 
 @Component({
   providers: [
@@ -74,14 +77,14 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
   @Output()
   filterDocuments = new EventEmitter<PaperlessTag[]>()
 
-  value: number[]
+  @ViewChild('tagSelect') select: NgSelectComponent
 
-  tags: PaperlessTag[]
+  value: number[] = []
+
+  tags: PaperlessTag[] = []
 
   public createTagRef: (name) => void
 
-  private _lastSearchTerm: string
-
   getTag(id: number) {
     if (this.tags) {
       return this.tags.find((tag) => tag.id == id)
@@ -111,15 +114,20 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
     })
     modal.componentInstance.dialogMode = EditDialogMode.CREATE
     if (name) modal.componentInstance.object = { name: name }
-    else if (this._lastSearchTerm)
+    else if (this.select.searchTerm)
-      modal.componentInstance.object = { name: this._lastSearchTerm }
+      modal.componentInstance.object = { name: this.select.searchTerm }
-    modal.componentInstance.succeeded.subscribe((newTag) => {
+    this.select.searchTerm = null
+    this.select.detectChanges()
+    return firstValueFrom(
+      (modal.componentInstance as TagEditDialogComponent).succeeded.pipe(
+        first(),
+        tap(() => {
           this.tagService.listAll().subscribe((tags) => {
             this.tags = tags.results
-        this.value = [...this.value, newTag.id]
-        this.onChange(this.value)
           })
         })
+      )
+    )
   }
 
   getSuggestions() {
@@ -137,20 +145,6 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
     this.onChange(this.value)
   }
 
-  clearLastSearchTerm() {
-    this._lastSearchTerm = null
-  }
-
-  onSearch($event) {
-    this._lastSearchTerm = $event.term
-  }
-
-  onBlur() {
-    setTimeout(() => {
-      this.clearLastSearchTerm()
-    }, 3000)
-  }
-
   get hasPrivate(): boolean {
     return this.value.some(
       (t) => this.tags?.find((t2) => t2.id === t) === undefined

src/documents/tests/test_api.py

@@ -3453,6 +3453,110 @@ class TestBulkEdit(DirectoriesMixin, APITestCase):
         self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
         self.assertEqual(len(kwargs["set_permissions"]["view"]["users"]), 2)
 
+    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    def test_insufficient_permissions_ownership(self, m):
+        """
+        GIVEN:
+            - Documents owned by user other than logged in user
+        WHEN:
+            - set_permissions bulk edit API endpoint is called
+        THEN:
+            - User is not able to change permissions
+        """
+        m.return_value = "OK"
+        self.doc1.owner = User.objects.get(username="temp_admin")
+        self.doc1.save()
+        user1 = User.objects.create(username="user1")
+        self.client.force_authenticate(user=user1)
+
+        permissions = {
+            "owner": user1.id,
+        }
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc1.id, self.doc2.id, self.doc3.id],
+                    "method": "set_permissions",
+                    "parameters": {"set_permissions": permissions},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+        m.assert_not_called()
+        self.assertEqual(response.content, b"Insufficient permissions")
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id, self.doc3.id],
+                    "method": "set_permissions",
+                    "parameters": {"set_permissions": permissions},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+
+    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    def test_insufficient_permissions_edit(self, m):
+        """
+        GIVEN:
+            - Documents for which current user only has view permissions
+        WHEN:
+            - API is called
+        THEN:
+            - set_storage_path is only called if user can edit all docs
+        """
+        m.return_value = "OK"
+        self.doc1.owner = User.objects.get(username="temp_admin")
+        self.doc1.save()
+        user1 = User.objects.create(username="user1")
+        assign_perm("view_document", user1, self.doc1)
+        self.client.force_authenticate(user=user1)
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc1.id, self.doc2.id, self.doc3.id],
+                    "method": "set_storage_path",
+                    "parameters": {"storage_path": self.sp1.id},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+        m.assert_not_called()
+        self.assertEqual(response.content, b"Insufficient permissions")
+
+        assign_perm("change_document", user1, self.doc1)
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc1.id, self.doc2.id, self.doc3.id],
+                    "method": "set_storage_path",
+                    "parameters": {"storage_path": self.sp1.id},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        m.assert_called_once()
+
 
 class TestBulkDownload(DirectoriesMixin, APITestCase):
     ENDPOINT = "/api/documents/bulk_download/"

src/documents/views.py

@@ -54,6 +54,7 @@ from rest_framework.viewsets import ModelViewSet
 from rest_framework.viewsets import ReadOnlyModelViewSet
 from rest_framework.viewsets import ViewSet
 
+from documents import bulk_edit
 from documents.filters import ObjectOwnedOrGrantedPermissionsFilter
 from documents.permissions import PaperlessAdminPermissions
 from documents.permissions import PaperlessObjectPermissions
@@ -694,7 +695,7 @@ class SavedViewViewSet(ModelViewSet, PassUserMixin):
         serializer.save(owner=self.request.user)
 
 
-class BulkEditView(GenericAPIView):
+class BulkEditView(GenericAPIView, PassUserMixin):
     permission_classes = (IsAuthenticated,)
     serializer_class = BulkEditSerializer
     parser_classes = (parsers.JSONParser,)
@@ -703,10 +704,25 @@ class BulkEditView(GenericAPIView):
         serializer = self.get_serializer(data=request.data)
         serializer.is_valid(raise_exception=True)
 
+        user = self.request.user
         method = serializer.validated_data.get("method")
         parameters = serializer.validated_data.get("parameters")
         documents = serializer.validated_data.get("documents")
 
+        if not user.is_superuser:
+            document_objs = Document.objects.filter(pk__in=documents)
+            has_perms = (
+                all((doc.owner == user or doc.owner is None) for doc in document_objs)
+                if method == bulk_edit.set_permissions
+                else all(
+                    has_perms_owner_aware(user, "change_document", doc)
+                    for doc in document_objs
+                )
+            )
+
+            if not has_perms:
+                return HttpResponseForbidden("Insufficient permissions")
+
         try:
             # TODO: parameter validation
             result = method(documents, **parameters)

src/paperless/checks.py

@@ -177,6 +177,23 @@ def settings_values_check(app_configs, **kwargs):
             )
         return msgs
 
-    return (
+    def _email_certificate_validate():
-        _ocrmypdf_settings_check() + _timezone_validate() + _barcode_scanner_validate()
+        msgs = []
+        # Existence checks
+        if (
+            settings.EMAIL_CERTIFICATE_FILE is not None
+            and not settings.EMAIL_CERTIFICATE_FILE.is_file()
+        ):
+            msgs.append(
+                Error(
+                    f"Email cert {settings.EMAIL_CERTIFICATE_FILE} is not a file",
+                ),
+            )
+        return msgs
+
+    return (
+        _ocrmypdf_settings_check()
+        + _timezone_validate()
+        + _barcode_scanner_validate()
+        + _email_certificate_validate()
     )

src/paperless/settings.py

@@ -67,11 +67,20 @@ def __get_float(key: str, default: float) -> float:
     return float(os.getenv(key, default))
 
 
-def __get_path(key: str, default: Union[PathLike, str]) -> Path:
+def __get_path(
+    key: str,
+    default: Optional[Union[PathLike, str]] = None,
+) -> Optional[Path]:
     """
-    Return a normalized, absolute path based on the environment variable or a default
+    Return a normalized, absolute path based on the environment variable or a default,
+    if provided.  If not set and no default, returns None
     """
-    return Path(os.environ.get(key, default)).resolve()
+    if key in os.environ:
+        return Path(os.environ[key]).resolve()
+    elif default is not None:
+        return Path(default).resolve()
+    else:
+        return None
 
 
 def __get_list(
@@ -477,6 +486,8 @@ CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken"
 SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid"
 LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language"
 
+EMAIL_CERTIFICATE_FILE = __get_path("PAPERLESS_EMAIL_CERTIFICATE_FILE")
+
 
 ###############################################################################
 # Database                                                                    #

src/paperless/tests/test_checks.py

@@ -1,9 +1,11 @@
 import os
+from pathlib import Path
 
 from django.test import TestCase
 from django.test import override_settings
 
 from documents.tests.utils import DirectoriesMixin
+from documents.tests.utils import FileSystemAssertsMixin
 from paperless.checks import binaries_check
 from paperless.checks import debug_mode_check
 from paperless.checks import paths_check
@@ -57,7 +59,7 @@ class TestChecks(DirectoriesMixin, TestCase):
         self.assertEqual(len(debug_mode_check(None)), 1)
 
 
-class TestSettingsChecks(DirectoriesMixin, TestCase):
+class TestSettingsChecksAgainstDefaults(DirectoriesMixin, TestCase):
     def test_all_valid(self):
         """
         GIVEN:
@@ -70,6 +72,8 @@ class TestSettingsChecks(DirectoriesMixin, TestCase):
         msgs = settings_values_check(None)
         self.assertEqual(len(msgs), 0)
 
+
+class TestOcrSettingsChecks(DirectoriesMixin, TestCase):
     @override_settings(OCR_OUTPUT_TYPE="notapdf")
     def test_invalid_output_type(self):
         """
@@ -160,6 +164,8 @@ class TestSettingsChecks(DirectoriesMixin, TestCase):
 
         self.assertIn('OCR clean mode "cleanme"', msg.msg)
 
+
+class TestTimezoneSettingsChecks(DirectoriesMixin, TestCase):
     @override_settings(TIME_ZONE="TheMoon\\MyCrater")
     def test_invalid_timezone(self):
         """
@@ -178,6 +184,8 @@ class TestSettingsChecks(DirectoriesMixin, TestCase):
 
         self.assertIn('Timezone "TheMoon\\MyCrater"', msg.msg)
 
+
+class TestBarcodeSettingsChecks(DirectoriesMixin, TestCase):
     @override_settings(CONSUMER_BARCODE_SCANNER="Invalid")
     def test_barcode_scanner_invalid(self):
         msgs = settings_values_check(None)
@@ -200,3 +208,26 @@ class TestSettingsChecks(DirectoriesMixin, TestCase):
     def test_barcode_scanner_valid(self):
         msgs = settings_values_check(None)
         self.assertEqual(len(msgs), 0)
+
+
+class TestEmailCertSettingsChecks(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
+    @override_settings(EMAIL_CERTIFICATE_FILE=Path("/tmp/not_actually_here.pem"))
+    def test_not_valid_file(self):
+        """
+        GIVEN:
+            - Default settings
+            - Email certificate is set
+        WHEN:
+            - Email certificate file doesn't exist
+        THEN:
+            - system check error reported for email certificate
+        """
+        self.assertIsNotFile("/tmp/not_actually_here.pem")
+
+        msgs = settings_values_check(None)
+
+        self.assertEqual(len(msgs), 1)
+
+        msg = msgs[0]
+
+        self.assertIn("Email cert /tmp/not_actually_here.pem is not a file", msg.msg)

src/paperless_mail/mail.py

@@ -395,12 +395,16 @@ def get_mailbox(server, port, security) -> MailBox:
     """
     Returns the correct MailBox instance for the given configuration.
     """
+    ssl_context = ssl.create_default_context()
+    if settings.EMAIL_CERTIFICATE_FILE is not None:  # pragma: nocover
+        ssl_context.load_verify_locations(cafile=settings.EMAIL_CERTIFICATE_FILE)
+
     if security == MailAccount.ImapSecurity.NONE:
         mailbox = MailBoxUnencrypted(server, port)
     elif security == MailAccount.ImapSecurity.STARTTLS:
-        mailbox = MailBoxTls(server, port, ssl_context=ssl.create_default_context())
+        mailbox = MailBoxTls(server, port, ssl_context=ssl_context)
     elif security == MailAccount.ImapSecurity.SSL:
-        mailbox = MailBox(server, port, ssl_context=ssl.create_default_context())
+        mailbox = MailBox(server, port, ssl_context=ssl_context)
     else:
         raise NotImplementedError("Unknown IMAP security")  # pragma: nocover
     return mailbox