From a6407d64e9c2e60925fac4fe5b514597a6c49e39 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 2 Jun 2024 22:36:53 -0700 Subject: [PATCH 01/30] Reset dev version string --- src-ui/src/environments/environment.prod.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index e8899a71a..d709e69ce 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -5,7 +5,7 @@ export const environment = { apiBaseUrl: document.baseURI + 'api/', apiVersion: '5', appTitle: 'Paperless-ngx', - version: '2.9.0', + version: '2.9.0-dev', webSocketHost: window.location.host, webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketBaseUrl: base_url.pathname + 'ws/', From de7c22e8d66fd261aaeb3b107fde887ac02a5baf Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:11:31 -0700 Subject: [PATCH 02/30] Fixes the logging of an email message to be something useful (#6901) --- src/paperless_mail/mail.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 7d41590ad..50435de5d 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -9,6 +9,7 @@ from datetime import date from datetime import timedelta from fnmatch import fnmatch from pathlib import Path +from typing import TYPE_CHECKING from typing import Optional from typing import Union @@ -584,12 +585,17 @@ class MailAccountHandler(LoggingMixin): total_processed_files = 0 for message in messages: + if TYPE_CHECKING: + assert isinstance(message, MailMessage) + if ProcessedMail.objects.filter( rule=rule, uid=message.uid, folder=rule.folder, ).exists(): - self.log.debug(f"Skipping mail {message}, already processed.") + self.log.debug( + f"Skipping mail '{message.uid}' subject '{message.subject}' from '{message.from_}', already processed.", + ) continue try: @@ -659,7 +665,7 @@ class MailAccountHandler(LoggingMixin): ): processed_attachments = 0 - consume_tasks = list() + consume_tasks = [] for att in message.attachments: if ( From 6d2ae3df1f398a44937b49c3fc3eb2c4d8bda339 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:33:46 -0700 Subject: [PATCH 03/30] Resolves test issues with Python 3.12 (#6902) --- src/documents/consumer.py | 105 ++++++++++++--------- src/documents/tests/test_api_app_config.py | 7 +- src/paperless/tests/test_settings.py | 7 +- 3 files changed, 67 insertions(+), 52 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 0d5514e2c..9447fb329 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -485,56 +485,65 @@ class ConsumerPlugin( Return the document object if it was successfully created. """ - self._send_progress( - 0, - 100, - ProgressStatusOptions.STARTED, - ConsumerStatusShortMessage.NEW_FILE, - ) + tempdir = None - # Make sure that preconditions for consuming the file are met. - - self.pre_check_file_exists() - self.pre_check_directories() - self.pre_check_duplicate() - self.pre_check_asn_value() - - self.log.info(f"Consuming {self.filename}") - - # For the actual work, copy the file into a tempdir - tempdir = tempfile.TemporaryDirectory( - prefix="paperless-ngx", - dir=settings.SCRATCH_DIR, - ) - self.working_copy = Path(tempdir.name) / Path(self.filename) - copy_file_with_basic_stats(self.input_doc.original_file, self.working_copy) - - # Determine the parser class. - - mime_type = magic.from_file(self.working_copy, mime=True) - - self.log.debug(f"Detected mime type: {mime_type}") - - # Based on the mime type, get the parser for that type - parser_class: Optional[type[DocumentParser]] = get_parser_class_for_mime_type( - mime_type, - ) - if not parser_class: - tempdir.cleanup() - self._fail( - ConsumerStatusShortMessage.UNSUPPORTED_TYPE, - f"Unsupported mime type {mime_type}", + try: + self._send_progress( + 0, + 100, + ProgressStatusOptions.STARTED, + ConsumerStatusShortMessage.NEW_FILE, ) - # Notify all listeners that we're going to do some work. + # Make sure that preconditions for consuming the file are met. - document_consumption_started.send( - sender=self.__class__, - filename=self.working_copy, - logging_group=self.logging_group, - ) + self.pre_check_file_exists() + self.pre_check_directories() + self.pre_check_duplicate() + self.pre_check_asn_value() - self.run_pre_consume_script() + self.log.info(f"Consuming {self.filename}") + + # For the actual work, copy the file into a tempdir + tempdir = tempfile.TemporaryDirectory( + prefix="paperless-ngx", + dir=settings.SCRATCH_DIR, + ) + self.working_copy = Path(tempdir.name) / Path(self.filename) + copy_file_with_basic_stats(self.input_doc.original_file, self.working_copy) + + # Determine the parser class. + + mime_type = magic.from_file(self.working_copy, mime=True) + + self.log.debug(f"Detected mime type: {mime_type}") + + # Based on the mime type, get the parser for that type + parser_class: Optional[type[DocumentParser]] = ( + get_parser_class_for_mime_type( + mime_type, + ) + ) + if not parser_class: + tempdir.cleanup() + self._fail( + ConsumerStatusShortMessage.UNSUPPORTED_TYPE, + f"Unsupported mime type {mime_type}", + ) + + # Notify all listeners that we're going to do some work. + + document_consumption_started.send( + sender=self.__class__, + filename=self.working_copy, + logging_group=self.logging_group, + ) + + self.run_pre_consume_script() + except: + if tempdir: + tempdir.cleanup() + raise def progress_callback(current_progress, max_progress): # pragma: no cover # recalculate progress to be within 20 and 80 @@ -593,6 +602,9 @@ class ConsumerPlugin( archive_path = document_parser.get_archive_path() except ParseError as e: + document_parser.cleanup() + if tempdir: + tempdir.cleanup() self._fail( str(e), f"Error occurred while consuming document {self.filename}: {e}", @@ -601,7 +613,8 @@ class ConsumerPlugin( ) except Exception as e: document_parser.cleanup() - tempdir.cleanup() + if tempdir: + tempdir.cleanup() self._fail( str(e), f"Unexpected error while consuming document {self.filename}: {e}", diff --git a/src/documents/tests/test_api_app_config.py b/src/documents/tests/test_api_app_config.py index ba14e664a..0d7771c07 100644 --- a/src/documents/tests/test_api_app_config.py +++ b/src/documents/tests/test_api_app_config.py @@ -70,12 +70,13 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): config.app_logo = "/logo/example.jpg" config.save() response = self.client.get("/api/ui_settings/", format="json") - self.assertDictContainsSubset( + self.assertDictEqual( + response.data["settings"], { "app_title": config.app_title, "app_logo": config.app_logo, - }, - response.data["settings"], + } + | response.data["settings"], ) def test_api_update_config(self): diff --git a/src/paperless/tests/test_settings.py b/src/paperless/tests/test_settings.py index e27630ffa..0051d40e7 100644 --- a/src/paperless/tests/test_settings.py +++ b/src/paperless/tests/test_settings.py @@ -339,11 +339,12 @@ class TestDBSettings(TestCase): ): databases = _parse_db_settings() - self.assertDictContainsSubset( - { + self.assertDictEqual( + databases["default"]["OPTIONS"], + databases["default"]["OPTIONS"] + | { "connect_timeout": 10.0, }, - databases["default"]["OPTIONS"], ) self.assertDictEqual( { From 3d6aa8a656f33cc3d17e2e526825114f6c1aad05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:37:46 -0700 Subject: [PATCH 04/30] Chore(deps): Bump tornado from 6.4 to 6.4.1 (#6930) Bumps [tornado](https://github.com/tornadoweb/tornado) from 6.4 to 6.4.1. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v6.4.0...v6.4.1) --- updated-dependencies: - dependency-name: tornado dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Pipfile.lock | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 3b71b0aaf..bc19eb4a6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1965,20 +1965,21 @@ }, "tornado": { "hashes": [ - "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0", - "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63", - "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263", - "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052", - "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f", - "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee", - "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78", - "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579", - "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212", - "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e", - "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2" + "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8", + "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f", + "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4", + "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3", + "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14", + "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842", + "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9", + "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698", + "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7", + "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d", + "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4" ], + "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==6.4" + "version": "==6.4.1" }, "tqdm": { "hashes": [ From d8c96b6e4a573d7aada11409d5f241d8bd1ff84d Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 7 Jun 2024 18:23:45 -0700 Subject: [PATCH 05/30] Enhancement: dont require document model permissions for notes (#6913) --- src/documents/permissions.py | 20 ++++++++++++++++++++ src/documents/views.py | 7 ++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/documents/permissions.py b/src/documents/permissions.py index 76f1835f2..a254f8377 100644 --- a/src/documents/permissions.py +++ b/src/documents/permissions.py @@ -138,3 +138,23 @@ def get_objects_for_user_owner_aware(user, perms, Model) -> QuerySet: def has_perms_owner_aware(user, perms, obj): checker = ObjectPermissionChecker(user) return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj) + + +class PaperlessNotePermissions(BasePermission): + """ + Permissions class that checks for model permissions for Notes. + """ + + perms_map = { + "GET": ["documents.view_note"], + "POST": ["documents.add_note"], + "DELETE": ["documents.delete_note"], + } + + def has_permission(self, request, view): + if not request.user or (not request.user.is_authenticated): # pragma: no cover + return False + + perms = self.perms_map[request.method] + + return request.user.has_perms(perms) diff --git a/src/documents/views.py b/src/documents/views.py index 91b99b610..02023b59f 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -123,6 +123,7 @@ from documents.models import WorkflowTrigger from documents.parsers import get_parser_class_for_mime_type from documents.parsers import parse_date_generator from documents.permissions import PaperlessAdminPermissions +from documents.permissions import PaperlessNotePermissions from documents.permissions import PaperlessObjectPermissions from documents.permissions import get_objects_for_user_owner_aware from documents.permissions import has_perms_owner_aware @@ -622,7 +623,11 @@ class DocumentViewSet( .order_by("-created") ] - @action(methods=["get", "post", "delete"], detail=True) + @action( + methods=["get", "post", "delete"], + detail=True, + permission_classes=[PaperlessNotePermissions], + ) def notes(self, request, pk=None): currentUser = request.user try: From 81e4092f53e7b47a6610e3cdca327084ff71b3fa Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 8 Jun 2024 11:28:47 -0700 Subject: [PATCH 06/30] Enhancement: unique mail rule names by owner --- .../tests/test_migration_workflows.py | 4 +-- .../0024_alter_mailrule_name_and_more.py | 33 +++++++++++++++++++ src/paperless_mail/models.py | 13 +++++++- 3 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 src/paperless_mail/migrations/0024_alter_mailrule_name_and_more.py diff --git a/src/documents/tests/test_migration_workflows.py b/src/documents/tests/test_migration_workflows.py index 742757783..507cb0c18 100644 --- a/src/documents/tests/test_migration_workflows.py +++ b/src/documents/tests/test_migration_workflows.py @@ -5,9 +5,7 @@ from documents.tests.utils import TestMigrations class TestMigrateWorkflow(TestMigrations): migrate_from = "1043_alter_savedviewfilterrule_rule_type" migrate_to = "1044_workflow_workflowaction_workflowtrigger_and_more" - dependencies = ( - ("paperless_mail", "0023_remove_mailrule_filter_attachment_filename_and_more"), - ) + dependencies = (("paperless_mail", "0024_alter_mailrule_name_and_more"),) def setUpBeforeMigration(self, apps): User = apps.get_model("auth", "User") diff --git a/src/paperless_mail/migrations/0024_alter_mailrule_name_and_more.py b/src/paperless_mail/migrations/0024_alter_mailrule_name_and_more.py new file mode 100644 index 000000000..c2840d0e4 --- /dev/null +++ b/src/paperless_mail/migrations/0024_alter_mailrule_name_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.11 on 2024-06-05 16:51 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("paperless_mail", "0023_remove_mailrule_filter_attachment_filename_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="mailrule", + name="name", + field=models.CharField(max_length=256, verbose_name="name"), + ), + migrations.AddConstraint( + model_name="mailrule", + constraint=models.UniqueConstraint( + fields=("name", "owner"), + name="paperless_mail_mailrule_unique_name_owner", + ), + ), + migrations.AddConstraint( + model_name="mailrule", + constraint=models.UniqueConstraint( + condition=models.Q(("owner__isnull", True)), + fields=("name",), + name="paperless_mail_mailrule_name_unique", + ), + ), + ] diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index 2343259a8..c53b16f1f 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -59,6 +59,17 @@ class MailRule(document_models.ModelWithOwner): class Meta: verbose_name = _("mail rule") verbose_name_plural = _("mail rules") + constraints = [ + models.UniqueConstraint( + fields=["name", "owner"], + name="%(app_label)s_%(class)s_unique_name_owner", + ), + models.UniqueConstraint( + name="%(app_label)s_%(class)s_name_unique", + fields=["name"], + condition=models.Q(owner__isnull=True), + ), + ] class ConsumptionScope(models.IntegerChoices): ATTACHMENTS_ONLY = 1, _("Only process attachments.") @@ -93,7 +104,7 @@ class MailRule(document_models.ModelWithOwner): FROM_NAME = 3, _("Use name (or mail address if not available)") FROM_CUSTOM = 4, _("Use correspondent selected below") - name = models.CharField(_("name"), max_length=256, unique=True) + name = models.CharField(_("name"), max_length=256) order = models.IntegerField(_("order"), default=0) From d1ac15baa94f029ca262d81f0dfadf297f562a9f Mon Sep 17 00:00:00 2001 From: Dominik Bruhn Date: Sat, 8 Jun 2024 18:56:25 +0200 Subject: [PATCH 07/30] Enhancement: support delete originals after split / merge (#6935) --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- docs/api.md | 5 + src-ui/messages.xlf | 44 ++++++--- .../merge-confirm-dialog.component.html | 4 + .../merge-confirm-dialog.component.ts | 11 ++- .../split-confirm-dialog.component.html | 4 + .../split-confirm-dialog.component.spec.ts | 9 ++ .../split-confirm-dialog.component.ts | 24 ++++- .../document-detail.component.spec.ts | 2 +- .../document-detail.component.ts | 1 + .../bulk-editor/bulk-editor.component.spec.ts | 19 ++++ .../bulk-editor/bulk-editor.component.ts | 3 + src/documents/bulk_edit.py | 46 +++++++-- src/documents/serialisers.py | 15 +++ src/documents/tests/test_api_bulk_edit.py | 94 +++++++++++++++++++ src/documents/tests/test_bulk_edit.py | 86 ++++++++++++++++- src/documents/views.py | 19 +++- 16 files changed, 354 insertions(+), 32 deletions(-) diff --git a/docs/api.md b/docs/api.md index c38018f71..94ece85ab 100644 --- a/docs/api.md +++ b/docs/api.md @@ -417,9 +417,14 @@ The following methods are supported: - The ordering of the merged document is determined by the list of IDs. - Optional `parameters`: - `"metadata_document_id": DOC_ID` apply metadata (tags, correspondent, etc.) from this document to the merged document. + - `"delete_originals": true` to delete the original documents. This requires the calling user being the owner of + all documents that are merged. - `split` - Requires `parameters`: - `"pages": [..]` The list should be a list of pages and/or a ranges, separated by commas e.g. `"[1,2-3,4,5-7]"` + - Optional `parameters`: + - `"delete_originals": true` to delete the original document after consumption. This requires the calling user being the owner of + the document. - The split operation only accepts a single document. - `rotate` - Requires `parameters`: diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 2cdb7a78a..56cfa9ae4 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -2302,11 +2302,11 @@ src/app/components/document-detail/document-detail.component.ts - 1151 + 1152 src/app/components/document-detail/document-detail.component.ts - 1192 + 1193 src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -2970,11 +2970,18 @@ 24 + + Delete original documents after successful merge + + src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html + 32 + + Note that only PDFs will be included. src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html - 30 + 34 @@ -2991,6 +2998,13 @@ 28 + + Delete original document after successful split + + src/app/components/common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component.html + 49 + + View @@ -5475,7 +5489,7 @@ src/app/components/document-detail/document-detail.component.ts - 1169 + 1170 src/app/guards/dirty-saved-view.guard.ts @@ -5957,21 +5971,21 @@ Split operation will begin in the background. src/app/components/document-detail/document-detail.component.ts - 1128 + 1129 Error executing split operation src/app/components/document-detail/document-detail.component.ts - 1137 + 1138 Rotate confirm src/app/components/document-detail/document-detail.component.ts - 1149 + 1150 src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -5982,49 +5996,49 @@ This operation will permanently rotate the original version of the current document. src/app/components/document-detail/document-detail.component.ts - 1150 + 1151 Rotation will begin in the background. Close and re-open the document after the operation has completed to see the changes. src/app/components/document-detail/document-detail.component.ts - 1166 + 1167 Error executing rotate operation src/app/components/document-detail/document-detail.component.ts - 1178 + 1179 Delete pages confirm src/app/components/document-detail/document-detail.component.ts - 1190 + 1191 This operation will permanently delete the selected pages from the original document. src/app/components/document-detail/document-detail.component.ts - 1191 + 1192 Delete pages operation will begin in the background. Close and re-open or reload this document after the operation has completed to see the changes. src/app/components/document-detail/document-detail.component.ts - 1206 + 1207 Error executing delete pages operation src/app/components/document-detail/document-detail.component.ts - 1215 + 1216 @@ -6417,7 +6431,7 @@ Merged document will be queued for consumption. src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 819 + 822 diff --git a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html index 0da291c94..576861ff2 100644 --- a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html @@ -27,6 +27,10 @@ } +
+ + +

Note that only PDFs will be included.

+
+ + +
} @else { - + } } @case (ContentRenderType.Text) { From fa7a5451db669768f5bb9721d35747864186ec27 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:37:42 -0700 Subject: [PATCH 19/30] Fix: use local pdf worker js (#6990) --- src-ui/src/app/app.component.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts index 7e8abdf34..c6de83cd1 100644 --- a/src-ui/src/app/app.component.ts +++ b/src-ui/src/app/app.component.ts @@ -35,6 +35,8 @@ export class AppComponent implements OnInit, OnDestroy { private permissionsService: PermissionsService, private hotKeyService: HotKeyService ) { + let anyWindow = window as any + anyWindow.pdfWorkerSrc = 'assets/js/pdf.worker.min.js' this.settings.updateAppearanceSettings() } From 61485b0f1d86280ab3c72c5949a843295edab66b Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:23:47 -0700 Subject: [PATCH 20/30] Fix: Document history could include extra fields (#6989) * Fixes creation of a custom field being included in a document's history even if not attached * Show custom field creation in UI --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- .../document-history.component.html | 50 ++++++++++--------- src/documents/tests/test_api_documents.py | 21 ++++++-- src/documents/views.py | 9 ++-- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src-ui/src/app/components/document-history/document-history.component.html b/src-ui/src/app/components/document-history/document-history.component.html index ea4a3c9bb..c63a1223c 100644 --- a/src-ui/src/app/components/document-history/document-history.component.html +++ b/src-ui/src/app/components/document-history/document-history.component.html @@ -27,31 +27,33 @@ } {{ entry.action | titlecase }} - @if (entry.action === AuditLogAction.Update) { -
    - @for (change of entry.changes | keyvalue; track change.key) { - @if (change.value["type"] === 'm2m') { -
  • - {{ change.value["operation"] | titlecase }}  - {{ change.key | titlecase }}:  - {{ change.value["objects"].join(', ') }} -
  • - } - @else if (change.value["type"] === 'custom_field') { -
  • - {{ change.value["field"] }}:  - {{ change.value["value"] }} -
  • - } - @else { -
  • - {{ change.key | titlecase }}:  - {{ change.value[1] }} -
  • - } +
      + @for (change of entry.changes | keyvalue; track change.key) { + @if (change.value["type"] === 'm2m') { +
    • + {{ change.value["operation"] | titlecase }}  + {{ change.key | titlecase }}:  + {{ change.value["objects"].join(', ') }} +
    • } -
    - } + @else if (change.value["type"] === 'custom_field') { +
  • + {{ change.value["field"] }}:  + {{ change.value["value"] }} +
  • + } + @else { +
  • + {{ change.key | titlecase }}:  + @if (change.key === 'content') { + {{ change.value[1]?.substring(0,100) }}... + } @else { + {{ change.value[1] }} + } +
  • + } + } +
} } diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index 7e69cb024..4ad6cb828 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -366,6 +366,16 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): data_type=CustomField.FieldDataType.STRING, ) self.client.force_login(user=self.user) + + # Initial response should include only document's creation + response = self.client.get(f"/api/documents/{doc.pk}/history/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data), 1) + + self.assertIsNone(response.data[0]["actor"]) + self.assertEqual(response.data[0]["action"], "create") + self.client.patch( f"/api/documents/{doc.pk}/", data={ @@ -379,12 +389,15 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): format="json", ) + # Second response should include custom field addition response = self.client.get(f"/api/documents/{doc.pk}/history/") + self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data[1]["actor"]["id"], self.user.id) - self.assertEqual(response.data[1]["action"], "create") + self.assertEqual(len(response.data), 2) + self.assertEqual(response.data[0]["actor"]["id"], self.user.id) + self.assertEqual(response.data[0]["action"], "create") self.assertEqual( - response.data[1]["changes"], + response.data[0]["changes"], { "custom_fields": { "type": "custom_field", @@ -393,6 +406,8 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): }, }, ) + self.assertIsNone(response.data[1]["actor"]) + self.assertEqual(response.data[1]["action"], "create") @override_settings(AUDIT_LOG_ENABLED=False) def test_document_history_action_disabled(self): diff --git a/src/documents/views.py b/src/documents/views.py index 68addd0f4..9e8e7301d 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -19,7 +19,6 @@ from django.apps import apps from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import User -from django.contrib.contenttypes.models import ContentType from django.db import connections from django.db.migrations.loader import MigrationLoader from django.db.migrations.recorder import MigrationRecorder @@ -106,7 +105,6 @@ from documents.matching import match_storage_paths from documents.matching import match_tags from documents.models import Correspondent from documents.models import CustomField -from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType from documents.models import Note @@ -799,15 +797,14 @@ class DocumentViewSet( else None ), } - for entry in LogEntry.objects.filter(object_pk=doc.pk).select_related( + for entry in LogEntry.objects.get_for_object(doc).select_related( "actor", ) ] # custom fields - for entry in LogEntry.objects.filter( - object_pk__in=list(doc.custom_fields.values_list("id", flat=True)), - content_type=ContentType.objects.get_for_model(CustomFieldInstance), + for entry in LogEntry.objects.get_for_objects( + doc.custom_fields.all(), ).select_related("actor"): entries.append( { From 22a6360edf610b8fdf3672084f5dcb8ab46f750c Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Thu, 13 Jun 2024 16:46:18 +0200 Subject: [PATCH 21/30] Fix: default order of documents gets lost in QuerySet pipeline (#6982) * Send ordered document list to Django REST pagination Currently, when pages of documents are requested from the API, the webserver logs a warning: ``` gunicorn[1550]: /home/madduck/code/paperless-ngx/.direnv/python-3.11.2/lib/python3.11/site-packages/rest_framework/pagination.py:200: UnorderedObjectListWarning: Pagination may yield inconsistent results with an unordered object_list: QuerySet. ``` This can yield unexpected and problematic results, including duplicate and missing IDs in the enumeration, as demonstrated in https://github.com/paperless-ngx/paperless-ngx/discussions/6859 The patch is simple: turn the unordered Documents QuerySet into one that's ordered by reverse creation date, which is the default ordering for `Document`. Note that the default ordering for `Document` means that `QuerySet.ordered` is actually `True` following the call to `distinct()`, but after `annotate()`, the flag changes to `False`, unless `order_by()` is used explicitly, as per this patch. Closes: https://github.com/paperless-ngx/paperless-ngx/discussions/6859 Signed-off-by: martin f. krafft * Ensure order of documents in permissions test The patch for #6982 changes the ordering of documents returned by the API, which was previously implicit, and is now explicit. Therefore, this patch masssages the API result to ensure the previous order. Signed-off-by: martin f. krafft --------- Signed-off-by: martin f. krafft --- src/documents/tests/test_api_permissions.py | 28 +++++++++++++-------- src/documents/views.py | 1 + 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/documents/tests/test_api_permissions.py b/src/documents/tests/test_api_permissions.py index d7131b834..7708b8541 100644 --- a/src/documents/tests/test_api_permissions.py +++ b/src/documents/tests/test_api_permissions.py @@ -432,13 +432,18 @@ class TestApiAuth(DirectoriesMixin, APITestCase): resp_data = response.json() - self.assertNotIn("permissions", resp_data["results"][0]) - self.assertIn("user_can_change", resp_data["results"][0]) - self.assertTrue(resp_data["results"][0]["user_can_change"]) # doc1 - self.assertFalse(resp_data["results"][0]["is_shared_by_requester"]) # doc1 - self.assertFalse(resp_data["results"][1]["user_can_change"]) # doc2 - self.assertTrue(resp_data["results"][2]["user_can_change"]) # doc3 - self.assertTrue(resp_data["results"][3]["is_shared_by_requester"]) # doc4 + # The response will contain the documents in reversed order of creation + # due to #6982, but previously this code relied on implicit ordering + # so let's ensure the order is as expected: + results = resp_data["results"][::-1] + + self.assertNotIn("permissions", results[0]) + self.assertIn("user_can_change", results[0]) + self.assertTrue(results[0]["user_can_change"]) # doc1 + self.assertFalse(results[0]["is_shared_by_requester"]) # doc1 + self.assertFalse(results[1]["user_can_change"]) # doc2 + self.assertTrue(results[2]["user_can_change"]) # doc3 + self.assertTrue(results[3]["is_shared_by_requester"]) # doc4 response = self.client.get( "/api/documents/?full_perms=true", @@ -449,9 +454,12 @@ class TestApiAuth(DirectoriesMixin, APITestCase): resp_data = response.json() - self.assertIn("permissions", resp_data["results"][0]) - self.assertNotIn("user_can_change", resp_data["results"][0]) - self.assertNotIn("is_shared_by_requester", resp_data["results"][0]) + # See above about response ordering + results = resp_data["results"][::-1] + + self.assertIn("permissions", results[0]) + self.assertNotIn("user_can_change", results[0]) + self.assertNotIn("is_shared_by_requester", results[0]) class TestApiUser(DirectoriesMixin, APITestCase): diff --git a/src/documents/views.py b/src/documents/views.py index 9e8e7301d..72414d4f0 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -362,6 +362,7 @@ class DocumentViewSet( def get_queryset(self): return ( Document.objects.distinct() + .order_by("-created") .annotate(num_notes=Count("notes")) .select_related("correspondent", "storage_path", "document_type", "owner") .prefetch_related("tags", "custom_fields", "notes") From 28db7e84e695a23513655ba3a3bfa0a1f91b9b3d Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Thu, 13 Jun 2024 11:53:34 -0700 Subject: [PATCH 22/30] Documentation: Corrections and clarifications for Python support (#6995) * Clarifies Python version support and a rough policy of what versions are supported --- CONTRIBUTING.md | 2 +- docs/development.md | 4 ---- docs/setup.md | 16 +++++++++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4136b547e..a7375000b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ If you want to implement something big: ## Python -Paperless supports python 3.9 - 3.11. We format Python code with [ruff](https://docs.astral.sh/ruff/formatter/). +Paperless supports python 3.9 - 3.11 at this time. We format Python code with [ruff](https://docs.astral.sh/ruff/formatter/). ## Branches diff --git a/docs/development.md b/docs/development.md index 969c293b1..bc9ef4c2b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -81,10 +81,6 @@ first-time setup. !!! note Using a virtual environment is highly recommended. You can spawn one via `pipenv shell`. - Make sure you're using Python 3.10.x or lower. Otherwise you might - get issues with building dependencies. You can use - [pyenv](https://github.com/pyenv/pyenv) to install a specific - Python version. 5. Install pre-commit hooks: diff --git a/docs/setup.md b/docs/setup.md index 19811e7c6..b0a0a5fed 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -250,9 +250,14 @@ a minimal installation of Debian/Buster, which is the current stable release at the time of writing. Windows is not and will never be supported. +Paperless requires Python 3. At this time, 3.9 - 3.11 are tested versions. +Newer versions may work, but some dependencies may not fully support newer versions. +Support for older Python versions may be dropped as they reach end of life or as newer versions +are released, dependency support is confirmed, etc. + 1. Install dependencies. Paperless requires the following packages. - - `python3` - 3.9 - 3.11 are supported + - `python3` - `python3-pip` - `python3-dev` - `default-libmysqlclient-dev` for MariaDB @@ -410,8 +415,7 @@ supported. sudo chown paperless:paperless /opt/paperless/consume ``` -8. Install python requirements from the `requirements.txt` file. It is - up to you if you wish to use a virtual environment or not. First you should update your pip, so it gets the actual packages. +8. Install python requirements from the `requirements.txt` file. ```shell-session sudo -Hu paperless pip3 install -r requirements.txt @@ -420,6 +424,12 @@ supported. This will install all python dependencies in the home directory of the new paperless user. + !!! tip + + It is up to you if you wish to use a virtual environment or not for the Python + dependencies. This is an alternative to the above and may require adjusting + the example scripts to utilize the virtual environment paths + 9. Go to `/opt/paperless/src`, and execute the following commands: ```bash From 9d4e2d4652450dc2a06f3b96a46516c4ceeec14d Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 15 Jun 2024 09:00:18 -0700 Subject: [PATCH 23/30] Enhancement: better boolean custom field display (#7001) --- .../custom-field-display.component.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src-ui/src/app/components/common/custom-field-display/custom-field-display.component.html b/src-ui/src/app/components/common/custom-field-display/custom-field-display.component.html index 07347f8e0..812c283e2 100644 --- a/src-ui/src/app/components/common/custom-field-display/custom-field-display.component.html +++ b/src-ui/src/app/components/common/custom-field-display/custom-field-display.component.html @@ -24,6 +24,12 @@ } } + @case (CustomFieldDataType.Boolean) { +
+ {{field.name}}: + +
+ } @default { {{value}} } From a796e58a94abf1f619b8e04fc33e1464594c50b5 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 17 Jun 2024 08:07:08 -0700 Subject: [PATCH 24/30] Feature: documents trash aka soft delete (#6944) --- Pipfile | 1 + Pipfile.lock | 9 + docs/configuration.md | 24 +- docs/usage.md | 9 + paperless.conf.example | 2 +- src-ui/messages.xlf | 405 ++++++++++++------ src-ui/src/app/app-routing.module.ts | 9 + src-ui/src/app/app.module.ts | 2 + .../admin/settings/settings.component.html | 2 +- .../admin/trash/trash.component.html | 98 +++++ .../admin/trash/trash.component.scss | 0 .../admin/trash/trash.component.spec.ts | 163 +++++++ .../components/admin/trash/trash.component.ts | 137 ++++++ .../app-frame/app-frame.component.html | 7 + .../confirm-dialog.component.spec.ts | 10 - .../confirm-dialog.component.ts | 20 - .../document-detail.component.ts | 8 +- .../bulk-editor/bulk-editor.component.spec.ts | 9 +- .../bulk-editor/bulk-editor.component.ts | 31 +- src-ui/src/app/data/document.ts | 2 + src-ui/src/app/data/ui-settings.ts | 6 + src-ui/src/app/services/trash.service.spec.ts | 59 +++ src-ui/src/app/services/trash.service.ts | 37 ++ ...ocument_deleted_at_document_restored_at.py | 23 + src/documents/models.py | 4 +- src/documents/serialisers.py | 24 ++ src/documents/signals/handlers.py | 8 +- src/documents/tasks.py | 30 ++ src/documents/tests/test_api_trash.py | 155 +++++++ src/documents/tests/test_api_uisettings.py | 1 + src/documents/tests/test_document_model.py | 30 ++ src/documents/tests/test_file_handling.py | 25 +- src/documents/tests/test_tasks.py | 34 ++ src/documents/views.py | 42 ++ src/paperless/checks.py | 2 +- src/paperless/settings.py | 23 +- src/paperless/tests/test_settings.py | 17 + src/paperless/urls.py | 6 + 38 files changed, 1283 insertions(+), 191 deletions(-) create mode 100644 src-ui/src/app/components/admin/trash/trash.component.html create mode 100644 src-ui/src/app/components/admin/trash/trash.component.scss create mode 100644 src-ui/src/app/components/admin/trash/trash.component.spec.ts create mode 100644 src-ui/src/app/components/admin/trash/trash.component.ts create mode 100644 src-ui/src/app/services/trash.service.spec.ts create mode 100644 src-ui/src/app/services/trash.service.ts create mode 100644 src/documents/migrations/1049_document_deleted_at_document_restored_at.py create mode 100644 src/documents/tests/test_api_trash.py diff --git a/Pipfile b/Pipfile index da26987cf..948e22273 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ django-extensions = "*" django-filter = "~=24.2" django-guardian = "*" django-multiselectfield = "*" +django-soft-delete = "*" djangorestframework = "==3.14.0" djangorestframework-guardian = "*" drf-writable-nested = "*" diff --git a/Pipfile.lock b/Pipfile.lock index f3753a0cd..e011e1c60 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -540,6 +540,15 @@ "index": "pypi", "version": "==0.1.12" }, + "django-soft-delete": { + "hashes": [ + "sha256:443c00a54c06d236ff8806c3260243d775cc536581d7377c2785080b1041ce1d", + "sha256:7cb4524231763a70ad79cfccd49d001b7e5fa666ec897cc044d897dd73e0146e" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==1.0.13" + }, "djangorestframework": { "hashes": [ "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", diff --git a/docs/configuration.md b/docs/configuration.md index f4c271ce1..0c3345145 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -219,10 +219,10 @@ database, classification model, etc). Defaults to "../data/", relative to the "src" directory. -#### [`PAPERLESS_TRASH_DIR=`](#PAPERLESS_TRASH_DIR) {#PAPERLESS_TRASH_DIR} +#### [`PAPERLESS_EMPTY_TRASH_DIR=`](#PAPERLESS_EMPTY_TRASH_DIR) {#PAPERLESS_EMPTY_TRASH_DIR} -: Instead of removing deleted documents, they are moved to this -directory. +: When documents are deleted (e.g. after emptying the trash) the original files will be moved here +instead of being removed from the filesystem. Only the original version is kept. This must be writeable by the user running paperless. When running inside docker, ensure that this path is within a permanent volume @@ -230,7 +230,9 @@ directory. Note that the directory must exist prior to using this setting. - Defaults to empty (i.e. really delete documents). + Defaults to empty (i.e. really delete files). + + This setting was previously named PAPERLESS_TRASH_DIR. #### [`PAPERLESS_MEDIA_ROOT=`](#PAPERLESS_MEDIA_ROOT) {#PAPERLESS_MEDIA_ROOT} @@ -1362,6 +1364,20 @@ processing. This only has an effect if Defaults to false. +## Trash + +#### [`EMPTY_TRASH_DELAY=`](#EMPTY_TRASH_DELAY) {#EMPTY_TRASH_DELAY} + +: Sets how long in days documents remain in the 'trash' before they are permanently deleted. + + Defaults to 30 days, minimum of 1 day. + +#### [`PAPERLESS_EMPTY_TRASH_TASK_CRON=`](#PAPERLESS_EMPTY_TRASH_TASK_CRON) {#PAPERLESS_EMPTY_TRASH_TASK_CRON} + +: Configures the schedule to empty the trash of expired deleted documents. + + Defaults to `0 1 * * *`, once per day. + ## Binaries There are a few external software packages that Paperless expects to diff --git a/docs/usage.md b/docs/usage.md index 5705be3da..034447d6e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -478,6 +478,15 @@ As of version 2.7, Paperless-ngx automatically records all changes to a document Changes to documents are visible under the "History" tab. Note that certain changes such as those made by workflows, record the 'actor' as "System". +## Document Trash + +When you first delete a document it is moved to the 'trash' until either it is explicitly deleted or it is automatically removed after a set amount of time has passed. +You can set how long documents remain in the trash before being automatically deleted with [`EMPTY_TRASH_DELAY`](configuration.md#EMPTY_TRASH_DELAY), which defaults +to 30 days. Until the file is actually deleted (e.g. the trash is emptied), all files and database content remains intact and can be restored at any point up until that time. + +Additionally you may configure a directory where deleted files are moved to when they the trash is emptied with [`PAPERLESS_EMPTY_TRASH_DIR`](configuration.md#PAPERLESS_EMPTY_TRASH_DIR). +Note that the empty trash directory only stores the original file, the archive file and all database information is permanently removed once a document is fully deleted. + ## Best practices {#basic-searching} Paperless offers a couple tools that help you organize your document diff --git a/paperless.conf.example b/paperless.conf.example index db557a7b6..63ee7be22 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -19,7 +19,7 @@ #PAPERLESS_CONSUMPTION_DIR=../consume #PAPERLESS_DATA_DIR=../data -#PAPERLESS_TRASH_DIR= +#PAPERLESS_EMPTY_TRASH_DIR= #PAPERLESS_MEDIA_ROOT=../media #PAPERLESS_STATICDIR=../static #PAPERLESS_FILENAME_FORMAT= diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 56cfa9ae4..492c160c9 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -240,18 +240,18 @@ Document was added to Paperless-ngx. src/app/app.component.ts - 83 + 85 src/app/app.component.ts - 92 + 94
Open document src/app/app.component.ts - 85 + 87 src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html @@ -274,21 +274,21 @@ Could not add : src/app/app.component.ts - 107 + 109 Document is being processed by Paperless-ngx. src/app/app.component.ts - 122 + 124 Dashboard src/app/app.component.ts - 129 + 131 src/app/components/app-frame/app-frame.component.html @@ -307,7 +307,7 @@ Documents src/app/app.component.ts - 140 + 142 src/app/components/app-frame/app-frame.component.html @@ -342,7 +342,7 @@ Settings src/app/app.component.ts - 152 + 154 src/app/components/admin/settings/settings.component.html @@ -369,14 +369,14 @@ Prev src/app/app.component.ts - 158 + 160 Next src/app/app.component.ts - 159 + 161 src/app/components/document-detail/document-detail.component.html @@ -387,56 +387,56 @@ End src/app/app.component.ts - 160 + 162 The dashboard can be used to show saved views, such as an 'Inbox'. Those settings are found under Settings > Saved Views once you have created some. src/app/app.component.ts - 166 + 168 Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training its machine learning algorithms. src/app/app.component.ts - 173 + 175 The documents list shows all of your documents and allows for filtering as well as bulk-editing. There are three different view styles: list, small cards and large cards. A list of documents currently opened for editing is shown in the sidebar. src/app/app.component.ts - 178 + 180 The filtering tools allow you to quickly find documents using various searches, dates, tags, etc. src/app/app.component.ts - 185 + 187 Any combination of filters can be saved as a 'view' which can then be displayed on the dashboard and / or sidebar. src/app/app.component.ts - 191 + 193 Tags, correspondents, document types and storage paths can all be managed using these pages. They can also be created from the document edit view. src/app/app.component.ts - 196 + 198 Manage e-mail accounts and rules for automatically importing documents. src/app/app.component.ts - 204 + 206 src/app/components/manage/mail/mail.component.html @@ -447,14 +447,14 @@ Workflows give you more control over the document pipeline. src/app/app.component.ts - 212 + 214 File Tasks shows you documents that have been consumed, are waiting to be, or may have failed during the process. src/app/app.component.ts - 220 + 222 src/app/components/admin/tasks/tasks.component.html @@ -465,28 +465,28 @@ Check out the settings for various tweaks to the web app and toggle settings for saved views. src/app/app.component.ts - 228 + 230 Thank you! 🙏 src/app/app.component.ts - 236 + 238 There are <em>tons</em> more features and info we didn't cover here, but this should get you started. Check out the documentation or visit the project on GitHub to learn more or to report issues. src/app/app.component.ts - 238 + 240 Lastly, on behalf of every contributor to this community-supported project, thank you for using Paperless-ngx! src/app/app.component.ts - 240 + 242 @@ -684,6 +684,10 @@ src/app/components/admin/tasks/tasks.component.html 23 + + src/app/components/admin/trash/trash.component.html + 45 + src/app/components/admin/users-groups/users-groups.component.html 92 @@ -976,13 +980,6 @@ 195 - - Deleting documents will always ask for confirmation. - - src/app/components/admin/settings/settings.component.html - 195 - - Apply on close @@ -1363,6 +1360,10 @@ src/app/components/admin/tasks/tasks.component.html 42 + + src/app/components/admin/trash/trash.component.html + 37 + src/app/components/admin/users-groups/users-groups.component.html 23 @@ -1422,6 +1423,22 @@ src/app/components/admin/settings/settings.component.html 369 + + src/app/components/admin/trash/trash.component.html + 67 + + + src/app/components/admin/trash/trash.component.html + 76 + + + src/app/components/admin/trash/trash.component.ts + 57 + + + src/app/components/admin/trash/trash.component.ts + 80 + src/app/components/admin/users-groups/users-groups.component.html 38 @@ -1765,6 +1782,10 @@ src/app/components/admin/tasks/tasks.component.html 9 + + src/app/components/admin/trash/trash.component.html + 8 + src/app/components/manage/management-list/management-list.component.html 3 @@ -1788,6 +1809,10 @@ src/app/components/admin/tasks/tasks.component.html 36 + + src/app/components/admin/trash/trash.component.html + 35 + src/app/components/admin/users-groups/users-groups.component.html 21 @@ -2045,6 +2070,188 @@ 141 + + Trash + + src/app/components/admin/trash/trash.component.html + 2 + + + src/app/components/app-frame/app-frame.component.html + 271 + + + src/app/components/app-frame/app-frame.component.html + 274 + + + + Manage trashed documents that are pending deletion. + + src/app/components/admin/trash/trash.component.html + 4 + + + + Restore selected + + src/app/components/admin/trash/trash.component.html + 11 + + + + Delete selected + + src/app/components/admin/trash/trash.component.html + 14 + + + + Empty trash + + src/app/components/admin/trash/trash.component.html + 17 + + + + Remaining + + src/app/components/admin/trash/trash.component.html + 36 + + + + days + + src/app/components/admin/trash/trash.component.html + 58 + + + + Restore + + src/app/components/admin/trash/trash.component.html + 66 + + + src/app/components/admin/trash/trash.component.html + 73 + + + + {VAR_PLURAL, plural, =1 {One document in trash} other { total documents in trash}} + + src/app/components/admin/trash/trash.component.html + 89 + + + + Confirm delete + + src/app/components/admin/trash/trash.component.ts + 53 + + + src/app/components/admin/trash/trash.component.ts + 74 + + + src/app/components/manage/management-list/management-list.component.ts + 203 + + + src/app/components/manage/management-list/management-list.component.ts + 320 + + + + This operation will permanently delete this document. + + src/app/components/admin/trash/trash.component.ts + 54 + + + + This operation cannot be undone. + + src/app/components/admin/trash/trash.component.ts + 55 + + + src/app/components/admin/trash/trash.component.ts + 78 + + + src/app/components/admin/users-groups/users-groups.component.ts + 116 + + + src/app/components/admin/users-groups/users-groups.component.ts + 166 + + + src/app/components/manage/custom-fields/custom-fields.component.ts + 73 + + + src/app/components/manage/mail/mail.component.ts + 114 + + + src/app/components/manage/mail/mail.component.ts + 173 + + + src/app/components/manage/management-list/management-list.component.ts + 322 + + + src/app/components/manage/workflows/workflows.component.ts + 97 + + + + Document deleted + + src/app/components/admin/trash/trash.component.ts + 63 + + + + This operation will permanently delete the selected documents. + + src/app/components/admin/trash/trash.component.ts + 76 + + + + This operation will permanently delete all documents in the trash. + + src/app/components/admin/trash/trash.component.ts + 77 + + + + Document(s) deleted + + src/app/components/admin/trash/trash.component.ts + 87 + + + + Document restored + + src/app/components/admin/trash/trash.component.ts + 97 + + + + Document(s) restored + + src/app/components/admin/trash/trash.component.ts + 106 + + Users & Groups @@ -2247,41 +2454,6 @@ 115 - - This operation cannot be undone. - - src/app/components/admin/users-groups/users-groups.component.ts - 116 - - - src/app/components/admin/users-groups/users-groups.component.ts - 166 - - - src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 714 - - - src/app/components/manage/custom-fields/custom-fields.component.ts - 73 - - - src/app/components/manage/mail/mail.component.ts - 114 - - - src/app/components/manage/mail/mail.component.ts - 173 - - - src/app/components/manage/management-list/management-list.component.ts - 322 - - - src/app/components/manage/workflows/workflows.component.ts - 97 - - Proceed @@ -2310,15 +2482,15 @@ src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 755 + 758 src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 788 + 791 src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 807 + 810 src/app/components/manage/custom-fields/custom-fields.component.ts @@ -2433,11 +2605,11 @@ src/app/components/app-frame/app-frame.component.html - 272 + 279 src/app/components/app-frame/app-frame.component.html - 275 + 282 @@ -2612,42 +2784,42 @@ GitHub src/app/components/app-frame/app-frame.component.html - 282 + 289 is available. src/app/components/app-frame/app-frame.component.html - 291,292 + 298,299 Click to view. src/app/components/app-frame/app-frame.component.html - 292 + 299 Paperless-ngx can automatically check for updates src/app/components/app-frame/app-frame.component.html - 296 + 303 How does this work? src/app/components/app-frame/app-frame.component.html - 303,305 + 310,312 Update available src/app/components/app-frame/app-frame.component.html - 316 + 323 @@ -2887,6 +3059,10 @@ src/app/components/common/permissions-dialog/permissions-dialog.component.html 26 + + src/app/components/document-detail/document-detail.component.ts + 776 + src/app/components/document-list/bulk-editor/bulk-editor.component.ts 401 @@ -2907,6 +3083,10 @@ src/app/components/document-list/bulk-editor/bulk-editor.component.ts 579 + + src/app/components/document-list/bulk-editor/bulk-editor.component.ts + 712 + Page @@ -5864,41 +6044,34 @@ 749 - - Confirm delete - - src/app/components/document-detail/document-detail.component.ts - 776 - - - src/app/components/manage/management-list/management-list.component.ts - 203 - - - src/app/components/manage/management-list/management-list.component.ts - 320 - - - - Do you really want to delete document ""? + + Do you really want to move the document "" to the trash? src/app/components/document-detail/document-detail.component.ts 777 - - The files for this document will be deleted permanently. This operation cannot be undone. + + Documents can be restored prior to permanent deletion. src/app/components/document-detail/document-detail.component.ts 778 + + src/app/components/document-list/bulk-editor/bulk-editor.component.ts + 714 + - - Delete document + + Move to trash src/app/components/document-detail/document-detail.component.ts 780 + + src/app/components/document-list/bulk-editor/bulk-editor.component.ts + 716 + Error deleting document @@ -5915,7 +6088,7 @@ src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 751 + 754 @@ -5989,7 +6162,7 @@ src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 784 + 787 @@ -6364,74 +6537,60 @@ 571,575 - - Delete confirm - - src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 712 - - - - This operation will permanently delete selected document(s). + + Move selected document(s) to the trash? src/app/components/document-list/bulk-editor/bulk-editor.component.ts 713 - - Delete document(s) - - src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 716 - - This operation will permanently recreate the archive files for selected document(s). src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 752 + 755 The archive files will be re-generated with the current settings. src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 753 + 756 This operation will permanently rotate the original version of document(s). src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 785 + 788 This will alter the original copy. src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 786 + 789 Merge confirm src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 805 + 808 This operation will merge selected documents into a new document. src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 806 + 809 Merged document will be queued for consumption. src/app/components/document-list/bulk-editor/bulk-editor.component.ts - 822 + 825 diff --git a/src-ui/src/app/app-routing.module.ts b/src-ui/src/app/app-routing.module.ts index 12b412f67..bbeba9e8a 100644 --- a/src-ui/src/app/app-routing.module.ts +++ b/src-ui/src/app/app-routing.module.ts @@ -26,6 +26,7 @@ import { MailComponent } from './components/manage/mail/mail.component' import { UsersAndGroupsComponent } from './components/admin/users-groups/users-groups.component' import { CustomFieldsComponent } from './components/manage/custom-fields/custom-fields.component' import { ConfigComponent } from './components/admin/config/config.component' +import { TrashComponent } from './components/admin/trash/trash.component' export const routes: Routes = [ { path: '', redirectTo: 'dashboard', pathMatch: 'full' }, @@ -144,6 +145,14 @@ export const routes: Routes = [ requireAdmin: true, }, }, + { + path: 'trash', + component: TrashComponent, + canActivate: [PermissionsGuard], + data: { + requireAdmin: true, + }, + }, // redirect old paths { path: 'settings/mail', diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index f9e04b069..2232dad19 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -125,6 +125,7 @@ import { CustomFieldDisplayComponent } from './components/common/custom-field-di import { GlobalSearchComponent } from './components/app-frame/global-search/global-search.component' import { HotkeyDialogComponent } from './components/common/hotkey-dialog/hotkey-dialog.component' import { DeletePagesConfirmDialogComponent } from './components/common/confirm-dialog/delete-pages-confirm-dialog/delete-pages-confirm-dialog.component' +import { TrashComponent } from './components/admin/trash/trash.component' import { airplane, archive, @@ -497,6 +498,7 @@ function initializeApp(settings: SettingsService) { GlobalSearchComponent, HotkeyDialogComponent, DeletePagesConfirmDialogComponent, + TrashComponent, ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/admin/settings/settings.component.html b/src-ui/src/app/components/admin/settings/settings.component.html index bcab7de33..285599639 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.html +++ b/src-ui/src/app/components/admin/settings/settings.component.html @@ -192,7 +192,7 @@
- +
diff --git a/src-ui/src/app/components/admin/trash/trash.component.html b/src-ui/src/app/components/admin/trash/trash.component.html new file mode 100644 index 000000000..1c66bdd44 --- /dev/null +++ b/src-ui/src/app/components/admin/trash/trash.component.html @@ -0,0 +1,98 @@ + + + + + + + +
+ +
+ +
+ + + + + + + + + + + @if (isLoading) { + + + + } + @for (document of documentsInTrash; track document.id) { + + + + + + + } + +
+
+ + +
+
NameRemainingActions
+
+ Loading... +
+
+ + +
+
{{ document.title }}{{ getDaysRemaining(document) }} days +
+
+ +
+ + +
+
+
+
+ + +
+
+
+ +@if (!isLoading) { +
+
+ {totalDocuments, plural, =1 {One document in trash} other {{{totalDocuments || 0}} total documents in trash}} + @if (selectedDocuments.size > 0) { +  ({{selectedDocuments.size}} selected) + } +
+ @if (documentsInTrash.length > 20) { + + } +
+} diff --git a/src-ui/src/app/components/admin/trash/trash.component.scss b/src-ui/src/app/components/admin/trash/trash.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/admin/trash/trash.component.spec.ts b/src-ui/src/app/components/admin/trash/trash.component.spec.ts new file mode 100644 index 000000000..063d4bb8f --- /dev/null +++ b/src-ui/src/app/components/admin/trash/trash.component.spec.ts @@ -0,0 +1,163 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' + +import { TrashComponent } from './trash.component' +import { HttpClientTestingModule } from '@angular/common/http/testing' +import { PageHeaderComponent } from '../../common/page-header/page-header.component' +import { + NgbModal, + NgbPaginationModule, + NgbPopoverModule, +} from '@ng-bootstrap/ng-bootstrap' +import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' +import { FormsModule, ReactiveFormsModule } from '@angular/forms' +import { TrashService } from 'src/app/services/trash.service' +import { of } from 'rxjs' +import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component' +import { By } from '@angular/platform-browser' + +const documentsInTrash = [ + { + id: 1, + name: 'test1', + created: new Date('2023-03-01T10:26:03.093116Z'), + deleted_at: new Date('2023-03-01T10:26:03.093116Z'), + }, + { + id: 2, + name: 'test2', + created: new Date('2023-03-01T10:26:03.093116Z'), + deleted_at: new Date('2023-03-01T10:26:03.093116Z'), + }, +] + +describe('TrashComponent', () => { + let component: TrashComponent + let fixture: ComponentFixture + let trashService: TrashService + let modalService: NgbModal + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ + TrashComponent, + PageHeaderComponent, + ConfirmDialogComponent, + ], + imports: [ + HttpClientTestingModule, + FormsModule, + ReactiveFormsModule, + NgbPopoverModule, + NgbPaginationModule, + NgxBootstrapIconsModule.pick(allIcons), + ], + }).compileComponents() + + fixture = TestBed.createComponent(TrashComponent) + trashService = TestBed.inject(TrashService) + modalService = TestBed.inject(NgbModal) + component = fixture.componentInstance + fixture.detectChanges() + }) + + it('should call correct service method on reload', () => { + const trashSpy = jest.spyOn(trashService, 'getTrash') + trashSpy.mockReturnValue( + of({ + count: 2, + all: documentsInTrash.map((d) => d.id), + results: documentsInTrash, + }) + ) + component.reload() + expect(trashSpy).toHaveBeenCalled() + expect(component.documentsInTrash).toEqual(documentsInTrash) + }) + + it('should support delete document', () => { + const trashSpy = jest.spyOn(trashService, 'emptyTrash') + let modal + modalService.activeInstances.subscribe((instances) => { + modal = instances[0] + }) + trashSpy.mockReturnValue(of('OK')) + component.delete(documentsInTrash[0]) + expect(modal).toBeDefined() + modal.componentInstance.confirmClicked.next() + expect(trashSpy).toHaveBeenCalled() + }) + + it('should support empty trash', () => { + const trashSpy = jest.spyOn(trashService, 'emptyTrash') + let modal + modalService.activeInstances.subscribe((instances) => { + modal = instances[instances.length - 1] + }) + trashSpy.mockReturnValue(of('OK')) + component.emptyTrash() + expect(modal).toBeDefined() + modal.componentInstance.confirmClicked.next() + expect(trashSpy).toHaveBeenCalled() + modal.close() + component.emptyTrash(new Set([1, 2])) + modal.componentInstance.confirmClicked.next() + expect(trashSpy).toHaveBeenCalledWith([1, 2]) + }) + + it('should support restore document', () => { + const restoreSpy = jest.spyOn(trashService, 'restoreDocuments') + const reloadSpy = jest.spyOn(component, 'reload') + restoreSpy.mockReturnValue(of('OK')) + component.restore(documentsInTrash[0]) + expect(restoreSpy).toHaveBeenCalledWith([documentsInTrash[0].id]) + expect(reloadSpy).toHaveBeenCalled() + }) + + it('should support restore all documents', () => { + const restoreSpy = jest.spyOn(trashService, 'restoreDocuments') + const reloadSpy = jest.spyOn(component, 'reload') + restoreSpy.mockReturnValue(of('OK')) + component.restoreAll() + expect(restoreSpy).toHaveBeenCalled() + expect(reloadSpy).toHaveBeenCalled() + component.restoreAll(new Set([1, 2])) + expect(restoreSpy).toHaveBeenCalledWith([1, 2]) + }) + + it('should support toggle all items in view', () => { + component.documentsInTrash = documentsInTrash + expect(component.selectedDocuments.size).toEqual(0) + const toggleAllSpy = jest.spyOn(component, 'toggleAll') + const checkButton = fixture.debugElement.queryAll( + By.css('input.form-check-input') + )[0] + checkButton.nativeElement.dispatchEvent(new Event('click')) + checkButton.nativeElement.checked = true + checkButton.nativeElement.dispatchEvent(new Event('click')) + expect(toggleAllSpy).toHaveBeenCalled() + expect(component.selectedDocuments.size).toEqual(documentsInTrash.length) + }) + + it('should support toggle item', () => { + component.selectedDocuments = new Set([1]) + component.toggleSelected(documentsInTrash[0]) + expect(component.selectedDocuments.size).toEqual(0) + component.toggleSelected(documentsInTrash[0]) + expect(component.selectedDocuments.size).toEqual(1) + }) + + it('should support clear selection', () => { + component.selectedDocuments = new Set([1]) + component.clearSelection() + expect(component.selectedDocuments.size).toEqual(0) + }) + + it('should correctly display days remaining', () => { + expect(component.getDaysRemaining(documentsInTrash[0])).toBeLessThan(0) + const tenDaysAgo = new Date() + tenDaysAgo.setDate(tenDaysAgo.getDate() - 10) + expect( + component.getDaysRemaining({ deleted_at: tenDaysAgo }) + ).toBeGreaterThan(0) // 10 days ago but depends on month + }) +}) diff --git a/src-ui/src/app/components/admin/trash/trash.component.ts b/src-ui/src/app/components/admin/trash/trash.component.ts new file mode 100644 index 000000000..b867f1706 --- /dev/null +++ b/src-ui/src/app/components/admin/trash/trash.component.ts @@ -0,0 +1,137 @@ +import { Component, OnDestroy } from '@angular/core' +import { NgbModal } from '@ng-bootstrap/ng-bootstrap' +import { Document } from 'src/app/data/document' +import { ToastService } from 'src/app/services/toast.service' +import { TrashService } from 'src/app/services/trash.service' +import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component' +import { Subject, takeUntil } from 'rxjs' +import { SettingsService } from 'src/app/services/settings.service' +import { SETTINGS_KEYS } from 'src/app/data/ui-settings' + +@Component({ + selector: 'pngx-trash', + templateUrl: './trash.component.html', + styleUrl: './trash.component.scss', +}) +export class TrashComponent implements OnDestroy { + public documentsInTrash: Document[] = [] + public selectedDocuments: Set = new Set() + public allToggled: boolean = false + public page: number = 1 + public totalDocuments: number + public isLoading: boolean = false + unsubscribeNotifier: Subject = new Subject() + + constructor( + private trashService: TrashService, + private toastService: ToastService, + private modalService: NgbModal, + private settingsService: SettingsService + ) { + this.reload() + } + + ngOnDestroy() { + this.unsubscribeNotifier.next() + this.unsubscribeNotifier.complete() + } + + reload() { + this.isLoading = true + this.trashService.getTrash(this.page).subscribe((r) => { + this.documentsInTrash = r.results + this.totalDocuments = r.count + this.isLoading = false + this.selectedDocuments.clear() + }) + } + + delete(document: Document) { + let modal = this.modalService.open(ConfirmDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.title = $localize`Confirm delete` + modal.componentInstance.messageBold = $localize`This operation will permanently delete this document.` + modal.componentInstance.message = $localize`This operation cannot be undone.` + modal.componentInstance.btnClass = 'btn-danger' + modal.componentInstance.btnCaption = $localize`Delete` + modal.componentInstance.confirmClicked + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(() => { + modal.componentInstance.buttonsEnabled = false + this.trashService.emptyTrash([document.id]).subscribe(() => { + this.toastService.showInfo($localize`Document deleted`) + modal.close() + this.reload() + }) + }) + } + + emptyTrash(documents?: Set) { + let modal = this.modalService.open(ConfirmDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.title = $localize`Confirm delete` + modal.componentInstance.messageBold = documents + ? $localize`This operation will permanently delete the selected documents.` + : $localize`This operation will permanently delete all documents in the trash.` + modal.componentInstance.message = $localize`This operation cannot be undone.` + modal.componentInstance.btnClass = 'btn-danger' + modal.componentInstance.btnCaption = $localize`Delete` + modal.componentInstance.confirmClicked + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(() => { + this.trashService + .emptyTrash(documents ? Array.from(documents) : null) + .subscribe(() => { + this.toastService.showInfo($localize`Document(s) deleted`) + this.allToggled = false + modal.close() + this.reload() + }) + }) + } + + restore(document: Document) { + this.trashService.restoreDocuments([document.id]).subscribe(() => { + this.toastService.showInfo($localize`Document restored`) + this.reload() + }) + } + + restoreAll(documents: Set = null) { + this.trashService + .restoreDocuments(documents ? Array.from(documents) : null) + .subscribe(() => { + this.toastService.showInfo($localize`Document(s) restored`) + this.allToggled = false + this.reload() + }) + } + + toggleAll(event: PointerEvent) { + if ((event.target as HTMLInputElement).checked) { + this.selectedDocuments = new Set(this.documentsInTrash.map((t) => t.id)) + } else { + this.clearSelection() + } + } + + toggleSelected(object: Document) { + this.selectedDocuments.has(object.id) + ? this.selectedDocuments.delete(object.id) + : this.selectedDocuments.add(object.id) + } + + clearSelection() { + this.allToggled = false + this.selectedDocuments.clear() + } + + getDaysRemaining(document: Document): number { + const delay = this.settingsService.get(SETTINGS_KEYS.EMPTY_TRASH_DELAY) + const diff = new Date().getTime() - new Date(document.deleted_at).getTime() + const days = Math.ceil(diff / (1000 * 3600 * 24)) + return delay - days + } +} diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index ab5759ec0..20c90b402 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -267,6 +267,13 @@ } +