From 814df94e8d4bad629e01fa8d80740b2cd9b94c2f Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 16 Jul 2025 17:03:03 -0700 Subject: [PATCH 01/27] Fix: dont use translated verbose_name for getting object perms (#10399) --- src/documents/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/documents/views.py b/src/documents/views.py index c56698e6f..de4277ad2 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -2490,7 +2490,7 @@ class BulkEditObjectsView(PassUserMixin): objs = object_class.objects.select_related("owner").filter(pk__in=object_ids) if not user.is_superuser: - model_name = object_class._meta.verbose_name + model_name = object_class._meta.model_name perm = ( f"documents.change_{model_name}" if operation == "set_permissions" From cebc227701665fe16af763a0b20a5a3e290794ab Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Thu, 17 Jul 2025 17:48:18 -0700 Subject: [PATCH 02/27] Fixhancement: add missing exact operator for boolean CF queries (#10402) --- src-ui/src/app/data/custom-field-query.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src-ui/src/app/data/custom-field-query.ts b/src-ui/src/app/data/custom-field-query.ts index 084b7a330..9f69fac76 100644 --- a/src-ui/src/app/data/custom-field-query.ts +++ b/src-ui/src/app/data/custom-field-query.ts @@ -85,7 +85,10 @@ export const CUSTOM_FIELD_QUERY_OPERATOR_GROUPS_BY_TYPE = { CustomFieldQueryOperatorGroups.Exact, CustomFieldQueryOperatorGroups.Date, ], - [CustomFieldDataType.Boolean]: [CustomFieldQueryOperatorGroups.Basic], + [CustomFieldDataType.Boolean]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.Exact, + ], [CustomFieldDataType.Integer]: [ CustomFieldQueryOperatorGroups.Basic, CustomFieldQueryOperatorGroups.Exact, From f8689c4819903f9d0ff5144677b71c8f7410a39e Mon Sep 17 00:00:00 2001 From: Boyuan Yang <073plan@gmail.com> Date: Fri, 18 Jul 2025 22:25:31 -0400 Subject: [PATCH 03/27] Documentation: Fix URL for PAPERLESS_OCR_LANGUAGE example in docker-compose.env (#10408) --- docker/compose/docker-compose.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/compose/docker-compose.env b/docker/compose/docker-compose.env index 7563ec069..75eeeed09 100644 --- a/docker/compose/docker-compose.env +++ b/docker/compose/docker-compose.env @@ -32,6 +32,6 @@ # Note that this is different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines # the language used for OCR. # The container installs English, German, Italian, Spanish and French by default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names # for available languages. #PAPERLESS_OCR_LANGUAGES=tur ces From 4b8f6ed6438a2c967accf5ff3e5b5e1407f9ed56 Mon Sep 17 00:00:00 2001 From: V0idC0de <26016825+V0idC0de@users.noreply.github.com> Date: Sun, 20 Jul 2025 15:31:49 +0200 Subject: [PATCH 04/27] Fixhancement: follow redirects in curl health check (#10415) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72c98b7cb..70226d41f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -265,4 +265,4 @@ ENTRYPOINT ["/init"] EXPOSE 8000 -HEALTHCHECK --interval=30s --timeout=10s --retries=5 CMD [ "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000" ] +HEALTHCHECK --interval=30s --timeout=10s --retries=5 CMD [ "curl", "-fs", "-S", "-L", "--max-time", "2", "http://localhost:8000" ] From 5410074062bef9aaaa4979cde7c6a87b39c0d9a3 Mon Sep 17 00:00:00 2001 From: Katrin Leinweber <9948149+katrinleinweber@users.noreply.github.com> Date: Sun, 20 Jul 2025 19:27:04 +0200 Subject: [PATCH 05/27] Documentation: copy-edits (#10417) --- CONTRIBUTING.md | 2 +- docker/compose/docker-compose.ci-test.yml | 2 +- .../compose/docker-compose.mariadb-tika.yml | 4 ++-- .../compose/docker-compose.postgres-tika.yml | 4 ++-- docker/compose/docker-compose.sqlite-tika.yml | 4 ++-- docs/administration.md | 2 +- docs/development.md | 12 +++++------ docs/index.md | 2 +- docs/setup.md | 2 +- docs/troubleshooting.md | 2 +- install-paperless-ngx.sh | 20 +++++++++---------- 11 files changed, 28 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93a529bfe..e0c52d00b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -141,7 +141,7 @@ The admins occasionally invite contributors directly if we believe having them o # Automatic Repository Maintenance The Paperless-ngx team appreciates all effort and interest from the community in filing bug reports, creating feature requests, sharing ideas and helping other -community members. That said, in an effort to keep the repository organized and managebale the project uses automatic handling of certain areas: +community members. That said, in an effort to keep the repository organized and manageable the project uses automatic handling of certain areas: - Issues that cannot be reproduced will be marked 'stale' after 7 days of inactivity and closed after 14 further days of inactivity. - Issues, pull requests and discussions that are closed will be locked after 30 days of inactivity. diff --git a/docker/compose/docker-compose.ci-test.yml b/docker/compose/docker-compose.ci-test.yml index 0d43ab923..9eb1afe19 100644 --- a/docker/compose/docker-compose.ci-test.yml +++ b/docker/compose/docker-compose.ci-test.yml @@ -1,4 +1,4 @@ -# Docker Compose file for running paperless testing with actual gotenberg +# Docker Compose file for running paperless testing with actual Gotenberg # and Tika containers for a more end to end test of the Tika related functionality # Can be used locally or by the CI to start the necessary containers with the # correct networking for the tests diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml index e48dd7dd3..b4522c26b 100644 --- a/docker/compose/docker-compose.mariadb-tika.yml +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -16,8 +16,8 @@ # - Instead of SQLite (default), MariaDB is used as the database server. # - Apache Tika and Gotenberg servers are started with paperless and paperless # is configured to use these services. These provide support for consuming -# Office documents (Word, Excel, Power Point and their LibreOffice counter- -# parts. +# Office documents (Word, Excel, PowerPoint and their LibreOffice counter- +# parts). # # To install and update paperless with this file, do the following: # diff --git a/docker/compose/docker-compose.postgres-tika.yml b/docker/compose/docker-compose.postgres-tika.yml index f247ec117..818e1b130 100644 --- a/docker/compose/docker-compose.postgres-tika.yml +++ b/docker/compose/docker-compose.postgres-tika.yml @@ -16,8 +16,8 @@ # - Instead of SQLite (default), PostgreSQL is used as the database server. # - Apache Tika and Gotenberg servers are started with paperless and paperless # is configured to use these services. These provide support for consuming -# Office documents (Word, Excel, Power Point and their LibreOffice counter- -# parts. +# Office documents (Word, Excel, PowerPoint and their LibreOffice counter- +# parts). # # To install and update paperless with this file, do the following: # diff --git a/docker/compose/docker-compose.sqlite-tika.yml b/docker/compose/docker-compose.sqlite-tika.yml index d2728fd6b..86a6f5031 100644 --- a/docker/compose/docker-compose.sqlite-tika.yml +++ b/docker/compose/docker-compose.sqlite-tika.yml @@ -16,8 +16,8 @@ # # - Apache Tika and Gotenberg servers are started with paperless and paperless # is configured to use these services. These provide support for consuming -# Office documents (Word, Excel, Power Point and their LibreOffice counter- -# parts. +# Office documents (Word, Excel, PowerPoint and their LibreOffice counter- +# parts). # # To install and update paperless with this file, do the following: # diff --git a/docs/administration.md b/docs/administration.md index 4bb4b34cc..b646c1f73 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -306,7 +306,7 @@ in dedicated folders according to their nature: `archive`, `originals`, If `-sm` or `--split-manifest` is provided, information about document will be placed in individual json files, instead of a single JSON file. The main manifest.json will still contain application wide information (e.g. tags, correspondent, -documenttype, etc) +document type, etc) If `-z` or `--zip` is provided, the export will be a zip file in the target directory, named according to the current local date or the diff --git a/docs/development.md b/docs/development.md index 5353c164d..c624f97c4 100644 --- a/docs/development.md +++ b/docs/development.md @@ -95,13 +95,13 @@ first-time setup. 7. You can now either ... - - install redis or + - install Redis or - - use the included `scripts/start_services.sh` to use docker to fire - up a redis instance (and some other services such as tika, - gotenberg and a database server) or + - use the included `scripts/start_services.sh` to use Docker to fire + up a Redis instance (and some other services such as Tika, + Gotenberg and a database server) or - - spin up a bare redis container + - spin up a bare Redis container ``` docker run -d -p 6379:6379 --restart unless-stopped redis:latest @@ -147,7 +147,7 @@ $ ng build --configuration production ### Testing - Run `pytest` in the `src/` directory to execute all tests. This also - generates a HTML coverage report. When runnings test, `paperless.conf` + generates a HTML coverage report. When running tests, `paperless.conf` is loaded as well. However, the tests rely on the default configuration. This is not ideal. But for now, make sure no settings except for DEBUG are overridden when testing. diff --git a/docs/index.md b/docs/index.md index b86112ed6..c1c06eb2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,7 +30,7 @@ physical documents into a searchable online archive so you can keep, well, _less - Utilizes the open-source Tesseract engine to recognize more than 100 languages. - Documents are saved as PDF/A format which is designed for long term storage, alongside the unaltered originals. - Uses machine-learning to automatically add tags, correspondents and document types to your documents. -- Supports PDF documents, images, plain text files, Office documents (Word, Excel, Powerpoint, and LibreOffice equivalents)[^1] and more. +- Supports PDF documents, images, plain text files, Office documents (Word, Excel, PowerPoint, and LibreOffice equivalents)[^1] and more. - Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and their format can be configured freely with different configurations assigned to different documents. - **Beautiful, modern web application** that features: - Customizable dashboard with statistics. diff --git a/docs/setup.md b/docs/setup.md index 9c75313af..f239e5429 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -445,7 +445,7 @@ are released, dependency support is confirmed, etc. 13. Configure ImageMagick to allow processing of PDF documents. Most distributions have this disabled by default, since PDF documents can contain malware. If you don't do this, paperless will fall back to - ghostscript for certain steps such as thumbnail generation. + Ghostscript for certain steps such as thumbnail generation. Edit `/etc/ImageMagick-6/policy.xml` and adjust diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6a7270e01..87348e32e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -335,7 +335,7 @@ You may see errors when deleting documents like: Data too long for column 'transaction_id' at row 1 ``` -This error can occur in installations which have upgraded from a version of Paperless-ngx that used Django 4 (Paperless-ngx versions prior to v2.13.0) with a MariaDB/MySQL database. Due to the backawards-incompatible change in Django 5, the column "documents_document.transaction_id" will need to be re-created, which can be done with a one-time run of the following management command: +This error can occur in installations which have upgraded from a version of Paperless-ngx that used Django 4 (Paperless-ngx versions prior to v2.13.0) with a MariaDB/MySQL database. Due to the backwards-incompatible change in Django 5, the column "documents_document.transaction_id" will need to be re-created, which can be done with a one-time run of the following management command: ```shell-session $ python3 manage.py convert_mariadb_uuid diff --git a/install-paperless-ngx.sh b/install-paperless-ngx.sh index 3977cece9..648196030 100755 --- a/install-paperless-ngx.sh +++ b/install-paperless-ngx.sh @@ -52,12 +52,12 @@ if ! command -v wget &> /dev/null ; then fi if ! command -v docker &> /dev/null ; then - echo "docker executable not found. Is docker installed?" + echo "docker executable not found. Is Docker installed?" exit 1 fi if ! docker compose &> /dev/null ; then - echo "docker compose plugin not found. Is docker compose installed?" + echo "docker compose plugin not found. Is Docker Compose installed?" exit 1 fi @@ -66,7 +66,7 @@ fi if ! docker stats --no-stream &> /dev/null ; then echo "" echo "WARN: It look like the current user does not have Docker permissions." - echo "WARN: Use 'sudo usermod -aG docker $USER' to assign Docker permissions to the user (may require restarting shell)." + echo "WARN: Use 'sudo usermod -aG docker $USER' to assign Docker permissions to the user (may require restarting the shell)." echo "" sleep 3 fi @@ -135,7 +135,7 @@ DATABASE_BACKEND=$ask_result echo "" echo "Paperless is able to use Apache Tika to support Office documents such as" -echo "Word, Excel, Powerpoint, and Libreoffice equivalents. This feature" +echo "Word, Excel, PowerPoint, and LibreOffice equivalents. This feature" echo "requires more resources due to the required services." echo "" @@ -157,7 +157,7 @@ echo "" echo "Specify the user id and group id you wish to run paperless as." echo "Paperless will also change ownership on the data, media and consume" echo "folder to the specified values, so it's a good idea to supply the user id" -echo "and group id of your unix user account." +echo "and group id of your Unix user account." echo "If unsure, leave default." echo "" @@ -212,7 +212,7 @@ if [[ "$DATABASE_BACKEND" == "sqlite" ]] ; then echo -n "SQLite database, the " fi echo "search index and other data." -echo "As with the media folder, leave empty to have this managed by docker." +echo "As with the media folder, leave empty to have this managed by Docker." echo "" echo "CAUTION: If specified, you must specify an absolute path starting with /" echo "or a relative path starting with ./ here." @@ -224,7 +224,7 @@ DATA_FOLDER=$ask_result if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then echo "" echo "The database folder, where your database stores its data." - echo "Leave empty to have this managed by docker." + echo "Leave empty to have this managed by Docker." echo "" echo "CAUTION: If specified, you must specify an absolute path starting with /" echo "or a relative path starting with ./ here." @@ -276,18 +276,18 @@ echo "" echo "Target folder: $TARGET_FOLDER" echo "Consume folder: $CONSUME_FOLDER" if [[ -z $MEDIA_FOLDER ]] ; then - echo "Media folder: Managed by docker" + echo "Media folder: Managed by Docker" else echo "Media folder: $MEDIA_FOLDER" fi if [[ -z $DATA_FOLDER ]] ; then - echo "Data folder: Managed by docker" + echo "Data folder: Managed by Docker" else echo "Data folder: $DATA_FOLDER" fi if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then if [[ -z $DATABASE_FOLDER ]] ; then - echo "Database folder: Managed by docker" + echo "Database folder: Managed by Docker" else echo "Database folder: $DATABASE_FOLDER" fi From 1fe85992660a3d6d6c935bdbe4927ca9f4c7fa0e Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 23 Jul 2025 22:05:55 -0700 Subject: [PATCH 06/27] Fix: Make some natural keyword date searches timezone-aware (#10416) --- src/documents/index.py | 39 +++++++++++++++++++++++++++++++ src/documents/tests/test_index.py | 37 +++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/documents/index.py b/src/documents/index.py index 10de04245..3d1030dca 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -2,10 +2,12 @@ from __future__ import annotations import logging import math +import re from collections import Counter from contextlib import contextmanager from datetime import datetime from datetime import time +from datetime import timedelta from datetime import timezone from shutil import rmtree from typing import TYPE_CHECKING @@ -13,6 +15,8 @@ from typing import Literal from django.conf import settings from django.utils import timezone as django_timezone +from django.utils.timezone import get_current_timezone +from django.utils.timezone import now from guardian.shortcuts import get_users_with_perms from whoosh import classify from whoosh import highlight @@ -344,6 +348,7 @@ class LocalDateParser(English): class DelayedFullTextQuery(DelayedQuery): def _get_query(self) -> tuple: q_str = self.query_params["query"] + q_str = rewrite_natural_date_keywords(q_str) qp = MultifieldParser( [ "content", @@ -450,3 +455,37 @@ def get_permissions_criterias(user: User | None = None) -> list: query.Term("viewer_id", str(user.id)), ) return user_criterias + + +def rewrite_natural_date_keywords(query_string: str) -> str: + """ + Rewrites natural date keywords (e.g. added:today or added:"yesterday") to UTC range syntax for Whoosh. + """ + + tz = get_current_timezone() + local_now = now().astimezone(tz) + + today = local_now.date() + yesterday = today - timedelta(days=1) + + ranges = { + "today": ( + datetime.combine(today, time.min, tzinfo=tz), + datetime.combine(today, time.max, tzinfo=tz), + ), + "yesterday": ( + datetime.combine(yesterday, time.min, tzinfo=tz), + datetime.combine(yesterday, time.max, tzinfo=tz), + ), + } + + pattern = r"(\b(?:added|created))\s*:\s*[\"']?(today|yesterday)[\"']?" + + def repl(m): + field, keyword = m.group(1), m.group(2) + start, end = ranges[keyword] + start_str = start.astimezone(timezone.utc).strftime("%Y%m%d%H%M%S") + end_str = end.astimezone(timezone.utc).strftime("%Y%m%d%H%M%S") + return f"{field}:[{start_str} TO {end_str}]" + + return re.sub(pattern, repl, query_string) diff --git a/src/documents/tests/test_index.py b/src/documents/tests/test_index.py index 24bc26d4c..2a41542e9 100644 --- a/src/documents/tests/test_index.py +++ b/src/documents/tests/test_index.py @@ -1,6 +1,11 @@ +from datetime import datetime from unittest import mock +from django.contrib.auth.models import User from django.test import TestCase +from django.test import override_settings +from django.utils.timezone import get_current_timezone +from django.utils.timezone import timezone from documents import index from documents.models import Document @@ -90,3 +95,35 @@ class TestAutoComplete(DirectoriesMixin, TestCase): _, kwargs = mocked_update_doc.call_args self.assertIsNone(kwargs["asn"]) + + @override_settings(TIME_ZONE="Pacific/Auckland") + def test_added_today_respects_local_timezone_boundary(self): + tz = get_current_timezone() + fixed_now = datetime(2025, 7, 20, 15, 0, 0, tzinfo=tz) + + # Fake a time near the local boundary (1 AM NZT = 13:00 UTC on previous UTC day) + local_dt = datetime(2025, 7, 20, 1, 0, 0).replace(tzinfo=tz) + utc_dt = local_dt.astimezone(timezone.utc) + + doc = Document.objects.create( + title="Time zone", + content="Testing added:today", + checksum="edgecase123", + added=utc_dt, + ) + + with index.open_index_writer() as writer: + index.update_document(writer, doc) + + superuser = User.objects.create_superuser(username="testuser") + self.client.force_login(superuser) + + with mock.patch("documents.index.now", return_value=fixed_now): + response = self.client.get("/api/documents/?query=added:today") + results = response.json()["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], doc.id) + + response = self.client.get("/api/documents/?query=added:yesterday") + results = response.json()["results"] + self.assertEqual(len(results), 0) From 293c84d871d8c8d80959bd8972d576967752cd96 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 23 Jul 2025 22:07:13 -0700 Subject: [PATCH 07/27] Enhancement: display saved view counts (#10246) --- .../admin/settings/settings.component.html | 1 + .../admin/settings/settings.component.spec.ts | 15 +++++- .../admin/settings/settings.component.ts | 11 +++++ .../app-frame/app-frame.component.html | 9 +++- .../app-frame/app-frame.component.spec.ts | 14 +++++- .../app-frame/app-frame.component.ts | 8 +++- .../saved-view-widget.component.html | 1 + .../saved-view-widget.component.ts | 3 ++ .../widget-frame/widget-frame.component.html | 5 +- .../widget-frame/widget-frame.component.ts | 3 ++ .../document-detail.component.ts | 4 ++ .../bulk-editor/bulk-editor.component.ts | 3 ++ src-ui/src/app/data/ui-settings.ts | 7 +++ .../services/rest/saved-view.service.spec.ts | 45 ++++++++++++++++-- .../app/services/rest/saved-view.service.ts | 46 +++++++++++++++++-- 15 files changed, 162 insertions(+), 13 deletions(-) diff --git a/src-ui/src/app/components/admin/settings/settings.component.html b/src-ui/src/app/components/admin/settings/settings.component.html index 9d235a0f3..ccd3cc7e3 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.html +++ b/src-ui/src/app/components/admin/settings/settings.component.html @@ -176,6 +176,7 @@