diff --git a/.editorconfig b/.editorconfig index 8111f01d8..a8b21f510 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,10 @@ end_of_line = lf charset = utf-8 max_line_length = 79 +[*.sh] +indent_style = tab +indent_size = 1 + [{*.html,*.css,*.js}] max_line_length = off diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76ab247fb..e1cc4c3ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,9 +353,9 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: cd src-ui && pnpm run build --configuration=production build-docker-image: - name: Build Docker image for ${{ github.ref_name }} + name: Build Docker image for ${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }} runs-on: ubuntu-24.04 - if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || startsWith(github.ref, 'refs/heads/fix-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/l10n_')) + if: (github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || startsWith(github.ref, 'refs/heads/fix-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/l10n_'))) || (github.event_name == 'pull_request' && (startsWith(github.head_ref, 'feature-') || startsWith(github.head_ref, 'fix-') || github.head_ref == 'dev' || github.head_ref == 'beta' || contains(github.head_ref, 'beta.rc') || startsWith(github.head_ref, 'l10n_'))) concurrency: group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }} cancel-in-progress: true @@ -364,6 +364,23 @@ jobs: - tests-frontend - tests-frontend-e2e steps: + - name: Prepare build variables + id: build-vars + uses: actions/github-script@v8 + with: + result-encoding: string + script: | + const isPR = context.eventName === 'pull_request'; + const defaultRefName = context.ref.replace('refs/heads/', ''); + const headRef = isPR ? context.payload.pull_request.head.ref : defaultRefName; + const buildRef = isPR ? `refs/heads/${headRef}` : context.ref; + const buildCacheKey = headRef.split('/').join('-'); + const canPush = context.eventName === 'push' || (isPR && context.payload.pull_request.head.repo.full_name === `${context.repo.owner}/${context.repo.repo}`); + + core.setOutput('build-ref', buildRef); + core.setOutput('build-ref-name', headRef); + core.setOutput('build-cache-key', buildCacheKey); + core.setOutput('can-push', canPush ? 'true' : 'false'); - name: Check pushing to Docker Hub id: push-other-places # Only push to Dockerhub from the main repo AND the ref is either: @@ -372,8 +389,11 @@ jobs: # beta # a tag # Otherwise forks would require a Docker Hub account and secrets setup + env: + BUILD_REF: ${{ steps.build-vars.outputs.build-ref }} + BUILD_REF_NAME: ${{ steps.build-vars.outputs.build-ref-name }} run: | - if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then + if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( "$BUILD_REF_NAME" == "dev" || "$BUILD_REF_NAME" == "beta" || $BUILD_REF == refs/tags/v* || $BUILD_REF == *beta.rc* ) ]] ; then echo "Enabling DockerHub image push" echo "enable=true" >> $GITHUB_OUTPUT else @@ -397,6 +417,8 @@ jobs: tags: | # Tag branches with branch name type=ref,event=branch + # Pull requests need a sanitized branch tag for pushing images + type=raw,value=${{ steps.build-vars.outputs.build-cache-key }},enable=${{ github.event_name == 'pull_request' }} # Process semver tags # For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag type=semver,pattern={{version}} @@ -439,7 +461,7 @@ jobs: context: . file: ./Dockerfile platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} + push: ${{ steps.build-vars.outputs.can-push == 'true' }} tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} build-args: | @@ -447,18 +469,20 @@ jobs: # Get cache layers from this branch, then dev # This allows new branches to get at least some cache benefits, generally from dev cache-from: | - type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} + type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ steps.build-vars.outputs.build-cache-key }} type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev - cache-to: | - type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} + cache-to: ${{ steps.build-vars.outputs.can-push == 'true' && format('type=registry,mode=max,ref=ghcr.io/{0}/builder/cache/app:{1}', steps.set-ghcr-repository.outputs.ghcr-repository, steps.build-vars.outputs.build-cache-key) || '' }} - name: Inspect image + if: steps.build-vars.outputs.can-push == 'true' run: | docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }} - name: Export frontend artifact from docker + if: steps.build-vars.outputs.can-push == 'true' run: | docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }} docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/ - name: Upload frontend artifact + if: steps.build-vars.outputs.can-push == 'true' uses: actions/upload-artifact@v5 with: name: frontend-compiled @@ -469,6 +493,7 @@ jobs: needs: - build-docker-image - documentation + if: github.event_name == 'push' runs-on: ubuntu-24.04 steps: - name: Checkout diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c0167d94..92e18bfbd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,12 +49,12 @@ repos: - 'prettier-plugin-organize-imports@4.1.0' # Python hooks - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.0 + rev: v0.14.5 hooks: - id: ruff-check - id: ruff-format - repo: https://github.com/tox-dev/pyproject-fmt - rev: "v2.11.0" + rev: "v2.11.1" hooks: - id: pyproject-fmt # Dockerfile hooks @@ -64,11 +64,11 @@ repos: - id: hadolint # Shell script hooks - repo: https://github.com/lovesegfault/beautysh - rev: v6.2.1 + rev: v6.4.2 hooks: - id: beautysh - additional_dependencies: - - setuptools + types: [file] + files: (\.sh$|/run$|/finish$) args: - "--tab" - repo: https://github.com/shellcheck-py/shellcheck-py @@ -76,7 +76,7 @@ repos: hooks: - id: shellcheck - repo: https://github.com/google/yamlfmt - rev: v0.18.0 + rev: v0.20.0 hooks: - id: yamlfmt exclude: "^src-ui/pnpm-lock.yaml" diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/init-env-file/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/init-env-file/run index d2c118ddb..2fc64e24e 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/init-env-file/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/init-env-file/run @@ -29,5 +29,5 @@ if find /run/s6/container_environment/*"_FILE" -maxdepth 1 > /dev/null 2>&1; the fi done else - echo "${log_prefix} No *_FILE environment found" + echo "${log_prefix} No *_FILE environment found" fi diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/init-wait-for-db/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/init-wait-for-db/run index 1739edd61..8ec900212 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/init-wait-for-db/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/init-wait-for-db/run @@ -12,24 +12,24 @@ declare -i DELAY=0 declare -i STARTED_AT=${EPOCHSECONDS:?EPOCHSECONDS var unset} delay_next_attempt() { - local -i elapsed=$(( EPOCHSECONDS - STARTED_AT )) - local -ri remaining=$(( TIMEOUT - elapsed )) + local -i elapsed=$(( EPOCHSECONDS - STARTED_AT )) + local -ri remaining=$(( TIMEOUT - elapsed )) - if (( remaining <= 0 )); then - echo "${LOG_PREFIX} Unable to connect after $elapsed seconds." - exit 1 - fi + if (( remaining <= 0 )); then + echo "${LOG_PREFIX} Unable to connect after $elapsed seconds." + exit 1 + fi - DELAY+=1 + DELAY+=1 - # clamp to remaining time - if (( DELAY > remaining )); then - DELAY=$remaining - fi + # clamp to remaining time + if (( DELAY > remaining )); then + DELAY=$remaining + fi - ATTEMPT+=1 - echo "${LOG_PREFIX} Attempt $ATTEMPT failed! Trying again in $DELAY seconds..." - sleep "$DELAY" + ATTEMPT+=1 + echo "${LOG_PREFIX} Attempt $ATTEMPT failed! Trying again in $DELAY seconds..." + sleep "$DELAY" } wait_for_postgres() { @@ -40,7 +40,7 @@ wait_for_postgres() { local -r user="${PAPERLESS_DBUSER:-paperless}" while ! pg_isready -h "${host}" -p "${port}" --username "${user}"; do - delay_next_attempt + delay_next_attempt done echo "${LOG_PREFIX} Connected to PostgreSQL" } @@ -51,8 +51,8 @@ wait_for_mariadb() { local -r host="${PAPERLESS_DBHOST:-localhost}" local -r port="${PAPERLESS_DBPORT:-3306}" - while ! true > "/dev/tcp/$host/$port"; do - delay_next_attempt + while ! mariadb-admin --host="$host" --port="$port" ping --silent >/dev/null 2>&1; do + delay_next_attempt done echo "${LOG_PREFIX} Connected to MariaDB" } diff --git a/docker/rootfs/etc/s6-overlay/s6-rc.d/svc-webserver/run b/docker/rootfs/etc/s6-overlay/s6-rc.d/svc-webserver/run index 841dad204..64b458150 100755 --- a/docker/rootfs/etc/s6-overlay/s6-rc.d/svc-webserver/run +++ b/docker/rootfs/etc/s6-overlay/s6-rc.d/svc-webserver/run @@ -10,11 +10,11 @@ export GRANIAN_WORKERS=${GRANIAN_WORKERS:-${PAPERLESS_WEBSERVER_WORKERS:-1}} # Only set GRANIAN_URL_PATH_PREFIX if PAPERLESS_FORCE_SCRIPT_NAME is set if [[ -n "${PAPERLESS_FORCE_SCRIPT_NAME}" ]]; then - export GRANIAN_URL_PATH_PREFIX=${PAPERLESS_FORCE_SCRIPT_NAME} + export GRANIAN_URL_PATH_PREFIX=${PAPERLESS_FORCE_SCRIPT_NAME} fi if [[ -n "${USER_IS_NON_ROOT}" ]]; then - exec granian --interface asginl --ws --loop uvloop "paperless.asgi:application" + exec granian --interface asginl --ws --loop uvloop "paperless.asgi:application" else - exec s6-setuidgid paperless granian --interface asginl --ws --loop uvloop "paperless.asgi:application" + exec s6-setuidgid paperless granian --interface asginl --ws --loop uvloop "paperless.asgi:application" fi diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 11a735f25..273393b46 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1157,7 +1157,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 192 + 208 @@ -3371,7 +3371,7 @@ src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 106 + 111 src/app/components/common/input/date/date.component.html @@ -3524,17 +3524,6 @@ 101 - - now - - src/app/components/common/dates-dropdown/dates-dropdown.component.html - 29 - - - src/app/components/common/dates-dropdown/dates-dropdown.component.html - 105 - - From @@ -3580,59 +3569,94 @@ 93 + + now + + src/app/components/common/dates-dropdown/dates-dropdown.component.html + 169 + + Within 1 week src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 76 + 81 Within 1 month src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 81 + 86 Within 3 months src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 86 + 91 Within 1 year src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 91 + 96 This year src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 96 + 101 This month src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 101 + 106 Yesterday src/app/components/common/dates-dropdown/dates-dropdown.component.ts - 111 + 116 src/app/pipes/custom-date.pipe.ts 29 + + Previous week + + src/app/components/common/dates-dropdown/dates-dropdown.component.ts + 121 + + + + Previous month + + src/app/components/common/dates-dropdown/dates-dropdown.component.ts + 135 + + + + Previous quarter + + src/app/components/common/dates-dropdown/dates-dropdown.component.ts + 141 + + + + Previous year + + src/app/components/common/dates-dropdown/dates-dropdown.component.ts + 155 + + Matching algorithm @@ -6811,7 +6835,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 179 + 195 src/app/data/document.ts @@ -7372,7 +7396,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 187 + 203 @@ -7999,7 +8023,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 184 + 200 src/app/data/document.ts @@ -8195,56 +8219,56 @@ Title & content src/app/components/document-list/filter-editor/filter-editor.component.ts - 182 + 198 File type src/app/components/document-list/filter-editor/filter-editor.component.ts - 189 + 205 More like src/app/components/document-list/filter-editor/filter-editor.component.ts - 198 + 214 equals src/app/components/document-list/filter-editor/filter-editor.component.ts - 204 + 220 is empty src/app/components/document-list/filter-editor/filter-editor.component.ts - 208 + 224 is not empty src/app/components/document-list/filter-editor/filter-editor.component.ts - 212 + 228 greater than src/app/components/document-list/filter-editor/filter-editor.component.ts - 216 + 232 less than src/app/components/document-list/filter-editor/filter-editor.component.ts - 220 + 236 @@ -8253,14 +8277,14 @@ )?.name"/> src/app/components/document-list/filter-editor/filter-editor.component.ts - 261,265 + 277,281 Without correspondent src/app/components/document-list/filter-editor/filter-editor.component.ts - 267 + 283 @@ -8269,14 +8293,14 @@ )?.name"/> src/app/components/document-list/filter-editor/filter-editor.component.ts - 273,277 + 289,293 Without document type src/app/components/document-list/filter-editor/filter-editor.component.ts - 279 + 295 @@ -8285,70 +8309,70 @@ )?.name"/> src/app/components/document-list/filter-editor/filter-editor.component.ts - 285,289 + 301,305 Without storage path src/app/components/document-list/filter-editor/filter-editor.component.ts - 291 + 307 Tag: src/app/components/document-list/filter-editor/filter-editor.component.ts - 295,297 + 311,313 Without any tag src/app/components/document-list/filter-editor/filter-editor.component.ts - 301 + 317 Custom fields query src/app/components/document-list/filter-editor/filter-editor.component.ts - 305 + 321 Title: src/app/components/document-list/filter-editor/filter-editor.component.ts - 308 + 324 ASN: src/app/components/document-list/filter-editor/filter-editor.component.ts - 311 + 327 Owner: src/app/components/document-list/filter-editor/filter-editor.component.ts - 314 + 330 Owner not in: src/app/components/document-list/filter-editor/filter-editor.component.ts - 317 + 333 Without an owner src/app/components/document-list/filter-editor/filter-editor.component.ts - 320 + 336 diff --git a/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.html b/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.html index 9b243d907..74b49bbdb 100644 --- a/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.html +++ b/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.html @@ -26,7 +26,7 @@ i18n-placeholder (change)="onSetCreatedRelativeDate($event)"> -
{{ item.name }}{{ item.date | customDate:'mediumDate' }} – now
+
@@ -102,7 +102,7 @@ i18n-placeholder (change)="onSetAddedRelativeDate($event)"> -
{{ item.name }}{{ item.date | customDate:'mediumDate' }} – now
+
@@ -158,3 +158,16 @@ + + +
+ {{ item.name }} + + @if (item.dateEnd) { + {{ item.date | customDate:'MMM d' }} – {{ item.dateEnd | customDate:'mediumDate' }} + } @else { + {{ item.date | customDate:'mediumDate' }} – now + } + +
+
diff --git a/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.ts b/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.ts index 501b43fab..e07b08959 100644 --- a/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.ts +++ b/src-ui/src/app/components/common/dates-dropdown/dates-dropdown.component.ts @@ -1,4 +1,4 @@ -import { NgClass } from '@angular/common' +import { NgClass, NgTemplateOutlet } from '@angular/common' import { Component, EventEmitter, @@ -42,6 +42,10 @@ export enum RelativeDate { THIS_MONTH = 6, TODAY = 7, YESTERDAY = 8, + PREVIOUS_WEEK = 9, + PREVIOUS_MONTH = 10, + PREVIOUS_QUARTER = 11, + PREVIOUS_YEAR = 12, } @Component({ @@ -59,6 +63,7 @@ export enum RelativeDate { FormsModule, ReactiveFormsModule, NgClass, + NgTemplateOutlet, ], }) export class DatesDropdownComponent implements OnInit, OnDestroy { @@ -111,6 +116,46 @@ export class DatesDropdownComponent implements OnInit, OnDestroy { name: $localize`Yesterday`, date: new Date().setDate(new Date().getDate() - 1), }, + { + id: RelativeDate.PREVIOUS_WEEK, + name: $localize`Previous week`, + date: new Date( + new Date().getFullYear(), + new Date().getMonth(), + new Date().getDate() - new Date().getDay() - 6 + ), + dateEnd: new Date( + new Date().getFullYear(), + new Date().getMonth(), + new Date().getDate() - new Date().getDay() + ), + }, + { + id: RelativeDate.PREVIOUS_MONTH, + name: $localize`Previous month`, + date: new Date(new Date().getFullYear(), new Date().getMonth() - 1, 1), + dateEnd: new Date(new Date().getFullYear(), new Date().getMonth(), 0), + }, + { + id: RelativeDate.PREVIOUS_QUARTER, + name: $localize`Previous quarter`, + date: new Date( + new Date().getFullYear(), + Math.floor(new Date().getMonth() / 3) * 3 - 3, + 1 + ), + dateEnd: new Date( + new Date().getFullYear(), + Math.floor(new Date().getMonth() / 3) * 3, + 0 + ), + }, + { + id: RelativeDate.PREVIOUS_YEAR, + name: $localize`Previous year`, + date: new Date('1/1/' + (new Date().getFullYear() - 1)), + dateEnd: new Date('12/31/' + (new Date().getFullYear() - 1)), + }, ] datePlaceHolder: string diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index 9ffcc380b..16b65c84d 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -173,6 +173,22 @@ const RELATIVE_DATE_QUERYSTRINGS = [ relativeDate: RelativeDate.YESTERDAY, dateQuery: 'yesterday', }, + { + relativeDate: RelativeDate.PREVIOUS_WEEK, + dateQuery: 'previous week', + }, + { + relativeDate: RelativeDate.PREVIOUS_MONTH, + dateQuery: 'previous month', + }, + { + relativeDate: RelativeDate.PREVIOUS_QUARTER, + dateQuery: 'previous quarter', + }, + { + relativeDate: RelativeDate.PREVIOUS_YEAR, + dateQuery: 'previous year', + }, ] const DEFAULT_TEXT_FILTER_TARGET_OPTIONS = [ diff --git a/src/documents/index.py b/src/documents/index.py index 9446c7db1..6b994ac8c 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -13,6 +13,7 @@ from shutil import rmtree from typing import TYPE_CHECKING from typing import Literal +from dateutil.relativedelta import relativedelta from django.conf import settings from django.utils import timezone as django_timezone from django.utils.timezone import get_current_timezone @@ -533,32 +534,84 @@ def get_permissions_criterias(user: User | None = None) -> list: def rewrite_natural_date_keywords(query_string: str) -> str: """ Rewrites natural date keywords (e.g. added:today or added:"yesterday") to UTC range syntax for Whoosh. + This resolves timezone issues with date parsing in Whoosh as well as adding support for more + natural date keywords. """ tz = get_current_timezone() local_now = now().astimezone(tz) - today = local_now.date() - yesterday = today - timedelta(days=1) - ranges = { - "today": ( - datetime.combine(today, time.min, tzinfo=tz), - datetime.combine(today, time.max, tzinfo=tz), - ), - "yesterday": ( - datetime.combine(yesterday, time.min, tzinfo=tz), - datetime.combine(yesterday, time.max, tzinfo=tz), - ), - } - - pattern = r"(\b(?:added|created))\s*:\s*[\"']?(today|yesterday)[\"']?" + # all supported Keywords + pattern = r"(\b(?:added|created|modified))\s*:\s*[\"']?(today|yesterday|this month|previous month|previous week|previous quarter|this year|previous year)[\"']?" def repl(m): - field, keyword = m.group(1), m.group(2) - start, end = ranges[keyword] + field = m.group(1) + keyword = m.group(2).lower() + + match keyword: + case "today": + start = datetime.combine(today, time.min, tzinfo=tz) + end = datetime.combine(today, time.max, tzinfo=tz) + + case "yesterday": + yesterday = today - timedelta(days=1) + start = datetime.combine(yesterday, time.min, tzinfo=tz) + end = datetime.combine(yesterday, time.max, tzinfo=tz) + + case "this month": + start = datetime(local_now.year, local_now.month, 1, 0, 0, 0, tzinfo=tz) + end = start + relativedelta(months=1) - timedelta(seconds=1) + + case "previous month": + this_month_start = datetime( + local_now.year, + local_now.month, + 1, + 0, + 0, + 0, + tzinfo=tz, + ) + start = this_month_start - relativedelta(months=1) + end = this_month_start - timedelta(seconds=1) + + case "this year": + start = datetime(local_now.year, 1, 1, 0, 0, 0, tzinfo=tz) + end = datetime.combine(today, time.max, tzinfo=tz) + + case "previous week": + days_since_monday = local_now.weekday() + this_week_start = datetime.combine( + today - timedelta(days=days_since_monday), + time.min, + tzinfo=tz, + ) + start = this_week_start - timedelta(days=7) + end = this_week_start - timedelta(seconds=1) + + case "previous quarter": + current_quarter = (local_now.month - 1) // 3 + 1 + this_quarter_start_month = (current_quarter - 1) * 3 + 1 + this_quarter_start = datetime( + local_now.year, + this_quarter_start_month, + 1, + 0, + 0, + 0, + tzinfo=tz, + ) + start = this_quarter_start - relativedelta(months=3) + end = this_quarter_start - timedelta(seconds=1) + + case "previous year": + start = datetime(local_now.year - 1, 1, 1, 0, 0, 0, tzinfo=tz) + end = datetime(local_now.year - 1, 12, 31, 23, 59, 59, tzinfo=tz) + + # Convert to UTC and format start_str = start.astimezone(timezone.utc).strftime("%Y%m%d%H%M%S") end_str = end.astimezone(timezone.utc).strftime("%Y%m%d%H%M%S") return f"{field}:[{start_str} TO {end_str}]" - return re.sub(pattern, repl, query_string) + return re.sub(pattern, repl, query_string, flags=re.IGNORECASE) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index bce376f76..0e53c9357 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -396,6 +396,7 @@ class CannotMoveFilesException(Exception): @receiver(models.signals.post_save, sender=CustomFieldInstance, weak=False) @receiver(models.signals.m2m_changed, sender=Document.tags.through, weak=False) @receiver(models.signals.post_save, sender=Document, weak=False) +@shared_task def update_filename_and_move_files( sender, instance: Document | CustomFieldInstance, @@ -559,7 +560,7 @@ def check_paths_and_prune_custom_fields(sender, instance: CustomField, **kwargs) cf_instance.save(update_fields=["value_select"]) # Update the filename and move files if necessary - update_filename_and_move_files(sender, cf_instance) + update_filename_and_move_files.delay(sender, cf_instance) @receiver(models.signals.post_delete, sender=CustomField) diff --git a/src/documents/tests/test_api_custom_fields.py b/src/documents/tests/test_api_custom_fields.py index 31dd14b88..b6e6c1342 100644 --- a/src/documents/tests/test_api_custom_fields.py +++ b/src/documents/tests/test_api_custom_fields.py @@ -4,6 +4,7 @@ from unittest.mock import ANY from django.contrib.auth.models import Permission from django.contrib.auth.models import User +from django.test import override_settings from rest_framework import status from rest_framework.test import APITestCase @@ -211,6 +212,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): ], ) + @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_custom_field_select_options_pruned(self): """ GIVEN: diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index c0070aa81..117a964ba 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -569,7 +569,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(generate_filename(doc), Path("document_apple.pdf")) # handler should not have been called - self.assertEqual(m.call_count, 0) + self.assertEqual(m.delay.call_count, 0) cf.extra_data = { "select_options": [ {"label": "aubergine", "id": "abc123"}, @@ -579,8 +579,8 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): } cf.save() self.assertEqual(generate_filename(doc), Path("document_aubergine.pdf")) - # handler should have been called - self.assertEqual(m.call_count, 1) + # handler should have been called via delay + self.assertEqual(m.delay.call_count, 1) class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, TestCase): diff --git a/src/documents/tests/test_index.py b/src/documents/tests/test_index.py index 2a41542e9..f216feedb 100644 --- a/src/documents/tests/test_index.py +++ b/src/documents/tests/test_index.py @@ -2,6 +2,7 @@ from datetime import datetime from unittest import mock from django.contrib.auth.models import User +from django.test import SimpleTestCase from django.test import TestCase from django.test import override_settings from django.utils.timezone import get_current_timezone @@ -127,3 +128,126 @@ class TestAutoComplete(DirectoriesMixin, TestCase): response = self.client.get("/api/documents/?query=added:yesterday") results = response.json()["results"] self.assertEqual(len(results), 0) + + +@override_settings(TIME_ZONE="UTC") +class TestRewriteNaturalDateKeywords(SimpleTestCase): + """ + Unit tests for rewrite_natural_date_keywords function. + """ + + def _rewrite_with_now(self, query: str, now_dt: datetime) -> str: + with mock.patch("documents.index.now", return_value=now_dt): + return index.rewrite_natural_date_keywords(query) + + def _assert_rewrite_contains( + self, + query: str, + now_dt: datetime, + *expected_fragments: str, + ) -> str: + result = self._rewrite_with_now(query, now_dt) + for fragment in expected_fragments: + self.assertIn(fragment, result) + return result + + def test_range_keywords(self): + """ + Test various different range keywords + """ + cases = [ + ( + "added:today", + datetime(2025, 7, 20, 15, 30, 45, tzinfo=timezone.utc), + ("added:[20250720", "TO 20250720"), + ), + ( + "added:yesterday", + datetime(2025, 7, 20, 15, 30, 45, tzinfo=timezone.utc), + ("added:[20250719", "TO 20250719"), + ), + ( + "added:this month", + datetime(2025, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + ("added:[20250701", "TO 20250731"), + ), + ( + "added:previous month", + datetime(2025, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + ("added:[20250601", "TO 20250630"), + ), + ( + "added:this year", + datetime(2025, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + ("added:[20250101", "TO 20250715"), + ), + ( + "added:previous year", + datetime(2025, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + ("added:[20240101", "TO 20241231"), + ), + # Previous quarter from July 15, 2025 is April-June. + ( + "added:previous quarter", + datetime(2025, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + ("added:[20250401", "TO 20250630"), + ), + # July 20, 2025 is a Sunday (weekday 6) so previous week is July 7-13. + ( + "added:previous week", + datetime(2025, 7, 20, 12, 0, 0, tzinfo=timezone.utc), + ("added:[20250707", "TO 20250713"), + ), + ] + + for query, now_dt, fragments in cases: + with self.subTest(query=query): + self._assert_rewrite_contains(query, now_dt, *fragments) + + def test_additional_fields(self): + fixed_now = datetime(2025, 7, 20, 15, 30, 45, tzinfo=timezone.utc) + # created + self._assert_rewrite_contains("created:today", fixed_now, "created:[20250720") + # modified + self._assert_rewrite_contains("modified:today", fixed_now, "modified:[20250720") + + def test_basic_syntax_variants(self): + """ + Test that quoting, casing, and multi-clause queries are parsed. + """ + fixed_now = datetime(2025, 7, 20, 15, 30, 45, tzinfo=timezone.utc) + + # quoted keywords + result1 = self._rewrite_with_now('added:"today"', fixed_now) + result2 = self._rewrite_with_now("added:'today'", fixed_now) + self.assertIn("added:[20250720", result1) + self.assertIn("added:[20250720", result2) + + # case insensitivity + for query in ("added:TODAY", "added:Today", "added:ToDaY"): + with self.subTest(case_variant=query): + self._assert_rewrite_contains(query, fixed_now, "added:[20250720") + + # multiple clauses + result = self._rewrite_with_now("added:today created:yesterday", fixed_now) + self.assertIn("added:[20250720", result) + self.assertIn("created:[20250719", result) + + def test_no_match(self): + """ + Test that queries without keywords are unchanged. + """ + query = "title:test content:example" + result = index.rewrite_natural_date_keywords(query) + self.assertEqual(query, result) + + @override_settings(TIME_ZONE="Pacific/Auckland") + def test_timezone_awareness(self): + """ + Test timezone conversion. + """ + # July 20, 2025 1:00 AM NZST = July 19, 2025 13:00 UTC + fixed_now = datetime(2025, 7, 20, 1, 0, 0, tzinfo=get_current_timezone()) + result = self._rewrite_with_now("added:today", fixed_now) + # Should convert to UTC properly + self.assertIn("added:[20250719", result) diff --git a/src/paperless_mail/templates/package-lock.json b/src/paperless_mail/templates/package-lock.json index e6c2db15a..38817d673 100644 --- a/src/paperless_mail/templates/package-lock.json +++ b/src/paperless_mail/templates/package-lock.json @@ -429,23 +429,21 @@ } }, "node_modules/glob": { - "version": "10.4.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.1.tgz", - "integrity": "sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dev": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, "funding": { "url": "https://github.com/sponsors/isaacs" } @@ -696,6 +694,12 @@ "node": ">= 6" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -1694,15 +1698,16 @@ "dev": true }, "glob": { - "version": "10.4.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.1.tgz", - "integrity": "sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", "dev": true, "requires": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" } }, @@ -1875,6 +1880,12 @@ "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", "dev": true }, + "package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true + }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",