diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 1ad06f12e..0d2302b72 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -42,7 +42,15 @@ body: id: logs attributes: label: Webserver logs - description: If available, post any logs from the web server related to your issue. + description: Logs from the web server related to your issue. + render: bash + validations: + required: true + - type: textarea + id: logs_browser + attributes: + label: Browser logs + description: Logs from the web browser related to your issue, if needed render: bash - type: input id: version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f075fbd9f..5b65245fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,15 +81,6 @@ jobs: matrix: python-version: ['3.8', '3.9', '3.10'] fail-fast: false - services: - tika: - image: ghcr.io/paperless-ngx/tika:latest - ports: - - "9998:9998/tcp" - gotenberg: - image: ghcr.io/p-h-a-i-l/gotenberg:7.6 - ports: - - "3000:3000/tcp" env: # Enable Tika end to end testing TIKA_LIVE: 1 @@ -107,6 +98,11 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + - + name: Start containers + run: | + docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet + docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach - name: Install pipenv run: | @@ -158,6 +154,12 @@ jobs: run: | cd src/ pipenv run coveralls --service=github + - + name: Stop containers + if: always() + run: | + docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml logs + docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml down tests-frontend: name: "Tests Frontend" @@ -482,10 +484,9 @@ jobs: - name: Upload release archive id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: shogo82148/actions-upload-release-asset@v1 with: + github_token: ${{ secrets.GITHUB_TOKEN }} upload_url: ${{ steps.create-release.outputs.upload_url }} asset_path: ./paperless-ngx.tar.xz asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz @@ -527,7 +528,7 @@ jobs: CURRENT_CHANGELOG=`tail --lines +2 changelog.md` echo -e "$CURRENT_CHANGELOG" >> changelog-new.md mv changelog-new.md changelog.md - pipenv run pre-commit --files changelog.md + pipenv run pre-commit run --files changelog.md git config --global user.name "github-actions" git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1bf47d14..c9f70ee81 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,8 +51,8 @@ repos: hooks: - id: add-trailing-comma exclude: "(migrations)" - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 hooks: - id: flake8 files: ^src/ @@ -63,7 +63,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/pyupgrade - rev: v3.1.0 + rev: v3.2.2 hooks: - id: pyupgrade exclude: "(migrations)" diff --git a/Dockerfile b/Dockerfile index 8b562e73e..b5b4d6e93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,34 +72,37 @@ COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/ # Packages need for running ARG RUNTIME_PACKAGES="\ + # Python + python3 \ + python3-pip \ + python3-setuptools \ + # General utils curl \ - file \ + # Docker specific + gosu \ + # Timezones support + tzdata \ # fonts for text file thumbnail generation fonts-liberation \ gettext \ ghostscript \ gnupg \ - gosu \ icc-profiles-free \ imagemagick \ - media-types \ + # Image processing liblept5 \ - libpq5 \ - libxml2 \ liblcms2-2 \ libtiff5 \ - libxslt1.1 \ libfreetype6 \ libwebp6 \ libopenjp2-7 \ libimagequant0 \ libraqm0 \ - libgnutls30 \ libjpeg62-turbo \ - python3 \ - python3-pip \ - python3-setuptools \ + # PostgreSQL + libpq5 \ postgresql-client \ + # MySQL / MariaDB mariadb-client \ # For Numpy libatlas3-base \ @@ -110,17 +113,23 @@ ARG RUNTIME_PACKAGES="\ tesseract-ocr-fra \ tesseract-ocr-ita \ tesseract-ocr-spa \ - # Suggested for OCRmyPDF - pngquant \ - # Suggested for pikepdf - jbig2dec \ - tzdata \ unpaper \ + pngquant \ + # pikepdf / qpdf + jbig2dec \ + libxml2 \ + libxslt1.1 \ + libgnutls30 \ # Mime type detection + file \ + libmagic1 \ + media-types \ zlib1g \ # Barcode splitter libzbar0 \ - poppler-utils" + poppler-utils \ + # RapidFuzz on armv7 + libatomic1" # Install basic runtime packages. # These change very infrequently diff --git a/README.md b/README.md index 8b1eb5e4a..9ae1443f4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx) [![Documentation Status](https://readthedocs.org/projects/paperless-ngx/badge/?version=latest)](https://paperless-ngx.readthedocs.io/en/latest/?badge=latest) [![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master) -[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/#paperless:adnidor.de) +[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)

@@ -105,6 +105,7 @@ Paperless has been around a while now, and people are starting to build stuff on - [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ng. - [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents. - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for Paperless. +- [Paperless Mobile](https://github.com/astubenbord/paperless-mobile): A modern, feature rich mobile application for Paperless. These projects also exist, but their status and compatibility with paperless-ngx is unknown. diff --git a/docker/compose/docker-compose.ci-test.yml b/docker/compose/docker-compose.ci-test.yml new file mode 100644 index 000000000..87bc8b7f2 --- /dev/null +++ b/docker/compose/docker-compose.ci-test.yml @@ -0,0 +1,22 @@ +# docker-compose file for running paperless testing with actual gotenberg +# and Tika containers for a more end to end test of the Tika related functionality +# Can be used locally or by the CI to start the nessecary containers with the +# correct networking for the tests + +version: "3.7" +services: + gotenberg: + image: docker.io/gotenberg/gotenberg:7.6 + hostname: gotenberg + container_name: gotenberg + network_mode: host + restart: unless-stopped + command: + - "gotenberg" + - "--chromium-disable-routes=true" + tika: + image: ghcr.io/paperless-ngx/tika:latest + hostname: tika + container_name: tika + network_mode: host + restart: unless-stopped diff --git a/docker/compose/docker-compose.mariadb-tika.yml b/docker/compose/docker-compose.mariadb-tika.yml index b3699e4d7..22f69ba4f 100644 --- a/docker/compose/docker-compose.mariadb-tika.yml +++ b/docker/compose/docker-compose.mariadb-tika.yml @@ -77,8 +77,8 @@ services: PAPERLESS_REDIS: redis://broker:6379 PAPERLESS_DBENGINE: mariadb PAPERLESS_DBHOST: db - PAPERLESS_DBUSER: paperless - PAPERLESS_DBPASSWORD: paperless + PAPERLESS_DBUSER: paperless # only needed if non-default username + PAPERLESS_DBPASS: paperless # only needed if non-default password PAPERLESS_DBPORT: 3306 PAPERLESS_TIKA_ENABLED: 1 PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000 diff --git a/docker/compose/docker-compose.mariadb.yml b/docker/compose/docker-compose.mariadb.yml index 8cb452464..a18e0fb3e 100644 --- a/docker/compose/docker-compose.mariadb.yml +++ b/docker/compose/docker-compose.mariadb.yml @@ -71,8 +71,8 @@ services: PAPERLESS_REDIS: redis://broker:6379 PAPERLESS_DBENGINE: mariadb PAPERLESS_DBHOST: db - PAPERLESS_DBUSER: paperless - PAPERLESS_DBPASSWORD: paperless + PAPERLESS_DBUSER: paperless # only needed if non-default username + PAPERLESS_DBPASS: paperless # only needed if non-default password PAPERLESS_DBPORT: 3306 diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index d1107feca..f227e18d8 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -137,8 +137,7 @@ initialize() { install_languages() { echo "Installing languages..." - local -r langs="$1" - read -ra langs <<<"$langs" + read -ra langs <<<"$1" # Check that it is not empty if [ ${#langs[@]} -eq 0 ]; then diff --git a/docs/administration.rst b/docs/administration.rst index ed2075a48..2984057ed 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -447,6 +447,14 @@ command: The command takes no arguments and processes all your mail accounts and rules. + .. note:: + + As of October 2022 Microsoft no longer supports IMAP authentication for Exchange + servers, thus Exchange is no longer supported until a solution is implemented in + the Python IMAP library used by Paperless. See `learn.microsoft.com`_ + +.. _learn.microsoft.com: https://learn.microsoft.com/en-us/exchange/clients-and-mobile-in-exchange-online/deprecation-of-basic-authentication-exchange-online + .. _utilities-archiver: Creating archived documents diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index eda2ca259..1fe3e3685 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -149,6 +149,9 @@ which will in turn call `pdf2pdfocr.py`_ on your document, which will then overwrite the file with an OCR'd version of the file and exit. At which point, the consumption process will begin with the newly modified file. +The script's stdout and stderr will be logged line by line to the webserver log, along +with the exit code of the script. + .. _pdf2pdfocr.py: https://github.com/LeoFCardoso/pdf2pdfocr .. _advanced-post_consume_script: @@ -178,6 +181,10 @@ example, you can take a look at `post-consumption-example.sh`_ in this project. The post consumption script cannot cancel the consumption process. +The script's stdout and stderr will be logged line by line to the webserver log, along +with the exit code of the script. + + Docker ------ Assumed you have ``/home/foo/paperless-ngx/scripts/post-consumption-example.sh``. @@ -417,3 +424,24 @@ For example, using Docker Compose: # ... volumes: - /path/to/my/scripts:/custom-cont-init.d:ro + +.. _advanced-mysql-caveats: + +MySQL Caveats +############# + +Case Sensitivity +================ + +The database interface does not provide a method to configure a MySQL database to +be case sensitive. This would prevent a user from creating a tag ``Name`` and ``NAME`` +as they are considered the same. + +Per Django documentation, to enable this requires manual intervention. To enable +case sensetive tables, you can execute the following command against each table: + +``ALTER TABLE CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`` + +You can also set the default for new tables (this does NOT affect existing tables) with: + +``ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`` diff --git a/docs/changelog.md b/docs/changelog.md index 1caa00220..2a037aebe 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,189 @@ # Changelog +## paperless-ngx 1.10.0 + +### Features + +- Feature: Capture stdout \& stderr of the pre/post consume scripts [@stumpylog](https://github.com/stumpylog) ([#1967](https://github.com/paperless-ngx/paperless-ngx/pull/1967)) +- Feature: Allow running custom container initialization scripts [@stumpylog](https://github.com/stumpylog) ([#1838](https://github.com/paperless-ngx/paperless-ngx/pull/1838)) +- Feature: Add more file name formatting options [@stumpylog](https://github.com/stumpylog) ([#1906](https://github.com/paperless-ngx/paperless-ngx/pull/1906)) +- Feature: 1.9.2 UI tweaks [@shamoon](https://github.com/shamoon) ([#1886](https://github.com/paperless-ngx/paperless-ngx/pull/1886)) +- Feature: Optional celery monitoring with Flower [@stumpylog](https://github.com/stumpylog) ([#1810](https://github.com/paperless-ngx/paperless-ngx/pull/1810)) +- Feature: Save pending tasks for frontend [@stumpylog](https://github.com/stumpylog) ([#1816](https://github.com/paperless-ngx/paperless-ngx/pull/1816)) +- Feature: Improved processing for automatic matching [@stumpylog](https://github.com/stumpylog) ([#1609](https://github.com/paperless-ngx/paperless-ngx/pull/1609)) +- Feature: Transition to celery for background tasks [@stumpylog](https://github.com/stumpylog) ([#1648](https://github.com/paperless-ngx/paperless-ngx/pull/1648)) +- Feature: UI Welcome Tour [@shamoon](https://github.com/shamoon) ([#1644](https://github.com/paperless-ngx/paperless-ngx/pull/1644)) +- Feature: slim sidebar [@shamoon](https://github.com/shamoon) ([#1641](https://github.com/paperless-ngx/paperless-ngx/pull/1641)) +- change default matching algo to auto and move to constant [@NiFNi](https://github.com/NiFNi) ([#1754](https://github.com/paperless-ngx/paperless-ngx/pull/1754)) +- Feature: Enable end to end Tika testing in CI [@stumpylog](https://github.com/stumpylog) ([#1757](https://github.com/paperless-ngx/paperless-ngx/pull/1757)) +- Feature: frontend update checking settings [@shamoon](https://github.com/shamoon) ([#1692](https://github.com/paperless-ngx/paperless-ngx/pull/1692)) +- Feature: Upgrade to qpdf 11, pikepdf 6 \& ocrmypdf 14 [@stumpylog](https://github.com/stumpylog) ([#1642](https://github.com/paperless-ngx/paperless-ngx/pull/1642)) + +### Bug Fixes + +- Bugfix: Fix created_date being a string [@stumpylog](https://github.com/stumpylog) ([#2023](https://github.com/paperless-ngx/paperless-ngx/pull/2023)) +- Bugfix: Fixes an issue with mixed text and images when redoing OCR [@stumpylog](https://github.com/stumpylog) ([#2017](https://github.com/paperless-ngx/paperless-ngx/pull/2017)) +- Bugfix: Always re-try barcodes with pdf2image [@stumpylog](https://github.com/stumpylog) ([#1953](https://github.com/paperless-ngx/paperless-ngx/pull/1953)) +- Fix: using `CONSUMER_SUBDIRS_AS_TAGS` causes failure with Celery in `dev` [@shamoon](https://github.com/shamoon) ([#1942](https://github.com/paperless-ngx/paperless-ngx/pull/1942)) +- Fix mail consumption broken in `dev` after move to celery [@shamoon](https://github.com/shamoon) ([#1934](https://github.com/paperless-ngx/paperless-ngx/pull/1934)) +- Bugfix: Prevent file handling from running with stale data [@stumpylog](https://github.com/stumpylog) ([#1905](https://github.com/paperless-ngx/paperless-ngx/pull/1905)) +- Chore: Reduce nuisance CI test failures [@stumpylog](https://github.com/stumpylog) ([#1922](https://github.com/paperless-ngx/paperless-ngx/pull/1922)) +- Bugfix: Unintentional deletion of feature tagged Docker images [@stumpylog](https://github.com/stumpylog) ([#1896](https://github.com/paperless-ngx/paperless-ngx/pull/1896)) +- Fix: independent control of saved views [@shamoon](https://github.com/shamoon) ([#1868](https://github.com/paperless-ngx/paperless-ngx/pull/1868)) +- Fix: frontend relative date searches [@shamoon](https://github.com/shamoon) ([#1865](https://github.com/paperless-ngx/paperless-ngx/pull/1865)) +- Chore: Fixes pipenv issues [@stumpylog](https://github.com/stumpylog) ([#1873](https://github.com/paperless-ngx/paperless-ngx/pull/1873)) +- Bugfix: Handle password protected PDFs during barcode detection [@stumpylog](https://github.com/stumpylog) ([#1858](https://github.com/paperless-ngx/paperless-ngx/pull/1858)) +- Fix: Allows configuring barcodes with pdf2image instead of pikepdf [@stumpylog](https://github.com/stumpylog) ([#1857](https://github.com/paperless-ngx/paperless-ngx/pull/1857)) +- Bugfix: Reverts the change around skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1829](https://github.com/paperless-ngx/paperless-ngx/pull/1829)) +- Fix: missing loadViewConfig breaks loading saved view [@shamoon](https://github.com/shamoon) ([#1792](https://github.com/paperless-ngx/paperless-ngx/pull/1792)) +- Bugfix: Fallback to pdf2image if pikepdf fails [@stumpylog](https://github.com/stumpylog) ([#1745](https://github.com/paperless-ngx/paperless-ngx/pull/1745)) +- Fix: creating new storage path on document edit fails to update menu [@shamoon](https://github.com/shamoon) ([#1777](https://github.com/paperless-ngx/paperless-ngx/pull/1777)) +- Bugfix: Files containing barcodes uploaded via web are not consumed after splitting [@stumpylog](https://github.com/stumpylog) ([#1762](https://github.com/paperless-ngx/paperless-ngx/pull/1762)) +- Bugfix: Fix email labeling for non-Gmail servers [@stumpylog](https://github.com/stumpylog) ([#1755](https://github.com/paperless-ngx/paperless-ngx/pull/1755)) +- Fix: allow preview for .csv files [@shamoon](https://github.com/shamoon) ([#1744](https://github.com/paperless-ngx/paperless-ngx/pull/1744)) +- Bugfix: csv recognition by consumer [@bin101](https://github.com/bin101) ([#1726](https://github.com/paperless-ngx/paperless-ngx/pull/1726)) +- Bugfix: Include document title when a duplicate is detected [@stumpylog](https://github.com/stumpylog) ([#1696](https://github.com/paperless-ngx/paperless-ngx/pull/1696)) +- Bugfix: Set MySql charset [@stumpylog](https://github.com/stumpylog) ([#1687](https://github.com/paperless-ngx/paperless-ngx/pull/1687)) +- Mariadb compose files should use `PAPERLESS_DBPASS` [@shamoon](https://github.com/shamoon) ([#1683](https://github.com/paperless-ngx/paperless-ngx/pull/1683)) + +### Documentation + +- Documentation: Update MariaDB docs to note some potential issues [@stumpylog](https://github.com/stumpylog) ([#2016](https://github.com/paperless-ngx/paperless-ngx/pull/2016)) +- Documentation: Add note re MS exchange servers [@shamoon](https://github.com/shamoon) ([#1780](https://github.com/paperless-ngx/paperless-ngx/pull/1780)) +- Chore: Updates Gotenberg versions [@stumpylog](https://github.com/stumpylog) ([#1768](https://github.com/paperless-ngx/paperless-ngx/pull/1768)) +- Documentation: Tweak LinuxServer [@stumpylog](https://github.com/stumpylog) ([#1761](https://github.com/paperless-ngx/paperless-ngx/pull/1761)) +- Documentation: Adds troubleshooting note about Kubernetes and ports [@stumpylog](https://github.com/stumpylog) ([#1731](https://github.com/paperless-ngx/paperless-ngx/pull/1731)) +- Documentation: LinuxServer.io Migration [@stumpylog](https://github.com/stumpylog) ([#1733](https://github.com/paperless-ngx/paperless-ngx/pull/1733)) +- [Documentation] Add v1.9.2 changelog [@github-actions](https://github.com/github-actions) ([#1671](https://github.com/paperless-ngx/paperless-ngx/pull/1671)) + +### Maintenance + +- Bump tj-actions/changed-files from 32 to 34 [@dependabot](https://github.com/dependabot) ([#1915](https://github.com/paperless-ngx/paperless-ngx/pull/1915)) +- Chore: Fix `dev` trying to build Pillow or lxml [@stumpylog](https://github.com/stumpylog) ([#1909](https://github.com/paperless-ngx/paperless-ngx/pull/1909)) +- Chore: Fixes pipenv issues [@stumpylog](https://github.com/stumpylog) ([#1873](https://github.com/paperless-ngx/paperless-ngx/pull/1873)) +- Chore: Simplified registry cleanup [@stumpylog](https://github.com/stumpylog) ([#1812](https://github.com/paperless-ngx/paperless-ngx/pull/1812)) +- Chore: Fixing deprecated workflow commands [@stumpylog](https://github.com/stumpylog) ([#1786](https://github.com/paperless-ngx/paperless-ngx/pull/1786)) +- Chore: Python library update + test fixes [@stumpylog](https://github.com/stumpylog) ([#1773](https://github.com/paperless-ngx/paperless-ngx/pull/1773)) +- Chore: Updates Gotenberg versions [@stumpylog](https://github.com/stumpylog) ([#1768](https://github.com/paperless-ngx/paperless-ngx/pull/1768)) +- Bump leonsteinhaeuser/project-beta-automations from 1.3.0 to 2.0.1 [@dependabot](https://github.com/dependabot) ([#1703](https://github.com/paperless-ngx/paperless-ngx/pull/1703)) +- Bump tj-actions/changed-files from 29.0.2 to 31.0.2 [@dependabot](https://github.com/dependabot) ([#1702](https://github.com/paperless-ngx/paperless-ngx/pull/1702)) +- Bump actions/checkout from 2 to 3 [@dependabot](https://github.com/dependabot) ([#1704](https://github.com/paperless-ngx/paperless-ngx/pull/1704)) +- Bump actions/setup-python from 3 to 4 [@dependabot](https://github.com/dependabot) ([#1705](https://github.com/paperless-ngx/paperless-ngx/pull/1705)) + +### Dependencies + +

+31 changes + +- Bugfix: Downgrade cryptography for armv7 compatibility [@stumpylog](https://github.com/stumpylog) ([#1954](https://github.com/paperless-ngx/paperless-ngx/pull/1954)) +- Chore: Bulk library updates + loosen restrictions [@stumpylog](https://github.com/stumpylog) ([#1949](https://github.com/paperless-ngx/paperless-ngx/pull/1949)) +- Bump tj-actions/changed-files from 32 to 34 [@dependabot](https://github.com/dependabot) ([#1915](https://github.com/paperless-ngx/paperless-ngx/pull/1915)) +- Bump scikit-learn from 1.1.2 to 1.1.3 [@dependabot](https://github.com/dependabot) ([#1903](https://github.com/paperless-ngx/paperless-ngx/pull/1903)) +- Bump angular packages as bundle [@dependabot](https://github.com/dependabot) ([#1910](https://github.com/paperless-ngx/paperless-ngx/pull/1910)) +- Bump ngx-ui-tour-ng-bootstrap from 11.0.0 to 11.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1911](https://github.com/paperless-ngx/paperless-ngx/pull/1911)) +- Bump jest-environment-jsdom from 29.1.2 to 29.2.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1914](https://github.com/paperless-ngx/paperless-ngx/pull/1914)) +- Bump pillow from 9.2.0 to 9.3.0 [@dependabot](https://github.com/dependabot) ([#1904](https://github.com/paperless-ngx/paperless-ngx/pull/1904)) +- Bump pytest from 7.1.3 to 7.2.0 [@dependabot](https://github.com/dependabot) ([#1902](https://github.com/paperless-ngx/paperless-ngx/pull/1902)) +- Bump tox from 3.26.0 to 3.27.0 [@dependabot](https://github.com/dependabot) ([#1901](https://github.com/paperless-ngx/paperless-ngx/pull/1901)) +- Bump zipp from 3.9.0 to 3.10.0 [@dependabot](https://github.com/dependabot) ([#1860](https://github.com/paperless-ngx/paperless-ngx/pull/1860)) +- Bump pytest-env from 0.6.2 to 0.8.1 [@dependabot](https://github.com/dependabot) ([#1859](https://github.com/paperless-ngx/paperless-ngx/pull/1859)) +- Bump sphinx from 5.2.3 to 5.3.0 [@dependabot](https://github.com/dependabot) ([#1817](https://github.com/paperless-ngx/paperless-ngx/pull/1817)) +- Chore: downgrade channels-redis [@stumpylog](https://github.com/stumpylog) ([#1802](https://github.com/paperless-ngx/paperless-ngx/pull/1802)) +- Chore: Update to qpdf 11.1.1 and update backend libraries [@stumpylog](https://github.com/stumpylog) ([#1749](https://github.com/paperless-ngx/paperless-ngx/pull/1749)) +- Bump myst-parser from 0.18.0 to 0.18.1 [@dependabot](https://github.com/dependabot) ([#1738](https://github.com/paperless-ngx/paperless-ngx/pull/1738)) +- Bump leonsteinhaeuser/project-beta-automations from 1.3.0 to 2.0.1 [@dependabot](https://github.com/dependabot) ([#1703](https://github.com/paperless-ngx/paperless-ngx/pull/1703)) +- Bump tj-actions/changed-files from 29.0.2 to 31.0.2 [@dependabot](https://github.com/dependabot) ([#1702](https://github.com/paperless-ngx/paperless-ngx/pull/1702)) +- Bump actions/checkout from 2 to 3 [@dependabot](https://github.com/dependabot) ([#1704](https://github.com/paperless-ngx/paperless-ngx/pull/1704)) +- Bump actions/setup-python from 3 to 4 [@dependabot](https://github.com/dependabot) ([#1705](https://github.com/paperless-ngx/paperless-ngx/pull/1705)) +- Bump rxjs from 7.5.6 to 7.5.7 in /src-ui [@dependabot](https://github.com/dependabot) ([#1720](https://github.com/paperless-ngx/paperless-ngx/pull/1720)) +- Bump uuid from 8.3.2 to 9.0.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1716](https://github.com/paperless-ngx/paperless-ngx/pull/1716)) +- Bump ng2-pdf-viewer from 9.1.0 to 9.1.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1717](https://github.com/paperless-ngx/paperless-ngx/pull/1717)) +- Bump ngx-color from 8.0.2 to 8.0.3 in /src-ui [@dependabot](https://github.com/dependabot) ([#1715](https://github.com/paperless-ngx/paperless-ngx/pull/1715)) +- Bump concurrently from 7.3.0 to 7.4.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1719](https://github.com/paperless-ngx/paperless-ngx/pull/1719)) +- Bump [@types/node from 18.7.14 to 18.7.23 in /src-ui @dependabot](https://github.com/types/node from 18.7.14 to 18.7.23 in /src-ui @dependabot) ([#1718](https://github.com/paperless-ngx/paperless-ngx/pull/1718)) +- Bump jest-environment-jsdom from 29.0.1 to 29.1.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1714](https://github.com/paperless-ngx/paperless-ngx/pull/1714)) +- Bump [@angular/cli @angular/core @dependabot](https://github.com/angular/cli @angular/core @dependabot) ([#1708](https://github.com/paperless-ngx/paperless-ngx/pull/1708)) +- Bump cypress from 10.7.0 to 10.9.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1707](https://github.com/paperless-ngx/paperless-ngx/pull/1707)) +- Bump bootstrap from 5.2.0 to 5.2.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1710](https://github.com/paperless-ngx/paperless-ngx/pull/1710)) +- Bump typescript from 4.7.4 to 4.8.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1706](https://github.com/paperless-ngx/paperless-ngx/pull/1706)) +
+ +### All App Changes + +- Add info that re-do OCR doesnt automatically refresh content [@shamoon](https://github.com/shamoon) ([#2025](https://github.com/paperless-ngx/paperless-ngx/pull/2025)) +- Bugfix: Fix created_date being a string [@stumpylog](https://github.com/stumpylog) ([#2023](https://github.com/paperless-ngx/paperless-ngx/pull/2023)) +- Bugfix: Fixes an issue with mixed text and images when redoing OCR [@stumpylog](https://github.com/stumpylog) ([#2017](https://github.com/paperless-ngx/paperless-ngx/pull/2017)) +- Bugfix: Don't allow exceptions during date parsing to fail consume [@stumpylog](https://github.com/stumpylog) ([#1998](https://github.com/paperless-ngx/paperless-ngx/pull/1998)) +- Feature: Capture stdout \& stderr of the pre/post consume scripts [@stumpylog](https://github.com/stumpylog) ([#1967](https://github.com/paperless-ngx/paperless-ngx/pull/1967)) +- Bugfix: Always re-try barcodes with pdf2image [@stumpylog](https://github.com/stumpylog) ([#1953](https://github.com/paperless-ngx/paperless-ngx/pull/1953)) +- Fix: using `CONSUMER_SUBDIRS_AS_TAGS` causes failure with Celery in `dev` [@shamoon](https://github.com/shamoon) ([#1942](https://github.com/paperless-ngx/paperless-ngx/pull/1942)) +- Fix mail consumption broken in `dev` after move to celery [@shamoon](https://github.com/shamoon) ([#1934](https://github.com/paperless-ngx/paperless-ngx/pull/1934)) +- Bugfix: Prevent file handling from running with stale data [@stumpylog](https://github.com/stumpylog) ([#1905](https://github.com/paperless-ngx/paperless-ngx/pull/1905)) +- Chore: Reduce nuisance CI test failures [@stumpylog](https://github.com/stumpylog) ([#1922](https://github.com/paperless-ngx/paperless-ngx/pull/1922)) +- Bump scikit-learn from 1.1.2 to 1.1.3 [@dependabot](https://github.com/dependabot) ([#1903](https://github.com/paperless-ngx/paperless-ngx/pull/1903)) +- Bump angular packages as bundle [@dependabot](https://github.com/dependabot) ([#1910](https://github.com/paperless-ngx/paperless-ngx/pull/1910)) +- Bump ngx-ui-tour-ng-bootstrap from 11.0.0 to 11.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1911](https://github.com/paperless-ngx/paperless-ngx/pull/1911)) +- Bump jest-environment-jsdom from 29.1.2 to 29.2.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1914](https://github.com/paperless-ngx/paperless-ngx/pull/1914)) +- Feature: Add more file name formatting options [@stumpylog](https://github.com/stumpylog) ([#1906](https://github.com/paperless-ngx/paperless-ngx/pull/1906)) +- Bump pillow from 9.2.0 to 9.3.0 [@dependabot](https://github.com/dependabot) ([#1904](https://github.com/paperless-ngx/paperless-ngx/pull/1904)) +- Bump pytest from 7.1.3 to 7.2.0 [@dependabot](https://github.com/dependabot) ([#1902](https://github.com/paperless-ngx/paperless-ngx/pull/1902)) +- Bump tox from 3.26.0 to 3.27.0 [@dependabot](https://github.com/dependabot) ([#1901](https://github.com/paperless-ngx/paperless-ngx/pull/1901)) +- directly use rapidfuzz [@maxbachmann](https://github.com/maxbachmann) ([#1899](https://github.com/paperless-ngx/paperless-ngx/pull/1899)) +- Feature: 1.9.2 UI tweaks [@shamoon](https://github.com/shamoon) ([#1886](https://github.com/paperless-ngx/paperless-ngx/pull/1886)) +- Bump zipp from 3.9.0 to 3.10.0 [@dependabot](https://github.com/dependabot) ([#1860](https://github.com/paperless-ngx/paperless-ngx/pull/1860)) +- Fix: independent control of saved views [@shamoon](https://github.com/shamoon) ([#1868](https://github.com/paperless-ngx/paperless-ngx/pull/1868)) +- Fix: frontend relative date searches [@shamoon](https://github.com/shamoon) ([#1865](https://github.com/paperless-ngx/paperless-ngx/pull/1865)) +- Django error W003 - MariaDB may not allow unique CharFields to have a max_length > 255. [@Sblop](https://github.com/Sblop) ([#1881](https://github.com/paperless-ngx/paperless-ngx/pull/1881)) +- Bump pytest-env from 0.6.2 to 0.8.1 [@dependabot](https://github.com/dependabot) ([#1859](https://github.com/paperless-ngx/paperless-ngx/pull/1859)) +- Fix: Allows configuring barcodes with pdf2image instead of pikepdf [@stumpylog](https://github.com/stumpylog) ([#1857](https://github.com/paperless-ngx/paperless-ngx/pull/1857)) +- Feature: Save pending tasks for frontend [@stumpylog](https://github.com/stumpylog) ([#1816](https://github.com/paperless-ngx/paperless-ngx/pull/1816)) +- Bugfix: Reverts the change around skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1829](https://github.com/paperless-ngx/paperless-ngx/pull/1829)) +- Bump sphinx from 5.2.3 to 5.3.0 [@dependabot](https://github.com/dependabot) ([#1817](https://github.com/paperless-ngx/paperless-ngx/pull/1817)) +- Fix: missing loadViewConfig breaks loading saved view [@shamoon](https://github.com/shamoon) ([#1792](https://github.com/paperless-ngx/paperless-ngx/pull/1792)) +- Bugfix: Fallback to pdf2image if pikepdf fails [@stumpylog](https://github.com/stumpylog) ([#1745](https://github.com/paperless-ngx/paperless-ngx/pull/1745)) +- Fix: creating new storage path on document edit fails to update menu [@shamoon](https://github.com/shamoon) ([#1777](https://github.com/paperless-ngx/paperless-ngx/pull/1777)) +- Chore: Python library update + test fixes [@stumpylog](https://github.com/stumpylog) ([#1773](https://github.com/paperless-ngx/paperless-ngx/pull/1773)) +- Feature: Improved processing for automatic matching [@stumpylog](https://github.com/stumpylog) ([#1609](https://github.com/paperless-ngx/paperless-ngx/pull/1609)) +- Feature: Transition to celery for background tasks [@stumpylog](https://github.com/stumpylog) ([#1648](https://github.com/paperless-ngx/paperless-ngx/pull/1648)) +- Feature: UI Welcome Tour [@shamoon](https://github.com/shamoon) ([#1644](https://github.com/paperless-ngx/paperless-ngx/pull/1644)) +- Feature: slim sidebar [@shamoon](https://github.com/shamoon) ([#1641](https://github.com/paperless-ngx/paperless-ngx/pull/1641)) +- Bugfix: Files containing barcodes uploaded via web are not consumed after splitting [@stumpylog](https://github.com/stumpylog) ([#1762](https://github.com/paperless-ngx/paperless-ngx/pull/1762)) +- change default matching algo to auto and move to constant [@NiFNi](https://github.com/NiFNi) ([#1754](https://github.com/paperless-ngx/paperless-ngx/pull/1754)) +- Bugfix: Fix email labeling for non-Gmail servers [@stumpylog](https://github.com/stumpylog) ([#1755](https://github.com/paperless-ngx/paperless-ngx/pull/1755)) +- Feature: frontend update checking settings [@shamoon](https://github.com/shamoon) ([#1692](https://github.com/paperless-ngx/paperless-ngx/pull/1692)) +- Fix: allow preview for .csv files [@shamoon](https://github.com/shamoon) ([#1744](https://github.com/paperless-ngx/paperless-ngx/pull/1744)) +- Bump myst-parser from 0.18.0 to 0.18.1 [@dependabot](https://github.com/dependabot) ([#1738](https://github.com/paperless-ngx/paperless-ngx/pull/1738)) +- Bugfix: csv recognition by consumer [@bin101](https://github.com/bin101) ([#1726](https://github.com/paperless-ngx/paperless-ngx/pull/1726)) +- Bugfix: Include document title when a duplicate is detected [@stumpylog](https://github.com/stumpylog) ([#1696](https://github.com/paperless-ngx/paperless-ngx/pull/1696)) +- Bump rxjs from 7.5.6 to 7.5.7 in /src-ui [@dependabot](https://github.com/dependabot) ([#1720](https://github.com/paperless-ngx/paperless-ngx/pull/1720)) +- Bump uuid from 8.3.2 to 9.0.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1716](https://github.com/paperless-ngx/paperless-ngx/pull/1716)) +- Bump ng2-pdf-viewer from 9.1.0 to 9.1.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1717](https://github.com/paperless-ngx/paperless-ngx/pull/1717)) +- Bump ngx-color from 8.0.2 to 8.0.3 in /src-ui [@dependabot](https://github.com/dependabot) ([#1715](https://github.com/paperless-ngx/paperless-ngx/pull/1715)) +- Bump concurrently from 7.3.0 to 7.4.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1719](https://github.com/paperless-ngx/paperless-ngx/pull/1719)) +- Bump [@types/node from 18.7.14 to 18.7.23 in /src-ui @dependabot](https://github.com/types/node from 18.7.14 to 18.7.23 in /src-ui @dependabot) ([#1718](https://github.com/paperless-ngx/paperless-ngx/pull/1718)) +- Bump jest-environment-jsdom from 29.0.1 to 29.1.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1714](https://github.com/paperless-ngx/paperless-ngx/pull/1714)) +- Bump [@angular/cli @angular/core @dependabot](https://github.com/angular/cli @angular/core @dependabot) ([#1708](https://github.com/paperless-ngx/paperless-ngx/pull/1708)) +- Bump cypress from 10.7.0 to 10.9.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1707](https://github.com/paperless-ngx/paperless-ngx/pull/1707)) +- Bump bootstrap from 5.2.0 to 5.2.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1710](https://github.com/paperless-ngx/paperless-ngx/pull/1710)) +- Bump typescript from 4.7.4 to 4.8.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1706](https://github.com/paperless-ngx/paperless-ngx/pull/1706)) +- Bugfix: Set MySql charset [@stumpylog](https://github.com/stumpylog) ([#1687](https://github.com/paperless-ngx/paperless-ngx/pull/1687)) + +## paperless-ngx 1.9.2 + +### Bug Fixes + +- Bugfix: Allow PAPERLESS_OCR_CLEAN=none [@shamoon](https://github.com/shamoon) ([#1670](https://github.com/paperless-ngx/paperless-ngx/pull/1670)) + +### All App Changes + +- Chore: Bumps version numbers to 1.9.2 [@stumpylog](https://github.com/stumpylog) ([#1666](https://github.com/paperless-ngx/paperless-ngx/pull/1666)) + ## paperless-ngx 1.9.1 +### Notes + +- Version 1.9.1 incorrectly displays the version string as 1.9.0 + ### Bug Fixes - Bugfix: Fixes missing OCR mode skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1645](https://github.com/paperless-ngx/paperless-ngx/pull/1645)) diff --git a/docs/configuration.rst b/docs/configuration.rst index 2523b0f5d..f9e2c8be4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -38,8 +38,14 @@ PAPERLESS_REDIS= PAPERLESS_DBENGINE= Optional, gives the ability to choose Postgres or MariaDB for database engine. Available options are `postgresql` and `mariadb`. + Default is `postgresql`. + .. warning:: + + Using MariaDB comes with some caveats. See :ref:`advanced-mysql-caveats` for details. + + PAPERLESS_DBHOST= By default, sqlite is used as the database backend. This can be changed here. diff --git a/docs/setup.rst b/docs/setup.rst index 15f16d193..db96c6b58 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -648,13 +648,56 @@ Migration to paperless-ngx is then performed in a few simple steps: 10. Optionally, follow the instructions below to migrate your existing data to PostgreSQL. +Migrating from LinuxServer.io Docker Image +========================================== + +As with any upgrades and large changes, it is highly recommended to create a backup before +starting. This assumes the image was running using Docker Compose, but the instructions +are translatable to Docker commands as well. + +1. Stop and remove the paperless container +2. If using an external database, stop the container +3. Update Redis configuration + + a) If ``REDIS_URL`` is already set, change it to ``PAPERLESS_REDIS`` and continue + to step 4. + b) Otherwise, in the ``docker-compose.yml`` add a new service for Redis, + following `the example compose files `_ + c) Set the environment variable ``PAPERLESS_REDIS`` so it points to the new Redis container + +4. Update user mapping + + a) If set, change the environment variable ``PUID`` to ``USERMAP_UID`` + b) If set, change the environment variable ``PGID`` to ``USERMAP_GID`` + +5. Update configuration paths + + a) Set the environment variable ``PAPERLESS_DATA_DIR`` + to ``/config`` + +6. Update media paths + + a) Set the environment variable ``PAPERLESS_MEDIA_ROOT`` + to ``/data/media`` + +7. Update timezone + + a) Set the environment variable ``PAPERLESS_TIME_ZONE`` + to the same value as ``TZ`` + +8. Modify the ``image:`` to point to ``ghcr.io/paperless-ngx/paperless-ngx:latest`` or + a specific version if preferred. + +9. Start the containers as before, using ``docker-compose``. + .. _setup-sqlite_to_psql: -Moving data from SQLite to PostgreSQL -===================================== +Moving data from SQLite to PostgreSQL or MySQL/MariaDB +====================================================== -Moving your data from SQLite to PostgreSQL is done via executing a series of django -management commands as below. +Moving your data from SQLite to PostgreSQL or MySQL/MariaDB is done via executing a series of django +management commands as below. The commands below use PostgreSQL, but are applicable to MySQL/MariaDB +with the .. caution:: @@ -671,6 +714,11 @@ management commands as below. and filenames (1024 characters). If you have data in these fields that surpasses these limits, migration to PostgreSQL is not possible and will fail with an error. +.. warning:: + + MySQL is case insensitive by default, treating values like "Name" and "NAME" as identical. + See :ref:`advanced-mysql-caveats` for details. + 1. Stop paperless, if it is running. 2. Tell paperless to use PostgreSQL: diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 644a38552..11736cf71 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -321,3 +321,12 @@ Uploading or consuming multiple files at once results in many workers attempting Consider changing to the PostgreSQL database if you will be processing many documents at once often. Otherwise, try tweaking the ``PAPERLESS_DB_TIMEOUT`` setting to allow more time for the database to unlock. This may have minor performance implications. + + +gunicorn fails to start with "is not a valid port number" +######################################################### + +You are likely running using Kubernetes, which automatically creates an environment variable named `${serviceName}_PORT`. +This is the same environment variable which is used by Paperless to optionally change the port gunicorn listens on. + +To fix this, set `PAPERLESS_PORT` again to your desired port, or the default of 8000. diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index daca389d1..0f38dd28f 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1957,8 +1957,8 @@ 391 - - Redo OCR operation will begin in the background. + + Redo OCR operation will begin in the background. Close and re-open or reload this document after the operation has completed to see new content. src/app/components/document-detail/document-detail.component.ts 494 diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 41bd50970..079c562b9 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -17,7 +17,7 @@ -