Merge remote-tracking branch 'upstream/master'

docker-compose.env.example

@@ -1,38 +1,22 @@
 # Environment variables to set for Paperless
-# Commented out variables will be replaced by a default within Paperless.
+# Commented out variables will be replaced with a default within Paperless.
+#
+# In addition to what you see here, you can also define any values you find in
+# paperless.conf.example here.  Values like:
+#
+# * PAPERLESS_PASSPHRASE
+# * PAPERLESS_CONSUMPTION_DIR
+# * PAPERLESS_CONSUME_MAIL_HOST
+#
+# ...are all explained in that file but can be defined here, since the Docker
+# installation doesn't make use of paperless.conf.
 
-# Passphrase Paperless uses to encrypt and decrypt your documents, if you want
-# encryption at all.
-# PAPERLESS_PASSPHRASE=CHANGE_ME
 
-# The amount of threads to use for text recognition
+# Additional languages to install for text recognition.  Note that this is
-# PAPERLESS_OCR_THREADS=4
+# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
-
+# default language used when guessing the language from the OCR output.
-# Additional languages to install for text recognition
 # PAPERLESS_OCR_LANGUAGES=deu ita
 
 # You can change the default user and group id to a custom one
 # USERMAP_UID=1000
 # USERMAP_GID=1000
-
-###############################################################################
-####                         Mail Consumption                              ####
-###############################################################################
-
-# These values are required if you want paperless to check a particular email
-# box every 10 minutes and attempt to consume documents from there.  If you
-# don't define a HOST, mail checking will just be disabled.
-# Don't use quotes after = or it will crash your docker
-# PAPERLESS_CONSUME_MAIL_HOST=
-# PAPERLESS_CONSUME_MAIL_PORT=
-# PAPERLESS_CONSUME_MAIL_USER=
-# PAPERLESS_CONSUME_MAIL_PASS=
-
-# Override the default IMAP inbox here. If it's not set, Paperless defaults to
-# INBOX.
-# PAPERLESS_CONSUME_MAIL_INBOX=INBOX
-
-# Any email sent to the target account that does not contain this text will be
-# ignored.  Mail checking won't work without this.
-# PAPERLESS_EMAIL_SECRET=
-

docs/changelog.rst

@@ -1,6 +1,27 @@
 Changelog
 #########
 
+2.4.0
+=====
+
+* A new set of actions are now available thanks to `jonaswinkler`_'s very first
+  pull request!  You can now do nifty things like tag documents in bulk, or set
+  correspondents in bulk.  `#405`_
+* The import/export system is now a little smarter.  By default, documents are
+  tagged as ``unencrypted``, since exports are by their nature unencrypted.
+  It's now in the import step that we decide the storage type.  This allows you
+  to export from an encrypted system and import into an unencrypted one, or
+  vice-versa.
+* The migration history has been slightly modified to accomodate PostgreSQL
+  users.  Additionally, you can now tell paperless to use PostgreSQL simply by
+  declaring ``PAPERLESS_DBUSER`` in your environment.  This will attempt to
+  connect to your Postgres database without a password unless you also set
+  ``PAPERLESS_DBPASS``.
+* A bug was found in the REST API filter system that was the result of an
+  update of django-filter some time ago.  This has now been patched `#412`_.
+  Thanks to `thepill`_ for spotting it!
+
+
 2.3.0
 =====
 
@@ -15,7 +36,8 @@ Changelog
 * As his last bit of effort on this release, Joshua also added some code to
   allow you to view the documents inline rather than download them as an
   attachment. `#400`_
-* Finally, `ahyear`_ found a slip in the Docker documentation and patched it. `#401`_
+* Finally, `ahyear`_ found a slip in the Docker documentation and patched it.
+  `#401`_
 
 
 2.2.1
@@ -32,14 +54,14 @@ Changelog
   version of Paperless that supports Django 2.0!  As a result of their hard
   work, you can now also run Paperless on Python 3.7 as well: `#386`_ &
   `#390`_.
-* `Stéphane Brunner`_ added a few lines of code that made tagging interface a lot
+* `Stéphane Brunner`_ added a few lines of code that made tagging interface a
-  easier on those of us with lots of different tags: `#391`_.
+  lot easier on those of us with lots of different tags: `#391`_.
 * `Kilian Koeltzsch`_ noticed a bug in how we capture & automatically create
   tags, so that's fixed now too: `#384`_.
 * `erikarvstedt`_ tweaked the behaviour of the test suite to be better behaved
   for packaging environments: `#383`_.
-* `Lukasz Soluch`_ added CORS support to make building a new Javascript-based front-end
+* `Lukasz Soluch`_ added CORS support to make building a new Javascript-based
-  cleaner & easier: `#387`_.
+  front-end cleaner & easier: `#387`_.
 
 
 2.1.0
@@ -499,8 +521,10 @@ bulk of the work on this big change.
 .. _Kilian Koeltzsch: https://github.com/kiliankoe
 .. _Lukasz Soluch: https://github.com/LukaszSolo
 .. _Joshua Taillon: https://github.com/jat255
-.. _dubit0:  https://github.com/dubit0
+.. _dubit0: https://github.com/dubit0
-.. _ahyear:  https://github.com/ahyear
+.. _ahyear: https://github.com/ahyear
+.. _jonaswinkler: https://github.com/jonaswinkler
+.. _thepill: https://github.com/thepill
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -587,6 +611,8 @@ bulk of the work on this big change.
 .. _#399: https://github.com/danielquinn/paperless/pull/399
 .. _#400: https://github.com/danielquinn/paperless/pull/400
 .. _#401: https://github.com/danielquinn/paperless/pull/401
+.. _#405: https://github.com/danielquinn/paperless/pull/405
+.. _#412: https://github.com/danielquinn/paperless/issues/412
 
 .. _pipenv: https://docs.pipenv.org/
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/

docs/consumption.rst

@@ -76,6 +76,31 @@ Pre-consumption script
 
 * Document file name
 
+A simple but common example for this would be creating a simple script like
+this:
+
+``/usr/local/bin/ocr-pdf``
+
+.. code:: bash
+
+    #!/usr/bin/env bash
+    pdf2pdfocr.py -i ${1}
+
+``/etc/paperless.conf``
+
+.. code:: bash
+
+    ...
+    PAPERLESS_PRE_CONSUME_SCRIPT="/usr/local/bin/ocr-pdf"
+    ...
+
+This will pass the path to the document about to be consumed to ``/usr/local/bin/ocr-pdf``,
+which will in turn call `pdf2pdfocr.py`_ on your document, which will then
+overwrite the file with an OCR'd version of the file and exit.  At which point,
+the consumption process will begin with the newly modified file.
+
+.. _pdf2pdfocr.py: https://github.com/LeoFCardoso/pdf2pdfocr
+
 
 .. _consumption-director-hook-variables-post:
 

docs/contributing.rst

@@ -0,0 +1,141 @@
+.. _contributing:
+
+Contributing to Paperless
+#########################
+
+Maybe you've been using Paperless for a while and want to add a feature or two,
+or maybe you've come across a bug that you have some ideas how to solve.  The
+beauty of Free software is that you can see what's wrong and help to get it
+fixed for everyone!
+
+
+How to Get Your Changes Rolled Into Paperless
+=============================================
+
+If you've found a bug, but don't know how to fix it, you can always post an
+issue on `GitHub`_ in the hopes that someone will have the time to fix it for
+you.  If however you're the one with the time, pull requests are always
+welcome, you just have to make sure that your code conforms to a few standards:
+
+Pep8
+----
+
+It's the standard for all Python development, so it's `very well documented`_.
+The short version is:
+
+* Lines should wrap at 79 characters
+* Use ``snake_case`` for variables, ``CamelCase`` for classes, and ``ALL_CAPS``
+  for constants.
+* Space out your operators: ``stuff + 7`` instead of ``stuff+7``
+* Two empty lines between classes, and functions, but 1 empty line between
+  class methods.
+
+There's more to it than that, but if you follow those, you'll probably be
+alright.  When you submit your pull request, there's a pep8 checker that'll
+look at your code to see if anything is off.  If it finds anything, it'll
+complain at you until you fix it.
+
+
+Additional Style Guides
+-----------------------
+
+Where pep8 is ambiguous, I've tried to be a little more specific.  These rules
+aren't hard-and-fast, but if you can conform to them, I'll appreciate it and
+spend less time trying to conform your PR before merging:
+
+
+Function calls
+..............
+
+If you're calling a function and that necessitates more than one line of code,
+please format it like this:
+
+.. code:: python
+
+    my_function(
+        argument1,
+        kwarg1="x",
+        kwarg2="y"
+        another_really_long_kwarg="some big value"
+        a_kwarg_calling_another_long_function=another_function(
+            another_arg,
+            another_kwarg="kwarg!"
+        )
+    )
+
+This is all in the interest of code uniformity rather than anything else.  If
+we stick to a style, everything is understandable in the same way.
+
+
+Quoting Strings
+...............
+
+pep8 is a little too open-minded on this for my liking.  Python strings should
+be quoted with double quotes (``"``) except in cases where the resulting string
+would require too much escaping of a double quote, in which case, a single
+quoted, or triple-quoted string will do:
+
+.. code:: python
+
+    my_string = "This is my string"
+    problematic_string = 'This is a "string" with "quotes" in it'
+
+In HTML templates, please use double-quotes for tag attributes, and single
+quotes for arguments passed to Django tempalte tags:
+
+.. code:: html
+
+    <div class="stuff">
+        <a href="{% url 'some-url-name' pk='w00t' %}">link this</a>
+    </div>
+
+This is to keep linters happy they look at an HTML file and see an attribute
+closing the ``"`` before it should have been.
+
+--
+
+That's all there is in terms of guidelines, so I hope it's not too daunting.
+
+
+Indentation & Spacing
+.....................
+
+When it comes to indentation:
+
+* For Python, the rule is: follow pep8 and use 4 spaces.
+* For Javascript, CSS, and HTML, please use 1 tab.
+
+Additionally, Django templates making use of block elements like ``{% if %}``,
+``{% for %}``, and ``{% block %}`` etc. should be indented:
+
+Good:
+
+.. code:: html
+
+    {% block stuff %}
+    	<h1>This is the stuff</h1>
+    {% endblock %}
+
+Bad:
+
+.. code:: html
+
+    {% block stuff %}
+    <h1>This is the stuff</h1>
+    {% endblock %}
+
+
+The Code of Conduct
+===================
+
+Paperless has a `code of conduct`_.  It's a lot like the other ones you see out
+there, with a few small changes, but basically it boils down to:
+
+> Don't be an ass, or you might get banned.
+
+I'm proud to say that the CoC has never had to be enforced because everyone has
+been awesome, friendly, and professional.
+
+.. _GitHub: https://github.com/danielquinn/paperless/issues
+.. _very well documented: https://www.python.org/dev/peps/pep-0008/
+.. _code of conduct: https://github.com/danielquinn/paperless/blob/master/CODE_OF_CONDUCT.md

docs/index.rst

@@ -43,5 +43,6 @@ Contents
    customising
    extending
    troubleshooting
+   contributing
    scanners
    changelog

requirements.txt

@@ -1,10 +1,10 @@
 -i https://pypi.python.org/simple
-apipkg==1.5; python_version != '3.1.*'
+apipkg==1.5; python_version != '3.3.*'
-atomicwrites==1.2.1; python_version != '3.1.*'
+atomicwrites==1.2.1; python_version != '3.3.*'
 attrs==18.2.0
 certifi==2018.8.24
 chardet==3.0.4
-coverage==4.5.1; python_version != '3.1.*'
+coverage==4.5.1; python_version < '4'
 coveralls==1.5.0
 dateparser==0.7.0
 django-cors-headers==2.4.0
@@ -14,9 +14,9 @@ django-filter==2.0.0
 django==2.0.8
 djangorestframework==3.8.2
 docopt==0.6.2
-execnet==1.5.0; python_version != '3.1.*'
+execnet==1.5.0; python_version != '3.3.*'
 factory-boy==2.11.1
-faker==0.9.0
+faker==0.9.0; python_version >= '2.7'
 filemagic==1.6
 fuzzywuzzy==0.15.0
 gunicorn==19.9.0
@@ -27,17 +27,17 @@ more-itertools==4.3.0
 numpy==1.15.1
 pdftotext==2.1.0
 pillow==5.2.0
-pluggy==0.7.1; python_version != '3.1.*'
+pluggy==0.7.1; python_version != '3.3.*'
-py==1.6.0; python_version != '3.1.*'
+py==1.6.0; python_version != '3.3.*'
 pycodestyle==2.4.0
 pyocr==0.5.3
-pytest-cov==2.5.1
+pytest-cov==2.6.0
 pytest-django==3.4.2
 pytest-env==0.6.2
-pytest-forked==0.2
+pytest-forked==0.2; python_version != '3.3.*'
 pytest-sugar==0.9.1
 pytest-xdist==1.23.0
-pytest==3.7.4
+pytest==3.8.0
 python-dateutil==2.7.3
 python-dotenv==0.9.1
 python-gnupg==0.4.3
@@ -51,4 +51,4 @@ scipy==1.1.0
 termcolor==1.1.0
 text-unidecode==1.2
 tzlocal==1.5.1
-urllib3==1.23; python_version != '3.0.*'
+urllib3==1.23; python_version != '3.3.*'

src/documents/actions.py

@@ -5,10 +5,13 @@ from django.core.exceptions import PermissionDenied
 from django.template.response import TemplateResponse
 
 from documents.classifier import DocumentClassifier
-from documents.models import Tag, Correspondent, DocumentType
+from documents.models import Correspondent, DocumentType, Tag
 
 
-def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
+def select_action(
+        modeladmin, request, queryset, title, action, modelclass,
+        success_message="", document_action=None, queryset_action=None):
+
     opts = modeladmin.model._meta
     app_label = opts.app_label
 
@@ -28,7 +31,9 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
                 queryset_action(queryset, selected_object)
 
             modeladmin.message_user(request, success_message % {
-                "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
+                "selected_object": selected_object.name,
+                "count": n,
+                "items": model_ngettext(modeladmin.opts, n)
             }, messages.SUCCESS)
 
         # Return None to display the change list page again.
@@ -48,10 +53,17 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
 
     request.current_app = modeladmin.admin_site.name
 
-    return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
+    return TemplateResponse(
+        request,
+        "admin/{}/{}/select_object.html".format(app_label, opts.model_name),
+        context
+    )
 
 
-def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
+def simple_action(
+        modeladmin, request, queryset, success_message="",
+        document_action=None, queryset_action=None):
+
     if not modeladmin.has_change_permission(request):
         raise PermissionDenied
 
@@ -73,40 +85,57 @@ def simple_action(modeladmin, request, queryset, success_message="", document_ac
 
 
 def add_tag_to_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+    return select_action(
-                         title="Add tag to multiple documents",
+        modeladmin=modeladmin,
-                         action="add_tag_to_selected",
+        request=request,
-                         modelclass=Tag,
+        queryset=queryset,
-                         success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
+        title="Add tag to multiple documents",
-                         document_action=lambda doc, tag: doc.tags.add(tag))
+        action="add_tag_to_selected",
-add_tag_to_selected.short_description = "Add tag to selected documents"
+        modelclass=Tag,
+        success_message="Successfully added tag %(selected_object)s to "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.add(tag)
+    )
 
 
 def remove_tag_from_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+    return select_action(
-                         title="Remove tag from multiple documents",
+        modeladmin=modeladmin,
-                         action="remove_tag_from_selected",
+        request=request,
-                         modelclass=Tag,
+        queryset=queryset,
-                         success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
+        title="Remove tag from multiple documents",
-                         document_action=lambda doc, tag: doc.tags.remove(tag))
+        action="remove_tag_from_selected",
-remove_tag_from_selected.short_description = "Remove tag from selected documents"
+        modelclass=Tag,
+        success_message="Successfully removed tag %(selected_object)s from "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.remove(tag)
+    )
 
 
 def set_correspondent_on_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
+
-                         title="Set correspondent on multiple documents",
+    return select_action(
-                         action="set_correspondent_on_selected",
+        modeladmin=modeladmin,
-                         modelclass=Correspondent,
+        request=request,
-                         success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
+        queryset=queryset,
-                         queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
+        title="Set correspondent on multiple documents",
-set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
+        action="set_correspondent_on_selected",
+        modelclass=Correspondent,
+        success_message="Successfully set correspondent %(selected_object)s "
+                        "on %(count)d %(items)s.",
+        queryset_action=lambda qs, corr: qs.update(correspondent=corr)
+    )
 
 
 def remove_correspondent_from_selected(modeladmin, request, queryset):
-    return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
+    return simple_action(
-                         success_message="Successfully removed correspondent from %(count)d %(items)s.",
+        modeladmin=modeladmin,
-                         queryset_action=lambda qs: qs.update(correspondent=None))
+        request=request,
-remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
+        queryset=queryset,
+        success_message="Successfully removed correspondent from %(count)d "
+                        "%(items)s.",
+        queryset_action=lambda qs: qs.update(correspondent=None)
+    )
 
 
 def set_document_type_on_selected(modeladmin, request, queryset):
@@ -116,14 +145,12 @@ def set_document_type_on_selected(modeladmin, request, queryset):
                          modelclass=DocumentType,
                          success_message="Successfully set document type %(selected_object)s on %(count)d %(items)s.",
                          queryset_action=lambda qs, document_type: qs.update(document_type=document_type))
-set_document_type_on_selected.short_description = "Set document type on selected documents"
 
 
 def remove_document_type_from_selected(modeladmin, request, queryset):
     return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
                          success_message="Successfully removed document type from %(count)d %(items)s.",
                          queryset_action=lambda qs: qs.update(document_type=None))
-remove_document_type_from_selected.short_description = "Remove document type from selected documents"
 
 
 def run_document_classifier_on_selected(modeladmin, request, queryset):
@@ -135,4 +162,16 @@ def run_document_classifier_on_selected(modeladmin, request, queryset):
     except FileNotFoundError:
         modeladmin.message_user(request, "Classifier model file not found.", messages.ERROR)
         return None
+
+
+add_tag_to_selected.short_description = "Add tag to selected documents"
+remove_tag_from_selected.short_description = \
+    "Remove tag from selected documents"
+set_correspondent_on_selected.short_description = \
+    "Set correspondent on selected documents"
+remove_correspondent_from_selected.short_description = \
+    "Remove correspondent from selected documents"
+set_document_type_on_selected.short_description = "Set document type on selected documents"
+remove_document_type_from_selected.short_description = "Remove document type from selected documents"
 run_document_classifier_on_selected.short_description = "Run document classifier on selected"
+

src/documents/admin.py

@@ -3,22 +3,26 @@ from datetime import datetime, timedelta
 from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import Group, User
+from django.db import models
 from django.http import HttpResponseRedirect
-try:
-    from django.core.urlresolvers import reverse
-except ImportError:
-    from django.urls import reverse
 from django.templatetags.static import static
+from django.urls import reverse
 from django.utils.html import format_html, format_html_join
 from django.utils.http import urlquote
 from django.utils.safestring import mark_safe
-from django.db import models
 
-from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
+from documents.actions import (
-    remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, \
+    add_tag_to_selected,
+    remove_correspondent_from_selected,
+    remove_tag_from_selected,
+    set_correspondent_on_selected,
+    set_document_type_on_selected,
+    remove_document_type_from_selected,
     run_document_classifier_on_selected
-from .models import Correspondent, Tag, Document, Log, DocumentType
+)
+
+from .models import Correspondent, Document, DocumentType, Log, Tag
 
 
 class FinancialYearFilter(admin.SimpleListFilter):
@@ -93,11 +97,18 @@ class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
         self.title = "correspondent (recent)"
 
     def field_choices(self, field, request, model_admin):
+
+        years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
+        days = 365 * years
+
         lookups = []
-        if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
+        if years and years > 0:
-            date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
+            correspondents = Correspondent.objects.filter(
-            for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
+                documents__created__gte=datetime.now() - timedelta(days=days)
+            ).distinct()
+            for c in correspondents:
                 lookups.append((c.id, c.name))
+
         return lookups
 
 
@@ -107,12 +118,20 @@ class CommonAdmin(admin.ModelAdmin):
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "automatic_classification", "document_count", "last_correspondence")
+    list_display = (
-    list_editable = ("automatic_classification",)
+        "name",
+        "automatic_classification",
+        "document_count",
+        "last_correspondence"
+    )
+    list_editable = ("automatic_classification")
 
     def get_queryset(self, request):
         qs = super(CorrespondentAdmin, self).get_queryset(request)
-        qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
+        qs = qs.annotate(
+            document_count=models.Count("documents"),
+            last_correspondence=models.Max("documents__created")
+        )
         return qs
 
     def document_count(self, obj):
@@ -160,24 +179,39 @@ class DocumentAdmin(CommonAdmin):
     class Media:
         css = {
             "all": ("paperless.css",)
-
         }
 
     search_fields = ("correspondent__name", "title", "content", "tags__name")
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_", "archive_serial_number", "document_type")
-    list_filter = ("document_type", "tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
+    list_filter = (
+        "document_type",
+        "tags",
+        ("correspondent", RecentCorrespondentFilter),
+        "correspondent",
+        FinancialYearFilter
+    )
 
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
-    actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, run_document_classifier_on_selected]
+    actions = [
+        add_tag_to_selected,
+        remove_tag_from_selected,
+        set_correspondent_on_selected,
+        remove_correspondent_from_selected,
+        set_document_type_on_selected,
+        remove_document_type_from_selected,
+        run_document_classifier_on_selected
+    ]
 
-    date_hierarchy = 'created'
+    date_hierarchy = "created"
 
-    document_queue = None
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.document_queue = []
 
     def has_add_permission(self, request):
         return False
@@ -187,27 +221,41 @@ class DocumentAdmin(CommonAdmin):
     created_.short_description = "Created"
 
     def changelist_view(self, request, extra_context=None):
-        response = super().changelist_view(request, extra_context)
 
-        if request.method == 'GET':
+        response = super().changelist_view(
+            request,
+            extra_context=extra_context
+        )
+
+        if request.method == "GET":
             cl = self.get_changelist_instance(request)
             self.document_queue = [doc.id for doc in cl.queryset]
 
         return response
 
-    def change_view(self, request, object_id=None, form_url='', extra_context=None):
+    def change_view(self, request, object_id=None, form_url='',
+                    extra_context=None):
+
         extra_context = extra_context or {}
         doc = Document.objects.get(id=object_id)
         extra_context['download_url'] = doc.download_url
         extra_context['file_type'] = doc.file_type
-        if self.document_queue and object_id and int(object_id) in self.document_queue:
+
-            # There is a queue of documents
+        if self.document_queue and object_id:
-            current_index = self.document_queue.index(int(object_id))
+            if int(object_id) in self.document_queue:
-            if current_index < len(self.document_queue) - 1:
+                # There is a queue of documents
-                # ... and there are still documents in the queue
+                current_index = self.document_queue.index(int(object_id))
-                extra_context['next_object'] = self.document_queue[current_index + 1]
+                if current_index < len(self.document_queue) - 1:
+                    # ... and there are still documents in the queue
+                    extra_context["next_object"] = self.document_queue[
+                        current_index + 1
+                    ]
+
         return super(DocumentAdmin, self).change_view(
-            request, object_id, form_url, extra_context=extra_context,
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
         )
 
     def response_change(self, request, obj):
@@ -217,25 +265,35 @@ class DocumentAdmin(CommonAdmin):
         preserved_filters = self.get_preserved_filters(request)
 
         msg_dict = {
-            'name': opts.verbose_name,
+            "name": opts.verbose_name,
-            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
+            "obj": format_html(
+                '<a href="{}">{}</a>',
+                urlquote(request.path),
+                obj
+            ),
         }
         if "_saveandeditnext" in request.POST:
             msg = format_html(
-                'The {name} "{obj}" was changed successfully. Editing next object.',
+                'The {name} "{obj}" was changed successfully. '
+                'Editing next object.',
                 **msg_dict
             )
             self.message_user(request, msg, messages.SUCCESS)
-            redirect_url = reverse('admin:%s_%s_change' %
+            redirect_url = reverse(
-                                   (opts.app_label, opts.model_name),
+                "admin:{}_{}_change".format(opts.app_label, opts.model_name),
-                                   args=(request.POST['_next_object'],),
+                args=(request.POST["_next_object"],),
-                                   current_app=self.admin_site.name)
+                current_app=self.admin_site.name
-            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
+            )
-            response = HttpResponseRedirect(redirect_url)
+            redirect_url = add_preserved_filters(
-        else:
+                {
-            response = super().response_change(request, obj)
+                    "preserved_filters": preserved_filters,
+                    "opts": opts
+                },
+                redirect_url
+            )
+            return HttpResponseRedirect(redirect_url)
 
-        return response
+        return super().response_change(request, obj)
 
     @mark_safe
     def thumbnail(self, obj):

src/documents/filters.py

@@ -1,8 +1,14 @@
-from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter
+from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter, ModelChoiceFilter
 
 from .models import Correspondent, Document, Tag, DocumentType
 
 
+CHAR_KWARGS = (
+    "startswith", "endswith", "contains",
+    "istartswith", "iendswith", "icontains"
+)
+
+
 class CorrespondentFilterSet(FilterSet):
 
     class Meta:
@@ -44,38 +50,27 @@ class DocumentTypeFilterSet(FilterSet):
 
 class DocumentFilterSet(FilterSet):
 
-    CHAR_KWARGS = {
+    tags_empty = BooleanFilter(
-        "lookup_expr": (
+        label="Is tagged",
-            "startswith",
+        field_name="tags",
-            "endswith",
+        lookup_expr="isnull",
-            "contains",
+        exclude=True
-            "istartswith",
+    )
-            "iendswith",
-            "icontains"
-        )
-    }
-
-    correspondent__name = CharFilter(
-        field_name="correspondent__name", **CHAR_KWARGS)
-    correspondent__slug = CharFilter(
-        field_name="correspondent__slug", **CHAR_KWARGS)
-    tags__name = CharFilter(
-        field_name="tags__name", **CHAR_KWARGS)
-    tags__slug = CharFilter(
-        field_name="tags__slug", **CHAR_KWARGS)
-    tags__empty = BooleanFilter(
-        field_name="tags", lookup_expr="isnull", distinct=True)
-    document_type__name = CharFilter(
-        name="document_type__name", **CHAR_KWARGS)
-    document_type__slug = CharFilter(
-        name="document_type__slug", **CHAR_KWARGS)
 
     class Meta:
         model = Document
         fields = {
-            "title": [
+
-                "startswith", "endswith", "contains",
+            "title": CHAR_KWARGS,
-                "istartswith", "iendswith", "icontains"
+            "content": ("contains", "icontains"),
-            ],
+
-            "content": ["contains", "icontains"],
+            "correspondent__name": CHAR_KWARGS,
+            "correspondent__slug": CHAR_KWARGS,
+
+            "tags__name": CHAR_KWARGS,
+            "tags__slug": CHAR_KWARGS,
+
+            "document_type__name": CHAR_KWARGS,
+            "document_type__slug": CHAR_KWARGS,
+
         }

src/documents/management/commands/document_exporter.py

@@ -55,7 +55,12 @@ class Command(Renderable, BaseCommand):
         documents = Document.objects.all()
         document_map = {d.pk: d for d in documents}
         manifest = json.loads(serializers.serialize("json", documents))
-        for document_dict in manifest:
+
+        for index, document_dict in enumerate(manifest):
+
+            # Force output to unencrypted as that will be the current state.
+            # The importer will make the decision to encrypt or not.
+            manifest[index]["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED  # NOQA: E501
 
             document = document_map[document_dict["pk"]]
 

src/documents/management/commands/document_importer.py

@@ -94,7 +94,7 @@ class Command(Renderable, BaseCommand):
             document_path = os.path.join(self.source, doc_file)
             thumbnail_path = os.path.join(self.source, thumb_file)
 
-            if document.storage_type == Document.STORAGE_TYPE_GPG:
+            if settings.PASSPHRASE:
 
                 with open(document_path, "rb") as unencrypted:
                     with open(document.source_path, "wb") as encrypted:
@@ -112,3 +112,15 @@ class Command(Renderable, BaseCommand):
 
                 shutil.copy(document_path, document.source_path)
                 shutil.copy(thumbnail_path, document.thumbnail_path)
+
+        # Reset the storage type to whatever we've used while importing
+
+        storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+        if settings.PASSPHRASE:
+            storage_type = Document.STORAGE_TYPE_GPG
+
+        Document.objects.filter(
+            pk__in=[r["pk"] for r in self.manifest]
+        ).update(
+            storage_type=storage_type
+        )

src/documents/migrations/0014_document_checksum.py

@@ -158,9 +158,4 @@ class Migration(migrations.Migration):
             name='modified',
             field=models.DateTimeField(auto_now=True, db_index=True),
         ),
-        migrations.AlterField(
-            model_name='document',
-            name='checksum',
-            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
-        ),
     ]

src/documents/migrations/0015_add_insensitive_to_match.py

@@ -12,6 +12,11 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='checksum',
+            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
+        ),
         migrations.AddField(
             model_name='correspondent',
             name='is_insensitive',

src/documents/templates/admin/documents/document/select_object.html

@@ -1,46 +1,50 @@
 {% extends "admin/base_site.html" %}
+
+
 {% load i18n l10n admin_urls static %}
 {% load staticfiles %}
 
-{% block extrahead %}
-{{ block.super }}
-{{ media }}
-<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 
+{% block extrahead %}
+	{{ block.super }}
+	{{ media }}
+	<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 {% endblock %}
 
+
 {% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
 
+
 {% block breadcrumbs %}
-<div class="breadcrumbs">
+	<div class="breadcrumbs">
-    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+		<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-    &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+		&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
-    &rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+		&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
-    &rsaquo; {{title}}
+		&rsaquo; {{ title }}
-</div>
+	</div>
 {% endblock %}
 
 {% block content %}
-<p>Please select the {{itemname}}.</p>
+	<p>Please select the {{itemname}}.</p>
-<form method="post">{% csrf_token %}
+	<form method="post">{% csrf_token %}
-    <div>
+		<div>
-        {% for obj in queryset %}
+			{% for obj in queryset %}
-        <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+			<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
-        {% endfor %}
+			{% endfor %}
-        <p>
+			<p>
-            <select name="obj_id">
+				<select name="obj_id">
-                {% for obj in objects %}
+					{% for obj in objects %}
-                <option value="{{obj.id}}">{{obj.name}}</option>
+					<option value="{{ obj.id }}">{{ obj.name }}</option>
-                {% endfor %}
+					{% endfor %}
-            </select>
+				</select>
-        </p>
+			</p>
 
-        <input type="hidden" name="action" value="{{action}}"/>
+			<input type="hidden" name="action" value="{{ action }}"/>
-        <input type="hidden" name="post" value="yes"/>
+			<input type="hidden" name="post" value="yes" />
-        <p>
+			<p>
-            <input type="submit" value="{% trans "Confirm" %}" />
+				<input type="submit" value="{% trans 'Confirm' %}" />
-            <a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+				<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
-        </p>
+			</p>
-    </div>
+		</div>
-</form>
+	</form>
 {% endblock %}

src/paperless/settings.py

@@ -149,8 +149,9 @@ if os.getenv("PAPERLESS_DBENGINE"):
         "ENGINE": os.getenv("PAPERLESS_DBENGINE"),
         "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
         "USER": os.getenv("PAPERLESS_DBUSER"),
-        "PASSWORD": os.getenv("PAPERLESS_DBPASS")
     }
+    if os.getenv("PAPERLESS_DBPASS"):
+        DATABASES["default"]["PASSWORD"] = os.getenv("PAPERLESS_DBPASS")
 
 
 # Password validation

src/paperless_tesseract/parsers.py

@@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser):
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
             raise OCRError(
-                "The guessed language is not available in this instance of "
+                "The guessed language ({}) is not available in this instance "
-                "Tesseract."
+                "of Tesseract.".format(guessed_language)
             )
 
     def _ocr(self, imgs, lang):

src/tox.ini

@@ -5,7 +5,7 @@
 
 [tox]
 skipsdist = True
-envlist = py34, py35, py36, pycodestyle, doc
+envlist = py34, py35, py36, py37, pycodestyle, doc
 
 [testenv]
 commands = pytest