Merge branch 'master' of https://github.com/asciimoo/searx into dos2unix

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.dir-locals.el

@@ -0,0 +1,133 @@
+;;; .dir-locals.el
+;;
+;; If you get ``*** EPC Error ***`` (even after a jedi:install-server) in your
+;; emacs session, mostly you have jedi-mode enabled but the python enviroment is
+;; missed.  The python environment has to be next to the
+;; ``<repo>/.dir-locals.el`` in::
+;;
+;;     ./local/py3
+;;
+;; In Emacs, some buffer locals are referencing the project environment:
+;;
+;; - prj-root                                --> <repo>/
+;; - python-environment-directory            --> <repo>/local
+;; - python-environment-default-root-name    --> py3
+;; - python-shell-virtualenv-root            --> <repo>/local/py3
+;;       When this variable is set with the path of the virtualenv to use,
+;;      `process-environment' and `exec-path' get proper values in order to run
+;;      shells inside the specified virtualenv, example::
+;;         (setq python-shell-virtualenv-root "/path/to/env/")
+;;
+;; To setup such an environment build target 'pyenv' or 'pyenvinstall'::
+;;
+;;   $ make pyenvinstall
+;;
+;; Alternatively create the virtualenv, source it and install jedi + epc
+;; (required by `emacs-jedi <https://tkf.github.io/emacs-jedi>`_)::
+;;
+;;     $ virtualenv --python=python3  "--no-site-packages" ./local/py3
+;;     ...
+;;     $ source ./local/py3/bin/activate
+;;     (py3)$ # now install into the activated 'py3' environment ..
+;;     (py3)$ pip install jedi epc
+;;     ...
+;;
+;; Here is what also I found useful to add to my .emacs::
+;;
+;;     (global-set-key [f6] 'flycheck-mode)
+;;     (add-hook 'python-mode-hook 'my:python-mode-hook)
+;;
+;;     (defun my:python-mode-hook ()
+;;       (add-to-list 'company-backends 'company-jedi)
+;;       (require 'jedi-core)
+;;       (jedi:setup)
+;;       (define-key python-mode-map (kbd "C-c C-d") 'jedi:show-doc)
+;;       (define-key python-mode-map (kbd "M-.")     'jedi:goto-definition)
+;;       (define-key python-mode-map (kbd "M-,")     'jedi:goto-definition-pop-marker)
+;;     )
+;;
+
+((nil
+  . ((fill-column . 80)
+     ))
+ (python-mode
+  . ((indent-tabs-mode . nil)
+
+     ;; project root folder is where the `.dir-locals.el' is located
+     (eval . (setq-local
+	      prj-root (locate-dominating-file  default-directory ".dir-locals.el")))
+
+     (eval . (setq-local
+	      python-environment-directory (expand-file-name "./local" prj-root)))
+
+     ;; use 'py3' enviroment as default
+     (eval . (setq-local
+	      python-environment-default-root-name "py3"))
+
+     (eval . (setq-local
+	      python-shell-virtualenv-root
+	      (concat python-environment-directory
+		      "/"
+		      python-environment-default-root-name)))
+
+     ;; python-shell-virtualenv-path is obsolete, use python-shell-virtualenv-root!
+     ;; (eval . (setq-local
+     ;; 	 python-shell-virtualenv-path python-shell-virtualenv-root))
+
+     (eval . (setq-local
+	      python-shell-interpreter
+	      (expand-file-name "bin/python" python-shell-virtualenv-root)))
+
+     (eval . (setq-local
+	      python-environment-virtualenv
+	      (list (expand-file-name "bin/virtualenv" python-shell-virtualenv-root)
+		    ;;"--system-site-packages"
+		    "--quiet")))
+
+     (eval . (setq-local
+	      pylint-command
+	      (expand-file-name "bin/pylint" python-shell-virtualenv-root)))
+
+     ;; pylint will find the '.pylintrc' file next to the CWD
+     ;;   https://pylint.readthedocs.io/en/latest/user_guide/run.html#command-line-options
+     (eval . (setq-local
+	      flycheck-pylintrc ".pylintrc"))
+
+     ;; flycheck & other python stuff should use the local py3 environment
+     (eval . (setq-local
+	      flycheck-python-pylint-executable python-shell-interpreter))
+
+     ;; use 'M-x jedi:show-setup-info'  and 'M-x epc:controller' to inspect jedi server
+
+     ;; https://tkf.github.io/emacs-jedi/latest/#jedi:environment-root -- You
+     ;; can specify a full path instead of a name (relative path). In that case,
+     ;; python-environment-directory is ignored and Python virtual environment
+     ;; is created at the specified path.
+     (eval . (setq-local  jedi:environment-root  python-shell-virtualenv-root))
+
+     ;; https://tkf.github.io/emacs-jedi/latest/#jedi:server-command
+     (eval .(setq-local
+	     jedi:server-command
+	     (list python-shell-interpreter
+		   jedi:server-script)
+	     ))
+
+     ;; jedi:environment-virtualenv --> see above 'python-environment-virtualenv'
+     ;; is set buffer local! No need to setup jedi:environment-virtualenv:
+     ;;
+     ;;    Virtualenv command to use.  A list of string.  If it is nil,
+     ;;    python-environment-virtualenv is used instead.  You must set non-nil
+     ;;    value to jedi:environment-root in order to make this setting work.
+     ;;
+     ;;    https://tkf.github.io/emacs-jedi/latest/#jedi:environment-virtualenv
+     ;;
+     ;; (eval . (setq-local
+     ;; 	      jedi:environment-virtualenv
+     ;; 	      (list (expand-file-name "bin/virtualenv" python-shell-virtualenv-root)
+     ;; 		    ;;"--python"
+     ;; 		    ;;"/usr/bin/python3.4"
+     ;; 		    )))
+
+     ;; jedi:server-args
+
+     )))

.gitignore

@@ -18,3 +18,6 @@ setup.cfg
 node_modules/
 
 .tx/
+
+local/
+searx.egg-info/

.pylintrc

@@ -0,0 +1,444 @@
+# -*- coding: utf-8; mode: conf -*-
+# lint Python modules using external checkers.
+#
+# This is the main checker controlling the other ones and the reports
+# generation. It is itself both a raw checker and an astng checker in order
+# to:
+# * handle message activation / deactivation at the module level
+# * handle some basic but necessary stats'data (number of classes, methods...)
+#
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS, .git, .svn
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint.
+jobs=1
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=bad-whitespace, duplicate-code
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+
+# HINT: do not set this here, use argument --msg-template=...
+#msg-template={path}:{line}: [{msg_id}({symbol}),{obj}] {msg}
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio).You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+
+# HINT: do not set this here, use argument --output-format=...
+#output-format=text
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+
+[BASIC]
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input
+
+# Naming hint for argument names
+argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct argument names
+argument-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Naming hint for attribute names
+attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct attribute names
+attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*)|([A-Z0-9_]*))$
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct constant names
+const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$
+#const-rgx=[f]?[A-Z_][a-zA-Z0-9_]{2,30}$
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming hint for function names
+function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct function names
+function-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_,log,cfg,id
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for method names
+method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct method names
+method-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression matching correct module names
+#module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+module-rgx=([a-z_][a-z0-9_]*)$
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+property-classes=abc.abstractproperty
+
+# Naming hint for variable names
+variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct variable names
+variable-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*)|([a-z]))$
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging  or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module
+max-module-lines=2000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.No config file found, using default configuration
+
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,future.builtins
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=8
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=20
+
+# Maximum number of boolean expressions in a if statement
+max-bool-expr=5
+
+# Maximum number of branch for function / method body
+max-branches=12
+
+# Maximum number of locals for function / method body
+max-locals=20
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

AUTHORS.rst

@@ -1,4 +1,4 @@
-Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament and Noémi Ványi.
+Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo and Markus Heiser.
 
 Major contributing authors:
 
@@ -9,6 +9,8 @@ Major contributing authors:
 - @Cqoicebordel
 - Noémi Ványi
 - Marc Abonce Seguin @a01200356
+- @pofilo
+- Markus Heiser @return42
 
 People who have submitted patches/translates, reported bugs, consulted features or
 generally made searx better:

Makefile

@@ -0,0 +1,62 @@
+# -*- coding: utf-8; mode: makefile-gmake -*-
+
+PYOBJECTS = searx
+PY_SETUP_EXTRAS ?= \[test\]
+
+include utils/makefile.include
+include utils/makefile.python
+
+all: clean install
+
+PHONY += help
+help:
+	@echo  '  test      - run developer tests'
+	@echo  '  run       - run developer instance'
+	@echo  '  install   - developer install (./local)'
+	@echo  '  uninstall - uninstall (./local)'
+	@echo  ''
+	@$(MAKE) -s -f utils/makefile.include make-help
+	@echo  ''
+	@$(MAKE) -s -f utils/makefile.python python-help
+
+PHONY += install
+install: pyenvinstall
+
+PHONY += uninstall
+uninstall: pyenvuninstall
+
+PHONY += clean
+clean: pyclean
+	$(call cmd,common_clean)
+
+PHONY += run
+run:  pyenvinstall
+	$(Q) ( \
+	sed -i -e "s/debug : False/debug : True/g" ./searx/settings.yml ; \
+	sleep 2 ; \
+	xdg-open http://127.0.0.1:8888/ ; \
+	sleep 3 ; \
+	sed -i -e "s/debug : True/debug : False/g" ./searx/settings.yml ; \
+	) &
+	$(PY_ENV)/bin/python ./searx/webapp.py
+
+# test
+# ----
+
+PHONY += test test.pylint test.pep8 test.unit test.robot
+
+# TODO: balance linting with pylint
+test: test.pep8 test.unit test.robot
+	- make pylint
+
+test.pep8: pyenvinstall
+	$(PY_ENV_ACT); ./manage.sh pep8_check
+
+test.unit: pyenvinstall
+	$(PY_ENV_ACT); ./manage.sh unit_tests
+
+test.robot: pyenvinstall
+	$(PY_ENV_ACT); ./manage.sh install_geckodriver
+	$(PY_ENV_ACT); ./manage.sh robot_tests
+
+.PHONY: $(PHONY)

searx/engines/google.py

@@ -107,13 +107,12 @@ images_path = '/images'
 supported_languages_url = 'https://www.google.com/preferences?#languages'
 
 # specific xpath variables
-results_xpath = '//div[@class="g"]'
-url_xpath = './/h3/a/@href'
-title_xpath = './/h3'
-content_xpath = './/span[@class="st"]'
-content_misc_xpath = './/div[@class="f slp"]'
-suggestion_xpath = '//p[@class="_Bmc"]'
-spelling_suggestion_xpath = '//a[@class="spell"]'
+results_xpath = '//div[contains(@class, "ZINbbc")]'
+url_xpath = './/div[@class="kCrYT"][1]/a/@href'
+title_xpath = './/div[@class="kCrYT"][1]/a/div[1]'
+content_xpath = './/div[@class="kCrYT"][2]//div[contains(@class, "BNeawe")]//div[contains(@class, "BNeawe")]'
+suggestion_xpath = '//div[contains(@class, "ZINbbc")][last()]//div[@class="rVLSBd"]/a//div[contains(@class, "BNeawe")]'
+spelling_suggestion_xpath = '//div[@id="scc"]//a'
 
 # map : detail location
 map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
@@ -199,10 +198,6 @@ def request(query, params):
     params['headers']['Accept-Language'] = language + ',' + language + '-' + country
     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
 
-    # Force Safari 3.1 on Mac OS X (Leopard) user agent to avoid loading the new UI that Searx can't parse
-    params['headers']['User-Agent'] = ("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4)"
-                                       "AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1")
-
     params['google_hostname'] = google_hostname
 
     return params
@@ -274,9 +269,7 @@ def response(resp):
                 content = extract_text_from_dom(result, content_xpath)
                 if content is None:
                     continue
-                content_misc = extract_text_from_dom(result, content_misc_xpath)
-                if content_misc is not None:
-                    content = content_misc + "<br />" + content
+
                 # append result
                 results.append({'url': url,
                                 'title': title,

searx/templates/oscar/advanced.html

@@ -1,16 +1,17 @@
 <input type="checkbox" name="advanced_search" id="check-advanced" {% if advanced_search %} checked="checked"{% endif %}>
-<label for="check-advanced">
+<label for="check-advanced">{{- "" -}}
     <span class="glyphicon glyphicon-cog"></span>
-    {{ _('Advanced settings') }}
+    {{- _('Advanced settings') -}}
 </label>
 <div id="advanced-search-container">
   {% include 'oscar/categories.html' %}
+
   <div class="row">
     <div class="col-xs-6">
-      {% include 'oscar/time-range.html' %}
+      {%- include 'oscar/time-range.html' -%}
     </div>
     <div class="col-xs-6">
-      {% include 'oscar/languages.html' %}
+      {%- include 'oscar/languages.html' -%}
     </div>
   </div>
 </div>

searx/templates/oscar/base.html

@@ -10,16 +10,17 @@
     <meta name="referrer" content="no-referrer">
     <meta name="viewport" content="width=device-width, initial-scale=1 , maximum-scale=1.0, user-scalable=1" />
     {% block meta %}{% endblock %}
-    <title>{% block title %}{% endblock %}{{ instance_name }}</title>
 
+    <title>{% block title %}{% endblock %}{{ instance_name }}</title>
     <link rel="stylesheet" href="{{ url_for('static', filename='css/bootstrap.min.css') }}" type="text/css" />
-    {% if preferences.get_value('oscar-style') %}
-        <link rel="stylesheet" href="{{ url_for('static', filename='css/'+preferences.get_value('oscar-style')+'.min.css') }}" type="text/css" />
-    {% else %}
-        <link rel="stylesheet" href="{{ url_for('static', filename='css/logicodev.min.css') }}" type="text/css" />
-    {% endif %}
+    {% if preferences.get_value('oscar-style') -%}
+    {{'    '}}<link rel="stylesheet" href="{{ url_for('static', filename='css/'+preferences.get_value('oscar-style')+'.min.css') }}" type="text/css" />
+    {%- else -%}
+    {{'    '}}<link rel="stylesheet" href="{{ url_for('static', filename='css/logicodev.min.css') }}" type="text/css" />
+    {%- endif %}
+
     <link rel="stylesheet" href="{{ url_for('static', filename='css/leaflet.min.css') }}" type="text/css" />
-    {% for css in styles %}
+    {%- for css in styles %}
         <link rel="stylesheet" href="{{ url_for('static', filename=css) }}" type="text/css" />
     {% endfor %}
 
@@ -48,6 +49,7 @@
 </head>
 <body>
     {% include 'oscar/navbar.html' %}
+
     <div class="container">
     {% if errors %}
         <div class="alert alert-danger fade in" role="alert">
@@ -93,13 +95,14 @@
     </div>
     <script src="{{ url_for('static', filename='js/jquery-1.11.1.min.js') }}"></script>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-    {% if autocomplete %}<script src="{{ url_for('static', filename='js/typeahead.bundle.min.js') }}"></script>{% endif %}
+    {% if autocomplete %}    <script src="{{ url_for('static', filename='js/typeahead.bundle.min.js') }}"></script>{% endif %}
+
     <script src="{{ url_for('static', filename='js/require-2.1.15.min.js') }}"></script>
     <script src="{{ url_for('static', filename='js/searx.min.js') }}"
             data-method="{{ method or 'POST' }}"
             data-autocompleter="{% if autocomplete %}true{% else %}false{% endif %}"></script>
     {% for script in scripts %}
-        <script src="{{ url_for('static', filename=script) }}"></script>
+    {{""}}<script src="{{ url_for('static', filename=script) }}"></script>
     {% endfor %}
     <noscript>
       <style>

searx/templates/oscar/categories.html

@@ -1,13 +1,13 @@
 <div id="categories">
-{% if rtl %}
-    {% for category in categories | reverse %}
-        <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />
+{%- if rtl -%}
+    {% for category in categories | reverse -%}
+        <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
-    {% endfor %}
-{% else %}
-    {% for category in categories %}
-        <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />
+    {%- endfor %}
+{%- else -%}
+    {% for category in categories -%}
+        <input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{- '' -}}
         <label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
-    {% endfor %}
-{% endif %}
+    {%- endfor %}
+{%- endif -%}
 </div>

searx/templates/oscar/infobox.html

@@ -1,34 +1,35 @@
 {% from 'oscar/macros.html' import result_link with context %}
 <div class="panel panel-default infobox">
-    <div class="panel-heading">
-        <h4 class="panel-title infobox_part"><bdi>{{ infobox.infobox }}</bdi></h4>
+    <div class="panel-heading">{{- "" -}}
+        <h4 class="panel-title infobox_part"><bdi>{{ infobox.infobox }}</bdi></h4>{{- "" -}}
     </div>
     <div class="panel-body">
         {% if infobox.img_src %}<img class="img-responsive center-block infobox_part" src="{{ image_proxify(infobox.img_src) }}" alt="{{ infobox.infobox }}" />{% endif %}
-        {% if infobox.content %}<bdi><p class="infobox_part">{{ infobox.content }}</bdi></p>{% endif %}
 
-        {% if infobox.attributes %}
+        {% if infobox.content %}<bdi><p class="infobox_part">{{ infobox.content }}</p></bdi>{% endif %}
+
+        {% if infobox.attributes -%}
         <table class="table table-striped infobox_part">
-            {% for attribute in infobox.attributes %}
-            <tr>
+            {% for attribute in infobox.attributes -%}
+            <tr>{{- "" -}}
                 <td><bdi>{{ attribute.label }}</bdi></td>
-                {% if attribute.image %}
+                {%- if attribute.image -%}
                 <td><img class="img-responsive" src="{{ image_proxify(attribute.image.src) }}" alt="{{ attribute.image.alt }}" /></td>
-                {% else %}
+                {%- else -%}
                 <td><bdi>{{ attribute.value }}</bdi></td>
-                {% endif %}
+                {%- endif -%}
             </tr>
-            {% endfor %}
+            {% endfor -%}
         </table>
         {% endif %}
 
-        {% if infobox.urls %}
-        <div class="infobox_part">
+        {% if infobox.urls -%}
+        <div class="infobox_part">{{- "\n" -}}
             <bdi>
-            {% for url in infobox.urls %}
-            <p class="btn btn-default btn-xs">{{ result_link(url.url, url.title) }}</a></p>
-            {% endfor %}
-            </bdi>
+            {%- for url in infobox.urls -%}
+            <p class="btn btn-default btn-xs">{{ result_link(url.url, url.title) }}</p>
+            {% endfor -%}
+            </bdi>{{- "" -}}
         </div>
         {% endif %}
     </div>

searx/templates/oscar/languages.html

@@ -1,12 +1,12 @@
-{% if preferences %}
+{% if preferences -%}
 <select class="custom-select form-control" name='language'>
-{% else %}
+{%- else -%}
 <select class="time_range custom-select form-control" id='language' name='language'>
-{% endif %}
+{%- endif -%}
 	<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
-		{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
+	{%- for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) -%}
 		<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
-			{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
+			{{- lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id -}}
 		</option>
-		{% endfor %}
-</select>
+	{%- endfor -%}
+</select>

searx/templates/oscar/macros.html

@@ -26,38 +26,38 @@
 
 <!-- Draw result footer -->
 {% macro result_footer(result) -%}
-    <div class="clearfix"></div>
+    <div class="clearfix"></div>{{- "" -}}
     <div class="pull-right">
-    {% for engine in result.engines %}
-        <span class="label label-default">{{ engine }}</span>
-    {% endfor %}
-    {% if result.url %}
-    <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
-    {% endif %}
-    {% if proxify %}
-    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
-    {% endif %}
-</div>
-{% if result.pretty_url %}
-<div class="external-link">{{ result.pretty_url }}</div>
-{% endif %}
+        {%- for engine in result.engines -%}
+            <span class="label label-default">{{ engine }}</span>
+        {%- endfor -%}
+        {%- if result.url -%}
+        <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+        {%- endif -%}
+        {%- if proxify -%}
+        <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+        {%- endif -%}
+    </div>
+    {%- if result.pretty_url -%}
+    <div class="external-link">{{ result.pretty_url }}</div>
+    {%- endif -%}
 {%- endmacro %}
 
 <!-- Draw result footer -->
 {% macro result_footer_rtl(result) -%}
-    <div class="clearfix"></div>
-    {% for engine in result.engines %}
+    <div class="clearfix"></div>{{- "" -}}
+    {% for engine in result.engines -%}
         <span class="label label-default">{{ engine }}</span>
-    {% endfor %}
-    {% if result.url %}
+    {%- endfor %}
+    {%- if result.url -%}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
-    {% endif %}
-    {% if proxify %}
+    {%- endif -%}
+    {% if proxify -%}
     <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
-    {% endif %}
-    {% if result.pretty_url %}
+    {%- endif %}
+    {%- if result.pretty_url -%}
     <div class="external-link">{{ result.pretty_url }}</div>
-    {% endif %}
+    {%- endif %}
 {%- endmacro %}
 
 {% macro preferences_item_header(info, label, rtl) -%}

searx/templates/oscar/navbar.html

@@ -1,9 +1,9 @@
-<div class="searx-navbar">
-    <span class="instance {% if rtl %}pull-right{% else %}pull-left{% endif%}">
-        <a href="{{ url_for('index') }}">{{ instance_name }}</a>
-    </span>
-    <span class="{% if rtl %}pull-left{% else %}pull-right{% endif %}">
-        <a href="{{ url_for('about') }}">{{ _('about') }}</a>
-        <a href="{{ url_for('preferences') }}">{{ _('preferences') }}</a>
-    </span>
+<div class="searx-navbar">{{- "" -}}
+    <span class="instance {% if rtl %}pull-right{% else %}pull-left{% endif%}">{{- "" -}}
+        <a href="{{ url_for('index') }}">{{ instance_name }}</a>{{- "" -}}
+    </span>{{- "" -}}
+    <span class="{% if rtl %}pull-left{% else %}pull-right{% endif %}">{{- "" -}}
+        <a href="{{ url_for('about') }}">{{ _('about') }}</a>{{- "" -}}
+        <a href="{{ url_for('preferences') }}">{{ _('preferences') }}</a>{{- "" -}}
+    </span>{{- "" -}}
 </div>

searx/templates/oscar/result_templates/default.html

@@ -1,31 +1,31 @@
 {% from 'oscar/macros.html' import result_header, result_sub_header, result_footer, result_footer_rtl, icon with context %}
 
-{{ result_header(result, favicons) }}
-{{ result_sub_header(result) }}
+{{- result_header(result, favicons) -}}
+{{- result_sub_header(result) -}}
 
-{% if result.embedded %}
+{%- if result.embedded -%}
     <small> &bull; <a class="text-info btn-collapse collapsed cursor-pointer media-loader disabled_if_nojs" data-toggle="collapse" data-target="#result-media-{{ index }}" data-btn-text-collapsed="{{ _('show media') }}" data-btn-text-not-collapsed="{{ _('hide media') }}">{{ icon('music') }} {{ _('show media') }}</a></small>
-{% endif %}
+{%- endif -%}
 
-{% if result.embedded %}
+{%- if result.embedded -%}
 <div id="result-media-{{ index }}" class="collapse">
    {{ result.embedded|safe }}
 </div>
-{% endif %}
+{%- endif -%}
 
-{% if result.img_src %}
+{%- if result.img_src -%}
 <div class="container-fluid">
     <div class="row">
 <img src="{{ image_proxify(result.img_src) }}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" style="width: auto; max-height: 60px; min-height: 60px;" class="col-xs-2 col-sm-4 col-md-4 result-content">
-{% if result.content %}<p class="result-content col-xs-8 col-sm-8 col-md-8">{{ result.content|safe }}</p>{% endif %}
+{% if result.content %}<p class="result-content col-xs-8 col-sm-8 col-md-8">{{ result.content|safe }}</p>{% endif -%}
     </div>
 </div>
-{% else %}
-{% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}
-{% endif %}
+{%- else -%}
+{%- if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif -%}
+{%- endif -%}
 
-{% if rtl %}
+{%- if rtl -%}
 {{ result_footer_rtl(result) }}
-{% else %}
+{%- else -%}
 {{ result_footer(result) }}
-{% endif %}
+{%- endif -%}

searx/templates/oscar/result_templates/images.html

@@ -1,49 +1,36 @@
-{% from 'oscar/macros.html' import draw_favicon %}
+{%- from 'oscar/macros.html' import draw_favicon -%}
 
-<a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} data-toggle="modal" data-target="#modal-{{ index }}-{{pageno}}">
-    <img src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" class="img-thumbnail">
+<a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} data-toggle="modal" data-target="#modal-{{ index }}-{{pageno}}">{{- "" -}}
+    <img src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" class="img-thumbnail">{{- "" -}}
 </a>
-
-<div class="modal fade" id="modal-{{ index }}-{{ pageno }}" tabindex="-1" role="dialog" aria-hidden="true">
-    <div class="modal-dialog">
-        <div class="modal-wrapper">
-            <div class="modal-header">
-                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
-                <h4 class="modal-title">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result.title|striptags }}</h4>
-            </div>
-            <div class="modal-body">
+<div class="modal fade" id="modal-{{ index }}-{{ pageno }}" tabindex="-1" role="dialog" aria-hidden="true">{{- "" -}}
+    <div class="modal-dialog">{{- "" -}}
+        <div class="modal-wrapper">{{- "" -}}
+            <div class="modal-header">{{- "" -}}
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>{{- "" -}}
+                <h4 class="modal-title">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result.title|striptags }}</h4>{{- "" -}}
+            </div>{{- "" -}}
+            <div class="modal-body">{{- "" -}}
                 <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
-                {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
-                {% if result.content %}
-                    <p class="result-content">
-                        {{ result.content|striptags }}
-                    </p>
-                {% endif %}
-                {% if result.img_format %}
-                    <p class="result-format">
-                        {{ result.img_format }}
-                    </p>
-                {% endif %}
-                {% if result.source %}
-                    <p class="result-source">
-                        {{ result.source }}
-                    </p>
-                {% endif %}
-            </div>
-            <div class="modal-footer">
-                <div class="clearfix"></div>
-                <span class="label label-default pull-right">{{ result.engine }}</span>
-                <p class="text-muted pull-left">{{ result.pretty_url }}</p>
-                <div class="clearfix"></div>
-                                <div class="row">
-                    <div class="col-md-6">
-                        <a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('Get image') }}</a>
-                    </div>
-                    <div class="col-md-6">
-                        <a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('View source') }}</a>
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-</div>
+                {%- if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif -%}
+                {%- if result.content %}<p class="result-content">{{ result.content|striptags }}</p>{% endif -%}
+                {%- if result.img_format %}<p class="result-format">{{ result.img_format }}</p>{% endif -%}
+                {%- if result.source %}<p class="result-source">{{ result.source }}</p>{% endif -%}
+            </div>{{- "" -}}
+            <div class="modal-footer">{{- "" -}}
+                <div class="clearfix"></div>{{- "" -}}
+                <span class="label label-default pull-right">{{ result.engine }}</span>{{- "" -}}
+                <p class="text-muted pull-left">{{ result.pretty_url }}</p>{{- "" -}}
+                <div class="clearfix"></div>{{- "" -}}
+				<div class="row">{{- "" -}}
+                    <div class="col-md-6">{{- "" -}}
+                        <a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('Get image') }}</a>{{- "" -}}
+                    </div>{{- "" -}}
+                    <div class="col-md-6">{{- "" -}}
+                        <a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('View source') }}</a>{{- "" -}}
+                    </div>{{- "" -}}
+                </div>{{- "" -}}
+            </div>{{- "" -}}
+        </div>{{- "" -}}
+    </div>{{- "" -}}
+</div>{{- "" -}}

searx/templates/oscar/results.html

@@ -1,89 +1,90 @@
 {% extends "oscar/base.html" %}
 {% macro search_form_attrs(pageno) -%}
-    {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
-    <input type="hidden" name="q" value="{{ q|e }}" />
-    <input type="hidden" name="pageno" value="{{ pageno }}" />
-    <input type="hidden" name="time_range" value="{{ time_range }}" />
-    <input type="hidden" name="language" value="{{ current_language }}" />
-    {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" />{% endif %}
+    {%- for category in selected_categories -%}<input type="hidden" name="category_{{ category }}" value="1"/>{%- endfor -%}
+    <input type="hidden" name="q" value="{{ q|e }}" />{{- "" -}}
+    <input type="hidden" name="pageno" value="{{ pageno }}" />{{- "" -}}
+    <input type="hidden" name="time_range" value="{{ time_range }}" />{{- "" -}}
+    <input type="hidden" name="language" value="{{ current_language }}" />{{- "" -}}
+    {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" />{% endif -%}
 {%- endmacro %}
 {%- macro search_url() %}{{ base_url }}?q={{ q|urlencode }}{% if selected_categories %}&amp;categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&amp;pageno={{ pageno }}{% endif %}{% if time_range %}&amp;time_range={{ time_range }}{% endif %}{% if current_language != 'all' %}&amp;language={{ current_language }}{% endif %}{% endmacro -%}
 
 {% block title %}{{ q|e }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ search_url() }}&amp;format=rss">{% endblock %}
+{% block meta %}{{"    "}}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ search_url() }}&amp;format=rss">{% endblock %}
 {% block content %}
     {% include 'oscar/search.html' %}
+
     <div class="row">
         <div class="col-sm-8" id="main_results">
             <h1 class="sr-only">{{ _('Search results') }}</h1>
 
-            {% if corrections %}
+            {% if corrections -%}
             <div class="result">
                 <span class="result_header text-muted form-inline pull-left suggestion_item">{{ _('Try searching for:') }}</span>
-                {% for correction in corrections %}
-                    <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-left suggestion_item">
-                        <input type="hidden" name="q" value="{{ correction.url }}">
-                        <button type="submit" class="btn btn-default btn-xs">{{ correction.title }}</button>
+                {% for correction in corrections -%}
+                    <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-left suggestion_item">{{- "" -}}
+                        <input type="hidden" name="q" value="{{ correction.url }}">{{- "" -}}
+                        <button type="submit" class="btn btn-default btn-xs">{{ correction.title }}</button>{{- "" -}}
                     </form>
                 {% endfor %}
             </div>
-            {% endif %}
+            {%- endif %}
 
-            {% if answers %}
-            {% for answer in answers %}
+            {% if answers -%}
+            {%- for answer in answers %}
             <div class="result well">
                 <span>{{ answer }}</span>
             </div>
-            {% endfor %}
-            {% endif %}
+            {%- endfor %}
+            {%- endif %}
 
-            {% for result in results %}
+            {% for result in results -%}
             <div class="result {% if result['template'] %}result-{{ result.template|replace('.html', '') }}{% else %}result-default{% endif %}">
-                {% set index = loop.index %}
-                {% if result.template %}
+                {%- set index = loop.index -%}
+                {%- if result.template -%}
                     {% include get_result_template('oscar', result['template']) %}
-                {% else %}
+                {%- else -%}
                     {% include 'oscar/result_templates/default.html' %}
-                {% endif %}
+                {%- endif -%}
             </div>
             {% endfor %}
 
-            {% if not results and not answers %}
+            {% if not results and not answers -%}
                 {% include 'oscar/messages/no_results.html' %}
             {% endif %}
 
             <div class="clearfix"></div>
 
-            {% if paging %}
+            {% if paging -%}
             {% if rtl %}
             <div id="pagination">
-                <div class="pull-left">
+                <div class="pull-left">{{- "" -}}
                   <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
-                        {{ search_form_attrs(pageno+1) }}
-                        <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>
-                    </form>
+                        {{- search_form_attrs(pageno+1) -}}
+                        <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>{{- "" -}}
+                    </form>{{- "" -}}
                 </div>
-                <div class="pull-right">
+                <div class="pull-right">{{- "" -}}
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"  class="pull-left">
-                        {{ search_form_attrs(pageno-1) }}
-                        <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-forward"></span> {{ _('previous page') }}</button>
-                    </form>
+                        {{- search_form_attrs(pageno-1) -}}
+                        <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-forward"></span> {{ _('previous page') }}</button>{{- "" -}}
+                    </form>{{- "" -}}
                 </div>
             </div><!-- /#pagination -->
             <div class="clearfix"></div>
             {% else %}
             <div id="pagination">
-                <div class="pull-left">
+                <div class="pull-left">{{- "" -}}
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
-                        {{ search_form_attrs(pageno-1) }}
-                        <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-backward"></span> {{ _('previous page') }}</button>
-                    </form>
+                        {{- search_form_attrs(pageno-1) -}}
+                        <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-backward"></span> {{ _('previous page') }}</button>{{- "" -}}
+                    </form>{{- "" -}}
                 </div>
-                <div class="pull-right">
+                <div class="pull-right">{{- "" -}}
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"  class="pull-left">
-                        {{ search_form_attrs(pageno+1) }}
-                        <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>
-                    </form>
+                        {{- search_form_attrs(pageno+1) -}}
+                        <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>{{- "" -}}
+                    </form>{{- "" -}}
                 </div>
             </div><!-- /#pagination -->
             <div class="clearfix"></div>
@@ -92,24 +93,24 @@
         </div><!-- /#main_results -->
 
         <div class="col-sm-4" id="sidebar_results">
-            {% if number_of_results != '0' %}
+            {% if number_of_results != '0' -%}
                 <p><small>{{ _('Number of results') }}: {{ number_of_results }}</small></p>
-            {% endif %}
+            {%- endif %}
 
-            {% if unresponsive_engines and results|length >= 1 %}
+            {% if unresponsive_engines and results|length >= 1 -%}
             <div class="alert alert-danger fade in" role="alert">
                 <p>{{ _('Engines cannot retrieve results') }}:</p>
-                {% for engine_name, error_type in unresponsive_engines %}
-                {{ engine_name }} ({{ error_type }}){% if not loop.last %}, {% endif %}
-                {% endfor %}
+                {%- for engine_name, error_type in unresponsive_engines -%}AAAA
+                {{- engine_name }} ({{ error_type }}){% if not loop.last %}, {% endif %}{{- "" -}}
+                {%- endfor -%}
             </div>
-            {% endif %}
+            {%- endif %}
 
-            {% if infoboxes %}
+            {% if infoboxes -%}
                 {% for infobox in infoboxes %}
-                    {% include 'oscar/infobox.html' %}
+                    {% include 'oscar/infobox.html' %}{{- "\n\n" -}}
                 {% endfor %}
-            {% endif %}
+            {%- endif %}
 
             {% if suggestions %}
             <div class="panel panel-default">
@@ -125,27 +126,26 @@
                     {% endfor %}
                 </div>
             </div>
-            {% endif %}
+            {%- endif %}
 
             <div class="panel panel-default">
-                <div class="panel-heading">
-                    <h4 class="panel-title">{{ _('Links') }}</h4>
+                <div class="panel-heading">{{- "" -}}
+                    <h4 class="panel-title">{{ _('Links') }}</h4>{{- "" -}}
                 </div>
                 <div class="panel-body">
-                    <form role="form">
-                        <div class="form-group">
-                            <label for="search_url">{{ _('Search URL') }}</label>
-                            <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ search_url() }}" readonly>
-                        </div>
+                    <form role="form">{{- "" -}}
+                        <div class="form-group">{{- "" -}}
+                            <label for="search_url">{{ _('Search URL') }}</label>{{- "" -}}
+                            <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ search_url() }}" readonly>{{- "" -}}
+                        </div>{{- "" -}}
                     </form>
-
                     <label>{{ _('Download results') }}</label>
                     <div class="clearfix"></div>
                     {% for output_type in ('csv', 'json', 'rss') %}
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
-                        {{ search_form_attrs(pageno) }}
-                        <input type="hidden" name="format" value="{{ output_type }}">
-                        <button type="submit" class="btn btn-default">{{ output_type }}</button>
+                        {{- search_form_attrs(pageno) -}}
+                        <input type="hidden" name="format" value="{{ output_type }}">{{- "" -}}
+                        <button type="submit" class="btn btn-default">{{ output_type }}</button>{{- "" -}}
                     </form>
                     {% endfor %}
                     <div class="clearfix"></div>

searx/templates/oscar/search.html

@@ -10,15 +10,15 @@
       </div>
     </div>
     <div class="col-xs-6 col-md-2 search-margin">
-        {% include 'oscar/time-range.html' %}
+        {%- include 'oscar/time-range.html' -%}
     </div>
     <div class="col-xs-6 col-md-2 search-margin">
-        {% include 'oscar/languages.html' %}
+        {%- include 'oscar/languages.html' -%}
     </div>
   </div>
   <div class="row">
     <div class="col-sm-12">
-        {% include 'oscar/categories.html' %}
+        {%- include 'oscar/categories.html' -%}
     </div>
   </div>
 </form><!-- / #search_form_full -->

searx/templates/oscar/time-range.html

@@ -1,17 +1,17 @@
-<select name="time_range" id="time-range" class="custom-select form-control">
+<select name="time_range" id="time-range" class="custom-select form-control">{{- "" -}}
     <option id="time-range-anytime" value="" {{ "selected" if time_range=="" or not time_range  else ""}}>
-        {{ _('Anytime') }}
-    </option>
+        {{- _('Anytime') -}}
+    </option>{{- "" -}}
     <option id="time-range-day" value="day" {{ "selected" if time_range=="day" else ""}}>
-        {{ _('Last day') }}
-    </option>
+        {{- _('Last day') -}}
+    </option>{{- "" -}}
     <option id="time-range-week" value="week" {{ "selected" if time_range=="week" else ""}}>
-        {{ _('Last week') }}
-    </option>
+        {{- _('Last week') -}}
+    </option>{{- "" -}}
     <option id="time-range-month" value="month" {{ "selected" if time_range=="month" else ""}}>
-        {{ _('Last month') }}
-    </option>
+        {{- _('Last month') -}}
+    </option>{{- "" -}}
     <option id="time-range-year" value="year" {{ "selected" if time_range=="year" else ""}}>
-        {{ _('Last year') }}
-    </option>
+        {{- _('Last year') -}}
+    </option>{{- "" -}}
 </select>

setup.py

@@ -11,14 +11,14 @@ import sys
 sys.path.insert(0, './searx')
 from version import VERSION_STRING
 
+with open('README.rst') as f:
+    long_description = f.read()
 
-def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+with open('requirements.txt') as f:
+    requirements = [ l.strip() for l in f.readlines()]
 
-
-long_description = read('README.rst')
-requirements = map(str.strip, open('requirements.txt').readlines())
-dev_requirements = map(str.strip, open('requirements-dev.txt').readlines())
+with open('requirements-dev.txt') as f:
+    dev_requirements = [ l.strip() for l in f.readlines()]
 
 setup(
     name='searx',

tests/unit/engines/test_google.py

@@ -58,93 +58,50 @@ class TestGoogleEngine(SearxTestCase):
         self.assertEqual(google.response(response), [])
 
         html = """
-        <div class="g">
-            <h3 class="r">
-                <a href="http://this.should.be.the.link/">
-                    <b>This</b> is <b>the</b> title
-                </a>
-            </h3>
-            <div class="s">
-                <div class="kv" style="margin-bottom:2px">
-                    <cite>
-                        <b>test</b>.psychologies.com/
-                    </cite>
-                    <div class="_nBb">‎
-                        <div style="display:inline" onclick="google.sham(this);" aria-expanded="false"
-                            aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">
-                            <span class="_O0">
-                            </span>
+        <div class="ZINbbc xpd O9g5cc uUPGi">
+            <div>
+                <div class="kCrYT">
+                    <a href="/url?q=http://this.should.be.the.link/">
+                        <div class="BNeawe">
+                            <b>This</b> is <b>the</b> title
                         </div>
-                        <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">
-                            <ul>
-                                <li class="_Ykb">
-                                    <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent
-                                        .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">
-                                        En cache
-                                    </a>
-                                </li>
-                                <li class="_Ykb">
-                                    <a class="_Zkb" href="/search?safe=off&amp;q=related:test.psy.com/">
-                                        Pages similaires
-                                    </a>
-                                </li>
-                            </ul>
+                        <div class="BNeawe">
+                            http://website
+                        </div>
+                    </a>
+                </div>
+                <div class="kCrYT">
+                    <div>
+                        <div class="BNeawe">
+                            <div>
+                                <div class="BNeawe">
+                                    This should be the content.
+                                </div>
+                            </div>
                         </div>
                     </div>
                 </div>
-                <span class="st">
-                    This should be the content.
-                </span>
-                <br>
-                <div class="osl">‎
-                    <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">
-                        Test Personnalité
-                    </a> - ‎
-                    <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">
-                        Tests - Moi
-                    </a> - ‎
-                    <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">
-                        Test Couple
-                    </a>
-                    - ‎
-                    <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">
-                        Test Amour
+            </div>
+        </p>
+        <div class="ZINbbc xpd O9g5cc uUPGi">
+            <div>
+                <div class="kCrYT">
+                    <span>
+                        <div class="BNeawe">
+                            Related searches
+                        </div>
+                    </span>
+                </div>
+                <div class="rVLSBd">
+                    <a>
+                        <div>
+                            <div class="BNeawe">
+                                suggestion title
+                            </div>
+                        </div>
                     </a>
                 </div>
             </div>
-        </div>
-        <div class="g">
-            <h3 class="r">
-                <a href="http://www.google.com/images?q=toto">
-                    <b>This</b>
-                </a>
-            </h3>
-        </div>
-        <div class="g">
-            <h3 class="r">
-                <a href="http://www.google.com/search?q=toto">
-                    <b>This</b> is
-                </a>
-            </h3>
-        </div>
-        <div class="g">
-            <h3 class="r">
-                <a href="€">
-                    <b>This</b> is <b>the</b>
-                </a>
-            </h3>
-        </div>
-        <div class="g">
-            <h3 class="r">
-                <a href="/url?q=url">
-                    <b>This</b> is <b>the</b>
-                </a>
-            </h3>
-        </div>
-        <p class="_Bmc" style="margin:3px 8px">
-            <a href="/search?num=20&amp;safe=off&amp;q=t&amp;revid=1754833769&amp;sa=X&amp;ei=-&amp;ved=">
-                suggestion <b>title</b>
-            </a>
         </p>
         """
         response = self.mock_response(html)

utils/makefile.include

@@ -0,0 +1,128 @@
+# -*- coding: utf-8; mode: makefile-gmake -*-
+
+make-help:
+	@echo  '  make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
+	@echo  '  make V=2   [targets] 2 => give reason for rebuild of target'
+
+quiet_cmd_common_clean = CLEAN     $@
+      cmd_common_clean = \
+	rm -rf tests/build ;\
+	find . -name '*.orig' -exec rm -f {} +     ;\
+	find . -name '*.rej' -exec rm -f {} +      ;\
+	find . -name '*~' -exec rm -f {} +         ;\
+	find . -name '*.bak' -exec rm -f {} +      ;\
+
+FMT = cat
+ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0)
+FMT = fmt
+endif
+
+# MS-Windows
+#
+# For a minimal *make-environment*, I'am using the gnu-tools from:
+#
+# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc.
+#   https://github.com/gnu-mcu-eclipse/windows-build-tools/releases
+#
+# - git for Windows, which brings 'find', 'grep' etc.
+#   https://git-scm.com/download/win
+
+
+# normpath
+#
+# System-dependent normalization of the path name
+#
+#   usage: $(call normpath,/path/to/file)
+
+normpath = $1
+ifeq ($(OS),Windows_NT)
+  normpath = $(subst /,\,$1)
+endif
+
+
+# stolen from linux/Makefile
+#
+
+ifeq ("$(origin V)", "command line")
+  KBUILD_VERBOSE = $(V)
+endif
+ifndef KBUILD_VERBOSE
+  KBUILD_VERBOSE = 0
+endif
+
+ifeq ($(KBUILD_VERBOSE),1)
+  quiet =
+  Q =
+else
+  quiet=quiet_
+  Q = @
+endif
+
+# stolen from linux/scripts/Kbuild.include
+#
+
+# Convenient variables
+comma   := ,
+quote   := "
+#" this comment is only for emacs highlighting
+squote  := '
+#' this comment is only for emacs highlighting
+empty   :=
+space   := $(empty) $(empty)
+space_escape := _-_SPACE_-_
+
+# Find any prerequisites that is newer than target or that does not exist.
+# PHONY targets skipped in both cases.
+any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^)
+#
+###
+# why - tell why a a target got build
+#       enabled by make V=2
+#       Output (listed in the order they are checked):
+#          (1) - due to target is PHONY
+#          (2) - due to target missing
+#          (3) - due to: file1.h file2.h
+#          (4) - due to command line change
+#          (5) - due to missing .cmd file
+#          (6) - due to target not in $(targets)
+# (1) PHONY targets are always build
+# (2) No target, so we better build it
+# (3) Prerequisite is newer than target
+# (4) The command line stored in the file named dir/.target.cmd
+#     differed from actual command line. This happens when compiler
+#     options changes
+# (5) No dir/.target.cmd file (used to store command line)
+# (6) No dir/.target.cmd file and target not listed in $(targets)
+#     This is a good hint that there is a bug in the kbuild file
+ifeq ($(KBUILD_VERBOSE),2)
+why =                                                                        \
+    $(if $(filter $@, $(PHONY)),- due to target is PHONY,                    \
+        $(if $(wildcard $@),                                                 \
+            $(if $(strip $(any-prereq)),- due to: $(any-prereq),             \
+                $(if $(arg-check),                                           \
+                    $(if $(cmd_$@),- due to command line change,             \
+                        $(if $(filter $@, $(targets)),                       \
+                            - due to missing .cmd file,                      \
+                            - due to $(notdir $@) not in $$(targets)         \
+                         )                                                   \
+                     )                                                       \
+                 )                                                           \
+             ),                                                              \
+             - due to target missing                                         \
+         )                                                                   \
+     )
+
+echo-why = $(call escsq, $(strip $(why)))
+endif
+#
+###
+# Escape single quote for use in echo statements
+escsq = $(subst $(squote),'\$(squote)',$1)
+#
+# echo command.
+# Short version is used, if $(quiet) equals `quiet_', otherwise full one.
+echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';)
+#
+# printing commands
+cmd = @$(echo-cmd) $(cmd_$(1))
+

utils/makefile.python

@@ -0,0 +1,290 @@
+# -*- coding: utf-8; mode: makefile-gmake -*-
+
+# list of python packages (folders) or modules (files) of this build
+PYOBJECTS ?=
+
+SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python
+export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
+
+# folder where the python distribution takes place
+PYDIST   ?= ./py_dist
+# folder where the python intermediate build files take place
+PYBUILD  ?= ./py_build
+# python version to use
+PY       ?=3
+PYTHON   ?= python$(PY)
+PIP      ?= pip$(PY)
+PIP_INST ?= --user
+
+# https://www.python.org/dev/peps/pep-0508/#extras
+#PY_SETUP_EXTRAS ?= \[develop,test\]
+PY_SETUP_EXTRAS ?=
+
+PYDEBUG  ?= --pdb
+PYLINT_RC ?= .pylintrc
+
+TEST_FOLDER  ?= ./tests
+TEST         ?= .
+
+VTENV_OPTS   = "--no-site-packages"
+PY_ENV       = ./local/py$(PY)
+PY_ENV_BIN   = $(PY_ENV)/bin
+PY_ENV_ACT   = . $(PY_ENV_BIN)/activate
+
+ifeq ($(OS),Windows_NT)
+  PYTHON     = python
+  PY_ENV_BIN = $(PY_ENV)/Scripts
+  PY_ENV_ACT = $(PY_ENV_BIN)/activate
+endif
+
+ifeq ($(PYTHON),python)
+  VIRTUALENV   = virtualenv
+else
+  VIRTUALENV   = virtualenv --python=$(PYTHON)
+endif
+
+ifeq ($(KBUILD_VERBOSE),1)
+  PIP_VERBOSE =
+  VIRTUALENV_VERBOSE =
+else
+  PIP_VERBOSE = "-q"
+  VIRTUALENV_VERBOSE = "-q"
+endif
+
+python-help::
+	@echo  'makefile.python:'
+	@echo  '  pyenv | pyenv[un]install'
+	@echo  '     build $(PY_ENV) & [un]install python objects'
+	@echo  '  targts using pyenv $(PY_ENV):'
+	@echo  '    pylint    - run pylint *linting*'
+	@echo  '    pytest    - run *tox* test on python objects'
+	@echo  '    pydebug   - run tests within a PDB debug session'
+	@echo  '    pybuild   - build python packages'
+	@echo  '    pyclean   - clean intermediate python objects'
+	@echo  '  targets using system users environment:'
+	@echo  '    py[un]install - [un]install python objects in editable mode'
+	@echo  '    upload-pypi   - upload $(PYDIST)/* files to PyPi'
+	@echo  'options:'
+	@echo  '  make PY=2  [targets] => to eval targets with python 2 ($(PY))'
+	@echo  '  make PIP_INST=       => to set/unset pip install options ($(PIP_INST))'
+	@echo  '  make TEST=.          => choose test from $(TEST_FOLDER) (default "." runs all)'
+	@echo  '  make DEBUG=          => target "debug": do not invoke PDB on errors'
+	@echo  '  make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]'
+	@echo  '  when using target "pydebug", set breakpoints within py-source by adding::'
+	@echo  '    DEBUG()'
+
+# ------------------------------------------------------------------------------
+# OS requirements
+# ------------------------------------------------------------------------------
+
+PHONY += msg-python-exe python-exe
+msg-python-exe:
+	@echo "\n  $(PYTHON) is required\n\n\
+  Make sure you have $(PYTHON) installed, grab it from\n\
+  https://www.python.org or install it from your package\n\
+  manager. On debian based OS these requirements are\n\
+  installed by::\n\n\
+    sudo -H apt-get install $(PYTHON)\n" | $(FMT)
+
+ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1)
+python-exe: msg-python-exe
+	$(error The '$(PYTHON)' command was not found)
+else
+python-exe:
+	@:
+endif
+
+msg-pip-exe:
+	@echo "\n  $(PIP) is required\n\n\
+  Make sure you have updated pip installed, grab it from\n\
+  https://pip.pypa.io or install it from your package\n\
+  manager. On debian based OS these requirements are\n\
+  installed by::\n\n\
+    sudo -H apt-get install python$(PY)-pip\n" | $(FMT)
+
+ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)
+pip-exe: msg-pip-exe
+	$(error The '$(PIP)' command was not found)
+else
+pip-exe:
+	@:
+endif
+
+PHONY += msg-virtualenv-exe virtualenv-exe
+msg-virtualenv-exe:
+	@echo "\n  virtualenv is required\n\n\
+  Make sure you have an updated virtualenv installed, grab it from\n\
+  https://virtualenv.pypa.io/en/stable/installation/ or install it\n\
+  via pip by::\n\n\
+    pip install --user https://github.com/pypa/virtualenv/tarball/master\n" | $(FMT)
+
+ifeq ($(shell which virtualenv >/dev/null 2>&1; echo $$?), 1)
+virtualenv-exe: msg-virtualenv-exe
+	$(error The 'virtualenv' command was not found)
+else
+virtualenv-exe:
+	@:
+endif
+
+# ------------------------------------------------------------------------------
+# commands
+# ------------------------------------------------------------------------------
+
+# $2 path to folder with setup.py, this uses pip from the OS
+quiet_cmd_pyinstall = INSTALL   $2
+      cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS)
+
+# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
+quiet_cmd_pyenvinstall = PYENV     install $2
+      cmd_pyenvinstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS)
+
+# Uninstall the package. Since pip does not uninstall the no longer needed
+# depencies (something like autoremove) the depencies remain.
+
+# $2 package name to uninstall, this uses pip from the OS.
+quiet_cmd_pyuninstall = UNINSTALL $2
+      cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2
+
+# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
+quiet_cmd_pyenvuninstall = PYENV     uninstall   $2
+      cmd_pyenvuninstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes $2
+
+# $2 path to folder where virtualenv take place
+quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
+      cmd_virtualenv  = \
+	if [ ! -d "./$(PY_ENV)" ];then                                  \
+		$(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2;   \
+	else                                                            \
+		echo "  PYENV     using virtualenv from $2";            \
+        fi
+
+# $2 path to lint
+quiet_cmd_pylint      = LINT      $@
+      cmd_pylint      = $(PY_ENV_BIN)/pylint --rcfile $(PYLINT_RC) $2
+
+quiet_cmd_pytest      = TEST      $@
+      cmd_pytest      = $(PY_ENV_BIN)/tox -vv
+
+# setuptools, pip, easy_install its a mess full of cracks, a documentation hell
+# and broken by design ... all sucks, I really, really hate all this ... aaargh!
+#
+# About python packaging see `Python Packaging Authority`_.  Most of the names
+# here are mapped to ``setup(<name1>=..., <name2>=...)`` arguments in
+# ``setup.py``.  See `Packaging and distributing projects`_ about ``setup(...)``
+# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_.
+#
+# Further read:
+#
+# - pythonwheels_
+# - setuptools_
+# - packaging_
+# - sdist_
+# - installing_
+#
+# .. _`Python Packaging Authority`: https://www.pypa.io
+# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/
+# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/
+# .. _pythonwheels: https://pythonwheels.com/
+# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html
+# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects
+# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions
+# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels
+# .. _installing: https://packaging.python.org/tutorials/installing-packages/
+#
+quiet_cmd_pybuild     = BUILD     $@
+      cmd_pybuild     = $(PY_ENV_BIN)/$(PYTHON) setup.py \
+			sdist -d $(PYDIST)  \
+			bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST)
+
+quiet_cmd_pyclean     = CLEAN     $@
+# remove 'build' folder since bdist_wheel does not care the --bdist-dir
+      cmd_pyclean     = \
+	rm -rf $(PYDIST) $(PYBUILD) ./local ./.tox *.egg-info     ;\
+	find . -name '*.pyc' -exec rm -f {} +      ;\
+	find . -name '*.pyo' -exec rm -f {} +      ;\
+	find . -name __pycache__ -exec rm -rf {} +
+
+# ------------------------------------------------------------------------------
+# targets
+# ------------------------------------------------------------------------------
+
+# for installation use the pip from the OS!
+PHONY += pyinstall
+pyinstall: pip-exe
+	$(call cmd,pyinstall,.)
+
+PHONY += pyuninstall
+pyuninstall: pip-exe
+	$(call cmd,pyuninstall,$(PYOBJECTS))
+
+# for installation use the pip from PY_ENV (not the OS)!
+PHONY += pyenvinstall
+pyenvinstall: $(PY_ENV)
+	$(call cmd,pyenvinstall,.)
+
+PHONY += pyenvuninstall
+pyenvuninstall: $(PY_ENV)
+	$(call cmd,pyenvuninstall,$(PYOBJECTS))
+
+PHONY += pyclean
+pyclean:
+	$(call cmd,pyclean)
+
+# to build *local* environment, python and virtualenv from the OS is needed!
+pyenv: $(PY_ENV)
+$(PY_ENV): virtualenv-exe python-exe
+	$(call cmd,virtualenv,$(PY_ENV))
+	@$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) -r requirements.txt
+
+PHONY += pylint-exe
+pylint-exe: $(PY_ENV)
+	@$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install pylint
+
+PHONY += pylint
+pylint: pylint-exe
+	$(call cmd,pylint,$(PYOBJECTS))
+
+PHONY += pybuild
+pybuild: $(PY_ENV)
+	$(call cmd,pybuild)
+
+PHONY += pytest
+pytest: $(PY_ENV)
+	$(call cmd,pytest)
+
+PHONY += pydebug
+# set breakpoint with:
+#    DEBUG()
+# e.g. to run tests in debug mode in emacs use:
+#   'M-x pdb' ... 'make pydebug'
+pydebug: $(PY_ENV)
+	DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST)
+
+# install / uninstall python objects into virtualenv (PYENV)
+pyenv-install: $(PY_ENV)
+	@$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e .
+	@echo "  ACTIVATE  $(call normpath,$(PY_ENV_ACT)) "
+
+pyenv-uninstall: $(PY_ENV)
+	@$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes .
+
+# runs python interpreter from ./local/py<N>/bin/python
+pyenv-python: pyenv-install
+	cd ./local; ../$(PY_ENV_BIN)/python -i
+
+# With 'dependency_links=' setuptools supports dependencies on packages hosted
+# on other reposetories then PyPi, see "Packages Not On PyPI" [1].  The big
+# drawback is, due to security reasons (I don't know where the security gate on
+# PyPi is), this feature is not supported by pip [2]. Thats why an upload to
+# PyPi is required and since uploads via setuptools is not recommended, we have
+# to imstall / use twine ... its really a mess.
+#
+# [1] http://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi
+# [2] https://github.com/pypa/pip/pull/1519
+
+# https://github.com/pypa/twine
+PHONY += upload-pypi
+upload-pypi: pyclean pybuild
+	@$(PY_ENV_BIN)/twine upload $(PYDIST)/*
+
+.PHONY: $(PHONY)