diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 99cd541456..c285549d31 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -8,6 +8,37 @@ trigger:
 
 jobs:
 
+- job: 'Python3'
+  pool:
+    vmImage: 'ubuntu-latest'
+  container:
+    image: 'archlinux/base:latest'
+    # See https://github.com/Microsoft/azure-pipelines-agent/issues/2043
+    # for why we need this hack to get root in the container
+    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
+
+  steps:
+  - script: |
+      set -xe
+      /tmp/docker exec -t -u 0 ci-container sh -c "pacman -Syu --noconfirm"
+      /tmp/docker exec -t -u 0 ci-container \
+      sh -c "pacman -S --noconfirm --needed base-devel sudo git sip chmlib icu jxrlib hunspell libmtp libusbx libwmf optipng podofo python-apsw python-beautifulsoup4 python-cssselect python-css-parser python-dateutil python-dbus python-dnspython python-dukpy python-feedparser python-html2text python-html5-parser python-lxml python-markdown python-mechanize python-msgpack python-netifaces python-unrardll python-pillow python-psutil python-pygments python-pyqt5 python-regex python-zeroconf python-pyqtwebengine qt5-x11extras qt5-svg qt5-imageformats udisks2"
+    displayName: 'Install calibre dependencies'
+
+  - checkout: self
+    fetchDepth: 1
+
+  - script: |
+      python setup.py bootstrap --ephemeral
+    displayName: 'Bootstrap calibre'
+    env: { 'CALIBRE_PY3_PORT': '1' }
+
+  - script: |
+      python setup.py test
+    displayName: 'Test calibre'
+    env: { 'CALIBRE_PY3_PORT': '1' }
+
+
 - job: 'Linux'
   pool:
     vmImage: 'ubuntu-latest'