new release: 0.10.7-stable

Bumps [django-debug-toolbar](https://github.com/jazzband/django-debug-toolbar) from 3.1 to 3.1.1.
- [Release notes](https://github.com/jazzband/django-debug-toolbar/releases)
- [Changelog](https://github.com/jazzband/django-debug-toolbar/blob/master/docs/changes.rst)
- [Commits](https://github.com/jazzband/django-debug-toolbar/compare/3.1...3.1.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.5-stable
+current_version = 0.10.7-stable
 tag = True
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\-(?P<release>.*)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Logs**
+Output of docker-compose logs or kubectl logs respectively
+
+**Version and Deployment (please complete the following information):**
+ - passbook version: [e.g. 0.10.0-stable]
+ - Deployment: [e.g. docker-compose, helm]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+- package-ecosystem: gomod
+  directory: "/proxy"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+- package-ecosystem: npm
+  directory: "/passbook/static/static"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  assignees:
+  - BeryJu

.github/workflows/release.yml

@@ -18,11 +18,11 @@ jobs:
       - name: Building Docker Image
         run: docker build
           --no-cache
-          -t beryju/passbook:0.10.5-stable
+          -t beryju/passbook:0.10.7-stable
           -t beryju/passbook:latest
           -f Dockerfile .
       - name: Push Docker Container to Registry (versioned)
-        run: docker push beryju/passbook:0.10.5-stable
+        run: docker push beryju/passbook:0.10.7-stable
       - name: Push Docker Container to Registry (latest)
         run: docker push beryju/passbook:latest
   build-proxy:
@@ -48,11 +48,11 @@ jobs:
           cd proxy
           docker build \
           --no-cache \
-          -t beryju/passbook-proxy:0.10.5-stable \
+          -t beryju/passbook-proxy:0.10.7-stable \
           -t beryju/passbook-proxy:latest \
           -f Dockerfile .
       - name: Push Docker Container to Registry (versioned)
-        run: docker push beryju/passbook-proxy:0.10.5-stable
+        run: docker push beryju/passbook-proxy:0.10.7-stable
       - name: Push Docker Container to Registry (latest)
         run: docker push beryju/passbook-proxy:latest
   build-static:
@@ -77,11 +77,11 @@ jobs:
         run: docker build
           --no-cache
           --network=$(docker network ls | grep github | awk '{print $1}')
-          -t beryju/passbook-static:0.10.5-stable
+          -t beryju/passbook-static:0.10.7-stable
           -t beryju/passbook-static:latest
           -f static.Dockerfile .
       - name: Push Docker Container to Registry (versioned)
-        run: docker push beryju/passbook-static:0.10.5-stable
+        run: docker push beryju/passbook-static:0.10.7-stable
       - name: Push Docker Container to Registry (latest)
         run: docker push beryju/passbook-static:latest
   test-release:
@@ -114,5 +114,5 @@ jobs:
           SENTRY_PROJECT: passbook
           SENTRY_URL: https://sentry.beryju.org
         with:
-          tagName: 0.10.5-stable
+          tagName: 0.10.7-stable
           environment: beryjuorg-prod

Makefile

@@ -8,12 +8,12 @@ coverage:
 
 lint-fix:
 	isort -rc .
-	black .
+	black passbook e2e lifecycle
 
 lint:
-	pyright
-	bandit -r .
-	pylint passbook
+	pyright pyright e2e lifecycle
+	bandit -r passbook e2e lifecycle
+	pylint passbook e2e lifecycle
 	prospector
 
 gen: coverage

Pipfile.lock

@@ -25,10 +25,10 @@
         },
         "amqp": {
             "hashes": [
-                "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21",
-                "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"
+                "sha256:9881f8e6fe23e3db9faa6cfd8c05390213e1d1b95c0162bc50552cad75bffa5f",
+                "sha256:a8fb8151eb9d12204c9f1784c0da920476077609fa0a70f2468001e3a4258484"
             ],
-            "version": "==2.6.1"
+            "version": "==5.0.1"
         },
         "asgiref": {
             "hashes": [
@@ -74,17 +74,18 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:44073b1b1823ffc9edcf9027afbca908dad6bd5000f512ca73f929f6a604ae24"
+                "sha256:0c464a7de522f88b581ca0d41ffa71e9be5e17fbb0456c275421f65b7c5f6a55",
+                "sha256:0fce548e19d6db8e11fd0e2ae7809e1e3282080636b4062b2452bfa20e4f0233"
             ],
             "index": "pypi",
-            "version": "==1.15.1"
+            "version": "==1.15.5"
         },
         "botocore": {
             "hashes": [
-                "sha256:6bdf60281c2e80360fe904851a1a07df3dcfe066fe88dc7fba2b5e626ac05c8c",
-                "sha256:d6bdf51c8880aa9974e6b61d2f7d9d1debe407287e2e9e60f36c789fe8ba6790"
+                "sha256:7ce7a05b98ffb3170396960273383e8aade9be6026d5a762f5f40969d5d6b761",
+                "sha256:e3bf44fba058f6df16006b94a67650418a080a525c82521abb3cb516a4cba362"
             ],
-            "version": "==1.18.1"
+            "version": "==1.18.5"
         },
         "cachetools": {
             "hashes": [
@@ -95,11 +96,11 @@
         },
         "celery": {
             "hashes": [
-                "sha256:a92e1d56e650781fb747032a3997d16236d037c8199eacd5217d1a72893bca45",
-                "sha256:d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f"
+                "sha256:313930fddde703d8e37029a304bf91429cd11aeef63c57de6daca9d958e1f255",
+                "sha256:72138dc3887f68dc58e1a2397e477256f80f1894c69fa4337f8ed70be460375b"
             ],
             "index": "pypi",
-            "version": "==4.4.7"
+            "version": "==5.0.0"
         },
         "certifi": {
             "hashes": [
@@ -179,6 +180,19 @@
             ],
             "version": "==7.1.2"
         },
+        "click-didyoumean": {
+            "hashes": [
+                "sha256:112229485c9704ff51362fe34b2d4f0b12fc71cc20f6d2b3afabed4b8bfa6aeb"
+            ],
+            "version": "==0.0.3"
+        },
+        "click-repl": {
+            "hashes": [
+                "sha256:9c4c3d022789cae912aad8a3f5e1d7c2cdd016ee1225b5212ad3e8691563cda5",
+                "sha256:b9f29d52abc4d6059f8e276132a111ab8d94980afe6a5432b9d996544afa95d5"
+            ],
+            "version": "==0.1.6"
+        },
         "constantly": {
             "hashes": [
                 "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35",
@@ -386,10 +400,10 @@
         },
         "google-auth": {
             "hashes": [
-                "sha256:7084c50c03f7a8a5696ef4500e65df0c525a0f6909f3c70b9ee65900a230c755",
-                "sha256:dcf86c5adc3a8a7659be190b12bb8912ae019cfd9ee2a571ea881e289fafbe39"
+                "sha256:31941bf019fb242c04d0de32845da10180788bfddb0de87d78c4bdf55555dda1",
+                "sha256:873051a6317294b083795cffc467bcd05b6df483ef542bfe0069ddbfbac0a096"
             ],
-            "version": "==1.21.2"
+            "version": "==1.21.3"
         },
         "gunicorn": {
             "hashes": [
@@ -533,10 +547,10 @@
         },
         "kombu": {
             "hashes": [
-                "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
-                "sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74"
+                "sha256:6dc509178ac4269b0e66ab4881f70a2035c33d3a622e20585f965986a5182006",
+                "sha256:f4965fba0a4718d47d470beeb5d6446e3357a62402b16c510b6a2f251e05ac3c"
             ],
-            "version": "==4.6.11"
+            "version": "==5.0.2"
         },
         "kubernetes": {
             "hashes": [
@@ -674,6 +688,13 @@
             ],
             "version": "==0.8.0"
         },
+        "prompt-toolkit": {
+            "hashes": [
+                "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489",
+                "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950"
+            ],
+            "version": "==3.0.7"
+        },
         "psycopg2-binary": {
             "hashes": [
                 "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
@@ -748,20 +769,25 @@
                 "sha256:54bdedd28476dea8a3cd86cb67c0df1f0e3d71cae8022354b0f879c41a3d27b2",
                 "sha256:55eb61aca2c883db770999f50d091ff7c14016f2769ad7bca3d9b75d1d7c1b68",
                 "sha256:6276478ada411aca97c0d5104916354b3d740d368407912722bd4d11aa9ee4c2",
+                "sha256:663f8de2b3df2e744d6e1610506e0ea4e213bde906795953c1e82279c169f0a7",
                 "sha256:67dcad1b8b201308586a8ca2ffe89df1e4f731d5a4cdd0610cc4ea790351c739",
                 "sha256:709b9f144d23e290b9863121d1ace14a72e01f66ea9c903fbdc690520dfdfcf0",
                 "sha256:8063a712fba642f78d3c506b0896846601b6de7f5c3d534e388ad0cc07f5a149",
                 "sha256:80d57177a0b7c14d4594c62bbb47fe2f6309ad3b0a34348a291d570925c97a82",
+                "sha256:87006cf0d81505408f1ae4f55cf8a5d95a8e029a4793360720ae17c6500f7ecc",
+                "sha256:9f62d21bc693f3d7d444f17ed2ad7a913b4c37c15cd807895d013c39c0517dfd",
                 "sha256:a207231a52426de3ff20f5608f0687261a3329d97a036c51f7d4c606a6f30c23",
                 "sha256:abc2e126c9490e58a36a0f83516479e781d83adfb134576a5cbe5c6af2a3e93c",
                 "sha256:b56638d58a3a4be13229c6a815cd448f9e3ce40c00880a5398471b42ee86f50e",
                 "sha256:bcd5b8416e73e4b0d48afba3704d8c826414764dafaed7a1a93c442188d90ccc",
                 "sha256:bec2bcdf7c9ce7f04d718e51887f3b05dc5c1cfaf5d2c2e9065ecddd1b2f6c9a",
                 "sha256:c8bf40cf6e281a4378e25846924327e728a887e8bf0ee83b2604a0f4b61692e8",
+                "sha256:cecbf67e81d6144a50dc615629772859463b2e4f815d0c082fa421db362f040e",
                 "sha256:d8074c8448cfd0705dfa71ca333277fce9786d0b9cac75d120545de6253f996a",
                 "sha256:dd302b6ae3965afeb5ef1b0d92486f986c0e65183cd7835973f0b593800590e6",
                 "sha256:de6e1cd75677423ff64712c337521e62e3a7a4fc84caabbd93207752e831a85a",
                 "sha256:ef39c98d9b8c0736d91937d193653e47c3b19ddf4fc3bccdc5e09aaa4b0c5d21",
+                "sha256:f2e045224074d5664dc9cbabbf4f4d4d46f1ee90f24780e3a9a668fd096ff17f",
                 "sha256:f521178e5a991ffd04182ed08f552daca1affcb826aeda0e1945cd989a9d4345",
                 "sha256:f78a68c2c820e4731e510a2df3eef0322f24fde1781ced970bf497b6c7d92982",
                 "sha256:fbe65d5cfe04ff2f7684160d50f5118bdefb01e3af4718eeb618bfed40f19d94"
@@ -778,12 +804,14 @@
                 "sha256:2275a663c9e744ee4eace816ef2d446b3060554c5773a92fbc79b05bf47debda",
                 "sha256:2710fc8d83b3352b370db932b3710033b9d630b970ff5aaa3e7458b5336e3b32",
                 "sha256:35b9c9177a9fe7288b19dd41554c9c8ca1063deb426dd5a02e7e2a7416b6bd11",
+                "sha256:3b23d63030819b7d9ac7db9360305fd1241e6870ca5b7e8d59fee4db4674a490",
                 "sha256:3caa32cf807422adf33c10c88c22e9e2e08b9d9d042f12e1e25fe23113dd618f",
                 "sha256:48cc2cfc251f04a6142badeb666d1ff49ca6fdfc303fd72579f62b768aaa52b9",
                 "sha256:4ae6379350a09339109e9b6f419bb2c3f03d3e441f4b0f5b8ca699d47cc9ff7e",
                 "sha256:4e0b27697fa1621c6d3d3b4edeec723c2e841285de6a8d378c1962da77b349be",
                 "sha256:58e19560814dabf5d788b95a13f6b98279cf41a49b1e49ee6cf6c79a57adb4c9",
                 "sha256:8044eae59301dd392fbb4a7c5d64e1aea8ef0be2540549807ecbe703d6233d68",
+                "sha256:85c108b42e47d4073344ff61d4e019f1d95bb7725ca0fe87d0a2deb237c10e49",
                 "sha256:89be1bf55e50116fe7e493a7c0c483099770dd7f81b87ac8d04a43b1a203e259",
                 "sha256:8fcdda24dddf47f716400d54fc7f75cadaaba1dd47cc127e59d752c9c0fc3c48",
                 "sha256:914fbb18e29c54585e6aa39d300385f90d0fa3b3cc02ed829b08f95c1acf60c2",
@@ -793,13 +821,16 @@
                 "sha256:a2ee8ba99d33e1a434fcd27d7d0aa7964163efeee0730fe2efc9d60edae1fc71",
                 "sha256:b2d756620078570d3f940c84bc94dd30aa362b795cce8b2723300a8800b87f1c",
                 "sha256:c0d085c8187a1e4d3402f626c9e438b5861151ab132d8761d9c5ce6491a87761",
+                "sha256:c315262e26d54a9684e323e37ac9254f481d57fcc4fd94002992460898ef5c04",
                 "sha256:c990f2c58f7c67688e9e86e6557ed05952669ff6f1343e77b459007d85f7df00",
                 "sha256:ccbbec59bf4b74226170c54476da5780c9176bae084878fc94d9a2c841218e34",
                 "sha256:dc2bed32c7b138f1331794e454a953360c8cedf3ee62ae31f063822da6007489",
+                "sha256:ddb1ae2891c8cb83a25da87a3e00111a9654fc5f0b70f18879c41aece45d6182",
                 "sha256:e070a1f91202ed34c396be5ea842b886f6fa2b90d2db437dc9fb35a26c80c060",
                 "sha256:e42860fbe1292668b682f6dabd225fbe2a7a4fa1632f0c39881c019e93dea594",
                 "sha256:e4e1c486bf226822c8dceac81d0ec59c0a2399dbd1b9e04f03c3efa3605db677",
                 "sha256:ea4d4b58f9bc34e224ef4b4604a6be03d72ef1f8c486391f970205f6733dbc46",
+                "sha256:f5bd6891380e0fb5467251daf22525644fdf6afd9ae8bc2fe065c78ea1882e0d",
                 "sha256:f60b3484ce4be04f5da3777c51c5140d3fe21cdd6674f2b6568f41c8130bcdeb"
             ],
             "version": "==3.9.8"
@@ -951,11 +982,11 @@
         },
         "sentry-sdk": {
             "hashes": [
-                "sha256:1a086486ff9da15791f294f6e9915eb3747d161ef64dee2d038a4d0b4a369b24",
-                "sha256:45486deb031cea6bbb25a540d7adb4dd48cd8a1cc31e6a5ce9fb4f792a572e9a"
+                "sha256:c9c0fa1412bad87104c4eee8dd36c7bbf60b0d92ae917ab519094779b22e6d9a",
+                "sha256:e159f7c919d19ae86e5a4ff370fccc45149fab461fbeb93fb5a735a0b33a9cb1"
             ],
             "index": "pypi",
-            "version": "==0.17.6"
+            "version": "==0.17.8"
         },
         "service-identity": {
             "hashes": [
@@ -1085,10 +1116,17 @@
         },
         "vine": {
             "hashes": [
-                "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
-                "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
+                "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30",
+                "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e"
             ],
-            "version": "==1.3.0"
+            "version": "==5.0.0"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
+                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+            ],
+            "version": "==0.2.5"
         },
         "websocket-client": {
             "hashes": [
@@ -1316,11 +1354,11 @@
         },
         "django-debug-toolbar": {
             "hashes": [
-                "sha256:eabbefe89881bbe4ca7c980ff102e3c35c8e8ad6eb725041f538988f2f39a943",
-                "sha256:ff94725e7aae74b133d0599b9bf89bd4eb8f5d2c964106e61d11750228c8774c"
+                "sha256:a1ce0665f7ef47d27b8df4b5d1058748e1f08500a01421a30d35164f38aaaf4c",
+                "sha256:c97921a9cd421d392e7860dc4b464db8e06c8628df4dc58fedab012888c293c6"
             ],
             "index": "pypi",
-            "version": "==2.2"
+            "version": "==3.1.1"
         },
         "docker": {
             "hashes": [

azure-pipelines.yml

@@ -8,6 +8,10 @@ variables:
   POSTGRES_DB: passbook
   POSTGRES_USER: passbook
   POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77"
+  ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}:
+    branchName: ${{ replace(variables['Build.SourceBranchName'], 'refs/heads/', '') }}
+  ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
+    branchName: ${{ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') }}
 
 stages:
   - stage: Lint
@@ -26,7 +30,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pylint passbook
+              script: pipenv run pylint passbook e2e lifecycle
       - job: black
         pool:
           vmImage: 'ubuntu-latest'
@@ -41,7 +45,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run black --check passbook
+              script: pipenv run black --check passbook e2e lifecycle
       - job: prospector
         pool:
           vmImage: 'ubuntu-latest'
@@ -57,7 +61,7 @@ stages:
                 pipenv install --dev prospector --skip-lock
           - task: CmdLine@2
             inputs:
-              script: pipenv run prospector passbook
+              script: pipenv run prospector
       - job: bandit
         pool:
           vmImage: 'ubuntu-latest'
@@ -72,7 +76,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run bandit -r passbook
+              script: pipenv run bandit -r passbook e2e lifecycle
       - job: pyright
         pool:
           vmImage: ubuntu-latest
@@ -93,7 +97,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pyright
+              script: pipenv run pyright e2e lifecycle
   - stage: Test
     jobs:
       - job: migrations
@@ -117,6 +121,41 @@ stages:
           - task: CmdLine@2
             inputs:
               script: pipenv run ./manage.py migrate
+      - job: migrations_from_previous_release
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.8'
+          - task: DockerCompose@0
+            displayName: Run services
+            inputs:
+              dockerComposeFile: 'scripts/ci.docker-compose.yml'
+              action: 'Run services'
+              buildImages: false
+          - task: CmdLine@2
+            displayName: Prepare Last tagged release
+            inputs:
+              script: |
+                git checkout $(git describe --abbrev=0 --match 'version/*')
+                sudo pip install -U wheel pipenv
+                pipenv install --dev
+          - task: CmdLine@2
+            displayName: Migrate to last tagged release
+            inputs:
+              script: pipenv run ./manage.py migrate
+          - task: CmdLine@2
+            displayName: Install current branch
+            inputs:
+              script: |
+                set -x
+                git checkout ${{ variables.branchName }}
+                pipenv sync --dev
+          - task: CmdLine@2
+            displayName: Migrate to current branch
+            inputs:
+              script: pipenv run ./manage.py migrate
       - job: coverage_unittest
         pool:
           vmImage: 'ubuntu-latest'
@@ -140,6 +179,9 @@ stages:
             inputs:
               script: |
                 pipenv run coverage run ./manage.py test passbook -v 3
+          - task: CmdLine@2
+            inputs:
+              script: |
                 mkdir output-unittest
                 mv unittest.xml output-unittest/unittest.xml
                 mv .coverage output-unittest/coverage
@@ -182,7 +224,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
-                pipenv run coverage run ./manage.py test e2e -v 3
+                pipenv run coverage run ./manage.py test e2e -v 3 --failfast
           - task: CmdLine@2
             condition: always()
             displayName: Cleanup
@@ -265,7 +307,7 @@ stages:
             repository: 'beryju/passbook'
             command: 'buildAndPush'
             Dockerfile: 'Dockerfile'
-            tags: 'gh-$(Build.SourceBranchName)'
+            tags: "gh-${{ variables.branchName }}"
       - job: build_static
         pool:
           vmImage: 'ubuntu-latest'
@@ -282,14 +324,14 @@ stages:
             repository: 'beryju/passbook-static'
             command: 'build'
             Dockerfile: 'static.Dockerfile'
-            tags: 'gh-$(Build.SourceBranchName)'
+            tags: "gh-${{ variables.branchName }}"
             arguments: "--network=beryjupassbook_default"
         - task: Docker@2
           inputs:
             containerRegistry: 'dockerhub'
             repository: 'beryju/passbook-static'
             command: 'push'
-            tags: 'gh-$(Build.SourceBranchName)'
+            tags: "gh-${{ variables.branchName }}"
   - stage: Deploy
     jobs:
       - job: deploy_dev

docker-compose.yml

@@ -23,7 +23,7 @@ services:
     labels:
       - traefik.enable=false
   server:
-    image: beryju/passbook:${PASSBOOK_TAG:-0.10.5-stable}
+    image: beryju/passbook:${PASSBOOK_TAG:-0.10.7-stable}
     command: server
     environment:
       PASSBOOK_REDIS__HOST: redis
@@ -41,7 +41,7 @@ services:
     env_file:
       - .env
   worker:
-    image: beryju/passbook:${PASSBOOK_TAG:-0.10.5-stable}
+    image: beryju/passbook:${PASSBOOK_TAG:-0.10.7-stable}
     command: worker
     networks:
       - internal
@@ -55,7 +55,7 @@ services:
     env_file:
       - .env
   static:
-    image: beryju/passbook-static:${PASSBOOK_TAG:-0.10.5-stable}
+    image: beryju/passbook-static:${PASSBOOK_TAG:-0.10.7-stable}
     networks:
       - internal
     labels:

docs/flow/examples/enrollment-email-verification.json

@@ -84,15 +84,6 @@
             }
         },
         {
-            "identifiers": {
-                "pk": "9922212c-47a2-475a-9905-abeb5e621652"
-            },
-            "model": "passbook_policies_expression.expressionpolicy",
-            "attrs": {
-                "name": "policy-enrollment-password-equals",
-                "expression": "# Verifies that the passwords are equal\r\nreturn request.context['password'] == request.context['password_repeat']"
-            }
-        },{
             "identifiers": {
                 "pk": "096e6282-6b30-4695-bd03-3b143eab5580",
                 "name": "default-enrollment-email-verficiation"
@@ -135,9 +126,6 @@
                     "cb954fd4-65a5-4ad9-b1ee-180ee9559cf4",
                     "7db91ee8-4290-4e08-8d39-63f132402515",
                     "d30b5eb4-7787-4072-b1ba-65b46e928920"
-                ],
-                "validation_policies": [
-                    "9922212c-47a2-475a-9905-abeb5e621652"
                 ]
             }
         },

docs/flow/examples/recovery-email-verification.json

@@ -55,16 +55,6 @@
                 "order": 1
             }
         },
-        {
-            "identifiers": {
-                "pk": "cd042fc6-cc92-4b98-b7e6-f4729df798d8"
-            },
-            "model": "passbook_policies_expression.expressionpolicy",
-            "attrs": {
-                "name": "default-password-change-password-equal",
-                "expression": "# Check that both passwords are equal.\nreturn request.context['password'] == request.context['password_repeat']"
-            }
-        },
         {
             "identifiers": {
                 "pk": "e54045a7-6ecb-4ad9-ad37-28e72d8e565e",
@@ -118,9 +108,6 @@
                 "fields": [
                     "7db91ee8-4290-4e08-8d39-63f132402515",
                     "d30b5eb4-7787-4072-b1ba-65b46e928920"
-                ],
-                "validation_policies": [
-                    "cd042fc6-cc92-4b98-b7e6-f4729df798d8"
                 ]
             }
         },

docs/installation/docker-compose.md

@@ -13,7 +13,7 @@ Download the latest `docker-compose.yml` from [here](https://raw.githubuserconte
 
 To optionally enable error-reporting, run `echo PASSBOOK_ERROR_REPORTING__ENABLED=true >> .env`
 
-To optionally deploy a different version run `echo PASSBOOK_TAG=0.10.5-stable >> .env`
+To optionally deploy a different version run `echo PASSBOOK_TAG=0.10.7-stable >> .env`
 
 If this is a fresh passbook install run the following commands to generate a password:
 

docs/installation/kubernetes.md

@@ -11,7 +11,7 @@ This installation automatically applies database migrations on startup. After th
 image:
   name: beryju/passbook
   name_static: beryju/passbook-static
-  tag: 0.10.5-stable
+  tag: 0.10.7-stable
 
 nameOverride: ""
 

docs/integrations/services/aws/index.md

@@ -24,10 +24,49 @@ You can of course use a custom signing certificate, and adjust durations.
 
 Create a role with the permissions you desire, and note the ARN.
 
-AWS requires two custom PropertyMappings; `Role` and `RoleSessionName`. Create them as following:
+After you've created the Property Mappings below, add them to the Provider.
 
-![](./property-mapping-role.png)
+Create an application, assign policies, and assign this provider.
 
-![](./property-mapping-role-session-name.png)
+Export the metadata from passbook, and create an Identity Provider [here](https://console.aws.amazon.com/iam/home#/providers).
 
-Afterwards export the metadata from passbook, and create an Identity Provider [here](https://console.aws.amazon.com/iam/home#/providers).
+#### Role Mapping
+
+The Role mapping specifies the AWS ARN(s) of the identity provider, and the role the user should assume ([see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_saml_assertions.html#saml_role-attribute)).
+
+This Mapping needs to have the SAML Name field set to "https://aws.amazon.com/SAML/Attributes/Role"
+
+As expression, you can return a static ARN like so
+
+```python
+return "arn:aws:iam::123412341234:role/saml_role,arn:aws:iam::123412341234:saml-provider/passbook"
+```
+
+Or, if you want to assign AWS Roles based on Group membership, you can add a custom attribute to the Groups, for example "aws_role", and use this snippet below. Groups are sorted by name and later groups overwrite earlier groups' attributes.
+
+```python
+role_name = user.group_attributes().get("aws_role", "")
+return f"arn:aws:iam::123412341234:role/{role_name},arn:aws:iam::123412341234:saml-provider/passbook"
+```
+
+If you want to allow a user to choose from multiple roles, use this snippet
+
+```python
+return [
+    "arn:aws:iam::123412341234:role/role_a,arn:aws:iam::123412341234:saml-provider/passbook",
+    "arn:aws:iam::123412341234:role/role_b,arn:aws:iam::123412341234:saml-provider/passbook",
+    "arn:aws:iam::123412341234:role/role_c,arn:aws:iam::123412341234:saml-provider/passbook",
+]
+```
+
+### RoleSessionName Mapping
+
+The RoleSessionMapping specifies what identifier will be shown at the top of the Management Console ([see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_saml_assertions.html#saml_role-session-attribute)).
+
+This mapping needs to have the SAML Name field set to "https://aws.amazon.com/SAML/Attributes/RoleSessionName".
+
+To use the user's username, use this snippet
+
+```python
+return user.username
+```

docs/integrations/services/gitlab/index.md

@@ -19,6 +19,7 @@ Create an application in passbook and note the slug, as this will be used later.
 - ACS URL: `https://gitlab.company/users/auth/saml/callback`
 - Audience: `https://gitlab.company`
 - Issuer: `https://gitlab.company`
+- Binding: `Post`
 
 You can of course use a custom signing certificate, and adjust durations. To get the value for `idp_cert_fingerprint`, you can use a tool like [this](https://www.samltool.com/fingerprint.php).
 
@@ -41,7 +42,7 @@ gitlab_rails['omniauth_providers'] = [
     args: {
       assertion_consumer_service_url: 'https://gitlab.company/users/auth/saml/callback',
       idp_cert_fingerprint: '4E:1E:CD:67:4A:67:5A:E9:6A:D0:3C:E6:DD:7A:F2:44:2E:76:00:6A',
-      idp_sso_target_url: 'https://passbook.company/application/saml/<passbook application slug>/login/',
+      idp_sso_target_url: 'https://passbook.company/application/saml/<passbook application slug>/sso/binding/post/',
       issuer: 'https://gitlab.company',
       name_identifier_format: 'urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress',
       attribute_statements: {

docs/policies/expression.md

@@ -27,4 +27,11 @@ return False
     - `request.context`: A dictionary with dynamic data. This depends on the origin of the execution.
 - `pb_is_sso_flow`: Boolean which is true if request was initiated by authenticating through an external provider.
 - `pb_client_ip`: Client's IP Address or '255.255.255.255' if no IP Address could be extracted. Can be [compared](../expressions/index.md#comparing-ip-addresses)
-- `pb_flow_plan`: Current Plan if Policy is called from the Flow Planner.
+
+Additionally, when the policy is executed from a flow, every variable from the flow's current context is accessible under the `context` object.
+
+This includes the following:
+
+- `prompt_data`: Data which has been saved from a prompt stage or an external source.
+- `application`: The application the user is in the process of authorizing.
+- `pending_user`: The currently pending user

e2e/ci.docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   chrome:
-    image: selenium/standalone-chrome:3.141.59-20200525
+    image: selenium/standalone-chrome:3.141
     volumes:
       - /dev/shm:/dev/shm
     network_mode: host

e2e/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   chrome:
-    image: selenium/standalone-chrome-debug:3.141.59-20200719
+    image: selenium/standalone-chrome-debug:3.141
     volumes:
       - /dev/shm:/dev/shm
     network_mode: host

e2e/test_flows_enroll.py

@@ -10,7 +10,6 @@ from selenium.webdriver.support import expected_conditions as ec
 
 from e2e.utils import USER, SeleniumTestCase
 from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
-from passbook.policies.expression.models import ExpressionPolicy
 from passbook.stages.email.models import EmailStage, EmailTemplates
 from passbook.stages.identification.models import IdentificationStage
 from passbook.stages.prompt.models import FieldTypes, Prompt, PromptStage
@@ -59,16 +58,9 @@ class TestFlowsEnroll(SeleniumTestCase):
             field_key="email", label="E-Mail", order=1, type=FieldTypes.EMAIL
         )
 
-        # Password checking policy
-        password_policy = ExpressionPolicy.objects.create(
-            name="policy-enrollment-password-equals",
-            expression="return request.context['password'] == request.context['password_repeat']",
-        )
-
         # Stages
         first_stage = PromptStage.objects.create(name="prompt-stage-first")
         first_stage.fields.set([username_prompt, password, password_repeat])
-        first_stage.validation_policies.set([password_policy])
         first_stage.save()
         second_stage = PromptStage.objects.create(name="prompt-stage-second")
         second_stage.fields.set([name_field, email])
@@ -152,16 +144,9 @@ class TestFlowsEnroll(SeleniumTestCase):
             field_key="email", label="E-Mail", order=1, type=FieldTypes.EMAIL
         )
 
-        # Password checking policy
-        password_policy = ExpressionPolicy.objects.create(
-            name="policy-enrollment-password-equals",
-            expression="return request.context['password'] == request.context['password_repeat']",
-        )
-
         # Stages
         first_stage = PromptStage.objects.create(name="prompt-stage-first")
         first_stage.fields.set([username_prompt, password, password_repeat])
-        first_stage.validation_policies.set([password_policy])
         first_stage.save()
         second_stage = PromptStage.objects.create(name="prompt-stage-second")
         second_stage.fields.set([name_field, email])

e2e/test_flows_otp.py

@@ -0,0 +1,141 @@
+"""test flow with otp stages"""
+from base64 import b32decode
+from sys import platform
+from time import sleep
+from unittest.case import skipUnless
+from urllib.parse import parse_qs, urlparse
+
+from django_otp.oath import TOTP
+from django_otp.plugins.otp_static.models import StaticDevice, StaticToken
+from django_otp.plugins.otp_totp.models import TOTPDevice
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+
+from e2e.utils import USER, SeleniumTestCase
+from passbook.flows.models import Flow, FlowStageBinding
+from passbook.stages.otp_validate.models import OTPValidateStage
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestFlowsOTP(SeleniumTestCase):
+    """test flow with otp stages"""
+
+    def test_otp_validate(self):
+        """test flow with otp stages"""
+        sleep(1)
+        # Setup TOTP Device
+        user = USER()
+        device = TOTPDevice.objects.create(user=user, confirmed=True, digits=6)
+
+        flow: Flow = Flow.objects.get(slug="default-authentication-flow")
+        # Move the user_login stage to order 3
+        FlowStageBinding.objects.filter(target=flow, order=2).update(order=3)
+        FlowStageBinding.objects.create(
+            target=flow, order=2, stage=OTPValidateStage.objects.create()
+        )
+
+        self.driver.get(f"{self.live_server_url}/flows/{flow.slug}/")
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        # Get expected token
+        totp = TOTP(device.bin_key, device.step, device.t0, device.digits, device.drift)
+        self.driver.find_element(By.ID, "id_code").send_keys(totp.token())
+        self.driver.find_element(By.ID, "id_code").send_keys(Keys.ENTER)
+        self.assertEqual(
+            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            USER().username,
+        )
+
+    def test_otp_totp_setup(self):
+        """test TOTP Setup stage"""
+        flow: Flow = Flow.objects.get(slug="default-authentication-flow")
+
+        self.driver.get(f"{self.live_server_url}/flows/{flow.slug}/")
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        self.assertEqual(
+            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            USER().username,
+        )
+
+        self.driver.find_element(By.CSS_SELECTOR, ".pf-c-page__header").click()
+        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.wait_for_url(self.url("passbook_core:user-settings"))
+
+        self.driver.find_element(By.LINK_TEXT, "Time-based OTP").click()
+
+        # Remember the current URL as we should end up back here
+        destination_url = self.driver.current_url
+
+        self.driver.find_element(
+            By.CSS_SELECTOR, ".pf-c-card__body a.pf-c-button"
+        ).click()
+
+        otp_uri = self.driver.find_element(
+            By.CSS_SELECTOR, "#flow-body > div > form > div:nth-child(3) > div"
+        ).get_attribute("aria-label")
+
+        # Parse the OTP URI, extract the secret and get the next token
+        otp_args = urlparse(otp_uri)
+        self.assertEqual(otp_args.scheme, "otpauth")
+        otp_qs = parse_qs(otp_args.query)
+        secret_key = b32decode(otp_qs["secret"][0])
+
+        totp = TOTP(secret_key)
+
+        self.driver.find_element(By.ID, "id_code").send_keys(totp.token())
+        self.driver.find_element(By.ID, "id_code").send_keys(Keys.ENTER)
+
+        self.wait_for_url(destination_url)
+        sleep(1)
+
+        self.assertTrue(TOTPDevice.objects.filter(user=USER(), confirmed=True).exists())
+
+    def test_otp_static_setup(self):
+        """test Static OTP Setup stage"""
+        flow: Flow = Flow.objects.get(slug="default-authentication-flow")
+
+        self.driver.get(f"{self.live_server_url}/flows/{flow.slug}/")
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        self.assertEqual(
+            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            USER().username,
+        )
+
+        self.driver.find_element(By.CSS_SELECTOR, ".pf-c-page__header").click()
+        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.wait_for_url(self.url("passbook_core:user-settings"))
+
+        self.driver.find_element(By.LINK_TEXT, "Static OTP").click()
+
+        # Remember the current URL as we should end up back here
+        destination_url = self.driver.current_url
+
+        self.driver.find_element(
+            By.CSS_SELECTOR, ".pf-c-card__body a.pf-c-button"
+        ).click()
+        token = self.driver.find_element(
+            By.CSS_SELECTOR, ".pb-otp-tokens li:nth-child(1)"
+        ).text
+
+        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
+
+        self.wait_for_url(destination_url)
+        sleep(1)
+
+        self.assertTrue(
+            StaticDevice.objects.filter(user=USER(), confirmed=True).exists()
+        )
+        device = StaticDevice.objects.filter(user=USER(), confirmed=True).first()
+        self.assertTrue(StaticToken.objects.filter(token=token, device=device).exists())

e2e/test_flows_stage_setup.py

@@ -20,12 +20,12 @@ class TestFlowsStageSetup(SeleniumTestCase):
         """test password change flow"""
         # Ensure that password stage has change_flow set
         flow = Flow.objects.get(
-            slug="default-password-change", designation=FlowDesignation.STAGE_SETUP,
+            slug="default-password-change",
+            designation=FlowDesignation.STAGE_CONFIGURATION,
         )
 
-        stages = PasswordStage.objects.filter(name="default-authentication-password")
-        stage = stages.first()
-        stage.change_flow = flow
+        stage = PasswordStage.objects.get(name="default-authentication-password")
+        stage.configure_flow = flow
         stage.save()
 
         new_password = generate_client_secret()

e2e/test_provider_oauth2_github.py

@@ -1,5 +1,6 @@
 """test OAuth Provider flow"""
 from sys import platform
+from time import sleep
 from typing import Any, Dict, Optional
 from unittest.case import skipUnless
 
@@ -139,6 +140,8 @@ class TestProviderOAuth2Github(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
 
+        sleep(1)
+
         self.assertIn(
             app.name,
             self.driver.find_element(

e2e/test_source_oauth.py

@@ -16,16 +16,18 @@ from yaml import safe_dump
 
 from e2e.utils import SeleniumTestCase
 from passbook.flows.models import Flow
-from passbook.providers.oauth2.generators import generate_client_secret
+from passbook.providers.oauth2.generators import (
+    generate_client_id,
+    generate_client_secret,
+)
 from passbook.sources.oauth.models import OAuthSource
 
-TOKEN_URL = "http://127.0.0.1:5556/dex/token"
 CONFIG_PATH = "/tmp/dex.yml"
 LOGGER = get_logger()
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")
-class TestSourceOAuth(SeleniumTestCase):
+class TestSourceOAuth2(SeleniumTestCase):
     """test OAuth Source flow"""
 
     container: Container
@@ -91,14 +93,14 @@ class TestSourceOAuth(SeleniumTestCase):
         authentication_flow = Flow.objects.get(slug="default-source-authentication")
         enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
 
-        OAuthSource.objects.create(
+        OAuthSource.objects.create(  # nosec
             name="dex",
             slug="dex",
             authentication_flow=authentication_flow,
             enrollment_flow=enrollment_flow,
             provider_type="openid-connect",
             authorization_url="http://127.0.0.1:5556/dex/auth",
-            access_token_url=TOKEN_URL,
+            access_token_url="http://127.0.0.1:5556/dex/token",
             profile_url="http://127.0.0.1:5556/dex/userinfo",
             consumer_key="example-app",
             consumer_secret=self.client_secret,
@@ -240,3 +242,99 @@ class TestSourceOAuth(SeleniumTestCase):
             self.driver.find_element(By.ID, "id_email").get_attribute("value"),
             "admin@example.com",
         )
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestSourceOAuth1(SeleniumTestCase):
+    """Test OAuth1 Source"""
+
+    def setUp(self) -> None:
+        self.client_id = generate_client_id()
+        self.client_secret = generate_client_secret()
+        self.source_slug = "oauth1-test"
+        super().setUp()
+
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "beryju/oauth1-test-server",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "environment": {
+                "OAUTH1_CLIENT_ID": self.client_id,
+                "OAUTH1_CLIENT_SECRET": self.client_secret,
+                "OAUTH1_REDIRECT_URI": (
+                    self.url(
+                        "passbook_sources_oauth:oauth-client-callback",
+                        source_slug=self.source_slug,
+                    )
+                ),
+            },
+        }
+
+    def create_objects(self):
+        """Create required objects"""
+        # Bootstrap all needed objects
+        authentication_flow = Flow.objects.get(slug="default-source-authentication")
+        enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
+
+        OAuthSource.objects.create(  # nosec
+            name="oauth1",
+            slug=self.source_slug,
+            authentication_flow=authentication_flow,
+            enrollment_flow=enrollment_flow,
+            provider_type="twitter",
+            request_token_url="http://localhost:5000/oauth/request_token",
+            access_token_url="http://localhost:5000/oauth/access_token",
+            authorization_url="http://localhost:5000/oauth/authorize",
+            profile_url="http://localhost:5000/api/me",
+            consumer_key=self.client_id,
+            consumer_secret=self.client_secret,
+        )
+
+    def test_oauth_enroll(self):
+        """test OAuth Source With With OIDC"""
+        self.create_objects()
+        self.driver.get(self.live_server_url)
+
+        self.wait.until(
+            ec.presence_of_element_located(
+                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
+            )
+        )
+        self.driver.find_element(
+            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
+        ).click()
+
+        # Now we should be at the IDP, wait for the login field
+        self.wait.until(ec.presence_of_element_located((By.NAME, "username")))
+        self.driver.find_element(By.NAME, "username").send_keys("example-user")
+        self.driver.find_element(By.NAME, "username").send_keys(Keys.ENTER)
+
+        # Wait until we're logged in
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "[name='confirm']"))
+        )
+        self.driver.find_element(By.CSS_SELECTOR, "[name='confirm']").click()
+
+        # Wait until we've loaded the user info page
+        self.wait.until(ec.presence_of_element_located((By.LINK_TEXT, "example-user")))
+        self.driver.find_element(By.LINK_TEXT, "example-user").click()
+
+        self.wait_for_url(self.url("passbook_core:user-settings"))
+        self.assertEqual(
+            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            "example-user",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_username").get_attribute("value"),
+            "example-user",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_name").get_attribute("value"),
+            "test name",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_email").get_attribute("value"),
+            "foo@example.com",
+        )

e2e/utils.py

@@ -1,5 +1,4 @@
 """passbook e2e testing utilities"""
-from functools import lru_cache
 from glob import glob
 from importlib.util import module_from_spec, spec_from_file_location
 from inspect import getmembers, isfunction
@@ -23,7 +22,6 @@ from structlog import get_logger
 from passbook.core.models import User
 
 
-@lru_cache
 # pylint: disable=invalid-name
 def USER() -> User:  # noqa
     """Cached function that always returns pbadmin"""

helm/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
-appVersion: "0.10.5-stable"
+appVersion: "0.10.7-stable"
 description: A Helm chart for passbook.
 name: passbook
-version: "0.10.5-stable"
+version: "0.10.7-stable"
 icon: https://github.com/BeryJu/passbook/blob/master/docs/images/logo.svg
 dependencies:
   - name: postgresql

helm/values.yaml

@@ -4,7 +4,7 @@
 image:
   name: beryju/passbook
   name_static: beryju/passbook-static
-  tag: 0.10.5-stable
+  tag: 0.10.7-stable
 
 nameOverride: ""
 

lifecycle/bootstrap.sh

@@ -4,7 +4,7 @@ printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap",
 if [[ "$1" == "server" ]]; then
     gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
 elif [[ "$1" == "worker" ]]; then
-    celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule -Q passbook,passbook_scheduled
+    celery -A passbook.root.celery worker --autoscale 10,3 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled
 elif [[ "$1" == "migrate" ]]; then
     # Run system migrations first, run normal migrations after
     python -m lifecycle.migrate

lifecycle/wait_for_db.py

@@ -1,16 +1,28 @@
 #!/usr/bin/env python
 """This file needs to be run from the root of the project to correctly
 import passbook. This is done by the dockerfile."""
+from json import dumps
+from sys import stderr
 from time import sleep
 
 from psycopg2 import OperationalError, connect
 from redis import Redis
 from redis.exceptions import RedisError
-from structlog import get_logger
 
 from passbook.lib.config import CONFIG
 
-LOGGER = get_logger()
+
+def j_print(event: str, log_level: str = "info", **kwargs):
+    """Print event in the same format as structlog with JSON.
+    Used before structlog is configured."""
+    data = {
+        "event": event,
+        "level": log_level,
+        "logger": __name__,
+    }
+    data.update(**kwargs)
+    print(dumps(data), file=stderr)
+
 
 while True:
     try:
@@ -24,7 +36,7 @@ while True:
         break
     except OperationalError:
         sleep(1)
-        LOGGER.warning("PostgreSQL Connection failed, retrying...")
+        j_print("PostgreSQL Connection failed, retrying...")
 
 while True:
     try:
@@ -38,4 +50,4 @@ while True:
         break
     except RedisError:
         sleep(1)
-        LOGGER.warning("Redis Connection failed, retrying...")
+        j_print("Redis Connection failed, retrying...")

passbook/__init__.py

@@ -1,2 +1,2 @@
 """passbook"""
-__version__ = "0.10.5-stable"
+__version__ = "0.10.7-stable"

passbook/admin/settings.py

@@ -0,0 +1,10 @@
+"""passbook admin settings"""
+from celery.schedules import crontab
+
+CELERY_BEAT_SCHEDULE = {
+    "admin_latest_version": {
+        "task": "passbook.admin.tasks.update_latest_version",
+        "schedule": crontab(minute=0),  # Run every hour
+        "options": {"queue": "passbook_scheduled"},
+    }
+}

passbook/admin/tasks.py

@@ -0,0 +1,23 @@
+"""passbook admin tasks"""
+from django.core.cache import cache
+from requests import RequestException, get
+from structlog import get_logger
+
+from passbook.root.celery import CELERY_APP
+
+LOGGER = get_logger()
+VERSION_CACHE_KEY = "passbook_latest_version"
+VERSION_CACHE_TIMEOUT = 2 * 60 * 60  # 2 hours
+
+
+@CELERY_APP.task()
+def update_latest_version():
+    """Update latest version info"""
+    try:
+        data = get(
+            "https://api.github.com/repos/beryju/passbook/releases/latest"
+        ).json()
+        tag_name = data.get("tag_name")
+        cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], VERSION_CACHE_TIMEOUT)
+    except (RequestException, IndexError):
+        cache.set(VERSION_CACHE_KEY, "0.0.0", VERSION_CACHE_TIMEOUT)

passbook/admin/views/applications.py

@@ -9,24 +9,30 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.core.forms.applications import ApplicationForm
 from passbook.core.models import Application
 from passbook.lib.views import CreateAssignPermView
 
 
-class ApplicationListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class ApplicationListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all applications"""
 
     model = Application
     permission_required = "passbook_core.view_application"
     ordering = "name"
-    paginate_by = 40
     template_name = "administration/application/list.html"
 
 
 class ApplicationCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -43,7 +49,11 @@ class ApplicationCreateView(
 
 
 class ApplicationUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update application"""
 

passbook/admin/views/certificate_key_pair.py

@@ -9,24 +9,30 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.crypto.forms import CertificateKeyPairForm
 from passbook.crypto.models import CertificateKeyPair
 from passbook.lib.views import CreateAssignPermView
 
 
-class CertificateKeyPairListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class CertificateKeyPairListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all keypairs"""
 
     model = CertificateKeyPair
     permission_required = "passbook_crypto.view_certificatekeypair"
     ordering = "name"
-    paginate_by = 40
     template_name = "administration/certificatekeypair/list.html"
 
 
 class CertificateKeyPairCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -43,7 +49,11 @@ class CertificateKeyPairCreateView(
 
 
 class CertificateKeyPairUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update certificatekeypair"""
 

passbook/admin/views/flows.py

@@ -11,7 +11,11 @@ from django.utils.translation import gettext as _
 from django.views.generic import DetailView, FormView, ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.flows.forms import FlowForm, FlowImportForm
 from passbook.flows.models import Flow
 from passbook.flows.planner import PLAN_CONTEXT_PENDING_USER
@@ -23,18 +27,20 @@ from passbook.lib.utils.urls import redirect_with_qs
 from passbook.lib.views import CreateAssignPermView
 
 
-class FlowListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class FlowListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all flows"""
 
     model = Flow
     permission_required = "passbook_flows.view_flow"
     ordering = "name"
-    paginate_by = 40
     template_name = "administration/flow/list.html"
 
 
 class FlowCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -51,7 +57,11 @@ class FlowCreateView(
 
 
 class FlowUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update flow"""
 

passbook/admin/views/groups.py

@@ -9,24 +9,30 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.core.forms.groups import GroupForm
 from passbook.core.models import Group
 from passbook.lib.views import CreateAssignPermView
 
 
-class GroupListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class GroupListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all groups"""
 
     model = Group
     permission_required = "passbook_core.view_group"
     ordering = "name"
-    paginate_by = 40
     template_name = "administration/group/list.html"
 
 
 class GroupCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -43,7 +49,11 @@ class GroupCreateView(
 
 
 class GroupUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update group"""
 

passbook/admin/views/outposts.py

@@ -12,24 +12,30 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.lib.views import CreateAssignPermView
 from passbook.outposts.forms import OutpostForm
 from passbook.outposts.models import Outpost, OutpostConfig
 
 
-class OutpostListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class OutpostListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all outposts"""
 
     model = Outpost
     permission_required = "passbook_outposts.view_outpost"
     ordering = "name"
-    paginate_by = 40
     template_name = "administration/outpost/list.html"
 
 
 class OutpostCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -53,7 +59,11 @@ class OutpostCreateView(
 
 
 class OutpostUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update outpost"""
 

passbook/admin/views/overview.py

@@ -5,32 +5,16 @@ from django.core.cache import cache
 from django.shortcuts import redirect, reverse
 from django.views.generic import TemplateView
 from packaging.version import LegacyVersion, Version, parse
-from requests import RequestException, get
 
 from passbook import __version__
 from passbook.admin.mixins import AdminRequiredMixin
+from passbook.admin.tasks import VERSION_CACHE_KEY, update_latest_version
 from passbook.core.models import Application, Provider, Source, User
 from passbook.flows.models import Flow, Stage
 from passbook.policies.models import Policy
 from passbook.root.celery import CELERY_APP
 from passbook.stages.invitation.models import Invitation
 
-VERSION_CACHE_KEY = "passbook_latest_version"
-
-
-def latest_version() -> Union[LegacyVersion, Version]:
-    """Get latest release from GitHub, cached"""
-    if not cache.get(VERSION_CACHE_KEY):
-        try:
-            data = get(
-                "https://api.github.com/repos/beryju/passbook/releases/latest"
-            ).json()
-            tag_name = data.get("tag_name")
-            cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], 30)
-        except (RequestException, IndexError):
-            cache.set(VERSION_CACHE_KEY, "0.0.0", 30)
-    return parse(cache.get(VERSION_CACHE_KEY))
-
 
 class AdministrationOverviewView(AdminRequiredMixin, TemplateView):
     """Overview View"""
@@ -44,6 +28,14 @@ class AdministrationOverviewView(AdminRequiredMixin, TemplateView):
             return redirect(reverse("passbook_flows:default-authentication"))
         return self.get(*args, **kwargs)
 
+    def get_latest_version(self) -> Union[LegacyVersion, Version]:
+        """Get latest version from cache"""
+        version_in_cache = cache.get(VERSION_CACHE_KEY)
+        if not version_in_cache:
+            update_latest_version.delay()
+            return parse(__version__)
+        return parse(version_in_cache)
+
     def get_context_data(self, **kwargs):
         kwargs["application_count"] = len(Application.objects.all())
         kwargs["policy_count"] = len(Policy.objects.all())
@@ -54,7 +46,7 @@ class AdministrationOverviewView(AdminRequiredMixin, TemplateView):
         kwargs["flow_count"] = len(Flow.objects.all())
         kwargs["invitation_count"] = len(Invitation.objects.all())
         kwargs["version"] = parse(__version__)
-        kwargs["version_latest"] = latest_version()
+        kwargs["version_latest"] = self.get_latest_version()
         kwargs["worker_count"] = len(CELERY_APP.control.ping(timeout=0.5))
         kwargs["providers_without_application"] = Provider.objects.filter(
             application=None

passbook/admin/views/policies.py

@@ -17,27 +17,31 @@ from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
 from passbook.admin.forms.policies import PolicyTestForm
 from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
     DeleteMessageView,
     InheritanceCreateView,
     InheritanceListView,
     InheritanceUpdateView,
+    UserPaginateListMixin,
 )
 from passbook.policies.models import Policy, PolicyBinding
 from passbook.policies.process import PolicyProcess, PolicyRequest
 
 
-class PolicyListView(LoginRequiredMixin, PermissionListMixin, InheritanceListView):
+class PolicyListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, InheritanceListView
+):
     """Show list of all policies"""
 
     model = Policy
     permission_required = "passbook_policies.view_policy"
-    paginate_by = 10
     ordering = "name"
     template_name = "administration/policy/list.html"
 
 
 class PolicyCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     InheritanceCreateView,
@@ -54,6 +58,7 @@ class PolicyCreateView(
 
 class PolicyUpdateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     PermissionRequiredMixin,
     InheritanceUpdateView,

passbook/admin/views/policies_bindings.py

@@ -11,18 +11,23 @@ from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 from guardian.shortcuts import get_objects_for_user
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.lib.views import CreateAssignPermView
 from passbook.policies.forms import PolicyBindingForm
 from passbook.policies.models import PolicyBinding
 
 
-class PolicyBindingListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class PolicyBindingListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all policies"""
 
     model = PolicyBinding
     permission_required = "passbook_policies.view_policybinding"
-    paginate_by = 10
     ordering = ["order", "target"]
     template_name = "administration/policy_binding/list.html"
 
@@ -47,6 +52,7 @@ class PolicyBindingListView(LoginRequiredMixin, PermissionListMixin, ListView):
 
 class PolicyBindingCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -63,7 +69,11 @@ class PolicyBindingCreateView(
 
 
 class PolicyBindingUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update policybinding"""
 

passbook/admin/views/property_mapping.py

@@ -9,16 +9,18 @@ from django.utils.translation import gettext as _
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
 from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
     DeleteMessageView,
     InheritanceCreateView,
     InheritanceListView,
     InheritanceUpdateView,
+    UserPaginateListMixin,
 )
 from passbook.core.models import PropertyMapping
 
 
 class PropertyMappingListView(
-    LoginRequiredMixin, PermissionListMixin, InheritanceListView
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, InheritanceListView
 ):
     """Show list of all property_mappings"""
 
@@ -26,11 +28,11 @@ class PropertyMappingListView(
     permission_required = "passbook_core.view_propertymapping"
     template_name = "administration/property_mapping/list.html"
     ordering = "name"
-    paginate_by = 40
 
 
 class PropertyMappingCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     InheritanceCreateView,
@@ -47,6 +49,7 @@ class PropertyMappingCreateView(
 
 class PropertyMappingUpdateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     PermissionRequiredMixin,
     InheritanceUpdateView,

passbook/admin/views/providers.py

@@ -9,26 +9,30 @@ from django.utils.translation import gettext as _
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
 from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
     DeleteMessageView,
     InheritanceCreateView,
     InheritanceListView,
     InheritanceUpdateView,
+    UserPaginateListMixin,
 )
 from passbook.core.models import Provider
 
 
-class ProviderListView(LoginRequiredMixin, PermissionListMixin, InheritanceListView):
+class ProviderListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, InheritanceListView
+):
     """Show list of all providers"""
 
     model = Provider
     permission_required = "passbook_core.add_provider"
     template_name = "administration/provider/list.html"
-    paginate_by = 10
     ordering = "id"
 
 
 class ProviderCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     InheritanceCreateView,
@@ -45,6 +49,7 @@ class ProviderCreateView(
 
 class ProviderUpdateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     PermissionRequiredMixin,
     InheritanceUpdateView,

passbook/admin/views/sources.py

@@ -9,26 +9,30 @@ from django.utils.translation import gettext as _
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
 from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
     DeleteMessageView,
     InheritanceCreateView,
     InheritanceListView,
     InheritanceUpdateView,
+    UserPaginateListMixin,
 )
 from passbook.core.models import Source
 
 
-class SourceListView(LoginRequiredMixin, PermissionListMixin, InheritanceListView):
+class SourceListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, InheritanceListView
+):
     """Show list of all sources"""
 
     model = Source
     permission_required = "passbook_core.view_source"
     ordering = "name"
-    paginate_by = 40
     template_name = "administration/source/list.html"
 
 
 class SourceCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     InheritanceCreateView,
@@ -45,6 +49,7 @@ class SourceCreateView(
 
 class SourceUpdateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     PermissionRequiredMixin,
     InheritanceUpdateView,

passbook/admin/views/stages.py

@@ -9,26 +9,30 @@ from django.utils.translation import gettext as _
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
 from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
     DeleteMessageView,
     InheritanceCreateView,
     InheritanceListView,
     InheritanceUpdateView,
+    UserPaginateListMixin,
 )
 from passbook.flows.models import Stage
 
 
-class StageListView(LoginRequiredMixin, PermissionListMixin, InheritanceListView):
+class StageListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, InheritanceListView
+):
     """Show list of all stages"""
 
     model = Stage
     template_name = "administration/stage/list.html"
     permission_required = "passbook_flows.view_stage"
     ordering = "name"
-    paginate_by = 40
 
 
 class StageCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     InheritanceCreateView,
@@ -45,6 +49,7 @@ class StageCreateView(
 
 class StageUpdateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     PermissionRequiredMixin,
     InheritanceUpdateView,

passbook/admin/views/stages_bindings.py

@@ -9,24 +9,30 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.flows.forms import FlowStageBindingForm
 from passbook.flows.models import FlowStageBinding
 from passbook.lib.views import CreateAssignPermView
 
 
-class StageBindingListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class StageBindingListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all flows"""
 
     model = FlowStageBinding
     permission_required = "passbook_flows.view_flowstagebinding"
-    paginate_by = 10
     ordering = ["target", "order"]
     template_name = "administration/stage_binding/list.html"
 
 
 class StageBindingCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -43,7 +49,11 @@ class StageBindingCreateView(
 
 
 class StageBindingUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update FlowStageBinding"""
 

passbook/admin/views/stages_invitations.py

@@ -10,25 +10,31 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.lib.views import CreateAssignPermView
 from passbook.stages.invitation.forms import InvitationForm
 from passbook.stages.invitation.models import Invitation
 from passbook.stages.invitation.signals import invitation_created
 
 
-class InvitationListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class InvitationListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all invitations"""
 
     model = Invitation
     permission_required = "passbook_stages_invitation.view_invitation"
     template_name = "administration/stage_invitation/list.html"
-    paginate_by = 10
     ordering = "-expires"
 
 
 class InvitationCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,

passbook/admin/views/stages_prompts.py

@@ -9,24 +9,30 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView, UpdateView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.lib.views import CreateAssignPermView
 from passbook.stages.prompt.forms import PromptAdminForm
 from passbook.stages.prompt.models import Prompt
 
 
-class PromptListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class PromptListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all prompts"""
 
     model = Prompt
     permission_required = "passbook_stages_prompt.view_prompt"
     ordering = "order"
-    paginate_by = 40
     template_name = "administration/stage_prompt/list.html"
 
 
 class PromptCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -43,7 +49,11 @@ class PromptCreateView(
 
 
 class PromptUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update prompt"""
 

passbook/admin/views/tokens.py

@@ -5,17 +5,18 @@ from django.utils.translation import gettext as _
 from django.views.generic import ListView
 from guardian.mixins import PermissionListMixin, PermissionRequiredMixin
 
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import DeleteMessageView, UserPaginateListMixin
 from passbook.core.models import Token
 
 
-class TokenListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class TokenListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all tokens"""
 
     model = Token
     permission_required = "passbook_core.view_token"
     ordering = "expires"
-    paginate_by = 40
     template_name = "administration/token/list.html"
 
 

passbook/admin/views/users.py

@@ -18,18 +18,23 @@ from guardian.mixins import (
 )
 
 from passbook.admin.forms.users import UserForm
-from passbook.admin.views.utils import DeleteMessageView
+from passbook.admin.views.utils import (
+    BackSuccessUrlMixin,
+    DeleteMessageView,
+    UserPaginateListMixin,
+)
 from passbook.core.models import Token, User
 from passbook.lib.views import CreateAssignPermView
 
 
-class UserListView(LoginRequiredMixin, PermissionListMixin, ListView):
+class UserListView(
+    LoginRequiredMixin, PermissionListMixin, UserPaginateListMixin, ListView
+):
     """Show list of all users"""
 
     model = User
     permission_required = "passbook_core.view_user"
     ordering = "username"
-    paginate_by = 40
     template_name = "administration/user/list.html"
 
     def get_queryset(self):
@@ -38,6 +43,7 @@ class UserListView(LoginRequiredMixin, PermissionListMixin, ListView):
 
 class UserCreateView(
     SuccessMessageMixin,
+    BackSuccessUrlMixin,
     LoginRequiredMixin,
     DjangoPermissionRequiredMixin,
     CreateAssignPermView,
@@ -54,7 +60,11 @@ class UserCreateView(
 
 
 class UserUpdateView(
-    SuccessMessageMixin, LoginRequiredMixin, PermissionRequiredMixin, UpdateView
+    SuccessMessageMixin,
+    BackSuccessUrlMixin,
+    LoginRequiredMixin,
+    PermissionRequiredMixin,
+    UpdateView,
 ):
     """Update user"""
 

passbook/admin/views/utils.py

@@ -1,9 +1,12 @@
 """passbook admin util views"""
-from typing import Any, Dict
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse
 
 from django.contrib import messages
 from django.contrib.messages.views import SuccessMessageMixin
+from django.db.models import QuerySet
 from django.http import Http404
+from django.http.request import HttpRequest
 from django.views.generic import DeleteView, ListView, UpdateView
 
 from passbook.lib.utils.reflection import all_subclasses
@@ -69,3 +72,31 @@ class InheritanceUpdateView(UpdateView):
             .select_subclasses()
             .first()
         )
+
+
+class BackSuccessUrlMixin:
+    """Checks if a relative URL has been given as ?back param, and redirect to it. Otherwise
+    default to self.success_url."""
+
+    request: HttpRequest
+
+    success_url: Optional[str]
+
+    def get_success_url(self) -> str:
+        """get_success_url from FormMixin"""
+        back_param = self.request.GET.get("back")
+        if back_param:
+            if not bool(urlparse(back_param).netloc):
+                return back_param
+        return str(self.success_url)
+
+
+class UserPaginateListMixin:
+    """Get paginate_by value from user's attributes, defaulting to 15"""
+
+    request: HttpRequest
+
+    # pylint: disable=unused-argument
+    def get_paginate_by(self, queryset: QuerySet) -> int:
+        """get_paginate_by Function of ListView"""
+        return self.request.user.attributes.get("paginate_by", 15)

passbook/audit/middleware.py

@@ -0,0 +1,85 @@
+"""Audit middleware"""
+from functools import partial
+from typing import Callable
+
+from django.contrib.auth.models import User
+from django.db.models import Model
+from django.db.models.signals import post_save, pre_delete
+from django.http import HttpRequest, HttpResponse
+
+from passbook.audit.models import Event, EventAction, model_to_dict
+from passbook.audit.signals import EventNewThread
+from passbook.core.middleware import LOCAL
+
+
+class AuditMiddleware:
+    """Register handlers for duration of request-response that log creation/update/deletion
+    of models"""
+
+    get_response: Callable[[HttpRequest], HttpResponse]
+
+    def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]):
+        self.get_response = get_response
+
+    def __call__(self, request: HttpRequest) -> HttpResponse:
+        # Connect signal for automatic logging
+        if hasattr(request, "user") and getattr(
+            request.user, "is_authenticated", False
+        ):
+            post_save_handler = partial(
+                self.post_save_handler, user=request.user, request=request
+            )
+            pre_delete_handler = partial(
+                self.pre_delete_handler, user=request.user, request=request
+            )
+            post_save.connect(
+                post_save_handler,
+                dispatch_uid=LOCAL.passbook["request_id"],
+                weak=False,
+            )
+            pre_delete.connect(
+                pre_delete_handler,
+                dispatch_uid=LOCAL.passbook["request_id"],
+                weak=False,
+            )
+
+        response = self.get_response(request)
+
+        post_save.disconnect(dispatch_uid=LOCAL.passbook["request_id"])
+        pre_delete.disconnect(dispatch_uid=LOCAL.passbook["request_id"])
+
+        return response
+
+    # pylint: disable=unused-argument
+    def process_exception(self, request: HttpRequest, exception: Exception):
+        """Unregister handlers in case of exception"""
+        post_save.disconnect(dispatch_uid=LOCAL.passbook["request_id"])
+        pre_delete.disconnect(dispatch_uid=LOCAL.passbook["request_id"])
+
+    @staticmethod
+    # pylint: disable=unused-argument
+    def post_save_handler(
+        user: User, request: HttpRequest, sender, instance: Model, created: bool, **_
+    ):
+        """Signal handler for all object's post_save"""
+        if isinstance(instance, Event):
+            return
+
+        action = EventAction.MODEL_CREATED if created else EventAction.MODEL_UPDATED
+        EventNewThread(action, request, user=user, model=model_to_dict(instance)).run()
+
+    @staticmethod
+    # pylint: disable=unused-argument
+    def pre_delete_handler(
+        user: User, request: HttpRequest, sender, instance: Model, **_
+    ):
+        """Signal handler for all object's pre_delete"""
+        if isinstance(instance, Event):
+            return
+
+        EventNewThread(
+            EventAction.MODEL_DELETED,
+            request,
+            user=user,
+            model=model_to_dict(instance),
+        ).run()

passbook/audit/migrations/0003_auto_20200917_1155.py

@@ -0,0 +1,59 @@
+# Generated by Django 3.1.1 on 2020-09-17 11:55
+from django.apps.registry import Apps
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+import passbook.audit.models
+
+
+def convert_user_to_json(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
+    Event = apps.get_model("passbook_audit", "Event")
+
+    db_alias = schema_editor.connection.alias
+    for event in Event.objects.all():
+        event.delete()
+        # Because event objects cannot be updated, we have to re-create them
+        event.pk = None
+        event.user_json = (
+            passbook.audit.models.get_user(event.user) if event.user else {}
+        )
+        event._state.adding = True
+        event.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_audit", "0002_auto_20200918_2116"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="event",
+            name="action",
+            field=models.TextField(
+                choices=[
+                    ("LOGIN", "login"),
+                    ("LOGIN_FAILED", "login_failed"),
+                    ("LOGOUT", "logout"),
+                    ("AUTHORIZE_APPLICATION", "authorize_application"),
+                    ("SUSPICIOUS_REQUEST", "suspicious_request"),
+                    ("SIGN_UP", "sign_up"),
+                    ("PASSWORD_RESET", "password_reset"),
+                    ("INVITE_CREATED", "invitation_created"),
+                    ("INVITE_USED", "invitation_used"),
+                    ("IMPERSONATION_STARTED", "impersonation_started"),
+                    ("IMPERSONATION_ENDED", "impersonation_ended"),
+                    ("CUSTOM", "custom"),
+                ]
+            ),
+        ),
+        migrations.AddField(
+            model_name="event", name="user_json", field=models.JSONField(default=dict),
+        ),
+        migrations.RunPython(convert_user_to_json),
+        migrations.RemoveField(model_name="event", name="user",),
+        migrations.RenameField(
+            model_name="event", old_name="user_json", new_name="user"
+        ),
+    ]

passbook/audit/migrations/0004_auto_20200921_1829.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.1.1 on 2020-09-21 18:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_audit", "0003_auto_20200917_1155"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="event",
+            name="action",
+            field=models.TextField(
+                choices=[
+                    ("login", "Login"),
+                    ("login_failed", "Login Failed"),
+                    ("logout", "Logout"),
+                    ("sign_up", "Sign Up"),
+                    ("authorize_application", "Authorize Application"),
+                    ("suspicious_request", "Suspicious Request"),
+                    ("password_set", "Password Set"),
+                    ("invitation_created", "Invite Created"),
+                    ("invitation_used", "Invite Used"),
+                    ("source_linked", "Source Linked"),
+                    ("impersonation_started", "Impersonation Started"),
+                    ("impersonation_ended", "Impersonation Ended"),
+                    ("model_created", "Model Created"),
+                    ("model_updated", "Model Updated"),
+                    ("model_deleted", "Model Deleted"),
+                    ("custom_", "Custom Prefix"),
+                ]
+            ),
+        ),
+    ]

passbook/audit/models.py

@@ -1,7 +1,6 @@
 """passbook audit models"""
-from enum import Enum
 from inspect import getmodule, stack
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Union
 from uuid import UUID, uuid4
 
 from django.conf import settings
@@ -12,16 +11,17 @@ from django.db.models.base import Model
 from django.http import HttpRequest
 from django.utils.translation import gettext as _
 from django.views.debug import SafeExceptionReporterFilter
-from guardian.shortcuts import get_anonymous_user
+from guardian.utils import get_anonymous_user
 from structlog import get_logger
 
 from passbook.core.middleware import (
     SESSION_IMPERSONATE_ORIGINAL_USER,
     SESSION_IMPERSONATE_USER,
 )
+from passbook.core.models import User
 from passbook.lib.utils.http import get_client_ip
 
-LOGGER = get_logger()
+LOGGER = get_logger("passbook.audit")
 
 
 def cleanse_dict(source: Dict[Any, Any]) -> Dict[Any, Any]:
@@ -53,6 +53,22 @@ def model_to_dict(model: Model) -> Dict[str, Any]:
     }
 
 
+def get_user(user: User, original_user: Optional[User] = None) -> Dict[str, Any]:
+    """Convert user object to dictionary, optionally including the original user"""
+    if isinstance(user, AnonymousUser):
+        user = get_anonymous_user()
+    user_data = {
+        "username": user.username,
+        "pk": user.pk,
+        "email": user.email,
+    }
+    if original_user:
+        original_data = get_user(original_user)
+        original_data["on_behalf_of"] = user_data
+        return original_data
+    return user_data
+
+
 def sanitize_dict(source: Dict[Any, Any]) -> Dict[Any, Any]:
     """clean source of all Models that would interfere with the JSONField.
     Models are replaced with a dictionary of {
@@ -73,38 +89,39 @@ def sanitize_dict(source: Dict[Any, Any]) -> Dict[Any, Any]:
     return final_dict
 
 
-class EventAction(Enum):
+class EventAction(models.TextChoices):
     """All possible actions to save into the audit log"""
 
     LOGIN = "login"
     LOGIN_FAILED = "login_failed"
     LOGOUT = "logout"
+
+    SIGN_UP = "sign_up"
     AUTHORIZE_APPLICATION = "authorize_application"
     SUSPICIOUS_REQUEST = "suspicious_request"
-    SIGN_UP = "sign_up"
-    PASSWORD_RESET = "password_reset"  # noqa # nosec
+    PASSWORD_SET = "password_set"  # noqa # nosec
+
     INVITE_CREATED = "invitation_created"
     INVITE_USED = "invitation_used"
+
+    SOURCE_LINKED = "source_linked"
+
     IMPERSONATION_STARTED = "impersonation_started"
     IMPERSONATION_ENDED = "impersonation_ended"
-    CUSTOM = "custom"
 
-    @staticmethod
-    def as_choices():
-        """Generate choices of actions used for database"""
-        return tuple(
-            (x, y.value) for x, y in getattr(EventAction, "__members__").items()
-        )
+    MODEL_CREATED = "model_created"
+    MODEL_UPDATED = "model_updated"
+    MODEL_DELETED = "model_deleted"
+
+    CUSTOM_PREFIX = "custom_"
 
 
 class Event(models.Model):
     """An individual audit log event"""
 
     event_uuid = models.UUIDField(primary_key=True, editable=False, default=uuid4)
-    user = models.ForeignKey(
-        settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL
-    )
-    action = models.TextField(choices=EventAction.as_choices())
+    user = models.JSONField(default=dict)
+    action = models.TextField(choices=EventAction.choices)
     date = models.DateTimeField(auto_now_add=True)
     app = models.TextField()
     context = models.JSONField(default=dict, blank=True)
@@ -119,20 +136,18 @@ class Event(models.Model):
 
     @staticmethod
     def new(
-        action: EventAction,
+        action: Union[str, EventAction],
         app: Optional[str] = None,
         _inspect_offset: int = 1,
         **kwargs,
     ) -> "Event":
         """Create new Event instance from arguments. Instance is NOT saved."""
         if not isinstance(action, EventAction):
-            raise ValueError(
-                f"action must be EventAction instance but was {type(action)}"
-            )
+            action = EventAction.CUSTOM_PREFIX + action
         if not app:
             app = getmodule(stack()[_inspect_offset][0]).__name__
         cleaned_kwargs = cleanse_dict(sanitize_dict(kwargs))
-        event = Event(action=action.value, app=app, context=cleaned_kwargs)
+        event = Event(action=action, app=app, context=cleaned_kwargs)
         return event
 
     def from_http(
@@ -142,17 +157,17 @@ class Event(models.Model):
         Events independently from requests.
         `user` arguments optionally overrides user from requests."""
         if hasattr(request, "user"):
-            if isinstance(request.user, AnonymousUser):
-                self.user = get_anonymous_user()
-            else:
-                self.user = request.user
+            self.user = get_user(
+                request.user,
+                request.session.get(SESSION_IMPERSONATE_ORIGINAL_USER, None),
+            )
         if user:
-            self.user = user
+            self.user = get_user(user)
         # Check if we're currently impersonating, and add that user
         if hasattr(request, "session"):
             if SESSION_IMPERSONATE_ORIGINAL_USER in request.session:
-                self.user = request.session[SESSION_IMPERSONATE_ORIGINAL_USER]
-                self.context["on_behalf_of"] = model_to_dict(
+                self.user = get_user(request.session[SESSION_IMPERSONATE_ORIGINAL_USER])
+                self.user["on_behalf_of"] = get_user(
                     request.session[SESSION_IMPERSONATE_USER]
                 )
         # User 255.255.255.255 as fallback if IP cannot be determined

passbook/audit/signals.py

@@ -20,15 +20,18 @@ from passbook.stages.user_write.signals import user_write
 class EventNewThread(Thread):
     """Create Event in background thread"""
 
-    action: EventAction
+    action: str
     request: HttpRequest
     kwargs: Dict[str, Any]
     user: Optional[User] = None
 
-    def __init__(self, action: EventAction, request: HttpRequest, **kwargs):
+    def __init__(
+        self, action: str, request: HttpRequest, user: Optional[User] = None, **kwargs
+    ):
         super().__init__()
         self.action = action
         self.request = request
+        self.user = user
         self.kwargs = kwargs
 
     def run(self):
@@ -57,7 +60,7 @@ def on_user_logged_out(sender, request: HttpRequest, user: User, **_):
 # pylint: disable=unused-argument
 def on_user_write(sender, request: HttpRequest, user: User, data: Dict[str, Any], **_):
     """Log User write"""
-    thread = EventNewThread(EventAction.CUSTOM, request, **data)
+    thread = EventNewThread("stages/user_write", request, **data)
     thread.user = user
     thread.run()
 

passbook/audit/templates/audit/list.html

@@ -40,12 +40,28 @@
                             </div>
                         </th>
                         <td role="cell">
+                            <div>
+                                <div>
                                     <code>{{ entry.context }}</code>
+                                </div>
+                                {% if entry.user.on_behalf_of %}
+                                <small>
+                                    {% blocktrans with username=entry.user.on_behalf_of.username %}
+                                    On behalf of {{ username }}
+                                    {% endblocktrans %}
+                                </small>
+                                {% endif %}
+                            </div>
                         </td>
                         <td role="cell">
-                            <span>
-                                {{ entry.user }}
-                            </span>
+                            <div>
+                                <div>{{ entry.user.username }}</div>
+                                <small>
+                                    {% blocktrans with pk=entry.user.pk %}
+                                    ID: {{ pk }}
+                                    {% endblocktrans %}
+                                </small>
+                            </div>
                         </td>
                         <td role="cell">
                             <span>

passbook/audit/tests/test_event.py

@@ -4,7 +4,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.test import TestCase
 from guardian.shortcuts import get_anonymous_user
 
-from passbook.audit.models import Event, EventAction
+from passbook.audit.models import Event
 from passbook.policies.dummy.models import DummyPolicy
 
 
@@ -13,7 +13,7 @@ class TestAuditEvent(TestCase):
 
     def test_new_with_model(self):
         """Create a new Event passing a model as kwarg"""
-        event = Event.new(EventAction.CUSTOM, test={"model": get_anonymous_user()})
+        event = Event.new("unittest", test={"model": get_anonymous_user()})
         event.save()  # We save to ensure nothing is un-saveable
         model_content_type = ContentType.objects.get_for_model(get_anonymous_user())
         self.assertEqual(
@@ -24,7 +24,7 @@ class TestAuditEvent(TestCase):
     def test_new_with_uuid_model(self):
         """Create a new Event passing a model (with UUID PK) as kwarg"""
         temp_model = DummyPolicy.objects.create(name="test", result=True)
-        event = Event.new(EventAction.CUSTOM, model=temp_model)
+        event = Event.new("unittest", model=temp_model)
         event.save()  # We save to ensure nothing is un-saveable
         model_content_type = ContentType.objects.get_for_model(temp_model)
         self.assertEqual(

passbook/audit/views.py

@@ -3,14 +3,16 @@ from django.contrib.auth.mixins import LoginRequiredMixin
 from django.views.generic import ListView
 from guardian.mixins import PermissionListMixin
 
+from passbook.admin.views.utils import UserPaginateListMixin
 from passbook.audit.models import Event
 
 
-class EventListView(PermissionListMixin, LoginRequiredMixin, ListView):
+class EventListView(
+    PermissionListMixin, LoginRequiredMixin, UserPaginateListMixin, ListView
+):
     """Show list of all invitations"""
 
     model = Event
     template_name = "audit/list.html"
     permission_required = "passbook_audit.view_event"
     ordering = "-created"
-    paginate_by = 20

passbook/core/admin.py

@@ -20,5 +20,5 @@ def admin_autoregister(app: AppConfig):
 
 for _app in apps.get_app_configs():
     if _app.label.startswith("passbook_"):
-        LOGGER.debug("Registering application for dj-admin", app=_app.label)
+        LOGGER.debug("Registering application for dj-admin", application=_app.label)
         admin_autoregister(_app)

passbook/core/middleware.py

@@ -1,11 +1,14 @@
 """passbook admin Middleware to impersonate users"""
-
+from logging import Logger
+from threading import local
 from typing import Callable
+from uuid import uuid4
 
 from django.http import HttpRequest, HttpResponse
 
 SESSION_IMPERSONATE_USER = "passbook_impersonate_user"
 SESSION_IMPERSONATE_ORIGINAL_USER = "passbook_impersonate_original_user"
+LOCAL = local()
 
 
 class ImpersonateMiddleware:
@@ -24,3 +27,30 @@ class ImpersonateMiddleware:
             request.user = request.session[SESSION_IMPERSONATE_USER]
 
         return self.get_response(request)
+
+
+class RequestIDMiddleware:
+    """Add a unique ID to every request"""
+
+    get_response: Callable[[HttpRequest], HttpResponse]
+
+    def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]):
+        self.get_response = get_response
+
+    def __call__(self, request: HttpRequest) -> HttpResponse:
+        if not hasattr(request, "request_id"):
+            request_id = uuid4().hex
+            setattr(request, "request_id", request_id)
+            LOCAL.passbook = {"request_id": request_id}
+        response = self.get_response(request)
+        response["X-passbook-id"] = request.request_id
+        del LOCAL.passbook["request_id"]
+        return response
+
+
+# pylint: disable=unused-argument
+def structlog_add_request_id(logger: Logger, method_name: str, event_dict):
+    """If threadlocal has passbook defined, add request_id to log"""
+    if hasattr(LOCAL, "passbook"):
+        event_dict["request_id"] = LOCAL.passbook.get("request_id", "")
+    return event_dict

passbook/core/migrations/0003_default_user.py

@@ -14,7 +14,7 @@ def create_default_user(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
     pbadmin, _ = User.objects.using(db_alias).get_or_create(
         username="pbadmin", email="root@localhost", name="passbook Default Admin"
     )
-    pbadmin.set_password("pbadmin")  # noqa # nosec
+    pbadmin.set_password("pbadmin", signal=False)  # noqa # nosec
     pbadmin.save()
 
 

passbook/core/models.py

@@ -1,6 +1,6 @@
 """passbook core models"""
 from datetime import timedelta
-from typing import Any, Optional, Type
+from typing import Any, Dict, Optional, Type
 from uuid import uuid4
 
 from django.contrib.auth.models import AbstractUser
@@ -9,6 +9,7 @@ from django.db import models
 from django.db.models import Q, QuerySet
 from django.forms import ModelForm
 from django.http import HttpRequest
+from django.utils.functional import cached_property
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from guardian.mixins import GuardianUserMixin
@@ -80,7 +81,14 @@ class User(GuardianUserMixin, AbstractUser):
 
     objects = UserManager()
 
-    @property
+    def group_attributes(self) -> Dict[str, Any]:
+        """Get a dictionary containing the attributes from all groups the user belongs to"""
+        final_attributes = {}
+        for group in self.pb_groups.all().order_by("name"):
+            final_attributes.update(group.attributes)
+        return final_attributes
+
+    @cached_property
     def is_superuser(self) -> bool:
         """Get supseruser status based on membership in a group with superuser status"""
         return self.pb_groups.filter(is_superuser=True).exists()
@@ -88,10 +96,10 @@ class User(GuardianUserMixin, AbstractUser):
     @property
     def is_staff(self) -> bool:
         """superuser == staff user"""
-        return self.is_superuser
+        return self.is_superuser  # type: ignore
 
-    def set_password(self, password):
-        if self.pk:
+    def set_password(self, password, signal=True):
+        if self.pk and signal:
             password_changed.send(sender=self, user=self, password=password)
         self.password_change_date = now()
         return super().set_password(password)

passbook/flows/migrations/0009_source_flows.py

@@ -12,7 +12,7 @@ FLOW_POLICY_EXPRESSION = """# This policy ensures that this flow can only be use
 return pb_is_sso_flow"""
 PROMPT_POLICY_EXPRESSION = """# Check if we've not been given a username by the external IdP
 # and trigger the enrollment flow
-return 'username' not in pb_flow_plan.context.get('prompt_data', {})"""
+return 'username' not in context.get('prompt_data', {})"""
 
 
 def create_default_source_enrollment_flow(
@@ -80,7 +80,9 @@ def create_default_source_enrollment_flow(
     )
 
     binding, _ = FlowStageBinding.objects.using(db_alias).update_or_create(
-        target=flow, stage=prompt_stage, defaults={"order": 0}
+        target=flow,
+        stage=prompt_stage,
+        defaults={"order": 0, "re_evaluate_policies": True},
     )
     PolicyBinding.objects.using(db_alias).update_or_create(
         policy=prompt_policy, target=binding, defaults={"order": 0}

passbook/flows/migrations/0013_auto_20200924_1605.py

@@ -0,0 +1,44 @@
+# Generated by Django 3.1.1 on 2020-09-24 16:05
+
+from django.apps.registry import Apps
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+from passbook.flows.models import FlowDesignation
+
+
+def update_flow_designation(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
+    Flow = apps.get_model("passbook_flows", "Flow")
+    db_alias = schema_editor.connection.alias
+
+    for flow in Flow.objects.using(db_alias).all():
+        if flow.designation == "stage_setup":
+            flow.designation = FlowDesignation.STAGE_CONFIGURATION
+            flow.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_flows", "0012_auto_20200908_1542"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="flow",
+            name="designation",
+            field=models.CharField(
+                choices=[
+                    ("authentication", "Authentication"),
+                    ("authorization", "Authorization"),
+                    ("invalidation", "Invalidation"),
+                    ("enrollment", "Enrollment"),
+                    ("unenrollment", "Unrenollment"),
+                    ("recovery", "Recovery"),
+                    ("stage_configuration", "Stage Configuration"),
+                ],
+                max_length=100,
+            ),
+        ),
+        migrations.RunPython(update_flow_designation),
+    ]

passbook/flows/migrations/0014_auto_20200925_2332.py

@@ -0,0 +1,51 @@
+# Generated by Django 3.1.1 on 2020-09-25 23:32
+
+from django.apps.registry import Apps
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+
+# First stage for default-source-enrollment flow (prompt stage)
+# needs to have its policy re-evaluated
+def update_default_source_enrollment_flow_binding(
+    apps: Apps, schema_editor: BaseDatabaseSchemaEditor
+):
+    Flow = apps.get_model("passbook_flows", "Flow")
+    FlowStageBinding = apps.get_model("passbook_flows", "FlowStageBinding")
+    db_alias = schema_editor.connection.alias
+
+    flows = Flow.objects.using(db_alias).filter(slug="default-source-enrollment")
+    if not flows.exists():
+        return
+    flow = flows.first()
+
+    binding = FlowStageBinding.objects.get(target=flow, order=0)
+    binding.re_evaluate_policies = True
+    binding.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_flows", "0013_auto_20200924_1605"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="flowstagebinding",
+            options={
+                "ordering": ["target", "order"],
+                "verbose_name": "Flow Stage Binding",
+                "verbose_name_plural": "Flow Stage Bindings",
+            },
+        ),
+        migrations.AlterField(
+            model_name="flowstagebinding",
+            name="re_evaluate_policies",
+            field=models.BooleanField(
+                default=False,
+                help_text="When this option is enabled, the planner will re-evaluate policies bound to this binding.",
+            ),
+        ),
+        migrations.RunPython(update_default_source_enrollment_flow_binding),
+    ]

passbook/flows/models.py

@@ -37,7 +37,7 @@ class FlowDesignation(models.TextChoices):
     ENROLLMENT = "enrollment"
     UNRENOLLMENT = "unenrollment"
     RECOVERY = "recovery"
-    STAGE_SETUP = "stage_setup"
+    STAGE_CONFIGURATION = "stage_configuration"
 
 
 class Stage(SerializerModel):
@@ -155,7 +155,10 @@ class FlowStageBinding(SerializerModel, PolicyBindingModel):
     re_evaluate_policies = models.BooleanField(
         default=False,
         help_text=_(
-            "When this option is enabled, the planner will re-evaluate policies bound to this."
+            (
+                "When this option is enabled, the planner will re-evaluate "
+                "policies bound to this binding."
+            )
         ),
     )
 
@@ -170,12 +173,35 @@ class FlowStageBinding(SerializerModel, PolicyBindingModel):
         return FlowStageBindingSerializer
 
     def __str__(self) -> str:
-        return f"'{self.target}' -> '{self.stage}' # {self.order}"
+        return f"{self.target} #{self.order} -> {self.stage}"
 
     class Meta:
 
-        ordering = ["order", "target"]
+        ordering = ["target", "order"]
 
         verbose_name = _("Flow Stage Binding")
         verbose_name_plural = _("Flow Stage Bindings")
         unique_together = (("target", "stage", "order"),)
+
+
+class ConfigurableStage(models.Model):
+    """Abstract base class for a Stage that can be configured by the enduser.
+    The stage should create a default flow with the configure_stage designation during
+    migration."""
+
+    configure_flow = models.ForeignKey(
+        Flow,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        help_text=_(
+            (
+                "Flow used by an authenticated user to configure this Stage. "
+                "If empty, user will not be able to configure this stage."
+            )
+        ),
+    )
+
+    class Meta:
+
+        abstract = True

passbook/flows/tests/test_transfer.py

@@ -105,15 +105,10 @@ class TestFlowTransfer(TransactionTestCase):
                 order=2,
                 type=FieldTypes.PASSWORD,
             )
-            # Password checking policy
-            password_policy = ExpressionPolicy.objects.create(
-                name=generate_client_id(), expression="return True",
-            )
 
             # Stages
             first_stage = PromptStage.objects.create(name=generate_client_id())
             first_stage.fields.set([username_prompt, password, password_repeat])
-            first_stage.validation_policies.set([password_policy])
             first_stage.save()
 
             flow = Flow.objects.create(

passbook/flows/urls.py

@@ -4,6 +4,7 @@ from django.urls import path
 from passbook.flows.models import FlowDesignation
 from passbook.flows.views import (
     CancelView,
+    ConfigureFlowInitView,
     FlowExecutorShellView,
     FlowExecutorView,
     ToDefaultFlow,
@@ -36,6 +37,11 @@ urlpatterns = [
         name="default-unenrollment",
     ),
     path("-/cancel/", CancelView.as_view(), name="cancel"),
+    path(
+        "-/configure/<uuid:stage_uuid>/",
+        ConfigureFlowInitView.as_view(),
+        name="configure",
+    ),
     path("b/<slug:flow_slug>/", FlowExecutorView.as_view(), name="flow-executor"),
     path(
         "<slug:flow_slug>/", FlowExecutorShellView.as_view(), name="flow-executor-shell"

passbook/flows/views.py

@@ -2,6 +2,7 @@
 from traceback import format_tb
 from typing import Any, Dict, Optional
 
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.http import (
     Http404,
     HttpRequest,
@@ -19,8 +20,8 @@ from structlog import get_logger
 from passbook.audit.models import cleanse_dict
 from passbook.core.models import PASSBOOK_USER_DEBUG
 from passbook.flows.exceptions import EmptyFlowException, FlowNonApplicableException
-from passbook.flows.models import Flow, FlowDesignation, Stage
-from passbook.flows.planner import FlowPlan, FlowPlanner
+from passbook.flows.models import ConfigurableStage, Flow, FlowDesignation, Stage
+from passbook.flows.planner import PLAN_CONTEXT_PENDING_USER, FlowPlan, FlowPlanner
 from passbook.lib.utils.reflection import class_to_path
 from passbook.lib.utils.urls import is_url_absolute, redirect_with_qs
 from passbook.policies.http import AccessDeniedResponse
@@ -156,10 +157,6 @@ class FlowExecutorView(View):
             stage_class=class_to_path(self.current_stage_view.__class__),
             flow_slug=self.flow.slug,
         )
-        # We call plan.next here to check for re-evaluate markers
-        # this is important so we can save the result
-        # and we don't have to re-evaluate the policies each request
-        self.plan.next()
         self.plan.pop()
         self.request.session[SESSION_KEY_PLAN] = self.plan
         if self.plan.stages:
@@ -295,3 +292,32 @@ def to_stage_response(request: HttpRequest, source: HttpResponse) -> HttpRespons
             {"type": "template", "body": source.content.decode("utf-8")}
         )
     return source
+
+
+class ConfigureFlowInitView(LoginRequiredMixin, View):
+    """Initiate planner for selected change flow and redirect to flow executor,
+    or raise Http404 if no configure_flow has been set."""
+
+    def get(self, request: HttpRequest, stage_uuid: str) -> HttpResponse:
+        """Initiate planner for selected change flow and redirect to flow executor,
+        or raise Http404 if no configure_flow has been set."""
+        try:
+            stage: Stage = Stage.objects.get_subclass(pk=stage_uuid)
+        except Stage.DoesNotExist as exc:
+            raise Http404 from exc
+        if not isinstance(stage, ConfigurableStage):
+            LOGGER.debug("Stage does not inherit ConfigurableStage", stage=stage)
+            raise Http404
+        if not stage.configure_flow:
+            LOGGER.debug("Stage has no configure_flow set", stage=stage)
+            raise Http404
+
+        plan = FlowPlanner(stage.configure_flow).plan(
+            request, {PLAN_CONTEXT_PENDING_USER: request.user}
+        )
+        request.session[SESSION_KEY_PLAN] = plan
+        return redirect_with_qs(
+            "passbook_flows:flow-executor-shell",
+            self.request.GET,
+            flow_slug=stage.configure_flow.slug,
+        )

passbook/lib/logging.py

@@ -1,9 +1,23 @@
 """logging helpers"""
+from logging import Logger
 from os import getpid
+from typing import Callable
 
 
 # pylint: disable=unused-argument
-def add_process_id(logger, method_name, event_dict):
+def add_process_id(logger: Logger, method_name: str, event_dict):
     """Add the current process ID"""
     event_dict["pid"] = getpid()
     return event_dict
+
+
+def add_common_fields(environment: str) -> Callable:
+    """Add a common field to easily search for passbook logs"""
+
+    def add_common_field(logger: Logger, method_name: str, event_dict):
+        """Add a common field to easily search for passbook logs"""
+        event_dict["app"] = "passbook"
+        event_dict["app_environment"] = environment
+        return event_dict
+
+    return add_common_field

passbook/lib/templatetags/passbook_utils.py

@@ -17,7 +17,7 @@ LOGGER = get_logger()
 
 @register.simple_tag(takes_context=True)
 def back(context: Context) -> str:
-    """Return a link back (either from GET paramter or referer."""
+    """Return a link back (either from GET parameter or referer."""
     if "request" not in context:
         return ""
     request = context.get("request")

passbook/policies/expression/evaluator.py

@@ -6,7 +6,6 @@ from django.http import HttpRequest
 from structlog import get_logger
 
 from passbook.flows.planner import PLAN_CONTEXT_SSO
-from passbook.flows.views import SESSION_KEY_PLAN
 from passbook.lib.expression.evaluator import BaseEvaluator
 from passbook.lib.utils.http import get_client_ip
 from passbook.policies.types import PolicyRequest, PolicyResult
@@ -31,23 +30,20 @@ class PolicyEvaluator(BaseEvaluator):
 
     def set_policy_request(self, request: PolicyRequest):
         """Update context based on policy request (if http request is given, update that too)"""
-        # update passbook/policies/expression/templates/policy/expression/form.html
         # update docs/policies/expression/index.md
         self._context["pb_is_sso_flow"] = request.context.get(PLAN_CONTEXT_SSO, False)
         if request.http_request:
             self.set_http_request(request.http_request)
         self._context["request"] = request
+        self._context["context"] = request.context
 
     def set_http_request(self, request: HttpRequest):
         """Update context based on http request"""
-        # update passbook/policies/expression/templates/policy/expression/form.html
         # update docs/policies/expression/index.md
         self._context["pb_client_ip"] = ip_address(
             get_client_ip(request) or "255.255.255.255"
         )
         self._context["request"] = request
-        if SESSION_KEY_PLAN in request.session:
-            self._context["pb_flow_plan"] = request.session[SESSION_KEY_PLAN]
 
     def evaluate(self, expression_source: str) -> PolicyResult:
         """Parse and evaluate expression. Policy is expected to return a truthy object.

passbook/policies/expression/migrations/0002_auto_20200926_1156.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.1 on 2020-09-26 11:56
+
+from django.apps.registry import Apps
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+
+def remove_pb_flow_plan(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
+    ExpressionPolicy = apps.get_model(
+        "passbook_policies_expression", "ExpressionPolicy"
+    )
+
+    db_alias = schema_editor.connection.alias
+
+    for policy in ExpressionPolicy.objects.using(db_alias).all():
+        policy.expression.replace("pb_flow_plan.", "context.")
+        policy.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_policies_expression", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_pb_flow_plan),
+    ]

passbook/policies/models.py

@@ -64,7 +64,7 @@ class PolicyBinding(SerializerModel):
         return PolicyBindingSerializer
 
     def __str__(self) -> str:
-        return f"PolicyBinding policy={self.policy} target={self.target} order={self.order}"
+        return f"Policy Binding {self.target} #{self.order} {self.policy}"
 
     class Meta:
 

passbook/providers/oauth2/migrations/0005_auto_20200920_1240.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.1.1 on 2020-09-20 12:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "passbook_providers_oauth2",
+            "0004_remove_oauth2provider_post_logout_redirect_uris",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="oauth2provider",
+            name="response_type",
+            field=models.TextField(
+                choices=[
+                    ("code", "code (Authorization Code Flow)"),
+                    (
+                        "code#adfs",
+                        "code (ADFS Compatibility Mode, sends id_token as access_token)",
+                    ),
+                    ("id_token", "id_token (Implicit Flow)"),
+                    ("id_token token", "id_token token (Implicit Flow)"),
+                    ("code token", "code token (Hybrid Flow)"),
+                    ("code id_token", "code id_token (Hybrid Flow)"),
+                    ("code id_token token", "code id_token token (Hybrid Flow)"),
+                ],
+                default="code",
+                help_text="Response Type required by the client.",
+            ),
+        ),
+    ]

passbook/providers/oauth2/views/token.py

@@ -218,10 +218,10 @@ class TokenView(View):
         if unauthorized_scopes:
             raise TokenError("invalid_scope")
 
-        refresh_token = self.params.refresh_token.provider.create_token(
-            user=self.params.refresh_token.user,
-            provider=self.params.refresh_token.provider,
-            scope=self.params.scope,
+        provider: OAuth2Provider = self.params.refresh_token.provider
+
+        refresh_token: RefreshToken = provider.create_refresh_token(
+            user=self.params.refresh_token.user, scope=self.params.scope,
         )
 
         # If the Token has an id_token it's an Authentication request.

passbook/providers/proxy/api.py

@@ -54,6 +54,7 @@ class ProxyProviderSerializer(ModelSerializer):
             "name",
             "internal_host",
             "external_host",
+            "internal_host_ssl_validation",
             "certificate",
             "skip_path_regex",
         ]
@@ -89,6 +90,7 @@ class ProxyOutpostConfigSerializer(ModelSerializer):
             "name",
             "internal_host",
             "external_host",
+            "internal_host_ssl_validation",
             "client_id",
             "client_secret",
             "oidc_configuration",

passbook/providers/proxy/forms.py

@@ -33,6 +33,7 @@ class ProxyProviderForm(forms.ModelForm):
             "name",
             "authorization_flow",
             "internal_host",
+            "internal_host_ssl_validation",
             "external_host",
             "certificate",
             "skip_path_regex",

passbook/providers/proxy/migrations/0007_auto_20200923_1017.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.1.1 on 2020-09-23 10:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_providers_proxy", "0006_proxyprovider_skip_path_regex"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="proxyprovider",
+            name="internal_host_ssl_validation",
+            field=models.BooleanField(
+                default=True, help_text="Validate SSL Certificates of upstream servers"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="proxyprovider",
+            name="skip_path_regex",
+            field=models.TextField(
+                blank=True,
+                default="",
+                help_text="Regular expressions for which authentication is not required. Each new line is interpreted as a new Regular Expression.",
+            ),
+        ),
+    ]

passbook/providers/proxy/models.py

@@ -46,15 +46,16 @@ class ProxyProvider(OutpostModel, OAuth2Provider):
     external_host = models.TextField(
         validators=[DomainlessURLValidator(schemes=("http", "https"))]
     )
-
-    cookie_secret = models.TextField(default=get_cookie_secret)
+    internal_host_ssl_validation = models.BooleanField(
+        default=True, help_text=_("Validate SSL Certificates of upstream servers")
+    )
 
     skip_path_regex = models.TextField(
         default="",
         blank=True,
         help_text=_(
             (
-                "Regular expression for which authentication is not required. "
+                "Regular expressions for which authentication is not required. "
                 "Each new line is interpreted as a new Regular Expression."
             )
         ),
@@ -64,6 +65,8 @@ class ProxyProvider(OutpostModel, OAuth2Provider):
         CertificateKeyPair, on_delete=models.SET_NULL, null=True, blank=True,
     )
 
+    cookie_secret = models.TextField(default=get_cookie_secret)
+
     def form(self) -> Type[ModelForm]:
         from passbook.providers.proxy.forms import ProxyProviderForm
 

passbook/root/settings.py

@@ -22,8 +22,9 @@ from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 
 from passbook import __version__
+from passbook.core.middleware import structlog_add_request_id
 from passbook.lib.config import CONFIG
-from passbook.lib.logging import add_process_id
+from passbook.lib.logging import add_common_fields, add_process_id
 from passbook.lib.sentry import before_send
 
 
@@ -175,6 +176,8 @@ MIDDLEWARE = [
     "django_prometheus.middleware.PrometheusBeforeMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "passbook.core.middleware.RequestIDMiddleware",
+    "passbook.audit.middleware.AuditMiddleware",
     "django.middleware.security.SecurityMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
@@ -330,6 +333,8 @@ structlog.configure_once(
         structlog.stdlib.add_log_level,
         structlog.stdlib.add_logger_name,
         add_process_id,
+        add_common_fields(CONFIG.y("error_reporting.environment", "customer")),
+        structlog_add_request_id,
         structlog.stdlib.PositionalArgumentsFormatter(),
         structlog.processors.TimeStamper(),
         structlog.processors.StackInfoRenderer(),

passbook/sources/ldap/api.py

@@ -24,6 +24,7 @@ class LDAPSourceSerializer(ModelSerializer):
             "user_group_membership_field",
             "object_uniqueness_field",
             "sync_users",
+            "sync_users_password",
             "sync_groups",
             "sync_parent_group",
             "property_mappings",

passbook/sources/ldap/auth.py

@@ -1,9 +1,12 @@
 """passbook LDAP Authentication Backend"""
+from typing import Optional
+
+import ldap3
 from django.contrib.auth.backends import ModelBackend
 from django.http import HttpRequest
 from structlog import get_logger
 
-from passbook.sources.ldap.connector import Connector
+from passbook.core.models import User
 from passbook.sources.ldap.models import LDAPSource
 
 LOGGER = get_logger()
@@ -18,7 +21,56 @@ class LDAPBackend(ModelBackend):
             return None
         for source in LDAPSource.objects.filter(enabled=True):
             LOGGER.debug("LDAP Auth attempt", source=source)
-            user = Connector(source).auth_user(**kwargs)
+            user = self.auth_user(source, **kwargs)
             if user:
                 return user
         return None
+
+    def auth_user(
+        self, source: LDAPSource, password: str, **filters: str
+    ) -> Optional[User]:
+        """Try to bind as either user_dn or mail with password.
+        Returns True on success, otherwise False"""
+        users = User.objects.filter(**filters)
+        if not users.exists():
+            return None
+        user: User = users.first()
+        if "distinguishedName" not in user.attributes:
+            LOGGER.debug(
+                "User doesn't have DN set, assuming not LDAP imported.", user=user
+            )
+            return None
+        # Either has unusable password,
+        # or has a password, but couldn't be authenticated by ModelBackend.
+        # This means we check with a bind to see if the LDAP password has changed
+        if self.auth_user_by_bind(source, user, password):
+            # Password given successfully binds to LDAP, so we save it in our Database
+            LOGGER.debug("Updating user's password in DB", user=user)
+            user.set_password(password, signal=False)
+            user.save()
+            return user
+        # Password doesn't match
+        LOGGER.debug("Failed to bind, password invalid")
+        return None
+
+    def auth_user_by_bind(
+        self, source: LDAPSource, user: User, password: str
+    ) -> Optional[User]:
+        """Attempt authentication by binding to the LDAP server as `user`. This
+        method should be avoided as its slow to do the bind."""
+        # Try to bind as new user
+        LOGGER.debug("Attempting Binding as user", user=user)
+        try:
+            temp_connection = ldap3.Connection(
+                source.connection.server,
+                user=user.attributes.get("distinguishedName"),
+                password=password,
+                raise_exceptions=True,
+            )
+            temp_connection.bind()
+            return user
+        except ldap3.core.exceptions.LDAPInvalidCredentialsResult as exception:
+            LOGGER.debug("LDAPInvalidCredentialsResult", user=user, error=exception)
+        except ldap3.core.exceptions.LDAPException as exception:
+            LOGGER.warning(exception)
+        return None

passbook/sources/ldap/forms.py

@@ -37,6 +37,7 @@ class LDAPSourceForm(forms.ModelForm):
             "user_group_membership_field",
             "object_uniqueness_field",
             "sync_users",
+            "sync_users_password",
             "sync_groups",
             "sync_parent_group",
             "property_mappings",

passbook/sources/ldap/migrations/0007_ldapsource_sync_users_password.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.1.1 on 2020-09-21 09:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_sources_ldap", "0006_auto_20200915_1919"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ldapsource",
+            name="sync_users_password",
+            field=models.BooleanField(
+                default=True,
+                help_text="When a user changes their password, sync it back to LDAP. This can only be enabled on a single LDAP source.",
+                unique=True,
+            ),
+        ),
+    ]

passbook/sources/ldap/models.py

@@ -6,7 +6,7 @@ from django.core.cache import cache
 from django.db import models
 from django.forms import ModelForm
 from django.utils.translation import gettext_lazy as _
-from ldap3 import Connection, Server
+from ldap3 import ALL, Connection, Server
 
 from passbook.core.models import Group, PropertyMapping, Source
 from passbook.lib.models import DomainlessURLValidator
@@ -52,6 +52,16 @@ class LDAPSource(Source):
     )
 
     sync_users = models.BooleanField(default=True)
+    sync_users_password = models.BooleanField(
+        default=True,
+        help_text=_(
+            (
+                "When a user changes their password, sync it back to LDAP. "
+                "This can only be enabled on a single LDAP source."
+            )
+        ),
+        unique=True,
+    )
     sync_groups = models.BooleanField(default=True)
     sync_parent_group = models.ForeignKey(
         Group, blank=True, null=True, default=None, on_delete=models.SET_DEFAULT
@@ -82,7 +92,7 @@ class LDAPSource(Source):
     def connection(self) -> Connection:
         """Get a fully connected and bound LDAP Connection"""
         if not self._connection:
-            server = Server(self.server_uri)
+            server = Server(self.server_uri, get_info=ALL)
             self._connection = Connection(
                 server,
                 raise_exceptions=True,
@@ -112,7 +122,7 @@ class LDAPPropertyMapping(PropertyMapping):
         return LDAPPropertyMappingForm
 
     def __str__(self):
-        return f"LDAP Property Mapping {self.expression} -> {self.object_field}"
+        return self.name
 
     class Meta:
 

passbook/sources/ldap/password.py

@@ -0,0 +1,155 @@
+"""Help validate and update passwords in LDAP"""
+from enum import IntFlag
+from re import split
+from typing import Optional
+
+import ldap3
+import ldap3.core.exceptions
+from structlog import get_logger
+
+from passbook.core.models import User
+from passbook.sources.ldap.models import LDAPSource
+
+LOGGER = get_logger()
+
+NON_ALPHA = r"~!@#$%^&*_-+=`|\(){}[]:;\"'<>,.?/"
+RE_DISPLAYNAME_SEPARATORS = r",\.–—_\s#\t"
+
+
+class PwdProperties(IntFlag):
+    """Possible values for the pwdProperties attribute"""
+
+    DOMAIN_PASSWORD_COMPLEX = 1
+    DOMAIN_PASSWORD_NO_ANON_CHANGE = 2
+    DOMAIN_PASSWORD_NO_CLEAR_CHANGE = 4
+    DOMAIN_LOCKOUT_ADMINS = 8
+    DOMAIN_PASSWORD_STORE_CLEARTEXT = 16
+    DOMAIN_REFUSE_PASSWORD_CHANGE = 32
+
+
+class PasswordCategories(IntFlag):
+    """Password categories as defined by Microsoft, a category can only be counted
+    once, hence intflag."""
+
+    NONE = 0
+    ALPHA_LOWER = 1
+    ALPHA_UPPER = 2
+    ALPHA_OTHER = 4
+    NUMERIC = 8
+    SYMBOL = 16
+
+
+class LDAPPasswordChanger:
+    """Help validate and update passwords in LDAP"""
+
+    _source: LDAPSource
+
+    def __init__(self, source: LDAPSource) -> None:
+        self._source = source
+
+    def get_domain_root_dn(self) -> str:
+        """Attempt to get root DN via MS specific fields or generic LDAP fields"""
+        info = self._source.connection.server.info
+        if "rootDomainNamingContext" in info.other:
+            return info.other["rootDomainNamingContext"][0]
+        naming_contexts = info.naming_contexts
+        naming_contexts.sort(key=len)
+        return naming_contexts[0]
+
+    def check_ad_password_complexity_enabled(self) -> bool:
+        """Check if DOMAIN_PASSWORD_COMPLEX is enabled"""
+        root_dn = self.get_domain_root_dn()
+        root_attrs = self._source.connection.extend.standard.paged_search(
+            search_base=root_dn,
+            search_filter="(objectClass=*)",
+            search_scope=ldap3.BASE,
+            attributes=["pwdProperties"],
+        )
+        root_attrs = list(root_attrs)[0]
+        pwd_properties = PwdProperties(root_attrs["attributes"]["pwdProperties"])
+        if PwdProperties.DOMAIN_PASSWORD_COMPLEX in pwd_properties:
+            return True
+
+        return False
+
+    def change_password(self, user: User, password: str):
+        """Change user's password"""
+        user_dn = user.attributes.get("distinguishedName", None)
+        if not user_dn:
+            raise AttributeError("User has no distinguishedName set.")
+        self._source.connection.extend.microsoft.modify_password(user_dn, password)
+
+    def _ad_check_password_existing(self, password: str, user_dn: str) -> bool:
+        """Check if a password contains sAMAccount or displayName"""
+        users = list(
+            self._source.connection.extend.standard.paged_search(
+                search_base=user_dn,
+                search_filter=self._source.user_object_filter,
+                search_scope=ldap3.BASE,
+                attributes=["displayName", "sAMAccountName"],
+            )
+        )
+        if len(users) != 1:
+            raise AssertionError()
+        user_attributes = users[0]["attributes"]
+        # If sAMAccountName is longer than 3 chars, check if its contained in password
+        if len(user_attributes["sAMAccountName"]) >= 3:
+            if password.lower() in user_attributes["sAMAccountName"].lower():
+                return False
+        display_name_tokens = split(
+            RE_DISPLAYNAME_SEPARATORS, user_attributes["displayName"]
+        )
+        for token in display_name_tokens:
+            # Ignore tokens under 3 chars
+            if len(token) < 3:
+                continue
+            if token.lower() in password.lower():
+                return False
+        return True
+
+    def ad_password_complexity(
+        self, password: str, user: Optional[User] = None
+    ) -> bool:
+        """Check if password matches Active direcotry password policies
+
+        https://docs.microsoft.com/en-us/windows/security/threat-protection/
+            security-policy-settings/password-must-meet-complexity-requirements
+        """
+        if user:
+            # Check if password contains sAMAccountName or displayNames
+            if "distinguishedName" in user.attributes:
+                existing_user_check = self._ad_check_password_existing(
+                    password, user.attributes.get("distinguishedName")
+                )
+                if not existing_user_check:
+                    LOGGER.debug("Password failed name check", user=user)
+                    return existing_user_check
+
+        # Step 2, match at least 3 of 5 categories
+        matched_categories = PasswordCategories.NONE
+        required = 3
+        for letter in password:
+            # Only match one category per letter,
+            if letter.islower():
+                matched_categories |= PasswordCategories.ALPHA_LOWER
+            elif letter.isupper():
+                matched_categories |= PasswordCategories.ALPHA_UPPER
+            elif not letter.isascii() and letter.isalpha():
+                # Not exactly matching microsoft's policy, but count it as "Other unicode" char
+                # when its alpha and not ascii
+                matched_categories |= PasswordCategories.ALPHA_OTHER
+            elif letter.isnumeric():
+                matched_categories |= PasswordCategories.NUMERIC
+            elif letter in NON_ALPHA:
+                matched_categories |= PasswordCategories.SYMBOL
+        if bin(matched_categories).count("1") < required:
+            LOGGER.debug(
+                "Password didn't match enough categories",
+                has=matched_categories,
+                must=required,
+            )
+            return False
+        LOGGER.debug(
+            "Password matched categories", has=matched_categories, must=required
+        )
+        return True

passbook/sources/ldap/signals.py

@@ -1,9 +1,19 @@
 """passbook ldap source signals"""
+from typing import Any, Dict
+
+from django.core.exceptions import ValidationError
 from django.db.models.signals import post_save
 from django.dispatch import receiver
+from django.utils.translation import gettext_lazy as _
+from ldap3.core.exceptions import LDAPException
 
+from passbook.core.models import User
+from passbook.core.signals import password_changed
+from passbook.flows.planner import PLAN_CONTEXT_PENDING_USER
 from passbook.sources.ldap.models import LDAPSource
+from passbook.sources.ldap.password import LDAPPasswordChanger
 from passbook.sources.ldap.tasks import sync_single
+from passbook.stages.prompt.signals import password_validate
 
 
 @receiver(post_save, sender=LDAPSource)
@@ -12,3 +22,38 @@ def sync_ldap_source_on_save(sender, instance: LDAPSource, **_):
     """Ensure that source is synced on save (if enabled)"""
     if instance.enabled:
         sync_single.delay(instance.pk)
+
+
+@receiver(password_validate)
+# pylint: disable=unused-argument
+def ldap_password_validate(sender, password: str, plan_context: Dict[str, Any], **__):
+    """if there's an LDAP Source with enabled password sync, check the password"""
+    sources = LDAPSource.objects.filter(sync_users_password=True)
+    if not sources.exists():
+        return
+    source = sources.first()
+    changer = LDAPPasswordChanger(source)
+    if changer.check_ad_password_complexity_enabled():
+        passing = changer.ad_password_complexity(
+            password, plan_context.get(PLAN_CONTEXT_PENDING_USER, None)
+        )
+        if not passing:
+            raise ValidationError(
+                _("Password does not match Active Direcory Complexity.")
+            )
+
+
+@receiver(password_changed)
+# pylint: disable=unused-argument
+def ldap_sync_password(sender, user: User, password: str, **_):
+    """Connect to ldap and update password. We do this in the background to get
+    automatic retries on error."""
+    sources = LDAPSource.objects.filter(sync_users_password=True)
+    if not sources.exists():
+        return
+    source = sources.first()
+    changer = LDAPPasswordChanger(source)
+    try:
+        changer.change_password(user, password)
+    except LDAPException as exc:
+        raise ValidationError("Failed to set password") from exc

passbook/sources/ldap/sync.py

@@ -1,5 +1,5 @@
-"""Wrapper for ldap3 to easily manage user"""
-from typing import Any, Dict, Optional
+"""Sync LDAP Users and groups into passbook"""
+from typing import Any, Dict
 
 import ldap3
 import ldap3.core.exceptions
@@ -13,19 +13,14 @@ from passbook.sources.ldap.models import LDAPPropertyMapping, LDAPSource
 LOGGER = get_logger()
 
 
-class Connector:
-    """Wrapper for ldap3 to easily manage user authentication and creation"""
+class LDAPSynchronizer:
+    """Sync LDAP Users and groups into passbook"""
 
     _source: LDAPSource
 
     def __init__(self, source: LDAPSource):
         self._source = source
 
-    @staticmethod
-    def encode_pass(password: str) -> bytes:
-        """Encodes a plain-text password so it can be used by AD"""
-        return '"{}"'.format(password).encode("utf-16-le")
-
     @property
     def base_dn_users(self) -> str:
         """Shortcut to get full base_dn for user lookups"""
@@ -187,48 +182,3 @@ class Connector:
             "distinguishedName"
         )
         return properties
-
-    def auth_user(self, password: str, **filters: str) -> Optional[User]:
-        """Try to bind as either user_dn or mail with password.
-        Returns True on success, otherwise False"""
-        users = User.objects.filter(**filters)
-        if not users.exists():
-            return None
-        user: User = users.first()
-        if "distinguishedName" not in user.attributes:
-            LOGGER.debug(
-                "User doesn't have DN set, assuming not LDAP imported.", user=user
-            )
-            return None
-        # Either has unusable password,
-        # or has a password, but couldn't be authenticated by ModelBackend.
-        # This means we check with a bind to see if the LDAP password has changed
-        if self.auth_user_by_bind(user, password):
-            # Password given successfully binds to LDAP, so we save it in our Database
-            LOGGER.debug("Updating user's password in DB", user=user)
-            user.set_password(password)
-            user.save()
-            return user
-        # Password doesn't match
-        LOGGER.debug("Failed to bind, password invalid")
-        return None
-
-    def auth_user_by_bind(self, user: User, password: str) -> Optional[User]:
-        """Attempt authentication by binding to the LDAP server as `user`. This
-        method should be avoided as its slow to do the bind."""
-        # Try to bind as new user
-        LOGGER.debug("Attempting Binding as user", user=user)
-        try:
-            temp_connection = ldap3.Connection(
-                self._source.connection.server,
-                user=user.attributes.get("distinguishedName"),
-                password=password,
-                raise_exceptions=True,
-            )
-            temp_connection.bind()
-            return user
-        except ldap3.core.exceptions.LDAPInvalidCredentialsResult as exception:
-            LOGGER.debug("LDAPInvalidCredentialsResult", user=user, error=exception)
-        except ldap3.core.exceptions.LDAPException as exception:
-            LOGGER.warning(exception)
-        return None

passbook/sources/ldap/tasks.py

@@ -4,8 +4,8 @@ from time import time
 from django.core.cache import cache
 
 from passbook.root.celery import CELERY_APP
-from passbook.sources.ldap.connector import Connector
 from passbook.sources.ldap.models import LDAPSource
+from passbook.sources.ldap.sync import LDAPSynchronizer
 
 
 @CELERY_APP.task()
@@ -19,9 +19,9 @@ def sync():
 def sync_single(source_pk):
     """Sync a single source"""
     source: LDAPSource = LDAPSource.objects.get(pk=source_pk)
-    connector = Connector(source)
-    connector.sync_users()
-    connector.sync_groups()
-    connector.sync_membership()
+    syncer = LDAPSynchronizer(source)
+    syncer.sync_users()
+    syncer.sync_groups()
+    syncer.sync_membership()
     cache_key = source.state_cache_prefix("last_sync")
     cache.set(cache_key, time(), timeout=60 * 60)

passbook/sources/ldap/tests.py

@@ -1,149 +0,0 @@
-"""LDAP Source tests"""
-from unittest.mock import Mock, PropertyMock, patch
-
-from django.test import TestCase
-from ldap3 import MOCK_SYNC, OFFLINE_AD_2012_R2, Connection, Server
-
-from passbook.core.models import Group, User
-from passbook.providers.oauth2.generators import generate_client_secret
-from passbook.sources.ldap.auth import LDAPBackend
-from passbook.sources.ldap.connector import Connector
-from passbook.sources.ldap.models import LDAPPropertyMapping, LDAPSource
-from passbook.sources.ldap.tasks import sync
-
-
-def _build_mock_connection() -> Connection:
-    """Create mock connection"""
-    server = Server("my_fake_server", get_info=OFFLINE_AD_2012_R2)
-    _pass = "foo"  # noqa # nosec
-    connection = Connection(
-        server,
-        user="cn=my_user,ou=test,o=lab",
-        password=_pass,
-        client_strategy=MOCK_SYNC,
-    )
-    connection.strategy.add_entry(
-        "cn=group1,ou=groups,ou=test,o=lab",
-        {
-            "name": "test-group",
-            "objectSid": "unique-test-group",
-            "objectCategory": "Group",
-            "distinguishedName": "cn=group1,ou=groups,ou=test,o=lab",
-        },
-    )
-    # Group without SID
-    connection.strategy.add_entry(
-        "cn=group2,ou=groups,ou=test,o=lab",
-        {
-            "name": "test-group",
-            "objectCategory": "Group",
-            "distinguishedName": "cn=group2,ou=groups,ou=test,o=lab",
-        },
-    )
-    connection.strategy.add_entry(
-        "cn=user0,ou=users,ou=test,o=lab",
-        {
-            "userPassword": LDAP_PASSWORD,
-            "sAMAccountName": "user0_sn",
-            "name": "user0_sn",
-            "revision": 0,
-            "objectSid": "user0",
-            "objectCategory": "Person",
-            "memberOf": "cn=group1,ou=groups,ou=test,o=lab",
-        },
-    )
-    # User without SID
-    connection.strategy.add_entry(
-        "cn=user1,ou=users,ou=test,o=lab",
-        {
-            "userPassword": "test1111",
-            "sAMAccountName": "user2_sn",
-            "name": "user1_sn",
-            "revision": 0,
-            "objectCategory": "Person",
-        },
-    )
-    # Duplicate users
-    connection.strategy.add_entry(
-        "cn=user2,ou=users,ou=test,o=lab",
-        {
-            "userPassword": "test2222",
-            "sAMAccountName": "user2_sn",
-            "name": "user2_sn",
-            "revision": 0,
-            "objectSid": "unique-test2222",
-            "objectCategory": "Person",
-        },
-    )
-    connection.strategy.add_entry(
-        "cn=user3,ou=users,ou=test,o=lab",
-        {
-            "userPassword": "test2222",
-            "sAMAccountName": "user2_sn",
-            "name": "user2_sn",
-            "revision": 0,
-            "objectSid": "unique-test2222",
-            "objectCategory": "Person",
-        },
-    )
-    connection.bind()
-    return connection
-
-
-LDAP_PASSWORD = generate_client_secret()
-LDAP_CONNECTION_PATCH = PropertyMock(return_value=_build_mock_connection())
-
-
-class LDAPSourceTests(TestCase):
-    """LDAP Source tests"""
-
-    def setUp(self):
-        self.source = LDAPSource.objects.create(
-            name="ldap",
-            slug="ldap",
-            base_dn="ou=test,o=lab",
-            additional_user_dn="ou=users",
-            additional_group_dn="ou=groups",
-        )
-        self.source.property_mappings.set(LDAPPropertyMapping.objects.all())
-        self.source.save()
-
-    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
-    def test_sync_users(self):
-        """Test user sync"""
-        connector = Connector(self.source)
-        connector.sync_users()
-        self.assertTrue(User.objects.filter(username="user0_sn").exists())
-        self.assertFalse(User.objects.filter(username="user1_sn").exists())
-
-    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
-    def test_sync_groups(self):
-        """Test group sync"""
-        connector = Connector(self.source)
-        connector.sync_groups()
-        connector.sync_membership()
-        group = Group.objects.filter(name="test-group")
-        self.assertTrue(group.exists())
-
-    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
-    def test_auth(self):
-        """Test Cached auth"""
-        connector = Connector(self.source)
-        connector.sync_users()
-
-        user = User.objects.get(username="user0_sn")
-        auth_user_by_bind = Mock(return_value=user)
-        with patch(
-            "passbook.sources.ldap.connector.Connector.auth_user_by_bind",
-            auth_user_by_bind,
-        ):
-            backend = LDAPBackend()
-            self.assertEqual(
-                backend.authenticate(None, username="user0_sn", password=LDAP_PASSWORD),
-                user,
-            )
-
-    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
-    def test_tasks(self):
-        """Test Scheduled tasks"""
-        sync()

passbook/sources/ldap/tests/test_auth.py

@@ -0,0 +1,47 @@
+"""LDAP Source tests"""
+from unittest.mock import Mock, PropertyMock, patch
+
+from django.test import TestCase
+
+from passbook.core.models import User
+from passbook.providers.oauth2.generators import generate_client_secret
+from passbook.sources.ldap.auth import LDAPBackend
+from passbook.sources.ldap.models import LDAPPropertyMapping, LDAPSource
+from passbook.sources.ldap.sync import LDAPSynchronizer
+from passbook.sources.ldap.tests.utils import _build_mock_connection
+
+LDAP_PASSWORD = generate_client_secret()
+LDAP_CONNECTION_PATCH = PropertyMock(return_value=_build_mock_connection(LDAP_PASSWORD))
+
+
+class LDAPSyncTests(TestCase):
+    """LDAP Sync tests"""
+
+    def setUp(self):
+        self.source = LDAPSource.objects.create(
+            name="ldap",
+            slug="ldap",
+            base_dn="DC=AD2012,DC=LAB",
+            additional_user_dn="ou=users",
+            additional_group_dn="ou=groups",
+        )
+        self.source.property_mappings.set(LDAPPropertyMapping.objects.all())
+        self.source.save()
+
+    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
+    def test_auth_synced_user(self):
+        """Test Cached auth"""
+        syncer = LDAPSynchronizer(self.source)
+        syncer.sync_users()
+
+        user = User.objects.get(username="user0_sn")
+        auth_user_by_bind = Mock(return_value=user)
+        with patch(
+            "passbook.sources.ldap.auth.LDAPBackend.auth_user_by_bind",
+            auth_user_by_bind,
+        ):
+            backend = LDAPBackend()
+            self.assertEqual(
+                backend.authenticate(None, username="user0_sn", password=LDAP_PASSWORD),
+                user,
+            )

passbook/sources/ldap/tests/test_password.py

@@ -0,0 +1,54 @@
+"""LDAP Source tests"""
+from unittest.mock import PropertyMock, patch
+
+from django.test import TestCase
+
+from passbook.core.models import User
+from passbook.providers.oauth2.generators import generate_client_secret
+from passbook.sources.ldap.models import LDAPPropertyMapping, LDAPSource
+from passbook.sources.ldap.password import LDAPPasswordChanger
+from passbook.sources.ldap.tests.utils import _build_mock_connection
+
+LDAP_PASSWORD = generate_client_secret()
+LDAP_CONNECTION_PATCH = PropertyMock(return_value=_build_mock_connection(LDAP_PASSWORD))
+
+
+class LDAPPasswordTests(TestCase):
+    """LDAP Password tests"""
+
+    def setUp(self):
+        self.source = LDAPSource.objects.create(
+            name="ldap",
+            slug="ldap",
+            base_dn="DC=AD2012,DC=LAB",
+            additional_user_dn="ou=users",
+            additional_group_dn="ou=groups",
+        )
+        self.source.property_mappings.set(LDAPPropertyMapping.objects.all())
+        self.source.save()
+
+    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
+    def test_password_complexity(self):
+        """Test password without user"""
+        pwc = LDAPPasswordChanger(self.source)
+        self.assertFalse(pwc.ad_password_complexity("test"))  # 1 category
+        self.assertFalse(pwc.ad_password_complexity("test1"))  # 2 categories
+        self.assertTrue(pwc.ad_password_complexity("test1!"))  # 2 categories
+
+    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
+    def test_password_complexity_user(self):
+        """test password with user"""
+        pwc = LDAPPasswordChanger(self.source)
+        user = User.objects.create(
+            username="test",
+            attributes={"distinguishedName": "cn=user,ou=users,DC=AD2012,DC=LAB"},
+        )
+        self.assertFalse(pwc.ad_password_complexity("test", user))  # 1 category
+        self.assertFalse(pwc.ad_password_complexity("test1", user))  # 2 categories
+        self.assertTrue(pwc.ad_password_complexity("test1!", user))  # 2 categories
+        self.assertFalse(
+            pwc.ad_password_complexity("erin!qewrqewr", user)
+        )  # displayName token
+        self.assertFalse(
+            pwc.ad_password_complexity("hagens!qewrqewr", user)
+        )  # displayName token

passbook/sources/ldap/tests/test_sync.py

@@ -0,0 +1,51 @@
+"""LDAP Source tests"""
+from unittest.mock import PropertyMock, patch
+
+from django.test import TestCase
+
+from passbook.core.models import Group, User
+from passbook.providers.oauth2.generators import generate_client_secret
+from passbook.sources.ldap.models import LDAPPropertyMapping, LDAPSource
+from passbook.sources.ldap.sync import LDAPSynchronizer
+from passbook.sources.ldap.tasks import sync
+from passbook.sources.ldap.tests.utils import _build_mock_connection
+
+LDAP_PASSWORD = generate_client_secret()
+LDAP_CONNECTION_PATCH = PropertyMock(return_value=_build_mock_connection(LDAP_PASSWORD))
+
+
+class LDAPSyncTests(TestCase):
+    """LDAP Sync tests"""
+
+    def setUp(self):
+        self.source = LDAPSource.objects.create(
+            name="ldap",
+            slug="ldap",
+            base_dn="DC=AD2012,DC=LAB",
+            additional_user_dn="ou=users",
+            additional_group_dn="ou=groups",
+        )
+        self.source.property_mappings.set(LDAPPropertyMapping.objects.all())
+        self.source.save()
+
+    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
+    def test_sync_users(self):
+        """Test user sync"""
+        syncer = LDAPSynchronizer(self.source)
+        syncer.sync_users()
+        self.assertTrue(User.objects.filter(username="user0_sn").exists())
+        self.assertFalse(User.objects.filter(username="user1_sn").exists())
+
+    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
+    def test_sync_groups(self):
+        """Test group sync"""
+        syncer = LDAPSynchronizer(self.source)
+        syncer.sync_groups()
+        syncer.sync_membership()
+        group = Group.objects.filter(name="test-group")
+        self.assertTrue(group.exists())
+
+    @patch("passbook.sources.ldap.models.LDAPSource.connection", LDAP_CONNECTION_PATCH)
+    def test_tasks(self):
+        """Test Scheduled tasks"""
+        sync()

passbook/sources/ldap/tests/utils.py

@@ -0,0 +1,93 @@
+"""ldap testing utils"""
+
+from ldap3 import MOCK_SYNC, OFFLINE_AD_2012_R2, Connection, Server
+
+
+def _build_mock_connection(password: str) -> Connection:
+    """Create mock connection"""
+    server = Server("my_fake_server", get_info=OFFLINE_AD_2012_R2)
+    _pass = "foo"  # noqa # nosec
+    connection = Connection(
+        server,
+        user="cn=my_user,DC=AD2012,DC=LAB",
+        password=_pass,
+        client_strategy=MOCK_SYNC,
+    )
+    # Entry for password checking
+    connection.strategy.add_entry(
+        "cn=user,ou=users,DC=AD2012,DC=LAB",
+        {
+            "name": "test-user",
+            "objectSid": "unique-test-group",
+            "objectCategory": "Person",
+            "displayName": "Erin M. Hagens",
+            "sAMAccountName": "sAMAccountName",
+            "distinguishedName": "cn=user,ou=users,DC=AD2012,DC=LAB",
+        },
+    )
+    connection.strategy.add_entry(
+        "cn=group1,ou=groups,DC=AD2012,DC=LAB",
+        {
+            "name": "test-group",
+            "objectSid": "unique-test-group",
+            "objectCategory": "Group",
+            "distinguishedName": "cn=group1,ou=groups,DC=AD2012,DC=LAB",
+        },
+    )
+    # Group without SID
+    connection.strategy.add_entry(
+        "cn=group2,ou=groups,DC=AD2012,DC=LAB",
+        {
+            "name": "test-group",
+            "objectCategory": "Group",
+            "distinguishedName": "cn=group2,ou=groups,DC=AD2012,DC=LAB",
+        },
+    )
+    connection.strategy.add_entry(
+        "cn=user0,ou=users,DC=AD2012,DC=LAB",
+        {
+            "userPassword": password,
+            "sAMAccountName": "user0_sn",
+            "name": "user0_sn",
+            "revision": 0,
+            "objectSid": "user0",
+            "objectCategory": "Person",
+            "memberOf": "cn=group1,ou=groups,DC=AD2012,DC=LAB",
+        },
+    )
+    # User without SID
+    connection.strategy.add_entry(
+        "cn=user1,ou=users,DC=AD2012,DC=LAB",
+        {
+            "userPassword": "test1111",
+            "sAMAccountName": "user2_sn",
+            "name": "user1_sn",
+            "revision": 0,
+            "objectCategory": "Person",
+        },
+    )
+    # Duplicate users
+    connection.strategy.add_entry(
+        "cn=user2,ou=users,DC=AD2012,DC=LAB",
+        {
+            "userPassword": "test2222",
+            "sAMAccountName": "user2_sn",
+            "name": "user2_sn",
+            "revision": 0,
+            "objectSid": "unique-test2222",
+            "objectCategory": "Person",
+        },
+    )
+    connection.strategy.add_entry(
+        "cn=user3,ou=users,DC=AD2012,DC=LAB",
+        {
+            "userPassword": "test2222",
+            "sAMAccountName": "user2_sn",
+            "name": "user2_sn",
+            "revision": 0,
+            "objectSid": "unique-test2222",
+            "objectCategory": "Person",
+        },
+    )
+    connection.bind()
+    return connection

passbook/sources/oauth/clients.py

@@ -1,278 +0,0 @@
-"""OAuth Clients"""
-import json
-from typing import TYPE_CHECKING, Any, Dict, Optional
-from urllib.parse import parse_qs, urlencode
-
-from django.http import HttpRequest
-from django.utils.crypto import constant_time_compare, get_random_string
-from django.utils.encoding import force_str
-from requests import Session
-from requests.exceptions import RequestException
-from requests_oauthlib import OAuth1
-from structlog import get_logger
-
-from passbook import __version__
-
-LOGGER = get_logger()
-if TYPE_CHECKING:
-    from passbook.sources.oauth.models import OAuthSource
-
-
-class BaseOAuthClient:
-    """Base OAuth Client"""
-
-    session: Session
-    source: "OAuthSource"
-
-    def __init__(self, source: "OAuthSource", token=""):  # nosec
-        self.source = source
-        self.token = token
-        self.session = Session()
-        self.session.headers.update({"User-Agent": "passbook %s" % __version__})
-
-    def get_access_token(
-        self, request: HttpRequest, callback=None
-    ) -> Optional[Dict[str, Any]]:
-        "Fetch access token from callback request."
-        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
-
-    def get_profile_info(self, token: Dict[str, str]) -> Optional[Dict[str, Any]]:
-        "Fetch user profile information."
-        try:
-            headers = {
-                "Authorization": f"{token['token_type']} {token['access_token']}"
-            }
-            response = self.session.request(
-                "get", self.source.profile_url, headers=headers,
-            )
-            response.raise_for_status()
-        except RequestException as exc:
-            LOGGER.warning("Unable to fetch user profile", exc=exc)
-            return None
-        else:
-            return response.json()
-
-    def get_redirect_args(self, request, callback) -> Dict[str, str]:
-        "Get request parameters for redirect url."
-        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
-
-    def get_redirect_url(self, request, callback, parameters=None):
-        "Build authentication redirect url."
-        args = self.get_redirect_args(request, callback=callback)
-        additional = parameters or {}
-        args.update(additional)
-        params = urlencode(args)
-        LOGGER.info("redirect args", **args)
-        return "{0}?{1}".format(self.source.authorization_url, params)
-
-    def parse_raw_token(self, raw_token):
-        "Parse token and secret from raw token response."
-        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
-
-    @property
-    def session_key(self):
-        """Return Session Key"""
-        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
-
-
-class OAuthClient(BaseOAuthClient):
-    """OAuth1 Client"""
-
-    _default_headers = {
-        "Accept": "application/json",
-    }
-
-    def get_access_token(
-        self, request: HttpRequest, callback=None
-    ) -> Optional[Dict[str, str]]:
-        "Fetch access token from callback request."
-        raw_token = request.session.get(self.session_key, None)
-        verifier = request.GET.get("oauth_verifier", None)
-        if raw_token is not None and verifier is not None:
-            data = {
-                "oauth_verifier": verifier,
-                "oauth_callback": callback,
-                "token": raw_token,
-            }
-            try:
-                response = self.session.request(
-                    "post",
-                    self.source.access_token_url,
-                    data=data,
-                    headers=self._default_headers,
-                )
-                response.raise_for_status()
-            except RequestException as exc:
-                LOGGER.warning("Unable to fetch access token", exc=exc)
-                return None
-            else:
-                return response.json()
-        return None
-
-    def get_request_token(self, request, callback):
-        "Fetch the OAuth request token. Only required for OAuth 1.0."
-        callback = force_str(request.build_absolute_uri(callback))
-        try:
-            response = self.session.request(
-                "post",
-                self.source.request_token_url,
-                data={"oauth_callback": callback},
-                headers=self._default_headers,
-            )
-            response.raise_for_status()
-        except RequestException as exc:
-            LOGGER.warning("Unable to fetch request token", exc=exc)
-            return None
-        else:
-            return response.text
-
-    def get_redirect_args(self, request, callback):
-        "Get request parameters for redirect url."
-        callback = force_str(request.build_absolute_uri(callback))
-        raw_token = self.get_request_token(request, callback)
-        token, secret = self.parse_raw_token(raw_token)
-        if token is not None and secret is not None:
-            request.session[self.session_key] = raw_token
-        return {
-            "oauth_token": token,
-            "oauth_callback": callback,
-        }
-
-    def parse_raw_token(self, raw_token):
-        "Parse token and secret from raw token response."
-        if raw_token is None:
-            return (None, None)
-        query_string = parse_qs(raw_token)
-        token = query_string.get("oauth_token", [None])[0]
-        secret = query_string.get("oauth_token_secret", [None])[0]
-        return (token, secret)
-
-    def request(self, method, url, **kwargs):
-        "Build remote url request. Constructs necessary auth."
-        user_token = kwargs.pop("token", self.token)
-        token, secret = self.parse_raw_token(user_token)
-        callback = kwargs.pop("oauth_callback", None)
-        verifier = kwargs.get("data", {}).pop("oauth_verifier", None)
-        oauth = OAuth1(
-            resource_owner_key=token,
-            resource_owner_secret=secret,
-            client_key=self.source.consumer_key,
-            client_secret=self.source.consumer_secret,
-            verifier=verifier,
-            callback_uri=callback,
-        )
-        kwargs["auth"] = oauth
-        return super(OAuthClient, self).session.request(method, url, **kwargs)
-
-    @property
-    def session_key(self):
-        return "oauth-client-{0}-request-token".format(self.source.name)
-
-
-class OAuth2Client(BaseOAuthClient):
-    """OAuth2 Client"""
-
-    _default_headers = {
-        "Accept": "application/json",
-    }
-
-    # pylint: disable=unused-argument
-    def check_application_state(self, request, callback):
-        "Check optional state parameter."
-        stored = request.session.get(self.session_key, None)
-        returned = request.GET.get("state", None)
-        check = False
-        if stored is not None:
-            if returned is not None:
-                check = constant_time_compare(stored, returned)
-            else:
-                LOGGER.warning("No state parameter returned by the source.")
-        else:
-            LOGGER.warning("No state stored in the sesssion.")
-        return check
-
-    def get_access_token(self, request, callback=None, **request_kwargs):
-        "Fetch access token from callback request."
-        callback = request.build_absolute_uri(callback or request.path)
-        if not self.check_application_state(request, callback):
-            LOGGER.warning("Application state check failed.")
-            return None
-        if "code" in request.GET:
-            args = {
-                "client_id": self.source.consumer_key,
-                "redirect_uri": callback,
-                "client_secret": self.source.consumer_secret,
-                "code": request.GET["code"],
-                "grant_type": "authorization_code",
-            }
-        else:
-            LOGGER.warning("No code returned by the source")
-            return None
-        try:
-            response = self.session.request(
-                "post",
-                self.source.access_token_url,
-                data=args,
-                headers=self._default_headers,
-                **request_kwargs,
-            )
-            response.raise_for_status()
-        except RequestException as exc:
-            LOGGER.warning("Unable to fetch access token", exc=exc)
-            return None
-        else:
-            return response.json()
-
-    # pylint: disable=unused-argument
-    def get_application_state(self, request, callback):
-        "Generate state optional parameter."
-        return get_random_string(32)
-
-    def get_redirect_args(self, request, callback):
-        "Get request parameters for redirect url."
-        callback = request.build_absolute_uri(callback)
-        args = {
-            "client_id": self.source.consumer_key,
-            "redirect_uri": callback,
-            "response_type": "code",
-        }
-        state = self.get_application_state(request, callback)
-        if state is not None:
-            args["state"] = state
-            request.session[self.session_key] = state
-        return args
-
-    def parse_raw_token(self, raw_token):
-        "Parse token and secret from raw token response."
-        if raw_token is None:
-            return (None, None)
-        # Load as json first then parse as query string
-        try:
-            token_data = json.loads(raw_token)
-        except ValueError:
-            token = parse_qs(raw_token).get("access_token", [None])[0]
-        else:
-            token = token_data.get("access_token", None)
-        return (token, None)
-
-    def request(self, method, url, **kwargs):
-        "Build remote url request. Constructs necessary auth."
-        user_token = kwargs.pop("token", self.token)
-        token, _ = self.parse_raw_token(user_token)
-        if token is not None:
-            params = kwargs.get("params", {})
-            params["access_token"] = token
-            kwargs["params"] = params
-        return super(OAuth2Client, self).session.request(method, url, **kwargs)
-
-    @property
-    def session_key(self):
-        return "oauth-client-{0}-request-state".format(self.source.name)
-
-
-def get_client(source, token=""):  # nosec
-    "Return the API client for the given source."
-    cls = OAuth2Client
-    if source.request_token_url:
-        cls = OAuthClient
-    return cls(source, token)

passbook/sources/oauth/clients/base.py

@@ -0,0 +1,75 @@
+"""OAuth Clients"""
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
+
+from django.http import HttpRequest
+from requests import Session
+from requests.exceptions import RequestException
+from requests.models import Response
+from structlog import get_logger
+
+from passbook import __version__
+from passbook.sources.oauth.models import OAuthSource
+
+LOGGER = get_logger()
+
+
+class BaseOAuthClient:
+    """Base OAuth Client"""
+
+    session: Session
+
+    source: OAuthSource
+    request: HttpRequest
+
+    callback: Optional[str]
+
+    def __init__(
+        self, source: OAuthSource, request: HttpRequest, callback: Optional[str] = None
+    ):
+        self.source = source
+        self.session = Session()
+        self.request = request
+        self.callback = callback
+        self.session.headers.update({"User-Agent": f"passbook {__version__}"})
+
+    def get_access_token(self, **request_kwargs) -> Optional[Dict[str, Any]]:
+        "Fetch access token from callback request."
+        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
+
+    def get_profile_info(self, token: Dict[str, str]) -> Optional[Dict[str, Any]]:
+        "Fetch user profile information."
+        try:
+            response = self.do_request("get", self.source.profile_url, token=token)
+            response.raise_for_status()
+        except RequestException as exc:
+            LOGGER.warning("Unable to fetch user profile", exc=exc)
+            return None
+        else:
+            return response.json()
+
+    def get_redirect_args(self) -> Dict[str, str]:
+        "Get request parameters for redirect url."
+        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
+
+    def get_redirect_url(self, parameters=None):
+        "Build authentication redirect url."
+        args = self.get_redirect_args()
+        additional = parameters or {}
+        args.update(additional)
+        params = urlencode(args)
+        LOGGER.info("redirect args", **args)
+        return f"{self.source.authorization_url}?{params}"
+
+    def parse_raw_token(self, raw_token: str) -> Dict[str, Any]:
+        "Parse token and secret from raw token response."
+        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover
+
+    def do_request(self, method: str, url: str, **kwargs) -> Response:
+        """Wrapper around self.session.request, which can add special headers"""
+        return self.session.request(method, url, **kwargs)
+
+    @property
+    def session_key(self) -> str:
+        """Return Session Key"""
+        raise NotImplementedError("Defined in a sub-class")  # pragma: no cover