release: 0.12.11-stable

* docs: initial migration to docusaurus

* website: add custom font, update blurbs and icons

* website: update splash

* root: update links to docs

* flows: use .pbflow extension so docusaurus doesn't mangle the files

* e2e: workaround prospector

* Squashed commit of the following:

commit 1248585dca
Author: Jens Langhammer <jens.langhammer@beryju.org>
Date:   Sun Nov 15 20:46:53 2020 +0100

    e2e: attempt to fix prospector error again

commit 1319c480c4
Author: Jens Langhammer <jens.langhammer@beryju.org>
Date:   Sun Nov 15 20:41:35 2020 +0100

    ci: install previous python version for upgrade testing

* web: update accent colours and format

* website: format markdown files

* website: fix colours for text

* website: switch to temporary accent colour to improve readability

* flows: fix path for TestTransferDocs

* flows: fix formatting of tests

.bumpversion.cfg

@@ -1,10 +1,10 @@
 [bumpversion]
-current_version = 0.9.0-rc1
+current_version = 0.12.11-stable
 tag = True
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\-(?P<release>.*)
 serialize = {major}.{minor}.{patch}-{release}
-message = new release: {new_version}
+message = release: {new_version}
 tag_name = version/{new_version}
 
 [bumpversion:part:release]
@@ -15,9 +15,11 @@ values =
 	beta
 	stable
 
-[bumpversion:file:README.md]
+[bumpversion:file:website/docs/installation/docker-compose.md]
 
-[bumpversion:file:docs/installation/docker-compose.md]
+[bumpversion:file:website/docs/installation/kubernetes.md]
+
+[bumpversion:file:docker-compose.yml]
 
 [bumpversion:file:helm/values.yaml]
 
@@ -26,3 +28,5 @@ values =
 [bumpversion:file:.github/workflows/release.yml]
 
 [bumpversion:file:passbook/__init__.py]
+
+[bumpversion:file:proxy/pkg/version.go]

.coveragerc

@@ -1,11 +1,12 @@
 [run]
 source = passbook
+relative_files = true
 omit =
-    */wsgi.py
+    */asgi.py
     manage.py
     */migrations/*
     */apps.py
-    docs/
+    website/
 
 [report]
 sort = Cover

.dockerignore

@@ -3,4 +3,4 @@ helm
 passbook-ui
 static
 *.env.yml
-node_modules/
+**/node_modules

.fossa.yml

@@ -1,20 +0,0 @@
-# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
-# Visit https://fossa.com to learn more
-
-version: 2
-cli:
-  server: https://app.fossa.com
-  fetcher: custom
-  project: git@github.com:BeryJu/passbook.git
-analyze:
-  modules:
-  - name: static
-    type: npm
-    target: passbook/static/static
-    path: passbook/static/static
-  - name: .
-    type: pip
-    target: .
-    path: .
-    options:
-      strategy: pipenv

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Logs**
+Output of docker-compose logs or kubectl logs respectively
+
+**Version and Deployment (please complete the following information):**
+ - passbook version: [e.g. 0.10.0-stable]
+ - Deployment: [e.g. docker-compose, helm]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,42 @@
+version: 2
+updates:
+- package-ecosystem: gomod
+  directory: "/proxy"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  assignees:
+  - BeryJu
+- package-ecosystem: npm
+  directory: "/passbook/static/static"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  assignees:
+  - BeryJu
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  assignees:
+  - BeryJu
+- package-ecosystem: docker
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  assignees:
+  - BeryJu
+- package-ecosystem: docker
+  directory: "/proxy"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  assignees:
+  - BeryJu

.github/workflows/release.yml

@@ -1,6 +1,8 @@
-name: passbook-release
+name: passbook-on-release
+
 on:
-  release
+  release:
+    types: [published, created]
 
 jobs:
   # Build
@@ -16,17 +18,26 @@ jobs:
       - name: Building Docker Image
         run: docker build
           --no-cache
-          -t beryju/passbook:0.9.0-rc1
+          -t beryju/passbook:0.12.11-stable
           -t beryju/passbook:latest
           -f Dockerfile .
       - name: Push Docker Container to Registry (versioned)
-        run: docker push beryju/passbook:0.9.0-rc1
+        run: docker push beryju/passbook:0.12.11-stable
       - name: Push Docker Container to Registry (latest)
         run: docker push beryju/passbook:latest
-  build-gatekeeper:
+  build-proxy:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "^1.15"
+      - name: prepare go api client
+        run: |
+          cd proxy
+          go get -u github.com/go-swagger/go-swagger/cmd/swagger
+          swagger generate client -f ../swagger.yaml -A passbook -t pkg/
+          go build -v .
       - name: Docker Login Registry
         env:
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
@@ -34,16 +45,16 @@ jobs:
         run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
       - name: Building Docker Image
         run: |
-          cd gatekeeper
+          cd proxy
           docker build \
           --no-cache \
-          -t beryju/passbook-gatekeeper:0.9.0-rc1 \
-          -t beryju/passbook-gatekeeper:latest \
+          -t beryju/passbook-proxy:0.12.11-stable \
+          -t beryju/passbook-proxy:latest \
           -f Dockerfile .
       - name: Push Docker Container to Registry (versioned)
-        run: docker push beryju/passbook-gatekeeper:0.9.0-rc1
+        run: docker push beryju/passbook-proxy:0.12.11-stable
       - name: Push Docker Container to Registry (latest)
-        run: docker push beryju/passbook-gatekeeper:latest
+        run: docker push beryju/passbook-proxy:latest
   build-static:
     runs-on: ubuntu-latest
     services:
@@ -66,11 +77,11 @@ jobs:
         run: docker build
           --no-cache
           --network=$(docker network ls | grep github | awk '{print $1}')
-          -t beryju/passbook-static:0.9.0-rc1
+          -t beryju/passbook-static:0.12.11-stable
           -t beryju/passbook-static:latest
           -f static.Dockerfile .
       - name: Push Docker Container to Registry (versioned)
-        run: docker push beryju/passbook-static:0.9.0-rc1
+        run: docker push beryju/passbook-static:0.12.11-stable
       - name: Push Docker Container to Registry (latest)
         run: docker push beryju/passbook-static:latest
   test-release:
@@ -82,10 +93,13 @@ jobs:
       - uses: actions/checkout@v1
       - name: Run test suite in final docker images
         run: |
+          sudo apt-get install -y pwgen
+          echo "PG_PASS=$(pwgen 40 1)" >> .env
+          echo "PASSBOOK_SECRET_KEY=$(pwgen 50 1)" >> .env
           docker-compose pull -q
           docker-compose up --no-start
           docker-compose start postgresql redis
-          docker-compose run -u root server bash -c "pip install --no-cache -r requirements-dev.txt && ./manage.py test"
+          docker-compose run -u root --entrypoint /bin/bash server -c "pip install --no-cache -r requirements-dev.txt && ./manage.py test passbook"
   sentry-release:
     needs:
       - test-release
@@ -100,5 +114,5 @@ jobs:
           SENTRY_PROJECT: passbook
           SENTRY_URL: https://sentry.beryju.org
         with:
-          tagName: 0.9.0-rc1
-          environment: production
+          tagName: 0.12.11-stable
+          environment: beryjuorg-prod

.github/workflows/tag.yml

@@ -1,10 +1,10 @@
+name: passbook-on-tag
+
 on:
   push:
     tags:
     - 'version/*'
 
-name: passbook-version-tag
-
 jobs:
   build:
     name: Create Release from Tag
@@ -13,6 +13,10 @@ jobs:
       - uses: actions/checkout@master
       - name: Pre-release test
         run: |
+          sudo apt-get install -y pwgen
+          echo "PASSBOOK_TAG=latest" >> .env
+          echo "PG_PASS=$(pwgen 40 1)" >> .env
+          echo "PASSBOOK_SECRET_KEY=$(pwgen 50 1)" >> .env
           docker-compose pull -q
           docker build \
             --no-cache \
@@ -20,7 +24,7 @@ jobs:
             -f Dockerfile .
           docker-compose up --no-start
           docker-compose start postgresql redis
-          docker-compose run -u root server bash -c "pip install --no-cache -r requirements-dev.txt && ./manage.py test"
+          docker-compose run -u root --entrypoint /bin/bash server -c "pip install --no-cache -r requirements-dev.txt && ./manage.py test passbook"
       - name: Install Helm
         run: |
           apt update && apt install -y curl
@@ -30,7 +34,7 @@ jobs:
           helm dependency update helm/
           helm package helm/
           mv passbook-*.tgz passbook-chart.tgz
-      - name: Extract verison number
+      - name: Extract version number
         id: get_version
         uses: actions/github-script@0.2.0
         with:
@@ -45,7 +49,7 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: Release ${{ steps.get_version.outputs.result }}
-          draft: false
+          draft: true
           prerelease: false
       - name: Upload packaged Helm Chart
         id: upload-release-asset

.gitignore

@@ -199,3 +199,4 @@ local.env.yml
 
 # Selenium Screenshots
 selenium_screenshots/**
+backups/

.prospector.yaml

@@ -9,3 +9,4 @@ ignore-paths:
 
 uses:
   - django
+  - celery

.pylintrc

@@ -1,9 +1,29 @@
 [MASTER]
 
-disable=redefined-outer-name,arguments-differ,no-self-use,cyclic-import,fixme,locally-disabled,too-many-ancestors,too-few-public-methods,import-outside-toplevel,bad-continuation,signature-differs
+disable =
+    arguments-differ,
+    no-self-use,
+    fixme,
+    locally-disabled,
+    too-many-ancestors,
+    too-few-public-methods,
+    import-outside-toplevel,
+    bad-continuation,
+    signature-differs,
+    similarities,
+    cyclic-import,
+    protected-access,
+    unsubscriptable-object # remove when pylint is upgraded to 2.6
+
 load-plugins=pylint_django,pylint.extensions.bad_builtin
-extension-pkg-whitelist=lxml
+
+extension-pkg-whitelist=lxml,xmlsec
+
+# Allow constants to be shorter than normal (and lowercase, for settings.py)
 const-rgx=[a-zA-Z0-9_]{1,40}$
+
 ignored-modules=django-otp
-jobs=12
+generated-members=xmlsec.constants.*,xmlsec.tree.*,xmlsec.template.*
 ignore=migrations
+max-attributes=12
+max-branches=20

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-buster as locker
+FROM python:3.9-slim-buster as locker
 
 COPY ./Pipfile /app/
 COPY ./Pipfile.lock /app/
@@ -9,27 +9,38 @@ RUN pip install pipenv && \
     pipenv lock -r > requirements.txt && \
     pipenv lock -rd > requirements-dev.txt
 
-FROM python:3.8-slim-buster
+FROM python:3.9-slim-buster
 
-COPY --from=locker /app/requirements.txt /app/
-COPY --from=locker /app/requirements-dev.txt /app/
-
-WORKDIR /app/
+WORKDIR /
+COPY --from=locker /app/requirements.txt /
+COPY --from=locker /app/requirements-dev.txt /
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends postgresql-client-11 && \
-    rm -rf /var/lib/apt/ && \
-    pip install -r requirements.txt  --no-cache-dir && \
-    adduser --system --no-create-home --uid 1000 --group --home /app passbook
+    apt-get install -y --no-install-recommends curl ca-certificates gnupg && \
+    curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
+    echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends postgresql-client-12 postgresql-client-11 build-essential libxmlsec1-dev pkg-config && \
+    apt-get clean && \
+    pip install -r /requirements.txt --no-cache-dir && \
+    apt-get remove --purge -y build-essential && \
+    apt-get autoremove --purge -y && \
+    # This is quite hacky, but docker has no guaranteed Group ID
+    # we could instead check for the GID of the socket and add the user dynamically,
+    # but then we have to drop permmissions later
+    groupadd -g 998 docker_998 && \
+    groupadd -g 999 docker_999 && \
+    adduser --system --no-create-home --uid 1000 --group --home /passbook passbook && \
+    usermod -a -G docker_998 passbook && \
+    usermod -a -G docker_999 passbook && \
+    mkdir /backups && \
+    chown passbook:passbook /backups
 
-COPY ./passbook/ /app/passbook
-COPY ./manage.py /app/
-COPY ./docker/uwsgi.ini /app/
-COPY ./docker/bootstrap.sh /bootstrap.sh
-COPY ./docker/wait_for_db.py /app/wait_for_db.py
-
-WORKDIR /app/
+COPY ./passbook/ /passbook
+COPY ./manage.py /
+COPY ./lifecycle/ /lifecycle
 
 USER passbook
+STOPSIGNAL SIGINT
 
-ENTRYPOINT [ "/bootstrap.sh" ]
+ENTRYPOINT [ "/lifecycle/bootstrap.sh" ]

Makefile

@@ -0,0 +1,26 @@
+all: lint-fix lint coverage gen
+
+coverage:
+	coverage run --concurrency=multiprocessing manage.py test --failfast -v 3
+	coverage combine
+	coverage html
+	coverage report
+
+lint-fix:
+	isort -rc .
+	black passbook e2e lifecycle
+
+lint:
+	pyright passbook e2e lifecycle
+	bandit -r passbook e2e lifecycle -x node_modules
+	pylint passbook e2e lifecycle
+	prospector
+
+gen: coverage
+	./manage.py generate_swagger -o swagger.yaml -f yaml
+
+local-stack:
+	export PASSBOOK_TAG=testing
+	docker build -t beryju/passbook:testng .
+	docker-compose up -d
+	docker-compose run --rm server migrate

Pipfile

@@ -13,52 +13,52 @@ django-dbbackup = "*"
 django-filter = "*"
 django-guardian = "*"
 django-model-utils = "*"
-django-oauth-toolkit = "*"
-django-oidc-provider = "*"
 django-otp = "*"
 django-prometheus = "*"
 django-recaptcha = "*"
 django-redis = "*"
-django-rest-framework = "*"
+djangorestframework = "*"
 django-storages = "*"
 djangorestframework-guardian = "*"
-drf-yasg = "*"
-kombu = "*"
+drf_yasg2 = "*"
+facebook-sdk = "*"
 ldap3 = "*"
 lxml = "*"
-oauthlib = "*"
 packaging = "*"
 psycopg2-binary = "*"
 pycryptodome = "*"
-pyuwsgi = "*"
+pyjwkest = "*"
+uvicorn = {extras = ["standard"],version = "*"}
+gunicorn = "*"
 pyyaml = "*"
 qrcode = "*"
 requests-oauthlib = "*"
 sentry-sdk = "*"
 service_identity = "*"
-signxml = "*"
 structlog = "*"
 swagger-spec-validator = "*"
 urllib3 = {extras = ["secure"],version = "*"}
-facebook-sdk = "*"
-elastic-apm = "*"
+dacite = "*"
+channels = "*"
+channels-redis = "*"
+kubernetes = "*"
+docker = "*"
+xmlsec = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.9"
 
 [dev-packages]
 autopep8 = "*"
 bandit = "*"
+black = "==20.8b1"
 bumpversion = "*"
 colorama = "*"
 coverage = "*"
 django-debug-toolbar = "*"
 pylint = "*"
 pylint-django = "*"
-unittest-xml-reporting = "*"
-black = "*"
 selenium = "*"
-docker = "*"
-
-[pipenv]
-allow_prereleases = true
+prospector = "*"
+pytest = "*"
+pytest-django = "*"

README.md

@@ -1,10 +1,9 @@
-<img src="passbook/static/static/passbook/logo.svg" height="50" alt="passbook logo"><img src="passbook/static/static/passbook/brand_inverted.svg" height="50" alt="passbook">
+<img src="website/static/img/logo.svg" height="50" alt="passbook logo"><img src="website/static/img/brand_inverted.svg" height="50" alt="passbook">
 
 [![CI Build status](https://img.shields.io/azure-devops/build/beryjuorg/passbook/1?style=flat-square)](https://dev.azure.com/beryjuorg/passbook/_build?definitionId=1)
 ![Tests](https://img.shields.io/azure-devops/tests/beryjuorg/passbook/1?compact_message&style=flat-square)
 [![Code Coverage](https://img.shields.io/codecov/c/gh/beryju/passbook?style=flat-square)](https://codecov.io/gh/BeryJu/passbook)
 ![Docker pulls](https://img.shields.io/docker/pulls/beryju/passbook.svg?style=flat-square)
-![Docker pulls (gatekeeper)](https://img.shields.io/docker/pulls/beryju/passbook-gatekeeper.svg?style=flat-square)
 ![Latest version](https://img.shields.io/docker/v/beryju/passbook?sort=semver&style=flat-square)
 ![LGTM Grade](https://img.shields.io/lgtm/grade/python/github/BeryJu/passbook?style=flat-square)
 
@@ -14,31 +13,18 @@ passbook is an open-source Identity Provider focused on flexibility and versatil
 
 ## Installation
 
-For small/test setups it is recommended to use docker-compose.
+For small/test setups it is recommended to use docker-compose, see the [documentation](https://passbook.beryju.org/website/docs/installation/docker-compose/)
 
-```
-wget https://raw.githubusercontent.com/BeryJu/passbook/master/docker-compose.yml
-# Optionally enable Error-reporting
-# export PASSBOOK_ERROR_REPORTING=true
-# Optionally deploy a different version
-# export PASSBOOK_TAG=0.9.0-rc1
-# If this is a productive installation, set a different PostgreSQL Password
-# export PG_PASS=$(pwgen 40 1)
-docker-compose pull
-docker-compose up -d
-docker-compose exec server ./manage.py migrate
-```
-
-For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org//installation/kubernetes/)
+For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org/website/docs/installation/kubernetes/)
 
 ## Screenshots
 
-![](docs/images/screen_apps.png)
-![](docs/images/screen_admin.png)
+![](website/static/img/screen_apps.png)
+![](website/static/img/screen_admin.png)
 
 ## Development
 
-To develop on passbook, you need a system with Python 3.7+ (3.8 is recommended). passbook uses [pipenv](https://pipenv.pypa.io/en/latest/) for managing dependencies.
+To develop on passbook, you need a system with Python 3.8+ (3.9 is recommended). passbook uses [pipenv](https://pipenv.pypa.io/en/latest/) for managing dependencies.
 
 To get started, run
 
@@ -61,7 +47,6 @@ postgresql:
   user: postgres
 
 log_level: debug
-error_reporting: false
 ```
 
 ## Security

SECURITY.md

@@ -6,8 +6,10 @@ As passbook is currently in a pre-stable, only the latest "stable" version is su
 
 | Version  | Supported          |
 | -------- | ------------------ |
-| 0.8.15   | :white_check_mark: |
+| 0.10.x   | :white_check_mark: |
+| 0.11.x   | :white_check_mark: |
+| 0.12.x   | :white_check_mark: |
 
 ## Reporting a Vulnerability
 
-To report a vulnerability, send am email to [security@beryju.org](mailto:security@beryju.org)
+To report a vulnerability, send an email to [security@beryju.org](mailto:security@beryju.org)

azure-pipelines.yml

@@ -8,6 +8,10 @@ variables:
   POSTGRES_DB: passbook
   POSTGRES_USER: passbook
   POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77"
+  ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}:
+    branchName: ${{ replace(variables['Build.SourceBranchName'], 'refs/heads/', '') }}
+  ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
+    branchName: ${{ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') }}
 
 stages:
   - stage: Lint
@@ -18,61 +22,65 @@ stages:
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pylint passbook
+              script: pipenv run pylint passbook e2e lifecycle
       - job: black
         pool:
           vmImage: 'ubuntu-latest'
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run black --check passbook
+              script: pipenv run black --check passbook e2e lifecycle
       - job: prospector
         pool:
           vmImage: 'ubuntu-latest'
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
                 pipenv install --dev prospector --skip-lock
           - task: CmdLine@2
             inputs:
-              script: pipenv run prospector passbook
+              script: pipenv run prospector
       - job: bandit
         pool:
           vmImage: 'ubuntu-latest'
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run bandit -r passbook
+              script: pipenv run bandit -r passbook e2e lifecycle
       - job: pyright
         pool:
           vmImage: ubuntu-latest
@@ -82,18 +90,19 @@ stages:
               version: '12.x'
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: CmdLine@2
             inputs:
-              script: npm install -g pyright
+              script: npm install -g pyright@1.1.79
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pyright
+              script: pipenv run pyright e2e lifecycle
   - stage: Test
     jobs:
       - job: migrations
@@ -102,7 +111,7 @@ stages:
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: DockerCompose@0
             displayName: Run services
             inputs:
@@ -112,18 +121,58 @@ stages:
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
               script: pipenv run ./manage.py migrate
+      - job: migrations_from_previous_release
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.8'
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.9'
+          - task: DockerCompose@0
+            displayName: Run services
+            inputs:
+              dockerComposeFile: 'scripts/ci.docker-compose.yml'
+              action: 'Run services'
+              buildImages: false
+          - task: CmdLine@2
+            displayName: Prepare Last tagged release
+            inputs:
+              script: |
+                git checkout $(git describe --abbrev=0 --match 'version/*')
+                sudo apt install -y libxmlsec1-dev pkg-config
+                sudo pip install -U wheel pipenv
+                pipenv install --dev
+          - task: CmdLine@2
+            displayName: Migrate to last tagged release
+            inputs:
+              script: pipenv run ./manage.py migrate
+          - task: CmdLine@2
+            displayName: Install current branch
+            inputs:
+              script: |
+                set -x
+                git checkout ${{ variables.branchName }}
+                pipenv sync --dev
+          - task: CmdLine@2
+            displayName: Migrate to current branch
+            inputs:
+              script: pipenv run ./manage.py migrate
       - job: coverage_unittest
         pool:
           vmImage: 'ubuntu-latest'
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: DockerCompose@0
             displayName: Run services
             inputs:
@@ -131,15 +180,27 @@ stages:
               action: 'Run services'
               buildImages: false
           - task: CmdLine@2
+            displayName: Install K3d and prepare
             inputs:
               script: |
+                wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+                k3d cluster create
+                k3d kubeconfig write -o ~/.kube/config --overwrite
+          - task: CmdLine@2
+            inputs:
+              script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: CmdLine@2
             displayName: Run full test suite
             inputs:
               script: |
-                pipenv run coverage run ./manage.py test passbook
+                export PB_TEST_K8S=true
+                pipenv run coverage run ./manage.py test passbook -v 3
+          - task: CmdLine@2
+            inputs:
+              script: |
                 mkdir output-unittest
                 mv unittest.xml output-unittest/unittest.xml
                 mv .coverage output-unittest/coverage
@@ -150,11 +211,11 @@ stages:
               publishLocation: 'pipeline'
       - job: coverage_e2e
         pool:
-          vmImage: 'ubuntu-latest'
+          name: coventry
         steps:
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: DockerCompose@0
             displayName: Run services
             inputs:
@@ -162,8 +223,16 @@ stages:
               action: 'Run services'
               buildImages: false
           - task: CmdLine@2
+            displayName: Install K3d and prepare
             inputs:
               script: |
+                wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+                k3d cluster create
+                k3d kubeconfig write -o ~/.kube/config --overwrite
+          - task: CmdLine@2
+            inputs:
+              script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
           - task: DockerCompose@0
@@ -177,11 +246,21 @@ stages:
             inputs:
               script: |
                 cd passbook/static/static
-                yarn
+                npm i
+                npm run build
           - task: CmdLine@2
             displayName: Run full test suite
             inputs:
-              script: pipenv run coverage run ./manage.py test e2e
+              script: |
+                export PB_TEST_K8S=true
+                pipenv run coverage run ./manage.py test e2e -v 3 --failfast
+          - task: CmdLine@2
+            condition: always()
+            displayName: Cleanup
+            inputs:
+              script: |
+                docker stop $(docker ps -aq)
+                docker container prune -f
           - task: CmdLine@2
             displayName: Prepare unittests and coverage for upload
             inputs:
@@ -219,17 +298,16 @@ stages:
               path: "coverage-unittest/"
           - task: UsePythonVersion@0
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - task: CmdLine@2
             inputs:
               script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
-                find .
                 pipenv run coverage combine coverage-e2e/coverage coverage-unittest/coverage
                 pipenv run coverage xml
                 pipenv run coverage html
-                find .
           - task: PublishCodeCoverageResults@1
             inputs:
               codeCoverageTool: 'Cobertura'
@@ -260,19 +338,7 @@ stages:
             repository: 'beryju/passbook'
             command: 'buildAndPush'
             Dockerfile: 'Dockerfile'
-            tags: 'gh-$(Build.SourceBranchName)'
-      - job: build_gatekeeper
-        pool:
-          vmImage: 'ubuntu-latest'
-        steps:
-        - task: Docker@2
-          inputs:
-            containerRegistry: 'dockerhub'
-            repository: 'beryju/passbook-gatekeeper'
-            command: 'buildAndPush'
-            Dockerfile: 'gatekeeper/Dockerfile'
-            buildContext: 'gatekeeper/'
-            tags: 'gh-$(Build.SourceBranchName)'
+            tags: "gh-${{ variables.branchName }}"
       - job: build_static
         pool:
           vmImage: 'ubuntu-latest'
@@ -289,11 +355,11 @@ stages:
             repository: 'beryju/passbook-static'
             command: 'build'
             Dockerfile: 'static.Dockerfile'
-            tags: 'gh-$(Build.SourceBranchName)'
+            tags: "gh-${{ variables.branchName }}"
             arguments: "--network=beryjupassbook_default"
         - task: Docker@2
           inputs:
             containerRegistry: 'dockerhub'
             repository: 'beryju/passbook-static'
             command: 'push'
-            tags: 'gh-$(Build.SourceBranchName)'
+            tags: "gh-${{ variables.branchName }}"

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.2'
 
 services:
   postgresql:
-    image: postgres
+    image: postgres:12
     volumes:
       - database:/var/lib/postgresql/data
     networks:
@@ -12,62 +12,68 @@ services:
       - POSTGRES_PASSWORD=${PG_PASS:-thisisnotagoodpassword}
       - POSTGRES_USER=passbook
       - POSTGRES_DB=passbook
-    labels:
-      - traefik.enable=false
+    env_file:
+      - .env
   redis:
     image: redis
     networks:
       - internal
-    labels:
-      - traefik.enable=false
   server:
-    image: beryju/passbook:${PASSBOOK_TAG:-latest}
-    command:
-      - uwsgi
-      - uwsgi.ini
+    image: beryju/passbook:${PASSBOOK_TAG:-0.12.11-stable}
+    command: server
     environment:
-      - PASSBOOK_REDIS__HOST=redis
-      - PASSBOOK_ERROR_REPORTING=${PASSBOOK_ERROR_REPORTING:-false}
-      - PASSBOOK_POSTGRESQL__HOST=postgresql
-      - PASSBOOK_POSTGRESQL__PASSWORD=${PG_PASS:-thisisnotagoodpassword}
+      PASSBOOK_REDIS__HOST: redis
+      PASSBOOK_POSTGRESQL__HOST: postgresql
+      PASSBOOK_POSTGRESQL__PASSWORD: ${PG_PASS}
     ports:
       - 8000
     networks:
       - internal
     labels:
-      - traefik.port=8000
-      - traefik.docker.network=internal
-      - traefik.frontend.rule=PathPrefix:/
+      traefik.enable: 'true'
+      traefik.docker.network: internal
+      traefik.http.routers.app-router.rule: PathPrefix(`/`)
+      traefik.http.routers.app-router.service: app-service
+      traefik.http.routers.app-router.tls: 'true'
+      traefik.http.services.app-service.loadbalancer.healthcheck.hostname: passbook-healthcheck-host
+      traefik.http.services.app-service.loadbalancer.server.port: '8000'
+    env_file:
+      - .env
   worker:
-    image: beryju/passbook:${PASSBOOK_TAG:-latest}
-    command:
-      - celery
-      - worker
-      - --autoscale=10,3
-      - -E
-      - -B
-      - -A=passbook.root.celery
-      - -s=/tmp/celerybeat-schedule
+    image: beryju/passbook:${PASSBOOK_TAG:-0.12.11-stable}
+    command: worker
     networks:
       - internal
-    labels:
-      - traefik.enable=false
     environment:
-      - PASSBOOK_REDIS__HOST=redis
-      - PASSBOOK_ERROR_REPORTING=${PASSBOOK_ERROR_REPORTING:-false}
-      - PASSBOOK_POSTGRESQL__HOST=postgresql
-      - PASSBOOK_POSTGRESQL__PASSWORD=${PG_PASS:-thisisnotagoodpassword}
+      PASSBOOK_REDIS__HOST: redis
+      PASSBOOK_POSTGRESQL__HOST: postgresql
+      PASSBOOK_POSTGRESQL__PASSWORD: ${PG_PASS}
+    volumes:
+      - ./backups:/backups
+      - /var/run/docker.sock:/var/run/docker.sock
+    env_file:
+      - .env
   static:
-    image: beryju/passbook-static:latest
+    image: beryju/passbook-static:${PASSBOOK_TAG:-0.12.11-stable}
     networks:
       - internal
     labels:
-      - traefik.frontend.rule=PathPrefix:/static, /robots.txt, /favicon.ico
-      - traefik.port=80
-      - traefik.docker.network=internal
+      traefik.enable: 'true'
+      traefik.docker.network: internal
+      traefik.http.routers.static-router.rule: PathPrefix(`/static`, `/robots.txt`, `/favicon.ico`)
+      traefik.http.routers.static-router.tls: 'true'
+      traefik.http.routers.static-router.service: static-service
+      traefik.http.services.static-service.loadbalancer.healthcheck.path: /
+      traefik.http.services.static-service.loadbalancer.server.port: '80'
   traefik:
-    image: traefik:1.7
-    command: --api --docker --defaultentrypoints=https --entryPoints='Name:http Address::80 Redirect.EntryPoint:https' --entryPoints='Name:https Address::443 TLS'
+    image: traefik:2.3
+    command:
+      - "--log.format=json"
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.http.address=:80"
+      - "--entrypoints.https.address=:443"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
     ports:

docker/bootstrap.sh

@@ -1,3 +0,0 @@
-#!/bin/bash -ex
-/app/wait_for_db.py
-"$@"

docker/uwsgi.ini

@@ -1,10 +0,0 @@
-[uwsgi]
-http = 0.0.0.0:8000
-wsgi-file = passbook/root/wsgi.py
-processes = 2
-master = true
-threads = 2
-enable-threads = true
-uid = passbook
-gid = passbook
-disable-logging = True

docs/build.sh

@@ -1,3 +0,0 @@
-#!/bin/bash -x
-pip install -U mkdocs mkdocs-material
-mkdocs gh-deploy

docs/expressions/reference/user-object.md

@@ -1,21 +0,0 @@
-# Passbook User Object
-
-The User object has the following attributes:
-
- - `username`: User's username.
- - `email` User's email.
- - `name` User's display mame.
- - `is_staff` Boolean field if user is staff.
- - `is_active` Boolean field if user is active.
- - `date_joined` Date user joined/was created.
- - `password_change_date` Date password was last changed.
- - `attributes` Dynamic attributes.
-
-## Examples
-
-List all the User's group names:
-
-```python
-for group in user.groups.all():
-    yield group.name
-```

docs/flow/examples/login.md

@@ -1,36 +0,0 @@
-# Login Flow
-
-This document describes how a simple authentication flow can be created.
-
-This flow is created automatically when passbook is installed.
-
-1. Create an **Identification** stage
-
-    > Here you can select whichever fields the user can identify themselves with
-    > Select the Template **Default Login**, as this template shows the (optional) Flows
-    > Here you can also link optional enrollment and recovery flows.
-
-2. Create a **Password** stage
-
-    > Select the Backend you want the password to be checked against. Select "passbook-internal Userdatabase".
-
-3. Create a **User Login** stage
-
-    > This stage doesn't have any options.
-
-4. Create a flow
-
-    > Create a flow with the delegation of **Authentication**
-    > Assign a name and a slug. The slug is used in the URL when the flow is executed.
-
-5. Bind the stages to the flow
-
-    > Bind the **Identification** Stage with an order of 0
-    > Bind the **Password** Stage with an order of 1
-    > Bind the **User Login** Stage with an order of 2
-
-    ![](login.png)
-
-!!! notice
-
-    This flow can used by any user, authenticated and un-authenticated. This means any authenticated user that visits this flow can login again.

docs/flow/stages/otp/index.md

@@ -1,7 +0,0 @@
-# OTP Stage
-
-This stage offers a generic Time-based One-time Password authentication step.
-
-You can optionally enforce this step, which will force every user without OTP setup to configure it.
-
-This stage uses a 6-digit Code with a 30 second time-drift. This is currently not changeable.

docs/flow/stages/user_delete.md

@@ -1,8 +0,0 @@
-# User Delete Stage
-
-!!! danger
-    This stage deletes the `pending_user` without any confirmation. You have to make sure the user is aware of this.
-
-This stage is intended for an unenrollment flow. It deletes the currently pending user.
-
-The pending user is also removed from the current session.

docs/installation/kubernetes.md

@@ -1,69 +0,0 @@
-# Kubernetes
-
-For a mid to high-load installation, Kubernetes is recommended. passbook is installed using a helm-chart.
-
-This installation automatically applies database migrations on startup. After the installation is done, you can use `pbadmin` as username and password.
-
-```
-# Default values for passbook.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-# passbook version to use. Defaults to latest stable version
-# image:
-#   tag:
-
-nameOverride: ""
-
-config:
-  # Optionally specify fixed secret_key, otherwise generated automatically
-  # secret_key: _k*@6h2u2@q-dku57hhgzb7tnx*ba9wodcb^s9g0j59@=y(@_o
-  # Enable error reporting
-  error_reporting: false
-  # Log level used by web and worker
-  # Can be either debug, info, warning, error
-  log_level: warning
-  # Optionally enable Elastic APM Support
-  apm:
-    enabled: false
-    server_url: ""
-    secret_token: ""
-    verify_server_cert: true
-
-# This Helm chart ships with built-in Prometheus ServiceMonitors and Rules.
-# This requires the CoreOS Prometheus Operator.
-monitoring:
-  enabled: false
-
-# Enable Database Backups to S3
-# backup:
-#   access_key: access-key
-#   secret_key: secret-key
-#   bucket: s3-bucket
-#   host: s3-host
-
-ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  path: /
-  hosts:
-    - passbook.k8s.local
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - passbook.k8s.local
-
-# These settings configure the packaged PostgreSQL and Redis chart.
-postgresql:
-  postgresqlDatabase: passbook
-
-redis:
-  cluster:
-    enabled: false
-  master:
-    persistence:
-      enabled: false
-    # https://stackoverflow.com/a/59189742
-    disableCommands: []
-```

docs/integrations/services/aws/index.md

@@ -1,32 +0,0 @@
-# Amazon Web Services Integration
-
-## What is AWS
-
-!!! note ""
-    Amazon Web Services (AWS) is the world’s most comprehensive and broadly adopted cloud platform, offering over 175 fully featured services from data centers globally. Millions of customers—including the fastest-growing startups, largest enterprises, and leading government agencies—are using AWS to lower costs, become more agile, and innovate faster.
-
-## Preparation
-
-The following placeholders will be used:
-
--   `passbook.company` is the FQDN of the passbook install.
-
-Create an application in passbook and note the slug, as this will be used later. Create a SAML provider with the following parameters:
-
--   ACS URL: `https://signin.aws.amazon.com/saml`
--   Audience: `urn:amazon:webservices`
--   Issuer: `passbook`
-
-You can of course use a custom signing certificate, and adjust durations.
-
-## AWS
-
-Create a role with the permissions you desire, and note the ARN.
-
-AWS requires two custom PropertyMappings; `Role` and `RoleSessionName`. Create them as following:
-
-![](./property-mapping-role.png)
-
-![](./property-mapping-role-session-name.png)
-
-Afterwards export the metadata from passbook, and create an Identity Provider [here](https://console.aws.amazon.com/iam/home#/providers).

docs/integrations/services/sentry/index.md

@@ -1,41 +0,0 @@
-# Sentry Integration
-
-## What is Sentry
-
-From https://sentry.io
-
-!!! note ""
-    Sentry provides self-hosted and cloud-based error monitoring that helps all software
-    teams discover, triage, and prioritize errors in real-time.
-
-    One million developers at over fifty thousand companies already ship
-    better software faster with Sentry. Won’t you join them?
-
-## Preparation
-
-The following placeholders will be used:
-
--   `sentry.company` is the FQDN of the Sentry install.
--   `passbook.company` is the FQDN of the passbook install.
-
-Create an application in passbook. Create an OpenID provider with the following parameters:
-
--   Client Type: `Confidential`
--   Response types: `code (Authorization Code Flow)`
--   JWT Algorithm: `RS256`
--   Redirect URIs: `https://sentry.company/auth/sso/`
--   Scopes: `openid email`
-
-## Sentry
-
-**This guide assumes you've installed Sentry using [getsentry/onpremise](https://github.com/getsentry/onpremise)**
-
-- Add `sentry-auth-oidc` to `onpremise/sentry/requirements.txt` (Create the file if it doesn't exist yet)
-- Add the following block to your `onpremise/sentry/sentry.conf.py`:
-```
-OIDC_ISSUER = "passbook"
-OIDC_CLIENT_ID = "<Client ID from passbook>"
-OIDC_CLIENT_SECRET = "<Client Secret from passbook>"
-OIDC_SCOPE = "openid email"
-OIDC_DOMAIN = "https://passbook.company/application/oidc/"
-```

docs/integrations/services/tower-awx/index.md

@@ -1,74 +0,0 @@
-# Ansible Tower / AWX Integration
-
-## What is Tower
-
-From https://docs.ansible.com/ansible/2.5/reference_appendices/tower.html
-
-!!! note ""
-    Ansible Tower (formerly ‘AWX’) is a web-based solution that makes Ansible even more easy to use for IT teams of all kinds. It’s designed to be the hub for all of your automation tasks.
-
-    Tower allows you to control access to who can access what, even allowing sharing of SSH credentials without someone being able to transfer those credentials. Inventory can be graphically managed or synced with a wide variety of cloud sources. It logs all of your jobs, integrates well with LDAP, and has an amazing browsable REST API. Command line tools are available for easy integration with Jenkins as well. Provisioning callbacks provide great support for autoscaling topologies.
-
-!!! note
-    AWX is the open-source version of Tower. The term "AWX" will be used interchangeably throughout this document.
-
-## Preparation
-
-The following placeholders will be used:
-
--   `awx.company` is the FQDN of the AWX/Tower install.
--   `passbook.company` is the FQDN of the passbook install.
-
-Create an application in passbook and note the slug, as this will be used later. Create a SAML provider with the following parameters:
-
--   ACS URL: `https://awx.company/sso/complete/saml/`
--   Audience: `awx`
--   Issuer: `https://awx.company/sso/metadata/saml/`
-
-You can of course use a custom signing certificate, and adjust durations.
-
-## AWX Configuration
-
-Navigate to `https://awx.company/#/settings/auth` to configure SAML. Set the Field `SAML SERVICE PROVIDER ENTITY ID` to `awx`.
-
-For the fields `SAML SERVICE PROVIDER PUBLIC CERTIFICATE` and `SAML SERVICE PROVIDER PRIVATE KEY`, you can either use custom certificates, or use the self-signed pair generated by passbook.
-
-Provide metadata in the `SAML Service Provider Organization Info` field:
-
-```json
-{
- "en-US": {
-  "name": "passbook",
-  "url": "https://passbook.company",
-  "displayname": "passbook"
- }
-}
-```
-
-Provide metadata in the  `SAML Service Provider Technical Contact` and `SAML Service Provider Technical Contact` fields:
-
-```json
-{
- "givenName": "Admin Name",
- "emailAddress": "admin@company"
-}
-```
-
-In the `SAML Enabled Identity Providers` paste the following configuration:
-
-```json
-{
- "passbook": {
-  "attr_username": "urn:oid:2.16.840.1.113730.3.1.241",
-  "attr_user_permanent_id": "urn:oid:0.9.2342.19200300.100.1.1",
-  "x509cert": "MIIDEjCCAfqgAwIBAgIRAJZ9pOZ1g0xjiHtQAAejsMEwDQYJKoZIhvcNAQELBQAwMDEuMCwGA1UEAwwlcGFzc2Jvb2sgU2VsZi1zaWduZWQgU0FNTCBDZXJ0aWZpY2F0ZTAeFw0xOTEyMjYyMDEwNDFaFw0yMDEyMjYyMDEwNDFaMFkxLjAsBgNVBAMMJXBhc3Nib29rIFNlbGYtc2lnbmVkIFNBTUwgQ2VydGlmaWNhdGUxETAPBgNVBAoMCHBhc3Nib29rMRQwEgYDVQQLDAtTZWxmLXNpZ25lZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAO/ktBYZkY9xAijF4acvzX6Q1K8KoIZeyde8fVgcWBz4L5FgDQ4/dni4k2YAcPdwteGL4nKVzetUzjbRCBUNuO6lqU4J4WNNX4Xg4Ir7XLRoAQeo+omTPBdpJ1p02HjtN5jT01umN3bK2yto1e37CJhK6WJiaXqRewPxh4lI4aqdj3BhFkJ3I3r2qxaWOAXQ6X7fg3w/ny7QP53//ouZo7hSLY3GIcRKgvdjjVM3OW5C3WLpOq5Dez5GWVJ17aeFCfGQ8bwFKde6qfYqyGcU9xHB36TtVHB9hSFP/tUFhkiSOxtsrYwCgCyXm4UTSpP+wiNyjKfFw7qGLBvA2hGTNw8CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAh9PeAqPRQk1/SSygIFADZBi08O/DPCshFwEHvJATIcTzcDD8UGAjXh+H5OlkDyX7KyrcaNvYaafCUo63A+WprdtdY5Ty6SBEwTYyiQyQfwM9BfK+imCoif1Ai7xAelD7p9lNazWq7JU+H/Ep7U7Q7LvpxAbK0JArt+IWTb2NcMb3OWE1r0gFbs44O1l6W9UbJTbyLMzbGbe5i+NHlgnwPwuhtRMh0NUYabGHKcHbhwyFhfGAQv2dAp5KF1E5gu6ZzCiFePzc0FrqXQyb2zpFYcJHXquiqaOeG7cZxRHYcjrl10Vxzki64XVA9BpdELgKSnupDGUEJsRUt3WVOmvZuA==",
-  "url": "https://passbook.company/application/saml/awx/login/",
-  "attr_last_name": "User.LastName",
-  "entity_id": "https://awx.company/sso/metadata/saml/",
-  "attr_email": "urn:oid:0.9.2342.19200300.100.1.3",
-  "attr_first_name": "urn:oid:2.5.4.3"
- }
-}
-```
-
-`x509cert` is the certificate configured in passbook. Remove the `--BEGIN CERTIFICATE--` and `--END CERTIFICATE--` headers, then enter the cert as one non-breaking string.

docs/policies/expression.md

@@ -1,30 +0,0 @@
-# Expression Policies
-
-!!! notice
-    These variables are available in addition to the common variables/functions defined in [**Expressions**](../expressions/index.md)
-
-The passing of the policy is determined by the return value of the code. Use `return True` to pass a policy and `return False` to fail it.
-
-### Available Functions
-
-#### `pb_message(message: str)`
-
-Add a message, visible by the end user. This can be used to show the reason why they were denied.
-
-Example:
-
-```python
-pb_message("Access denied")
-return False
-```
-
-### Context variables
-
-- `request`: A PolicyRequest object, which has the following properties:
-    - `request.user`: The current user, against which the policy is applied. ([ref](../expressions/reference/user-object.md))
-    - `request.http_request`: The Django HTTP Request. ([ref](https://docs.djangoproject.com/en/3.0/ref/request-response/#httprequest-objects))
-    - `request.obj`: A Django Model instance. This is only set if the policy is ran against an object.
-    - `request.context`: A dictionary with dynamic data. This depends on the origin of the execution.
-- `pb_is_sso_flow`: Boolean which is true if request was initiated by authenticating through an external provider.
-- `pb_client_ip`: Client's IP Address or '255.255.255.255' if no IP Address could be extracted. Can be [compared](../expressions/index.md#comparing-ip-addresses)
-- `pb_flow_plan`: Current Plan if Policy is called from the Flow Planner.

docs/property-mappings/expression.md

@@ -1,12 +0,0 @@
-# Property Mapping Expressions
-
-The property mapping should return a value that is expected by the Provider/Source. Supported types are documented in the individual Provider/Source. Returning `None` is always accepted and would simply skip the mapping for which `None` was returned.
-
-!!! notice
-    These variables are available in addition to the common variables/functions defined in [**Expressions**](../expressions/index.md)
-
-### Context Variables
-
-- `user`: The current user. This may be `None` if there is no contextual user. ([ref](../expressions/reference/user-object.md))
-- `request`: The current request. This may be `None` if there is no contextual request. ([ref](https://docs.djangoproject.com/en/3.0/ref/request-response/#httprequest-objects))
-- Other arbitrary arguments given by the provider, this is documented on the Provider/Source.

docs/providers.md

@@ -1,24 +0,0 @@
-# Providers
-
-Providers allow external applications to authenticate against passbook and use its user information.
-
-## OpenID Provider
-
-This provider utilises the commonly used OpenID Connect variation of OAuth2.
-
-## OAuth2 Provider
-
-This provider is slightly different than the OpenID Provider. While it uses the same basic OAuth2 Protocol, it provides a GitHub-compatible endpoint. This allows you to integrate applications which don't support custom OpenID providers.
-The API exposes username, email, name, and groups in a GitHub-compatible format.
-This provider currently supports the following scopes:
-
-- `openid`: Access OpenID Userinfo
-- `userinfo`: Access OpenID Userinfo
-- `email`: Access OpenID Email
-- `user:email`: GitHub Compatibility: User Email
-- `read:org`: GitHub Compatibility: User Groups
-
-## SAML Provider
-
-This provider allows you to integrate enterprise software using the SAML2 Protocol. It supports signed requests and uses [Property Mappings](property-mappings/index.md#saml-property-mapping) to determine which fields are exposed and what values they return. This makes it possible to expose vendor-specific fields.
-Default fields are exposed through auto-generated Property Mappings, which are prefixed with "Autogenerated".

docs/requirements.txt

@@ -1,2 +0,0 @@
-mkdocs
-mkdocs-material

docs/runtime.txt

@@ -1 +0,0 @@
-3.7

e2e/ci.docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   chrome:
-    image: selenium/standalone-chrome
+    image: selenium/standalone-chrome:3.141
     volumes:
       - /dev/shm:/dev/shm
     network_mode: host

e2e/docker-compose.yml

@@ -2,7 +2,8 @@ version: '3.7'
 
 services:
   chrome:
-    image: selenium/standalone-chrome-debug:3.141.59-20200525
+    image: selenium/standalone-chrome-debug:3.141
     volumes:
       - /dev/shm:/dev/shm
     network_mode: host
+    restart: always

e2e/setup.sh

@@ -9,7 +9,7 @@ curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
 sudo apt-get install -y nodejs
 sudo npm install -g yarn
 # Setup python
-sudo apt install -y python3.8 python3-pip
+sudo apt install -y python3.9 python3-pip libxmlsec1-dev pkg-config
 # Setup docker
 sudo pip3 install pipenv
 

e2e/test_flows_enroll.py

@@ -1,17 +1,15 @@
 """Test Enroll flow"""
-from time import sleep
+from sys import platform
+from typing import Any, Dict, Optional
+from unittest.case import skipUnless
 
 from django.test import override_settings
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as ec
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
-from e2e.utils import USER, SeleniumTestCase
+from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
-from passbook.policies.expression.models import ExpressionPolicy
-from passbook.policies.models import PolicyBinding
 from passbook.stages.email.models import EmailStage, EmailTemplates
 from passbook.stages.identification.models import IdentificationStage
 from passbook.stages.prompt.models import FieldTypes, Prompt, PromptStage
@@ -19,38 +17,24 @@ from passbook.stages.user_login.models import UserLoginStage
 from passbook.stages.user_write.models import UserWriteStage
 
 
+@skipUnless(platform.startswith("linux"), "requires local docker")
 class TestFlowsEnroll(SeleniumTestCase):
     """Test Enroll flow"""
 
-    def setUp(self):
-        self.container = self.setup_client()
-        super().setUp()
-
-    def setup_client(self) -> Container:
-        """Setup test IdP container"""
-        client: DockerClient = from_env()
-        container = client.containers.run(
-            image="mailhog/mailhog",
-            detach=True,
-            network_mode="host",
-            auto_remove=True,
-            healthcheck=Healthcheck(
-                test=["CMD", "wget", "-s", "http://localhost:8025"],
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "docker.beryju.org/proxy/mailhog/mailhog:v1.0.1",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "healthcheck": Healthcheck(
+                test=["CMD", "wget", "--spider", "http://localhost:8025"],
                 interval=5 * 100 * 1000000,
                 start_period=1 * 100 * 1000000,
             ),
-        )
-        while True:
-            container.reload()
-            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
-            if status == "healthy":
-                return container
-            sleep(1)
-
-    def tearDown(self):
-        self.container.kill()
-        super().tearDown()
+        }
 
+    @retry()
     def test_enroll_2_step(self):
         """Test 2-step enroll flow"""
         # First stage fields
@@ -85,15 +69,6 @@ class TestFlowsEnroll(SeleniumTestCase):
         user_write = UserWriteStage.objects.create(name="enroll-user-write")
         user_login = UserLoginStage.objects.create(name="enroll-user-login")
 
-        # Password checking policy
-        password_policy = ExpressionPolicy.objects.create(
-            name="policy-enrollment-password-equals",
-            expression="return request.context['password'] == request.context['password_repeat']",
-        )
-        PolicyBinding.objects.create(
-            target=first_stage, policy=password_policy, order=0
-        )
-
         flow = Flow.objects.create(
             name="default-enrollment-flow",
             slug="default-enrollment-flow",
@@ -130,7 +105,7 @@ class TestFlowsEnroll(SeleniumTestCase):
 
         self.wait_for_url(self.url("passbook_core:user-settings"))
         self.assertEqual(
-            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            self.driver.find_element(By.ID, "user-settings").text,
             "foo",
         )
         self.assertEqual(
@@ -145,6 +120,7 @@ class TestFlowsEnroll(SeleniumTestCase):
             "foo@bar.baz",
         )
 
+    @retry()
     @override_settings(EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend")
     def test_enroll_email(self):
         """Test enroll with Email verification"""
@@ -186,15 +162,6 @@ class TestFlowsEnroll(SeleniumTestCase):
         user_write = UserWriteStage.objects.create(name="enroll-user-write")
         user_login = UserLoginStage.objects.create(name="enroll-user-login")
 
-        # Password checking policy
-        password_policy = ExpressionPolicy.objects.create(
-            name="policy-enrollment-password-equals",
-            expression="return request.context['password'] == request.context['password_repeat']",
-        )
-        PolicyBinding.objects.create(
-            target=first_stage, policy=password_policy, order=0
-        )
-
         flow = Flow.objects.create(
             name="default-enrollment-flow",
             slug="default-enrollment-flow",
@@ -221,30 +188,30 @@ class TestFlowsEnroll(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_name").send_keys("some name")
         self.driver.find_element(By.ID, "id_email").send_keys("foo@bar.baz")
         self.driver.find_element(By.CSS_SELECTOR, ".pf-c-button").click()
-        sleep(3)
+        # Wait for the success message so we know the email is sent
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, ".pf-c-form > p"))
+        )
 
         # Open Mailhog
         self.driver.get("http://localhost:8025")
 
         # Click on first message
+        self.wait.until(
+            ec.presence_of_element_located((By.CLASS_NAME, "msglist-message"))
+        )
         self.driver.find_element(By.CLASS_NAME, "msglist-message").click()
-        sleep(3)
         self.driver.switch_to.frame(self.driver.find_element(By.CLASS_NAME, "tab-pane"))
         self.driver.find_element(By.ID, "confirm").click()
         self.driver.close()
         self.driver.switch_to.window(self.driver.window_handles[0])
 
         # We're now logged in
-        sleep(3)
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.XPATH, "//a[contains(@href, '/-/user/')]")
-            )
-        )
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.find_element(By.ID, "user-settings").click()
 
         self.assertEqual(
-            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            self.driver.find_element(By.ID, "user-settings").text,
             "foo",
         )
         self.assertEqual(

e2e/test_flows_login.py

@@ -1,13 +1,18 @@
 """test default login flow"""
+from sys import platform
+from unittest.case import skipUnless
+
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
-from e2e.utils import USER, SeleniumTestCase
+from e2e.utils import USER, SeleniumTestCase, retry
 
 
+@skipUnless(platform.startswith("linux"), "requires local docker")
 class TestFlowsLogin(SeleniumTestCase):
     """test default login flow"""
 
+    @retry()
     def test_login(self):
         """test default login flow"""
         self.driver.get(f"{self.live_server_url}/flows/default-authentication-flow/")
@@ -17,6 +22,6 @@ class TestFlowsLogin(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
         self.assertEqual(
-            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
+            self.driver.find_element(By.ID, "user-settings").text,
             USER().username,
         )

e2e/test_flows_otp.py

@@ -0,0 +1,144 @@
+"""test flow with otp stages"""
+from base64 import b32decode
+from sys import platform
+from time import sleep
+from unittest.case import skipUnless
+from urllib.parse import parse_qs, urlparse
+
+from django_otp.oath import TOTP
+from django_otp.plugins.otp_static.models import StaticDevice, StaticToken
+from django_otp.plugins.otp_totp.models import TOTPDevice
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as ec
+
+from e2e.utils import USER, SeleniumTestCase, retry
+from passbook.flows.models import Flow, FlowStageBinding
+from passbook.stages.otp_validate.models import OTPValidateStage
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestFlowsOTP(SeleniumTestCase):
+    """test flow with otp stages"""
+
+    @retry()
+    def test_otp_validate(self):
+        """test flow with otp stages"""
+        sleep(1)
+        # Setup TOTP Device
+        user = USER()
+        device = TOTPDevice.objects.create(user=user, confirmed=True, digits=6)
+
+        flow: Flow = Flow.objects.get(slug="default-authentication-flow")
+        # Move the user_login stage to order 3
+        FlowStageBinding.objects.filter(target=flow, order=2).update(order=3)
+        FlowStageBinding.objects.create(
+            target=flow, order=2, stage=OTPValidateStage.objects.create()
+        )
+
+        self.driver.get(f"{self.live_server_url}/flows/{flow.slug}/")
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        # Get expected token
+        totp = TOTP(device.bin_key, device.step, device.t0, device.digits, device.drift)
+        self.driver.find_element(By.ID, "id_code").send_keys(totp.token())
+        self.driver.find_element(By.ID, "id_code").send_keys(Keys.ENTER)
+        self.wait_for_url(self.url("passbook_core:overview"))
+        self.assertEqual(
+            self.driver.find_element(By.ID, "user-settings").text,
+            USER().username,
+        )
+
+    @retry()
+    def test_otp_totp_setup(self):
+        """test TOTP Setup stage"""
+        flow: Flow = Flow.objects.get(slug="default-authentication-flow")
+
+        self.driver.get(f"{self.live_server_url}/flows/{flow.slug}/")
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        self.assertEqual(
+            self.driver.find_element(By.ID, "user-settings").text,
+            USER().username,
+        )
+
+        self.driver.find_element(By.CSS_SELECTOR, ".pf-c-page__header").click()
+        self.driver.get(self.url("passbook_core:user-settings"))
+
+        self.driver.find_element(By.LINK_TEXT, "Time-based OTP").click()
+
+        # Remember the current URL as we should end up back here
+        destination_url = self.driver.current_url
+
+        self.driver.find_element(
+            By.CSS_SELECTOR, ".pf-c-card__body a.pf-c-button"
+        ).click()
+
+        self.wait.until(ec.presence_of_element_located((By.ID, "qr")))
+        otp_uri = self.driver.find_element(By.ID, "qr").get_attribute("data-otpuri")
+
+        # Parse the OTP URI, extract the secret and get the next token
+        otp_args = urlparse(otp_uri)
+        self.assertEqual(otp_args.scheme, "otpauth")
+        otp_qs = parse_qs(otp_args.query)
+        secret_key = b32decode(otp_qs["secret"][0])
+
+        totp = TOTP(secret_key)
+
+        self.driver.find_element(By.ID, "id_code").send_keys(totp.token())
+        self.driver.find_element(By.ID, "id_code").send_keys(Keys.ENTER)
+
+        self.wait_for_url(destination_url)
+        sleep(1)
+
+        self.assertTrue(TOTPDevice.objects.filter(user=USER(), confirmed=True).exists())
+
+    @retry()
+    def test_otp_static_setup(self):
+        """test Static OTP Setup stage"""
+        flow: Flow = Flow.objects.get(slug="default-authentication-flow")
+
+        self.driver.get(f"{self.live_server_url}/flows/{flow.slug}/")
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        self.assertEqual(
+            self.driver.find_element(By.ID, "user-settings").text,
+            USER().username,
+        )
+
+        self.driver.find_element(By.CSS_SELECTOR, ".pf-c-page__header").click()
+        self.driver.find_element(By.ID, "user-settings").click()
+        self.wait_for_url(self.url("passbook_core:user-settings"))
+
+        self.driver.find_element(By.LINK_TEXT, "Static OTP").click()
+
+        # Remember the current URL as we should end up back here
+        destination_url = self.driver.current_url
+
+        self.driver.find_element(
+            By.CSS_SELECTOR, ".pf-c-card__body a.pf-c-button"
+        ).click()
+        token = self.driver.find_element(
+            By.CSS_SELECTOR, ".pb-otp-tokens li:nth-child(1)"
+        ).text
+
+        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
+
+        self.wait_for_url(destination_url)
+        sleep(1)
+
+        self.assertTrue(
+            StaticDevice.objects.filter(user=USER(), confirmed=True).exists()
+        )
+        device = StaticDevice.objects.filter(user=USER(), confirmed=True).first()
+        self.assertTrue(StaticToken.objects.filter(token=token, device=device).exists())

e2e/test_flows_stage_setup.py

@@ -1,25 +1,36 @@
 """test stage setup flows (password change)"""
-import string
-from random import SystemRandom
-from time import sleep
+from sys import platform
+from unittest.case import skipUnless
 
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
-from e2e.utils import USER, SeleniumTestCase
+from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import User
+from passbook.flows.models import Flow, FlowDesignation
+from passbook.providers.oauth2.generators import generate_client_secret
+from passbook.stages.password.models import PasswordStage
 
 
+@skipUnless(platform.startswith("linux"), "requires local docker")
 class TestFlowsStageSetup(SeleniumTestCase):
     """test stage setup flows"""
 
+    @retry()
     def test_password_change(self):
         """test password change flow"""
-        new_password = "".join(
-            SystemRandom().choice(string.ascii_uppercase + string.digits)
-            for _ in range(8)
+        # Ensure that password stage has change_flow set
+        flow = Flow.objects.get(
+            slug="default-password-change",
+            designation=FlowDesignation.STAGE_CONFIGURATION,
         )
 
+        stage = PasswordStage.objects.get(name="default-authentication-password")
+        stage.configure_flow = flow
+        stage.save()
+
+        new_password = generate_client_secret()
+
         self.driver.get(
             f"{self.live_server_url}/flows/default-authentication-flow/?next=%2F"
         )
@@ -28,14 +39,15 @@ class TestFlowsStageSetup(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
         self.driver.find_element(By.CSS_SELECTOR, ".pf-c-page__header").click()
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.driver.find_element(By.ID, "user-settings").click()
+        self.wait_for_url(self.url("passbook_core:user-settings"))
         self.driver.find_element(By.LINK_TEXT, "Change password").click()
         self.driver.find_element(By.ID, "id_password").send_keys(new_password)
         self.driver.find_element(By.ID, "id_password_repeat").click()
         self.driver.find_element(By.ID, "id_password_repeat").send_keys(new_password)
         self.driver.find_element(By.CSS_SELECTOR, ".pf-c-button").click()
 
-        sleep(2)
+        self.wait_for_url(self.url("passbook_core:user-settings"))
         # Because USER() is cached, we need to get the user manually here
         user = User.objects.get(username=USER().username)
         self.assertTrue(user.check_password(new_password))

e2e/test_provider_oauth2_github.py

@@ -1,91 +1,86 @@
 """test OAuth Provider flow"""
+from sys import platform
 from time import sleep
+from typing import Any, Dict, Optional
+from unittest.case import skipUnless
 
-from oauth2_provider.generators import generate_client_id, generate_client_secret
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as ec
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
-from e2e.utils import USER, SeleniumTestCase
+from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import Application
 from passbook.flows.models import Flow
 from passbook.policies.expression.models import ExpressionPolicy
 from passbook.policies.models import PolicyBinding
-from passbook.providers.oauth.models import OAuth2Provider
+from passbook.providers.oauth2.generators import (
+    generate_client_id,
+    generate_client_secret,
+)
+from passbook.providers.oauth2.models import ClientTypes, OAuth2Provider, ResponseTypes
 
 
-class TestProviderOAuth(SeleniumTestCase):
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestProviderOAuth2Github(SeleniumTestCase):
     """test OAuth Provider flow"""
 
     def setUp(self):
         self.client_id = generate_client_id()
         self.client_secret = generate_client_secret()
-        self.container = self.setup_client()
         super().setUp()
 
-    def setup_client(self) -> Container:
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
         """Setup client grafana container which we test OAuth against"""
-        client: DockerClient = from_env()
-        container = client.containers.run(
-            image="grafana/grafana:latest",
-            detach=True,
-            network_mode="host",
-            auto_remove=True,
-            healthcheck=Healthcheck(
+        return {
+            "image": "docker.beryju.org/proxy/grafana/grafana:7.1.0",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "healthcheck": Healthcheck(
                 test=["CMD", "wget", "--spider", "http://localhost:3000"],
                 interval=5 * 100 * 1000000,
                 start_period=1 * 100 * 1000000,
             ),
-            environment={
+            "environment": {
                 "GF_AUTH_GITHUB_ENABLED": "true",
-                "GF_AUTH_GITHUB_allow_sign_up": "true",
+                "GF_AUTH_GITHUB_ALLOW_SIGN_UP": "true",
                 "GF_AUTH_GITHUB_CLIENT_ID": self.client_id,
                 "GF_AUTH_GITHUB_CLIENT_SECRET": self.client_secret,
                 "GF_AUTH_GITHUB_SCOPES": "user:email,read:org",
                 "GF_AUTH_GITHUB_AUTH_URL": self.url(
-                    "passbook_providers_oauth:github-authorize"
+                    "passbook_providers_oauth2_github:github-authorize"
                 ),
                 "GF_AUTH_GITHUB_TOKEN_URL": self.url(
-                    "passbook_providers_oauth:github-access-token"
+                    "passbook_providers_oauth2_github:github-access-token"
                 ),
                 "GF_AUTH_GITHUB_API_URL": self.url(
-                    "passbook_providers_oauth:github-user"
+                    "passbook_providers_oauth2_github:github-user"
                 ),
                 "GF_LOG_LEVEL": "debug",
             },
-        )
-        while True:
-            container.reload()
-            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
-            if status == "healthy":
-                return container
-            sleep(1)
-
-    def tearDown(self):
-        self.container.kill()
-        super().tearDown()
+        }
 
+    @retry()
     def test_authorization_consent_implied(self):
         """test OAuth Provider flow (default authorization flow with implied consent)"""
-        sleep(1)
         # Bootstrap all needed objects
         authorization_flow = Flow.objects.get(
             slug="default-provider-authorization-implicit-consent"
         )
         provider = OAuth2Provider.objects.create(
             name="grafana",
-            client_type=OAuth2Provider.CLIENT_CONFIDENTIAL,
-            authorization_grant_type=OAuth2Provider.GRANT_AUTHORIZATION_CODE,
             client_id=self.client_id,
             client_secret=self.client_secret,
+            client_type=ClientTypes.CONFIDENTIAL,
+            response_type=ResponseTypes.CODE,
             redirect_uris="http://localhost:3000/login/github",
-            skip_authorization=True,
             authorization_flow=authorization_flow,
         )
         Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
+            name="Grafana",
+            slug="grafana",
+            provider=provider,
         )
 
         self.driver.get("http://localhost:3000")
@@ -97,52 +92,50 @@ class TestProviderOAuth(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
 
         self.wait_for_url("http://localhost:3000/?orgId=1")
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/profile')]").click()
+        self.driver.get("http://localhost:3000/profile")
         self.assertEqual(
             self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
             USER().username,
         )
         self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[1]/div/input",
-            ).get_attribute("value"),
+            self.driver.find_element(By.CSS_SELECTOR, "input[name=name]").get_attribute(
+                "value"
+            ),
             USER().username,
         )
         self.assertEqual(
             self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[2]/div/input",
+                By.CSS_SELECTOR, "input[name=email]"
             ).get_attribute("value"),
             USER().email,
         )
         self.assertEqual(
             self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[3]/div/input",
+                By.CSS_SELECTOR, "input[name=login]"
             ).get_attribute("value"),
             USER().username,
         )
 
+    @retry()
     def test_authorization_consent_explicit(self):
         """test OAuth Provider flow (default authorization flow with explicit consent)"""
-        sleep(1)
         # Bootstrap all needed objects
         authorization_flow = Flow.objects.get(
             slug="default-provider-authorization-explicit-consent"
         )
         provider = OAuth2Provider.objects.create(
             name="grafana",
-            client_type=OAuth2Provider.CLIENT_CONFIDENTIAL,
-            authorization_grant_type=OAuth2Provider.GRANT_AUTHORIZATION_CODE,
             client_id=self.client_id,
             client_secret=self.client_secret,
+            client_type=ClientTypes.CONFIDENTIAL,
+            response_type=ResponseTypes.CODE,
             redirect_uris="http://localhost:3000/login/github",
-            skip_authorization=True,
             authorization_flow=authorization_flow,
         )
         app = Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
+            name="Grafana",
+            slug="grafana",
+            provider=provider,
         )
 
         self.driver.get("http://localhost:3000")
@@ -153,68 +146,66 @@ class TestProviderOAuth(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
 
-        self.assertIn(
+        sleep(1)
+
+        self.assertEqual(
             app.name,
-            self.driver.find_element(
-                By.XPATH, "/html/body/div[2]/div/main/div/form/div[2]/p[1]"
-            ).text,
+            self.driver.find_element(By.ID, "application-name").text,
         )
         self.assertEqual(
-            "GitHub Compatibility: User Email",
-            self.driver.find_element(
-                By.XPATH, "/html/body/div[2]/div/main/div/form/div[2]/ul/li[1]"
-            ).text,
+            "GitHub Compatibility: Access you Email addresses",
+            self.driver.find_element(By.ID, "scope-user:email").text,
         )
-        sleep(1)
-        self.driver.find_element(By.CSS_SELECTOR, "[type=submit]").click()
+        self.driver.find_element(
+            By.CSS_SELECTOR,
+            ("[type=submit]"),
+        ).click()
 
         self.wait_for_url("http://localhost:3000/?orgId=1")
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/profile')]").click()
+        self.driver.get("http://localhost:3000/profile")
         self.assertEqual(
             self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
             USER().username,
         )
         self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[1]/div/input",
-            ).get_attribute("value"),
+            self.driver.find_element(By.CSS_SELECTOR, "input[name=name]").get_attribute(
+                "value"
+            ),
             USER().username,
         )
         self.assertEqual(
             self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[2]/div/input",
+                By.CSS_SELECTOR, "input[name=email]"
             ).get_attribute("value"),
             USER().email,
         )
         self.assertEqual(
             self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[3]/div/input",
+                By.CSS_SELECTOR, "input[name=login]"
             ).get_attribute("value"),
             USER().username,
         )
 
+    @retry()
     def test_denied(self):
         """test OAuth Provider flow (default authorization flow, denied)"""
-        sleep(1)
         # Bootstrap all needed objects
         authorization_flow = Flow.objects.get(
             slug="default-provider-authorization-explicit-consent"
         )
         provider = OAuth2Provider.objects.create(
             name="grafana",
-            client_type=OAuth2Provider.CLIENT_CONFIDENTIAL,
-            authorization_grant_type=OAuth2Provider.GRANT_AUTHORIZATION_CODE,
             client_id=self.client_id,
             client_secret=self.client_secret,
+            client_type=ClientTypes.CONFIDENTIAL,
+            response_type=ResponseTypes.CODE,
             redirect_uris="http://localhost:3000/login/github",
-            skip_authorization=True,
             authorization_flow=authorization_flow,
         )
         app = Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
+            name="Grafana",
+            slug="grafana",
+            provider=provider,
         )
 
         negative_policy = ExpressionPolicy.objects.create(
@@ -229,7 +220,10 @@ class TestProviderOAuth(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-        self.wait_for_url(self.url("passbook_flows:denied"))
+
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "header > h1"))
+        )
         self.assertEqual(
             self.driver.find_element(By.CSS_SELECTOR, "header > h1").text,
             "Permission denied",

e2e/test_provider_oauth2_grafana.py

@@ -0,0 +1,380 @@
+"""test OAuth2 OpenID Provider flow"""
+from sys import platform
+from time import sleep
+from typing import Any, Dict, Optional
+from unittest.case import skipUnless
+
+from docker.types import Healthcheck
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as ec
+from structlog import get_logger
+
+from e2e.utils import USER, SeleniumTestCase, retry
+from passbook.core.models import Application
+from passbook.crypto.models import CertificateKeyPair
+from passbook.flows.models import Flow
+from passbook.policies.expression.models import ExpressionPolicy
+from passbook.policies.models import PolicyBinding
+from passbook.providers.oauth2.constants import (
+    SCOPE_OPENID,
+    SCOPE_OPENID_EMAIL,
+    SCOPE_OPENID_PROFILE,
+)
+from passbook.providers.oauth2.generators import (
+    generate_client_id,
+    generate_client_secret,
+)
+from passbook.providers.oauth2.models import (
+    ClientTypes,
+    OAuth2Provider,
+    ResponseTypes,
+    ScopeMapping,
+)
+
+LOGGER = get_logger()
+APPLICATION_SLUG = "grafana"
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestProviderOAuth2OAuth(SeleniumTestCase):
+    """test OAuth with OAuth Provider flow"""
+
+    def setUp(self):
+        self.client_id = generate_client_id()
+        self.client_secret = generate_client_secret()
+        super().setUp()
+
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "docker.beryju.org/proxy/grafana/grafana:7.1.0",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "healthcheck": Healthcheck(
+                test=["CMD", "wget", "--spider", "http://localhost:3000"],
+                interval=5 * 100 * 1000000,
+                start_period=1 * 100 * 1000000,
+            ),
+            "environment": {
+                "GF_AUTH_GENERIC_OAUTH_ENABLED": "true",
+                "GF_AUTH_GENERIC_OAUTH_CLIENT_ID": self.client_id,
+                "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET": self.client_secret,
+                "GF_AUTH_GENERIC_OAUTH_SCOPES": "openid email profile",
+                "GF_AUTH_GENERIC_OAUTH_AUTH_URL": (
+                    self.url("passbook_providers_oauth2:authorize")
+                ),
+                "GF_AUTH_GENERIC_OAUTH_TOKEN_URL": (
+                    self.url("passbook_providers_oauth2:token")
+                ),
+                "GF_AUTH_GENERIC_OAUTH_API_URL": (
+                    self.url("passbook_providers_oauth2:userinfo")
+                ),
+                "GF_AUTH_SIGNOUT_REDIRECT_URL": (
+                    self.url(
+                        "passbook_providers_oauth2:end-session",
+                        application_slug=APPLICATION_SLUG,
+                    )
+                ),
+                "GF_LOG_LEVEL": "debug",
+            },
+        }
+
+    @retry()
+    def test_redirect_uri_error(self):
+        """test OpenID Provider flow (invalid redirect URI, check error message)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-implicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name="grafana",
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:3000/",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        Application.objects.create(
+            name="Grafana",
+            slug=APPLICATION_SLUG,
+            provider=provider,
+        )
+
+        self.driver.get("http://localhost:3000")
+        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        sleep(2)
+        self.assertEqual(
+            self.driver.find_element(By.CLASS_NAME, "pf-c-title").text,
+            "Redirect URI Error",
+        )
+
+    @retry()
+    def test_authorization_consent_implied(self):
+        """test OpenID Provider flow (default authorization flow with implied consent)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-implicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name="grafana",
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:3000/login/generic_oauth",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        Application.objects.create(
+            name="Grafana",
+            slug=APPLICATION_SLUG,
+            provider=provider,
+        )
+
+        self.driver.get("http://localhost:3000")
+        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        self.wait_for_url("http://localhost:3000/?orgId=1")
+        self.driver.get("http://localhost:3000/profile")
+        self.assertEqual(
+            self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
+            USER().name,
+        )
+        self.assertEqual(
+            self.driver.find_element(By.CSS_SELECTOR, "input[name=name]").get_attribute(
+                "value"
+            ),
+            USER().name,
+        )
+        self.assertEqual(
+            self.driver.find_element(
+                By.CSS_SELECTOR, "input[name=email]"
+            ).get_attribute("value"),
+            USER().email,
+        )
+        self.assertEqual(
+            self.driver.find_element(
+                By.CSS_SELECTOR, "input[name=login]"
+            ).get_attribute("value"),
+            USER().email,
+        )
+
+    @retry()
+    def test_authorization_logout(self):
+        """test OpenID Provider flow with logout"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-implicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name="grafana",
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:3000/login/generic_oauth",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        Application.objects.create(
+            name="Grafana",
+            slug=APPLICATION_SLUG,
+            provider=provider,
+        )
+
+        self.driver.get("http://localhost:3000")
+        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        self.wait_for_url("http://localhost:3000/?orgId=1")
+        self.driver.get("http://localhost:3000/profile")
+        self.assertEqual(
+            self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
+            USER().name,
+        )
+        self.assertEqual(
+            self.driver.find_element(By.CSS_SELECTOR, "input[name=name]").get_attribute(
+                "value"
+            ),
+            USER().name,
+        )
+        self.assertEqual(
+            self.driver.find_element(
+                By.CSS_SELECTOR, "input[name=email]"
+            ).get_attribute("value"),
+            USER().email,
+        )
+        self.assertEqual(
+            self.driver.find_element(
+                By.CSS_SELECTOR, "input[name=login]"
+            ).get_attribute("value"),
+            USER().email,
+        )
+        self.driver.get("http://localhost:3000/logout")
+        self.wait_for_url(
+            self.url(
+                "passbook_providers_oauth2:end-session",
+                application_slug=APPLICATION_SLUG,
+            )
+        )
+        self.driver.find_element(By.ID, "logout").click()
+
+    @retry()
+    def test_authorization_consent_explicit(self):
+        """test OpenID Provider flow (default authorization flow with explicit consent)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-explicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name="grafana",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:3000/login/generic_oauth",
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        app = Application.objects.create(
+            name="Grafana",
+            slug=APPLICATION_SLUG,
+            provider=provider,
+        )
+
+        self.driver.get("http://localhost:3000")
+        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        self.assertEqual(
+            app.name,
+            self.driver.find_element(By.ID, "application-name").text,
+        )
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "[type=submit]"))
+        )
+        sleep(1)
+        self.driver.find_element(By.CSS_SELECTOR, "[type=submit]").click()
+
+        self.wait_for_url("http://localhost:3000/?orgId=1")
+        self.driver.get("http://localhost:3000/profile")
+
+        self.assertEqual(
+            self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
+            USER().name,
+        )
+        self.assertEqual(
+            self.driver.find_element(By.CSS_SELECTOR, "input[name=name]").get_attribute(
+                "value"
+            ),
+            USER().name,
+        )
+        self.assertEqual(
+            self.driver.find_element(
+                By.CSS_SELECTOR, "input[name=email]"
+            ).get_attribute("value"),
+            USER().email,
+        )
+        self.assertEqual(
+            self.driver.find_element(
+                By.CSS_SELECTOR, "input[name=login]"
+            ).get_attribute("value"),
+            USER().email,
+        )
+
+    @retry()
+    def test_authorization_denied(self):
+        """test OpenID Provider flow (default authorization with access deny)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-explicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name="grafana",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:3000/login/generic_oauth",
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        app = Application.objects.create(
+            name="Grafana",
+            slug=APPLICATION_SLUG,
+            provider=provider,
+        )
+
+        negative_policy = ExpressionPolicy.objects.create(
+            name="negative-static", expression="return False"
+        )
+        PolicyBinding.objects.create(target=app, policy=negative_policy, order=0)
+        self.driver.get("http://localhost:3000")
+        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "header > h1"))
+        )
+        self.assertEqual(
+            self.driver.find_element(By.CSS_SELECTOR, "header > h1").text,
+            "Permission denied",
+        )

e2e/test_provider_oauth2_oidc.py

@@ -0,0 +1,285 @@
+"""test OAuth2 OpenID Provider flow"""
+from json import loads
+from sys import platform
+from time import sleep
+from unittest.case import skipUnless
+
+from docker import DockerClient, from_env
+from docker.models.containers import Container
+from docker.types import Healthcheck
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as ec
+from structlog import get_logger
+
+from e2e.utils import USER, SeleniumTestCase, retry
+from passbook.core.models import Application
+from passbook.crypto.models import CertificateKeyPair
+from passbook.flows.models import Flow
+from passbook.policies.expression.models import ExpressionPolicy
+from passbook.policies.models import PolicyBinding
+from passbook.providers.oauth2.constants import (
+    SCOPE_OPENID,
+    SCOPE_OPENID_EMAIL,
+    SCOPE_OPENID_PROFILE,
+)
+from passbook.providers.oauth2.generators import (
+    generate_client_id,
+    generate_client_secret,
+)
+from passbook.providers.oauth2.models import (
+    ClientTypes,
+    OAuth2Provider,
+    ResponseTypes,
+    ScopeMapping,
+)
+
+LOGGER = get_logger()
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestProviderOAuth2OIDC(SeleniumTestCase):
+    """test OAuth with OpenID Provider flow"""
+
+    def setUp(self):
+        self.client_id = generate_client_id()
+        self.client_secret = generate_client_secret()
+        self.application_slug = "test"
+        super().setUp()
+
+    def setup_client(self) -> Container:
+        """Setup client saml-sp container which we test SAML against"""
+        sleep(1)
+        client: DockerClient = from_env()
+        client.images.pull("beryju/oidc-test-client")
+        container = client.containers.run(
+            image="docker.beryju.org/proxy/beryju/oidc-test-client",
+            detach=True,
+            network_mode="host",
+            auto_remove=True,
+            healthcheck=Healthcheck(
+                test=["CMD", "wget", "--spider", "http://localhost:9009/health"],
+                interval=5 * 100 * 1000000,
+                start_period=1 * 100 * 1000000,
+            ),
+            environment={
+                "OIDC_CLIENT_ID": self.client_id,
+                "OIDC_CLIENT_SECRET": self.client_secret,
+                "OIDC_PROVIDER": f"{self.live_server_url}/application/o/{self.application_slug}/",
+            },
+        )
+        while True:
+            container.reload()
+            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
+            if status == "healthy":
+                return container
+            LOGGER.info("Container failed healthcheck")
+            sleep(1)
+
+    @retry()
+    def test_redirect_uri_error(self):
+        """test OpenID Provider flow (invalid redirect URI, check error message)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-implicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name=self.application_slug,
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:9009/",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        Application.objects.create(
+            name=self.application_slug,
+            slug=self.application_slug,
+            provider=provider,
+        )
+        self.container = self.setup_client()
+
+        self.driver.get("http://localhost:9009")
+
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        sleep(2)
+        self.assertEqual(
+            self.driver.find_element(By.CLASS_NAME, "pf-c-title").text,
+            "Redirect URI Error",
+        )
+
+    @retry()
+    def test_authorization_consent_implied(self):
+        """test OpenID Provider flow (default authorization flow with implied consent)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-implicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name=self.application_slug,
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:9009/auth/callback",
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        Application.objects.create(
+            name=self.application_slug,
+            slug=self.application_slug,
+            provider=provider,
+        )
+        self.container = self.setup_client()
+
+        self.driver.get("http://localhost:9009")
+
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        self.wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, "pre")))
+        body = loads(self.driver.find_element(By.CSS_SELECTOR, "pre").text)
+
+        self.assertEqual(body["IDTokenClaims"]["nickname"], USER().username)
+        self.assertEqual(body["UserInfo"]["nickname"], USER().username)
+
+        self.assertEqual(body["IDTokenClaims"]["name"], USER().name)
+        self.assertEqual(body["UserInfo"]["name"], USER().name)
+
+        self.assertEqual(body["IDTokenClaims"]["email"], USER().email)
+        self.assertEqual(body["UserInfo"]["email"], USER().email)
+
+    @retry()
+    def test_authorization_consent_explicit(self):
+        """test OpenID Provider flow (default authorization flow with explicit consent)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-explicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name=self.application_slug,
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:9009/auth/callback",
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        app = Application.objects.create(
+            name=self.application_slug,
+            slug=self.application_slug,
+            provider=provider,
+        )
+        self.container = self.setup_client()
+
+        self.driver.get("http://localhost:9009")
+
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        self.assertEqual(
+            app.name,
+            self.driver.find_element(By.ID, "application-name").text,
+        )
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "[type=submit]"))
+        )
+        sleep(1)
+        self.driver.find_element(By.CSS_SELECTOR, "[type=submit]").click()
+
+        self.wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, "pre")))
+        body = loads(self.driver.find_element(By.CSS_SELECTOR, "pre").text)
+
+        self.assertEqual(body["IDTokenClaims"]["nickname"], USER().username)
+        self.assertEqual(body["UserInfo"]["nickname"], USER().username)
+
+        self.assertEqual(body["IDTokenClaims"]["name"], USER().name)
+        self.assertEqual(body["UserInfo"]["name"], USER().name)
+
+        self.assertEqual(body["IDTokenClaims"]["email"], USER().email)
+        self.assertEqual(body["UserInfo"]["email"], USER().email)
+
+    @retry()
+    def test_authorization_denied(self):
+        """test OpenID Provider flow (default authorization with access deny)"""
+        sleep(1)
+        # Bootstrap all needed objects
+        authorization_flow = Flow.objects.get(
+            slug="default-provider-authorization-explicit-consent"
+        )
+        provider = OAuth2Provider.objects.create(
+            name=self.application_slug,
+            authorization_flow=authorization_flow,
+            response_type=ResponseTypes.CODE,
+            client_type=ClientTypes.CONFIDENTIAL,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            rsa_key=CertificateKeyPair.objects.first(),
+            redirect_uris="http://localhost:9009/auth/callback",
+        )
+        provider.property_mappings.set(
+            ScopeMapping.objects.filter(
+                scope_name__in=[SCOPE_OPENID, SCOPE_OPENID_EMAIL, SCOPE_OPENID_PROFILE]
+            )
+        )
+        provider.save()
+        app = Application.objects.create(
+            name=self.application_slug,
+            slug=self.application_slug,
+            provider=provider,
+        )
+
+        negative_policy = ExpressionPolicy.objects.create(
+            name="negative-static", expression="return False"
+        )
+        PolicyBinding.objects.create(target=app, policy=negative_policy, order=0)
+
+        self.container = self.setup_client()
+        self.driver.get("http://localhost:9009")
+
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "header > h1"))
+        )
+        self.assertEqual(
+            self.driver.find_element(By.CSS_SELECTOR, "header > h1").text,
+            "Permission denied",
+        )

e2e/test_provider_oidc.py

@@ -1,304 +0,0 @@
-"""test OpenID Provider flow"""
-from time import sleep
-
-from django.shortcuts import reverse
-from oauth2_provider.generators import generate_client_id, generate_client_secret
-from oidc_provider.models import Client, ResponseType
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.support import expected_conditions as ec
-
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
-from e2e.utils import USER, SeleniumTestCase, ensure_rsa_key
-from passbook.core.models import Application
-from passbook.flows.models import Flow
-from passbook.policies.expression.models import ExpressionPolicy
-from passbook.policies.models import PolicyBinding
-from passbook.providers.oidc.models import OpenIDProvider
-
-
-class TestProviderOIDC(SeleniumTestCase):
-    """test OpenID Provider flow"""
-
-    def setUp(self):
-        self.client_id = generate_client_id()
-        self.client_secret = generate_client_secret()
-        self.container = self.setup_client()
-        super().setUp()
-
-    def setup_client(self) -> Container:
-        """Setup client grafana container which we test OIDC against"""
-        client: DockerClient = from_env()
-        container = client.containers.run(
-            image="grafana/grafana:latest",
-            detach=True,
-            network_mode="host",
-            auto_remove=True,
-            healthcheck=Healthcheck(
-                test=["CMD", "wget", "--spider", "http://localhost:3000"],
-                interval=5 * 100 * 1000000,
-                start_period=1 * 100 * 1000000,
-            ),
-            environment={
-                "GF_AUTH_GENERIC_OAUTH_ENABLED": "true",
-                "GF_AUTH_GENERIC_OAUTH_CLIENT_ID": self.client_id,
-                "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET": self.client_secret,
-                "GF_AUTH_GENERIC_OAUTH_SCOPES": "openid email profile",
-                "GF_AUTH_GENERIC_OAUTH_AUTH_URL": (
-                    self.live_server_url + reverse("passbook_providers_oidc:authorize")
-                ),
-                "GF_AUTH_GENERIC_OAUTH_TOKEN_URL": (
-                    self.live_server_url + reverse("oidc_provider:token")
-                ),
-                "GF_AUTH_GENERIC_OAUTH_API_URL": (
-                    self.live_server_url + reverse("oidc_provider:userinfo")
-                ),
-                "GF_LOG_LEVEL": "debug",
-            },
-        )
-        while True:
-            container.reload()
-            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
-            if status == "healthy":
-                return container
-            sleep(1)
-
-    def tearDown(self):
-        self.container.kill()
-        super().tearDown()
-
-    def test_redirect_uri_error(self):
-        """test OpenID Provider flow (invalid redirect URI, check error message)"""
-        sleep(1)
-        # Bootstrap all needed objects
-        authorization_flow = Flow.objects.get(
-            slug="default-provider-authorization-implicit-consent"
-        )
-        client = Client.objects.create(
-            name="grafana",
-            client_type="confidential",
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            _redirect_uris="http://localhost:3000/",
-            _scope="openid userinfo",
-        )
-        # At least one of these objects must exist
-        ensure_rsa_key()
-        # This response_code object might exist or not, depending on the order the tests are run
-        rp_type, _ = ResponseType.objects.get_or_create(value="code")
-        client.response_types.set([rp_type])
-        client.save()
-        provider = OpenIDProvider.objects.create(
-            oidc_client=client, authorization_flow=authorization_flow,
-        )
-        Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
-        )
-
-        self.driver.get("http://localhost:3000")
-        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
-        self.driver.find_element(By.ID, "id_uid_field").click()
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
-        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-        sleep(2)
-        self.assertEqual(
-            self.driver.find_element(By.CLASS_NAME, "pf-c-title").text,
-            "Redirect URI Error",
-        )
-
-    def test_authorization_consent_implied(self):
-        """test OpenID Provider flow (default authorization flow with implied consent)"""
-        sleep(1)
-        # Bootstrap all needed objects
-        authorization_flow = Flow.objects.get(
-            slug="default-provider-authorization-implicit-consent"
-        )
-        client = Client.objects.create(
-            name="grafana",
-            client_type="confidential",
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            _redirect_uris="http://localhost:3000/login/generic_oauth",
-            _scope="openid profile email",
-            reuse_consent=False,
-            require_consent=False,
-        )
-        # At least one of these objects must exist
-        ensure_rsa_key()
-        # This response_code object might exist or not, depending on the order the tests are run
-        rp_type, _ = ResponseType.objects.get_or_create(value="code")
-        client.response_types.set([rp_type])
-        client.save()
-        provider = OpenIDProvider.objects.create(
-            oidc_client=client, authorization_flow=authorization_flow,
-        )
-        Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
-        )
-
-        self.driver.get("http://localhost:3000")
-        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
-        self.driver.find_element(By.ID, "id_uid_field").click()
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
-        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/profile')]").click()
-        self.assertEqual(
-            self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
-            USER().name,
-        )
-        self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[1]/div/input",
-            ).get_attribute("value"),
-            USER().name,
-        )
-        self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[2]/div/input",
-            ).get_attribute("value"),
-            USER().email,
-        )
-        self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[3]/div/input",
-            ).get_attribute("value"),
-            USER().email,
-        )
-
-    def test_authorization_consent_explicit(self):
-        """test OpenID Provider flow (default authorization flow with explicit consent)"""
-        sleep(1)
-        # Bootstrap all needed objects
-        authorization_flow = Flow.objects.get(
-            slug="default-provider-authorization-explicit-consent"
-        )
-        client = Client.objects.create(
-            name="grafana",
-            client_type="confidential",
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            _redirect_uris="http://localhost:3000/login/generic_oauth",
-            _scope="openid profile email",
-            reuse_consent=False,
-            require_consent=False,
-        )
-        # At least one of these objects must exist
-        ensure_rsa_key()
-        # This response_code object might exist or not, depending on the order the tests are run
-        rp_type, _ = ResponseType.objects.get_or_create(value="code")
-        client.response_types.set([rp_type])
-        client.save()
-        provider = OpenIDProvider.objects.create(
-            oidc_client=client, authorization_flow=authorization_flow,
-        )
-        app = Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
-        )
-
-        self.driver.get("http://localhost:3000")
-        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
-        self.driver.find_element(By.ID, "id_uid_field").click()
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
-        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-
-        self.assertIn(
-            app.name,
-            self.driver.find_element(
-                By.XPATH, "/html/body/div[2]/div/main/div/form/div[2]/p[1]"
-            ).text,
-        )
-        self.wait.until(
-            ec.presence_of_element_located((By.CSS_SELECTOR, "[type=submit]"))
-        )
-        sleep(1)
-        self.driver.find_element(By.CSS_SELECTOR, "[type=submit]").click()
-
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.XPATH, "//a[contains(@href, '/profile')]")
-            )
-        )
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/profile')]").click()
-        self.assertEqual(
-            self.driver.find_element(By.CLASS_NAME, "page-header__title").text,
-            USER().name,
-        )
-        self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[1]/div/input",
-            ).get_attribute("value"),
-            USER().name,
-        )
-        self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[2]/div/input",
-            ).get_attribute("value"),
-            USER().email,
-        )
-        self.assertEqual(
-            self.driver.find_element(
-                By.XPATH,
-                "/html/body/grafana-app/div/div/div/react-profile-wrapper/form[1]/div[3]/div/input",
-            ).get_attribute("value"),
-            USER().email,
-        )
-
-    def test_authorization_denied(self):
-        """test OpenID Provider flow (default authorization with access deny)"""
-        sleep(1)
-        # Bootstrap all needed objects
-        authorization_flow = Flow.objects.get(
-            slug="default-provider-authorization-explicit-consent"
-        )
-        client = Client.objects.create(
-            name="grafana",
-            client_type="confidential",
-            client_id=self.client_id,
-            client_secret=self.client_secret,
-            _redirect_uris="http://localhost:3000/login/generic_oauth",
-            _scope="openid profile email",
-            reuse_consent=False,
-            require_consent=False,
-        )
-        # At least one of these objects must exist
-        ensure_rsa_key()
-        # This response_code object might exist or not, depending on the order the tests are run
-        rp_type, _ = ResponseType.objects.get_or_create(value="code")
-        client.response_types.set([rp_type])
-        client.save()
-        provider = OpenIDProvider.objects.create(
-            oidc_client=client, authorization_flow=authorization_flow,
-        )
-        app = Application.objects.create(
-            name="Grafana", slug="grafana", provider=provider,
-        )
-
-        negative_policy = ExpressionPolicy.objects.create(
-            name="negative-static", expression="return False"
-        )
-        PolicyBinding.objects.create(target=app, policy=negative_policy, order=0)
-        self.driver.get("http://localhost:3000")
-        self.driver.find_element(By.CLASS_NAME, "btn-service--oauth").click()
-        self.driver.find_element(By.ID, "id_uid_field").click()
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
-        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
-        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-        self.wait_for_url(self.url("passbook_flows:denied"))
-        self.assertEqual(
-            self.driver.find_element(By.CSS_SELECTOR, "header > h1").text,
-            "Permission denied",
-        )

e2e/test_provider_proxy.py

@@ -0,0 +1,157 @@
+"""Proxy and Outpost e2e tests"""
+from dataclasses import asdict
+from sys import platform
+from time import sleep
+from typing import Any, Dict, Optional
+from unittest.case import skipUnless
+
+from channels.testing import ChannelsLiveServerTestCase
+from docker.client import DockerClient, from_env
+from docker.models.containers import Container
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+
+from e2e.utils import USER, SeleniumTestCase, retry
+from passbook import __version__
+from passbook.core.models import Application
+from passbook.flows.models import Flow
+from passbook.outposts.models import (
+    DockerServiceConnection,
+    Outpost,
+    OutpostConfig,
+    OutpostType,
+)
+from passbook.providers.proxy.models import ProxyProvider
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestProviderProxy(SeleniumTestCase):
+    """Proxy and Outpost e2e tests"""
+
+    proxy_container: Container
+
+    def tearDown(self) -> None:
+        super().tearDown()
+        self.proxy_container.kill()
+
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "docker.beryju.org/proxy/traefik/whoami:latest",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+        }
+
+    def start_proxy(self, outpost: Outpost) -> Container:
+        """Start proxy container based on outpost created"""
+        client: DockerClient = from_env()
+        container = client.containers.run(
+            image=f"beryju/passbook-proxy:{__version__}",
+            detach=True,
+            network_mode="host",
+            auto_remove=True,
+            environment={
+                "PASSBOOK_HOST": self.live_server_url,
+                "PASSBOOK_TOKEN": outpost.token.key,
+            },
+        )
+        return container
+
+    @retry()
+    def test_proxy_simple(self):
+        """Test simple outpost setup with single provider"""
+        proxy: ProxyProvider = ProxyProvider.objects.create(
+            name="proxy_provider",
+            authorization_flow=Flow.objects.get(
+                slug="default-provider-authorization-implicit-consent"
+            ),
+            internal_host="http://localhost:80",
+            external_host="http://localhost:4180",
+        )
+        # Ensure OAuth2 Params are set
+        proxy.set_oauth_defaults()
+        proxy.save()
+        # we need to create an application to actually access the proxy
+        Application.objects.create(name="proxy", slug="proxy", provider=proxy)
+        outpost: Outpost = Outpost.objects.create(
+            name="proxy_outpost",
+            type=OutpostType.PROXY,
+        )
+        outpost.providers.add(proxy)
+        outpost.save()
+
+        self.proxy_container = self.start_proxy(outpost)
+
+        # Wait until outpost healthcheck succeeds
+        healthcheck_retries = 0
+        while healthcheck_retries < 50:
+            if len(outpost.state) > 0:
+                state = outpost.state[0]
+                if state.last_seen:
+                    break
+            healthcheck_retries += 1
+            sleep(0.5)
+
+        self.driver.get("http://localhost:4180")
+
+        self.driver.find_element(By.ID, "id_uid_field").click()
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
+        self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
+        self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+
+        sleep(1)
+
+        full_body_text = self.driver.find_element(By.CSS_SELECTOR, "pre").text
+        self.assertIn("X-Forwarded-Preferred-Username: pbadmin", full_body_text)
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestProviderProxyConnect(ChannelsLiveServerTestCase):
+    """Test Proxy connectivity over websockets"""
+
+    @retry()
+    def test_proxy_connectivity(self):
+        """Test proxy connectivity over websocket"""
+        SeleniumTestCase().apply_default_data()
+        proxy: ProxyProvider = ProxyProvider.objects.create(
+            name="proxy_provider",
+            authorization_flow=Flow.objects.get(
+                slug="default-provider-authorization-implicit-consent"
+            ),
+            internal_host="http://localhost:80",
+            external_host="http://localhost:4180",
+        )
+        # Ensure OAuth2 Params are set
+        proxy.set_oauth_defaults()
+        proxy.save()
+        # we need to create an application to actually access the proxy
+        Application.objects.create(name="proxy", slug="proxy", provider=proxy)
+        service_connection = DockerServiceConnection.objects.get(local=True)
+        outpost: Outpost = Outpost.objects.create(
+            name="proxy_outpost",
+            type=OutpostType.PROXY,
+            service_connection=service_connection,
+            _config=asdict(
+                OutpostConfig(passbook_host=self.live_server_url, log_level="debug")
+            ),
+        )
+        outpost.providers.add(proxy)
+        outpost.save()
+
+        # Wait until outpost healthcheck succeeds
+        healthcheck_retries = 0
+        while healthcheck_retries < 50:
+            if len(outpost.state) > 0:
+                state = outpost.state[0]
+                if state.last_seen and state.version:
+                    break
+            healthcheck_retries += 1
+            sleep(0.5)
+
+        state = outpost.state
+        self.assertTrue(len(state), 1)
+        self.assertEqual(state[0].version, __version__)
+
+        # Make sure to delete the outpost to remove the container
+        outpost.delete()

e2e/test_provider_saml.py

@@ -1,13 +1,18 @@
 """test SAML Provider flow"""
+from json import loads
+from sys import platform
 from time import sleep
-
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
+from unittest.case import skipUnless
 
 from docker import DockerClient, from_env
 from docker.models.containers import Container
 from docker.types import Healthcheck
-from e2e.utils import USER, SeleniumTestCase
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as ec
+from structlog import get_logger
+
+from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import Application
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
@@ -19,7 +24,10 @@ from passbook.providers.saml.models import (
     SAMLProvider,
 )
 
+LOGGER = get_logger()
 
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
 class TestProviderSAML(SeleniumTestCase):
     """test SAML Provider flow"""
 
@@ -28,8 +36,9 @@ class TestProviderSAML(SeleniumTestCase):
     def setup_client(self, provider: SAMLProvider) -> Container:
         """Setup client saml-sp container which we test SAML against"""
         client: DockerClient = from_env()
+        client.images.pull("beryju/oidc-test-client")
         container = client.containers.run(
-            image="beryju/saml-test-sp",
+            image="docker.beryju.org/proxy/beryju/saml-test-sp",
             detach=True,
             network_mode="host",
             auto_remove=True,
@@ -54,12 +63,10 @@ class TestProviderSAML(SeleniumTestCase):
             status = container.attrs.get("State", {}).get("Health", {}).get("Status")
             if status == "healthy":
                 return container
+            LOGGER.info("Container failed healthcheck")
             sleep(1)
 
-    def tearDown(self):
-        self.container.kill()
-        super().tearDown()
-
+    @retry()
     def test_sp_initiated_implicit(self):
         """test SAML Provider flow SP-initiated flow (implicit consent)"""
         # Bootstrap all needed objects
@@ -78,7 +85,9 @@ class TestProviderSAML(SeleniumTestCase):
         provider.property_mappings.set(SAMLPropertyMapping.objects.all())
         provider.save()
         Application.objects.create(
-            name="SAML", slug="passbook-saml", provider=provider,
+            name="SAML",
+            slug="passbook-saml",
+            provider=provider,
         )
         self.container = self.setup_client(provider)
         self.driver.get("http://localhost:9009")
@@ -88,11 +97,16 @@ class TestProviderSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
         self.wait_for_url("http://localhost:9009/")
-        self.assertEqual(
-            self.driver.find_element(By.XPATH, "/html/body/pre").text,
-            f"Hello, {USER().name}!",
-        )
 
+        body = loads(self.driver.find_element(By.CSS_SELECTOR, "pre").text)
+
+        self.assertEqual(body["attr"]["cn"], [USER().name])
+        self.assertEqual(body["attr"]["displayName"], [USER().username])
+        self.assertEqual(body["attr"]["eduPersonPrincipalName"], [USER().email])
+        self.assertEqual(body["attr"]["mail"], [USER().email])
+        self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
+
+    @retry()
     def test_sp_initiated_explicit(self):
         """test SAML Provider flow SP-initiated flow (explicit consent)"""
         # Bootstrap all needed objects
@@ -111,7 +125,9 @@ class TestProviderSAML(SeleniumTestCase):
         provider.property_mappings.set(SAMLPropertyMapping.objects.all())
         provider.save()
         app = Application.objects.create(
-            name="SAML", slug="passbook-saml", provider=provider,
+            name="SAML",
+            slug="passbook-saml",
+            provider=provider,
         )
         self.container = self.setup_client(provider)
         self.driver.get("http://localhost:9009")
@@ -120,20 +136,23 @@ class TestProviderSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-        self.assertIn(
+        self.assertEqual(
             app.name,
-            self.driver.find_element(
-                By.XPATH, "/html/body/div[2]/div/main/div/form/div[2]/p[1]"
-            ).text,
+            self.driver.find_element(By.ID, "application-name").text,
         )
         sleep(1)
         self.driver.find_element(By.CSS_SELECTOR, "[type=submit]").click()
         self.wait_for_url("http://localhost:9009/")
-        self.assertEqual(
-            self.driver.find_element(By.XPATH, "/html/body/pre").text,
-            f"Hello, {USER().name}!",
-        )
 
+        body = loads(self.driver.find_element(By.CSS_SELECTOR, "pre").text)
+
+        self.assertEqual(body["attr"]["cn"], [USER().name])
+        self.assertEqual(body["attr"]["displayName"], [USER().username])
+        self.assertEqual(body["attr"]["eduPersonPrincipalName"], [USER().email])
+        self.assertEqual(body["attr"]["mail"], [USER().email])
+        self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
+
+    @retry()
     def test_idp_initiated_implicit(self):
         """test SAML Provider flow IdP-initiated flow (implicit consent)"""
         # Bootstrap all needed objects
@@ -152,7 +171,9 @@ class TestProviderSAML(SeleniumTestCase):
         provider.property_mappings.set(SAMLPropertyMapping.objects.all())
         provider.save()
         Application.objects.create(
-            name="SAML", slug="passbook-saml", provider=provider,
+            name="SAML",
+            slug="passbook-saml",
+            provider=provider,
         )
         self.container = self.setup_client(provider)
         self.driver.get(
@@ -166,12 +187,18 @@ class TestProviderSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
+        sleep(1)
         self.wait_for_url("http://localhost:9009/")
-        self.assertEqual(
-            self.driver.find_element(By.XPATH, "/html/body/pre").text,
-            f"Hello, {USER().name}!",
-        )
 
+        body = loads(self.driver.find_element(By.CSS_SELECTOR, "pre").text)
+
+        self.assertEqual(body["attr"]["cn"], [USER().name])
+        self.assertEqual(body["attr"]["displayName"], [USER().username])
+        self.assertEqual(body["attr"]["eduPersonPrincipalName"], [USER().email])
+        self.assertEqual(body["attr"]["mail"], [USER().email])
+        self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
+
+    @retry()
     def test_sp_initiated_denied(self):
         """test SAML Provider flow SP-initiated flow (Policy denies access)"""
         # Bootstrap all needed objects
@@ -193,7 +220,9 @@ class TestProviderSAML(SeleniumTestCase):
         provider.property_mappings.set(SAMLPropertyMapping.objects.all())
         provider.save()
         app = Application.objects.create(
-            name="SAML", slug="passbook-saml", provider=provider,
+            name="SAML",
+            slug="passbook-saml",
+            provider=provider,
         )
         PolicyBinding.objects.create(target=app, policy=negative_policy, order=0)
         self.container = self.setup_client(provider)
@@ -203,7 +232,10 @@ class TestProviderSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
         self.driver.find_element(By.ID, "id_password").send_keys(USER().username)
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
-        self.wait_for_url(self.url("passbook_flows:denied"))
+
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "header > h1"))
+        )
         self.assertEqual(
             self.driver.find_element(By.CSS_SELECTOR, "header > h1").text,
             "Permission denied",

e2e/test_source_oauth.py

@@ -0,0 +1,343 @@
+"""test OAuth Source"""
+from os.path import abspath
+from sys import platform
+from time import sleep
+from typing import Any, Dict, Optional
+from unittest.case import skipUnless
+
+from django.test import override_settings
+from docker.models.containers import Container
+from docker.types import Healthcheck
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as ec
+from structlog import get_logger
+from yaml import safe_dump
+
+from e2e.utils import SeleniumTestCase, retry
+from passbook.flows.models import Flow
+from passbook.providers.oauth2.generators import (
+    generate_client_id,
+    generate_client_secret,
+)
+from passbook.sources.oauth.models import OAuthSource
+
+CONFIG_PATH = "/tmp/dex.yml"
+LOGGER = get_logger()
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestSourceOAuth2(SeleniumTestCase):
+    """test OAuth Source flow"""
+
+    container: Container
+
+    def setUp(self):
+        self.client_secret = generate_client_secret()
+        self.prepare_dex_config()
+        super().setUp()
+
+    def prepare_dex_config(self):
+        """Since Dex does not document which environment
+        variables can be used to configure clients"""
+        config = {
+            "enablePasswordDB": True,
+            "issuer": "http://127.0.0.1:5556/dex",
+            "logger": {"level": "debug"},
+            "staticClients": [
+                {
+                    "id": "example-app",
+                    "name": "Example App",
+                    "redirectURIs": [
+                        self.url(
+                            "passbook_sources_oauth:oauth-client-callback",
+                            source_slug="dex",
+                        )
+                    ],
+                    "secret": self.client_secret,
+                }
+            ],
+            "staticPasswords": [
+                {
+                    "email": "admin@example.com",
+                    # hash for password
+                    "hash": "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W",
+                    "userID": "08a8684b-db88-4b73-90a9-3cd1661f5466",
+                    "username": "admin",
+                }
+            ],
+            "storage": {"config": {"file": "/tmp/dex.db"}, "type": "sqlite3"},
+            "web": {"http": "0.0.0.0:5556"},
+        }
+        with open(CONFIG_PATH, "w+") as _file:
+            safe_dump(config, _file)
+
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "quay.io/dexidp/dex:v2.24.0",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "command": "serve /config.yml",
+            "healthcheck": Healthcheck(
+                test=["CMD", "wget", "--spider", "http://localhost:5556/dex/healthz"],
+                interval=5 * 100 * 1000000,
+                start_period=1 * 100 * 1000000,
+            ),
+            "volumes": {abspath(CONFIG_PATH): {"bind": "/config.yml", "mode": "ro"}},
+        }
+
+    def create_objects(self):
+        """Create required objects"""
+        # Bootstrap all needed objects
+        authentication_flow = Flow.objects.get(slug="default-source-authentication")
+        enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
+
+        OAuthSource.objects.create(  # nosec
+            name="dex",
+            slug="dex",
+            authentication_flow=authentication_flow,
+            enrollment_flow=enrollment_flow,
+            provider_type="openid-connect",
+            authorization_url="http://127.0.0.1:5556/dex/auth",
+            access_token_url="http://127.0.0.1:5556/dex/token",
+            profile_url="http://127.0.0.1:5556/dex/userinfo",
+            consumer_key="example-app",
+            consumer_secret=self.client_secret,
+        )
+
+    @retry()
+    def test_oauth_enroll(self):
+        """test OAuth Source With With OIDC"""
+        self.create_objects()
+        self.driver.get(self.live_server_url)
+
+        self.wait.until(
+            ec.presence_of_element_located(
+                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
+            )
+        )
+        self.driver.find_element(
+            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
+        ).click()
+
+        # Now we should be at the IDP, wait for the login field
+        self.wait.until(ec.presence_of_element_located((By.ID, "login")))
+        self.driver.find_element(By.ID, "login").send_keys("admin@example.com")
+        self.driver.find_element(By.ID, "password").send_keys("password")
+        self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
+
+        # Wait until we're logged in
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "button[type=submit]"))
+        )
+        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
+
+        self.wait.until(ec.presence_of_element_located((By.NAME, "username")))
+        # At this point we've been redirected back
+        # and we're asked for the username
+        self.driver.find_element(By.NAME, "username").click()
+        self.driver.find_element(By.NAME, "username").send_keys("foo")
+        self.driver.find_element(By.NAME, "username").send_keys(Keys.ENTER)
+
+        # Wait until we've loaded the user info page
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.get(self.url("passbook_core:user-settings"))
+
+        self.assertEqual(
+            self.driver.find_element(By.ID, "user-settings").text,
+            "foo",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_username").get_attribute("value"), "foo"
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_name").get_attribute("value"),
+            "admin",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_email").get_attribute("value"),
+            "admin@example.com",
+        )
+
+    @retry()
+    @override_settings(SESSION_COOKIE_SAMESITE="strict")
+    def test_oauth_samesite_strict(self):
+        """test OAuth Source With SameSite set to strict
+        (=will fail because session is not carried over)"""
+        self.create_objects()
+        self.driver.get(self.live_server_url)
+
+        self.wait.until(
+            ec.presence_of_element_located(
+                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
+            )
+        )
+        self.driver.find_element(
+            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
+        ).click()
+
+        # Now we should be at the IDP, wait for the login field
+        self.wait.until(ec.presence_of_element_located((By.ID, "login")))
+        self.driver.find_element(By.ID, "login").send_keys("admin@example.com")
+        self.driver.find_element(By.ID, "password").send_keys("password")
+        self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
+
+        # Wait until we're logged in
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "button[type=submit]"))
+        )
+        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
+
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, ".pf-c-alert__title"))
+        )
+        self.assertEqual(
+            self.driver.find_element(By.CSS_SELECTOR, ".pf-c-alert__title").text,
+            "Authentication Failed.",
+        )
+
+    @retry()
+    def test_oauth_enroll_auth(self):
+        """test OAuth Source With With OIDC (enroll and authenticate again)"""
+        self.test_oauth_enroll()
+        # We're logged in at the end of this, log out and re-login
+        self.driver.find_element(By.ID, "logout").click()
+
+        self.wait.until(
+            ec.presence_of_element_located(
+                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
+            )
+        )
+        sleep(1)
+        self.driver.find_element(
+            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
+        ).click()
+        sleep(1)
+        # Now we should be at the IDP, wait for the login field
+        self.wait.until(ec.presence_of_element_located((By.ID, "login")))
+        self.driver.find_element(By.ID, "login").send_keys("admin@example.com")
+        self.driver.find_element(By.ID, "password").send_keys("password")
+        self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
+
+        # Wait until we're logged in
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "button[type=submit]"))
+        )
+        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
+
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.get(self.url("passbook_core:user-settings"))
+
+        self.assertEqual(
+            self.driver.find_element(By.ID, "user-settings").text,
+            "foo",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_username").get_attribute("value"), "foo"
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_name").get_attribute("value"),
+            "admin",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_email").get_attribute("value"),
+            "admin@example.com",
+        )
+
+
+@skipUnless(platform.startswith("linux"), "requires local docker")
+class TestSourceOAuth1(SeleniumTestCase):
+    """Test OAuth1 Source"""
+
+    def setUp(self) -> None:
+        self.client_id = generate_client_id()
+        self.client_secret = generate_client_secret()
+        self.source_slug = "oauth1-test"
+        super().setUp()
+
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "docker.beryju.org/proxy/beryju/oauth1-test-server",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "environment": {
+                "OAUTH1_CLIENT_ID": self.client_id,
+                "OAUTH1_CLIENT_SECRET": self.client_secret,
+                "OAUTH1_REDIRECT_URI": (
+                    self.url(
+                        "passbook_sources_oauth:oauth-client-callback",
+                        source_slug=self.source_slug,
+                    )
+                ),
+            },
+        }
+
+    def create_objects(self):
+        """Create required objects"""
+        # Bootstrap all needed objects
+        authentication_flow = Flow.objects.get(slug="default-source-authentication")
+        enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
+
+        OAuthSource.objects.create(  # nosec
+            name="oauth1",
+            slug=self.source_slug,
+            authentication_flow=authentication_flow,
+            enrollment_flow=enrollment_flow,
+            provider_type="twitter",
+            request_token_url="http://localhost:5000/oauth/request_token",
+            access_token_url="http://localhost:5000/oauth/access_token",
+            authorization_url="http://localhost:5000/oauth/authorize",
+            profile_url="http://localhost:5000/api/me",
+            consumer_key=self.client_id,
+            consumer_secret=self.client_secret,
+        )
+
+    @retry()
+    def test_oauth_enroll(self):
+        """test OAuth Source With With OIDC"""
+        self.create_objects()
+        self.driver.get(self.live_server_url)
+
+        self.wait.until(
+            ec.presence_of_element_located(
+                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
+            )
+        )
+        self.driver.find_element(
+            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
+        ).click()
+
+        # Now we should be at the IDP, wait for the login field
+        self.wait.until(ec.presence_of_element_located((By.NAME, "username")))
+        self.driver.find_element(By.NAME, "username").send_keys("example-user")
+        self.driver.find_element(By.NAME, "username").send_keys(Keys.ENTER)
+
+        # Wait until we're logged in
+        self.wait.until(
+            ec.presence_of_element_located((By.CSS_SELECTOR, "[name='confirm']"))
+        )
+        self.driver.find_element(By.CSS_SELECTOR, "[name='confirm']").click()
+
+        # Wait until we've loaded the user info page
+        sleep(2)
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.get(self.url("passbook_core:user-settings"))
+
+        self.assertEqual(
+            self.driver.find_element(By.ID, "user-settings").text,
+            "example-user",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_username").get_attribute("value"),
+            "example-user",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_name").get_attribute("value"),
+            "test name",
+        )
+        self.assertEqual(
+            self.driver.find_element(By.ID, "id_email").get_attribute("value"),
+            "foo@example.com",
+        )

e2e/test_source_saml.py

@@ -1,18 +1,22 @@
 """test SAML Source"""
+from sys import platform
 from time import sleep
+from typing import Any, Dict, Optional
+from unittest.case import skipUnless
 
+from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
+from structlog import get_logger
 
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
-from e2e.utils import SeleniumTestCase
+from e2e.utils import SeleniumTestCase, retry
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
 from passbook.sources.saml.models import SAMLBindingTypes, SAMLSource
 
+LOGGER = get_logger()
+
 IDP_CERT = """-----BEGIN CERTIFICATE-----
 MIIDXTCCAkWgAwIBAgIJALmVVuDWu4NYMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
 BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
@@ -65,52 +69,39 @@ Sm75WXsflOxuTn08LbgGc4s=
 -----END PRIVATE KEY-----"""
 
 
+@skipUnless(platform.startswith("linux"), "requires local docker")
 class TestSourceSAML(SeleniumTestCase):
     """test SAML Source flow"""
 
-    def setUp(self):
-        self.container = self.setup_client()
-        super().setUp()
-
-    def setup_client(self) -> Container:
-        """Setup test IdP container"""
-        client: DockerClient = from_env()
-        container = client.containers.run(
-            image="kristophjunge/test-saml-idp",
-            detach=True,
-            network_mode="host",
-            auto_remove=True,
-            healthcheck=Healthcheck(
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        return {
+            "image": "docker.beryju.org/proxy/kristophjunge/test-saml-idp:1.15",
+            "detach": True,
+            "network_mode": "host",
+            "auto_remove": True,
+            "healthcheck": Healthcheck(
                 test=["CMD", "curl", "http://localhost:8080"],
                 interval=5 * 100 * 1000000,
                 start_period=1 * 100 * 1000000,
             ),
-            environment={
+            "environment": {
                 "SIMPLESAMLPHP_SP_ENTITY_ID": "entity-id",
                 "SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE": (
                     f"{self.live_server_url}/source/saml/saml-idp-test/acs/"
                 ),
             },
-        )
-        while True:
-            container.reload()
-            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
-            if status == "healthy":
-                return container
-            sleep(1)
-
-    def tearDown(self):
-        self.container.kill()
-        super().tearDown()
+        }
 
+    @retry()
     def test_idp_redirect(self):
         """test SAML Source With redirect binding"""
-        sleep(1)
         # Bootstrap all needed objects
         authentication_flow = Flow.objects.get(slug="default-source-authentication")
         enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
         keypair = CertificateKeyPair.objects.create(
-            name="test-idp-cert", certificate_data=IDP_CERT, key_data=IDP_KEY,
+            name="test-idp-cert",
+            certificate_data=IDP_CERT,
+            key_data=IDP_KEY,
         )
 
         SAMLSource.objects.create(
@@ -142,12 +133,8 @@ class TestSourceSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
 
         # Wait until we're logged in
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.XPATH, "//a[contains(@href, '/-/user/')]")
-            )
-        )
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.get(self.url("passbook_core:user-settings"))
 
         # Wait until we've loaded the user info page
         self.wait.until(ec.presence_of_element_located((By.ID, "id_username")))
@@ -155,14 +142,16 @@ class TestSourceSAML(SeleniumTestCase):
             self.driver.find_element(By.ID, "id_username").get_attribute("value"), ""
         )
 
+    @retry()
     def test_idp_post(self):
         """test SAML Source With post binding"""
-        sleep(1)
         # Bootstrap all needed objects
         authentication_flow = Flow.objects.get(slug="default-source-authentication")
         enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
         keypair = CertificateKeyPair.objects.create(
-            name="test-idp-cert", certificate_data=IDP_CERT, key_data=IDP_KEY,
+            name="test-idp-cert",
+            certificate_data=IDP_CERT,
+            key_data=IDP_KEY,
         )
 
         SAMLSource.objects.create(
@@ -196,12 +185,8 @@ class TestSourceSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
 
         # Wait until we're logged in
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.XPATH, "//a[contains(@href, '/-/user/')]")
-            )
-        )
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.get(self.url("passbook_core:user-settings"))
 
         # Wait until we've loaded the user info page
         self.wait.until(ec.presence_of_element_located((By.ID, "id_username")))
@@ -209,14 +194,16 @@ class TestSourceSAML(SeleniumTestCase):
             self.driver.find_element(By.ID, "id_username").get_attribute("value"), ""
         )
 
+    @retry()
     def test_idp_post_auto(self):
         """test SAML Source With post binding (auto redirect)"""
-        sleep(1)
         # Bootstrap all needed objects
         authentication_flow = Flow.objects.get(slug="default-source-authentication")
         enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
         keypair = CertificateKeyPair.objects.create(
-            name="test-idp-cert", certificate_data=IDP_CERT, key_data=IDP_KEY,
+            name="test-idp-cert",
+            certificate_data=IDP_CERT,
+            key_data=IDP_KEY,
         )
 
         SAMLSource.objects.create(
@@ -248,12 +235,8 @@ class TestSourceSAML(SeleniumTestCase):
         self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
 
         # Wait until we're logged in
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.XPATH, "//a[contains(@href, '/-/user/')]")
-            )
-        )
-        self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").click()
+        self.wait.until(ec.presence_of_element_located((By.ID, "user-settings")))
+        self.driver.get(self.url("passbook_core:user-settings"))
 
         # Wait until we've loaded the user info page
         self.wait.until(ec.presence_of_element_located((By.ID, "id_username")))

e2e/test_sources_oauth.py

@@ -1,212 +0,0 @@
-"""test OAuth Source"""
-from os.path import abspath
-from time import sleep
-
-from oauth2_provider.generators import generate_client_secret
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-from selenium.webdriver.support import expected_conditions as ec
-from yaml import safe_dump
-
-from docker import DockerClient, from_env
-from docker.models.containers import Container
-from docker.types import Healthcheck
-from e2e.utils import SeleniumTestCase
-from passbook.flows.models import Flow
-from passbook.sources.oauth.models import OAuthSource
-
-TOKEN_URL = "http://127.0.0.1:5556/dex/token"
-CONFIG_PATH = "/tmp/dex.yml"
-
-
-class TestSourceOAuth(SeleniumTestCase):
-    """test OAuth Source flow"""
-
-    container: Container
-
-    def setUp(self):
-        self.client_secret = generate_client_secret()
-        self.container = self.setup_client()
-        super().setUp()
-
-    def prepare_dex_config(self):
-        """Since Dex does not document which environment
-        variables can be used to configure clients"""
-        config = {
-            "enablePasswordDB": True,
-            "issuer": "http://127.0.0.1:5556/dex",
-            "logger": {"level": "debug"},
-            "staticClients": [
-                {
-                    "id": "example-app",
-                    "name": "Example App",
-                    "redirectURIs": [
-                        self.url(
-                            "passbook_sources_oauth:oauth-client-callback",
-                            source_slug="dex",
-                        )
-                    ],
-                    "secret": self.client_secret,
-                }
-            ],
-            "staticPasswords": [
-                {
-                    "email": "admin@example.com",
-                    # hash for password
-                    "hash": "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W",
-                    "userID": "08a8684b-db88-4b73-90a9-3cd1661f5466",
-                    "username": "admin",
-                }
-            ],
-            "storage": {"config": {"file": "/tmp/dex.db"}, "type": "sqlite3"},
-            "web": {"http": "0.0.0.0:5556"},
-        }
-        with open(CONFIG_PATH, "w+") as _file:
-            safe_dump(config, _file)
-
-    def setup_client(self) -> Container:
-        """Setup test Dex container"""
-        self.prepare_dex_config()
-        client: DockerClient = from_env()
-        container = client.containers.run(
-            image="quay.io/dexidp/dex:v2.24.0",
-            detach=True,
-            network_mode="host",
-            auto_remove=True,
-            command="serve /config.yml",
-            healthcheck=Healthcheck(
-                test=["CMD", "wget", "--spider", "http://localhost:5556/dex/healthz"],
-                interval=5 * 100 * 1000000,
-                start_period=1 * 100 * 1000000,
-            ),
-            volumes={abspath(CONFIG_PATH): {"bind": "/config.yml", "mode": "ro",}},
-        )
-        while True:
-            container.reload()
-            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
-            if status == "healthy":
-                return container
-            sleep(1)
-
-    def create_objects(self):
-        """Create required objects"""
-        sleep(1)
-        # Bootstrap all needed objects
-        authentication_flow = Flow.objects.get(slug="default-source-authentication")
-        enrollment_flow = Flow.objects.get(slug="default-source-enrollment")
-
-        OAuthSource.objects.create(
-            name="dex",
-            slug="dex",
-            authentication_flow=authentication_flow,
-            enrollment_flow=enrollment_flow,
-            provider_type="openid-connect",
-            authorization_url="http://127.0.0.1:5556/dex/auth",
-            access_token_url=TOKEN_URL,
-            profile_url="http://127.0.0.1:5556/dex/userinfo",
-            consumer_key="example-app",
-            consumer_secret=self.client_secret,
-        )
-
-    def tearDown(self):
-        self.container.kill()
-        super().tearDown()
-
-    def test_oauth_enroll(self):
-        """test OAuth Source With With OIDC"""
-        self.create_objects()
-        self.driver.get(self.live_server_url)
-
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
-            )
-        )
-        self.driver.find_element(
-            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
-        ).click()
-
-        # Now we should be at the IDP, wait for the login field
-        self.wait.until(ec.presence_of_element_located((By.ID, "login")))
-        self.driver.find_element(By.ID, "login").send_keys("admin@example.com")
-        self.driver.find_element(By.ID, "password").send_keys("password")
-        self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
-
-        # Wait until we're logged in
-        self.wait.until(
-            ec.presence_of_element_located((By.CSS_SELECTOR, "button[type=submit]"))
-        )
-        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
-
-        # At this point we've been redirected back
-        # and we're asked for the username
-        self.driver.find_element(By.NAME, "username").click()
-        self.driver.find_element(By.NAME, "username").send_keys("foo")
-        self.driver.find_element(By.NAME, "username").send_keys(Keys.ENTER)
-
-        # Wait until we've loaded the user info page
-        self.wait.until(ec.presence_of_element_located((By.LINK_TEXT, "foo")))
-        self.driver.find_element(By.LINK_TEXT, "foo").click()
-
-        self.wait_for_url(self.url("passbook_core:user-settings"))
-        self.assertEqual(
-            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
-            "foo",
-        )
-        self.assertEqual(
-            self.driver.find_element(By.ID, "id_username").get_attribute("value"), "foo"
-        )
-        self.assertEqual(
-            self.driver.find_element(By.ID, "id_name").get_attribute("value"), "admin",
-        )
-        self.assertEqual(
-            self.driver.find_element(By.ID, "id_email").get_attribute("value"),
-            "admin@example.com",
-        )
-
-    def test_oauth_enroll_auth(self):
-        """test OAuth Source With With OIDC (enroll and authenticate again)"""
-        self.test_oauth_enroll()
-        # We're logged in at the end of this, log out and re-login
-        self.driver.find_element(By.CSS_SELECTOR, "[aria-label=logout]").click()
-
-        self.wait.until(
-            ec.presence_of_element_located(
-                (By.CLASS_NAME, "pf-c-login__main-footer-links-item-link")
-            )
-        )
-        self.driver.find_element(
-            By.CLASS_NAME, "pf-c-login__main-footer-links-item-link"
-        ).click()
-
-        # Now we should be at the IDP, wait for the login field
-        self.wait.until(ec.presence_of_element_located((By.ID, "login")))
-        self.driver.find_element(By.ID, "login").send_keys("admin@example.com")
-        self.driver.find_element(By.ID, "password").send_keys("password")
-        self.driver.find_element(By.ID, "password").send_keys(Keys.ENTER)
-
-        # Wait until we're logged in
-        self.wait.until(
-            ec.presence_of_element_located((By.CSS_SELECTOR, "button[type=submit]"))
-        )
-        self.driver.find_element(By.CSS_SELECTOR, "button[type=submit]").click()
-
-        # Wait until we've loaded the user info page
-        self.wait.until(ec.presence_of_element_located((By.LINK_TEXT, "foo")))
-        self.driver.find_element(By.LINK_TEXT, "foo").click()
-
-        self.wait_for_url(self.url("passbook_core:user-settings"))
-        self.assertEqual(
-            self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,
-            "foo",
-        )
-        self.assertEqual(
-            self.driver.find_element(By.ID, "id_username").get_attribute("value"), "foo"
-        )
-        self.assertEqual(
-            self.driver.find_element(By.ID, "id_name").get_attribute("value"), "admin",
-        )
-        self.assertEqual(
-            self.driver.find_element(By.ID, "id_email").get_attribute("value"),
-            "admin@example.com",
-        )

e2e/utils.py

@@ -1,18 +1,22 @@
 """passbook e2e testing utilities"""
-from functools import lru_cache
+from functools import wraps
 from glob import glob
 from importlib.util import module_from_spec, spec_from_file_location
 from inspect import getmembers, isfunction
 from os import environ, makedirs
-from time import time
+from time import sleep, time
+from typing import Any, Callable, Dict, Optional
 
-from Cryptodome.PublicKey import RSA
 from django.apps import apps
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
 from django.db import connection, transaction
 from django.db.utils import IntegrityError
 from django.shortcuts import reverse
+from django.test.testcases import TransactionTestCase
+from docker import DockerClient, from_env
+from docker.models.containers import Container
 from selenium import webdriver
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.support.ui import WebDriverWait
@@ -21,35 +25,47 @@ from structlog import get_logger
 from passbook.core.models import User
 
 
-@lru_cache
 # pylint: disable=invalid-name
 def USER() -> User:  # noqa
     """Cached function that always returns pbadmin"""
     return User.objects.get(username="pbadmin")
 
 
-def ensure_rsa_key():
-    """Ensure that at least one RSAKey Object exists, create one if none exist"""
-    from oidc_provider.models import RSAKey
-
-    if not RSAKey.objects.exists():
-        key = RSA.generate(2048)
-        rsakey = RSAKey(key=key.exportKey("PEM").decode("utf8"))
-        rsakey.save()
-
-
 class SeleniumTestCase(StaticLiveServerTestCase):
     """StaticLiveServerTestCase which automatically creates a Webdriver instance"""
 
+    container: Optional[Container] = None
+
     def setUp(self):
         super().setUp()
         makedirs("selenium_screenshots/", exist_ok=True)
         self.driver = self._get_driver()
         self.driver.maximize_window()
         self.driver.implicitly_wait(30)
-        self.wait = WebDriverWait(self.driver, 50)
+        self.wait = WebDriverWait(self.driver, 60)
         self.apply_default_data()
         self.logger = get_logger()
+        if specs := self.get_container_specs():
+            self.container = self._start_container(specs)
+
+    def _start_container(self, specs: Dict[str, Any]) -> Container:
+        client: DockerClient = from_env()
+        client.images.pull(specs["image"])
+        container = client.containers.run(**specs)
+        if "healthcheck" not in specs:
+            return container
+        while True:
+            container.reload()
+            status = container.attrs.get("State", {}).get("Health", {}).get("Status")
+            if status == "healthy":
+                return container
+            self.logger.info("Container failed healthcheck")
+            sleep(1)
+
+    def get_container_specs(self) -> Optional[Dict[str, Any]]:
+        """Optionally get container specs which will launched on setup, wait for the container to
+        be healthy, and deleted again on tearDown"""
+        return None
 
     def _get_driver(self) -> WebDriver:
         return webdriver.Remote(
@@ -58,7 +74,7 @@ class SeleniumTestCase(StaticLiveServerTestCase):
         )
 
     def tearDown(self):
-        if "CI" in environ:
+        if "TF_BUILD" in environ:
             screenshot_file = (
                 f"selenium_screenshots/{self.__class__.__name__}_{time()}.png"
             )
@@ -68,6 +84,8 @@ class SeleniumTestCase(StaticLiveServerTestCase):
             self.logger.warning(
                 line["message"], source=line["source"], level=line["level"]
             )
+        if self.container:
+            self.container.kill()
         self.driver.quit()
         super().tearDown()
 
@@ -108,3 +126,40 @@ class SeleniumTestCase(StaticLiveServerTestCase):
                             func(apps, schema_editor)
                         except IntegrityError:
                             pass
+
+
+def retry(max_retires=3, exceptions=None):
+    """Retry test multiple times. Default to catching Selenium Timeout Exception"""
+
+    if not exceptions:
+        exceptions = [TimeoutException, NoSuchElementException]
+
+    logger = get_logger()
+
+    def retry_actual(func: Callable):
+        """Retry test multiple times"""
+        count = 1
+
+        @wraps(func)
+        def wrapper(self: TransactionTestCase, *args, **kwargs):
+            """Run test again if we're below max_retries, including tearDown and
+            setUp. Otherwise raise the error"""
+            nonlocal count
+            try:
+                return func(self, *args, **kwargs)
+            # pylint: disable=catching-non-exception
+            except tuple(exceptions) as exc:
+                count += 1
+                if count > max_retires:
+                    logger.debug("Exceeded retry count", exc=exc, test=self)
+                    # pylint: disable=raising-non-exception
+                    raise exc
+                logger.debug("Retrying on error", exc=exc, test=self)
+                self.tearDown()
+                self._post_teardown()  # noqa
+                self.setUp()
+                return wrapper(self, *args, **kwargs)
+
+        return wrapper
+
+    return retry_actual

gatekeeper/Dockerfile

@@ -1,8 +0,0 @@
-FROM quay.io/oauth2-proxy/oauth2-proxy
-
-COPY templates /templates
-
-ENV OAUTH2_PROXY_EMAIL_DOMAINS=*
-ENV OAUTH2_PROXY_PROVIDER=oidc
-ENV OAUTH2_PROXY_CUSTOM_TEMPLATES_DIR=/templates
-ENV OAUTH2_PROXY_HTTP_ADDRESS=:4180

gatekeeper/templates/error.html

@@ -1,18 +0,0 @@
-{{define "error.html"}}
-<!DOCTYPE html>
-<html lang="en" charset="utf-8">
-
-<head>
-    <title>{{.Title}}</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
-</head>
-
-<body>
-    <h2>{{.Title}}</h2>
-    <p>{{.Message}}</p>
-    <hr>
-    <p><a href="{{.ProxyPrefix}}/sign_in">Sign In</a></p>
-</body>
-
-</html>
-{{end}}

gatekeeper/templates/sign_in.html

@@ -1,119 +0,0 @@
-{{define "sign_in.html"}}
-<!DOCTYPE html>
-<html lang="en" charset="utf-8">
-<head>
-    <title>Sign In with passbook</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
-    <style>
-        body {
-            font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-            font-size: 14px;
-            line-height: 1.42857143;
-            color: #333;
-            background: #f0f0f0;
-        }
-
-        .signin {
-            display: block;
-            margin: 20px auto;
-            max-width: 400px;
-            background: #fff;
-            border: 1px solid #ccc;
-            border-radius: 10px;
-            padding: 20px;
-        }
-
-        .center {
-            text-align: center;
-        }
-
-        .btn {
-            color: #fff;
-            background-color: #428bca;
-            border: 1px solid #357ebd;
-            -webkit-border-radius: 4;
-            -moz-border-radius: 4;
-            border-radius: 4px;
-            font-size: 14px;
-            padding: 6px 12px;
-            text-decoration: none;
-            cursor: pointer;
-        }
-
-        .btn:hover {
-            background-color: #3071a9;
-            border-color: #285e8e;
-            text-decoration: none;
-        }
-
-        label {
-            display: inline-block;
-            max-width: 100%;
-            margin-bottom: 5px;
-            font-weight: 700;
-        }
-
-        input {
-            display: block;
-            width: 100%;
-            height: 34px;
-            padding: 6px 12px;
-            font-size: 14px;
-            line-height: 1.42857143;
-            color: #555;
-            background-color: #fff;
-            background-image: none;
-            border: 1px solid #ccc;
-            border-radius: 4px;
-            -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075);
-            box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075);
-            -webkit-transition: border-color ease-in-out .15s, -webkit-box-shadow ease-in-out .15s;
-            -o-transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
-            transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s;
-            margin: 0;
-            box-sizing: border-box;
-        }
-
-        footer {
-            display: block;
-            font-size: 10px;
-            color: #aaa;
-            text-align: center;
-            margin-bottom: 10px;
-        }
-
-        footer a {
-            display: inline-block;
-            height: 25px;
-            line-height: 25px;
-            color: #aaa;
-            text-decoration: underline;
-        }
-
-        footer a:hover {
-            color: #aaa;
-        }
-    </style>
-</head>
-
-<body>
-    <div class="signin center">
-        <form method="GET" action="{{.ProxyPrefix}}/start">
-            <input type="hidden" name="rd" value="{{.Redirect}}">
-            <button type="submit" class="btn">Sign in with passbook</button><br />
-        </form>
-    </div>
-    <script>
-        if (window.location.hash) {
-            (function () {
-                var inputs = document.getElementsByName('rd');
-                for (var i = 0; i < inputs.length; i++) {
-                    inputs[i].value += window.location.hash;
-                }
-            })();
-        }
-    </script>
-</body>
-
-</html>
-{{end}}

helm/Chart.lock

@@ -1,9 +1,9 @@
 dependencies:
 - name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com/
-  version: 6.5.8
+  repository: https://charts.bitnami.com/bitnami
+  version: 9.3.2
 - name: redis
-  repository: https://kubernetes-charts.storage.googleapis.com/
-  version: 9.5.1
-digest: sha256:f18b5dc8d0be13d584407405c60d10b6b84d25f7fa8aaa3dd0e5385c38f5c516
-generated: "2019-12-14T13:33:48.4341939Z"
+  repository: https://charts.bitnami.com/bitnami
+  version: 10.7.16
+digest: sha256:fd31e2e2b9ff17a5ed906a77a4f15ffa1ab7f5aecaea1e5db77f0d199ae4f19e
+generated: "2020-08-25T17:57:49.684549+02:00"

helm/Chart.yaml

@@ -1,6 +1,17 @@
-apiVersion: v1
-appVersion: "0.9.0-rc1"
-description: A Helm chart for passbook.
+apiVersion: v2
+description: passbook is an open-source Identity Provider focused on flexibility and versatility. You can use passbook in an existing environment to add support for new protocols. passbook is also a great solution for implementing signup/recovery/etc in your application, so you don't have to deal with it.
 name: passbook
-version: "0.9.0-rc1"
-icon: https://git.beryju.org/uploads/-/system/project/avatar/108/logo.png
+home: https://passbook.beryju.org
+sources:
+  - https://github.com/BeryJu/passbook
+version: "0.12.11-stable"
+icon: https://raw.githubusercontent.com/BeryJu/passbook/master/website/static/img/logo.svg
+dependencies:
+  - name: postgresql
+    version: 9.4.1
+    repository: https://charts.bitnami.com/bitnami
+    condition: install.postgresql
+  - name: redis
+    version: 10.9.0
+    repository: https://charts.bitnami.com/bitnami
+    condition: install.redis

helm/README.md

@@ -0,0 +1,28 @@
+# passbook Helm Chart
+
+| Name                              | Default                 | Description |
+|-----------------------------------|-------------------------|-------------|
+| image.name                        | beryju/passbook         | Image used to run the passbook server and worker |
+| image.name_static                 | beryju/passbook-static  | Image used to run the passbook static server (CSS and JS Files) |
+| image.tag                         | 0.12.5-stable           | Image tag |
+| serverReplicas                    | 1                       | Replicas for the Server deployment |
+| workerReplicas                    | 1                       | Replicas for the Worker deployment |
+| kubernetesIntegration             | true                    | Enable/disable the Kubernetes integration for passbook. This will create a service account for passbook to create and update outposts in passbook |
+| config.secretKey                  |                         | Secret key used to sign session cookies, generate with `pwgen 50 1` for example. |
+| config.errorReporting.enabled     | false                   | Enable/disable error reporting |
+| config.errorReporting.environment | customer                | Environment sent with the error reporting |
+| config.errorReporting.sendPii     | false                   | Whether to send Personally-identifiable data with the error reporting |
+| config.logLevel                   | warning                 | Log level of passbook |
+| backup.accessKey                  |                         | Optionally enable S3 Backup, Access Key |
+| backup.secretKey                  |                         | Optionally enable S3 Backup, Secret Key |
+| backup.bucket                     |                         | Optionally enable S3 Backup, Bucket |
+| backup.region                     |                         | Optionally enable S3 Backup, Region |
+| backup.host                       |                         | Optionally enable S3 Backup, to custom Endpoint like minio |
+| ingress.annotations               | {}                      | Annotations for the ingress object |
+| ingress.hosts                     | [passbook.k8s.local]    | Hosts which the ingress will match |
+| ingress.tls                       | []                      | TLS Configuration, same as Ingress objects |
+| install.postgresql                | true                    | Enables/disables the packaged PostgreSQL Chart
+| install.redis                     | true                    | Enables/disables the packaged Redis Chart
+| postgresql.postgresqlPassword     |                         | Password used for PostgreSQL, generated automatically.
+
+For more info, see https://passbook.beryju.org/ and https://passbook.beryju.org/docs/installation/kubernetes/

helm/requirements.lock

@@ -1,9 +0,0 @@
-dependencies:
-- name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com/
-  version: 6.5.8
-- name: redis
-  repository: https://kubernetes-charts.storage.googleapis.com/
-  version: 9.5.1
-digest: sha256:476834fb82f66bc7242c4a5e7343d0a859d8307cb301256beb0eb749983014e4
-generated: "2019-11-07T10:21:30.902415+01:00"

helm/requirements.yaml

@@ -1,7 +0,0 @@
-dependencies:
-- name: postgresql
-  version: 6.5.8
-  repository: https://kubernetes-charts.storage.googleapis.com/
-- name: redis
-  version: 9.5.1
-  repository: https://kubernetes-charts.storage.googleapis.com/

helm/templates/NOTES.txt

@@ -1,19 +1,5 @@
-1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
+1. Access passbook using the following URL:
 {{- range .Values.ingress.hosts }}
   http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }}
 {{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "passbook.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get svc -w {{ include "passbook.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "passbook.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "passbook.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl port-forward $POD_NAME 8080:8000
-{{- end }}
+2. Login to passbook using the user "pbadmin" and the password "pbadmin".

helm/templates/_helpers.tpl

@@ -3,7 +3,7 @@
 Expand the name of the chart.
 */}}
 {{- define "passbook.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- default .Chart.Name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*
@@ -12,17 +12,13 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 If release name contains chart name it will be used as a full name.
 */}}
 {{- define "passbook.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- $name := default .Chart.Name -}}
 {{- if contains $name .Release.Name -}}
 {{- .Release.Name | trunc 63 | trimSuffix "-" -}}
 {{- else -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 {{- end -}}
-{{- end -}}
 
 {{/*
 Create chart name and version as used by the chart label.

helm/templates/configmap.yaml

@@ -3,26 +3,18 @@ kind: ConfigMap
 metadata:
   name: {{ include "passbook.fullname" . }}-config
 data:
-  config.yml: |
-    postgresql:
-      host: "{{ .Release.Name }}-postgresql"
-      name: "{{ .Values.postgresql.postgresqlDatabase }}"
-      user: postgres
-      {{- if .Values.backup }}
-      backup:
-        access_key: "{{ .Values.backup.access_key }}"
-        secret_key: "{{ .Values.backup.secret_key }}"
-        bucket: "{{ .Values.backup.bucket }}"
-        host: "{{ .Values.backup.host }}"
-      {{- end}}
-    redis:
-      host: "{{ .Release.Name }}-redis-master"
-      cache_db: 0
-      message_queue_db: 1
-    error_reporting: {{ .Values.config.error_reporting }}
-    log_level: "{{ .Values.config.log_level }}"
-    apm:
-      enabled: {{ .Values.config.apm.enabled }}
-      server_url: "{{ .Values.config.apm.server_url }}"
-      secret_token: "{{ .Values.config.apm.server_token }}"
-      verify_server_cert: {{ .Values.config.apm.verify_server_cert }}
+  POSTGRESQL__HOST: "{{ .Release.Name }}-postgresql"
+  POSTGRESQL__NAME: "{{ .Values.postgresql.postgresqlDatabase }}"
+  POSTGRESQL__USER: "{{ .Values.postgresql.postgresqlUsername }}"
+  {{- if .Values.backup }}
+  POSTGRESQL__S3_BACKUP__ACCESS_KEY: "{{ .Values.backup.accessKey }}"
+  POSTGRESQL__S3_BACKUP__SECRET_KEY: "{{ .Values.backup.secretKey }}"
+  POSTGRESQL__S3_BACKUP__BUCKET: "{{ .Values.backup.bucket }}"
+  POSTGRESQL__S3_BACKUP__REGION: "{{ .Values.backup.region }}"
+  POSTGRESQL__S3_BACKUP__HOST: "{{ .Values.backup.host }}"
+  {{- end}}
+  REDIS__HOST: "{{ .Release.Name }}-redis-master"
+  ERROR_REPORTING__ENABLED: "{{ .Values.config.errorReporting.enabled }}"
+  ERROR_REPORTING__ENVIRONMENT: "{{ .Values.config.errorReporting.environment }}"
+  ERROR_REPORTING__SEND_PII: "{{ .Values.config.errorReporting.sendPii }}"
+  LOG_LEVEL: "{{ .Values.config.logLevel }}"

helm/templates/ingress.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.ingress.enabled -}}
 {{- $fullName := include "passbook.fullname" . -}}
 apiVersion: extensions/v1beta1
 kind: Ingress
@@ -46,4 +45,3 @@ spec:
               serviceName: {{ $fullName }}-static
               servicePort: http
   {{- end }}
-{{- end }}

helm/templates/prom-rules.yaml

@@ -1,121 +0,0 @@
-{{- if .Values.monitoring.enabled -}}
----
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: {{ include "passbook.fullname" . }}-static-rules
-  labels:
-    app.kubernetes.io/name: {{ include "passbook.name" . }}
-    helm.sh/chart: {{ include "passbook.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-spec:
-  groups:
-  - name: Aggregate request counters
-    rules:
-      - record: job:django_http_requests_before_middlewares_total:sum_rate30s
-        expr: sum(rate(django_http_requests_before_middlewares_total[30s])) by (job)
-      - record: job:django_http_requests_unknown_latency_total:sum_rate30s
-        expr: sum(rate(django_http_requests_unknown_latency_total[30s])) by (job)
-      - record: job:django_http_ajax_requests_total:sum_rate30s
-        expr: sum(rate(django_http_ajax_requests_total[30s])) by (job)
-      - record: job:django_http_responses_before_middlewares_total:sum_rate30s
-        expr: sum(rate(django_http_responses_before_middlewares_total[30s])) by (job)
-      - record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s
-        expr: sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s])) by (job)
-      - record: job:django_http_requests_body_total_bytes:sum_rate30s
-        expr: sum(rate(django_http_requests_body_total_bytes[30s])) by (job)
-      - record: job:django_http_responses_streaming_total:sum_rate30s
-        expr: sum(rate(django_http_responses_streaming_total[30s])) by (job)
-      - record: job:django_http_responses_body_total_bytes:sum_rate30s
-        expr: sum(rate(django_http_responses_body_total_bytes[30s])) by (job)
-      - record: job:django_http_requests_total:sum_rate30s
-        expr: sum(rate(django_http_requests_total_by_method[30s])) by (job)
-      - record: job:django_http_requests_total_by_method:sum_rate30s
-        expr: sum(rate(django_http_requests_total_by_method[30s])) by (job,method)
-      - record: job:django_http_requests_total_by_transport:sum_rate30s
-        expr: sum(rate(django_http_requests_total_by_transport[30s])) by (job,transport)
-      - record: job:django_http_requests_total_by_view:sum_rate30s
-        expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) by (job,view)
-      - record: job:django_http_requests_total_by_view_transport_method:sum_rate30s
-        expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) by (job,view,transport,method)
-      - record: job:django_http_responses_total_by_templatename:sum_rate30s
-        expr: sum(rate(django_http_responses_total_by_templatename[30s])) by (job,templatename)
-      - record: job:django_http_responses_total_by_status:sum_rate30s
-        expr: sum(rate(django_http_responses_total_by_status[30s])) by (job,status)
-      - record: job:django_http_responses_total_by_status_name_method:sum_rate30s
-        expr: sum(rate(django_http_responses_total_by_status_name_method[30s])) by (job,status,name,method)
-      - record: job:django_http_responses_total_by_charset:sum_rate30s
-        expr: sum(rate(django_http_responses_total_by_charset[30s])) by (job,charset)
-      - record: job:django_http_exceptions_total_by_type:sum_rate30s
-        expr: sum(rate(django_http_exceptions_total_by_type[30s])) by (job,type)
-      - record: job:django_http_exceptions_total_by_view:sum_rate30s
-        expr: sum(rate(django_http_exceptions_total_by_view[30s])) by (job,view)
-  - name: Aggregate latency histograms
-    rules:
-      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
-        expr: histogram_quantile(0.50, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "50"
-      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
-        expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "95"
-      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
-        expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "99"
-      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
-        expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "99.9"
-      - record: job:django_http_requests_latency_seconds:quantile_rate30s
-        expr: histogram_quantile(0.50, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "50"
-      - record: job:django_http_requests_latency_seconds:quantile_rate30s
-        expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "95"
-      - record: job:django_http_requests_latency_seconds:quantile_rate30s
-        expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "99"
-      - record: job:django_http_requests_latency_seconds:quantile_rate30s
-        expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s])) by (job, le))
-        labels:
-          quantile: "99.9"
-  - name: Aggregate model operations
-    rules:
-      - record: job:django_model_inserts_total:sum_rate1m
-        expr: sum(rate(django_model_inserts_total[1m])) by (job, model)
-      - record: job:django_model_updates_total:sum_rate1m
-        expr: sum(rate(django_model_updates_total[1m])) by (job, model)
-      - record: job:django_model_deletes_total:sum_rate1m
-        expr: sum(rate(django_model_deletes_total[1m])) by (job, model)
-  - name: Aggregate database operations
-    rules:
-      - record: job:django_db_new_connections_total:sum_rate30s
-        expr: sum(rate(django_db_new_connections_total[30s])) by (alias, vendor)
-      - record: job:django_db_new_connection_errors_total:sum_rate30s
-        expr: sum(rate(django_db_new_connection_errors_total[30s])) by (alias, vendor)
-      - record: job:django_db_execute_total:sum_rate30s
-        expr: sum(rate(django_db_execute_total[30s])) by (alias, vendor)
-      - record: job:django_db_execute_many_total:sum_rate30s
-        expr: sum(rate(django_db_execute_many_total[30s])) by (alias, vendor)
-      - record: job:django_db_errors_total:sum_rate30s
-        expr: sum(rate(django_db_errors_total[30s])) by (alias, vendor, type)
-  - name: Aggregate migrations
-    rules:
-      - record: job:django_migrations_applied_total:max
-        expr: max(django_migrations_applied_total) by (job, connection)
-      - record: job:django_migrations_unapplied_total:max
-        expr: max(django_migrations_unapplied_total) by (job, connection)
-  - name: Alerts
-    rules:
-      - alert: UnappliedMigrations
-        expr: job:django_migrations_unapplied_total:max > 0
-        for: 1m
-        labels:
-          severity: testing
-{{- end }}

helm/templates/secret.yaml

@@ -5,8 +5,8 @@ metadata:
   name: {{ include "passbook.fullname" . }}-secret-key
 data:
   monitoring_username: bW9uaXRvcg== # monitor in base64
-  {{- if .Values.config.secret_key }}
-  secret_key: {{ .Values.config.secret_key | b64enc | quote }}
+  {{- if .Values.config.secretKey }}
+  secret_key: {{ .Values.config.secretKey | b64enc | quote }}
   {{- else }}
   secret_key: {{ randAlphaNum 50 | b64enc | quote}}
   {{- end }}

helm/templates/service-account.yaml

@@ -0,0 +1,64 @@
+{{- if .Values.kubernetesIntegration }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "passbook.fullname" . }}-sa-role
+rules:
+- apiGroups:
+    - ""
+  resources:
+    - secrets
+    - services
+  verbs:
+    - "get"
+    - "create"
+    - "delete"
+    - "read"
+    - "patch"
+- apiGroups:
+    - "extensions"
+    - "apps"
+  resources:
+    - "deployments"
+  verbs:
+    - "get"
+    - "create"
+    - "delete"
+    - "read"
+    - "patch"
+- apiGroups:
+    - "extensions"
+    - "networking.k8s.io"
+  resources:
+    - "ingresses"
+  verbs:
+    - "get"
+    - "create"
+    - "delete"
+    - "read"
+    - "patch"
+- apiGroups:
+    - ""
+  resources:
+    - namespaces
+  verbs:
+    - list
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "passbook.fullname" . }}-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "passbook.fullname" . }}-sa-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "passbook.fullname" . }}-sa-role
+subjects:
+- kind: ServiceAccount
+  name: {{ include "passbook.fullname" . }}-sa
+  namespace: {{ .Release.Namespace }}
+{{- end }}

helm/templates/static-deployment.yaml

@@ -7,23 +7,23 @@ metadata:
     helm.sh/chart: {{ include "passbook.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-    k8s.passbook.io/component: static
+    k8s.passbook.beryju.org/component: static
 spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "passbook.name" . }}
       app.kubernetes.io/instance: {{ .Release.Name }}
-      k8s.passbook.io/component: static
+      k8s.passbook.beryju.org/component: static
   template:
     metadata:
       labels:
         app.kubernetes.io/name: {{ include "passbook.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
-        k8s.passbook.io/component: static
+        k8s.passbook.beryju.org/component: static
     spec:
       containers:
         - name: {{ .Chart.Name }}-static
-          image: "beryju/passbook-static:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.name_static }}:{{ .Values.image.tag }}"
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

helm/templates/static-service.yaml

@@ -7,7 +7,7 @@ metadata:
     helm.sh/chart: {{ include "passbook.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-    k8s.passbook.io/component: static
+    k8s.passbook.beryju.org/component: static
 spec:
   type: ClusterIP
   ports:
@@ -18,4 +18,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ include "passbook.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-    k8s.passbook.io/component: static
+    k8s.passbook.beryju.org/component: static

helm/templates/static-sm.yaml

@@ -1,17 +0,0 @@
-{{- if .Values.monitoring.enabled -}}
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ include "passbook.name" . }}
-    helm.sh/chart: {{ include "passbook.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-  name: {{ include "passbook.fullname" . }}-static-monitoring
-spec:
-  endpoints:
-  - port: http
-  selector:
-    matchLabels:
-      k8s.passbook.io/component: static
-{{- end }}

helm/templates/web-deployment.yaml

@@ -7,35 +7,45 @@ metadata:
     helm.sh/chart: {{ include "passbook.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-    k8s.passbook.io/component: web
+    k8s.passbook.beryju.org/component: web
 spec:
-  replicas: 2
+  replicas: {{ .Values.serverReplicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "passbook.name" . }}
       app.kubernetes.io/instance: {{ .Release.Name }}
-      k8s.passbook.io/component: web
+      k8s.passbook.beryju.org/component: web
   template:
     metadata:
       labels:
         app.kubernetes.io/name: {{ include "passbook.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
-        k8s.passbook.io/component: web
+        k8s.passbook.beryju.org/component: web
     spec:
-      volumes:
-        - name: config-volume
-          configMap:
-            name: {{ include "passbook.fullname" . }}-config
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - {{ include "passbook.name" . }}
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - {{ .Release.Name }}
+                - key: k8s.passbook.beryju.org/component
+                  operator: In
+                  values:
+                  - web
+              topologyKey: "kubernetes.io/hostname"
       initContainers:
         - name: passbook-database-migrations
-          image: "beryju/passbook:{{ .Values.image.tag }}"
-          imagePullPolicy: Always
-          args:
-            - ./manage.py
-            - migrate
-          volumeMounts:
-            - mountPath: /etc/passbook
-              name: config-volume
+          image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
+          args: [migrate]
           envFrom:
             - configMapRef:
                 name: {{ include "passbook.fullname" . }}-config
@@ -58,14 +68,8 @@ spec:
                   key: postgresql-password
       containers:
         - name: {{ .Chart.Name }}
-          image: "beryju/passbook:{{ .Values.image.tag }}"
-          imagePullPolicy: Always
-          args:
-            - uwsgi
-            - uwsgi.ini
-          volumeMounts:
-            - mountPath: /etc/passbook
-              name: config-volume
+          image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
+          args: [server]
           envFrom:
             - configMapRef:
                 name: {{ include "passbook.fullname" . }}-config
@@ -74,18 +78,18 @@ spec:
             - name: PASSBOOK_SECRET_KEY
               valueFrom:
                 secretKeyRef:
-                  name: {{ include "passbook.fullname" . }}-secret-key
-                  key: secret_key
+                  name: "{{ include "passbook.fullname" . }}-secret-key"
+                  key: "secret_key"
             - name: PASSBOOK_REDIS__PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ .Release.Name }}-redis"
-                  key: redis-password
+                  key: "redis-password"
             - name: PASSBOOK_POSTGRESQL__PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ .Release.Name }}-postgresql"
-                  key: postgresql-password
+                  key: "postgresql-password"
           ports:
             - name: http
               containerPort: 8000
@@ -96,18 +100,18 @@ spec:
               port: http
               httpHeaders:
                 - name: Host
-                  value: kubernetes-healthcheck-host
+                  value: passbook-healthcheck-host
           readinessProbe:
             httpGet:
               path: /
               port: http
               httpHeaders:
                 - name: Host
-                  value: kubernetes-healthcheck-host
+                  value: passbook-healthcheck-host
           resources:
             requests:
               cpu: 100m
-              memory: 200M
+              memory: 300M
             limits:
               cpu: 300m
-              memory: 350M
+              memory: 500M

helm/templates/web-service.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     helm.sh/chart: {{ include "passbook.chart" . }}
-    k8s.passbook.io/component: web
+    k8s.passbook.beryju.org/component: web
 spec:
   type: ClusterIP
   ports:
@@ -18,4 +18,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ include "passbook.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-    k8s.passbook.io/component: web
+    k8s.passbook.beryju.org/component: web

helm/templates/web-sm.yaml

@@ -1,26 +0,0 @@
-{{- if .Values.monitoring.enabled -}}
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ include "passbook.name" . }}
-    helm.sh/chart: {{ include "passbook.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-  name: {{ include "passbook.fullname" . }}-web-monitoring
-spec:
-  endpoints:
-  - basicAuth:
-      password:
-        name: {{ include "passbook.fullname" . }}-secret-key
-        key: secret_key
-      username:
-        name: {{ include "passbook.fullname" . }}-secret-key
-        key: monitoring_username
-    port: http
-    path: /metrics/
-    interval: 10s
-  selector:
-    matchLabels:
-      k8s.passbook.io/component: web
-{{- end }}

helm/templates/worker-deployment.yaml

@@ -7,64 +7,73 @@ metadata:
     helm.sh/chart: {{ include "passbook.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-    k8s.passbook.io/component: worker
+    k8s.passbook.beryju.org/component: worker
 spec:
-  replicas: 1
+  replicas: {{ .Values.workerReplicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "passbook.name" . }}
       app.kubernetes.io/instance: {{ .Release.Name }}
-      k8s.passbook.io/component: worker
+      k8s.passbook.beryju.org/component: worker
   template:
     metadata:
       labels:
         app.kubernetes.io/name: {{ include "passbook.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
-        k8s.passbook.io/component: worker
+        k8s.passbook.beryju.org/component: worker
     spec:
-      volumes:
-        - name: config-volume
-          configMap:
-            name: {{ include "passbook.fullname" . }}-config
+      {{- if .Values.kubernetesIntegration }}
+      serviceAccountName: {{ include "passbook.fullname" . }}-sa
+      {{- end }}
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - {{ include "passbook.name" . }}
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - {{ .Release.Name }}
+                - key: k8s.passbook.beryju.org/component
+                  operator: In
+                  values:
+                  - worker
+              topologyKey: "kubernetes.io/hostname"
       containers:
         - name: {{ .Chart.Name }}
-          image: "beryju/passbook:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
           imagePullPolicy: IfNotPresent
-          args:
-            - celery
-            - worker
-            - --autoscale=10,3
-            - -E
-            - -B
-            - -A=passbook.root.celery
-            - -s=/tmp/celerybeat-schedule
-          volumeMounts:
-            - mountPath: /etc/passbook
-              name: config-volume
+          args: [worker]
           envFrom:
             - configMapRef:
-                name: {{ include "passbook.fullname" . }}-config
-              prefix: PASSBOOK_
+                name: "{{ include "passbook.fullname" . }}-config"
+              prefix: "PASSBOOK_"
           env:
             - name: PASSBOOK_SECRET_KEY
               valueFrom:
                 secretKeyRef:
-                  name: {{ include "passbook.fullname" . }}-secret-key
+                  name: "{{ include "passbook.fullname" . }}-secret-key"
                   key: secret_key
             - name: PASSBOOK_REDIS__PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ .Release.Name }}-redis"
-                  key: redis-password
+                  key: "redis-password"
             - name: PASSBOOK_POSTGRESQL__PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: "{{ .Release.Name }}-postgresql"
-                  key: postgresql-password
+                  key: "postgresql-password"
           resources:
             requests:
               cpu: 150m
-              memory: 300M
+              memory: 400M
             limits:
               cpu: 300m
-              memory: 500M
+              memory: 600M

helm/values.test.yaml

@@ -0,0 +1,21 @@
+image:
+  tag: gh-master
+
+serverReplicas: 1
+workerReplicas: 1
+
+config:
+  # Log level used by web and worker
+  # Can be either debug, info, warning, error
+  logLevel: debug
+
+ingress:
+  hosts:
+    - passbook.127.0.0.1.nip.io
+
+# These values influence the bundled postgresql and redis charts, but are also used by passbook to connect
+postgresql:
+  postgresqlPassword: EK-5jnKfjrGRm<77
+
+redis:
+  password: password

helm/values.yaml

@@ -1,44 +1,41 @@
-# Default values for passbook.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
+###################################
+# Values directly affecting passbook
+###################################
 image:
-  tag: 0.9.0-rc1
+  name: beryju/passbook
+  name_static: beryju/passbook-static
+  tag: 0.12.11-stable
 
-nameOverride: ""
+serverReplicas: 1
+workerReplicas: 1
+
+# Enable the Kubernetes integration which lets passbook deploy outposts into kubernetes
+kubernetesIntegration: true
 
 config:
   # Optionally specify fixed secret_key, otherwise generated automatically
-  # secret_key: _k*@6h2u2@q-dku57hhgzb7tnx*ba9wodcb^s9g0j59@=y(@_o
+  # secretKey: _k*@6h2u2@q-dku57hhgzb7tnx*ba9wodcb^s9g0j59@=y(@_o
   # Enable error reporting
-  error_reporting: false
+  errorReporting:
+    enabled: false
+    environment: customer
+    sendPii: false
   # Log level used by web and worker
   # Can be either debug, info, warning, error
-  log_level: warning
-  # Optionally enable Elastic APM Support
-  apm:
-    enabled: false
-    server_url: ""
-    secret_token: ""
-    verify_server_cert: true
-
-# This Helm chart ships with built-in Prometheus ServiceMonitors and Rules.
-# This requires the CoreOS Prometheus Operator.
-monitoring:
-  enabled: false
+  logLevel: warning
 
 # Enable Database Backups to S3
 # backup:
-#   access_key: access-key
-#   secret_key: secret-key
+#   accessKey: access-key
+#   secretKey: secret-key
 #   bucket: s3-bucket
+#   region: eu-central-1
 #   host: s3-host
 
 ingress:
-  enabled: false
   annotations: {}
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
-  path: /
   hosts:
     - passbook.k8s.local
   tls: []
@@ -46,7 +43,15 @@ ingress:
   #    hosts:
   #      - passbook.k8s.local
 
-# These settings configure the packaged PostgreSQL and Redis chart.
+###################################
+# Values controlling dependencies
+###################################
+
+install:
+  postgresql: true
+  redis: true
+
+# These values influence the bundled postgresql and redis charts, but are also used by passbook to connect
 postgresql:
   postgresqlDatabase: passbook
 
@@ -54,7 +59,5 @@ redis:
   cluster:
     enabled: false
   master:
-    persistence:
-      enabled: false
     # https://stackoverflow.com/a/59189742
     disableCommands: []

lifecycle/bootstrap.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+python -m lifecycle.wait_for_db
+printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@" > /dev/stderr
+if [[ "$1" == "server" ]]; then
+    gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
+elif [[ "$1" == "worker" ]]; then
+    celery -A passbook.root.celery worker --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled
+elif [[ "$1" == "migrate" ]]; then
+    # Run system migrations first, run normal migrations after
+    python -m lifecycle.migrate
+    python -m manage migrate
+elif [[ "$1" == "backup" ]]; then
+    python -m manage dbbackup --clean
+elif [[ "$1" == "restore" ]]; then
+    python -m manage dbrestore ${@:2}
+elif [[ "$1" == "bash" ]]; then
+    /bin/bash
+else
+    python -m manage "$@"
+fi

lifecycle/gunicorn.conf.py

@@ -0,0 +1,57 @@
+"""Gunicorn config"""
+import os
+import warnings
+from multiprocessing import cpu_count
+from pathlib import Path
+
+import structlog
+
+bind = "0.0.0.0:8000"
+
+user = "passbook"
+group = "passbook"
+
+worker_class = "uvicorn.workers.UvicornWorker"
+# Docker containers don't have /tmp as tmpfs
+worker_tmp_dir = "/dev/shm"
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "passbook.root.settings")
+
+logconfig_dict = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "json_formatter": {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.processors.JSONRenderer(),
+            "foreign_pre_chain": [
+                structlog.stdlib.add_log_level,
+                structlog.stdlib.add_logger_name,
+                structlog.processors.TimeStamper(),
+                structlog.processors.StackInfoRenderer(),
+                structlog.processors.format_exc_info,
+            ],
+        }
+    },
+    "handlers": {
+        "error_console": {
+            "class": "logging.StreamHandler",
+            "formatter": "json_formatter",
+        },
+        "console": {"class": "logging.StreamHandler", "formatter": "json_formatter"},
+    },
+    "loggers": {
+        "uvicorn": {"handlers": ["console"], "level": "WARNING", "propagate": False},
+        "gunicorn": {"handlers": ["console"], "level": "INFO", "propagate": False},
+    },
+}
+
+# if we're running in kubernetes, use fixed workers because we can scale with more pods
+# otherwise (assume docker-compose), use as much as we can
+if Path("/var/run/secrets/kubernetes.io").exists():
+    workers = 2
+else:
+    worker = cpu_count() * 2 + 1
+threads = 4
+
+warnings.simplefilter("once")

lifecycle/migrate.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""System Migration handler"""
+from importlib.util import module_from_spec, spec_from_file_location
+from inspect import getmembers, isclass
+from pathlib import Path
+from typing import Any
+
+from psycopg2 import connect
+from structlog import get_logger
+
+from passbook.lib.config import CONFIG
+
+LOGGER = get_logger()
+
+
+class BaseMigration:
+    """Base System Migration"""
+
+    cur: Any
+    con: Any
+
+    def __init__(self, cur: Any, con: Any):
+        self.cur = cur
+        self.con = con
+
+    def needs_migration(self) -> bool:
+        """Return true if Migration needs to be run"""
+        return False
+
+    def run(self):
+        """Run the actual migration"""
+
+
+if __name__ == "__main__":
+
+    conn = connect(
+        dbname=CONFIG.y("postgresql.name"),
+        user=CONFIG.y("postgresql.user"),
+        password=CONFIG.y("postgresql.password"),
+        host=CONFIG.y("postgresql.host"),
+    )
+    curr = conn.cursor()
+
+    for migration in Path(__file__).parent.absolute().glob("system_migrations/*.py"):
+        spec = spec_from_file_location("lifecycle.system_migrations", migration)
+        mod = module_from_spec(spec)
+        # pyright: reportGeneralTypeIssues=false
+        spec.loader.exec_module(mod)
+
+        for name, sub in getmembers(mod, isclass):
+            if name != "Migration":
+                continue
+            migration = sub(curr, conn)
+            if migration.needs_migration():
+                LOGGER.info("Migration needs to be applied", migration=sub)
+                migration.run()
+                LOGGER.info("Migration finished applying", migration=sub)

lifecycle/system_migrations/to_0_10.py

@@ -0,0 +1,50 @@
+from os import system
+
+from lifecycle.migrate import BaseMigration
+
+SQL_STATEMENT = """delete from django_migrations where app = 'passbook_stages_prompt';
+drop table passbook_stages_prompt_prompt cascade;
+drop table passbook_stages_prompt_promptstage cascade;
+drop table passbook_stages_prompt_promptstage_fields;
+drop table corsheaders_corsmodel cascade;
+drop table oauth2_provider_accesstoken cascade;
+drop table oauth2_provider_grant cascade;
+drop table oauth2_provider_refreshtoken cascade;
+drop table oidc_provider_client cascade;
+drop table oidc_provider_client_response_types cascade;
+drop table oidc_provider_code cascade;
+drop table oidc_provider_responsetype cascade;
+drop table oidc_provider_rsakey cascade;
+drop table oidc_provider_token cascade;
+drop table oidc_provider_userconsent cascade;
+drop table passbook_providers_app_gw_applicationgatewayprovider cascade;
+delete from django_migrations where app = 'passbook_flows' and name = '0008_default_flows';
+delete from django_migrations where app = 'passbook_flows' and name = '0009_source_flows';
+delete from django_migrations where app = 'passbook_flows' and name = '0010_provider_flows';
+delete from django_migrations where app = 'passbook_stages_password' and
+name = '0002_passwordstage_change_flow';"""
+
+
+class Migration(BaseMigration):
+    def needs_migration(self) -> bool:
+        self.cur.execute(
+            "select * from information_schema.tables where table_name='oidc_provider_client'"
+        )
+        return bool(self.cur.rowcount)
+
+    def system_crit(self, command):
+        retval = system(command)  # nosec
+        if retval != 0:
+            raise Exception("Migration error")
+
+    def run(self):
+        self.cur.execute(SQL_STATEMENT)
+        self.con.commit()
+        self.system_crit("./manage.py migrate passbook_stages_prompt")
+        self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake")
+        self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake")
+        self.system_crit(
+            "./manage.py migrate passbook_flows 0010_provider_flows --fake"
+        )
+        self.system_crit("./manage.py migrate passbook_flows")
+        self.system_crit("./manage.py migrate passbook_stages_password --fake")

lifecycle/wait_for_db.py

@@ -1,16 +1,28 @@
 #!/usr/bin/env python
 """This file needs to be run from the root of the project to correctly
 import passbook. This is done by the dockerfile."""
+from json import dumps
+from sys import stderr
 from time import sleep
 
 from psycopg2 import OperationalError, connect
 from redis import Redis
 from redis.exceptions import RedisError
-from structlog import get_logger
 
 from passbook.lib.config import CONFIG
 
-LOGGER = get_logger()
+
+def j_print(event: str, log_level: str = "info", **kwargs):
+    """Print event in the same format as structlog with JSON.
+    Used before structlog is configured."""
+    data = {
+        "event": event,
+        "level": log_level,
+        "logger": __name__,
+    }
+    data.update(**kwargs)
+    print(dumps(data), file=stderr)
+
 
 while True:
     try:
@@ -24,7 +36,7 @@ while True:
         break
     except OperationalError:
         sleep(1)
-        LOGGER.warning("PostgreSQL Connection failed, retrying...")
+        j_print("PostgreSQL Connection failed, retrying...")
 
 while True:
     try:
@@ -38,4 +50,4 @@ while True:
         break
     except RedisError:
         sleep(1)
-        LOGGER.warning("Redis Connection failed, retrying...")
+        j_print("Redis Connection failed, retrying...")

mkdocs.yml

@@ -1,73 +0,0 @@
-site_name: passbook Docs
-site_url: https://passbook.beryju.org/
-copyright: "Copyright © 2019 - 2020 BeryJu.org"
-
-nav:
-  - Home: index.md
-  - Terminology: terminology.md
-  - Installation:
-    - docker-compose: installation/docker-compose.md
-    - Kubernetes: installation/kubernetes.md
-  - Flows:
-      Overview: flow/flows.md
-      Examples:
-        - Login: flow/examples/login.md
-  - Stages:
-      - Captcha Stage: flow/stages/captcha/index.md
-      - Dummy Stage: flow/stages/dummy/index.md
-      - Email Stage: flow/stages/email/index.md
-      - Identification Stage: flow/stages/identification/index.md
-      - Invitation Stage: flow/stages/invitation/index.md
-      - OTP Stage: flow/stages/otp/index.md
-      - Password Stage: flow/stages/password/index.md
-      - Prompt Stage: flow/stages/prompt/index.md
-      - Prompt Stage Validation: flow/stages/prompt/validation.md
-      - User Delete Stage: flow/stages/user_delete.md
-      - User Login Stage: flow/stages/user_login.md
-      - User Logout Stage: flow/stages/user_logout.md
-      - User Write Stage: flow/stages/user_write.md
-  - Sources: sources.md
-  - Providers: providers.md
-  - Expressions:
-    - Overview: expressions/index.md
-    - Reference:
-      - User Object: expressions/reference/user-object.md
-  - Property Mappings:
-    - Overview: property-mappings/index.md
-    - Expressions: property-mappings/expression.md
-  - Policies:
-    - Overview: policies/index.md
-    - Expression: policies/expression.md
-  - Integrations:
-    - as Provider:
-        - Amazon Web Services: integrations/services/aws/index.md
-        - GitLab: integrations/services/gitlab/index.md
-        - Rancher: integrations/services/rancher/index.md
-        - Harbor: integrations/services/harbor/index.md
-        - Sentry: integrations/services/sentry/index.md
-        - Ansible Tower/AWX: integrations/services/tower-awx/index.md
-  - Upgrading from v0.8.x: upgrading-from-0.8.x.md
-
-repo_name: "BeryJu/passbook"
-repo_url: https://github.com/BeryJu/passbook
-theme:
-  name: material
-  logo: images/logo.svg
-  favicon: images/logo.svg
-  palette:
-    scheme: slate
-    primary: white
-
-markdown_extensions:
-  - toc:
-      permalink: "¶"
-  - admonition
-  - codehilite
-  - pymdownx.betterem:
-      smart_enable: all
-  - pymdownx.inlinehilite
-  - pymdownx.magiclink
-  - attr_list
-
-plugins:
-  - search

passbook/__init__.py

@@ -1,2 +1,2 @@
 """passbook"""
-__version__ = "0.9.0-rc1"
+__version__ = "0.12.11-stable"

passbook/admin/api/overview.py

@@ -0,0 +1,80 @@
+"""passbook administration overview"""
+from django.core.cache import cache
+from django.http import response
+from drf_yasg2.utils import swagger_auto_schema
+from rest_framework.fields import SerializerMethodField
+from rest_framework.permissions import IsAdminUser
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.serializers import Serializer
+from rest_framework.viewsets import ViewSet
+
+from passbook import __version__
+from passbook.admin.tasks import VERSION_CACHE_KEY, update_latest_version
+from passbook.core.models import Provider
+from passbook.policies.models import Policy
+from passbook.root.celery import CELERY_APP
+
+
+class AdministrationOverviewSerializer(Serializer):
+    """Overview View"""
+
+    version = SerializerMethodField()
+    version_latest = SerializerMethodField()
+    worker_count = SerializerMethodField()
+    providers_without_application = SerializerMethodField()
+    policies_without_binding = SerializerMethodField()
+    cached_policies = SerializerMethodField()
+    cached_flows = SerializerMethodField()
+
+    def get_version(self, _) -> str:
+        """Get current version"""
+        return __version__
+
+    def get_version_latest(self, _) -> str:
+        """Get latest version from cache"""
+        version_in_cache = cache.get(VERSION_CACHE_KEY)
+        if not version_in_cache:
+            update_latest_version.delay()
+            return __version__
+        return version_in_cache
+
+    def get_worker_count(self, _) -> int:
+        """Ping workers"""
+        return len(CELERY_APP.control.ping(timeout=0.5))
+
+    def get_providers_without_application(self, _) -> int:
+        """Count of providers without application"""
+        return len(Provider.objects.filter(application=None))
+
+    def get_policies_without_binding(self, _) -> int:
+        """Count of policies not bound or use in prompt stages"""
+        return len(
+            Policy.objects.filter(bindings__isnull=True, promptstage__isnull=True)
+        )
+
+    def get_cached_policies(self, _) -> int:
+        """Get cached policy count"""
+        return len(cache.keys("policy_*"))
+
+    def get_cached_flows(self, _) -> int:
+        """Get cached flow count"""
+        return len(cache.keys("flow_*"))
+
+    def create(self, request: Request) -> response:
+        raise NotImplementedError
+
+    def update(self, request: Request) -> Response:
+        raise NotImplementedError
+
+
+class AdministrationOverviewViewSet(ViewSet):
+    """Return single instance of AdministrationOverviewSerializer"""
+
+    permission_classes = [IsAdminUser]
+
+    @swagger_auto_schema(responses={200: AdministrationOverviewSerializer(many=True)})
+    def list(self, request: Request) -> Response:
+        """Return single instance of AdministrationOverviewSerializer"""
+        serializer = AdministrationOverviewSerializer(True)
+        return Response(serializer.data)

passbook/admin/api/overview_metrics.py

@@ -0,0 +1,80 @@
+"""passbook administration overview"""
+import time
+from collections import Counter
+from datetime import timedelta
+from typing import Dict, List
+
+from django.db.models import Count, ExpressionWrapper, F
+from django.db.models.fields import DurationField
+from django.db.models.functions import ExtractHour
+from django.http import response
+from django.utils.timezone import now
+from drf_yasg2.utils import swagger_auto_schema
+from rest_framework.fields import SerializerMethodField
+from rest_framework.permissions import IsAdminUser
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.serializers import Serializer
+from rest_framework.viewsets import ViewSet
+
+from passbook.audit.models import Event, EventAction
+
+
+class AdministrationMetricsSerializer(Serializer):
+    """Overview View"""
+
+    logins_per_1h = SerializerMethodField()
+    logins_failed_per_1h = SerializerMethodField()
+
+    def get_events_per_1h(self, action: str) -> List[Dict[str, int]]:
+        """Get event count by hour in the last day, fill with zeros"""
+        date_from = now() - timedelta(days=1)
+        result = (
+            Event.objects.filter(action=action, created__gte=date_from)
+            .annotate(
+                age=ExpressionWrapper(
+                    now() - F("created"), output_field=DurationField()
+                )
+            )
+            .annotate(age_hours=ExtractHour("age"))
+            .values("age_hours")
+            .annotate(count=Count("pk"))
+            .order_by("age_hours")
+        )
+        data = Counter({d["age_hours"]: d["count"] for d in result})
+        results = []
+        _now = now()
+        for hour in range(0, -24, -1):
+            results.append(
+                {
+                    "x": time.mktime((_now + timedelta(hours=hour)).timetuple()) * 1000,
+                    "y": data[hour * -1],
+                }
+            )
+        return results
+
+    def get_logins_per_1h(self, _):
+        """Get successful logins per hour for the last 24 hours"""
+        return self.get_events_per_1h(EventAction.LOGIN)
+
+    def get_logins_failed_per_1h(self, _):
+        """Get failed logins per hour for the last 24 hours"""
+        return self.get_events_per_1h(EventAction.LOGIN_FAILED)
+
+    def create(self, request: Request) -> response:
+        raise NotImplementedError
+
+    def update(self, request: Request) -> Response:
+        raise NotImplementedError
+
+
+class AdministrationMetricsViewSet(ViewSet):
+    """Return single instance of AdministrationMetricsSerializer"""
+
+    permission_classes = [IsAdminUser]
+
+    @swagger_auto_schema(responses={200: AdministrationMetricsSerializer(many=True)})
+    def list(self, request: Request) -> Response:
+        """Return single instance of AdministrationMetricsSerializer"""
+        serializer = AdministrationMetricsSerializer(True)
+        return Response(serializer.data)

passbook/admin/api/tasks.py

@@ -0,0 +1,72 @@
+"""Tasks API"""
+from importlib import import_module
+
+from django.contrib import messages
+from django.http.response import Http404
+from django.utils.translation import gettext_lazy as _
+from drf_yasg2.utils import swagger_auto_schema
+from rest_framework.decorators import action
+from rest_framework.fields import CharField, DateTimeField, IntegerField, ListField
+from rest_framework.permissions import IsAdminUser
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.serializers import Serializer
+from rest_framework.viewsets import ViewSet
+
+from passbook.lib.tasks import TaskInfo
+
+
+class TaskSerializer(Serializer):
+    """Serialize TaskInfo and TaskResult"""
+
+    task_name = CharField()
+    task_description = CharField()
+    task_finish_timestamp = DateTimeField(source="finish_timestamp")
+
+    status = IntegerField(source="result.status.value")
+    messages = ListField(source="result.messages")
+
+    def create(self, request: Request) -> Response:
+        raise NotImplementedError
+
+    def update(self, request: Request) -> Response:
+        raise NotImplementedError
+
+
+class TaskViewSet(ViewSet):
+    """Read-only view set that returns all background tasks"""
+
+    permission_classes = [IsAdminUser]
+
+    @swagger_auto_schema(responses={200: TaskSerializer(many=True)})
+    def list(self, request: Request) -> Response:
+        """List current messages and pass into Serializer"""
+        return Response(TaskSerializer(TaskInfo.all().values(), many=True).data)
+
+    @action(detail=True, methods=["post"])
+    # pylint: disable=invalid-name
+    def retry(self, request: Request, pk=None) -> Response:
+        """Retry task"""
+        task = TaskInfo.by_name(pk)
+        if not task:
+            raise Http404
+        try:
+            task_module = import_module(task.task_call_module)
+            task_func = getattr(task_module, task.task_call_func)
+            task_func.delay(*task.task_call_args, **task.task_call_kwargs)
+            messages.success(
+                self.request,
+                _(
+                    "Successfully re-scheduled Task %(name)s!"
+                    % {"name": task.task_name}
+                ),
+            )
+            return Response(
+                {
+                    "successful": True,
+                }
+            )
+        except ImportError:
+            # if we get an import error, the module path has probably changed
+            task.delete()
+            return Response({"successful": False})

passbook/admin/fields.py

@@ -1,9 +1,35 @@
-"""YAML fields"""
+"""Additional fields"""
 import yaml
 from django import forms
+from django.utils.datastructures import MultiValueDict
 from django.utils.translation import gettext_lazy as _
 
 
+class ArrayFieldSelectMultiple(forms.SelectMultiple):
+    """This is a Form Widget for use with a Postgres ArrayField. It implements
+    a multi-select interface that can be given a set of `choices`.
+    You can provide a `delimiter` keyword argument to specify the delimeter used.
+
+    https://gist.github.com/stephane/00e73c0002de52b1c601"""
+
+    def __init__(self, *args, **kwargs):
+        # Accept a `delimiter` argument, and grab it (defaulting to a comma)
+        self.delimiter = kwargs.pop("delimiter", ",")
+        super().__init__(*args, **kwargs)
+
+    def value_from_datadict(self, data, files, name):
+        if isinstance(data, MultiValueDict):
+            # Normally, we'd want a list here, which is what we get from the
+            # SelectMultiple superclass, but the SimpleArrayField expects to
+            # get a delimited string, so we're doing a little extra work.
+            return self.delimiter.join(data.getlist(name))
+
+        return data.get(name)
+
+    def get_context(self, name, value, attrs):
+        return super().get_context(name, value.split(self.delimiter), attrs)
+
+
 class CodeMirrorWidget(forms.Textarea):
     """Custom Textarea-based Widget that triggers a CodeMirror editor"""
 
@@ -15,11 +41,8 @@ class CodeMirrorWidget(forms.Textarea):
         self.mode = mode
 
     def render(self, *args, **kwargs):
-        if "attrs" not in kwargs:
-            kwargs["attrs"] = {}
-        attrs = kwargs["attrs"]
-        if "class" not in attrs:
-            attrs["class"] = ""
+        attrs = kwargs.setdefault("attrs", {})
+        attrs.setdefault("class", "")
         attrs["class"] += " codemirror"
         attrs["data-cm-mode"] = self.mode
         return super().render(*args, **kwargs)
@@ -33,7 +56,7 @@ class YAMLString(str):
     """YAML String type"""
 
 
-class YAMLField(forms.CharField):
+class YAMLField(forms.JSONField):
     """Django's JSON Field converted to YAML"""
 
     default_error_messages = {
@@ -52,10 +75,14 @@ class YAMLField(forms.CharField):
             converted = yaml.safe_load(value)
         except yaml.YAMLError:
             raise forms.ValidationError(
-                self.error_messages["invalid"], code="invalid", params={"value": value},
+                self.error_messages["invalid"],
+                code="invalid",
+                params={"value": value},
             )
         if isinstance(converted, str):
             return YAMLString(converted)
+        if converted is None:
+            return {}
         return converted
 
     def bound_data(self, data, initial):

passbook/admin/forms/users.py

@@ -12,7 +12,7 @@ class UserForm(forms.ModelForm):
     class Meta:
 
         model = User
-        fields = ["username", "name", "email", "is_staff", "is_active", "attributes"]
+        fields = ["username", "name", "email", "is_active", "attributes"]
         widgets = {
             "name": forms.TextInput,
             "attributes": CodeMirrorWidget,

passbook/admin/middleware.py

@@ -1,26 +0,0 @@
-"""passbook admin Middleware to impersonate users"""
-
-from passbook.core.models import User
-
-
-def impersonate(get_response):
-    """Middleware to impersonate users"""
-
-    def middleware(request):
-        """Middleware to impersonate users"""
-
-        # User is superuser and has __impersonate ID set
-        if request.user.is_superuser and "__impersonate" in request.GET:
-            request.session["impersonate_id"] = request.GET["__impersonate"]
-        # user wants to stop impersonation
-        elif "__unimpersonate" in request.GET and "impersonate_id" in request.session:
-            del request.session["impersonate_id"]
-
-        # Actually impersonate user
-        if request.user.is_superuser and "impersonate_id" in request.session:
-            request.user = User.objects.get(pk=request.session["impersonate_id"])
-
-        response = get_response(request)
-        return response
-
-    return middleware

passbook/admin/settings.py

@@ -1,5 +1,10 @@
 """passbook admin settings"""
+from celery.schedules import crontab
 
-MIDDLEWARE = [
-    "passbook.admin.middleware.impersonate",
-]
+CELERY_BEAT_SCHEDULE = {
+    "admin_latest_version": {
+        "task": "passbook.admin.tasks.update_latest_version",
+        "schedule": crontab(minute=0),  # Run every hour
+        "options": {"queue": "passbook_scheduled"},
+    }
+}

passbook/admin/tasks.py

@@ -0,0 +1,30 @@
+"""passbook admin tasks"""
+from django.core.cache import cache
+from requests import RequestException, get
+from structlog import get_logger
+
+from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
+from passbook.root.celery import CELERY_APP
+
+LOGGER = get_logger()
+VERSION_CACHE_KEY = "passbook_latest_version"
+VERSION_CACHE_TIMEOUT = 2 * 60 * 60  # 2 hours
+
+
+@CELERY_APP.task(bind=True, base=MonitoredTask)
+def update_latest_version(self: MonitoredTask):
+    """Update latest version info"""
+    try:
+        data = get(
+            "https://api.github.com/repos/beryju/passbook/releases/latest"
+        ).json()
+        tag_name = data.get("tag_name")
+        cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], VERSION_CACHE_TIMEOUT)
+        self.set_status(
+            TaskResult(
+                TaskResultStatus.SUCCESSFUL, ["Successfully updated latest Version"]
+            )
+        )
+    except (RequestException, IndexError) as exc:
+        cache.set(VERSION_CACHE_KEY, "0.0.0", VERSION_CACHE_TIMEOUT)
+        self.set_status(TaskResult(TaskResultStatus.ERROR).with_error(exc))

passbook/admin/templates/administration/application/list.html

@@ -18,6 +18,7 @@
         {% if object_list %}
         <div class="pf-c-toolbar">
             <div class="pf-c-toolbar__content">
+                {% include 'partials/toolbar_search.html' %}
                 <div class="pf-c-toolbar__bulk-select">
                     <a href="{% url 'passbook_admin:application-create' %}?back={{ request.get_full_path }}" class="pf-c-button pf-m-primary" type="button">{% trans 'Create' %}</a>
                 </div>
@@ -62,18 +63,27 @@
                 {% endfor %}
             </tbody>
         </table>
-        <div class="pf-c-toolbar" id="page-layout-table-simple-toolbar-bottom">
+        <div class="pf-c-pagination pf-m-bottom">
             {% include 'partials/pagination.html' %}
         </div>
         {% else %}
+        <div class="pf-c-toolbar">
+            <div class="pf-c-toolbar__content">
+                {% include 'partials/toolbar_search.html' %}
+            </div>
+        </div>
         <div class="pf-c-empty-state">
             <div class="pf-c-empty-state__content">
-                <i class="fas fa-cubes pf-c-empty-state__icon" aria-hidden="true"></i>
+                <i class="pf-icon pf-icon-applications pf-c-empty-state__icon" aria-hidden="true"></i>
                 <h1 class="pf-c-title pf-m-lg">
                     {% trans 'No Applications.' %}
                 </h1>
                 <div class="pf-c-empty-state__body">
+                {% if request.GET.search != "" %}
+                    {% trans "Your search query doesn't match any application." %}
+                {% else %}
                     {% trans 'Currently no applications exist. Click the button below to create one.' %}
+                {% endif %}
                 </div>
                 <a href="{% url 'passbook_admin:application-create' %}?back={{ request.get_full_path }}" class="pf-c-button pf-m-primary" type="button">{% trans 'Create' %}</a>
             </div>