make it kinda work

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

blueprints/testing/oidc-conformance.yaml

@@ -1,7 +1,8 @@
-# yaml-language-server: $schema=https://goauthentik.io/blueprints/schema.json
 version: 1
 metadata:
-  name: OIDC conformance testing
+  name: OpenID Conformance testing
+  labels:
+    blueprints.goauthentik.io/instantiate: "false"
 entries:
   - identifiers:
       managed: goauthentik.io/providers/oauth2/scope-address
@@ -32,7 +33,7 @@ entries:
   - model: authentik_providers_oauth2.oauth2provider
     id: provider
     identifiers:
-      name: provider
+      name: oidc-conformance-1
     attrs:
       authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
       invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
@@ -43,7 +44,8 @@ entries:
         - matching_mode: strict
           url: https://localhost:8443/test/a/authentik/callback
         - matching_mode: strict
-          url: https://localhost.emobix.co.uk:8443/test/a/authentik/callback
+          url: https://10.120.20.76:8443/test/a/authentik/callback
+          # url: !Format [https://%s:8443/test/a/authentik/callback, !Context host_ip]
       property_mappings:
         - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
         - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]
@@ -72,7 +74,8 @@ entries:
         - matching_mode: strict
           url: https://localhost:8443/test/a/authentik/callback
         - matching_mode: strict
-          url: https://localhost.emobix.co.uk:8443/test/a/authentik/callback
+          url: https://10.120.20.76:8443/test/a/authentik/callback
+          # url: !Format [https://%s:8443/test/a/authentik/callback, !Context host_ip]
       property_mappings:
         - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-openid]]
         - !Find [authentik_providers_oauth2.scopemapping, [managed, goauthentik.io/providers/oauth2/scope-email]]

tests/e2e/utils.py

@@ -176,6 +176,7 @@ class SeleniumTestCase(DockerTestCase, StaticLiveServerTestCase):
     def _get_driver(self) -> WebDriver:
         count = 0
         opts = webdriver.ChromeOptions()
+        opts.accept_insecure_certs = True
         opts.add_argument("--disable-search-engine-choice-screen")
         # This breaks selenium when running remotely...?
         # opts.set_capability("goog:loggingPrefs", {"browser": "ALL"})
@@ -260,7 +261,6 @@ class SeleniumTestCase(DockerTestCase, StaticLiveServerTestCase):
 
     def login(self, shadow_dom=True):
         """Do entire login flow"""
-
         if shadow_dom:
             flow_executor = self.get_shadow_root("ak-flow-executor")
             identification_stage = self.get_shadow_root("ak-stage-identification", flow_executor)

tests/openid-conformance/run.py

@@ -1,121 +0,0 @@
-#!/usr/bin/env python3
-
-import asyncio
-import json
-import os
-
-from conformance import Conformance
-
-CONFORMANCE_SERVER = "https://localhost:8443/"
-
-# This is the name of the Basic OP test plan:
-test_plan_name = "oidcc-basic-certification-test-plan"
-
-# This is the variant configuration of the test,
-# i.e. static or dynamic metadata location and client registration:
-test_variant_config = {"server_metadata": "discovery", "client_registration": "static_client"}
-
-# This is the required configuration for the test run:
-test_plan_config = {
-    "alias": "authentik",
-    "description": "authentik",
-    "server": {
-        "discoveryUrl": "http://10.120.20.76:9000/application/o/conformance/.well-known/openid-configuration"
-    },
-    "client": {
-        "client_id": "4054d882aff59755f2f279968b97ce8806a926e1",
-        "client_secret": "4c7e4933009437fb486b5389d15b173109a0555dc47e0cc0949104f1925bcc6565351cb1dffd7e6818cf074f5bd50c210b565121a7328ee8bd40107fc4bbd867",
-    },
-    "client_secret_post": {
-        "client_id": "4054d882aff59755f2f279968b97ce8806a926e1",
-        "client_secret": "4c7e4933009437fb486b5389d15b173109a0555dc47e0cc0949104f1925bcc6565351cb1dffd7e6818cf074f5bd50c210b565121a7328ee8bd40107fc4bbd867",
-    },
-    "client2": {
-        "client_id": "ad64aeaf1efe388ecf4d28fcc537e8de08bcae26",
-        "client_secret": "ff2e34a5b04c99acaf7241e25a950e7f6134c86936923d8c698d8f38bd57647750d661069612c0ee55045e29fe06aa101804bdae38e8360647d595e771fea789",
-    },
-    "consent": {},
-    "browser": [
-        {
-            "match": "http://10.120.20.76:9000/application/o/authorize*",
-            "tasks": [
-                {
-                    "task": "Login",
-                    "optional": True,
-                    "match": "http://10.120.20.76:9000/if/flow/default-authentication-flow*",
-                    "commands": [
-                        ["wait", "css", "[name=uid_field]", 10],
-                        ["text", "css", "[name=uid_field]", "akadmin"],
-                        ["wait", "css", "button[type=submit]", 10],
-                        ["click", "css", "button[type=submit]"],
-                        ["wait", "css", "[name=password]", 10],
-                        ["text", "css", "[name=password]", "foo"],
-                        ["click", "css", "button[type=submit]"],
-                        ["wait", "css", "#loading-text", 10],
-                        ["wait", "css", "#foo", 10],
-                        ["click", "css", "#foo"],
-                        # ["wait", "contains", "application/o/authorize", 10],
-                    ],
-                },
-                {
-                    "task": "Authorize",
-                    "match": "http://10.120.20.76:9000/application/o/authorize*",
-                    "optional": True,
-                    "commands": [
-                        ["wait", "css", "#loading-text", 10],
-                    ],
-                },
-                {
-                    "task": "Authorize 2",
-                    "optional": True,
-                    "match": "http://10.120.20.76:9000/if/flow/default-provider-authorization-implicit-consent*",
-                },
-            ],
-        }
-    ],
-}
-
-
-# Create a Conformance instance...
-conformance = Conformance(CONFORMANCE_SERVER, None, verify_ssl=False)
-
-# Create a test plan instance and print the id of it
-test_plan = asyncio.run(
-    conformance.create_test_plan(test_plan_name, json.dumps(test_plan_config), test_variant_config)
-)
-plan_id = test_plan["id"]
-
-print(f"----------------\nBegin {test_plan_name}.")
-print(f"Plan URL: {CONFORMANCE_SERVER}plan-detail.html?plan={plan_id}\n")
-
-# Iterate over the tests in the plan and run them one by one
-for test in test_plan["modules"]:
-
-    # Fetch name and variant of the next test to run
-    module_name = test["testModule"]
-    variant = test["variant"]
-    print(f"Module name: {module_name}")
-    print(f"Variant: {json.dumps(variant)}")
-
-    # Create an instance of that test
-    module_instance = asyncio.run(
-        conformance.create_test_from_plan_with_variant(plan_id, module_name, variant)
-    )
-    module_id = module_instance["id"]
-    print(f"Test URL: {CONFORMANCE_SERVER}log-detail.html?log={module_id}")
-
-    # Run the test and wait for it to finish
-    state = asyncio.run(conformance.wait_for_state(module_id, ["FINISHED"]))
-    print("")
-
-print(f"Plan URL: {CONFORMANCE_SERVER}plan-detail.html?plan={plan_id}\n")
-print(f"\nEnd {test_plan_name}\n----------------")
-
-print("Creating certification package")
-asyncio.run(
-    conformance.create_certification_package(
-        plan_id=plan_id,
-        conformance_pdf_path="OpenID-Certification-of-Conformance.pdf",
-        output_zip_directory="./zips/",
-    )
-)

tests/openid_conformance/compose.yml

@@ -19,7 +19,7 @@ services:
       -Xdebug -Xrunjdwp:transport=dt_socket,address=*:9999,server=y,suspend=n
       -jar /server/fapi-test-suite.jar
       -Djdk.tls.maxHandshakeMessageSize=65536
-      --fintechlabs.base_url=https://localhost.emobix.co.uk:8443
+      --fintechlabs.base_url=https://10.120.20.76:8443
       --fintechlabs.base_mtls_url=https://localhost.emobix.co.uk:8444
       --fintechlabs.devmode=true
       --fintechlabs.startredir=true

tests/openid_conformance/conformance.py

@@ -75,6 +75,17 @@ class Conformance:
             )
         return response.json()
 
+    def get_test_status(self,module_id):
+        """Returns an array containing a dictionary per test module"""
+        api_url = f"{self.api_url_base}api/runner/{module_id}"
+        response = self.httpclient.get(api_url)
+
+        if response.status_code != 200:
+            raise Exception(
+                f"get_test_status failed - HTTP {response.status_code:d} {response.content}"
+            )
+        return response.json()
+
     async def exporthtml(self, plan_id, path):
         for i in range(5):
             api_url = f"{self.api_url_base}api/plan/exporthtml/{plan_id}"
@@ -148,7 +159,7 @@ class Conformance:
             certificationOfConformancePdf.close()
             clientSideData.close()
 
-    async def create_test_plan(self, name, configuration, variant=None):
+    def create_test_plan(self, name, configuration, variant=None):
         api_url = f"{self.api_url_base}api/plan"
         payload = {"planName": name}
         if variant != None:
@@ -161,7 +172,7 @@ class Conformance:
             )
         return response.json()
 
-    async def create_test(self, test_name, configuration):
+    def create_test(self, test_name, configuration):
         api_url = f"{self.api_url_base}api/runner"
         payload = {"test": test_name}
         response = self.httpclient.post(api_url, params=payload, data=configuration)
@@ -183,7 +194,7 @@ class Conformance:
             )
         return response.json()
 
-    async def create_test_from_plan_with_variant(self, plan_id, test_name, variant):
+    def create_test_from_plan_with_variant(self, plan_id, test_name, variant):
         api_url = f"{self.api_url_base}api/runner"
         payload = {"test": test_name, "plan": plan_id}
         if variant != None:
@@ -196,7 +207,7 @@ class Conformance:
             )
         return response.json()
 
-    async def get_module_info(self, module_id):
+    def get_module_info(self, module_id):
         api_url = f"{self.api_url_base}api/info/{module_id}"
         response = self.httpclient.get(api_url)
 
@@ -206,7 +217,7 @@ class Conformance:
             )
         return response.json()
 
-    async def get_test_log(self, module_id):
+    def get_test_log(self, module_id):
         api_url = f"{self.api_url_base}api/log/{module_id}"
         response = self.httpclient.get(api_url)
 
@@ -216,6 +227,18 @@ class Conformance:
             )
         return response.json()
 
+    def upload_image(self, log_id, placeholder, data):
+        api_url = f"{self.api_url_base}api/log/{log_id}/images/{placeholder}"
+        response = self.httpclient.post(api_url, data=data, headers={
+            "Content-Type": "text/plain"
+        })
+
+        if response.status_code != 200:
+            raise Exception(
+                f"upload_image failed - HTTP {response.status_code:d} {response.content}"
+            )
+        return response.json()
+
     async def start_test(self, module_id):
         api_url = f"{self.api_url_base}api/runner/{module_id}"
         response = self.httpclient.post(api_url)

tests/openid_conformance/test_conformance.py

@@ -0,0 +1,103 @@
+from json import dumps
+from time import sleep
+from authentik.blueprints.tests import apply_blueprint, reconcile_app
+from tests.e2e.utils import SeleniumTestCase, retry
+from tests.openid_conformance.conformance import Conformance
+from selenium.webdriver.common.by import By
+
+from selenium.webdriver.support import expected_conditions as ec
+
+
+class TestOpenIDConformance(SeleniumTestCase):
+
+    @retry()
+    @apply_blueprint(
+        "default/flow-default-authentication-flow.yaml",
+        "default/flow-default-invalidation-flow.yaml",
+    )
+    @apply_blueprint(
+        "default/flow-default-provider-authorization-implicit-consent.yaml",
+        "default/flow-default-provider-invalidation.yaml",
+    )
+    @apply_blueprint("system/providers-oauth2.yaml")
+    @reconcile_app("authentik_crypto")
+    @apply_blueprint("testing/oidc-conformance.yaml")
+    def test_oidcc_basic_certification_test(self):
+        test_plan_name = "oidcc-basic-certification-test-plan"
+        test_variant_config = {
+            "server_metadata": "discovery",
+            "client_registration": "static_client",
+        }
+        test_plan_config = {
+            "alias": "authentik",
+            "description": "authentik",
+            "server": {
+                "discoveryUrl": f"{self.live_server_url}/application/o/conformance/.well-known/openid-configuration"
+            },
+            "client": {
+                "client_id": "4054d882aff59755f2f279968b97ce8806a926e1",
+                "client_secret": "4c7e4933009437fb486b5389d15b173109a0555dc47e0cc0949104f1925bcc6565351cb1dffd7e6818cf074f5bd50c210b565121a7328ee8bd40107fc4bbd867",
+            },
+            "client_secret_post": {
+                "client_id": "4054d882aff59755f2f279968b97ce8806a926e1",
+                "client_secret": "4c7e4933009437fb486b5389d15b173109a0555dc47e0cc0949104f1925bcc6565351cb1dffd7e6818cf074f5bd50c210b565121a7328ee8bd40107fc4bbd867",
+            },
+            "client2": {
+                "client_id": "ad64aeaf1efe388ecf4d28fcc537e8de08bcae26",
+                "client_secret": "ff2e34a5b04c99acaf7241e25a950e7f6134c86936923d8c698d8f38bd57647750d661069612c0ee55045e29fe06aa101804bdae38e8360647d595e771fea789",
+            },
+            "consent": {},
+        }
+
+        # Create a Conformance instance...
+        conformance = Conformance(f"https://{self.host}:8443/", None, verify_ssl=False)
+
+        # Create a test plan instance and print the id of it
+        test_plan = conformance.create_test_plan(
+            test_plan_name, dumps(test_plan_config), test_variant_config
+        )
+        plan_id = test_plan["id"]
+        n = 0
+        for test in test_plan["modules"]:
+            with self.subTest(test["testModule"]):
+                # Fetch name and variant of the next test to run
+                module_name = test["testModule"]
+                variant = test["variant"]
+                print(f"Module name: {module_name}")
+                print(f"Variant: {dumps(variant)}")
+
+                # Create an instance of that test
+                module_instance = conformance.create_test_from_plan_with_variant(
+                    plan_id, module_name, variant
+                )
+                module_id = module_instance["id"]
+                while True:
+                    test_status = conformance.get_test_status(module_id)
+                    print(test_status)
+                    browser_urls = test_status.get("browser", {}).get("urls", [])
+                    print(browser_urls)
+                    if len(browser_urls) < 1:
+                        continue
+                    self.do_browser(browser_urls[0])
+                    # Check if we need to upload any items
+                    test_log = conformance.get_test_log(module_id)
+                    upload_items = [x for x in test_log if "upload" in x]
+                    if len(upload_items) > 0:
+                        sleep(10)
+                        for item in upload_items:
+                            conformance.upload_image(
+                                module_id, item, self.driver.get_screenshot_as_base64()
+                            )
+                    # Close tab we've opened earlier
+                    # self.driver.close()
+                    break
+            sleep(2)
+            n += 1
+
+    def do_browser(self, url):
+        """For any specific OpenID Conformance test, execute the operations required"""
+        # self.driver.switch_to.new_window("tab")
+        self.driver.get(url)
+        if "if/flow/default-authentication-flow" in self.driver.current_url:
+            self.login()
+            self.wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, "#complete")))