habsi/authentik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

e2e: add @retry decorator to make e2e tests more reliable

Browse Source
This commit is contained in:
Jens Langhammer
2020-10-20 18:42:26 +02:00
parent ef021495ef
commit aeee3ad7f9
13 changed files with 80 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
e2e/test_flows_enroll.py
View File

@ -8,7 +8,7 @@ from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
from passbook.stages.email.models import EmailStage, EmailTemplates
from passbook.stages.identification.models import IdentificationStage
@ -34,6 +34,7 @@ class TestFlowsEnroll(SeleniumTestCase):
),
}
@retry()
def test_enroll_2_step(self):
"""Test 2-step enroll flow"""
# First stage fields
@ -119,6 +120,7 @@ class TestFlowsEnroll(SeleniumTestCase):
"foo@bar.baz",
)
@retry()
@override_settings(EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend")
def test_enroll_email(self):
"""Test enroll with Email verification"""

3
e2e/test_flows_login.py
View File

@ -5,13 +5,14 @@ from unittest.case import skipUnless
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
@skipUnless(platform.startswith("linux"), "requires local docker")
class TestFlowsLogin(SeleniumTestCase):
"""test default login flow"""
@retry()
def test_login(self):
"""test default login flow"""
self.driver.get(f"{self.live_server_url}/flows/default-authentication-flow/")

5
e2e/test_flows_otp.py
View File

@ -12,7 +12,7 @@ from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.flows.models import Flow, FlowStageBinding
from passbook.stages.otp_validate.models import OTPValidateStage
@ -21,6 +21,7 @@ from passbook.stages.otp_validate.models import OTPValidateStage
class TestFlowsOTP(SeleniumTestCase):
"""test flow with otp stages"""
@retry()
def test_otp_validate(self):
"""test flow with otp stages"""
sleep(1)
@ -52,6 +53,7 @@ class TestFlowsOTP(SeleniumTestCase):
USER().username,
)
@retry()
def test_otp_totp_setup(self):
"""test TOTP Setup stage"""
flow: Flow = Flow.objects.get(slug="default-authentication-flow")
@ -98,6 +100,7 @@ class TestFlowsOTP(SeleniumTestCase):
self.assertTrue(TOTPDevice.objects.filter(user=USER(), confirmed=True).exists())
@retry()
def test_otp_static_setup(self):
"""test Static OTP Setup stage"""
flow: Flow = Flow.objects.get(slug="default-authentication-flow")

3
e2e/test_flows_stage_setup.py
View File

@ -5,7 +5,7 @@ from unittest.case import skipUnless
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.core.models import User
from passbook.flows.models import Flow, FlowDesignation
from passbook.providers.oauth2.generators import generate_client_secret
@ -16,6 +16,7 @@ from passbook.stages.password.models import PasswordStage
class TestFlowsStageSetup(SeleniumTestCase):
"""test stage setup flows"""
@retry()
def test_password_change(self):
"""test password change flow"""
# Ensure that password stage has change_flow set

5
e2e/test_provider_oauth2_github.py
View File

@ -9,7 +9,7 @@ from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.core.models import Application
from passbook.flows.models import Flow
from passbook.policies.expression.models import ExpressionPolicy
@ -61,6 +61,7 @@ class TestProviderOAuth2Github(SeleniumTestCase):
},
}
@retry()
def test_authorization_consent_implied(self):
"""test OAuth Provider flow (default authorization flow with implied consent)"""
# Bootstrap all needed objects
@ -115,6 +116,7 @@ class TestProviderOAuth2Github(SeleniumTestCase):
USER().username,
)
@retry()
def test_authorization_consent_explicit(self):
"""test OAuth Provider flow (default authorization flow with explicit consent)"""
# Bootstrap all needed objects
@ -184,6 +186,7 @@ class TestProviderOAuth2Github(SeleniumTestCase):
USER().username,
)
@retry()
def test_denied(self):
"""test OAuth Provider flow (default authorization flow, denied)"""
# Bootstrap all needed objects

7
e2e/test_provider_oauth2_grafana.py
View File

@ -10,7 +10,7 @@ from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair
from passbook.flows.models import Flow
@ -80,6 +80,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
},
}
@retry()
def test_redirect_uri_error(self):
"""test OpenID Provider flow (invalid redirect URI, check error message)"""
sleep(1)
@ -122,6 +123,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
"Redirect URI Error",
)
@retry()
def test_authorization_consent_implied(self):
"""test OpenID Provider flow (default authorization flow with implied consent)"""
sleep(1)
@ -183,6 +185,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
USER().email,
)
@retry()
def test_authorization_logout(self):
"""test OpenID Provider flow with logout"""
sleep(1)
@ -252,6 +255,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
)
self.driver.find_element(By.ID, "logout").click()
@retry()
def test_authorization_consent_explicit(self):
"""test OpenID Provider flow (default authorization flow with explicit consent)"""
sleep(1)
@ -325,6 +329,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
USER().email,
)
@retry()
def test_authorization_denied(self):
"""test OpenID Provider flow (default authorization with access deny)"""
sleep(1)

6
e2e/test_provider_oauth2_oidc.py
View File

@ -12,7 +12,7 @@ from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair
from passbook.flows.models import Flow
@ -76,6 +76,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
LOGGER.info("Container failed healthcheck")
sleep(1)
@retry()
def test_redirect_uri_error(self):
"""test OpenID Provider flow (invalid redirect URI, check error message)"""
sleep(1)
@ -119,6 +120,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
"Redirect URI Error",
)
@retry()
def test_authorization_consent_implied(self):
"""test OpenID Provider flow (default authorization flow with implied consent)"""
sleep(1)
@ -169,6 +171,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
self.assertEqual(body["IDTokenClaims"]["email"], USER().email)
self.assertEqual(body["UserInfo"]["email"], USER().email)
@retry()
def test_authorization_consent_explicit(self):
"""test OpenID Provider flow (default authorization flow with explicit consent)"""
sleep(1)
@ -229,6 +232,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
self.assertEqual(body["IDTokenClaims"]["email"], USER().email)
self.assertEqual(body["UserInfo"]["email"], USER().email)
@retry()
def test_authorization_denied(self):
"""test OpenID Provider flow (default authorization with access deny)"""
sleep(1)

4
e2e/test_provider_proxy.py
View File

@ -11,7 +11,7 @@ from docker.models.containers import Container
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook import __version__
from passbook.core.models import Application
from passbook.flows.models import Flow
@ -57,6 +57,7 @@ class TestProviderProxy(SeleniumTestCase):
)
return container
@retry()
def test_proxy_simple(self):
"""Test simple outpost setup with single provider"""
proxy: ProxyProvider = ProxyProvider.objects.create(
@ -110,6 +111,7 @@ class TestProviderProxy(SeleniumTestCase):
class TestProviderProxyConnect(ChannelsLiveServerTestCase):
"""Test Proxy connectivity over websockets"""
@retry()
def test_proxy_connectivity(self):
"""Test proxy connectivity over websocket"""
SeleniumTestCase().apply_default_data()

6
e2e/test_provider_saml.py
View File

@ -12,7 +12,7 @@ from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase
from e2e.utils import USER, SeleniumTestCase, retry
from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair
from passbook.flows.models import Flow
@ -66,6 +66,7 @@ class TestProviderSAML(SeleniumTestCase):
LOGGER.info("Container failed healthcheck")
sleep(1)
@retry()
def test_sp_initiated_implicit(self):
"""test SAML Provider flow SP-initiated flow (implicit consent)"""
# Bootstrap all needed objects
@ -105,6 +106,7 @@ class TestProviderSAML(SeleniumTestCase):
self.assertEqual(body["attr"]["mail"], [USER().email])
self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
@retry()
def test_sp_initiated_explicit(self):
"""test SAML Provider flow SP-initiated flow (explicit consent)"""
# Bootstrap all needed objects
@ -150,6 +152,7 @@ class TestProviderSAML(SeleniumTestCase):
self.assertEqual(body["attr"]["mail"], [USER().email])
self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
@retry()
def test_idp_initiated_implicit(self):
"""test SAML Provider flow IdP-initiated flow (implicit consent)"""
# Bootstrap all needed objects
@ -195,6 +198,7 @@ class TestProviderSAML(SeleniumTestCase):
self.assertEqual(body["attr"]["mail"], [USER().email])
self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
@retry()
def test_sp_initiated_denied(self):
"""test SAML Provider flow SP-initiated flow (Policy denies access)"""
# Bootstrap all needed objects

6
e2e/test_source_oauth.py
View File

@ -14,7 +14,7 @@ from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from yaml import safe_dump
from e2e.utils import SeleniumTestCase
from e2e.utils import SeleniumTestCase, retry
from passbook.flows.models import Flow
from passbook.providers.oauth2.generators import (
generate_client_id,
@ -106,6 +106,7 @@ class TestSourceOAuth2(SeleniumTestCase):
consumer_secret=self.client_secret,
)
@retry()
def test_oauth_enroll(self):
"""test OAuth Source With With OIDC"""
self.create_objects()
@ -159,6 +160,7 @@ class TestSourceOAuth2(SeleniumTestCase):
"admin@example.com",
)
@retry()
@override_settings(SESSION_COOKIE_SAMESITE="strict")
def test_oauth_samesite_strict(self):
"""test OAuth Source With SameSite set to strict
@ -195,6 +197,7 @@ class TestSourceOAuth2(SeleniumTestCase):
"Authentication Failed.",
)
@retry()
def test_oauth_enroll_auth(self):
"""test OAuth Source With With OIDC (enroll and authenticate again)"""
self.test_oauth_enroll()
@ -291,6 +294,7 @@ class TestSourceOAuth1(SeleniumTestCase):
consumer_secret=self.client_secret,
)
@retry()
def test_oauth_enroll(self):
"""test OAuth Source With With OIDC"""
self.create_objects()

5
e2e/test_source_saml.py
View File

@ -10,7 +10,7 @@ from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from e2e.utils import SeleniumTestCase
from e2e.utils import SeleniumTestCase, retry
from passbook.crypto.models import CertificateKeyPair
from passbook.flows.models import Flow
from passbook.sources.saml.models import SAMLBindingTypes, SAMLSource
@ -92,6 +92,7 @@ class TestSourceSAML(SeleniumTestCase):
},
}
@retry()
def test_idp_redirect(self):
"""test SAML Source With redirect binding"""
# Bootstrap all needed objects
@ -141,6 +142,7 @@ class TestSourceSAML(SeleniumTestCase):
self.driver.find_element(By.ID, "id_username").get_attribute("value"), ""
)
@retry()
def test_idp_post(self):
"""test SAML Source With post binding"""
# Bootstrap all needed objects
@ -192,6 +194,7 @@ class TestSourceSAML(SeleniumTestCase):
self.driver.find_element(By.ID, "id_username").get_attribute("value"), ""
)
@retry()
def test_idp_post_auto(self):
"""test SAML Source With post binding (auto redirect)"""
# Bootstrap all needed objects

37
e2e/utils.py
View File

@ -1,19 +1,22 @@
"""passbook e2e testing utilities"""
from functools import wraps
from glob import glob
from importlib.util import module_from_spec, spec_from_file_location
from inspect import getmembers, isfunction
from os import environ, makedirs
from time import sleep, time
from typing import Any, Dict, Optional
from typing import Any, Callable, Dict, Optional
from django.apps import apps
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from django.db import connection, transaction
from django.db.utils import IntegrityError
from django.shortcuts import reverse
from django.test.testcases import TestCase
from docker import DockerClient, from_env
from docker.models.containers import Container
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.webdriver.support.ui import WebDriverWait
@ -123,3 +126,35 @@ class SeleniumTestCase(StaticLiveServerTestCase):
func(apps, schema_editor)
except IntegrityError:
pass
def retry(max_retires=3, exceptions=None):
"""Retry test multiple times. Default to catching Selenium Timeout Exception"""
if not exceptions:
exceptions = [TimeoutException]
def retry_actual(func: Callable):
"""Retry test multiple times"""
count = 1
@wraps(func)
def wrapper(self: TestCase, *args, **kwargs):
"""Run test again if we're below max_retries, including tearDown and
setUp. Otherwise raise the error"""
nonlocal count
try:
return func(self, *args, **kwargs)
# pylint: disable=catching-non-exception
except tuple(exceptions) as exc:
count += 1
if count > max_retires:
# pylint: disable=raising-non-exception
raise exc
self.tearDown()
self.setUp()
return wrapper(self, *args, **kwargs)
return wrapper
return retry_actual

2
swagger.yaml
View File

@ -6348,7 +6348,7 @@ definitions:
for input-based policies.
type: boolean
re_evaluate_policies:
title: Evaluate on call
title: Re evaluate policies
description: Evaluate policies when the Stage is present to the user.
type: boolean
order: