From 82930ee8070332f95804148711c98e07ce7cea18 Mon Sep 17 00:00:00 2001 From: "gcp-cherry-pick-bot[bot]" <98988430+gcp-cherry-pick-bot[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 19:01:06 +0100 Subject: [PATCH] root: expose CONN_MAX_AGE, CONN_HEALTH_CHECKS and DISABLE_SERVER_SIDE_CURSORS for PostgreSQL config (cherry-pick #10159) (#12419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit root: expose CONN_MAX_AGE, CONN_HEALTH_CHECKS and DISABLE_SERVER_SIDE_CURSORS for PostgreSQL config (#10159) Co-authored-by: Tomás Farías Santana Co-authored-by: Marc 'risson' Schmitt Co-authored-by: Tana M Berry --- authentik/lib/config.py | 30 ++++++- authentik/lib/default.yml | 2 - authentik/lib/tests/test_config.py | 82 +++++++++++++++++++ .../configuration/configuration.mdx | 28 ++++++- website/docs/releases/2024/v2024.12.md | 10 +++ 5 files changed, 145 insertions(+), 7 deletions(-) diff --git a/authentik/lib/config.py b/authentik/lib/config.py index b1e1898f17..d7e78ea2dc 100644 --- a/authentik/lib/config.py +++ b/authentik/lib/config.py @@ -280,9 +280,24 @@ class ConfigLoader: self.log("warning", "Failed to parse config as int", path=path, exc=str(exc)) return default + def get_optional_int(self, path: str, default=None) -> int | None: + """Wrapper for get that converts value into int or None if set""" + value = self.get(path, default) + + try: + return int(value) + except (ValueError, TypeError) as exc: + if value is None or (isinstance(value, str) and value.lower() == "null"): + return None + self.log("warning", "Failed to parse config as int", path=path, exc=str(exc)) + return default + def get_bool(self, path: str, default=False) -> bool: """Wrapper for get that converts value into boolean""" - return str(self.get(path, default)).lower() == "true" + value = self.get(path, UNSET) + if value is UNSET: + return default + return str(self.get(path)).lower() == "true" def get_keys(self, path: str, sep=".") -> list[str]: """List attribute keys by using yaml path""" @@ -354,20 +369,33 @@ def django_db_config(config: ConfigLoader | None = None) -> dict: "sslcert": config.get("postgresql.sslcert"), "sslkey": config.get("postgresql.sslkey"), }, + "CONN_MAX_AGE": CONFIG.get_optional_int("postgresql.conn_max_age", 0), + "CONN_HEALTH_CHECKS": CONFIG.get_bool("postgresql.conn_health_checks", False), + "DISABLE_SERVER_SIDE_CURSORS": CONFIG.get_bool( + "postgresql.disable_server_side_cursors", False + ), "TEST": { "NAME": config.get("postgresql.test.name"), }, } } + conn_max_age = CONFIG.get_optional_int("postgresql.conn_max_age", UNSET) + disable_server_side_cursors = CONFIG.get_bool("postgresql.disable_server_side_cursors", UNSET) if config.get_bool("postgresql.use_pgpool", False): db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True + if disable_server_side_cursors is not UNSET: + db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = disable_server_side_cursors if config.get_bool("postgresql.use_pgbouncer", False): # https://docs.djangoproject.com/en/4.0/ref/databases/#transaction-pooling-server-side-cursors db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True # https://docs.djangoproject.com/en/4.0/ref/databases/#persistent-connections db["default"]["CONN_MAX_AGE"] = None # persistent + if disable_server_side_cursors is not UNSET: + db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = disable_server_side_cursors + if conn_max_age is not UNSET: + db["default"]["CONN_MAX_AGE"] = conn_max_age for replica in config.get_keys("postgresql.read_replicas"): _database = deepcopy(db["default"]) diff --git a/authentik/lib/default.yml b/authentik/lib/default.yml index 9a8289a0df..492af91f37 100644 --- a/authentik/lib/default.yml +++ b/authentik/lib/default.yml @@ -6,8 +6,6 @@ postgresql: user: authentik port: 5432 password: "env://POSTGRES_PASSWORD" - use_pgbouncer: false - use_pgpool: false test: name: test_authentik read_replicas: {} diff --git a/authentik/lib/tests/test_config.py b/authentik/lib/tests/test_config.py index eca6540c80..bbe286db46 100644 --- a/authentik/lib/tests/test_config.py +++ b/authentik/lib/tests/test_config.py @@ -214,6 +214,9 @@ class TestConfig(TestCase): "PORT": "foo", "TEST": {"NAME": "foo"}, "USER": "foo", + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, + "DISABLE_SERVER_SIDE_CURSORS": False, } }, ) @@ -251,6 +254,9 @@ class TestConfig(TestCase): "PORT": "foo", "TEST": {"NAME": "foo"}, "USER": "foo", + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, + "DISABLE_SERVER_SIDE_CURSORS": False, }, "replica_0": { "ENGINE": "authentik.root.db", @@ -266,6 +272,72 @@ class TestConfig(TestCase): "PORT": "foo", "TEST": {"NAME": "foo"}, "USER": "foo", + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, + "DISABLE_SERVER_SIDE_CURSORS": False, + }, + }, + ) + + def test_db_read_replicas_pgbouncer(self): + """Test read replicas""" + config = ConfigLoader() + config.set("postgresql.host", "foo") + config.set("postgresql.name", "foo") + config.set("postgresql.user", "foo") + config.set("postgresql.password", "foo") + config.set("postgresql.port", "foo") + config.set("postgresql.sslmode", "foo") + config.set("postgresql.sslrootcert", "foo") + config.set("postgresql.sslcert", "foo") + config.set("postgresql.sslkey", "foo") + config.set("postgresql.test.name", "foo") + config.set("postgresql.use_pgbouncer", True) + # Read replica + config.set("postgresql.read_replicas.0.host", "bar") + # Override conn_max_age + config.set("postgresql.read_replicas.0.conn_max_age", 10) + # This isn't supported + config.set("postgresql.read_replicas.0.use_pgbouncer", False) + conf = django_db_config(config) + self.assertEqual( + conf, + { + "default": { + "DISABLE_SERVER_SIDE_CURSORS": True, + "CONN_MAX_AGE": None, + "CONN_HEALTH_CHECKS": False, + "ENGINE": "authentik.root.db", + "HOST": "foo", + "NAME": "foo", + "OPTIONS": { + "sslcert": "foo", + "sslkey": "foo", + "sslmode": "foo", + "sslrootcert": "foo", + }, + "PASSWORD": "foo", + "PORT": "foo", + "TEST": {"NAME": "foo"}, + "USER": "foo", + }, + "replica_0": { + "DISABLE_SERVER_SIDE_CURSORS": True, + "CONN_MAX_AGE": 10, + "CONN_HEALTH_CHECKS": False, + "ENGINE": "authentik.root.db", + "HOST": "bar", + "NAME": "foo", + "OPTIONS": { + "sslcert": "foo", + "sslkey": "foo", + "sslmode": "foo", + "sslrootcert": "foo", + }, + "PASSWORD": "foo", + "PORT": "foo", + "TEST": {"NAME": "foo"}, + "USER": "foo", }, }, ) @@ -294,6 +366,8 @@ class TestConfig(TestCase): { "default": { "DISABLE_SERVER_SIDE_CURSORS": True, + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, "ENGINE": "authentik.root.db", "HOST": "foo", "NAME": "foo", @@ -310,6 +384,8 @@ class TestConfig(TestCase): }, "replica_0": { "DISABLE_SERVER_SIDE_CURSORS": True, + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, "ENGINE": "authentik.root.db", "HOST": "bar", "NAME": "foo", @@ -362,6 +438,9 @@ class TestConfig(TestCase): "PORT": "foo", "TEST": {"NAME": "foo"}, "USER": "foo", + "DISABLE_SERVER_SIDE_CURSORS": False, + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, }, "replica_0": { "ENGINE": "authentik.root.db", @@ -377,6 +456,9 @@ class TestConfig(TestCase): "PORT": "foo", "TEST": {"NAME": "foo"}, "USER": "foo", + "DISABLE_SERVER_SIDE_CURSORS": False, + "CONN_MAX_AGE": 0, + "CONN_HEALTH_CHECKS": False, }, }, ) diff --git a/website/docs/install-config/configuration/configuration.mdx b/website/docs/install-config/configuration/configuration.mdx index eb7c0e62b5..ae9178a048 100644 --- a/website/docs/install-config/configuration/configuration.mdx +++ b/website/docs/install-config/configuration/configuration.mdx @@ -70,14 +70,17 @@ To check if your config has been applied correctly, you can run the following co - `AUTHENTIK_POSTGRESQL__USER`: Database user - `AUTHENTIK_POSTGRESQL__PORT`: Database port, defaults to 5432 - `AUTHENTIK_POSTGRESQL__PASSWORD`: Database password, defaults to the environment variable `POSTGRES_PASSWORD` -- `AUTHENTIK_POSTGRESQL__USE_PGBOUNCER`: Adjust configuration to support connection to PgBouncer -- `AUTHENTIK_POSTGRESQL__USE_PGPOOL`: Adjust configuration to support connection to Pgpool +- `AUTHENTIK_POSTGRESQL__USE_PGBOUNCER`: Adjust configuration to support connection to PgBouncer. Deprecated, see below +- `AUTHENTIK_POSTGRESQL__USE_PGPOOL`: Adjust configuration to support connection to Pgpool. Deprecated, see below - `AUTHENTIK_POSTGRESQL__SSLMODE`: Strictness of ssl verification. Defaults to `"verify-ca"` - `AUTHENTIK_POSTGRESQL__SSLROOTCERT`: CA root for server ssl verification - `AUTHENTIK_POSTGRESQL__SSLCERT`: Path to x509 client certificate to authenticate to server - `AUTHENTIK_POSTGRESQL__SSLKEY`: Path to private key of `SSLCERT` certificate +- `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE`: Database connection lifetime. Defaults to `0` (no persistent connections). Can be set to `null` for unlimited persistent connections. See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/settings/#conn-max-age) for more details. +- `AUTHENTIK_POSTGRESQL__CONN_HEALTH_CHECK`: Existing persistent database connections will be health checked before they are reused if set to `true`. Defaults to `false`. See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/settings/#conn-health-checks) for more details. +- `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS`: Disable server side cursors when set to `true`. Defaults to `false`. See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/settings/#disable-server-side-cursors) for more details. -All PostgreSQL settings, apart from `USE_PGBOUNCER` and `USE_PGPOOL`, support hot-reloading. Adding and removing read replicas doesn't support hot-reloading. +The PostgreSQL settings `HOST`, `PORT`, `USER`, and `PASSWORD` support hot-reloading. Adding and removing read replicas doesn't support hot-reloading. ### Read replicas @@ -96,8 +99,25 @@ The same PostgreSQL settings as described above are used for each read replica. - `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__SSLROOTCERT` - `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__SSLCERT` - `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__SSLKEY` +- `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__CONN_MAX_AGE` +- `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__CONN_HEALTH_CHECK` +- `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__DISABLE_SERVER_SIDE_CURSORS` -Note that `USE_PGBOUNCER` and `USE_PGPOOL` are inherited from the main database configuration and are _not_ overridable on read replicas. +### Using a PostgreSQL connection pooler (PgBouncer or PgPool) + +When your PostgreSQL database(s) are running behind a connection pooler, like PgBouncer or PgPool, two settings need to be overridden: + +- `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE` + + A connection pooler running in session pool mode (PgBouncer default) can be incompatible with unlimited persistent connections enabled by setting this to `null`: If the connection from the connection pooler to the database server is dropped, the connection pooler will wait for the client to disconnect before releasing the connection; however this will **never** happen as authentik is configured to keep the connection to the connection pooler forever. + + To address this incompatibility, either configure the connection pooler to run in transaction pool mode, or update this setting to a value lower than any timeouts that may cause the connection to the database to be dropped (up to `0`). + +- `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS` + + Using a connection pooler in transaction pool mode (e.g. PgPool, or PgBouncer in transaction or statement pool mode) requires disabling server-side cursors, so this setting must be set to `false`. + +Additionally, you can set `AUTHENTIK_POSTGRESQL__CONN_HEALTH_CHECK` to perform health checks on persistent database connections before they are re-used. ## Redis Settings diff --git a/website/docs/releases/2024/v2024.12.md b/website/docs/releases/2024/v2024.12.md index d4c8aa1149..292c0b9ea4 100644 --- a/website/docs/releases/2024/v2024.12.md +++ b/website/docs/releases/2024/v2024.12.md @@ -24,6 +24,16 @@ To try out the release candidate, replace your Docker image tag with the latest You can disable this behavior in the **Admin interface** under **System** > **Settings**. +- **Deprecated PostgreSQL `USE_PGBOUNCER` and `USE_PGPOOL` settings** + + With this release, the `AUTHENTIK_POSTGRESQL__USE_PGBOUNCER` and `AUTHENTIK_POSTGRESQL__USE_PGPOOL` settings have been deprecated in favor of exposing the underlying database settings: `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE` and `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS`. + + If you are using PgBouncer or PgPool as connection poolers and wish to maintain the same behavior as previous versions, `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS` must be set to `true`. Moreover, if you are using PgBouncer `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE` must be set to `null`. + + The newly exposed settings allow supporting a wider set of connection pooler configurations. For details on how these settings interact with different configurations of connection poolers, please refer to the [PostgreSQL documentation](../../install-config/configuration/configuration.mdx#postgresql-settings). + + These settings will be removed in a future version. + ## New features - **Redirect stage**