From 1ed24a5eef2fc93564f38a43dc04d861e7727770 Mon Sep 17 00:00:00 2001 From: Jens L Date: Tue, 10 Jan 2023 22:00:34 +0100 Subject: [PATCH] blueprints: internal storage (#4397) * rework oci client Signed-off-by: Jens Langhammer * add blueprint content Signed-off-by: Jens Langhammer * add UI Signed-off-by: Jens Langhammer * make path optional Signed-off-by: Jens Langhammer * add validation Signed-off-by: Jens Langhammer * add tests Signed-off-by: Jens Langhammer * fix Signed-off-by: Jens Langhammer Signed-off-by: Jens Langhammer Signed-off-by: Jens Langhammer --- authentik/blueprints/api.py | 18 ++++ .../0002_blueprintinstance_content.py | 23 +++++ authentik/blueprints/models.py | 74 +++----------- authentik/blueprints/tests/test_oci.py | 3 +- authentik/blueprints/tests/test_v1_api.py | 25 +++++ authentik/blueprints/v1/oci.py | 98 +++++++++++++++++++ schema.yml | 13 ++- web/src/admin/blueprints/BlueprintForm.ts | 57 ++++++++--- web/src/elements/CodeMirror.ts | 6 ++ website/developer-docs/blueprints/index.md | 12 ++- 10 files changed, 249 insertions(+), 80 deletions(-) create mode 100644 authentik/blueprints/migrations/0002_blueprintinstance_content.py create mode 100644 authentik/blueprints/v1/oci.py diff --git a/authentik/blueprints/api.py b/authentik/blueprints/api.py index ae976f48e4..c805990e16 100644 --- a/authentik/blueprints/api.py +++ b/authentik/blueprints/api.py @@ -1,4 +1,5 @@ """Serializer mixin for managed models""" +from django.utils.translation import gettext_lazy as _ from drf_spectacular.utils import extend_schema, inline_serializer from rest_framework.decorators import action from rest_framework.exceptions import ValidationError @@ -11,6 +12,7 @@ from rest_framework.viewsets import ModelViewSet from authentik.api.decorators import permission_required from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed +from authentik.blueprints.v1.importer import Importer from authentik.blueprints.v1.tasks import apply_blueprint, blueprints_find_dict from authentik.core.api.used_by import UsedByMixin from authentik.core.api.utils import PassiveSerializer @@ -40,6 +42,21 @@ class BlueprintInstanceSerializer(ModelSerializer): raise ValidationError(exc) from exc return path + def validate_content(self, content: str) -> str: + """Ensure content (if set) is a valid blueprint""" + if content == "": + return content + context = self.instance.context if self.instance else {} + valid, logs = Importer(content, context).validate() + if not valid: + raise ValidationError(_("Failed to validate blueprint"), *[x["msg"] for x in logs]) + return content + + def validate(self, attrs: dict) -> dict: + if attrs.get("path", "") == "" and attrs.get("content", "") == "": + raise ValidationError(_("Either path or content must be set.")) + return super().validate(attrs) + class Meta: model = BlueprintInstance @@ -54,6 +71,7 @@ class BlueprintInstanceSerializer(ModelSerializer): "enabled", "managed_models", "metadata", + "content", ] extra_kwargs = { "status": {"read_only": True}, diff --git a/authentik/blueprints/migrations/0002_blueprintinstance_content.py b/authentik/blueprints/migrations/0002_blueprintinstance_content.py new file mode 100644 index 0000000000..ba11effc0c --- /dev/null +++ b/authentik/blueprints/migrations/0002_blueprintinstance_content.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.5 on 2023-01-10 19:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("authentik_blueprints", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="blueprintinstance", + name="content", + field=models.TextField(blank=True, default=""), + ), + migrations.AlterField( + model_name="blueprintinstance", + name="path", + field=models.TextField(blank=True, default=""), + ), + ] diff --git a/authentik/blueprints/models.py b/authentik/blueprints/models.py index 121ae8f1ca..a8a7a2e930 100644 --- a/authentik/blueprints/models.py +++ b/authentik/blueprints/models.py @@ -1,30 +1,18 @@ """blueprint models""" from pathlib import Path -from urllib.parse import urlparse from uuid import uuid4 from django.contrib.postgres.fields import ArrayField from django.db import models from django.utils.translation import gettext_lazy as _ -from opencontainers.distribution.reggie import ( - NewClient, - WithDebug, - WithDefaultName, - WithDigest, - WithReference, - WithUserAgent, - WithUsernamePassword, -) -from requests.exceptions import RequestException from rest_framework.serializers import Serializer from structlog import get_logger +from authentik.blueprints.v1.oci import BlueprintOCIClient, OCIException from authentik.lib.config import CONFIG from authentik.lib.models import CreatedUpdatedModel, SerializerModel from authentik.lib.sentry import SentryIgnoredException -from authentik.lib.utils.http import authentik_user_agent -OCI_MEDIA_TYPE = "application/vnd.goauthentik.blueprint.v1+yaml" LOGGER = get_logger() @@ -74,7 +62,8 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel): name = models.TextField() metadata = models.JSONField(default=dict) - path = models.TextField() + path = models.TextField(default="", blank=True) + content = models.TextField(default="", blank=True) context = models.JSONField(default=dict) last_applied = models.DateTimeField(auto_now=True) last_applied_hash = models.TextField() @@ -86,60 +75,29 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel): def retrieve_oci(self) -> str: """Get blueprint from an OCI registry""" - url = urlparse(self.path) - ref = "latest" - path = url.path[1:] - if ":" in url.path: - path, _, ref = path.partition(":") - client = NewClient( - f"https://{url.hostname}", - WithUserAgent(authentik_user_agent()), - WithUsernamePassword(url.username, url.password), - WithDefaultName(path), - WithDebug(True), - ) - LOGGER.info("Fetching OCI manifests for blueprint", instance=self) - manifest_request = client.NewRequest( - "GET", - "/v2//manifests/", - WithReference(ref), - ).SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json") + client = BlueprintOCIClient(self.path.replace("oci://", "https://")) try: - manifest_response = client.Do(manifest_request) - manifest_response.raise_for_status() - except RequestException as exc: + manifests = client.fetch_manifests() + return client.fetch_blobs(manifests) + except OCIException as exc: raise BlueprintRetrievalFailed(exc) from exc - manifest = manifest_response.json() - if "errors" in manifest: - raise BlueprintRetrievalFailed(manifest["errors"]) - blob = None - for layer in manifest.get("layers", []): - if layer.get("mediaType", "") == OCI_MEDIA_TYPE: - blob = layer.get("digest") - LOGGER.debug("Found layer with matching media type", instance=self, blob=blob) - if not blob: - raise BlueprintRetrievalFailed("Blob not found") - - blob_request = client.NewRequest( - "GET", - "/v2//blobs/", - WithDigest(blob), - ) + def retrieve_file(self) -> str: + """Get blueprint from path""" try: - blob_response = client.Do(blob_request) - blob_response.raise_for_status() - return blob_response.text - except RequestException as exc: + full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path)) + with full_path.open("r", encoding="utf-8") as _file: + return _file.read() + except (IOError, OSError) as exc: raise BlueprintRetrievalFailed(exc) from exc def retrieve(self) -> str: """Retrieve blueprint contents""" if self.path.startswith("oci://"): return self.retrieve_oci() - full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path)) - with full_path.open("r", encoding="utf-8") as _file: - return _file.read() + if self.path != "": + return self.retrieve_file() + return self.content @property def serializer(self) -> Serializer: diff --git a/authentik/blueprints/tests/test_oci.py b/authentik/blueprints/tests/test_oci.py index 2eec45ea2d..dd54e2602f 100644 --- a/authentik/blueprints/tests/test_oci.py +++ b/authentik/blueprints/tests/test_oci.py @@ -2,7 +2,8 @@ from django.test import TransactionTestCase from requests_mock import Mocker -from authentik.blueprints.models import OCI_MEDIA_TYPE, BlueprintInstance, BlueprintRetrievalFailed +from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed +from authentik.blueprints.v1.oci import OCI_MEDIA_TYPE class TestBlueprintOCI(TransactionTestCase): diff --git a/authentik/blueprints/tests/test_v1_api.py b/authentik/blueprints/tests/test_v1_api.py index 35dc8f8ff1..276ac3abfe 100644 --- a/authentik/blueprints/tests/test_v1_api.py +++ b/authentik/blueprints/tests/test_v1_api.py @@ -43,3 +43,28 @@ class TestBlueprintsV1API(APITestCase): "6871c0003f5c07be5c3316d9d4a08444bd8fed1b3f03294e51e44522" ), ) + + def test_api_blank(self): + """Test blank""" + res = self.client.post( + reverse("authentik_api:blueprintinstance-list"), + data={ + "name": "foo", + }, + ) + self.assertEqual(res.status_code, 400) + self.assertJSONEqual( + res.content.decode(), {"non_field_errors": ["Either path or content must be set."]} + ) + + def test_api_content(self): + """Test blank""" + res = self.client.post( + reverse("authentik_api:blueprintinstance-list"), + data={ + "name": "foo", + "content": '{"version": 3}', + }, + ) + self.assertEqual(res.status_code, 400) + self.assertJSONEqual(res.content.decode(), {"content": ["Failed to validate blueprint"]}) diff --git a/authentik/blueprints/v1/oci.py b/authentik/blueprints/v1/oci.py new file mode 100644 index 0000000000..b05d3c52f0 --- /dev/null +++ b/authentik/blueprints/v1/oci.py @@ -0,0 +1,98 @@ +"""OCI Client""" +from typing import Any +from urllib.parse import ParseResult, urlparse + +from opencontainers.distribution.reggie import ( + NewClient, + WithDebug, + WithDefaultName, + WithDigest, + WithReference, + WithUserAgent, + WithUsernamePassword, +) +from requests.exceptions import RequestException +from structlog import get_logger +from structlog.stdlib import BoundLogger + +from authentik.lib.sentry import SentryIgnoredException +from authentik.lib.utils.http import authentik_user_agent + +OCI_MEDIA_TYPE = "application/vnd.goauthentik.blueprint.v1+yaml" + + +class OCIException(SentryIgnoredException): + """OCI-related errors""" + + +class BlueprintOCIClient: + """Blueprint OCI Client""" + + url: ParseResult + sanitized_url: str + logger: BoundLogger + ref: str + client: NewClient + + def __init__(self, url: str) -> None: + self._parse_url(url) + self.logger = get_logger().bind(url=self.sanitized_url) + + self.ref = "latest" + path = self.url.path[1:] + if ":" in self.url.path: + path, _, self.ref = path.partition(":") + self.client = NewClient( + f"https://{self.url.hostname}", + WithUserAgent(authentik_user_agent()), + WithUsernamePassword(self.url.username, self.url.password), + WithDefaultName(path), + WithDebug(True), + ) + + def _parse_url(self, url: str): + self.url = urlparse(url) + netloc = self.url.netloc + if "@" in netloc: + netloc = netloc[netloc.index("@") + 1 :] + self.sanitized_url = self.url._replace(netloc=netloc).geturl() + + def fetch_manifests(self) -> dict[str, Any]: + """Fetch manifests for ref""" + self.logger.info("Fetching OCI manifests for blueprint") + manifest_request = self.client.NewRequest( + "GET", + "/v2//manifests/", + WithReference(self.ref), + ).SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json") + try: + manifest_response = self.client.Do(manifest_request) + manifest_response.raise_for_status() + except RequestException as exc: + raise OCIException(exc) from exc + manifest = manifest_response.json() + if "errors" in manifest: + raise OCIException(manifest["errors"]) + return manifest + + def fetch_blobs(self, manifest: dict[str, Any]): + """Fetch blob based on manifest info""" + blob = None + for layer in manifest.get("layers", []): + if layer.get("mediaType", "") == OCI_MEDIA_TYPE: + blob = layer.get("digest") + self.logger.debug("Found layer with matching media type", blob=blob) + if not blob: + raise OCIException("Blob not found") + + blob_request = self.client.NewRequest( + "GET", + "/v2//blobs/", + WithDigest(blob), + ) + try: + blob_response = self.client.Do(blob_request) + blob_response.raise_for_status() + return blob_response.text + except RequestException as exc: + raise OCIException(exc) from exc diff --git a/schema.yml b/schema.yml index 9fd8decb7b..d77bceaf3a 100644 --- a/schema.yml +++ b/schema.yml @@ -25918,6 +25918,7 @@ components: type: string path: type: string + default: '' context: type: object additionalProperties: {} @@ -25943,13 +25944,14 @@ components: type: object additionalProperties: {} readOnly: true + content: + type: string required: - last_applied - last_applied_hash - managed_models - metadata - name - - path - pk - status BlueprintInstanceRequest: @@ -25961,15 +25963,16 @@ components: minLength: 1 path: type: string - minLength: 1 + default: '' context: type: object additionalProperties: {} enabled: type: boolean + content: + type: string required: - name - - path BlueprintInstanceStatusEnum: enum: - successful @@ -33147,12 +33150,14 @@ components: minLength: 1 path: type: string - minLength: 1 + default: '' context: type: object additionalProperties: {} enabled: type: boolean + content: + type: string PatchedCaptchaStageRequest: type: object description: CaptchaStage Serializer diff --git a/web/src/admin/blueprints/BlueprintForm.ts b/web/src/admin/blueprints/BlueprintForm.ts index 8aa8ae793e..0836eb09b9 100644 --- a/web/src/admin/blueprints/BlueprintForm.ts +++ b/web/src/admin/blueprints/BlueprintForm.ts @@ -20,26 +20,27 @@ import PFToggleGroup from "@patternfly/patternfly/components/ToggleGroup/toggle- import { BlueprintFile, BlueprintInstance, ManagedApi } from "@goauthentik/api"; enum blueprintSource { - local, + file, oci, + internal, } @customElement("ak-blueprint-form") export class BlueprintForm extends ModelForm { @state() - source: blueprintSource = blueprintSource.local; + source: blueprintSource = blueprintSource.file; - loadInstance(pk: string): Promise { - return new ManagedApi(DEFAULT_CONFIG) - .managedBlueprintsRetrieve({ - instanceUuid: pk, - }) - .then((inst) => { - if (inst.path.startsWith("oci://")) { - this.source = blueprintSource.oci; - } - return inst; - }); + async loadInstance(pk: string): Promise { + const inst = await new ManagedApi(DEFAULT_CONFIG).managedBlueprintsRetrieve({ + instanceUuid: pk, + }); + if (inst.path?.startsWith("oci://")) { + this.source = blueprintSource.oci; + } + if (inst.content !== "") { + this.source = blueprintSource.internal; + } + return inst; } getSuccessMessage(): string { @@ -102,12 +103,12 @@ export class BlueprintForm extends ModelForm {
+ +
+ +
- ${this.source === blueprintSource.local + ${this.source === blueprintSource.file ? html` {

` : html``} + ${this.source === blueprintSource.internal + ? html` + + ` + : html``}
diff --git a/web/src/elements/CodeMirror.ts b/web/src/elements/CodeMirror.ts index cd12db95ac..0cf551dfb3 100644 --- a/web/src/elements/CodeMirror.ts +++ b/web/src/elements/CodeMirror.ts @@ -28,6 +28,9 @@ export class CodeMirrorTextarea extends AKElement { @property() name?: string; + @property({ type: Boolean }) + parseValue = true; + editor?: EditorView; _value?: string; @@ -67,6 +70,9 @@ export class CodeMirrorTextarea extends AKElement { } get value(): T | string { + if (!this.parseValue) { + return this.getInnerValue(); + } try { switch (this.mode.toLowerCase()) { case "yaml": diff --git a/website/developer-docs/blueprints/index.md b/website/developer-docs/blueprints/index.md index 0f666ef2ce..bd7a653f39 100644 --- a/website/developer-docs/blueprints/index.md +++ b/website/developer-docs/blueprints/index.md @@ -22,7 +22,7 @@ Blueprints are yaml files, whose format is described further in [File structure] Starting with authentik 2022.8, blueprints are used to manage authentik default flows and other system objects. These blueprints can be disabled/replaced with custom blueprints in certain circumstances. -## Storage - Local +## Storage - File The authentik container by default looks for blueprints in `/blueprints`. Underneath this directory, there are a couple default subdirectories: @@ -49,3 +49,13 @@ To push a blueprint to an OCI-compatible registry, [ORAS](https://oras.land/) ca ``` oras push ghcr.io//blueprint/:latest :application/vnd.goauthentik.blueprint.v1+yaml ``` + +## Storage - Internal + +:::info +Requires authentik 2023.1 +::: + +Blueprints can be stored in authentik's database, which allows blueprints to be managed via external configuration management tools like Terraform. + +Modifying the contents of a blueprint will trigger its reconciliation. Blueprints are validated on submission to prevent invalid blueprints from being saved.