habsi/authentik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

events: add ASN Database reader (#7793)

Browse Source 
* events: add ASN Database reader

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add tests

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix test config generator

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* de-duplicate code

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add enrich_context

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* rename to context processors?

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix cache

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* use config deprecation system, update docs

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* update more docs and tests

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add test asn db

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* re-build schema with latest versions

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

---------

Signed-off-by: Jens Langhammer <jens@goauthentik.io>
This commit is contained in:
Jens L
2023-12-20 22:16:50 +01:00
committed by GitHub
parent 4ff3915d59
commit 50860d7ffe
27 changed files with 393 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
authentik/api/v3/config.py
View File

@ -19,7 +19,7 @@ from rest_framework.response import Response
from rest_framework.views import APIView
from authentik.core.api.utils import PassiveSerializer
from authentik.events.geo import GEOIP_READER
from authentik.events.context_processors.base import get_context_processors
from authentik.lib.config import CONFIG
capabilities = Signal()
@ -30,6 +30,7 @@ class Capabilities(models.TextChoices):
CAN_SAVE_MEDIA = "can_save_media"
CAN_GEO_IP = "can_geo_ip"
CAN_ASN = "can_asn"
CAN_IMPERSONATE = "can_impersonate"
CAN_DEBUG = "can_debug"
IS_ENTERPRISE = "is_enterprise"
@ -68,8 +69,9 @@ class ConfigView(APIView):
deb_test = settings.DEBUG or settings.TEST
if Path(settings.MEDIA_ROOT).is_mount() or deb_test:
caps.append(Capabilities.CAN_SAVE_MEDIA)
if GEOIP_READER.enabled:
caps.append(Capabilities.CAN_GEO_IP)
for processor in get_context_processors():
if cap := processor.capability():
caps.append(cap)
if CONFIG.get_bool("impersonation"):
caps.append(Capabilities.CAN_IMPERSONATE)
if settings.DEBUG: # pragma: no cover

13
authentik/core/api/authenticated_sessions.py
View File

@ -14,7 +14,8 @@ from ua_parser import user_agent_parser
from authentik.api.authorization import OwnerSuperuserPermissions
from authentik.core.api.used_by import UsedByMixin
from authentik.core.models import AuthenticatedSession
from authentik.events.geo import GEOIP_READER, GeoIPDict
from authentik.events.context_processors.asn import ASN_CONTEXT_PROCESSOR, ASNDict
from authentik.events.context_processors.geoip import GEOIP_CONTEXT_PROCESSOR, GeoIPDict
class UserAgentDeviceDict(TypedDict):
@ -59,6 +60,7 @@ class AuthenticatedSessionSerializer(ModelSerializer):
current = SerializerMethodField()
user_agent = SerializerMethodField()
geo_ip = SerializerMethodField()
asn = SerializerMethodField()
def get_current(self, instance: AuthenticatedSession) -> bool:
"""Check if session is currently active session"""
@ -70,8 +72,12 @@ class AuthenticatedSessionSerializer(ModelSerializer):
return user_agent_parser.Parse(instance.last_user_agent)
def get_geo_ip(self, instance: AuthenticatedSession) -> Optional[GeoIPDict]: # pragma: no cover
"""Get parsed user agent"""
return GEOIP_READER.city_dict(instance.last_ip)
"""Get GeoIP Data"""
return GEOIP_CONTEXT_PROCESSOR.city_dict(instance.last_ip)
def get_asn(self, instance: AuthenticatedSession) -> Optional[ASNDict]: # pragma: no cover
"""Get ASN Data"""
return ASN_CONTEXT_PROCESSOR.asn_dict(instance.last_ip)
class Meta:
model = AuthenticatedSession
@ -80,6 +86,7 @@ class AuthenticatedSessionSerializer(ModelSerializer):
"current",
"user_agent",
"geo_ip",
"asn",
"user",
"last_ip",
"last_user_agent",

0
authentik/events/context_processors/__init__.py Normal file
View File

79
authentik/events/context_processors/asn.py Normal file
View File

@ -0,0 +1,79 @@
"""ASN Enricher"""
from typing import TYPE_CHECKING, Optional, TypedDict
from django.http import HttpRequest
from geoip2.errors import GeoIP2Error
from geoip2.models import ASN
from sentry_sdk import Hub
from authentik.events.context_processors.mmdb import MMDBContextProcessor
from authentik.lib.config import CONFIG
from authentik.root.middleware import ClientIPMiddleware
if TYPE_CHECKING:
from authentik.api.v3.config import Capabilities
from authentik.events.models import Event
class ASNDict(TypedDict):
"""ASN Details"""
asn: int
as_org: str | None
network: str | None
class ASNContextProcessor(MMDBContextProcessor):
"""ASN Database reader wrapper"""
def capability(self) -> Optional["Capabilities"]:
from authentik.api.v3.config import Capabilities
return Capabilities.CAN_ASN
def path(self) -> str | None:
return CONFIG.get("events.context_processors.asn")
def enrich_event(self, event: "Event"):
asn = self.asn_dict(event.client_ip)
if not asn:
return
event.context["asn"] = asn
def enrich_context(self, request: HttpRequest) -> dict:
return {
"asn": self.asn_dict(ClientIPMiddleware.get_client_ip(request)),
}
def asn(self, ip_address: str) -> Optional[ASN]:
"""Wrapper for Reader.asn"""
with Hub.current.start_span(
op="authentik.events.asn.asn",
description=ip_address,
):
if not self.enabled:
return None
self.check_expired()
try:
return self.reader.asn(ip_address)
except (GeoIP2Error, ValueError):
return None
def asn_to_dict(self, asn: ASN) -> ASNDict:
"""Convert ASN to dict"""
asn_dict: ASNDict = {
"asn": asn.autonomous_system_number,
"as_org": asn.autonomous_system_organization,
"network": str(asn.network) if asn.network else None,
}
return asn_dict
def asn_dict(self, ip_address: str) -> Optional[ASNDict]:
"""Wrapper for self.asn that returns a dict"""
asn = self.asn(ip_address)
if not asn:
return None
return self.asn_to_dict(asn)
ASN_CONTEXT_PROCESSOR = ASNContextProcessor()

43
authentik/events/context_processors/base.py Normal file
View File

@ -0,0 +1,43 @@
"""Base event enricher"""
from functools import cache
from typing import TYPE_CHECKING, Optional
from django.http import HttpRequest
if TYPE_CHECKING:
from authentik.api.v3.config import Capabilities
from authentik.events.models import Event
class EventContextProcessor:
"""Base event enricher"""
def capability(self) -> Optional["Capabilities"]:
"""Return the capability this context processor provides"""
return None
def configured(self) -> bool:
"""Return true if this context processor is configured"""
return False
def enrich_event(self, event: "Event"):
"""Modify event"""
raise NotImplementedError
def enrich_context(self, request: HttpRequest) -> dict:
"""Modify context"""
raise NotImplementedError
@cache
def get_context_processors() -> list[EventContextProcessor]:
"""Get a list of all configured context processors"""
from authentik.events.context_processors.asn import ASN_CONTEXT_PROCESSOR
from authentik.events.context_processors.geoip import GEOIP_CONTEXT_PROCESSOR
processors_types = [ASN_CONTEXT_PROCESSOR, GEOIP_CONTEXT_PROCESSOR]
processors = []
for _type in processors_types:
if _type.configured():
processors.append(_type)
return processors

84
authentik/events/context_processors/geoip.py Normal file
View File

@ -0,0 +1,84 @@
"""events GeoIP Reader"""
from typing import TYPE_CHECKING, Optional, TypedDict
from django.http import HttpRequest
from geoip2.errors import GeoIP2Error
from geoip2.models import City
from sentry_sdk.hub import Hub
from authentik.events.context_processors.mmdb import MMDBContextProcessor
from authentik.lib.config import CONFIG
from authentik.root.middleware import ClientIPMiddleware
if TYPE_CHECKING:
from authentik.api.v3.config import Capabilities
from authentik.events.models import Event
class GeoIPDict(TypedDict):
"""GeoIP Details"""
continent: str
country: str
lat: float
long: float
city: str
class GeoIPContextProcessor(MMDBContextProcessor):
"""Slim wrapper around GeoIP API"""
def capability(self) -> Optional["Capabilities"]:
from authentik.api.v3.config import Capabilities
return Capabilities.CAN_GEO_IP
def path(self) -> str | None:
return CONFIG.get("events.context_processors.geoip")
def enrich_event(self, event: "Event"):
city = self.city_dict(event.client_ip)
if not city:
return
event.context["geo"] = city
def enrich_context(self, request: HttpRequest) -> dict:
# Different key `geoip` vs `geo` for legacy reasons
return {"geoip": self.city(ClientIPMiddleware.get_client_ip(request))}
def city(self, ip_address: str) -> Optional[City]:
"""Wrapper for Reader.city"""
with Hub.current.start_span(
op="authentik.events.geo.city",
description=ip_address,
):
if not self.enabled:
return None
self.check_expired()
try:
return self.reader.city(ip_address)
except (GeoIP2Error, ValueError):
return None
def city_to_dict(self, city: City) -> GeoIPDict:
"""Convert City to dict"""
city_dict: GeoIPDict = {
"continent": city.continent.code,
"country": city.country.iso_code,
"lat": city.location.latitude,
"long": city.location.longitude,
"city": "",
}
if city.city.name:
city_dict["city"] = city.city.name
return city_dict
def city_dict(self, ip_address: str) -> Optional[GeoIPDict]:
"""Wrapper for self.city that returns a dict"""
city = self.city(ip_address)
if not city:
return None
return self.city_to_dict(city)
GEOIP_CONTEXT_PROCESSOR = GeoIPContextProcessor()

54
authentik/events/context_processors/mmdb.py Normal file
View File

@ -0,0 +1,54 @@
"""Common logic for reading MMDB files"""
from pathlib import Path
from typing import Optional
from geoip2.database import Reader
from structlog.stdlib import get_logger
from authentik.events.context_processors.base import EventContextProcessor
class MMDBContextProcessor(EventContextProcessor):
"""Common logic for reading MaxMind DB files, including re-loading if the file has changed"""
def __init__(self):
self.reader: Optional[Reader] = None
self._last_mtime: float = 0.0
self.logger = get_logger()
self.open()
def path(self) -> str | None:
"""Get the path to the MMDB file to load"""
raise NotImplementedError
def open(self):
"""Get GeoIP Reader, if configured, otherwise none"""
path = self.path()
if path == "" or not path:
return
try:
self.reader = Reader(path)
self._last_mtime = Path(path).stat().st_mtime
self.logger.info("Loaded MMDB database", last_write=self._last_mtime, file=path)
except OSError as exc:
self.logger.warning("Failed to load MMDB database", exc=exc)
def check_expired(self):
"""Check if the modification date of the MMDB database has
changed, and reload it if so"""
path = self.path()
if path == "" or not path:
return
try:
mtime = Path(path).stat().st_mtime
diff = self._last_mtime < mtime
if diff > 0:
self.logger.info("Found new MMDB Database, reopening", diff=diff, path=path)
self.open()
except OSError as exc:
self.logger.warning("Failed to check MMDB age", exc=exc)
@property
def enabled(self) -> bool:
"""Check if MMDB is enabled"""
return bool(self.reader)

100
authentik/events/geo.py
View File

@ -1,100 +0,0 @@
"""events GeoIP Reader"""
from os import stat
from typing import Optional, TypedDict
from geoip2.database import Reader
from geoip2.errors import GeoIP2Error
from geoip2.models import City
from sentry_sdk.hub import Hub
from structlog.stdlib import get_logger
from authentik.lib.config import CONFIG
LOGGER = get_logger()
class GeoIPDict(TypedDict):
"""GeoIP Details"""
continent: str
country: str
lat: float
long: float
city: str
class GeoIPReader:
"""Slim wrapper around GeoIP API"""
def __init__(self):
self.__reader: Optional[Reader] = None
self.__last_mtime: float = 0.0
self.__open()
def __open(self):
"""Get GeoIP Reader, if configured, otherwise none"""
path = CONFIG.get("geoip")
if path == "" or not path:
return
try:
self.__reader = Reader(path)
self.__last_mtime = stat(path).st_mtime
LOGGER.info("Loaded GeoIP database", last_write=self.__last_mtime)
except OSError as exc:
LOGGER.warning("Failed to load GeoIP database", exc=exc)
def __check_expired(self):
"""Check if the modification date of the GeoIP database has
changed, and reload it if so"""
path = CONFIG.get("geoip")
try:
mtime = stat(path).st_mtime
diff = self.__last_mtime < mtime
if diff > 0:
LOGGER.info("Found new GeoIP Database, reopening", diff=diff)
self.__open()
except OSError as exc:
LOGGER.warning("Failed to check GeoIP age", exc=exc)
return
@property
def enabled(self) -> bool:
"""Check if GeoIP is enabled"""
return bool(self.__reader)
def city(self, ip_address: str) -> Optional[City]:
"""Wrapper for Reader.city"""
with Hub.current.start_span(
op="authentik.events.geo.city",
description=ip_address,
):
if not self.enabled:
return None
self.__check_expired()
try:
return self.__reader.city(ip_address)
except (GeoIP2Error, ValueError):
return None
def city_to_dict(self, city: City) -> GeoIPDict:
"""Convert City to dict"""
city_dict: GeoIPDict = {
"continent": city.continent.code,
"country": city.country.iso_code,
"lat": city.location.latitude,
"long": city.location.longitude,
"city": "",
}
if city.city.name:
city_dict["city"] = city.city.name
return city_dict
def city_dict(self, ip_address: str) -> Optional[GeoIPDict]:
"""Wrapper for self.city that returns a dict"""
city = self.city(ip_address)
if not city:
return None
return self.city_to_dict(city)
GEOIP_READER = GeoIPReader()

14
authentik/events/models.py
View File

@ -26,7 +26,7 @@ from authentik.core.middleware import (
SESSION_KEY_IMPERSONATE_USER,
)
from authentik.core.models import ExpiringModel, Group, PropertyMapping, User
from authentik.events.geo import GEOIP_READER
from authentik.events.context_processors.base import get_context_processors
from authentik.events.utils import (
cleanse_dict,
get_user,
@ -246,21 +246,15 @@ class Event(SerializerModel, ExpiringModel):
self.user["on_behalf_of"] = get_user(request.session[SESSION_KEY_IMPERSONATE_USER])
# User 255.255.255.255 as fallback if IP cannot be determined
self.client_ip = ClientIPMiddleware.get_client_ip(request)
# Apply GeoIP Data, when enabled
self.with_geoip()
# Enrich event data
for processor in get_context_processors():
processor.enrich_event(self)
# If there's no app set, we get it from the requests too
if not self.app:
self.app = Event._get_app_from_request(request)
self.save()
return self
def with_geoip(self): # pragma: no cover
"""Apply GeoIP Data, when enabled"""
city = GEOIP_READER.city_dict(self.client_ip)
if not city:
return
self.context["geo"] = city
def save(self, *args, **kwargs):
if self._state.adding:
LOGGER.info(

24
authentik/events/tests/test_enrich_asn.py Normal file
View File

@ -0,0 +1,24 @@
"""Test ASN Wrapper"""
from django.test import TestCase
from authentik.events.context_processors.asn import ASNContextProcessor
class TestASN(TestCase):
"""Test ASN Wrapper"""
def setUp(self) -> None:
self.reader = ASNContextProcessor()
def test_simple(self):
"""Test simple asn wrapper"""
# IPs from
# https://github.com/maxmind/MaxMind-DB/blob/main/source-data/GeoLite2-ASN-Test.json
self.assertEqual(
self.reader.asn_dict("1.0.0.1"),
{
"asn": 15169,
"as_org": "Google Inc.",
"network": "1.0.0.0/24",
},
)

4
authentik/events/tests/test_geoip.py → authentik/events/tests/test_enrich_geoip.py
View File

@ -1,14 +1,14 @@
"""Test GeoIP Wrapper"""
from django.test import TestCase
from authentik.events.geo import GeoIPReader
from authentik.events.context_processors.geoip import GeoIPContextProcessor
class TestGeoIP(TestCase):
"""Test GeoIP Wrapper"""
def setUp(self) -> None:
self.reader = GeoIPReader()
self.reader = GeoIPContextProcessor()
def test_simple(self):
"""Test simple city wrapper"""

9
authentik/events/utils.py
View File

@ -17,12 +17,13 @@ from django.db.models.base import Model
from django.http.request import HttpRequest
from django.utils import timezone
from django.views.debug import SafeExceptionReporterFilter
from geoip2.models import City
from geoip2.models import ASN, City
from guardian.utils import get_anonymous_user
from authentik.blueprints.v1.common import YAMLTag
from authentik.core.models import User
from authentik.events.geo import GEOIP_READER
from authentik.events.context_processors.asn import ASN_CONTEXT_PROCESSOR
from authentik.events.context_processors.geoip import GEOIP_CONTEXT_PROCESSOR
from authentik.policies.types import PolicyRequest
# Special keys which are *not* cleaned, even when the default filter
@ -123,7 +124,9 @@ def sanitize_item(value: Any) -> Any:
if isinstance(value, (HttpRequest, WSGIRequest)):
return ...
if isinstance(value, City):
return GEOIP_READER.city_to_dict(value)
return GEOIP_CONTEXT_PROCESSOR.city_to_dict(value)
if isinstance(value, ASN):
return ASN_CONTEXT_PROCESSOR.asn_to_dict(value)
if isinstance(value, Path):
return str(value)
if isinstance(value, Exception):

1
authentik/lib/config.py
View File

@ -35,6 +35,7 @@ REDIS_ENV_KEYS = [
]
DEPRECATIONS = {
"geoip": "events.context_processors.geoip",
"redis.broker_url": "broker.url",
"redis.broker_transport_options": "broker.transport_options",
"redis.cache_timeout": "cache.timeout",

5
authentik/lib/default.yml
View File

@ -108,7 +108,10 @@ cookie_domain: null
disable_update_check: false
disable_startup_analytics: false
avatars: env://AUTHENTIK_AUTHENTIK__AVATARS?gravatar,initials
geoip: "/geoip/GeoLite2-City.mmdb"
events:
context_processors:
geoip: "/geoip/GeoLite2-City.mmdb"
asn: "/geoip/GeoLite2-ASN.mmdb"
footer_links: []

1
authentik/policies/reputation/api.py
View File

@ -47,6 +47,7 @@ class ReputationSerializer(ModelSerializer):
"identifier",
"ip",
"ip_geo_data",
"ip_asn_data",
"score",
"updated",
]

17
authentik/policies/reputation/migrations/0006_reputation_ip_asn_data.py Normal file
View File

@ -0,0 +1,17 @@
# Generated by Django 4.2.7 on 2023-12-05 22:20
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("authentik_policies_reputation", "0005_reputation_expires_reputation_expiring"),
]
operations = [
migrations.AddField(
model_name="reputation",
name="ip_asn_data",
field=models.JSONField(default=dict),
),
]

1
authentik/policies/reputation/models.py
View File

@ -76,6 +76,7 @@ class Reputation(ExpiringModel, SerializerModel):
identifier = models.TextField()
ip = models.GenericIPAddressField()
ip_geo_data = models.JSONField(default=dict)
ip_asn_data = models.JSONField(default=dict)
score = models.BigIntegerField(default=0)
expires = models.DateTimeField(default=reputation_expiry)

6
authentik/policies/reputation/tasks.py
View File

@ -2,7 +2,8 @@
from django.core.cache import cache
from structlog.stdlib import get_logger
from authentik.events.geo import GEOIP_READER
from authentik.events.context_processors.asn import ASN_CONTEXT_PROCESSOR
from authentik.events.context_processors.geoip import GEOIP_CONTEXT_PROCESSOR
from authentik.events.monitored_tasks import (
MonitoredTask,
TaskResult,
@ -26,7 +27,8 @@ def save_reputation(self: MonitoredTask):
ip=score["ip"],
identifier=score["identifier"],
)
rep.ip_geo_data = GEOIP_READER.city_dict(score["ip"]) or {}
rep.ip_geo_data = GEOIP_CONTEXT_PROCESSOR.city_dict(score["ip"]) or {}
rep.ip_asn_data = ASN_CONTEXT_PROCESSOR.asn_dict(score["ip"]) or {}
rep.score = score["score"]
objects_to_update.append(rep)
Reputation.objects.bulk_update(objects_to_update, ["score", "ip_geo_data"])

12
authentik/policies/types.py
View File

@ -8,7 +8,7 @@ from django.db.models import Model
from django.http import HttpRequest
from structlog.stdlib import get_logger
from authentik.events.geo import GEOIP_READER
from authentik.events.context_processors.base import get_context_processors
if TYPE_CHECKING:
from authentik.core.models import User
@ -37,15 +37,9 @@ class PolicyRequest:
def set_http_request(self, request: HttpRequest): # pragma: no cover
"""Load data from HTTP request, including geoip when enabled"""
from authentik.root.middleware import ClientIPMiddleware
self.http_request = request
if not GEOIP_READER.enabled:
return
client_ip = ClientIPMiddleware.get_client_ip(request)
if not client_ip:
return
self.context["geoip"] = GEOIP_READER.city(client_ip)
for processor in get_context_processors():
self.context.update(processor.enrich_context(request))
@property
def should_cache(self) -> bool:

3
authentik/root/test_runner.py
View File

@ -32,7 +32,8 @@ class PytestTestRunner(DiscoverRunner): # pragma: no cover
settings.TEST = True
settings.CELERY["task_always_eager"] = True
CONFIG.set("avatars", "none")
CONFIG.set("geoip", "tests/GeoLite2-City-Test.mmdb")
CONFIG.set("events.context_processors.geoip", "tests/GeoLite2-City-Test.mmdb")
CONFIG.set("events.context_processors.asn", "tests/GeoLite2-ASN-Test.mmdb")
CONFIG.set("blueprints_dir", "./blueprints")
CONFIG.set(
"outposts.container_image_base",