From d8585eb8726e58df8db47987c858d9b2135eb249 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Wed, 10 Apr 2019 18:48:55 +0200 Subject: [PATCH 01/23] trigger autoreload from config files --- passbook/lib/config.py | 12 ++++++++++++ passbook/lib/default.yml | 2 ++ 2 files changed, 14 insertions(+) diff --git a/passbook/lib/config.py b/passbook/lib/config.py index 1158673aa7..a8db9bab84 100644 --- a/passbook/lib/config.py +++ b/passbook/lib/config.py @@ -8,6 +8,7 @@ from typing import Any import yaml from django.conf import ImproperlyConfigured +from django.utils.autoreload import autoreload_started SEARCH_PATHS = [ 'passbook/lib/default.yml', @@ -21,6 +22,8 @@ ENVIRONMENT = os.getenv('PASSBOOK_ENV', 'local') class ConfigLoader: """Search through SEARCH_PATHS and load configuration""" + loaded_file = [] + __config = {} __context_default = None __sub_dicts = [] @@ -69,6 +72,8 @@ class ConfigLoader: with open(path) as file: try: self.update(self.__config, yaml.safe_load(file)) + LOGGER.debug("Loaded %s", path) + self.loaded_file.append(path) except yaml.YAMLError as exc: raise ImproperlyConfigured from exc except PermissionError as exc: @@ -126,3 +131,10 @@ class ConfigLoader: CONFIG = ConfigLoader() + +# pylint: disable=unused-argument +def signal_handler(sender, **kwargs): + """Add all loaded config files to autoreload watcher""" + for path in CONFIG.loaded_file: + sender.watch_file(path) +autoreload_started.connect(signal_handler) diff --git a/passbook/lib/default.yml b/passbook/lib/default.yml index d51dd7d708..7e11d089cc 100644 --- a/passbook/lib/default.yml +++ b/passbook/lib/default.yml @@ -35,6 +35,8 @@ redis: localhost/0 error_report_enabled: true secret_key: 9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s +domains: + - passbook.local primary_domain: 'localhost' passbook: From 40866f9ecd720e815edec29c217317a09f7b1f68 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Wed, 10 Apr 2019 18:49:33 +0200 Subject: [PATCH 02/23] Choose upstream more cleverly --- passbook/app_gw/middleware.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/passbook/app_gw/middleware.py b/passbook/app_gw/middleware.py index 99dccdfdb1..5e2563a2c5 100644 --- a/passbook/app_gw/middleware.py +++ b/passbook/app_gw/middleware.py @@ -1,6 +1,7 @@ """passbook app_gw middleware""" import mimetypes from logging import getLogger +from random import SystemRandom from urllib.parse import urlparse import certifi @@ -18,6 +19,7 @@ from passbook.core.models import Application from passbook.core.policies import PolicyEngine from passbook.lib.config import CONFIG +SESSION_UPSTREAM_KEY = 'passbook_app_gw_upstream' IGNORED_HOSTNAMES_KEY = 'passbook_app_gw_ignored' LOGGER = getLogger(__name__) QUOTE_SAFE = r'<.;>\(}*+|~=-$/_:^@)[{]&\'!,"`' @@ -68,11 +70,11 @@ class ApplicationGatewayMiddleware: return True, None # At this point we're certain there's a matching ApplicationGateway if len(matches) > 1: - # TODO This should never happen + # This should never happen raise ValueError app_gw = matches.first() try: - # Check if ApplicationGateway is associcaited with application + # Check if ApplicationGateway is associated with application getattr(app_gw, 'application') return False, app_gw except Application.DoesNotExist: @@ -87,10 +89,18 @@ class ApplicationGatewayMiddleware: self.request = request return self.dispatch(request) + def _get_upstream(self): + """Choose random upstream and save in session""" + if SESSION_UPSTREAM_KEY not in self.request.session: + self.request.session[SESSION_UPSTREAM_KEY] = {} + if self.app_gw.pk not in self.request.session[SESSION_UPSTREAM_KEY]: + upstream_index = SystemRandom().randrange(len(self.app_gw.upstream)) + self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk] = upstream_index + return self.app_gw.upstream[self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk]] + def get_upstream(self): """Get upstream as parsed url""" - # TODO: How to choose upstream? - upstream = self.app_gw.upstream[0] + upstream = self._get_upstream() self._parsed_url = urlparse(upstream) From 04d613cb28d1f9e31115becdd8e20cc68d3fd45a Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Wed, 10 Apr 2019 19:03:22 +0200 Subject: [PATCH 03/23] Move code from django-revproxy to app_gw to fix cookie bug --- passbook/app_gw/middleware.py | 4 +- passbook/app_gw/proxy/__init__.py | 0 passbook/app_gw/proxy/exceptions.py | 8 + passbook/app_gw/proxy/response.py | 63 ++++++++ passbook/app_gw/proxy/utils.py | 227 ++++++++++++++++++++++++++++ 5 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 passbook/app_gw/proxy/__init__.py create mode 100644 passbook/app_gw/proxy/exceptions.py create mode 100644 passbook/app_gw/proxy/response.py create mode 100644 passbook/app_gw/proxy/utils.py diff --git a/passbook/app_gw/middleware.py b/passbook/app_gw/middleware.py index 99dccdfdb1..8b0c629cd4 100644 --- a/passbook/app_gw/middleware.py +++ b/passbook/app_gw/middleware.py @@ -9,10 +9,10 @@ from django.core.cache import cache from django.utils.http import urlencode from django.views.generic import RedirectView from revproxy.exceptions import InvalidUpstream -from revproxy.response import get_django_response -from revproxy.utils import encode_items, normalize_request_headers from passbook.app_gw.models import ApplicationGatewayProvider +from passbook.app_gw.proxy.response import get_django_response +from passbook.app_gw.proxy.utils import encode_items, normalize_request_headers from passbook.app_gw.rewrite import Rewriter from passbook.core.models import Application from passbook.core.policies import PolicyEngine diff --git a/passbook/app_gw/proxy/__init__.py b/passbook/app_gw/proxy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/passbook/app_gw/proxy/exceptions.py b/passbook/app_gw/proxy/exceptions.py new file mode 100644 index 0000000000..9d2b0dc8a7 --- /dev/null +++ b/passbook/app_gw/proxy/exceptions.py @@ -0,0 +1,8 @@ +"""Exception classes""" + +class ReverseProxyException(Exception): + """Base for revproxy exception""" + + +class InvalidUpstream(ReverseProxyException): + """Invalid upstream set""" diff --git a/passbook/app_gw/proxy/response.py b/passbook/app_gw/proxy/response.py new file mode 100644 index 0000000000..426533a5ac --- /dev/null +++ b/passbook/app_gw/proxy/response.py @@ -0,0 +1,63 @@ +"""response functions from django-revproxy""" +import logging + +from django.http import HttpResponse, StreamingHttpResponse + +from passbook.app_gw.proxy.utils import (cookie_from_string, + set_response_headers, should_stream) + +#: Default number of bytes that are going to be read in a file lecture +DEFAULT_AMT = 2 ** 16 + +logger = logging.getLogger('revproxy.response') + + +def get_django_response(proxy_response, strict_cookies=False): + """This method is used to create an appropriate response based on the + Content-Length of the proxy_response. If the content is bigger than + MIN_STREAMING_LENGTH, which is found on utils.py, + than django.http.StreamingHttpResponse will be created, + else a django.http.HTTPResponse will be created instead + + :param proxy_response: An Instance of urllib3.response.HTTPResponse that + will create an appropriate response + :param strict_cookies: Whether to only accept RFC-compliant cookies + :returns: Returns an appropriate response based on the proxy_response + content-length + """ + status = proxy_response.status + headers = proxy_response.headers + + logger.debug('Proxy response headers: %s', headers) + + content_type = headers.get('Content-Type') + + logger.debug('Content-Type: %s', content_type) + + if should_stream(proxy_response): + logger.info('Content-Length is bigger than %s', DEFAULT_AMT) + response = StreamingHttpResponse(proxy_response.stream(DEFAULT_AMT), + status=status, + content_type=content_type) + else: + content = proxy_response.data or b'' + response = HttpResponse(content, status=status, + content_type=content_type) + + logger.info('Normalizing response headers') + set_response_headers(response, headers) + + logger.debug('Response headers: %s', getattr(response, '_headers')) + + cookies = proxy_response.headers.getlist('set-cookie') + logger.info('Checking for invalid cookies') + for cookie_string in cookies: + cookie_dict = cookie_from_string(cookie_string, + strict_cookies=strict_cookies) + # if cookie is invalid cookie_dict will be None + if cookie_dict: + response.set_cookie(**cookie_dict) + + logger.debug('Response cookies: %s', response.cookies) + + return response diff --git a/passbook/app_gw/proxy/utils.py b/passbook/app_gw/proxy/utils.py new file mode 100644 index 0000000000..812d4a622d --- /dev/null +++ b/passbook/app_gw/proxy/utils.py @@ -0,0 +1,227 @@ +"""Utils from django-revproxy, slightly adjusted""" +import logging +import re +from wsgiref.util import is_hop_by_hop + +try: + from http.cookies import SimpleCookie + COOKIE_PREFIX = '' +except ImportError: + from Cookie import SimpleCookie + COOKIE_PREFIX = 'Set-Cookie: ' + + +#: List containing string constant that are used to represent headers that can +#: be ignored in the required_header function +IGNORE_HEADERS = ( + 'HTTP_ACCEPT_ENCODING', # We want content to be uncompressed so + # we remove the Accept-Encoding from + # original request + 'HTTP_HOST', + 'HTTP_REMOTE_USER', +) + + +# Default from HTTP RFC 2616 +# See: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1 +#: Variable that represent the default charset used +DEFAULT_CHARSET = 'latin-1' + +#: List containing string constants that represents possible html content type +HTML_CONTENT_TYPES = ( + 'text/html', + 'application/xhtml+xml' +) + +#: Variable used to represent a minimal content size required for response +#: to be turned into stream +MIN_STREAMING_LENGTH = 4 * 1024 # 4KB + +#: Regex used to find charset in a html content type +_get_charset_re = re.compile(r';\s*charset=(?P[^\s;]+)', re.I) + + +def is_html_content_type(content_type): + """Function used to verify if the parameter is a proper html content type + + :param content_type: String variable that represent a content-type + :returns: A boolean value stating if the content_type is a valid html + content type + """ + for html_content_type in HTML_CONTENT_TYPES: + if content_type.startswith(html_content_type): + return True + + return False + + +def should_stream(proxy_response): + """Function to verify if the proxy_response must be converted into + a stream.This will be done by checking the proxy_response content-length + and verify if its length is bigger than one stipulated + by MIN_STREAMING_LENGTH. + + :param proxy_response: An Instance of urllib3.response.HTTPResponse + :returns: A boolean stating if the proxy_response should + be treated as a stream + """ + content_type = proxy_response.headers.get('Content-Type') + + if is_html_content_type(content_type): + return False + + try: + content_length = int(proxy_response.headers.get('Content-Length', 0)) + except ValueError: + content_length = 0 + + if not content_length or content_length > MIN_STREAMING_LENGTH: + return True + + return False + + +def get_charset(content_type): + """Function used to retrieve the charset from a content-type.If there is no + charset in the content type then the charset defined on DEFAULT_CHARSET + will be returned + + :param content_type: A string containing a Content-Type header + :returns: A string containing the charset + """ + if not content_type: + return DEFAULT_CHARSET + + matched = _get_charset_re.search(content_type) + if matched: + # Extract the charset and strip its double quotes + return matched.group('charset').replace('"', '') + return DEFAULT_CHARSET + + +def required_header(header): + """Function that verify if the header parameter is a essential header + + :param header: A string represented a header + :returns: A boolean value that represent if the header is required + """ + if header in IGNORE_HEADERS: + return False + + if header.startswith('HTTP_') or header == 'CONTENT_TYPE': + return True + + return False + + +def set_response_headers(response, response_headers): + """Set response's header""" + for header, value in response_headers.items(): + if is_hop_by_hop(header) or header.lower() == 'set-cookie': + continue + + response[header.title()] = value + + logger.debug('Response headers: %s', getattr(response, '_headers')) + + +def normalize_request_headers(request): + """Function used to transform header, replacing 'HTTP\\_' to '' + and replace '_' to '-' + + :param request: A HttpRequest that will be transformed + :returns: A dictionary with the normalized headers + """ + norm_headers = {} + for header, value in request.META.items(): + if required_header(header): + norm_header = header.replace('HTTP_', '').title().replace('_', '-') + norm_headers[norm_header] = value + + return norm_headers + + +def encode_items(items): + """Function that encode all elements in the list of items passed as + a parameter + + :param items: A list of tuple + :returns: A list of tuple with all items encoded in 'utf-8' + """ + encoded = [] + for key, values in items: + for value in values: + encoded.append((key.encode('utf-8'), value.encode('utf-8'))) + return encoded + + +logger = logging.getLogger('revproxy.cookies') + + +def cookie_from_string(cookie_string, strict_cookies=False): + """Parser for HTTP header set-cookie + The return from this function will be used as parameters for + django's response.set_cookie method. Because set_cookie doesn't + have parameter comment, this cookie attribute will be ignored. + + :param cookie_string: A string representing a valid cookie + :param strict_cookies: Whether to only accept RFC-compliant cookies + :returns: A dictionary containing the cookie_string attributes + """ + + if strict_cookies: + + cookies = SimpleCookie(COOKIE_PREFIX + cookie_string) + if not cookies.keys(): + return None + cookie_name, = cookies.keys() + cookie_dict = {k: v for k, v in cookies[cookie_name].items() + if v and k != 'comment'} + cookie_dict['key'] = cookie_name + cookie_dict['value'] = cookies[cookie_name].value + return cookie_dict + valid_attrs = ('path', 'domain', 'comment', 'expires', + 'max_age', 'httponly', 'secure') + + cookie_dict = {} + + cookie_parts = cookie_string.split(';') + try: + cookie_dict['key'], cookie_dict['value'] = \ + cookie_parts[0].split('=', 1) + cookie_dict['value'] = cookie_dict['value'].replace('"', '') + # print('aaaaaaaaaaaaaaaaaaaaaaaaaaaa') + # print(cookie_parts[0].split('=', 1)) + except ValueError: + logger.warning('Invalid cookie: `%s`', cookie_string) + return None + + if cookie_dict['value'].startswith('='): + logger.warning('Invalid cookie: `%s`', cookie_string) + return None + + for part in cookie_parts[1:]: + if '=' in part: + attr, value = part.split('=', 1) + value = value.strip() + else: + attr = part + value = '' + + attr = attr.strip().lower() + if not attr: + continue + + if attr in valid_attrs: + if attr in ('httponly', 'secure'): + cookie_dict[attr] = True + elif attr in 'comment': + # ignoring comment attr as explained in the + # function docstring + continue + else: + cookie_dict[attr] = value + else: + logger.warning('Unknown cookie attribute %s', attr) + + return cookie_dict From c9ac10f6f6e68e069bc49980d47c4ce90d27ce40 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Wed, 10 Apr 2019 19:03:42 +0200 Subject: [PATCH 04/23] Implement websocket proxy --- passbook/app_gw/requirements.txt | 3 + passbook/app_gw/settings.py | 8 +-- passbook/app_gw/websocket/__init__.py | 0 passbook/app_gw/websocket/consumer.py | 83 +++++++++++++++++++++++++++ passbook/app_gw/websocket/routing.py | 17 ++++++ 5 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 passbook/app_gw/websocket/__init__.py create mode 100644 passbook/app_gw/websocket/consumer.py create mode 100644 passbook/app_gw/websocket/routing.py diff --git a/passbook/app_gw/requirements.txt b/passbook/app_gw/requirements.txt index ae3eaf219b..934e9cc253 100644 --- a/passbook/app_gw/requirements.txt +++ b/passbook/app_gw/requirements.txt @@ -1,2 +1,5 @@ django-revproxy urllib3[secure] +channels +service_identity +websocket-client diff --git a/passbook/app_gw/settings.py b/passbook/app_gw/settings.py index 6e5808d8d2..2fabd10ef4 100644 --- a/passbook/app_gw/settings.py +++ b/passbook/app_gw/settings.py @@ -1,5 +1,5 @@ """Application Security Gateway settings""" - -# INSTALLED_APPS = [ -# 'revproxy' -# ] +INSTALLED_APPS = [ + 'channels' +] +ASGI_APPLICATION = "passbook.app_gw.websocket.routing.application" diff --git a/passbook/app_gw/websocket/__init__.py b/passbook/app_gw/websocket/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/passbook/app_gw/websocket/consumer.py b/passbook/app_gw/websocket/consumer.py new file mode 100644 index 0000000000..bedfa41adb --- /dev/null +++ b/passbook/app_gw/websocket/consumer.py @@ -0,0 +1,83 @@ +"""websocket proxy consumer""" +import threading +from logging import getLogger +from ssl import CERT_NONE + +import websocket +from channels.generic.websocket import WebsocketConsumer + +from passbook.app_gw.models import ApplicationGatewayProvider + +LOGGER = getLogger(__name__) + +class ProxyConsumer(WebsocketConsumer): + """Proxy websocket connection to upstream""" + + _headers_dict = {} + _app_gw = None + _client = None + _thread = None + + def _fix_headers(self, input_dict): + """Fix headers from bytestrings to normal strings""" + return { + key.decode('utf-8'): value.decode('utf-8') + for key, value in dict(input_dict).items() + } + + def connect(self): + """Extract host header, lookup in database and proxy connection""" + self._headers_dict = self._fix_headers(dict(self.scope.get('headers'))) + host = self._headers_dict.pop('host') + query_string = self.scope.get('query_string').decode('utf-8') + matches = ApplicationGatewayProvider.objects.filter( + server_name__contains=[host], + enabled=True) + if matches.exists(): + self._app_gw = matches.first() + # TODO: Get upstream that starts with wss or + upstream = self._app_gw.upstream[0].replace('http', 'ws') + self.scope.get('path') + if query_string: + upstream += '?' + query_string + sslopt = {} + if not self._app_gw.upstream_ssl_verification: + sslopt = {"cert_reqs": CERT_NONE} + self._client = websocket.WebSocketApp( + url=upstream, + subprotocols=self.scope.get('subprotocols'), + header=self._headers_dict, + on_message=self._client_on_message_handler(), + on_error=self._client_on_error_handler(), + on_close=self._client_on_close_handler(), + on_open=self._client_on_open_handler()) + LOGGER.debug("Accepting connection for %s", host) + self._thread = threading.Thread(target=lambda: self._client.run_forever(sslopt=sslopt)) + self._thread.start() + + def _client_on_open_handler(self): + return lambda ws: self.accept(self._client.sock.handshake_response.subprotocol) + + def _client_on_message_handler(self): + # pylint: disable=unused-argument,invalid-name + def message_handler(ws, message): + if isinstance(message, str): + self.send(text_data=message) + else: + self.send(bytes_data=message) + return message_handler + + def _client_on_error_handler(self): + return lambda ws, error: print(error) + + def _client_on_close_handler(self): + return lambda ws: self.disconnect(0) + + def disconnect(self, code): + self._client.close() + + def receive(self, text_data=None, bytes_data=None): + if text_data: + opcode = websocket.ABNF.OPCODE_TEXT + if bytes_data: + opcode = websocket.ABNF.OPCODE_BINARY + self._client.send(text_data or bytes_data, opcode) diff --git a/passbook/app_gw/websocket/routing.py b/passbook/app_gw/websocket/routing.py new file mode 100644 index 0000000000..bbf7b7a831 --- /dev/null +++ b/passbook/app_gw/websocket/routing.py @@ -0,0 +1,17 @@ +"""app_gw websocket proxy""" +from channels.auth import AuthMiddlewareStack +from channels.routing import ProtocolTypeRouter, URLRouter +from django.conf.urls import url + +from passbook.app_gw.websocket.consumer import ProxyConsumer + +websocket_urlpatterns = [ + url(r'^(.*)$', ProxyConsumer), +] + +application = ProtocolTypeRouter({ + # (http->django views is added by default) + 'websocket': AuthMiddlewareStack( + URLRouter(websocket_urlpatterns) + ), +}) From 11630c9a743ba1fed18cc9805dd78c35af7131b6 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Wed, 10 Apr 2019 22:38:25 +0200 Subject: [PATCH 05/23] switch kubernetes deployment to daphne server --- Dockerfile | 4 ++-- .../passbook/templates/passbook-web-deployment.yaml | 2 +- passbook/app_gw/requirements.txt | 2 ++ passbook/core/asgi.py | 13 +++++++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 passbook/core/asgi.py diff --git a/Dockerfile b/Dockerfile index 45bf8be752..0647936bfc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY ./requirements.txt /app/ WORKDIR /app/ -RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev -y && \ +RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \ mkdir /app/static/ && \ pip install -r requirements.txt && \ pip install psycopg2 && \ @@ -23,7 +23,7 @@ COPY --from=build /app/static /app/static/ WORKDIR /app/ -RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev -y && \ +RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \ pip install -r requirements.txt && \ pip install psycopg2 && \ adduser --system --home /app/ passbook && \ diff --git a/helm/passbook/templates/passbook-web-deployment.yaml b/helm/passbook/templates/passbook-web-deployment.yaml index 2308300cab..df98947a26 100644 --- a/helm/passbook/templates/passbook-web-deployment.yaml +++ b/helm/passbook/templates/passbook-web-deployment.yaml @@ -29,7 +29,7 @@ spec: image: "docker.pkg.beryju.org/passbook:{{ .Values.image.tag }}" imagePullPolicy: IfNotPresent command: ["/bin/sh","-c"] - args: ["./manage.py migrate && ./manage.py web"] + args: ["./manage.py migrate && daphne -p 8000 passbook.core.asgi:application"] ports: - name: http containerPort: 8000 diff --git a/passbook/app_gw/requirements.txt b/passbook/app_gw/requirements.txt index 934e9cc253..19b5cf2d4d 100644 --- a/passbook/app_gw/requirements.txt +++ b/passbook/app_gw/requirements.txt @@ -3,3 +3,5 @@ urllib3[secure] channels service_identity websocket-client +daphne<2.3.0 +asgiref~=2.3 diff --git a/passbook/core/asgi.py b/passbook/core/asgi.py new file mode 100644 index 0000000000..249debe64f --- /dev/null +++ b/passbook/core/asgi.py @@ -0,0 +1,13 @@ +""" +ASGI entrypoint. Configures Django and then runs the application +defined in the ASGI_APPLICATION setting. +""" + +import os + +import django +from channels.routing import get_default_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "passbook.core.settings") +django.setup() +application = get_default_application() From b369eb28f1f1808f9ad3d0448a770dc2c5eaadee Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 10:43:13 +0200 Subject: [PATCH 06/23] set default log level to warn, fix clean_nonces not working --- helm/passbook/templates/passbook-configmap.yaml | 4 ++-- passbook/core/tasks.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/passbook/templates/passbook-configmap.yaml b/helm/passbook/templates/passbook-configmap.yaml index 1c9a3fe1ce..26618935ad 100644 --- a/helm/passbook/templates/passbook-configmap.yaml +++ b/helm/passbook/templates/passbook-configmap.yaml @@ -15,8 +15,8 @@ data: port: '' log: level: - console: DEBUG - file: DEBUG + console: WARNING + file: WARNING file: /dev/null syslog: host: 127.0.0.1 diff --git a/passbook/core/tasks.py b/passbook/core/tasks.py index 0b691ce2a4..e7339c46b3 100644 --- a/passbook/core/tasks.py +++ b/passbook/core/tasks.py @@ -24,5 +24,5 @@ def send_email(to_address, subject, template, context): @CELERY_APP.task() def clean_nonces(): """Remove expired nonces""" - amount = Nonce.objects.filter(expires__lt=datetime.now(), expiring=True).delete() + amount, _ = Nonce.objects.filter(expires__lt=datetime.now(), expiring=True).delete() LOGGER.debug("Deleted expired %d nonces", amount) From c723b0233f4f6b847072b874d6448709fb0a83f0 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 10:48:28 +0200 Subject: [PATCH 07/23] prepare 0.1.28 --- debian/changelog | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/debian/changelog b/debian/changelog index d8af718df2..6fb7c17abd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,20 @@ +passbook (0.1.28) stable; urgency=medium + + * bump version: 0.1.26-beta -> 0.1.27-beta + * fix allauth client's formatting + * switch from raven to sentry_sdk + * add ability to have non-expiring nonces, clean up expired nonces + * fully remove raven and switch WSGI and logging to sentry_sdk + * fix failing CI + * trigger autoreload from config files + * Choose upstream more cleverly + * Move code from django-revproxy to app_gw to fix cookie bug + * Implement websocket proxy + * switch kubernetes deployment to daphne server + * set default log level to warn, fix clean_nonces not working + + -- Jens Langhammer Thu, 11 Apr 2019 08:46:44 +0000 + passbook (0.1.27) stable; urgency=medium * bump version: 0.1.25-beta -> 0.1.26-beta From a1a5223b588fa69b16d3335d28e71d9a673c42eb Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 10:48:31 +0200 Subject: [PATCH 08/23] bump version: 0.1.27-beta -> 0.1.28-beta --- .bumpversion.cfg | 2 +- .gitlab-ci.yml | 2 +- client-packages/allauth/setup.py | 2 +- client-packages/sentry-auth-passbook/setup.py | 2 +- helm/passbook/Chart.yaml | 4 ++-- helm/passbook/values.yaml | 2 +- passbook/__init__.py | 2 +- passbook/admin/__init__.py | 2 +- passbook/api/__init__.py | 2 +- passbook/app_gw/__init__.py | 2 +- passbook/audit/__init__.py | 2 +- passbook/captcha_factor/__init__.py | 2 +- passbook/core/__init__.py | 2 +- passbook/hibp_policy/__init__.py | 2 +- passbook/ldap/__init__.py | 2 +- passbook/lib/__init__.py | 2 +- passbook/oauth_client/__init__.py | 2 +- passbook/oauth_provider/__init__.py | 2 +- passbook/otp/__init__.py | 2 +- passbook/password_expiry_policy/__init__.py | 2 +- passbook/saml_idp/__init__.py | 2 +- passbook/suspicious_policy/__init__.py | 2 +- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e2223c826e..938fee0f1d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.27-beta +current_version = 0.1.28-beta tag = True commit = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)\-(?P.*) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6439c5154..3e582e7adc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ package-docker: before_script: - echo "{\"auths\":{\"docker.$NEXUS_URL\":{\"auth\":\"$NEXUS_AUTH\"}}}" > /kaniko/.docker/config.json script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.27-beta + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.28-beta stage: build only: - tags diff --git a/client-packages/allauth/setup.py b/client-packages/allauth/setup.py index a1c5c5286e..8c0afcb9fa 100644 --- a/client-packages/allauth/setup.py +++ b/client-packages/allauth/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup( name='django-allauth-passbook', - version='0.1.27-beta', + version='0.1.28-beta', description='passbook support for django-allauth', # long_description='\n'.join(read_simple('docs/index.md')[2:]), long_description_content_type='text/markdown', diff --git a/client-packages/sentry-auth-passbook/setup.py b/client-packages/sentry-auth-passbook/setup.py index 79b8ef320e..16f455fe11 100644 --- a/client-packages/sentry-auth-passbook/setup.py +++ b/client-packages/sentry-auth-passbook/setup.py @@ -18,7 +18,7 @@ tests_require = [ setup( name='sentry-auth-passbook', - version='0.1.27-beta', + version='0.1.28-beta', author='BeryJu.org', author_email='support@beryju.org', url='https://passbook.beryju.org', diff --git a/helm/passbook/Chart.yaml b/helm/passbook/Chart.yaml index aa86ae0fc7..d306069552 100644 --- a/helm/passbook/Chart.yaml +++ b/helm/passbook/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -appVersion: "0.1.27-beta" +appVersion: "0.1.28-beta" description: A Helm chart for passbook. name: passbook -version: "0.1.27-beta" +version: "0.1.28-beta" icon: https://passbook.beryju.org/images/logo.png diff --git a/helm/passbook/values.yaml b/helm/passbook/values.yaml index 594381af6c..51fb71c2dc 100644 --- a/helm/passbook/values.yaml +++ b/helm/passbook/values.yaml @@ -5,7 +5,7 @@ replicaCount: 1 image: - tag: 0.1.27-beta + tag: 0.1.28-beta nameOverride: "" diff --git a/passbook/__init__.py b/passbook/__init__.py index 4d753cee24..4bc9185a48 100644 --- a/passbook/__init__.py +++ b/passbook/__init__.py @@ -1,2 +1,2 @@ """passbook""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/admin/__init__.py b/passbook/admin/__init__.py index 8d3a5d9ed0..89ce55b4e0 100644 --- a/passbook/admin/__init__.py +++ b/passbook/admin/__init__.py @@ -1,2 +1,2 @@ """passbook admin""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/api/__init__.py b/passbook/api/__init__.py index 1ae1882a31..d547c82013 100644 --- a/passbook/api/__init__.py +++ b/passbook/api/__init__.py @@ -1,2 +1,2 @@ """passbook api""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/app_gw/__init__.py b/passbook/app_gw/__init__.py index ddde0cc7d3..147588d2ab 100644 --- a/passbook/app_gw/__init__.py +++ b/passbook/app_gw/__init__.py @@ -1,2 +1,2 @@ """passbook Application Security Gateway Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/audit/__init__.py b/passbook/audit/__init__.py index 4f60a518d1..784a85e069 100644 --- a/passbook/audit/__init__.py +++ b/passbook/audit/__init__.py @@ -1,2 +1,2 @@ """passbook audit Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/captcha_factor/__init__.py b/passbook/captcha_factor/__init__.py index 5ae0deff43..e3fcfebd8b 100644 --- a/passbook/captcha_factor/__init__.py +++ b/passbook/captcha_factor/__init__.py @@ -1,2 +1,2 @@ """passbook captcha_factor Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/core/__init__.py b/passbook/core/__init__.py index 6be23b4c07..840f2b287c 100644 --- a/passbook/core/__init__.py +++ b/passbook/core/__init__.py @@ -1,2 +1,2 @@ """passbook core""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/hibp_policy/__init__.py b/passbook/hibp_policy/__init__.py index 36804c8cf5..5be707c227 100644 --- a/passbook/hibp_policy/__init__.py +++ b/passbook/hibp_policy/__init__.py @@ -1,2 +1,2 @@ """passbook hibp_policy""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/ldap/__init__.py b/passbook/ldap/__init__.py index 63bf0fdc03..edf5fb6c44 100644 --- a/passbook/ldap/__init__.py +++ b/passbook/ldap/__init__.py @@ -1,2 +1,2 @@ """Passbook ldap app Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/lib/__init__.py b/passbook/lib/__init__.py index cf777953dd..37bcb6450c 100644 --- a/passbook/lib/__init__.py +++ b/passbook/lib/__init__.py @@ -1,2 +1,2 @@ """passbook lib""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/oauth_client/__init__.py b/passbook/oauth_client/__init__.py index f20c40ce8d..93b5d5e11e 100644 --- a/passbook/oauth_client/__init__.py +++ b/passbook/oauth_client/__init__.py @@ -1,2 +1,2 @@ """passbook oauth_client Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/oauth_provider/__init__.py b/passbook/oauth_provider/__init__.py index 0b3a57986d..15e26c879b 100644 --- a/passbook/oauth_provider/__init__.py +++ b/passbook/oauth_provider/__init__.py @@ -1,2 +1,2 @@ """passbook oauth_provider Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/otp/__init__.py b/passbook/otp/__init__.py index e4e20a91f3..e886d0c6cc 100644 --- a/passbook/otp/__init__.py +++ b/passbook/otp/__init__.py @@ -1,2 +1,2 @@ """passbook otp Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/password_expiry_policy/__init__.py b/passbook/password_expiry_policy/__init__.py index 780ee4d94c..63782d25fc 100644 --- a/passbook/password_expiry_policy/__init__.py +++ b/passbook/password_expiry_policy/__init__.py @@ -1,2 +1,2 @@ """passbook password_expiry""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/saml_idp/__init__.py b/passbook/saml_idp/__init__.py index d84d876ee6..0f41ea548c 100644 --- a/passbook/saml_idp/__init__.py +++ b/passbook/saml_idp/__init__.py @@ -1,2 +1,2 @@ """passbook saml_idp Header""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' diff --git a/passbook/suspicious_policy/__init__.py b/passbook/suspicious_policy/__init__.py index bd4ffe5b17..fc556da5c4 100644 --- a/passbook/suspicious_policy/__init__.py +++ b/passbook/suspicious_policy/__init__.py @@ -1,2 +1,2 @@ """passbook suspicious_policy""" -__version__ = '0.1.27-beta' +__version__ = '0.1.28-beta' From a9031a6abc88f83a24d9f48bb173c15726de58af Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 12:44:26 +0200 Subject: [PATCH 09/23] Add libpq-dev dependency so psycopg2 build works --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 24d4909f8b..32e2530b94 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: admin Priority: optional Maintainer: BeryJu.org Uploaders: Jens Langhammer , BeryJu.org -Build-Depends: debhelper (>= 10), dh-systemd (>= 1.5), dh-exec, wget, dh-exec, python3 (>= 3.5) | python3.6 | python3.7 +Build-Depends: debhelper (>= 10), dh-systemd (>= 1.5), dh-exec, wget, dh-exec, python3 (>= 3.5) | python3.6 | python3.7, libpq-dev Standards-Version: 3.9.6 Package: passbook From 366ef352c60fc4d328f7b25090b968f1ee42b699 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 13:43:08 +0200 Subject: [PATCH 10/23] switch to whitenoise for static files --- passbook/core/requirements.txt | 1 + passbook/core/settings.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/passbook/core/requirements.txt b/passbook/core/requirements.txt index fd80ba3f11..5d7793425c 100644 --- a/passbook/core/requirements.txt +++ b/passbook/core/requirements.txt @@ -12,3 +12,4 @@ psycopg2 PyYAML sentry-sdk pip +whitenoise diff --git a/passbook/core/settings.py b/passbook/core/settings.py index 421d9c1229..47dda43c33 100644 --- a/passbook/core/settings.py +++ b/passbook/core/settings.py @@ -121,6 +121,7 @@ CACHES = { MIDDLEWARE = [ 'django.contrib.sessions.middleware.SessionMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'passbook.app_gw.middleware.ApplicationGatewayMiddleware', 'django.middleware.security.SecurityMiddleware', @@ -246,6 +247,7 @@ with CONFIG.cd('web'): # https://docs.djangoproject.com/en/2.1/howto/static-files/ STATIC_URL = '/static/' +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' LOG_HANDLERS = ['console', 'syslog', 'file'] From 19cd1624c161d09958299f54c4bcf675c2c72bce Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 13:43:49 +0200 Subject: [PATCH 11/23] replace cherrypy with daphne --- .../templates/passbook-web-deployment.yaml | 2 +- passbook/core/management/commands/web.py | 34 ++++++++----------- passbook/core/requirements.txt | 1 - passbook/core/settings.py | 17 +++------- 4 files changed, 21 insertions(+), 33 deletions(-) diff --git a/helm/passbook/templates/passbook-web-deployment.yaml b/helm/passbook/templates/passbook-web-deployment.yaml index df98947a26..2308300cab 100644 --- a/helm/passbook/templates/passbook-web-deployment.yaml +++ b/helm/passbook/templates/passbook-web-deployment.yaml @@ -29,7 +29,7 @@ spec: image: "docker.pkg.beryju.org/passbook:{{ .Values.image.tag }}" imagePullPolicy: IfNotPresent command: ["/bin/sh","-c"] - args: ["./manage.py migrate && daphne -p 8000 passbook.core.asgi:application"] + args: ["./manage.py migrate && ./manage.py web"] ports: - name: http containerPort: 8000 diff --git a/passbook/core/management/commands/web.py b/passbook/core/management/commands/web.py index da8c66879f..329e070e65 100644 --- a/passbook/core/management/commands/web.py +++ b/passbook/core/management/commands/web.py @@ -2,11 +2,11 @@ from logging import getLogger -import cherrypy -from django.conf import settings +from daphne.cli import CommandLineInterface from django.core.management.base import BaseCommand +from django.utils import autoreload -from passbook.core.wsgi import application +from passbook.lib.config import CONFIG LOGGER = getLogger(__name__) @@ -16,19 +16,15 @@ class Command(BaseCommand): def handle(self, *args, **options): """passbook cherrypy server""" - config = settings.CHERRYPY_SERVER - config.update(**options) - cherrypy.config.update(config) - cherrypy.tree.graft(application, '/') - # Mount NullObject to serve static files - cherrypy.tree.mount(None, '/static', config={ - '/': { - 'tools.staticdir.on': True, - 'tools.staticdir.dir': settings.STATIC_ROOT, - 'tools.expires.on': True, - 'tools.expires.secs': 86400, - 'tools.gzip.on': True, - } - }) - cherrypy.engine.start() - cherrypy.engine.block() + autoreload.run_with_reloader(self.daphne_server) + + def daphne_server(self): + """Run daphne server within autoreload""" + autoreload.raise_last_exception() + with CONFIG.cd('web'): + CommandLineInterface().run([ + '-p', str(CONFIG.get('port', 8000)), + '-b', CONFIG.get('listen', '0.0.0.0'), # nosec + '--access-log', '/dev/null', + 'passbook.core.asgi:application' + ]) diff --git a/passbook/core/requirements.txt b/passbook/core/requirements.txt index 5d7793425c..78e07757ae 100644 --- a/passbook/core/requirements.txt +++ b/passbook/core/requirements.txt @@ -1,5 +1,4 @@ celery -cherrypy colorlog django-ipware django-model-utils diff --git a/passbook/core/settings.py b/passbook/core/settings.py index 47dda43c33..144dbd5161 100644 --- a/passbook/core/settings.py +++ b/passbook/core/settings.py @@ -231,18 +231,6 @@ sentry_init( send_default_pii=True ) - -# CherryPY settings -with CONFIG.cd('web'): - CHERRYPY_SERVER = { - 'server.socket_host': CONFIG.get('listen', '0.0.0.0'), # nosec - 'server.socket_port': CONFIG.get('port', 8000), - 'server.thread_pool': CONFIG.get('threads', 30), - 'log.screen': False, - 'log.access_file': '', - 'log.error_file': '', - } - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.1/howto/static-files/ @@ -325,6 +313,11 @@ with CONFIG.cd('log'): 'level': 'DEBUG', 'propagate': True, }, + 'daphne': { + 'handlers': LOG_HANDLERS, + 'level': 'DEBUG', + 'propagate': True, + } } } From a3ef26b7adee1fe28524dc739c2b7fae3ff2041c Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 13:54:11 +0200 Subject: [PATCH 12/23] Run collectstatic before coverage, use autoreload on celery worker --- .gitlab-ci.yml | 1 + passbook/core/management/commands/web.py | 2 +- passbook/core/management/commands/worker.py | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e582e7adc..abde87ea39 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,7 @@ pylint: stage: test coverage: script: + - python manage.py collectstatic --no-input - coverage run manage.py test - coverage report stage: test diff --git a/passbook/core/management/commands/web.py b/passbook/core/management/commands/web.py index 329e070e65..fe5b157bd0 100644 --- a/passbook/core/management/commands/web.py +++ b/passbook/core/management/commands/web.py @@ -15,7 +15,7 @@ class Command(BaseCommand): """Run CherryPy webserver""" def handle(self, *args, **options): - """passbook cherrypy server""" + """passbook daphne server""" autoreload.run_with_reloader(self.daphne_server) def daphne_server(self): diff --git a/passbook/core/management/commands/worker.py b/passbook/core/management/commands/worker.py index 00971ca7b0..1a20f22d58 100644 --- a/passbook/core/management/commands/worker.py +++ b/passbook/core/management/commands/worker.py @@ -3,6 +3,7 @@ from logging import getLogger from django.core.management.base import BaseCommand +from django.utils import autoreload from passbook.core.celery import CELERY_APP @@ -14,4 +15,9 @@ class Command(BaseCommand): def handle(self, *args, **options): """celery worker""" + autoreload.run_with_reloader(self.celery_worker) + + def celery_worker(self): + """Run celery worker within autoreload""" + autoreload.raise_last_exception() CELERY_APP.worker_main(['worker', '--autoscale=10,3', '-E', '-B']) From 3ff2ec929f7771a84744e38af288d30a2ce53339 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 14:03:05 +0200 Subject: [PATCH 13/23] prepare 0.1.29 --- debian/changelog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debian/changelog b/debian/changelog index 6fb7c17abd..95a6097e05 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +passbook (0.1.29) stable; urgency=medium + + * bump version: 0.1.27-beta -> 0.1.28-beta + * Add libpq-dev dependency so psycopg2 build works + * switch to whitenoise for static files + * replace cherrypy with daphne + * Run collectstatic before coverage, use autoreload on celery worker + + -- Jens Langhammer Thu, 11 Apr 2019 12:00:27 +0000 + passbook (0.1.28) stable; urgency=medium * bump version: 0.1.26-beta -> 0.1.27-beta From c90d8ddcffe66907be0ddd70bbf1d5bcf297c013 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 14:03:08 +0200 Subject: [PATCH 14/23] bump version: 0.1.28-beta -> 0.1.29-beta --- .bumpversion.cfg | 2 +- .gitlab-ci.yml | 2 +- client-packages/allauth/setup.py | 2 +- client-packages/sentry-auth-passbook/setup.py | 2 +- helm/passbook/Chart.yaml | 4 ++-- helm/passbook/values.yaml | 2 +- passbook/__init__.py | 2 +- passbook/admin/__init__.py | 2 +- passbook/api/__init__.py | 2 +- passbook/app_gw/__init__.py | 2 +- passbook/audit/__init__.py | 2 +- passbook/captcha_factor/__init__.py | 2 +- passbook/core/__init__.py | 2 +- passbook/hibp_policy/__init__.py | 2 +- passbook/ldap/__init__.py | 2 +- passbook/lib/__init__.py | 2 +- passbook/oauth_client/__init__.py | 2 +- passbook/oauth_provider/__init__.py | 2 +- passbook/otp/__init__.py | 2 +- passbook/password_expiry_policy/__init__.py | 2 +- passbook/saml_idp/__init__.py | 2 +- passbook/suspicious_policy/__init__.py | 2 +- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 938fee0f1d..4e245d77b4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.28-beta +current_version = 0.1.29-beta tag = True commit = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)\-(?P.*) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abde87ea39..801bdf118f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,7 +56,7 @@ package-docker: before_script: - echo "{\"auths\":{\"docker.$NEXUS_URL\":{\"auth\":\"$NEXUS_AUTH\"}}}" > /kaniko/.docker/config.json script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.28-beta + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.29-beta stage: build only: - tags diff --git a/client-packages/allauth/setup.py b/client-packages/allauth/setup.py index 8c0afcb9fa..73bb3ec10d 100644 --- a/client-packages/allauth/setup.py +++ b/client-packages/allauth/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup( name='django-allauth-passbook', - version='0.1.28-beta', + version='0.1.29-beta', description='passbook support for django-allauth', # long_description='\n'.join(read_simple('docs/index.md')[2:]), long_description_content_type='text/markdown', diff --git a/client-packages/sentry-auth-passbook/setup.py b/client-packages/sentry-auth-passbook/setup.py index 16f455fe11..e188f8749b 100644 --- a/client-packages/sentry-auth-passbook/setup.py +++ b/client-packages/sentry-auth-passbook/setup.py @@ -18,7 +18,7 @@ tests_require = [ setup( name='sentry-auth-passbook', - version='0.1.28-beta', + version='0.1.29-beta', author='BeryJu.org', author_email='support@beryju.org', url='https://passbook.beryju.org', diff --git a/helm/passbook/Chart.yaml b/helm/passbook/Chart.yaml index d306069552..4d87dd4a6b 100644 --- a/helm/passbook/Chart.yaml +++ b/helm/passbook/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -appVersion: "0.1.28-beta" +appVersion: "0.1.29-beta" description: A Helm chart for passbook. name: passbook -version: "0.1.28-beta" +version: "0.1.29-beta" icon: https://passbook.beryju.org/images/logo.png diff --git a/helm/passbook/values.yaml b/helm/passbook/values.yaml index 51fb71c2dc..1ed0101626 100644 --- a/helm/passbook/values.yaml +++ b/helm/passbook/values.yaml @@ -5,7 +5,7 @@ replicaCount: 1 image: - tag: 0.1.28-beta + tag: 0.1.29-beta nameOverride: "" diff --git a/passbook/__init__.py b/passbook/__init__.py index 4bc9185a48..d81e47a621 100644 --- a/passbook/__init__.py +++ b/passbook/__init__.py @@ -1,2 +1,2 @@ """passbook""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/admin/__init__.py b/passbook/admin/__init__.py index 89ce55b4e0..2d422e9efb 100644 --- a/passbook/admin/__init__.py +++ b/passbook/admin/__init__.py @@ -1,2 +1,2 @@ """passbook admin""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/api/__init__.py b/passbook/api/__init__.py index d547c82013..91f45ad7fa 100644 --- a/passbook/api/__init__.py +++ b/passbook/api/__init__.py @@ -1,2 +1,2 @@ """passbook api""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/app_gw/__init__.py b/passbook/app_gw/__init__.py index 147588d2ab..0d9a77d50c 100644 --- a/passbook/app_gw/__init__.py +++ b/passbook/app_gw/__init__.py @@ -1,2 +1,2 @@ """passbook Application Security Gateway Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/audit/__init__.py b/passbook/audit/__init__.py index 784a85e069..d1be966a11 100644 --- a/passbook/audit/__init__.py +++ b/passbook/audit/__init__.py @@ -1,2 +1,2 @@ """passbook audit Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/captcha_factor/__init__.py b/passbook/captcha_factor/__init__.py index e3fcfebd8b..13332c7c6b 100644 --- a/passbook/captcha_factor/__init__.py +++ b/passbook/captcha_factor/__init__.py @@ -1,2 +1,2 @@ """passbook captcha_factor Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/core/__init__.py b/passbook/core/__init__.py index 840f2b287c..97df8c221e 100644 --- a/passbook/core/__init__.py +++ b/passbook/core/__init__.py @@ -1,2 +1,2 @@ """passbook core""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/hibp_policy/__init__.py b/passbook/hibp_policy/__init__.py index 5be707c227..9fec995273 100644 --- a/passbook/hibp_policy/__init__.py +++ b/passbook/hibp_policy/__init__.py @@ -1,2 +1,2 @@ """passbook hibp_policy""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/ldap/__init__.py b/passbook/ldap/__init__.py index edf5fb6c44..3c5c0545ff 100644 --- a/passbook/ldap/__init__.py +++ b/passbook/ldap/__init__.py @@ -1,2 +1,2 @@ """Passbook ldap app Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/lib/__init__.py b/passbook/lib/__init__.py index 37bcb6450c..b06d9a1392 100644 --- a/passbook/lib/__init__.py +++ b/passbook/lib/__init__.py @@ -1,2 +1,2 @@ """passbook lib""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/oauth_client/__init__.py b/passbook/oauth_client/__init__.py index 93b5d5e11e..424aa297be 100644 --- a/passbook/oauth_client/__init__.py +++ b/passbook/oauth_client/__init__.py @@ -1,2 +1,2 @@ """passbook oauth_client Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/oauth_provider/__init__.py b/passbook/oauth_provider/__init__.py index 15e26c879b..5b22d389be 100644 --- a/passbook/oauth_provider/__init__.py +++ b/passbook/oauth_provider/__init__.py @@ -1,2 +1,2 @@ """passbook oauth_provider Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/otp/__init__.py b/passbook/otp/__init__.py index e886d0c6cc..f6cbbadb73 100644 --- a/passbook/otp/__init__.py +++ b/passbook/otp/__init__.py @@ -1,2 +1,2 @@ """passbook otp Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/password_expiry_policy/__init__.py b/passbook/password_expiry_policy/__init__.py index 63782d25fc..590a0bdec9 100644 --- a/passbook/password_expiry_policy/__init__.py +++ b/passbook/password_expiry_policy/__init__.py @@ -1,2 +1,2 @@ """passbook password_expiry""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/saml_idp/__init__.py b/passbook/saml_idp/__init__.py index 0f41ea548c..1fd1fb3173 100644 --- a/passbook/saml_idp/__init__.py +++ b/passbook/saml_idp/__init__.py @@ -1,2 +1,2 @@ """passbook saml_idp Header""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' diff --git a/passbook/suspicious_policy/__init__.py b/passbook/suspicious_policy/__init__.py index fc556da5c4..97d1123775 100644 --- a/passbook/suspicious_policy/__init__.py +++ b/passbook/suspicious_policy/__init__.py @@ -1,2 +1,2 @@ """passbook suspicious_policy""" -__version__ = '0.1.28-beta' +__version__ = '0.1.29-beta' From 045a802365bcd97a0ba3d51c9d757b44a461c4fe Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 14:22:32 +0200 Subject: [PATCH 15/23] don't use context manager in web command --- debian/changelog | 7 +++++++ passbook/core/management/commands/web.py | 13 ++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index 95a6097e05..619c907de1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +passbook (0.1.30) stable; urgency=medium + + * bump version: 0.1.28-beta -> 0.1.29-beta + * don't use context manager in web command + + -- Jens Langhammer Thu, 11 Apr 2019 12:21:58 +0000 + passbook (0.1.29) stable; urgency=medium * bump version: 0.1.27-beta -> 0.1.28-beta diff --git a/passbook/core/management/commands/web.py b/passbook/core/management/commands/web.py index fe5b157bd0..5bd865adc1 100644 --- a/passbook/core/management/commands/web.py +++ b/passbook/core/management/commands/web.py @@ -21,10 +21,9 @@ class Command(BaseCommand): def daphne_server(self): """Run daphne server within autoreload""" autoreload.raise_last_exception() - with CONFIG.cd('web'): - CommandLineInterface().run([ - '-p', str(CONFIG.get('port', 8000)), - '-b', CONFIG.get('listen', '0.0.0.0'), # nosec - '--access-log', '/dev/null', - 'passbook.core.asgi:application' - ]) + CommandLineInterface().run([ + '-p', str(CONFIG.y('web.port', 8000)), + '-b', CONFIG.y('web.listen', '0.0.0.0'), # nosec + '--access-log', '/dev/null', + 'passbook.core.asgi:application' + ]) From 146edb45d40cea09a8ad71febe47038576faea7d Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 14:22:34 +0200 Subject: [PATCH 16/23] bump version: 0.1.29-beta -> 0.1.30-beta --- .bumpversion.cfg | 2 +- .gitlab-ci.yml | 2 +- client-packages/allauth/setup.py | 2 +- client-packages/sentry-auth-passbook/setup.py | 2 +- helm/passbook/Chart.yaml | 4 ++-- helm/passbook/values.yaml | 2 +- passbook/__init__.py | 2 +- passbook/admin/__init__.py | 2 +- passbook/api/__init__.py | 2 +- passbook/app_gw/__init__.py | 2 +- passbook/audit/__init__.py | 2 +- passbook/captcha_factor/__init__.py | 2 +- passbook/core/__init__.py | 2 +- passbook/hibp_policy/__init__.py | 2 +- passbook/ldap/__init__.py | 2 +- passbook/lib/__init__.py | 2 +- passbook/oauth_client/__init__.py | 2 +- passbook/oauth_provider/__init__.py | 2 +- passbook/otp/__init__.py | 2 +- passbook/password_expiry_policy/__init__.py | 2 +- passbook/saml_idp/__init__.py | 2 +- passbook/suspicious_policy/__init__.py | 2 +- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4e245d77b4..93603d73fb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.29-beta +current_version = 0.1.30-beta tag = True commit = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)\-(?P.*) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 801bdf118f..8e69f2f520 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,7 +56,7 @@ package-docker: before_script: - echo "{\"auths\":{\"docker.$NEXUS_URL\":{\"auth\":\"$NEXUS_AUTH\"}}}" > /kaniko/.docker/config.json script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.29-beta + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.30-beta stage: build only: - tags diff --git a/client-packages/allauth/setup.py b/client-packages/allauth/setup.py index 73bb3ec10d..7fc5537511 100644 --- a/client-packages/allauth/setup.py +++ b/client-packages/allauth/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup( name='django-allauth-passbook', - version='0.1.29-beta', + version='0.1.30-beta', description='passbook support for django-allauth', # long_description='\n'.join(read_simple('docs/index.md')[2:]), long_description_content_type='text/markdown', diff --git a/client-packages/sentry-auth-passbook/setup.py b/client-packages/sentry-auth-passbook/setup.py index e188f8749b..0cf7a587ee 100644 --- a/client-packages/sentry-auth-passbook/setup.py +++ b/client-packages/sentry-auth-passbook/setup.py @@ -18,7 +18,7 @@ tests_require = [ setup( name='sentry-auth-passbook', - version='0.1.29-beta', + version='0.1.30-beta', author='BeryJu.org', author_email='support@beryju.org', url='https://passbook.beryju.org', diff --git a/helm/passbook/Chart.yaml b/helm/passbook/Chart.yaml index 4d87dd4a6b..1d48529c86 100644 --- a/helm/passbook/Chart.yaml +++ b/helm/passbook/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -appVersion: "0.1.29-beta" +appVersion: "0.1.30-beta" description: A Helm chart for passbook. name: passbook -version: "0.1.29-beta" +version: "0.1.30-beta" icon: https://passbook.beryju.org/images/logo.png diff --git a/helm/passbook/values.yaml b/helm/passbook/values.yaml index 1ed0101626..cdf354c937 100644 --- a/helm/passbook/values.yaml +++ b/helm/passbook/values.yaml @@ -5,7 +5,7 @@ replicaCount: 1 image: - tag: 0.1.29-beta + tag: 0.1.30-beta nameOverride: "" diff --git a/passbook/__init__.py b/passbook/__init__.py index d81e47a621..834d18188f 100644 --- a/passbook/__init__.py +++ b/passbook/__init__.py @@ -1,2 +1,2 @@ """passbook""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/admin/__init__.py b/passbook/admin/__init__.py index 2d422e9efb..0ff885c8aa 100644 --- a/passbook/admin/__init__.py +++ b/passbook/admin/__init__.py @@ -1,2 +1,2 @@ """passbook admin""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/api/__init__.py b/passbook/api/__init__.py index 91f45ad7fa..e3e8de0fa3 100644 --- a/passbook/api/__init__.py +++ b/passbook/api/__init__.py @@ -1,2 +1,2 @@ """passbook api""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/app_gw/__init__.py b/passbook/app_gw/__init__.py index 0d9a77d50c..e2040c202d 100644 --- a/passbook/app_gw/__init__.py +++ b/passbook/app_gw/__init__.py @@ -1,2 +1,2 @@ """passbook Application Security Gateway Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/audit/__init__.py b/passbook/audit/__init__.py index d1be966a11..05b4dda7ad 100644 --- a/passbook/audit/__init__.py +++ b/passbook/audit/__init__.py @@ -1,2 +1,2 @@ """passbook audit Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/captcha_factor/__init__.py b/passbook/captcha_factor/__init__.py index 13332c7c6b..87c6285c00 100644 --- a/passbook/captcha_factor/__init__.py +++ b/passbook/captcha_factor/__init__.py @@ -1,2 +1,2 @@ """passbook captcha_factor Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/core/__init__.py b/passbook/core/__init__.py index 97df8c221e..7ee0e2b2d1 100644 --- a/passbook/core/__init__.py +++ b/passbook/core/__init__.py @@ -1,2 +1,2 @@ """passbook core""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/hibp_policy/__init__.py b/passbook/hibp_policy/__init__.py index 9fec995273..9ca1709332 100644 --- a/passbook/hibp_policy/__init__.py +++ b/passbook/hibp_policy/__init__.py @@ -1,2 +1,2 @@ """passbook hibp_policy""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/ldap/__init__.py b/passbook/ldap/__init__.py index 3c5c0545ff..2e36a67610 100644 --- a/passbook/ldap/__init__.py +++ b/passbook/ldap/__init__.py @@ -1,2 +1,2 @@ """Passbook ldap app Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/lib/__init__.py b/passbook/lib/__init__.py index b06d9a1392..14a931e2b0 100644 --- a/passbook/lib/__init__.py +++ b/passbook/lib/__init__.py @@ -1,2 +1,2 @@ """passbook lib""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/oauth_client/__init__.py b/passbook/oauth_client/__init__.py index 424aa297be..fc22248ca2 100644 --- a/passbook/oauth_client/__init__.py +++ b/passbook/oauth_client/__init__.py @@ -1,2 +1,2 @@ """passbook oauth_client Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/oauth_provider/__init__.py b/passbook/oauth_provider/__init__.py index 5b22d389be..f518cd8fba 100644 --- a/passbook/oauth_provider/__init__.py +++ b/passbook/oauth_provider/__init__.py @@ -1,2 +1,2 @@ """passbook oauth_provider Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/otp/__init__.py b/passbook/otp/__init__.py index f6cbbadb73..dd171e616a 100644 --- a/passbook/otp/__init__.py +++ b/passbook/otp/__init__.py @@ -1,2 +1,2 @@ """passbook otp Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/password_expiry_policy/__init__.py b/passbook/password_expiry_policy/__init__.py index 590a0bdec9..1885035134 100644 --- a/passbook/password_expiry_policy/__init__.py +++ b/passbook/password_expiry_policy/__init__.py @@ -1,2 +1,2 @@ """passbook password_expiry""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/saml_idp/__init__.py b/passbook/saml_idp/__init__.py index 1fd1fb3173..d2fe9d1c3d 100644 --- a/passbook/saml_idp/__init__.py +++ b/passbook/saml_idp/__init__.py @@ -1,2 +1,2 @@ """passbook saml_idp Header""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' diff --git a/passbook/suspicious_policy/__init__.py b/passbook/suspicious_policy/__init__.py index 97d1123775..2b5bb7024d 100644 --- a/passbook/suspicious_policy/__init__.py +++ b/passbook/suspicious_policy/__init__.py @@ -1,2 +1,2 @@ """passbook suspicious_policy""" -__version__ = '0.1.29-beta' +__version__ = '0.1.30-beta' From f69f959bdb9274c0f776be25a86bb27cbd8ec635 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 15:29:01 +0200 Subject: [PATCH 17/23] allow setting authentication_header to empty string (disabling the header) --- .../migrations/0003_auto_20190411_1314.py | 18 ++++++++++++++++++ passbook/app_gw/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 passbook/app_gw/migrations/0003_auto_20190411_1314.py diff --git a/passbook/app_gw/migrations/0003_auto_20190411_1314.py b/passbook/app_gw/migrations/0003_auto_20190411_1314.py new file mode 100644 index 0000000000..28434b016f --- /dev/null +++ b/passbook/app_gw/migrations/0003_auto_20190411_1314.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2019-04-11 13:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('passbook_app_gw', '0002_auto_20190321_1521'), + ] + + operations = [ + migrations.AlterField( + model_name='applicationgatewayprovider', + name='authentication_header', + field=models.TextField(blank=True, default='X-Remote-User'), + ), + ] diff --git a/passbook/app_gw/models.py b/passbook/app_gw/models.py index 1bdf2b80b6..dec46bdc0c 100644 --- a/passbook/app_gw/models.py +++ b/passbook/app_gw/models.py @@ -15,7 +15,7 @@ class ApplicationGatewayProvider(Provider): upstream = ArrayField(models.TextField()) enabled = models.BooleanField(default=True) - authentication_header = models.TextField(default='X-Remote-User') + authentication_header = models.TextField(default='X-Remote-User', blank=True) default_content_type = models.TextField(default='application/octet-stream') upstream_ssl_verification = models.BooleanField(default=True) From 61478db94eb194fa6eb446ac67e055d87d22143d Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 15:29:35 +0200 Subject: [PATCH 18/23] use global urllib Pools --- passbook/app_gw/middleware.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/passbook/app_gw/middleware.py b/passbook/app_gw/middleware.py index f491468798..d1a064308f 100644 --- a/passbook/app_gw/middleware.py +++ b/passbook/app_gw/middleware.py @@ -27,6 +27,11 @@ ERRORS_MESSAGES = { 'upstream-no-scheme': ("Upstream URL scheme must be either " "'http' or 'https' (%s).") } +HTTP_NO_VERIFY = urllib3.PoolManager() +HTTP = urllib3.PoolManager( + cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where()) + # pylint: disable=too-many-instance-attributes class ApplicationGatewayMiddleware: @@ -45,10 +50,6 @@ class ApplicationGatewayMiddleware: def __init__(self, get_response): self.get_response = get_response self.ignored_hosts = cache.get(IGNORED_HOSTNAMES_KEY, []) - self.http_no_verify = urllib3.PoolManager() - self.http = urllib3.PoolManager( - cert_reqs='CERT_REQUIRED', - ca_certs=certifi.where()) def precheck(self, request): """Check if a request should be proxied or forwarded to passbook""" @@ -137,6 +138,8 @@ class ApplicationGatewayMiddleware: .. versionadded:: 0.9.8 """ request_headers = self.get_proxy_request_headers(self.request) + if not self.app_gw.authentication_header: + return request_headers request_headers[self.app_gw.authentication_header] = self.request.user.get_username() LOGGER.info("%s set", self.app_gw.authentication_header) @@ -171,9 +174,9 @@ class ApplicationGatewayMiddleware: request_url += '?' + self.get_encoded_query_params() LOGGER.debug("Request URL: %s", request_url) - http = self.http + http = HTTP if not self.app_gw.upstream_ssl_verification: - http = self.http_no_verify + http = HTTP_NO_VERIFY try: proxy_response = http.urlopen(request.method, From 755045b22667ffa9b7830f446c9f052230bbadd4 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 15:30:07 +0200 Subject: [PATCH 19/23] try to fix app_gw being null --- passbook/app_gw/middleware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/passbook/app_gw/middleware.py b/passbook/app_gw/middleware.py index d1a064308f..53ab5d6179 100644 --- a/passbook/app_gw/middleware.py +++ b/passbook/app_gw/middleware.py @@ -77,10 +77,10 @@ class ApplicationGatewayMiddleware: try: # Check if ApplicationGateway is associated with application getattr(app_gw, 'application') - return False, app_gw + if app_gw: + return False, app_gw except Application.DoesNotExist: LOGGER.debug("ApplicationGateway not associated with Application") - return True, None return True, None def __call__(self, request): From 16eb629b71531a6d92bd2daae086f2cfcfef349b Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Thu, 11 Apr 2019 15:30:42 +0200 Subject: [PATCH 20/23] only enable sentry when not DEBUG --- passbook/core/settings.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/passbook/core/settings.py b/passbook/core/settings.py index 144dbd5161..d0d52f71c5 100644 --- a/passbook/core/settings.py +++ b/passbook/core/settings.py @@ -217,19 +217,21 @@ CELERY_BEAT_SCHEDULE = { } } -sentry_init( - dsn=("https://55b5dd780bc14f4c96bba69b7a9abbcc:449af483bd0745" - "0d83be640d834e5458@sentry.services.beryju.org/8"), - integrations=[ - DjangoIntegration(), - CeleryIntegration(), - LoggingIntegration( - level=logging.INFO, - event_level=logging.ERROR - ) - ], - send_default_pii=True -) + +if not DEBUG: + sentry_init( + dsn=("https://55b5dd780bc14f4c96bba69b7a9abbcc:449af483bd0745" + "0d83be640d834e5458@sentry.services.beryju.org/8"), + integrations=[ + DjangoIntegration(), + CeleryIntegration(), + LoggingIntegration( + level=logging.INFO, + event_level=logging.ERROR + ) + ], + send_default_pii=True, + ) # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.1/howto/static-files/ @@ -315,7 +317,7 @@ with CONFIG.cd('log'): }, 'daphne': { 'handlers': LOG_HANDLERS, - 'level': 'DEBUG', + 'level': 'INFO', 'propagate': True, } } From 940b3eb94323fd4b60c64c76ae6aceb688dd231a Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Sat, 13 Apr 2019 16:04:48 +0200 Subject: [PATCH 21/23] move logging to separate thread --- passbook/core/settings.py | 29 ++++++++++++++++++----------- passbook/lib/log.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 passbook/lib/log.py diff --git a/passbook/core/settings.py b/passbook/core/settings.py index d0d52f71c5..971b1bcfa1 100644 --- a/passbook/core/settings.py +++ b/passbook/core/settings.py @@ -239,8 +239,6 @@ if not DEBUG: STATIC_URL = '/static/' STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' -LOG_HANDLERS = ['console', 'syslog', 'file'] - with CONFIG.cd('log'): LOGGING = { 'version': 1, @@ -283,41 +281,50 @@ with CONFIG.cd('log'): 'formatter': 'verbose', 'filename': CONFIG.get('file'), }, + 'queue': { + 'level': CONFIG.get('level').get('console'), + 'class': 'passbook.lib.log.QueueListenerHandler', + 'handlers': [ + 'cfg://handlers.console', + # 'cfg://handlers.syslog', + 'cfg://handlers.file', + ], + } }, 'loggers': { 'passbook': { - 'handlers': LOG_HANDLERS, + 'handlers': ['queue'], 'level': 'DEBUG', 'propagate': True, }, 'django': { - 'handlers': LOG_HANDLERS, - 'level': 'INFO', + 'handlers': ['queue'], + 'level': 'DEBUG', 'propagate': True, }, 'tasks': { - 'handlers': LOG_HANDLERS, + 'handlers': ['queue'], 'level': 'DEBUG', 'propagate': True, }, 'cherrypy': { - 'handlers': LOG_HANDLERS, + 'handlers': ['queue'], 'level': 'DEBUG', 'propagate': True, }, 'oauthlib': { - 'handlers': LOG_HANDLERS, + 'handlers': ['queue'], 'level': 'DEBUG', 'propagate': True, }, 'oauth2_provider': { - 'handlers': LOG_HANDLERS, + 'handlers': ['queue'], 'level': 'DEBUG', 'propagate': True, }, 'daphne': { - 'handlers': LOG_HANDLERS, - 'level': 'INFO', + 'handlers': ['queue'], + 'level': 'DEBUG', 'propagate': True, } } diff --git a/passbook/lib/log.py b/passbook/lib/log.py new file mode 100644 index 0000000000..b4a72f9170 --- /dev/null +++ b/passbook/lib/log.py @@ -0,0 +1,38 @@ +from atexit import register +from logging.config import ConvertingDict, ConvertingList, valid_ident +from logging.handlers import QueueHandler, QueueListener +from queue import Queue + +from django.conf import settings + + +def _resolve_handlers(l): + # import pudb; pu.db + if not isinstance(l, ConvertingList): + return l + + # Indexing the list performs the evaluation. + return [l[i] for i in range(len(l))] + + +class QueueListenerHandler(QueueHandler): + + def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)): + super().__init__(queue) + handlers = _resolve_handlers(handlers) + self._listener = QueueListener( + self.queue, + *handlers, + respect_handler_level=respect_handler_level) + if auto_run: + self.start() + register(self.stop) + + def start(self): + self._listener.start() + + def stop(self): + self._listener.stop() + + def emit(self, record): + return super().emit(record) From 9b5b03647b29469177fff38c122ac994352b38ac Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Sat, 13 Apr 2019 16:05:11 +0200 Subject: [PATCH 22/23] move actual proxying logic to separate class --- passbook/app_gw/middleware.py | 236 ++----------------------------- passbook/app_gw/proxy/handler.py | 222 +++++++++++++++++++++++++++++ passbook/app_gw/signals.py | 2 +- passbook/core/policies.py | 20 ++- 4 files changed, 254 insertions(+), 226 deletions(-) create mode 100644 passbook/app_gw/proxy/handler.py diff --git a/passbook/app_gw/middleware.py b/passbook/app_gw/middleware.py index 53ab5d6179..919e5c6b5f 100644 --- a/passbook/app_gw/middleware.py +++ b/passbook/app_gw/middleware.py @@ -1,241 +1,33 @@ """passbook app_gw middleware""" -import mimetypes -from logging import getLogger -from random import SystemRandom -from urllib.parse import urlparse - -import certifi -import urllib3 -from django.core.cache import cache -from django.utils.http import urlencode from django.views.generic import RedirectView -from revproxy.exceptions import InvalidUpstream -from passbook.app_gw.models import ApplicationGatewayProvider -from passbook.app_gw.proxy.response import get_django_response -from passbook.app_gw.proxy.utils import encode_items, normalize_request_headers -from passbook.app_gw.rewrite import Rewriter -from passbook.core.models import Application -from passbook.core.policies import PolicyEngine +from passbook.app_gw.proxy.handler import RequestHandler from passbook.lib.config import CONFIG -SESSION_UPSTREAM_KEY = 'passbook_app_gw_upstream' -IGNORED_HOSTNAMES_KEY = 'passbook_app_gw_ignored' -LOGGER = getLogger(__name__) -QUOTE_SAFE = r'<.;>\(}*+|~=-$/_:^@)[{]&\'!,"`' -ERRORS_MESSAGES = { - 'upstream-no-scheme': ("Upstream URL scheme must be either " - "'http' or 'https' (%s).") -} -HTTP_NO_VERIFY = urllib3.PoolManager() -HTTP = urllib3.PoolManager( - cert_reqs='CERT_REQUIRED', - ca_certs=certifi.where()) - -# pylint: disable=too-many-instance-attributes class ApplicationGatewayMiddleware: """Check if request should be proxied or handeled normally""" - ignored_hosts = [] - request = None - app_gw = None - http = None - http_no_verify = None - host_header = '' - - _parsed_url = None - _request_headers = None + _app_gw_cache = {} def __init__(self, get_response): self.get_response = get_response - self.ignored_hosts = cache.get(IGNORED_HOSTNAMES_KEY, []) - - def precheck(self, request): - """Check if a request should be proxied or forwarded to passbook""" - # Check if hostname is in cached list of ignored hostnames - # This saves us having to query the database on each request - self.host_header = request.META.get('HTTP_HOST') - if self.host_header in self.ignored_hosts: - LOGGER.debug("%s is ignored", self.host_header) - return True, None - # Look through all ApplicationGatewayProviders and check hostnames - matches = ApplicationGatewayProvider.objects.filter( - server_name__contains=[self.host_header], - enabled=True) - if not matches.exists(): - # Mo matching Providers found, add host header to ignored list - self.ignored_hosts.append(self.host_header) - cache.set(IGNORED_HOSTNAMES_KEY, self.ignored_hosts) - LOGGER.debug("Ignoring %s", self.host_header) - return True, None - # At this point we're certain there's a matching ApplicationGateway - if len(matches) > 1: - # This should never happen - raise ValueError - app_gw = matches.first() - try: - # Check if ApplicationGateway is associated with application - getattr(app_gw, 'application') - if app_gw: - return False, app_gw - except Application.DoesNotExist: - LOGGER.debug("ApplicationGateway not associated with Application") - return True, None def __call__(self, request): - forward, self.app_gw = self.precheck(request) - if forward: - return self.get_response(request) - self.request = request - return self.dispatch(request) + # Rudimentary cache + host_header = request.META.get('HTTP_HOST') + if host_header not in self._app_gw_cache: + self._app_gw_cache[host_header] = RequestHandler.find_app_gw_for_request(request) + if self._app_gw_cache[host_header]: + return self.dispatch(request, self._app_gw_cache[host_header]) + return self.get_response(request) - def _get_upstream(self): - """Choose random upstream and save in session""" - if SESSION_UPSTREAM_KEY not in self.request.session: - self.request.session[SESSION_UPSTREAM_KEY] = {} - if self.app_gw.pk not in self.request.session[SESSION_UPSTREAM_KEY]: - upstream_index = SystemRandom().randrange(len(self.app_gw.upstream)) - self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk] = upstream_index - return self.app_gw.upstream[self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk]] - - def get_upstream(self): - """Get upstream as parsed url""" - upstream = self._get_upstream() - - self._parsed_url = urlparse(upstream) - - if self._parsed_url.scheme not in ('http', 'https'): - raise InvalidUpstream(ERRORS_MESSAGES['upstream-no-scheme'] % - upstream) - - return upstream - - def _format_path_to_redirect(self, request): - LOGGER.debug("Path before: %s", request.get_full_path()) - rewriter = Rewriter(self.app_gw, request) - after = rewriter.build() - LOGGER.debug("Path after: %s", after) - return after - - def get_proxy_request_headers(self, request): - """Get normalized headers for the upstream - Gets all headers from the original request and normalizes them. - Normalization occurs by removing the prefix ``HTTP_`` and - replacing and ``_`` by ``-``. Example: ``HTTP_ACCEPT_ENCODING`` - becames ``Accept-Encoding``. - .. versionadded:: 0.9.1 - :param request: The original HTTPRequest instance - :returns: Normalized headers for the upstream - """ - return normalize_request_headers(request) - - def get_request_headers(self): - """Return request headers that will be sent to upstream. - The header REMOTE_USER is set to the current user - if AuthenticationMiddleware is enabled and - the view's add_remote_user property is True. - .. versionadded:: 0.9.8 - """ - request_headers = self.get_proxy_request_headers(self.request) - if not self.app_gw.authentication_header: - return request_headers - request_headers[self.app_gw.authentication_header] = self.request.user.get_username() - LOGGER.info("%s set", self.app_gw.authentication_header) - - return request_headers - - def check_permission(self): - """Check if user is authenticated and has permission to access app""" - if not hasattr(self.request, 'user'): - return False - if not self.request.user.is_authenticated: - return False - policy_engine = PolicyEngine(self.app_gw.application.policies.all()) - policy_engine.for_user(self.request.user).with_request(self.request).build() - passing, _messages = policy_engine.result - - return passing - - def get_encoded_query_params(self): - """Return encoded query params to be used in proxied request""" - get_data = encode_items(self.request.GET.lists()) - return urlencode(get_data) - - def _created_proxy_response(self, request, path): - request_payload = request.body - - LOGGER.debug("Request headers: %s", self._request_headers) - - request_url = self.get_upstream() + path - LOGGER.debug("Request URL: %s", request_url) - - if request.GET: - request_url += '?' + self.get_encoded_query_params() - LOGGER.debug("Request URL: %s", request_url) - - http = HTTP - if not self.app_gw.upstream_ssl_verification: - http = HTTP_NO_VERIFY - - try: - proxy_response = http.urlopen(request.method, - request_url, - redirect=False, - retries=None, - headers=self._request_headers, - body=request_payload, - decode_content=False, - preload_content=False) - LOGGER.debug("Proxy response header: %s", - proxy_response.getheaders()) - except urllib3.exceptions.HTTPError as error: - LOGGER.exception(error) - raise - - return proxy_response - - def _replace_host_on_redirect_location(self, request, proxy_response): - location = proxy_response.headers.get('Location') - if location: - if request.is_secure(): - scheme = 'https://' - else: - scheme = 'http://' - request_host = scheme + self.host_header - - upstream_host_http = 'http://' + self._parsed_url.netloc - upstream_host_https = 'https://' + self._parsed_url.netloc - - location = location.replace(upstream_host_http, request_host) - location = location.replace(upstream_host_https, request_host) - proxy_response.headers['Location'] = location - LOGGER.debug("Proxy response LOCATION: %s", - proxy_response.headers['Location']) - - def _set_content_type(self, request, proxy_response): - content_type = proxy_response.headers.get('Content-Type') - if not content_type: - content_type = (mimetypes.guess_type(request.path)[0] or - self.app_gw.default_content_type) - proxy_response.headers['Content-Type'] = content_type - LOGGER.debug("Proxy response CONTENT-TYPE: %s", - proxy_response.headers['Content-Type']) - - def dispatch(self, request): + def dispatch(self, request, app_gw): """Build proxied request and pass to upstream""" - if not self.check_permission(): + handler = RequestHandler(app_gw, request) + + if not handler.check_permission(): to_url = 'https://%s/?next=%s' % (CONFIG.get('domains')[0], request.get_full_path()) return RedirectView.as_view(url=to_url)(request) - self._request_headers = self.get_request_headers() - - path = self._format_path_to_redirect(request) - proxy_response = self._created_proxy_response(request, path) - - self._replace_host_on_redirect_location(request, proxy_response) - self._set_content_type(request, proxy_response) - response = get_django_response(proxy_response, strict_cookies=False) - - LOGGER.debug("RESPONSE RETURNED: %s", response) - return response + return handler.get_response() diff --git a/passbook/app_gw/proxy/handler.py b/passbook/app_gw/proxy/handler.py new file mode 100644 index 0000000000..2f63ac6ada --- /dev/null +++ b/passbook/app_gw/proxy/handler.py @@ -0,0 +1,222 @@ +"""passbook app_gw request handler""" +import mimetypes +from logging import getLogger +from random import SystemRandom +from urllib.parse import urlparse + +import certifi +import urllib3 +from django.core.cache import cache +from django.utils.http import urlencode + +from passbook.app_gw.models import ApplicationGatewayProvider +from passbook.app_gw.proxy.exceptions import InvalidUpstream +from passbook.app_gw.proxy.response import get_django_response +from passbook.app_gw.proxy.utils import encode_items, normalize_request_headers +from passbook.app_gw.rewrite import Rewriter +from passbook.core.models import Application +from passbook.core.policies import PolicyEngine + +SESSION_UPSTREAM_KEY = 'passbook_app_gw_upstream' +IGNORED_HOSTNAMES_KEY = 'passbook_app_gw_ignored' +LOGGER = getLogger(__name__) +QUOTE_SAFE = r'<.;>\(}*+|~=-$/_:^@)[{]&\'!,"`' +ERRORS_MESSAGES = { + 'upstream-no-scheme': ("Upstream URL scheme must be either " + "'http' or 'https' (%s).") +} +HTTP_NO_VERIFY = urllib3.PoolManager() +HTTP = urllib3.PoolManager( + cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where()) +IGNORED_HOSTS = cache.get(IGNORED_HOSTNAMES_KEY, []) + + +class RequestHandler: + """Forward requests""" + + _parsed_url = None + _request_headers = None + + def __init__(self, app_gw, request): + self.app_gw = app_gw + self.request = request + + @staticmethod + def find_app_gw_for_request(request): + """Check if a request should be proxied or forwarded to passbook""" + # Check if hostname is in cached list of ignored hostnames + # This saves us having to query the database on each request + host_header = request.META.get('HTTP_HOST') + if host_header in IGNORED_HOSTS: + LOGGER.debug("%s is ignored", host_header) + return False + # Look through all ApplicationGatewayProviders and check hostnames + matches = ApplicationGatewayProvider.objects.filter( + server_name__contains=[host_header], + enabled=True) + if not matches.exists(): + # Mo matching Providers found, add host header to ignored list + IGNORED_HOSTS.append(host_header) + cache.set(IGNORED_HOSTNAMES_KEY, IGNORED_HOSTS) + LOGGER.debug("Ignoring %s", host_header) + return False + # At this point we're certain there's a matching ApplicationGateway + if len(matches) > 1: + # This should never happen + raise ValueError + app_gw = matches.first() + try: + # Check if ApplicationGateway is associated with application + getattr(app_gw, 'application') + if app_gw: + return app_gw + except Application.DoesNotExist: + LOGGER.debug("ApplicationGateway not associated with Application") + return True + + def _get_upstream(self): + """Choose random upstream and save in session""" + if SESSION_UPSTREAM_KEY not in self.request.session: + self.request.session[SESSION_UPSTREAM_KEY] = {} + if self.app_gw.pk not in self.request.session[SESSION_UPSTREAM_KEY]: + upstream_index = int(SystemRandom().random() * len(self.app_gw.upstream)) + self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk] = upstream_index + return self.app_gw.upstream[self.request.session[SESSION_UPSTREAM_KEY][self.app_gw.pk]] + + def get_upstream(self): + """Get upstream as parsed url""" + upstream = self._get_upstream() + + self._parsed_url = urlparse(upstream) + + if self._parsed_url.scheme not in ('http', 'https'): + raise InvalidUpstream(ERRORS_MESSAGES['upstream-no-scheme'] % + upstream) + + return upstream + + def _format_path_to_redirect(self): + LOGGER.debug("Path before: %s", self.request.get_full_path()) + rewriter = Rewriter(self.app_gw, self.request) + after = rewriter.build() + LOGGER.debug("Path after: %s", after) + return after + + def get_proxy_request_headers(self): + """Get normalized headers for the upstream + Gets all headers from the original request and normalizes them. + Normalization occurs by removing the prefix ``HTTP_`` and + replacing and ``_`` by ``-``. Example: ``HTTP_ACCEPT_ENCODING`` + becames ``Accept-Encoding``. + .. versionadded:: 0.9.1 + :param request: The original HTTPRequest instance + :returns: Normalized headers for the upstream + """ + return normalize_request_headers(self.request) + + def get_request_headers(self): + """Return request headers that will be sent to upstream. + The header REMOTE_USER is set to the current user + if AuthenticationMiddleware is enabled and + the view's add_remote_user property is True. + .. versionadded:: 0.9.8 + """ + request_headers = self.get_proxy_request_headers() + if not self.app_gw.authentication_header: + return request_headers + request_headers[self.app_gw.authentication_header] = self.request.user.get_username() + LOGGER.info("%s set", self.app_gw.authentication_header) + + return request_headers + + def check_permission(self): + """Check if user is authenticated and has permission to access app""" + if not hasattr(self.request, 'user'): + return False + if not self.request.user.is_authenticated: + return False + policy_engine = PolicyEngine(self.app_gw.application.policies.all()) + policy_engine.for_user(self.request.user).with_request(self.request).build() + passing, _messages = policy_engine.result + + return passing + + def get_encoded_query_params(self): + """Return encoded query params to be used in proxied request""" + get_data = encode_items(self.request.GET.lists()) + return urlencode(get_data) + + def _created_proxy_response(self, path): + request_payload = self.request.body + + LOGGER.debug("Request headers: %s", self._request_headers) + + request_url = self.get_upstream() + path + LOGGER.debug("Request URL: %s", request_url) + + if self.request.GET: + request_url += '?' + self.get_encoded_query_params() + LOGGER.debug("Request URL: %s", request_url) + + http = HTTP + if not self.app_gw.upstream_ssl_verification: + http = HTTP_NO_VERIFY + + try: + proxy_response = http.urlopen(self.request.method, + request_url, + redirect=False, + retries=None, + headers=self._request_headers, + body=request_payload, + decode_content=False, + preload_content=False) + LOGGER.debug("Proxy response header: %s", + proxy_response.getheaders()) + except urllib3.exceptions.HTTPError as error: + LOGGER.exception(error) + raise + + return proxy_response + + def _replace_host_on_redirect_location(self, proxy_response): + location = proxy_response.headers.get('Location') + if location: + if self.request.is_secure(): + scheme = 'https://' + else: + scheme = 'http://' + request_host = scheme + self.request.META.get('HTTP_HOST') + + upstream_host_http = 'http://' + self._parsed_url.netloc + upstream_host_https = 'https://' + self._parsed_url.netloc + + location = location.replace(upstream_host_http, request_host) + location = location.replace(upstream_host_https, request_host) + proxy_response.headers['Location'] = location + LOGGER.debug("Proxy response LOCATION: %s", + proxy_response.headers['Location']) + + def _set_content_type(self, proxy_response): + content_type = proxy_response.headers.get('Content-Type') + if not content_type: + content_type = (mimetypes.guess_type(self.request.path)[0] or + self.app_gw.default_content_type) + proxy_response.headers['Content-Type'] = content_type + LOGGER.debug("Proxy response CONTENT-TYPE: %s", + proxy_response.headers['Content-Type']) + + def get_response(self): + """Pass request to upstream and return response""" + self._request_headers = self.get_request_headers() + + path = self._format_path_to_redirect() + proxy_response = self._created_proxy_response(path) + + self._replace_host_on_redirect_location(proxy_response) + self._set_content_type(proxy_response) + response = get_django_response(proxy_response, strict_cookies=False) + + LOGGER.debug("RESPONSE RETURNED: %s", response) + return response diff --git a/passbook/app_gw/signals.py b/passbook/app_gw/signals.py index cb07171eb0..163432681d 100644 --- a/passbook/app_gw/signals.py +++ b/passbook/app_gw/signals.py @@ -6,8 +6,8 @@ from django.core.cache import cache from django.db.models.signals import post_save from django.dispatch import receiver -from passbook.app_gw.middleware import IGNORED_HOSTNAMES_KEY from passbook.app_gw.models import ApplicationGatewayProvider +from passbook.app_gw.proxy.handler import IGNORED_HOSTNAMES_KEY LOGGER = getLogger(__name__) diff --git a/passbook/core/policies.py b/passbook/core/policies.py index e495f018e3..4d9c52aead 100644 --- a/passbook/core/policies.py +++ b/passbook/core/policies.py @@ -1,4 +1,5 @@ """passbook core policy engine""" +import cProfile from logging import getLogger from amqp.exceptions import UnexpectedFrame @@ -10,6 +11,18 @@ from ipware import get_client_ip from passbook.core.celery import CELERY_APP from passbook.core.models import Policy, User + +def profileit(func): + def wrapper(*args, **kwargs): + datafn = func.__name__ + ".profile" # Name the data file sensibly + prof = cProfile.Profile() + retval = prof.runcall(func, *args, **kwargs) + prof.dump_stats(datafn) + return retval + + return wrapper + + LOGGER = getLogger(__name__) def _cache_key(policy, user): @@ -66,6 +79,7 @@ class PolicyEngine: self.__request = request return self + @profileit def build(self): """Build task group""" if not self.__user: @@ -82,16 +96,16 @@ class PolicyEngine: for policy in self.policies: cached_policy = cache.get(_cache_key(policy, self.__user), None) if cached_policy: - LOGGER.debug("Taking result from cache for %s", policy.pk.hex) + LOGGER.warning("Taking result from cache for %s", policy.pk.hex) cached_policies.append(cached_policy) else: - LOGGER.debug("Evaluating policy %s", policy.pk.hex) + LOGGER.warning("Evaluating policy %s", policy.pk.hex) signatures.append(_policy_engine_task.signature( args=(self.__user.pk, policy.pk.hex), kwargs=kwargs, time_limit=policy.timeout)) self.__get_timeout += policy.timeout - LOGGER.debug("Set total policy timeout to %r", self.__get_timeout) + LOGGER.warning("Set total policy timeout to %r", self.__get_timeout) # If all policies are cached, we have an empty list here. if signatures: self.__group = group(signatures)() From dda41af5c8c3aa72519dccdd19c8327313d16f9d Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Sat, 13 Apr 2019 17:22:03 +0200 Subject: [PATCH 23/23] remove logging to increase speed, add more caching to policy and rewriter --- passbook/app_gw/proxy/handler.py | 41 ++++++++++++++------------ passbook/app_gw/{ => proxy}/rewrite.py | 6 +++- passbook/core/policies.py | 35 +++++++--------------- passbook/core/settings.py | 4 +-- passbook/lib/log.py | 25 ++++++++-------- 5 files changed, 51 insertions(+), 60 deletions(-) rename passbook/app_gw/{ => proxy}/rewrite.py (82%) diff --git a/passbook/app_gw/proxy/handler.py b/passbook/app_gw/proxy/handler.py index 2f63ac6ada..10d4055090 100644 --- a/passbook/app_gw/proxy/handler.py +++ b/passbook/app_gw/proxy/handler.py @@ -12,8 +12,8 @@ from django.utils.http import urlencode from passbook.app_gw.models import ApplicationGatewayProvider from passbook.app_gw.proxy.exceptions import InvalidUpstream from passbook.app_gw.proxy.response import get_django_response +from passbook.app_gw.proxy.rewrite import Rewriter from passbook.app_gw.proxy.utils import encode_items, normalize_request_headers -from passbook.app_gw.rewrite import Rewriter from passbook.core.models import Application from passbook.core.policies import PolicyEngine @@ -30,7 +30,7 @@ HTTP = urllib3.PoolManager( cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) IGNORED_HOSTS = cache.get(IGNORED_HOSTNAMES_KEY, []) - +POLICY_CACHE = {} class RequestHandler: """Forward requests""" @@ -41,6 +41,8 @@ class RequestHandler: def __init__(self, app_gw, request): self.app_gw = app_gw self.request = request + if self.app_gw.pk not in POLICY_CACHE: + POLICY_CACHE[self.app_gw.pk] = self.app_gw.application.policies.all() @staticmethod def find_app_gw_for_request(request): @@ -49,7 +51,7 @@ class RequestHandler: # This saves us having to query the database on each request host_header = request.META.get('HTTP_HOST') if host_header in IGNORED_HOSTS: - LOGGER.debug("%s is ignored", host_header) + # LOGGER.debug("%s is ignored", host_header) return False # Look through all ApplicationGatewayProviders and check hostnames matches = ApplicationGatewayProvider.objects.filter( @@ -59,7 +61,7 @@ class RequestHandler: # Mo matching Providers found, add host header to ignored list IGNORED_HOSTS.append(host_header) cache.set(IGNORED_HOSTNAMES_KEY, IGNORED_HOSTS) - LOGGER.debug("Ignoring %s", host_header) + # LOGGER.debug("Ignoring %s", host_header) return False # At this point we're certain there's a matching ApplicationGateway if len(matches) > 1: @@ -72,7 +74,8 @@ class RequestHandler: if app_gw: return app_gw except Application.DoesNotExist: - LOGGER.debug("ApplicationGateway not associated with Application") + pass + # LOGGER.debug("ApplicationGateway not associated with Application") return True def _get_upstream(self): @@ -97,10 +100,10 @@ class RequestHandler: return upstream def _format_path_to_redirect(self): - LOGGER.debug("Path before: %s", self.request.get_full_path()) + # LOGGER.debug("Path before: %s", self.request.get_full_path()) rewriter = Rewriter(self.app_gw, self.request) after = rewriter.build() - LOGGER.debug("Path after: %s", after) + # LOGGER.debug("Path after: %s", after) return after def get_proxy_request_headers(self): @@ -126,7 +129,7 @@ class RequestHandler: if not self.app_gw.authentication_header: return request_headers request_headers[self.app_gw.authentication_header] = self.request.user.get_username() - LOGGER.info("%s set", self.app_gw.authentication_header) + # LOGGER.debug("%s set", self.app_gw.authentication_header) return request_headers @@ -136,7 +139,7 @@ class RequestHandler: return False if not self.request.user.is_authenticated: return False - policy_engine = PolicyEngine(self.app_gw.application.policies.all()) + policy_engine = PolicyEngine(POLICY_CACHE[self.app_gw.pk]) policy_engine.for_user(self.request.user).with_request(self.request).build() passing, _messages = policy_engine.result @@ -150,14 +153,14 @@ class RequestHandler: def _created_proxy_response(self, path): request_payload = self.request.body - LOGGER.debug("Request headers: %s", self._request_headers) + # LOGGER.debug("Request headers: %s", self._request_headers) request_url = self.get_upstream() + path - LOGGER.debug("Request URL: %s", request_url) + # LOGGER.debug("Request URL: %s", request_url) if self.request.GET: request_url += '?' + self.get_encoded_query_params() - LOGGER.debug("Request URL: %s", request_url) + # LOGGER.debug("Request URL: %s", request_url) http = HTTP if not self.app_gw.upstream_ssl_verification: @@ -172,8 +175,8 @@ class RequestHandler: body=request_payload, decode_content=False, preload_content=False) - LOGGER.debug("Proxy response header: %s", - proxy_response.getheaders()) + # LOGGER.debug("Proxy response header: %s", + # proxy_response.getheaders()) except urllib3.exceptions.HTTPError as error: LOGGER.exception(error) raise @@ -195,8 +198,8 @@ class RequestHandler: location = location.replace(upstream_host_http, request_host) location = location.replace(upstream_host_https, request_host) proxy_response.headers['Location'] = location - LOGGER.debug("Proxy response LOCATION: %s", - proxy_response.headers['Location']) + # LOGGER.debug("Proxy response LOCATION: %s", + # proxy_response.headers['Location']) def _set_content_type(self, proxy_response): content_type = proxy_response.headers.get('Content-Type') @@ -204,8 +207,8 @@ class RequestHandler: content_type = (mimetypes.guess_type(self.request.path)[0] or self.app_gw.default_content_type) proxy_response.headers['Content-Type'] = content_type - LOGGER.debug("Proxy response CONTENT-TYPE: %s", - proxy_response.headers['Content-Type']) + # LOGGER.debug("Proxy response CONTENT-TYPE: %s", + # proxy_response.headers['Content-Type']) def get_response(self): """Pass request to upstream and return response""" @@ -218,5 +221,5 @@ class RequestHandler: self._set_content_type(proxy_response) response = get_django_response(proxy_response, strict_cookies=False) - LOGGER.debug("RESPONSE RETURNED: %s", response) + # LOGGER.debug("RESPONSE RETURNED: %s", response) return response diff --git a/passbook/app_gw/rewrite.py b/passbook/app_gw/proxy/rewrite.py similarity index 82% rename from passbook/app_gw/rewrite.py rename to passbook/app_gw/proxy/rewrite.py index dc8d6531f3..20eac9a9a1 100644 --- a/passbook/app_gw/rewrite.py +++ b/passbook/app_gw/proxy/rewrite.py @@ -2,6 +2,7 @@ from passbook.app_gw.models import RewriteRule +RULE_CACHE = {} class Context: """Empty class which we dynamically add attributes to""" @@ -15,6 +16,9 @@ class Rewriter: def __init__(self, application, request): self.__application = application self.__request = request + if self.__application.pk not in RULE_CACHE: + RULE_CACHE[self.__application.pk] = RewriteRule.objects.filter( + provider__in=[self.__application]) def __build_context(self, matches): """Build object with .0, .1, etc as groups and give access to request""" @@ -27,7 +31,7 @@ class Rewriter: def build(self): """Run all rules over path and return final path""" path = self.__request.get_full_path() - for rule in RewriteRule.objects.filter(provider__in=[self.__application]): + for rule in RULE_CACHE[self.__application.pk]: matches = rule.compiled_matcher.search(path) if not matches: continue diff --git a/passbook/core/policies.py b/passbook/core/policies.py index 4d9c52aead..84ae94e96d 100644 --- a/passbook/core/policies.py +++ b/passbook/core/policies.py @@ -1,7 +1,5 @@ """passbook core policy engine""" -import cProfile -from logging import getLogger - +# from logging import getLogger from amqp.exceptions import UnexpectedFrame from celery import group from celery.exceptions import TimeoutError as CeleryTimeoutError @@ -11,19 +9,7 @@ from ipware import get_client_ip from passbook.core.celery import CELERY_APP from passbook.core.models import Policy, User - -def profileit(func): - def wrapper(*args, **kwargs): - datafn = func.__name__ + ".profile" # Name the data file sensibly - prof = cProfile.Profile() - retval = prof.runcall(func, *args, **kwargs) - prof.dump_stats(datafn) - return retval - - return wrapper - - -LOGGER = getLogger(__name__) +# LOGGER = getLogger(__name__) def _cache_key(policy, user): return "%s#%s" % (policy.uuid, user.pk) @@ -37,8 +23,8 @@ def _policy_engine_task(user_pk, policy_pk, **kwargs): user_obj = User.objects.get(pk=user_pk) for key, value in kwargs.items(): setattr(user_obj, key, value) - LOGGER.debug("Running policy `%s`#%s for user %s...", policy_obj.name, - policy_obj.pk.hex, user_obj) + # LOGGER.debug("Running policy `%s`#%s for user %s...", policy_obj.name, + # policy_obj.pk.hex, user_obj) policy_result = policy_obj.passes(user_obj) # Handle policy result correctly if result, message or just result message = None @@ -47,10 +33,10 @@ def _policy_engine_task(user_pk, policy_pk, **kwargs): # Invert result if policy.negate is set if policy_obj.negate: policy_result = not policy_result - LOGGER.debug("Policy %r#%s got %s", policy_obj.name, policy_obj.pk.hex, policy_result) + # LOGGER.debug("Policy %r#%s got %s", policy_obj.name, policy_obj.pk.hex, policy_result) cache_key = _cache_key(policy_obj, user_obj) cache.set(cache_key, (policy_obj.action, policy_result, message)) - LOGGER.debug("Cached entry as %s", cache_key) + # LOGGER.debug("Cached entry as %s", cache_key) return policy_obj.action, policy_result, message class PolicyEngine: @@ -79,7 +65,6 @@ class PolicyEngine: self.__request = request return self - @profileit def build(self): """Build task group""" if not self.__user: @@ -96,16 +81,16 @@ class PolicyEngine: for policy in self.policies: cached_policy = cache.get(_cache_key(policy, self.__user), None) if cached_policy: - LOGGER.warning("Taking result from cache for %s", policy.pk.hex) + # LOGGER.debug("Taking result from cache for %s", policy.pk.hex) cached_policies.append(cached_policy) else: - LOGGER.warning("Evaluating policy %s", policy.pk.hex) + # LOGGER.debug("Evaluating policy %s", policy.pk.hex) signatures.append(_policy_engine_task.signature( args=(self.__user.pk, policy.pk.hex), kwargs=kwargs, time_limit=policy.timeout)) self.__get_timeout += policy.timeout - LOGGER.warning("Set total policy timeout to %r", self.__get_timeout) + # LOGGER.debug("Set total policy timeout to %r", self.__get_timeout) # If all policies are cached, we have an empty list here. if signatures: self.__group = group(signatures)() @@ -134,7 +119,7 @@ class PolicyEngine: for policy_action, policy_result, policy_message in result: passing = (policy_action == Policy.ACTION_ALLOW and policy_result) or \ (policy_action == Policy.ACTION_DENY and not policy_result) - LOGGER.debug('Action=%s, Result=%r => %r', policy_action, policy_result, passing) + # LOGGER.debug('Action=%s, Result=%r => %r', policy_action, policy_result, passing) if policy_message: messages.append(policy_message) if not passing: diff --git a/passbook/core/settings.py b/passbook/core/settings.py index 971b1bcfa1..2d0b26b18a 100644 --- a/passbook/core/settings.py +++ b/passbook/core/settings.py @@ -299,7 +299,7 @@ with CONFIG.cd('log'): }, 'django': { 'handlers': ['queue'], - 'level': 'DEBUG', + 'level': 'INFO', 'propagate': True, }, 'tasks': { @@ -324,7 +324,7 @@ with CONFIG.cd('log'): }, 'daphne': { 'handlers': ['queue'], - 'level': 'DEBUG', + 'level': 'INFO', 'propagate': True, } } diff --git a/passbook/lib/log.py b/passbook/lib/log.py index b4a72f9170..4bf85d556e 100644 --- a/passbook/lib/log.py +++ b/passbook/lib/log.py @@ -1,38 +1,37 @@ +"""QueueListener that can be configured from logging.dictConfig""" from atexit import register -from logging.config import ConvertingDict, ConvertingList, valid_ident +from logging.config import ConvertingList from logging.handlers import QueueHandler, QueueListener from queue import Queue -from django.conf import settings - -def _resolve_handlers(l): - # import pudb; pu.db - if not isinstance(l, ConvertingList): - return l +def _resolve_handlers(_list): + """Evaluates ConvertingList by iterating over it""" + if not isinstance(_list, ConvertingList): + return _list # Indexing the list performs the evaluation. - return [l[i] for i in range(len(l))] + return [_list[i] for i in range(len(_list))] class QueueListenerHandler(QueueHandler): + """QueueListener that can be configured from logging.dictConfig""" - def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)): + def __init__(self, handlers, auto_run=True, queue=Queue(-1)): super().__init__(queue) handlers = _resolve_handlers(handlers) self._listener = QueueListener( self.queue, *handlers, - respect_handler_level=respect_handler_level) + respect_handler_level=True) if auto_run: self.start() register(self.stop) def start(self): + """start background thread""" self._listener.start() def stop(self): + """stop background thread""" self._listener.stop() - - def emit(self, record): - return super().emit(record)