diff --git a/internal/outpost/ak/crypto.go b/internal/outpost/ak/crypto.go
new file mode 100644
index 0000000000..21880a17f1
--- /dev/null
+++ b/internal/outpost/ak/crypto.go
@@ -0,0 +1,85 @@
+package ak
+
+import (
+	"context"
+	"crypto/tls"
+
+	log "github.com/sirupsen/logrus"
+	"goauthentik.io/api"
+)
+
+type CryptoStore struct {
+	api *api.CryptoApiService
+
+	log *log.Entry
+
+	fingerprints map[string]string
+	certificates map[string]*tls.Certificate
+}
+
+func NewCryptoStore(cryptoApi *api.CryptoApiService) *CryptoStore {
+	return &CryptoStore{
+		api:          cryptoApi,
+		log:          log.WithField("logger", "authentik.outpost.cryptostore"),
+		fingerprints: make(map[string]string),
+		certificates: make(map[string]*tls.Certificate),
+	}
+}
+
+func (cs *CryptoStore) AddKeypair(uuid string) error {
+	// If they keypair was already added, don't
+	// do it again
+	if _, ok := cs.fingerprints[uuid]; ok {
+		return nil
+	}
+	// reset fingerprint to force update
+	cs.fingerprints[uuid] = ""
+	err := cs.Fetch(uuid)
+	if err != nil {
+		return err
+	}
+	cs.fingerprints[uuid] = cs.getFingerprint(uuid)
+	return nil
+}
+
+func (cs *CryptoStore) getFingerprint(uuid string) string {
+	kp, _, err := cs.api.CryptoCertificatekeypairsRetrieve(context.Background(), uuid).Execute()
+	if err != nil {
+		cs.log.WithField("uuid", uuid).WithError(err).Warning("Failed to fetch certificate's fingerprint")
+		return ""
+	}
+	return kp.FingerprintSha256
+}
+
+func (cs *CryptoStore) Fetch(uuid string) error {
+	cfp := cs.getFingerprint(uuid)
+	if cfp == cs.fingerprints[uuid] {
+		cs.log.WithField("uuid", uuid).Info("Fingerprint hasn't changed, not fetching cert")
+		return nil
+	}
+	cs.log.WithField("uuid", uuid).Info("Fetching certificate and private key")
+
+	cert, _, err := cs.api.CryptoCertificatekeypairsViewCertificateRetrieve(context.Background(), uuid).Execute()
+	if err != nil {
+		return err
+	}
+	key, _, err := cs.api.CryptoCertificatekeypairsViewPrivateKeyRetrieve(context.Background(), uuid).Execute()
+	if err != nil {
+		return err
+	}
+
+	x509cert, err := tls.X509KeyPair([]byte(cert.Data), []byte(key.Data))
+	if err != nil {
+		return err
+	}
+	cs.certificates[uuid] = &x509cert
+	return nil
+}
+
+func (cs *CryptoStore) Get(uuid string) *tls.Certificate {
+	err := cs.Fetch(uuid)
+	if err != nil {
+		cs.log.WithError(err).Warning("failed to fetch certificate")
+	}
+	return cs.certificates[uuid]
+}
diff --git a/internal/outpost/ak/global.go b/internal/outpost/ak/global.go
index 56d929b35d..c2d90d1c1d 100644
--- a/internal/outpost/ak/global.go
+++ b/internal/outpost/ak/global.go
@@ -1,8 +1,6 @@
 package ak
 
 import (
-	"context"
-	"crypto/tls"
 	"net/http"
 	"os"
 	"strings"
@@ -11,7 +9,6 @@ import (
 	"github.com/getsentry/sentry-go"
 	httptransport "github.com/go-openapi/runtime/client"
 	log "github.com/sirupsen/logrus"
-	"goauthentik.io/api"
 	"goauthentik.io/internal/constants"
 )
 
@@ -69,21 +66,3 @@ func GetTLSTransport() http.RoundTripper {
 	}
 	return tlsTransport
 }
-
-// ParseCertificate Load certificate from Keyepair UUID and parse it into a go Certificate
-func ParseCertificate(kpUuid string, cryptoApi *api.CryptoApiService) (*tls.Certificate, error) {
-	cert, _, err := cryptoApi.CryptoCertificatekeypairsViewCertificateRetrieve(context.Background(), kpUuid).Execute()
-	if err != nil {
-		return nil, err
-	}
-	key, _, err := cryptoApi.CryptoCertificatekeypairsViewPrivateKeyRetrieve(context.Background(), kpUuid).Execute()
-	if err != nil {
-		return nil, err
-	}
-
-	x509cert, err := tls.X509KeyPair([]byte(cert.Data), []byte(key.Data))
-	if err != nil {
-		return nil, err
-	}
-	return &x509cert, nil
-}
diff --git a/internal/outpost/ldap/api.go b/internal/outpost/ldap/api.go
index 89f12eff7b..f769969c70 100644
--- a/internal/outpost/ldap/api.go
+++ b/internal/outpost/ldap/api.go
@@ -13,7 +13,6 @@ import (
 	"github.com/go-openapi/strfmt"
 	"github.com/pires/go-proxyproto"
 	log "github.com/sirupsen/logrus"
-	"goauthentik.io/internal/outpost/ak"
 )
 
 func (ls *LDAPServer) Refresh() error {
@@ -45,14 +44,12 @@ func (ls *LDAPServer) Refresh() error {
 			gidStartNumber:      *provider.GidStartNumber,
 		}
 		if provider.Certificate.Get() != nil {
-			logger.WithField("provider", provider.Name).Debug("Enabling TLS")
-			cert, err := ak.ParseCertificate(*provider.Certificate.Get(), ls.ac.Client.CryptoApi)
+			kp := provider.Certificate.Get()
+			err := ls.cs.AddKeypair(*kp)
 			if err != nil {
-				logger.WithField("provider", provider.Name).WithError(err).Warning("Failed to fetch certificate")
-			} else {
-				providers[idx].cert = cert
-				logger.WithField("provider", provider.Name).Debug("Loaded certificates")
+				ls.log.WithError(err).Warning("Failed to initially fetch certificate")
 			}
+			providers[idx].cert = ls.cs.Get(*kp)
 		}
 	}
 	ls.providers = providers
diff --git a/internal/outpost/ldap/ldap.go b/internal/outpost/ldap/ldap.go
index 79f4fa2f27..74bbc71f39 100644
--- a/internal/outpost/ldap/ldap.go
+++ b/internal/outpost/ldap/ldap.go
@@ -47,6 +47,7 @@ type LDAPServer struct {
 	s           *ldap.Server
 	log         *log.Entry
 	ac          *ak.APIController
+	cs          *ak.CryptoStore
 	defaultCert *tls.Certificate
 	providers   []*ProviderInstance
 }
@@ -69,6 +70,7 @@ func NewServer(ac *ak.APIController) *LDAPServer {
 		s:         s,
 		log:       log.WithField("logger", "authentik.outpost.ldap"),
 		ac:        ac,
+		cs:        ak.NewCryptoStore(ac.Client.CryptoApi),
 		providers: []*ProviderInstance{},
 	}
 	defaultCert, err := crypto.GenerateSelfSignedCert()
diff --git a/internal/outpost/proxy/api_bundle.go b/internal/outpost/proxy/api_bundle.go
index 4480a65978..d819ed2b99 100644
--- a/internal/outpost/proxy/api_bundle.go
+++ b/internal/outpost/proxy/api_bundle.go
@@ -15,7 +15,6 @@ import (
 	"github.com/oauth2-proxy/oauth2-proxy/pkg/validation"
 	log "github.com/sirupsen/logrus"
 	"goauthentik.io/api"
-	"goauthentik.io/internal/outpost/ak"
 )
 
 type providerBundle struct {
@@ -89,14 +88,12 @@ func (pb *providerBundle) prepareOpts(provider api.ProxyOutpostConfig) *options.
 	}
 
 	if provider.Certificate.Get() != nil {
-		pb.log.WithField("provider", provider.Name).Debug("Enabling TLS")
-		cert, err := ak.ParseCertificate(*provider.Certificate.Get(), pb.s.ak.Client.CryptoApi)
+		kp := provider.Certificate.Get()
+		err := pb.s.cs.AddKeypair(*kp)
 		if err != nil {
-			pb.log.WithField("provider", provider.Name).WithError(err).Warning("Failed to fetch certificate")
-			return providerOpts
+			pb.log.WithError(err).Warning("Failed to initially fetch certificate")
 		}
-		pb.cert = cert
-		pb.log.WithField("provider", provider.Name).Debug("Loaded certificates")
+		pb.cert = pb.s.cs.Get(*kp)
 	}
 	return providerOpts
 }
diff --git a/internal/outpost/proxy/server.go b/internal/outpost/proxy/server.go
index 5666f74feb..bffdc8789d 100644
--- a/internal/outpost/proxy/server.go
+++ b/internal/outpost/proxy/server.go
@@ -21,6 +21,7 @@ type Server struct {
 	stop        chan struct{} // channel for waiting shutdown
 	logger      *log.Entry
 	ak          *ak.APIController
+	cs          *ak.CryptoStore
 	defaultCert tls.Certificate
 }
 
@@ -35,6 +36,7 @@ func NewServer(ac *ak.APIController) *Server {
 		logger:      log.WithField("logger", "authentik.outpost.proxy-http-server"),
 		defaultCert: defaultCert,
 		ak:          ac,
+		cs:          ak.NewCryptoStore(ac.Client.CryptoApi),
 	}
 }