diff --git a/authentik/core/tasks.py b/authentik/core/tasks.py
index 3e18c50b44..2e23ae9974 100644
--- a/authentik/core/tasks.py
+++ b/authentik/core/tasks.py
@@ -63,8 +63,6 @@ def should_backup() -> bool:
         return False
     if not CONFIG.y_bool("postgresql.backup.enabled"):
         return False
-    if settings.DEBUG:
-        return False
     return True
 
 
diff --git a/authentik/outposts/docker_ssh.py b/authentik/outposts/docker_ssh.py
index 9f01912447..46d9f6ebbe 100644
--- a/authentik/outposts/docker_ssh.py
+++ b/authentik/outposts/docker_ssh.py
@@ -62,16 +62,21 @@ class DockerInlineSSH:
 
     def cleanup(self):
         """Cleanup when we're done"""
-        os.unlink(self.key_path)
-        with open(self.config_path, "r+", encoding="utf-8") as ssh_config:
-            start = 0
-            end = 0
-            lines = ssh_config.readlines()
-            for idx, line in enumerate(lines):
-                if line == self.header:
-                    start = idx
-                if start != 0 and line == f"{FOOTER}\n":
-                    end = idx
-        with open(self.config_path, "w+", encoding="utf-8") as ssh_config:
-            lines = lines[:start] + lines[end + 2 :]
-            ssh_config.writelines(lines)
+        try:
+            os.unlink(self.key_path)
+            with open(self.config_path, "r+", encoding="utf-8") as ssh_config:
+                start = 0
+                end = 0
+                lines = ssh_config.readlines()
+                for idx, line in enumerate(lines):
+                    if line == self.header:
+                        start = idx
+                    if start != 0 and line == f"{FOOTER}\n":
+                        end = idx
+            with open(self.config_path, "w+", encoding="utf-8") as ssh_config:
+                lines = lines[:start] + lines[end + 2 :]
+                ssh_config.writelines(lines)
+        except OSError:
+            # If we fail deleting a file it doesn't matter that much
+            # since we're just in a container
+            pass