lifecycle: migrate internal healthcheck to use go (#5322)
Signed-off-by: Jens Langhammer <jens@goauthentik.io>
This commit is contained in:
Jens L
77
cmd/server/healthcheck.go
Normal file
77
cmd/server/healthcheck.go
Normal file
@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"goauthentik.io/internal/config"
|
||||
"goauthentik.io/internal/utils/web"
|
||||
)
|
||||
|
||||
var workerHeartbeat = path.Join(os.TempDir(), "authentik-worker")
|
||||
|
||||
const workerThreshold = 30
|
||||
|
||||
var healthcheckCmd = &cobra.Command{
|
||||
Use: "healthcheck",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) < 1 {
|
||||
os.Exit(1)
|
||||
}
|
||||
mode := args[0]
|
||||
config.Get()
|
||||
exitCode := 1
|
||||
log.WithField("mode", mode).Debug("checking health")
|
||||
switch strings.ToLower(mode) {
|
||||
case "server":
|
||||
exitCode = checkServer()
|
||||
case "worker":
|
||||
exitCode = checkWorker()
|
||||
default:
|
||||
log.Warn("Invalid mode")
|
||||
}
|
||||
os.Exit(exitCode)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(healthcheckCmd)
|
||||
}
|
||||
|
||||
func checkServer() int {
|
||||
h := &http.Client{
|
||||
Transport: web.NewUserAgentTransport("goauthentik.io/healthcheck", http.DefaultTransport),
|
||||
}
|
||||
url := fmt.Sprintf("http://%s/-/health/ready/", config.Get().Listen.HTTP)
|
||||
res, err := h.Head(url)
|
||||
if err != nil {
|
||||
log.WithError(err).Warning("failed to send healthcheck request")
|
||||
return 1
|
||||
}
|
||||
if res.StatusCode >= 400 {
|
||||
log.WithField("status", res.StatusCode).Warning("unhealthy status code")
|
||||
return 1
|
||||
}
|
||||
log.Debug("successfully checked health")
|
||||
return 0
|
||||
}
|
||||
|
||||
func checkWorker() int {
|
||||
stat, err := os.Stat(workerHeartbeat)
|
||||
if err != nil {
|
||||
log.WithError(err).Warning("failed to check worker heartbeat file")
|
||||
return 1
|
||||
}
|
||||
delta := time.Since(stat.ModTime()).Seconds()
|
||||
if delta > workerThreshold {
|
||||
log.WithField("threshold", workerThreshold).WithField("delta", delta).Warning("Worker hasn't updated heartbeat in threshold")
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@ -1,132 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"goauthentik.io/internal/common"
|
||||
"goauthentik.io/internal/config"
|
||||
"goauthentik.io/internal/constants"
|
||||
"goauthentik.io/internal/debug"
|
||||
"goauthentik.io/internal/gounicorn"
|
||||
"goauthentik.io/internal/outpost/ak"
|
||||
"goauthentik.io/internal/outpost/proxyv2"
|
||||
sentryutils "goauthentik.io/internal/utils/sentry"
|
||||
webutils "goauthentik.io/internal/utils/web"
|
||||
"goauthentik.io/internal/web"
|
||||
"goauthentik.io/internal/web/tenant_tls"
|
||||
)
|
||||
|
||||
var running = true
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
log.SetFormatter(&log.JSONFormatter{
|
||||
FieldMap: log.FieldMap{
|
||||
log.FieldKeyMsg: "event",
|
||||
log.FieldKeyTime: "timestamp",
|
||||
},
|
||||
DisableHTMLEscape: true,
|
||||
})
|
||||
debug.EnableDebugServer()
|
||||
l := log.WithField("logger", "authentik.root")
|
||||
|
||||
if config.Get().ErrorReporting.Enabled {
|
||||
err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: config.Get().ErrorReporting.SentryDSN,
|
||||
AttachStacktrace: true,
|
||||
EnableTracing: true,
|
||||
TracesSampler: sentryutils.SamplerFunc(config.Get().ErrorReporting.SampleRate),
|
||||
Release: fmt.Sprintf("authentik@%s", constants.VERSION),
|
||||
Environment: config.Get().ErrorReporting.Environment,
|
||||
HTTPTransport: webutils.NewUserAgentTransport(constants.UserAgent(), http.DefaultTransport),
|
||||
IgnoreErrors: []string{
|
||||
http.ErrAbortHandler.Error(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
l.WithError(err).Warning("failed to init sentry")
|
||||
}
|
||||
}
|
||||
|
||||
ex := common.Init()
|
||||
defer common.Defer()
|
||||
|
||||
u, _ := url.Parse("http://localhost:8000")
|
||||
|
||||
g := gounicorn.New()
|
||||
defer func() {
|
||||
l.Info("shutting down gunicorn")
|
||||
g.Kill()
|
||||
}()
|
||||
ws := web.NewWebServer(g)
|
||||
g.HealthyCallback = func() {
|
||||
if !config.Get().Outposts.DisableEmbeddedOutpost {
|
||||
go attemptProxyStart(ws, u)
|
||||
}
|
||||
}
|
||||
go web.RunMetricsServer()
|
||||
go attemptStartBackend(g)
|
||||
ws.Start()
|
||||
<-ex
|
||||
running = false
|
||||
l.Info("shutting down webserver")
|
||||
go ws.Shutdown()
|
||||
}
|
||||
|
||||
func attemptStartBackend(g *gounicorn.GoUnicorn) {
|
||||
for {
|
||||
if !running {
|
||||
return
|
||||
}
|
||||
err := g.Start()
|
||||
log.WithField("logger", "authentik.router").WithError(err).Warning("gunicorn process died, restarting")
|
||||
}
|
||||
}
|
||||
|
||||
func attemptProxyStart(ws *web.WebServer, u *url.URL) {
|
||||
maxTries := 100
|
||||
attempt := 0
|
||||
l := log.WithField("logger", "authentik.server")
|
||||
for {
|
||||
l.Debug("attempting to init outpost")
|
||||
ac := ak.NewAPIController(*u, config.Get().SecretKey)
|
||||
if ac == nil {
|
||||
attempt += 1
|
||||
time.Sleep(1 * time.Second)
|
||||
if attempt > maxTries {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Init tenant_tls here too since it requires an API Client,
|
||||
// so we just re-use the same one as the outpost uses
|
||||
tw := tenant_tls.NewWatcher(ac.Client)
|
||||
go tw.Start()
|
||||
ws.TenantTLS = tw
|
||||
ac.AddRefreshHandler(func() {
|
||||
tw.Check()
|
||||
})
|
||||
|
||||
srv := proxyv2.NewProxyServer(ac)
|
||||
ws.ProxyServer = srv
|
||||
ac.Server = srv
|
||||
l.Debug("attempting to start outpost")
|
||||
err := ac.StartBackgroundTasks()
|
||||
if err != nil {
|
||||
l.WithError(err).Warning("outpost failed to start")
|
||||
attempt += 1
|
||||
time.Sleep(15 * time.Second)
|
||||
if attempt > maxTries {
|
||||
break
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
select {}
|
||||
}
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
141
cmd/server/server.go
Normal file
141
cmd/server/server.go
Normal file
@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"goauthentik.io/internal/common"
|
||||
"goauthentik.io/internal/config"
|
||||
"goauthentik.io/internal/constants"
|
||||
"goauthentik.io/internal/debug"
|
||||
"goauthentik.io/internal/gounicorn"
|
||||
"goauthentik.io/internal/outpost/ak"
|
||||
"goauthentik.io/internal/outpost/proxyv2"
|
||||
sentryutils "goauthentik.io/internal/utils/sentry"
|
||||
webutils "goauthentik.io/internal/utils/web"
|
||||
"goauthentik.io/internal/web"
|
||||
"goauthentik.io/internal/web/tenant_tls"
|
||||
)
|
||||
|
||||
var running = true
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "authentik",
|
||||
Short: "Start authentik instance",
|
||||
Version: constants.FullVersion(),
|
||||
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
log.SetFormatter(&log.JSONFormatter{
|
||||
FieldMap: log.FieldMap{
|
||||
log.FieldKeyMsg: "event",
|
||||
log.FieldKeyTime: "timestamp",
|
||||
},
|
||||
DisableHTMLEscape: true,
|
||||
})
|
||||
},
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
debug.EnableDebugServer()
|
||||
l := log.WithField("logger", "authentik.root")
|
||||
|
||||
if config.Get().ErrorReporting.Enabled {
|
||||
err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: config.Get().ErrorReporting.SentryDSN,
|
||||
AttachStacktrace: true,
|
||||
EnableTracing: true,
|
||||
TracesSampler: sentryutils.SamplerFunc(config.Get().ErrorReporting.SampleRate),
|
||||
Release: fmt.Sprintf("authentik@%s", constants.VERSION),
|
||||
Environment: config.Get().ErrorReporting.Environment,
|
||||
HTTPTransport: webutils.NewUserAgentTransport(constants.UserAgent(), http.DefaultTransport),
|
||||
IgnoreErrors: []string{
|
||||
http.ErrAbortHandler.Error(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
l.WithError(err).Warning("failed to init sentry")
|
||||
}
|
||||
}
|
||||
|
||||
ex := common.Init()
|
||||
defer common.Defer()
|
||||
|
||||
u, _ := url.Parse("http://localhost:8000")
|
||||
|
||||
g := gounicorn.New()
|
||||
defer func() {
|
||||
l.Info("shutting down gunicorn")
|
||||
g.Kill()
|
||||
}()
|
||||
ws := web.NewWebServer(g)
|
||||
g.HealthyCallback = func() {
|
||||
if !config.Get().Outposts.DisableEmbeddedOutpost {
|
||||
go attemptProxyStart(ws, u)
|
||||
}
|
||||
}
|
||||
go web.RunMetricsServer()
|
||||
go attemptStartBackend(g)
|
||||
ws.Start()
|
||||
<-ex
|
||||
running = false
|
||||
l.Info("shutting down webserver")
|
||||
go ws.Shutdown()
|
||||
|
||||
},
|
||||
}
|
||||
|
||||
func attemptStartBackend(g *gounicorn.GoUnicorn) {
|
||||
for {
|
||||
if !running {
|
||||
return
|
||||
}
|
||||
err := g.Start()
|
||||
log.WithField("logger", "authentik.router").WithError(err).Warning("gunicorn process died, restarting")
|
||||
}
|
||||
}
|
||||
|
||||
func attemptProxyStart(ws *web.WebServer, u *url.URL) {
|
||||
maxTries := 100
|
||||
attempt := 0
|
||||
l := log.WithField("logger", "authentik.server")
|
||||
for {
|
||||
l.Debug("attempting to init outpost")
|
||||
ac := ak.NewAPIController(*u, config.Get().SecretKey)
|
||||
if ac == nil {
|
||||
attempt += 1
|
||||
time.Sleep(1 * time.Second)
|
||||
if attempt > maxTries {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Init tenant_tls here too since it requires an API Client,
|
||||
// so we just re-use the same one as the outpost uses
|
||||
tw := tenant_tls.NewWatcher(ac.Client)
|
||||
go tw.Start()
|
||||
ws.TenantTLS = tw
|
||||
ac.AddRefreshHandler(func() {
|
||||
tw.Check()
|
||||
})
|
||||
|
||||
srv := proxyv2.NewProxyServer(ac)
|
||||
ws.ProxyServer = srv
|
||||
ac.Server = srv
|
||||
l.Debug("attempting to start outpost")
|
||||
err := ac.StartBackgroundTasks()
|
||||
if err != nil {
|
||||
l.WithError(err).Warning("outpost failed to start")
|
||||
attempt += 1
|
||||
time.Sleep(15 * time.Second)
|
||||
if attempt > maxTries {
|
||||
break
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
select {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user