outposts: make metrics compliant with Prometheus best-practices (#6398)

web/outpost: make metrics compliant with Prometheus best-practices

Today, all NewHistogramVec store values in nanoseconds without changing
the default histogram bucket, which are made for seconds, making them
a bit useless. In addition, some metrics names are not self-explanatoryand
and do not comply with Prometheus best practices.

This commit tries to fix all of this "issues".

NOTE: I kept old metrics in order to avoid breaking changes with
existing dashboards and metrics.

Signed-off-by: Alexandre NICOLAIE <xunleii@users.noreply.github.com>

internal/outpost/proxyv2/application/application.go

@@ -163,13 +163,19 @@ func NewApplication(p api.ProxyOutpostConfig, c *http.Client, server Server) (*A
 			}
 			before := time.Now()
 			inner.ServeHTTP(rw, r)
-			after := time.Since(before)
+			elapsed := time.Since(before)
 			metrics.Requests.With(prometheus.Labels{
 				"outpost_name": a.outpostName,
 				"type":         "app",
 				"method":       r.Method,
 				"host":         web.GetHost(r),
-			}).Observe(float64(after))
+			}).Observe(float64(elapsed) / float64(time.Second))
+			metrics.RequestsLegacy.With(prometheus.Labels{
+				"outpost_name": a.outpostName,
+				"type":         "app",
+				"method":       r.Method,
+				"host":         web.GetHost(r),
+			}).Observe(float64(elapsed))
 		})
 	})
 	if server.API().GlobalConfig.ErrorReporting.Enabled {

internal/outpost/proxyv2/application/mode_proxy.go

@@ -55,7 +55,7 @@ func (a *Application) configureProxy() error {
 		}
 		before := time.Now()
 		rp.ServeHTTP(rw, r)
-		after := time.Since(before)
+		elapsed := time.Since(before)
 
 		metrics.UpstreamTiming.With(prometheus.Labels{
 			"outpost_name":  a.outpostName,
@@ -63,7 +63,14 @@ func (a *Application) configureProxy() error {
 			"method":        r.Method,
 			"scheme":        r.URL.Scheme,
 			"host":          web.GetHost(r),
-		}).Observe(float64(after))
+		}).Observe(float64(elapsed) / float64(time.Second))
+		metrics.UpstreamTimingLegacy.With(prometheus.Labels{
+			"outpost_name":  a.outpostName,
+			"upstream_host": r.URL.Host,
+			"method":        r.Method,
+			"scheme":        r.URL.Scheme,
+			"host":          web.GetHost(r),
+		}).Observe(float64(elapsed))
 	})
 	return nil
 }