From c8459e1e5933f8bf5bc25635ce13724d492e5ebe Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Tue, 25 Jun 2019 17:38:29 +0300 Subject: prometheus --- cmd/rr/http/metrics.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 cmd/rr/http/metrics.go (limited to 'cmd/rr/http/metrics.go') diff --git a/cmd/rr/http/metrics.go b/cmd/rr/http/metrics.go new file mode 100644 index 00000000..f9d3c079 --- /dev/null +++ b/cmd/rr/http/metrics.go @@ -0,0 +1,86 @@ +package http + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/cobra" + rr "github.com/spiral/roadrunner/cmd/rr/cmd" + rrhttp "github.com/spiral/roadrunner/service/http" + "github.com/spiral/roadrunner/service/metrics" + "strconv" +) + +func init() { + cobra.OnInitialize(func() { + svc, _ := rr.Container.Get(metrics.ID) + mtr, ok := svc.(*metrics.Service) + if !ok || !mtr.Enabled() { + return + } + + ht, _ := rr.Container.Get(rrhttp.ID) + if ht, ok := ht.(*rrhttp.Service); ok { + collector := newCollector() + + // register metrics + mtr.MustRegister(collector.requestCounter) + mtr.MustRegister(collector.requestDuration) + + // collect events + ht.AddListener(collector.listener) + } + }) +} + +// listener provide debug callback for system events. With colors! +type metricCollector struct { + requestCounter *prometheus.CounterVec + requestDuration *prometheus.HistogramVec +} + +func newCollector() *metricCollector { + return &metricCollector{ + requestCounter: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rr_http_total", + Help: "Total number of handled http requests after server restart.", + }, + []string{"status"}, + ), + requestDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rr_http_request_duration", + Help: "HTTP request duration.", + Buckets: []float64{0.25, 0.5, 1, 10, 20, 60}, + }, + []string{"status"}, + ), + } +} + +// listener listens to http events and generates nice looking output. +func (c *metricCollector) listener(event int, ctx interface{}) { + // http events + switch event { + case rrhttp.EventResponse: + e := ctx.(*rrhttp.ResponseEvent) + + c.requestCounter.With(prometheus.Labels{ + "status": strconv.Itoa(e.Response.Status), + }).Inc() + + c.requestDuration.With(prometheus.Labels{ + "status": strconv.Itoa(e.Response.Status), + }).Observe(e.Elapsed().Seconds()) + + case rrhttp.EventError: + e := ctx.(*rrhttp.ErrorEvent) + + c.requestCounter.With(prometheus.Labels{ + "status": "500", + }).Inc() + + c.requestDuration.With(prometheus.Labels{ + "status": "500", + }).Observe(e.Elapsed().Seconds()) + } +} -- cgit v1.2.3 From 1fe259d1fba89732abe0229ce5b8c133b7a75981 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Wed, 26 Jun 2019 13:03:39 +0300 Subject: worker memory usage tracking for http --- cmd/rr/http/metrics.go | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) (limited to 'cmd/rr/http/metrics.go') diff --git a/cmd/rr/http/metrics.go b/cmd/rr/http/metrics.go index f9d3c079..32e4a6cd 100644 --- a/cmd/rr/http/metrics.go +++ b/cmd/rr/http/metrics.go @@ -6,7 +6,9 @@ import ( rr "github.com/spiral/roadrunner/cmd/rr/cmd" rrhttp "github.com/spiral/roadrunner/service/http" "github.com/spiral/roadrunner/service/metrics" + "github.com/spiral/roadrunner/util" "strconv" + "time" ) func init() { @@ -24,9 +26,13 @@ func init() { // register metrics mtr.MustRegister(collector.requestCounter) mtr.MustRegister(collector.requestDuration) + mtr.MustRegister(collector.workersMemory) // collect events ht.AddListener(collector.listener) + + // update memory usage every 10 seconds + go collector.collectMemory(ht, time.Second*10) } }) } @@ -35,25 +41,31 @@ func init() { type metricCollector struct { requestCounter *prometheus.CounterVec requestDuration *prometheus.HistogramVec + workersMemory prometheus.Gauge } func newCollector() *metricCollector { return &metricCollector{ requestCounter: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "rr_http_total", + Name: "rr_http_request_total", Help: "Total number of handled http requests after server restart.", }, []string{"status"}, ), requestDuration: prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "rr_http_request_duration", - Help: "HTTP request duration.", - Buckets: []float64{0.25, 0.5, 1, 10, 20, 60}, + Name: "rr_http_request_seconds", + Help: "HTTP request duration.", }, []string{"status"}, ), + workersMemory: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "rr_http_workers_memory_bytes", + Help: "Memory usage by HTTP workers.", + }, + ), } } @@ -84,3 +96,28 @@ func (c *metricCollector) listener(event int, ctx interface{}) { }).Observe(e.Elapsed().Seconds()) } } + +// collect memory usage by server workers +func (c *metricCollector) collectMemory(service *rrhttp.Service, tick time.Duration) { + started := false + for { + server := service.Server() + if server == nil && started { + // stopped + return + } + + started = true + + if workers, err := util.ServerState(server); err == nil { + sum := 0.0 + for _, w := range workers { + sum = sum + float64(w.MemoryUsage) + } + + c.workersMemory.Set(sum) + } + + time.Sleep(tick) + } +} -- cgit v1.2.3 From de5b5a37eb362daeebbec04f82a4265e5414b6e3 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Thu, 27 Jun 2019 11:03:11 +0300 Subject: more rpc methods for app metrics --- cmd/rr/http/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmd/rr/http/metrics.go') diff --git a/cmd/rr/http/metrics.go b/cmd/rr/http/metrics.go index 32e4a6cd..21bbbaf1 100644 --- a/cmd/rr/http/metrics.go +++ b/cmd/rr/http/metrics.go @@ -55,7 +55,7 @@ func newCollector() *metricCollector { ), requestDuration: prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "rr_http_request_seconds", + Name: "rr_http_request_duration_seconds", Help: "HTTP request duration.", }, []string{"status"}, -- cgit v1.2.3