From 5d43fc2c8d952801b1942932cea446f9d3c537c3 Mon Sep 17 00:00:00 2001 From: Rob Landers Date: Thu, 26 Sep 2024 09:53:37 +0200 Subject: [PATCH] add basic metrics (#966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add metrics * change how counting works * also replace dots * check that metrics exist * rename NullMetrics to nullMetrics * update go.sum * register collectors only once * add tests * add tests for metrics and fix bugs * keep old metrics around for test * properly reset during shutdown * use the same method as frankenphp * Revert "keep old metrics around for test" This reverts commit 1f0df6f6bdaebf32aec346f068d6f42a0b5f4007. * change to require.NoError * compile regex only once * remove name sanitizer * use require * parameterize host port because security software sucks * remove need for renaming workers * increase number of threads and add tests * fix where frankenphp configuration was bleeding into later tests * adds basic docs for metrics * Add caddy metrics link Co-authored-by: Kévin Dunglas * Fix typos Co-authored-by: Kévin Dunglas * address feedback * change comment to be much more "dangerous" --------- Co-authored-by: Kévin Dunglas --- caddy/caddy.go | 10 +- caddy/caddy_test.go | 293 ++++++++++++++++++++++++++++++++++++++++---- caddy/go.mod | 6 +- docs/metrics.md | 12 ++ frankenphp.go | 81 +++++++++--- go.mod | 10 +- go.sum | 16 +++ metrics.go | 252 +++++++++++++++++++++++++++++++++++++ metrics_test.go | 82 +++++++++++++ options.go | 9 ++ worker.go | 4 + 11 files changed, 723 insertions(+), 52 deletions(-) create mode 100644 docs/metrics.md create mode 100644 metrics.go create mode 100644 metrics_test.go diff --git a/caddy/caddy.go b/caddy/caddy.go index e09e864b..d689769d 100644 --- a/caddy/caddy.go +++ b/caddy/caddy.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/prometheus/client_golang/prometheus" "net/http" "path/filepath" "strconv" @@ -44,6 +45,8 @@ var mainPHPInterpreterKey mainPHPinterpreterKeyType var phpInterpreter = caddy.NewUsagePool() +var metrics = frankenphp.NewPrometheusMetrics(prometheus.DefaultRegisterer) + type phpInterpreterDestructor struct{} func (phpInterpreterDestructor) Destruct() error { @@ -80,7 +83,7 @@ func (f *FrankenPHPApp) Start() error { repl := caddy.NewReplacer() logger := caddy.Log() - opts := []frankenphp.Option{frankenphp.WithNumThreads(f.NumThreads), frankenphp.WithLogger(logger)} + opts := []frankenphp.Option{frankenphp.WithNumThreads(f.NumThreads), frankenphp.WithLogger(logger), frankenphp.WithMetrics(metrics)} for _, w := range f.Workers { opts = append(opts, frankenphp.WithWorkers(repl.ReplaceKnown(w.FileName, ""), w.Num, w.Env)) } @@ -106,8 +109,11 @@ func (f *FrankenPHPApp) Start() error { return nil } -func (*FrankenPHPApp) Stop() error { +func (f *FrankenPHPApp) Stop() error { caddy.Log().Info("FrankenPHP stopped 🐘") + // reset configuration so it doesn't bleed into later tests + f.Workers = nil + f.NumThreads = 0 return nil } diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index b44d425b..cc750d6c 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -3,6 +3,10 @@ package caddy_test import ( "bytes" "fmt" + "github.com/dunglas/frankenphp" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" "net/http" "strings" "sync" @@ -11,6 +15,8 @@ import ( "github.com/caddyserver/caddy/v2/caddytest" ) +var testPort = "9080" + func TestPHP(t *testing.T) { var wg sync.WaitGroup tester := caddytest.NewTester(t) @@ -18,13 +24,13 @@ func TestPHP(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp } - localhost:9080 { + localhost:`+testPort+` { route { php { root ../testdata @@ -37,7 +43,7 @@ func TestPHP(t *testing.T) { wg.Add(1) go func(i int) { - tester.AssertGetResponse(fmt.Sprintf("http://localhost:9080/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) + tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) wg.Done() }(i) } @@ -50,13 +56,13 @@ func TestLargeRequest(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp } - localhost:9080 { + localhost:`+testPort+` { route { php { root ../testdata @@ -66,7 +72,7 @@ func TestLargeRequest(t *testing.T) { `, "caddyfile") tester.AssertPostResponseBody( - "http://localhost:9080/large-request.php", + "http://localhost:"+testPort+"/large-request.php", []string{}, bytes.NewBufferString(strings.Repeat("f", 1_048_576)), http.StatusOK, @@ -81,7 +87,7 @@ func TestWorker(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp { @@ -89,7 +95,7 @@ func TestWorker(t *testing.T) { } } - localhost:9080 { + localhost:`+testPort+` { route { php { root ../testdata @@ -102,7 +108,7 @@ func TestWorker(t *testing.T) { wg.Add(1) go func(i int) { - tester.AssertGetResponse(fmt.Sprintf("http://localhost:9080/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) + tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) wg.Done() }(i) } @@ -115,7 +121,7 @@ func TestEnv(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp { @@ -127,7 +133,7 @@ func TestEnv(t *testing.T) { } } - localhost:9080 { + localhost:`+testPort+` { route { php { root ../testdata @@ -137,7 +143,7 @@ func TestEnv(t *testing.T) { } `, "caddyfile") - tester.AssertGetResponse("http://localhost:9080/worker-env.php", http.StatusOK, "bazbar") + tester.AssertGetResponse("http://localhost:"+testPort+"/worker-env.php", http.StatusOK, "bazbar") } func TestJsonEnv(t *testing.T) { @@ -160,12 +166,12 @@ func TestJsonEnv(t *testing.T) { ] }, "http": { - "http_port": 9080, + "http_port": `+testPort+`, "https_port": 9443, "servers": { "srv0": { "listen": [ - ":9080" + ":`+testPort+`" ], "routes": [ { @@ -220,7 +226,7 @@ func TestJsonEnv(t *testing.T) { } `, "json") - tester.AssertGetResponse("http://localhost:9080/worker-env.php", http.StatusOK, "bazbar") + tester.AssertGetResponse("http://localhost:"+testPort+"/worker-env.php", http.StatusOK, "bazbar") } func TestCustomCaddyVariablesInEnv(t *testing.T) { @@ -229,7 +235,7 @@ func TestCustomCaddyVariablesInEnv(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp { @@ -241,7 +247,7 @@ func TestCustomCaddyVariablesInEnv(t *testing.T) { } } - localhost:9080 { + localhost:`+testPort+` { route { map 1 {my_customvar} { default "hello " @@ -254,7 +260,7 @@ func TestCustomCaddyVariablesInEnv(t *testing.T) { } `, "caddyfile") - tester.AssertGetResponse("http://localhost:9080/worker-env.php", http.StatusOK, "hello world") + tester.AssertGetResponse("http://localhost:"+testPort+"/worker-env.php", http.StatusOK, "hello world") } func TestPHPServerDirective(t *testing.T) { @@ -263,21 +269,21 @@ func TestPHPServerDirective(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp } - localhost:9080 { + localhost:`+testPort+` { root * ../testdata php_server } `, "caddyfile") - tester.AssertGetResponse("http://localhost:9080", http.StatusOK, "I am by birth a Genevese (i not set)") - tester.AssertGetResponse("http://localhost:9080/hello.txt", http.StatusOK, "Hello") - tester.AssertGetResponse("http://localhost:9080/not-found.txt", http.StatusOK, "I am by birth a Genevese (i not set)") + tester.AssertGetResponse("http://localhost:"+testPort, http.StatusOK, "I am by birth a Genevese (i not set)") + tester.AssertGetResponse("http://localhost:"+testPort+"/hello.txt", http.StatusOK, "Hello") + tester.AssertGetResponse("http://localhost:"+testPort+"/not-found.txt", http.StatusOK, "I am by birth a Genevese (i not set)") } func TestPHPServerDirectiveDisableFileServer(t *testing.T) { @@ -286,14 +292,14 @@ func TestPHPServerDirectiveDisableFileServer(t *testing.T) { { skip_install_trust admin localhost:2999 - http_port 9080 + http_port `+testPort+` https_port 9443 frankenphp order php_server before respond } - localhost:9080 { + localhost:`+testPort+` { root * ../testdata php_server { file_server off @@ -302,6 +308,239 @@ func TestPHPServerDirectiveDisableFileServer(t *testing.T) { } `, "caddyfile") - tester.AssertGetResponse("http://localhost:9080", http.StatusOK, "I am by birth a Genevese (i not set)") - tester.AssertGetResponse("http://localhost:9080/hello.txt", http.StatusNotFound, "Not found") + tester.AssertGetResponse("http://localhost:"+testPort, http.StatusOK, "I am by birth a Genevese (i not set)") + tester.AssertGetResponse("http://localhost:"+testPort+"/hello.txt", http.StatusNotFound, "Not found") +} + +func TestMetrics(t *testing.T) { + var wg sync.WaitGroup + tester := caddytest.NewTester(t) + tester.InitServer(` + { + skip_install_trust + admin localhost:2999 + http_port `+testPort+` + https_port 9443 + + frankenphp + } + + localhost:`+testPort+` { + route { + php { + root ../testdata + } + } + } + `, "caddyfile") + + // Make some requests + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) + wg.Done() + }(i) + } + wg.Wait() + + // Fetch metrics + resp, err := http.Get("http://localhost:2999/metrics") + if err != nil { + t.Fatalf("failed to fetch metrics: %v", err) + } + defer resp.Body.Close() + + // Read and parse metrics + metrics := new(bytes.Buffer) + _, err = metrics.ReadFrom(resp.Body) + if err != nil { + t.Fatalf("failed to read metrics: %v", err) + } + + cpus := fmt.Sprintf("%d", frankenphp.MaxThreads) + + // Check metrics + expectedMetrics := ` + # HELP frankenphp_total_threads Total number of PHP threads + # TYPE frankenphp_total_threads counter + frankenphp_total_threads ` + cpus + ` + + # HELP frankenphp_busy_threads Number of busy PHP threads + # TYPE frankenphp_busy_threads gauge + frankenphp_busy_threads 0 + ` + + require.NoError(t, testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expectedMetrics), "frankenphp_total_threads", "frankenphp_busy_threads")) +} + +func TestWorkerMetrics(t *testing.T) { + var wg sync.WaitGroup + tester := caddytest.NewTester(t) + tester.InitServer(` + { + skip_install_trust + admin localhost:2999 + http_port `+testPort+` + https_port 9443 + + frankenphp { + worker ../testdata/index.php 2 + } + } + + localhost:`+testPort+` { + route { + php { + root ../testdata + } + } + } + `, "caddyfile") + + // Make some requests + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) + wg.Done() + }(i) + } + wg.Wait() + + // Fetch metrics + resp, err := http.Get("http://localhost:2999/metrics") + if err != nil { + t.Fatalf("failed to fetch metrics: %v", err) + } + defer resp.Body.Close() + + // Read and parse metrics + metrics := new(bytes.Buffer) + _, err = metrics.ReadFrom(resp.Body) + if err != nil { + t.Fatalf("failed to read metrics: %v", err) + } + + cpus := fmt.Sprintf("%d", frankenphp.MaxThreads) + + // Check metrics + expectedMetrics := ` + # HELP frankenphp_total_threads Total number of PHP threads + # TYPE frankenphp_total_threads counter + frankenphp_total_threads ` + cpus + ` + + # HELP frankenphp_busy_threads Number of busy PHP threads + # TYPE frankenphp_busy_threads gauge + frankenphp_busy_threads 2 + + # HELP frankenphp_testdata_index_php_busy_workers Number of busy PHP workers for this worker + # TYPE frankenphp_testdata_index_php_busy_workers gauge + frankenphp_testdata_index_php_busy_workers 0 + + # HELP frankenphp_testdata_index_php_total_workers Total number of PHP workers for this worker + # TYPE frankenphp_testdata_index_php_total_workers gauge + frankenphp_testdata_index_php_total_workers 2 + + # HELP frankenphp_testdata_index_php_worker_request_count + # TYPE frankenphp_testdata_index_php_worker_request_count counter + frankenphp_testdata_index_php_worker_request_count 10 + ` + + require.NoError(t, + testutil.GatherAndCompare( + prometheus.DefaultGatherer, + strings.NewReader(expectedMetrics), + "frankenphp_total_threads", + "frankenphp_busy_threads", + "frankenphp_testdata_index_php_busy_workers", + "frankenphp_testdata_index_php_total_workers", + "frankenphp_testdata_index_php_worker_request_count", + )) +} + +func TestAutoWorkerConfig(t *testing.T) { + var wg sync.WaitGroup + tester := caddytest.NewTester(t) + tester.InitServer(` + { + skip_install_trust + admin localhost:2999 + http_port `+testPort+` + https_port 9443 + + frankenphp { + worker ../testdata/index.php + } + } + + localhost:`+testPort+` { + route { + php { + root ../testdata + } + } + } + `, "caddyfile") + + // Make some requests + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) + wg.Done() + }(i) + } + wg.Wait() + + // Fetch metrics + resp, err := http.Get("http://localhost:2999/metrics") + if err != nil { + t.Fatalf("failed to fetch metrics: %v", err) + } + defer resp.Body.Close() + + // Read and parse metrics + metrics := new(bytes.Buffer) + _, err = metrics.ReadFrom(resp.Body) + if err != nil { + t.Fatalf("failed to read metrics: %v", err) + } + + cpus := fmt.Sprintf("%d", frankenphp.MaxThreads) + workers := fmt.Sprintf("%d", frankenphp.MaxThreads-1) + + // Check metrics + expectedMetrics := ` + # HELP frankenphp_total_threads Total number of PHP threads + # TYPE frankenphp_total_threads counter + frankenphp_total_threads ` + cpus + ` + + # HELP frankenphp_busy_threads Number of busy PHP threads + # TYPE frankenphp_busy_threads gauge + frankenphp_busy_threads ` + workers + ` + + # HELP frankenphp_testdata_index_php_busy_workers Number of busy PHP workers for this worker + # TYPE frankenphp_testdata_index_php_busy_workers gauge + frankenphp_testdata_index_php_busy_workers 0 + + # HELP frankenphp_testdata_index_php_total_workers Total number of PHP workers for this worker + # TYPE frankenphp_testdata_index_php_total_workers gauge + frankenphp_testdata_index_php_total_workers ` + workers + ` + + # HELP frankenphp_testdata_index_php_worker_request_count + # TYPE frankenphp_testdata_index_php_worker_request_count counter + frankenphp_testdata_index_php_worker_request_count 10 + ` + + require.NoError(t, + testutil.GatherAndCompare( + prometheus.DefaultGatherer, + strings.NewReader(expectedMetrics), + "frankenphp_total_threads", + "frankenphp_busy_threads", + "frankenphp_testdata_index_php_busy_workers", + "frankenphp_testdata_index_php_total_workers", + "frankenphp_testdata_index_php_worker_request_count", + )) } diff --git a/caddy/go.mod b/caddy/go.mod index be9b993b..d7543e20 100644 --- a/caddy/go.mod +++ b/caddy/go.mod @@ -13,7 +13,9 @@ require ( github.com/dunglas/frankenphp v1.2.5 github.com/dunglas/mercure/caddy v0.16.3 github.com/dunglas/vulcain/caddy v1.0.5 + github.com/prometheus/client_golang v1.20.2 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 ) @@ -42,6 +44,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect @@ -97,6 +100,7 @@ require ( github.com/kevburnsjr/skipfilter v0.0.1 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -119,7 +123,7 @@ require ( github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pires/go-proxyproto v0.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 00000000..3ee4c84e --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,12 @@ +# Metrics + +When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenPHP exposes the following metrics: + +* `frankenphp_[worker]_total_workers`: The total number of workers. +* `frankenphp_[worker]_busy_workers`: The number of workers currently processing a request. +* `frankenphp_[worker]_worker_request_time`: The time spent processing requests by all workers. +* `frankenphp_[worker]_worker_request_count`: The number of requests processed by all workers. +* `frankenphp_total_threads`: The total number of PHP threads. +* `frankenphp_busy_threads`: The number of PHP threads currently processing a request (running workers always consume a thread). + +For worker metrics, the `[worker]` placeholder is replaced by the worker script path in the Caddyfile. diff --git a/frankenphp.go b/frankenphp.go index f540d0b1..9b4f4924 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -39,6 +39,7 @@ import ( "strconv" "strings" "sync" + "time" "unsafe" "github.com/maypok86/otter" @@ -71,6 +72,8 @@ var ( loggerMu sync.RWMutex logger *zap.Logger + + metrics Metrics = nullMetrics{} ) type syslogLevel int @@ -127,6 +130,7 @@ type FrankenPHPContext struct { done chan interface{} currentWorkerRequest cgo.Handle + startedAt time.Time } func clientHasClosed(r *http.Request) bool { @@ -242,6 +246,40 @@ func Config() PHPConfig { } } +// MaxThreads is internally used during tests. It is written to, but never read and may go away in the future. +var MaxThreads int + +func calculateMaxThreads(opt *opt) error { + maxProcs := runtime.GOMAXPROCS(0) * 2 + + var numWorkers int + for i, w := range opt.workers { + if w.num <= 0 { + // https://github.com/dunglas/frankenphp/issues/126 + opt.workers[i].num = maxProcs + } + metrics.TotalWorkers(w.fileName, w.num) + + numWorkers += opt.workers[i].num + } + + if opt.numThreads <= 0 { + if numWorkers >= maxProcs { + // Start at least as many threads as workers, and keep a free thread to handle requests in non-worker mode + opt.numThreads = numWorkers + 1 + } else { + opt.numThreads = maxProcs + } + } else if opt.numThreads <= numWorkers { + return NotEnoughThreads + } + + metrics.TotalThreads(opt.numThreads) + MaxThreads = opt.numThreads + + return nil +} + // Init starts the PHP runtime and the configured workers. func Init(options ...Option) error { if requestChan != nil { @@ -270,27 +308,13 @@ func Init(options ...Option) error { loggerMu.Unlock() } - maxProcs := runtime.GOMAXPROCS(0) - - var numWorkers int - for i, w := range opt.workers { - if w.num <= 0 { - // https://github.com/dunglas/frankenphp/issues/126 - opt.workers[i].num = maxProcs * 2 - } - - numWorkers += opt.workers[i].num + if opt.metrics != nil { + metrics = opt.metrics } - if opt.numThreads <= 0 { - if numWorkers >= maxProcs { - // Start at least as many threads as workers, and keep a free thread to handle requests in non-worker mode - opt.numThreads = numWorkers + 1 - } else { - opt.numThreads = maxProcs - } - } else if opt.numThreads <= numWorkers { - return NotEnoughThreads + err := calculateMaxThreads(opt) + if err != nil { + return err } config := Config() @@ -337,6 +361,7 @@ func Shutdown() { stopWorkers() close(done) shutdownWG.Wait() + metrics.Shutdown() requestChan = nil // Always reset the WaitGroup to ensure we're in a clean state @@ -449,12 +474,20 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error } fc.responseWriter = responseWriter + fc.startedAt = time.Now() + + isWorker := fc.responseWriter == nil + isWorkerRequest := false rc := requestChan // Detect if a worker is available to handle this request - if nil != fc.responseWriter { + if !isWorker { if v, ok := workersRequestChans.Load(fc.scriptFilename); ok { + isWorkerRequest = true + metrics.StartWorkerRequest(fc.scriptFilename) rc = v.(chan *http.Request) + } else { + metrics.StartRequest() } } @@ -464,6 +497,14 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error <-fc.done } + if !isWorker { + if isWorkerRequest { + metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) + } else { + metrics.StopRequest() + } + } + return nil } diff --git a/go.mod b/go.mod index ade40d11..27270765 100644 --- a/go.mod +++ b/go.mod @@ -6,20 +6,26 @@ retract v1.0.0-rc.1 // Human error require ( github.com/maypok86/otter v1.2.2 + github.com/prometheus/client_golang v1.19.1 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 golang.org/x/net v0.27.0 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dolthub/maphash v0.1.0 // indirect github.com/gammazero/deque v0.2.1 // indirect - github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4d010a6f..93880394 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -17,6 +21,14 @@ github.com/maypok86/otter v1.2.2/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdH github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= @@ -30,8 +42,12 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/metrics.go b/metrics.go new file mode 100644 index 00000000..3b083237 --- /dev/null +++ b/metrics.go @@ -0,0 +1,252 @@ +package frankenphp + +import ( + "github.com/prometheus/client_golang/prometheus" + "path/filepath" + "regexp" + "sync" + "time" +) + +var metricsNameRegex = regexp.MustCompile(`\W+`) +var metricsNameFixRegex = regexp.MustCompile(`^_+|_+$`) + +type Metrics interface { + // StartWorker collects started workers + StartWorker(name string) + // StopWorker collects stopped workers + StopWorker(name string) + // TotalWorkers collects expected workers + TotalWorkers(name string, num int) + // TotalThreads collects total threads + TotalThreads(num int) + // StartRequest collects started requests + StartRequest() + // StopRequest collects stopped requests + StopRequest() + // StopWorkerRequest collects stopped worker requests + StopWorkerRequest(name string, duration time.Duration) + // StartWorkerRequest collects started worker requests + StartWorkerRequest(name string) + Shutdown() +} + +type nullMetrics struct{} + +func (n nullMetrics) StartWorker(name string) { +} + +func (n nullMetrics) StopWorker(name string) { +} + +func (n nullMetrics) TotalWorkers(name string, num int) { +} + +func (n nullMetrics) TotalThreads(num int) { +} + +func (n nullMetrics) StartRequest() { +} + +func (n nullMetrics) StopRequest() { +} + +func (n nullMetrics) StopWorkerRequest(name string, duration time.Duration) { +} + +func (n nullMetrics) StartWorkerRequest(name string) { +} + +func (n nullMetrics) Shutdown() { +} + +type PrometheusMetrics struct { + registry prometheus.Registerer + totalThreads prometheus.Counter + busyThreads prometheus.Gauge + totalWorkers map[string]prometheus.Gauge + busyWorkers map[string]prometheus.Gauge + workerRequestTime map[string]prometheus.Counter + workerRequestCount map[string]prometheus.Counter + mu sync.Mutex +} + +func (m *PrometheusMetrics) StartWorker(name string) { + m.busyThreads.Inc() + + // tests do not register workers before starting them + if _, ok := m.totalWorkers[name]; !ok { + return + } + m.totalWorkers[name].Inc() +} + +func (m *PrometheusMetrics) StopWorker(name string) { + m.busyThreads.Dec() + + // tests do not register workers before starting them + if _, ok := m.totalWorkers[name]; !ok { + return + } + m.totalWorkers[name].Dec() +} + +func (m *PrometheusMetrics) getIdentity(name string) (string, error) { + actualName, err := filepath.Abs(name) + if err != nil { + return name, err + } + + return actualName, nil +} + +func (m *PrometheusMetrics) TotalWorkers(name string, num int) { + m.mu.Lock() + defer m.mu.Unlock() + + identity, err := m.getIdentity(name) + if err != nil { + // do not create metrics, let error propagate when worker is started + return + } + + subsystem := getWorkerNameForMetrics(name) + + if _, ok := m.totalWorkers[identity]; !ok { + m.totalWorkers[identity] = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "frankenphp", + Subsystem: subsystem, + Name: "total_workers", + Help: "Total number of PHP workers for this worker", + }) + m.registry.MustRegister(m.totalWorkers[identity]) + } + + if _, ok := m.busyWorkers[identity]; !ok { + m.busyWorkers[identity] = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "frankenphp", + Subsystem: subsystem, + Name: "busy_workers", + Help: "Number of busy PHP workers for this worker", + }) + m.registry.MustRegister(m.busyWorkers[identity]) + } + + if _, ok := m.workerRequestTime[identity]; !ok { + m.workerRequestTime[identity] = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "frankenphp", + Subsystem: subsystem, + Name: "worker_request_time", + }) + m.registry.MustRegister(m.workerRequestTime[identity]) + } + + if _, ok := m.workerRequestCount[identity]; !ok { + m.workerRequestCount[identity] = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "frankenphp", + Subsystem: subsystem, + Name: "worker_request_count", + }) + m.registry.MustRegister(m.workerRequestCount[identity]) + } +} + +func (m *PrometheusMetrics) TotalThreads(num int) { + m.totalThreads.Add(float64(num)) +} + +func (m *PrometheusMetrics) StartRequest() { + m.busyThreads.Inc() +} + +func (m *PrometheusMetrics) StopRequest() { + m.busyThreads.Dec() +} + +func (m *PrometheusMetrics) StopWorkerRequest(name string, duration time.Duration) { + if _, ok := m.workerRequestTime[name]; !ok { + return + } + + m.workerRequestCount[name].Inc() + m.busyWorkers[name].Dec() + m.workerRequestTime[name].Add(duration.Seconds()) +} + +func (m *PrometheusMetrics) StartWorkerRequest(name string) { + if _, ok := m.busyWorkers[name]; !ok { + return + } + m.busyWorkers[name].Inc() +} + +func (m *PrometheusMetrics) Shutdown() { + m.registry.Unregister(m.totalThreads) + m.registry.Unregister(m.busyThreads) + + for _, g := range m.totalWorkers { + m.registry.Unregister(g) + } + + for _, g := range m.busyWorkers { + m.registry.Unregister(g) + } + + for _, c := range m.workerRequestTime { + m.registry.Unregister(c) + } + + for _, c := range m.workerRequestCount { + m.registry.Unregister(c) + } + + m.totalThreads = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "frankenphp_total_threads", + Help: "Total number of PHP threads", + }) + m.busyThreads = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "frankenphp_busy_threads", + Help: "Number of busy PHP threads", + }) + m.totalWorkers = map[string]prometheus.Gauge{} + m.busyWorkers = map[string]prometheus.Gauge{} + m.workerRequestTime = map[string]prometheus.Counter{} + m.workerRequestCount = map[string]prometheus.Counter{} + + m.registry.MustRegister(m.totalThreads) + m.registry.MustRegister(m.busyThreads) +} + +func getWorkerNameForMetrics(name string) string { + name = metricsNameRegex.ReplaceAllString(name, "_") + name = metricsNameFixRegex.ReplaceAllString(name, "") + + return name +} + +func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { + if registry == nil { + registry = prometheus.NewRegistry() + } + + m := &PrometheusMetrics{ + registry: registry, + totalThreads: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "frankenphp_total_threads", + Help: "Total number of PHP threads", + }), + busyThreads: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "frankenphp_busy_threads", + Help: "Number of busy PHP threads", + }), + totalWorkers: map[string]prometheus.Gauge{}, + busyWorkers: map[string]prometheus.Gauge{}, + workerRequestTime: map[string]prometheus.Counter{}, + workerRequestCount: map[string]prometheus.Counter{}, + } + + m.registry.MustRegister(m.totalThreads) + m.registry.MustRegister(m.busyThreads) + + return m +} diff --git a/metrics_test.go b/metrics_test.go new file mode 100644 index 00000000..2e6640d7 --- /dev/null +++ b/metrics_test.go @@ -0,0 +1,82 @@ +package frankenphp + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetWorkerNameForMetrics(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"worker-1", "worker_1"}, + {"worker@name", "worker_name"}, + {"worker name", "worker_name"}, + {"worker/name", "worker_name"}, + {"worker.name", "worker_name"}, + {"////worker////name...//worker", "worker_name_worker"}, + } + + for _, test := range tests { + result := getWorkerNameForMetrics(test.input) + assert.Equal(t, test.expected, result) + } +} + +func createPrometheusMetrics() *PrometheusMetrics { + return &PrometheusMetrics{ + registry: prometheus.NewRegistry(), + totalThreads: prometheus.NewCounter(prometheus.CounterOpts{Name: "total_threads"}), + busyThreads: prometheus.NewGauge(prometheus.GaugeOpts{Name: "busy_threads"}), + totalWorkers: make(map[string]prometheus.Gauge), + busyWorkers: make(map[string]prometheus.Gauge), + workerRequestTime: make(map[string]prometheus.Counter), + workerRequestCount: make(map[string]prometheus.Counter), + mu: sync.Mutex{}, + } +} + +func TestPrometheusMetrics_TotalWorkers(t *testing.T) { + m := createPrometheusMetrics() + + tests := []struct { + name string + worker string + num int + }{ + {"SetWorkers", "test_worker", 5}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m.TotalWorkers(tt.worker, tt.num) + actualName, _ := m.getIdentity(tt.worker) + _, ok := m.totalWorkers[actualName] + require.True(t, ok) + }) + } +} + +func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) { + m := createPrometheusMetrics() + m.StopWorkerRequest("test_worker", 2*time.Second) + + name := "test_worker" + _, ok := m.workerRequestTime[name] + require.False(t, ok) +} + +func TestPrometheusMetrics_StartWorkerRequest(t *testing.T) { + m := createPrometheusMetrics() + m.StartWorkerRequest("test_worker") + + name := "test_worker" + _, ok := m.workerRequestCount[name] + require.False(t, ok) +} diff --git a/options.go b/options.go index 0c5622b8..70b90c6a 100644 --- a/options.go +++ b/options.go @@ -14,6 +14,7 @@ type opt struct { numThreads int workers []workerOpt logger *zap.Logger + metrics Metrics } type workerOpt struct { @@ -31,6 +32,14 @@ func WithNumThreads(numThreads int) Option { } } +func WithMetrics(m Metrics) Option { + return func(o *opt) error { + o.metrics = m + + return nil + } +} + // WithWorkers configures the PHP workers to start. func WithWorkers(fileName string, num int, env map[string]string) Option { return func(o *opt) error { diff --git a/worker.go b/worker.go index 72dc8a89..fb86ee15 100644 --- a/worker.go +++ b/worker.go @@ -63,6 +63,9 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { for { // Create main dummy request r, err := http.NewRequest(http.MethodGet, filepath.Base(absFileName), nil) + + metrics.StartWorker(absFileName) + if err != nil { panic(err) } @@ -93,6 +96,7 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { // TODO: make the max restart configurable if _, ok := workersRequestChans.Load(absFileName); ok { + metrics.StopWorker(absFileName) workersReadyWG.Add(1) if fc.exitStatus == 0 { if c := l.Check(zapcore.InfoLevel, "restarting"); c != nil {