From 81a2c0bec6d76dc67cd4ca8495d3578e9763498a Mon Sep 17 00:00:00 2001 From: David Dobmeier Date: Mon, 3 May 2021 11:33:53 -0400 Subject: [PATCH 1/3] Update go mod and dependencies to be compatible with 'go get' --- .gitignore | 3 ++- go.mod | 17 ++++++++--------- go.sum | 7 +++---- main.go | 4 ++-- pkg/metrics/get_billable_from_github.go | 3 ++- pkg/metrics/get_runners_from_github.go | 3 ++- .../get_runners_organization_from_github.go | 3 ++- pkg/metrics/get_workflow_runs_from_github.go | 3 ++- pkg/metrics/metrics.go | 3 ++- pkg/metrics/workflows_cache.go | 2 +- pkg/server/server.go | 4 ++-- 11 files changed, 28 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 66fd13c..bb18ae9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.dll *.so *.dylib +github-actions-exporter # Test binary, built with `go test -c` *.test @@ -12,4 +13,4 @@ *.out # Dependency directories (remove the comment below to include it) -# vendor/ +vendor/ diff --git a/go.mod b/go.mod index d46bb1a..6165079 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,13 @@ -module github-actions-exporter +module github.com/Spendesk/github-actions-exporter go 1.16 require ( - github.com/fasthttp/router v1.3.9 // indirect - github.com/google/go-github/v33 v33.0.1-0.20210311004518-0540c33dca8b // indirect - github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect - github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/prometheus/client_golang v1.9.0 // indirect - github.com/urfave/cli/v2 v2.3.0 // indirect - github.com/valyala/fasthttp v1.22.0 // indirect - golang.org/x/oauth2 v0.0.0-20210311163135-5366d9dc1934 // indirect + github.com/fasthttp/router v1.3.9 + github.com/google/go-github/v33 v33.0.1-0.20210311004518-0540c33dca8b + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 + github.com/prometheus/client_golang v1.9.0 + github.com/urfave/cli/v2 v2.3.0 + github.com/valyala/fasthttp v1.22.0 + golang.org/x/oauth2 v0.0.0-20210311163135-5366d9dc1934 ) diff --git a/go.sum b/go.sum index 97f1384..44cba5d 100644 --- a/go.sum +++ b/go.sum @@ -142,7 +142,6 @@ github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -150,6 +149,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v33 v33.0.1-0.20210311004518-0540c33dca8b h1:EUf9qFA+oxcRcZVR3UlG4FM031+ZA8DQWHgedI7fnm8= github.com/google/go-github/v33 v33.0.1-0.20210311004518-0540c33dca8b/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= @@ -275,14 +275,13 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -567,6 +566,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -646,7 +646,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= diff --git a/main.go b/main.go index f194e6f..846421f 100644 --- a/main.go +++ b/main.go @@ -6,8 +6,8 @@ import ( "github.com/urfave/cli/v2" - "github-actions-exporter/pkg/config" - "github-actions-exporter/pkg/server" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/Spendesk/github-actions-exporter/pkg/server" ) var ( diff --git a/pkg/metrics/get_billable_from_github.go b/pkg/metrics/get_billable_from_github.go index 9394a53..000c8b8 100644 --- a/pkg/metrics/get_billable_from_github.go +++ b/pkg/metrics/get_billable_from_github.go @@ -2,12 +2,13 @@ package metrics import ( "context" - "github-actions-exporter/pkg/config" "log" "strconv" "strings" "time" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/prometheus/client_golang/prometheus" ) diff --git a/pkg/metrics/get_runners_from_github.go b/pkg/metrics/get_runners_from_github.go index f29c7c0..9704ec8 100644 --- a/pkg/metrics/get_runners_from_github.go +++ b/pkg/metrics/get_runners_from_github.go @@ -2,12 +2,13 @@ package metrics import ( "context" - "github-actions-exporter/pkg/config" "log" "strconv" "strings" "time" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/prometheus/client_golang/prometheus" ) diff --git a/pkg/metrics/get_runners_organization_from_github.go b/pkg/metrics/get_runners_organization_from_github.go index 2d4a661..da46abf 100644 --- a/pkg/metrics/get_runners_organization_from_github.go +++ b/pkg/metrics/get_runners_organization_from_github.go @@ -2,11 +2,12 @@ package metrics import ( "context" - "github-actions-exporter/pkg/config" "log" "strconv" "time" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/google/go-github/v33/github" "github.com/prometheus/client_golang/prometheus" ) diff --git a/pkg/metrics/get_workflow_runs_from_github.go b/pkg/metrics/get_workflow_runs_from_github.go index f13bac1..bde2edf 100644 --- a/pkg/metrics/get_workflow_runs_from_github.go +++ b/pkg/metrics/get_workflow_runs_from_github.go @@ -2,12 +2,13 @@ package metrics import ( "context" - "github-actions-exporter/pkg/config" "log" "strconv" "strings" "time" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/prometheus/client_golang/prometheus" ) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4f2ebf4..2961613 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -1,10 +1,11 @@ package metrics import ( - "github-actions-exporter/pkg/config" "log" "net/http" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/google/go-github/v33/github" "github.com/gregjones/httpcache" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/metrics/workflows_cache.go b/pkg/metrics/workflows_cache.go index cbd2dd1..6205fab 100644 --- a/pkg/metrics/workflows_cache.go +++ b/pkg/metrics/workflows_cache.go @@ -8,7 +8,7 @@ import ( "github.com/google/go-github/v33/github" - "github-actions-exporter/pkg/config" + "github.com/Spendesk/github-actions-exporter/pkg/config" ) var ( diff --git a/pkg/server/server.go b/pkg/server/server.go index 2a9f3c7..d50351d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -8,8 +8,8 @@ import ( "github.com/urfave/cli/v2" "github.com/valyala/fasthttp" - "github-actions-exporter/pkg/config" - "github-actions-exporter/pkg/metrics" + "github.com/Spendesk/github-actions-exporter/pkg/config" + "github.com/Spendesk/github-actions-exporter/pkg/metrics" ) // RunServer - run http server for expose metrics From 10c3514d2b68e12904e952976dfd0c7c44f74a31 Mon Sep 17 00:00:00 2001 From: David Dobmeier Date: Mon, 3 May 2021 12:43:57 -0400 Subject: [PATCH 2/3] Fix segfault if repo has no workflows or token doesnt have permissions Fixes a segfault if a provided repo has no workflows, or if the token used in the exporter doesnt have all the required permissions. Still logs errors appropriately. --- pkg/metrics/workflows_cache.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/workflows_cache.go b/pkg/metrics/workflows_cache.go index 6205fab..b180faa 100644 --- a/pkg/metrics/workflows_cache.go +++ b/pkg/metrics/workflows_cache.go @@ -29,8 +29,10 @@ func workflowCache() { } s := make(map[int64]github.Workflow) - for _, w := range resp.Workflows { - s[*w.ID] = *w + if resp != nil { + for _, w := range resp.Workflows { + s[*w.ID] = *w + } } ww[repo] = s From 1a0f90ed860ae1dd7de1b0703ab2b2ab6774a688 Mon Sep 17 00:00:00 2001 From: David Dobmeier Date: Mon, 3 May 2021 12:46:55 -0400 Subject: [PATCH 3/3] Add metrics for runner 'busy' status Adds metrics for runner busy status to allow monitoring of when to scale runners based on how many runners are idle or in use at any given time. This was tested with PR #19 for ORG runner metrics as well without issue. --- pkg/metrics/get_runners_from_github.go | 12 ++++++++++++ pkg/metrics/get_runners_organization_from_github.go | 12 ++++++++++++ pkg/metrics/metrics.go | 2 ++ 3 files changed, 26 insertions(+) diff --git a/pkg/metrics/get_runners_from_github.go b/pkg/metrics/get_runners_from_github.go index 9704ec8..7198545 100644 --- a/pkg/metrics/get_runners_from_github.go +++ b/pkg/metrics/get_runners_from_github.go @@ -20,6 +20,13 @@ var ( }, []string{"repo", "os", "name", "id"}, ) + runnersBusyGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "github_runner_busy", + Help: "runner busy status", + }, + []string{"repo", "os", "name", "id"}, + ) ) // getRunnersFromGithub - return information about runners and their status for a specific repo @@ -37,6 +44,11 @@ func getRunnersFromGithub() { } else { runnersGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(0) } + if runner.GetBusy() { + runnersBusyGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(1) + } else { + runnersBusyGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(0) + } } } } diff --git a/pkg/metrics/get_runners_organization_from_github.go b/pkg/metrics/get_runners_organization_from_github.go index da46abf..ca01ac5 100644 --- a/pkg/metrics/get_runners_organization_from_github.go +++ b/pkg/metrics/get_runners_organization_from_github.go @@ -20,6 +20,13 @@ var ( }, []string{"organization", "os", "name", "id"}, ) + runnersOrganizationBusyGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "github_runner_organization_busy", + Help: "runner busy status", + }, + []string{"organization", "os", "name", "id"}, + ) ) // getRunnersOrganizationFromGithub - return information about runners and their status for an organization @@ -38,6 +45,11 @@ func getRunnersOrganizationFromGithub() { } else { runnersOrganizationGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(0) } + if runner.GetBusy() { + runnersOrganizationBusyGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(1) + } else { + runnersOrganizationBusyGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(0) + } } if rr.NextPage == 0 { break diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2961613..cc5a40e 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -20,7 +20,9 @@ var ( // InitMetrics - register metrics in prometheus lib and start func for monitor func InitMetrics() { prometheus.MustRegister(runnersGauge) + prometheus.MustRegister(runnersBusyGauge) prometheus.MustRegister(runnersOrganizationGauge) + prometheus.MustRegister(runnersOrganizationBusyGauge) prometheus.MustRegister(workflowRunStatusGauge) prometheus.MustRegister(workflowRunStatusDeprecatedGauge) prometheus.MustRegister(workflowRunDurationGauge)