Open
Description
Something seems to have gotten stuck running alertmanager tests that hit the 30m timeout:
panic: test timed out after 30m0s
running tests:
TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled/receiver=opsgenie_firewall_enabled=false (29m48s)
goroutine 4962 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:2484 +0x605
created by time.goFunc
/usr/local/go/src/time/sleep.go:215 +0x45
goroutine 1 [chan receive, 29 minutes]:
testing.(*T).Run(0xc000252380, {0x285c950, 0x4c}, 0x297bf90)
/usr/local/go/src/testing/testing.go:1859 +0x91e
testing.runTests.func1(0xc000252380)
/usr/local/go/src/testing/testing.go:2279 +0x86
testing.tRunner(0xc000252380, 0xc000915ae0)
/usr/local/go/src/testing/testing.go:1792 +0x226
testing.runTests(0xc000303608, {0x3d47300, 0x41, 0x41}, {0x1?, 0x0?, 0x3d59f80?})
/usr/local/go/src/testing/testing.go:2277 +0x96d
testing.(*M).Run(0xc00077b5e0)
/usr/local/go/src/testing/testing.go:2142 +0xeeb
main.main()
_testmain.go:173 +0x165
goroutine 6 [select]:
github.com/golang/glog.(*fileSink).flushDaemon(0x3d5a018)
/__w/mimir/mimir/vendor/github.com/golang/glog/glog_file.go:380 +0x179
created by github.com/golang/glog.init.1 in goroutine 1
/__w/mimir/mimir/vendor/github.com/golang/glog/glog_file.go:188 +0x2be
goroutine 7 [select, 30 minutes]:
github.com/dgraph-io/ristretto.(*defaultPolicy).processItems(0xc0004d6600)
/__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/policy.go:102 +0xfd
created by github.com/dgraph-io/ristretto.newDefaultPolicy in goroutine 1
/__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/policy.go:86 +0x2a5
goroutine 8 [select]:
github.com/dgraph-io/ristretto.(*Cache).processItems(0xc0004ce000)
/__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/cache.go:452 +0x229
created by github.com/dgraph-io/ristretto.NewCache in goroutine 1
/__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/cache.go:207 +0xe46
goroutine 189 [chan receive, 29 minutes]:
testing.(*T).Parallel(0xc000aa2540)
/usr/local/go/src/testing/testing.go:1577 +0x4ff
github.com/grafana/mimir/pkg/alertmanager.TestIsHealthyForAlertmanagerOperations(0xc000aa2540)
/__w/mimir/mimir/pkg/alertmanager/alertmanager_ring_test.go:17 +0x3f
testing.tRunner(0xc000aa2540, 0x297bf58)
/usr/local/go/src/testing/testing.go:1792 +0x226
created by testing.(*T).Run in goroutine 1
/usr/local/go/src/testing/testing.go:1851 +0x8f3
goroutine 37 [chan receive, 30 minutes]:
github.com/grafana/mimir/pkg/alertmanager.init.0.func1()
/__w/mimir/mimir/pkg/alertmanager/alertmanager.go:166 +0x54
created by github.com/grafana/mimir/pkg/alertmanager.init.0 in goroutine 1
/__w/mimir/mimir/pkg/alertmanager/alertmanager.go:161 +0x25
goroutine 4683 [sync.WaitGroup.Wait, 29 minutes]:
sync.runtime_SemacquireWaitGroup(0xc000cd5d60?)
/usr/local/go/src/runtime/sema.go:110 +0x25
sync.(*WaitGroup).Wait(0xc000cd5d60)
/usr/local/go/src/sync/waitgroup.go:118 +0x89
github.com/prometheus/alertmanager/notify.FanoutStage.Exec({0xc001d666d0, 0x1, 0xc002279740?}, {0x2e8eb98, 0xc0013d0750}, {0x2e78ae0, 0xc00112fc20}, {0xc0021c73e8, 0x1, 0x1})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:505 +0x370
github.com/prometheus/alertmanager/notify.MultiStage.Exec({0xc001c00600, 0x6, 0xc0019756a8?}, {0x2e8eb98, 0xc0013d0750}, {0x2e78ae0, 0xc00112fc20}, {0xc0021c73d0, 0x1, 0x1})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:477 +0x168
github.com/prometheus/alertmanager/notify.RoutingStage.Exec(0xc001bdea20, {0x2e8eb98, 0xc0013d0750}, {0x2e78ae0, 0xc00112fc20}, {0xc0021c73d0, 0x1, 0x1})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:463 +0x1b0
github.com/prometheus/alertmanager/dispatch.(*Dispatcher).processAlert.func1({0x2e8eb98, 0xc0013d06f0}, {0xc0021c73d0, 0x1, 0x1})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:384 +0x76c
github.com/prometheus/alertmanager/dispatch.(*aggrGroup).flush(0xc00139d3b0, {0x2e8eb98, 0xc0013d06f0}, 0xc00074d1f0)
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:565 +0x925
github.com/prometheus/alertmanager/dispatch.(*aggrGroup).run(0xc00139d3b0, 0xc00074d1f0)
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:500 +0x71a
created by github.com/prometheus/alertmanager/dispatch.(*Dispatcher).processAlert in goroutine 4619
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:373 +0x7bd
goroutine 4562 [select]:
github.com/grafana/dskit/kv/consul.(*mockKV).loop(0xc001a9cbe0)
/__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:90 +0x1ab
created by github.com/grafana/dskit/kv/consul.NewInMemoryClientWithConfig in goroutine 2120
/__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:61 +0x286
goroutine 2010 [chan receive, 29 minutes]:
testing.tRunner.func1()
/usr/local/go/src/testing/testing.go:1753 +0x965
testing.tRunner(0xc000cd0e00, 0x297bf90)
/usr/local/go/src/testing/testing.go:1798 +0x25f
created by testing.(*T).Run in goroutine 1
/usr/local/go/src/testing/testing.go:1851 +0x8f3
goroutine 2120 [select, 29 minutes]:
github.com/grafana/dskit/services.(*BasicService).awaitState(0xc001672640, {0x2e8e930, 0x3d807c0}, 0x4, 0xc00227fce0)
/__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:297 +0xca
github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc001672640, {0x2e8e930, 0x3d807c0})
/__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:293 +0x54
github.com/grafana/dskit/services.StopAndAwaitTerminated({0x2e8e930, 0x3d807c0}, {0x2e95cd0, 0xc0012b38c0})
/__w/mimir/mimir/vendor/github.com/grafana/dskit/services/services.go:122 +0x5f
github.com/grafana/mimir/pkg/alertmanager.setupSingleMultitenantAlertmanager.func2()
/__w/mimir/mimir/pkg/alertmanager/multitenant_test.go:149 +0x49
testing.(*common).Cleanup.func1()
/usr/local/go/src/testing/testing.go:1211 +0x170
testing.(*common).runCleanup(0xc000f16a80, 0x0)
/usr/local/go/src/testing/testing.go:1445 +0x2b4
testing.tRunner.func2()
/usr/local/go/src/testing/testing.go:1786 +0x4d
testing.tRunner(0xc000f16a80, 0xc00121b758)
/usr/local/go/src/testing/testing.go:1798 +0x25f
created by testing.(*T).Run in goroutine 2010
/usr/local/go/src/testing/testing.go:1851 +0x8f3
goroutine 4646 [sync.RWMutex.RLock, 29 minutes]:
sync.runtime_SemacquireRWMutexR(0xc002872ac0?, 0x1?, 0x3d59f80?)
/usr/local/go/src/runtime/sema.go:100 +0x25
sync.(*RWMutex).RLock(0xc002872ab0)
/usr/local/go/src/sync/rwmutex.go:74 +0x5b
github.com/prometheus/alertmanager/nflog.(*Log).Snapshot(0xc002872a80, {0x2e78de0, 0xc002569ab8})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:500 +0x145
github.com/prometheus/alertmanager/nflog.(*Log).Maintenance.func1()
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:324 +0x88
github.com/prometheus/alertmanager/nflog.(*Log).Maintenance.func2(0xc0001c8ef8)
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:339 +0x3c8
github.com/prometheus/alertmanager/nflog.(*Log).Maintenance(0xc002872a80, 0xd18c2e2800, {0xc000fea3c0, 0x9c}, 0xc0018c1490, 0x0)
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:365 +0x795
github.com/grafana/mimir/pkg/alertmanager.New.func1()
/__w/mimir/mimir/pkg/alertmanager/alertmanager.go:236 +0x88
created by github.com/grafana/mimir/pkg/alertmanager.New in goroutine 4587
/__w/mimir/mimir/pkg/alertmanager/alertmanager.go:235 +0xd45
goroutine 4587 [sync.WaitGroup.Wait, 29 minutes]:
sync.runtime_SemacquireWaitGroup(0xc0028682c0?)
/usr/local/go/src/runtime/sema.go:110 +0x25
sync.(*WaitGroup).Wait(0xc0028682c0)
/usr/local/go/src/sync/waitgroup.go:118 +0x89
github.com/grafana/mimir/pkg/alertmanager.(*Alertmanager).StopAndWait(0xc002868248)
/__w/mimir/mimir/pkg/alertmanager/alertmanager.go:582 +0x633
github.com/grafana/mimir/pkg/alertmanager.(*MultitenantAlertmanager).stopping(0xc0012b38c0, {0x2e772a0?, 0x3d148e0?})
/__w/mimir/mimir/pkg/alertmanager/multitenant.go:623 +0x10f
github.com/grafana/dskit/services.(*BasicService).main(0xc001672640)
/__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:210 +0x542
created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 2120
/__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:122 +0x1cb
goroutine 4684 [chan send, 29 minutes]:
github.com/grafana/mimir/pkg/alertmanager.(*state).broadcast(...)
/__w/mimir/mimir/pkg/alertmanager/state_replication.go:329
github.com/grafana/mimir/pkg/alertmanager.(*stateChannel).Broadcast(0xc0014689d8, {0xc00269d600, 0x48, 0x80})
/__w/mimir/mimir/pkg/alertmanager/state_replication.go:342 +0x191
github.com/prometheus/alertmanager/nflog.(*Log).Log(0xc002872a80, 0xc000bca550, {0xc000cd5ccb, 0x5}, {0xc000cd5d78, 0x1, 0x1}, {0x3d807c0, 0x0, 0x0}, ...)
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:417 +0x76b
github.com/prometheus/alertmanager/notify.SetNotifiesStage.Exec({{0x2e81480?, 0xc002872a80?}, 0xc000bca550?}, {0x2e8eb98, 0xc0013d07b0}, {0xc001e3dda0?, 0x44ad74?}, {0xc0021c73e8, 0x1, 0x1})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:952 +0x267
github.com/prometheus/alertmanager/notify.MultiStage.Exec({0xc001ec2800, 0x4, 0x0?}, {0x2e8eb98, 0xc0013d0750}, {0x2e78ae0, 0xc00112fc20}, {0xc0021c73e8, 0x1, 0x1})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:477 +0x168
github.com/prometheus/alertmanager/notify.FanoutStage.Exec.func1({0x2e78c40, 0xc000010270})
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:499 +0xc7
created by github.com/prometheus/alertmanager/notify.FanoutStage.Exec in goroutine 4683
/__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:498 +0xf0
FAIL github.com/grafana/mimir/pkg/alertmanager 1800.100s