Added internal count metrics in addition to the external counts that (c…

…adence-workflow#6135)

are already there

What changed?
Added new metric which gives the max per workflow ID rps we have seen for the domain every second

Why?
This will help users understand how close they are to the per workflow ID rate limits

How did you test it?
Unit tests

Potential risks
Just adding a metric, so should be safe

Release notes

Documentation Changes

common/metrics/defs.go

@@ -2476,6 +2476,7 @@ const (
 	WorkflowIDCacheSizeGauge
 	WorkflowIDCacheRequestsExternalRatelimitedCounter
 	WorkflowIDCacheRequestsExternalMaxRequestsPerSecondsTimer
+	WorkflowIDCacheRequestsInternalMaxRequestsPerSecondsTimer
 	WorkflowIDCacheRequestsInternalRatelimitedCounter
 	NumHistoryMetrics
 )
@@ -3115,6 +3116,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		WorkflowIDCacheSizeGauge:                                     {metricName: "workflow_id_cache_size", metricType: Gauge},
 		WorkflowIDCacheRequestsExternalRatelimitedCounter:            {metricName: "workflow_id_external_requests_ratelimited", metricType: Counter},
 		WorkflowIDCacheRequestsExternalMaxRequestsPerSecondsTimer:    {metricName: "workflow_id_external_requests_max_requests_per_seconds", metricType: Timer},
+		WorkflowIDCacheRequestsInternalMaxRequestsPerSecondsTimer:    {metricName: "workflow_id_internal_requests_max_requests_per_seconds", metricType: Timer},
 		WorkflowIDCacheRequestsInternalRatelimitedCounter:            {metricName: "workflow_id_internal_requests_ratelimited", metricType: Counter},
 	},
 	Matching: {

service/history/workflowcache/cache.go

@@ -70,7 +70,8 @@ type cacheKey struct {
 type cacheValue struct {
 	externalRateLimiter quotas.Limiter
 	internalRateLimiter quotas.Limiter
-	countMetric         workflowIDCountMetric
+	externalCountMetric workflowIDCountMetric
+	internalCountMetric workflowIDCountMetric
 }
 
 // Params is the parameters for a new WFCache
@@ -144,13 +145,14 @@ func (c *wfCache) allow(domainID string, workflowID string, rateLimitType rateLi
 
 	switch rateLimitType {
 	case external:
-		value.countMetric.updatePerDomainMaxWFRequestCount(domainName, c.timeSource, c.metricsClient)
+		value.externalCountMetric.updatePerDomainMaxWFRequestCount(domainName, c.timeSource, c.metricsClient, metrics.WorkflowIDCacheRequestsExternalMaxRequestsPerSecondsTimer)
 		if !value.externalRateLimiter.Allow() {
 			c.emitRateLimitMetrics(domainID, workflowID, domainName, "external", metrics.WorkflowIDCacheRequestsExternalRatelimitedCounter)
 			return false
 		}
 		return true
 	case internal:
+		value.internalCountMetric.updatePerDomainMaxWFRequestCount(domainName, c.timeSource, c.metricsClient, metrics.WorkflowIDCacheRequestsInternalMaxRequestsPerSecondsTimer)
 		if !value.internalRateLimiter.Allow() {
 			c.emitRateLimitMetrics(domainID, workflowID, domainName, "internal", metrics.WorkflowIDCacheRequestsInternalRatelimitedCounter)
 			return false

service/history/workflowcache/metrics.go

@@ -50,6 +50,7 @@ func (cm *workflowIDCountMetric) updatePerDomainMaxWFRequestCount(
 	domainName string,
 	timeSource clock.TimeSource,
 	metricsClient metrics.Client,
+	metric int,
 ) {
 	cm.Lock()
 	defer cm.Unlock()
@@ -61,5 +62,5 @@ func (cm *workflowIDCountMetric) updatePerDomainMaxWFRequestCount(
 
 	// We can just use the upper of the metric, so it is not an issue to emit all the counts
 	metricsClient.Scope(metrics.HistoryClientWfIDCacheScope, metrics.DomainTag(domainName)).
-		RecordTimer(metrics.WorkflowIDCacheRequestsExternalMaxRequestsPerSecondsTimer, time.Duration(cm.count))
+		RecordTimer(metric, time.Duration(cm.count))
 }

service/history/workflowcache/metrics_test.go

@@ -35,6 +35,7 @@ import (
 
 func TestUpdatePerDomainMaxWFRequestCount(t *testing.T) {
 	domainName := "some domain name"
+	metric := metrics.WorkflowIDCacheRequestsInternalMaxRequestsPerSecondsTimer
 
 	cases := []struct {
 		name                             string
@@ -45,23 +46,23 @@ func TestUpdatePerDomainMaxWFRequestCount(t *testing.T) {
 			name: "Single workflowID",
 			updatePerDomainMaxWFRequestCount: func(metricsClient metrics.Client, timeSource clock.MockedTimeSource) {
 				workflowID1 := &workflowIDCountMetric{}
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 1
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 2
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 1
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 2
 			},
 			expecetMetrics: []time.Duration{1, 2},
 		},
 		{
 			name: "Separate workflowIDs",
 			updatePerDomainMaxWFRequestCount: func(metricsClient metrics.Client, timeSource clock.MockedTimeSource) {
 				workflowID1 := &workflowIDCountMetric{}
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 1
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 1
 
 				workflowID2 := &workflowIDCountMetric{}
-				workflowID2.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 1
-				workflowID2.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 2
-				workflowID2.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 3
+				workflowID2.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 1
+				workflowID2.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 2
+				workflowID2.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 3
 
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 2
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 2
 
 			},
 			expecetMetrics: []time.Duration{1, 1, 2, 3, 2},
@@ -70,11 +71,11 @@ func TestUpdatePerDomainMaxWFRequestCount(t *testing.T) {
 			name: "Reset",
 			updatePerDomainMaxWFRequestCount: func(metricsClient metrics.Client, timeSource clock.MockedTimeSource) {
 				workflowID1 := &workflowIDCountMetric{}
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 1
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 2
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 1
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 2
 
 				timeSource.Advance(1100 * time.Millisecond)
-				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient) // Emits 1
+				workflowID1.updatePerDomainMaxWFRequestCount(domainName, timeSource, metricsClient, metric) // Emits 1
 			},
 			expecetMetrics: []time.Duration{1, 2, 1},
 		},