diff --git a/mapic/mistmetrics.go b/mapic/mistmetrics.go index 14aa5caa..67dc2d02 100644 --- a/mapic/mistmetrics.go +++ b/mapic/mistmetrics.go @@ -57,16 +57,7 @@ func (mc *mac) enrichMistMetrics(metrics string) string { res := strings.Builder{} lines := strings.Split(metrics, "\n") for i, line := range lines { - playbackID, ok := mc.parsePlaybackID(line) - if ok { - // Enrich labels for the lines that contains playbackID - oldStr := mc.streamLabel(playbackID) - newStr := mc.enrichLabels(playbackID) - res.WriteString(strings.Replace(line, oldStr, newStr, -1)) - } else { - // Do not enrich labels for lines that do not contain playbackID - res.WriteString(line) - } + res.WriteString(mc.enrichLine(line)) // Skip last end of line to preserve the same number of lines as Mist if i < len(lines)-1 { @@ -76,6 +67,42 @@ func (mc *mac) enrichMistMetrics(metrics string) string { return res.String() } +func (mc *mac) enrichLine(line string) string { + res := mc.enrichPlaybackSpecificLabels(line) + return mc.enrichConstLabels(res) +} + +func (mc *mac) enrichPlaybackSpecificLabels(line string) string { + playbackID, ok := mc.parsePlaybackID(line) + if ok { + // Enrich labels for the lines that contains playbackID + oldStr := mc.streamLabel(playbackID) + newStr := mc.enrichLabels(playbackID) + return strings.Replace(line, oldStr, newStr, 1) + } + return line +} + +func (mc *mac) enrichConstLabels(line string) string { + constLabels := fmt.Sprintf(`catalyst="true",catalyst_node="%s"`, mc.nodeID) + if len(line) == 0 || strings.HasPrefix(line, "#") { + // empty lines or comments + return line + } + if strings.Contains(line, "}") { + // metrics with labels + return strings.Replace(line, "}", fmt.Sprintf(",%s}", constLabels), 1) + } + // metrics without labels + lineSplit := strings.Split(line, " ") + if len(lineSplit) < 2 { + // invalid metric, do not enrich + return line + } + metricName := lineSplit[0] + return strings.Replace(line, metricName, fmt.Sprintf("%s{%s}", metricName, constLabels), 1) +} + func (mc *mac) parsePlaybackID(line string) (string, bool) { match := mc.streamMetricsRe.FindStringSubmatch(line) if len(match) > 1 { @@ -86,7 +113,6 @@ func (mc *mac) parsePlaybackID(line string) (string, bool) { func (mc *mac) enrichLabels(playbackID string) string { res := mc.streamLabel(playbackID) - res += `,catalyst="true"` si, err := mc.getStreamInfo(playbackID) if err != nil { glog.Warning("could not enrich Mist metrics for stream=%s err=%v", playbackID, err) diff --git a/mapic/mistmetrics_test.go b/mapic/mistmetrics_test.go index 02e943fd..0903c797 100644 --- a/mapic/mistmetrics_test.go +++ b/mapic/mistmetrics_test.go @@ -19,12 +19,15 @@ mist_sessions{stream="video+077bh6xx5bx5tdua",sessType="viewers"}1 mist_latency{stream="video+077bh6xx5bx5tdua",source="sin-prod-catalyst-3.lp-playback.studio"}1795 mist_sessions{stream="video+51b13mqy7sgw520w",sessType="viewers"}5 mist_latency{stream="video+51b13mqy7sgw520w",source="prg-prod-catalyst-0.lp-playback.studio"}1156 +mist_sessions_count{sessType="viewers"}93 +mist_logs 14860632 ` func TestEnrichMistMetrics(t *testing.T) { // given mc := mac{ baseStreamName: "video", + nodeID: "fra-staging-staging-catalyst-0.livepeer.monster", streamInfo: map[string]*streamInfo{ "077bh6xx5bx5tdua": {stream: &api.Stream{UserID: "abcdefgh-123456789"}}, "51b13mqy7sgw520w": {stream: &api.Stream{UserID: "hgfedcba-987654321"}}, @@ -41,11 +44,13 @@ func TestEnrichMistMetrics(t *testing.T) { require.Equal(t, len(inLines), len(resLines)) expLines := []string{ - `version{app="MistServer",version="729ddd4b42980d0124c72a46f13d8e0697293e94",release="Generic_x86_64"} 1`, - `mist_sessions{stream="video+077bh6xx5bx5tdua",catalyst="true",user_id="abcdefgh-123456789",sessType="viewers"}1`, - `mist_latency{stream="video+077bh6xx5bx5tdua",catalyst="true",user_id="abcdefgh-123456789",source="sin-prod-catalyst-3.lp-playback.studio"}1795`, - `mist_sessions{stream="video+51b13mqy7sgw520w",catalyst="true",user_id="hgfedcba-987654321",sessType="viewers"}5`, - `mist_latency{stream="video+51b13mqy7sgw520w",catalyst="true",user_id="hgfedcba-987654321",source="prg-prod-catalyst-0.lp-playback.studio"}1156`, + `version{app="MistServer",version="729ddd4b42980d0124c72a46f13d8e0697293e94",release="Generic_x86_64",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"} 1`, + `mist_sessions{stream="video+077bh6xx5bx5tdua",user_id="abcdefgh-123456789",sessType="viewers",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}1`, + `mist_latency{stream="video+077bh6xx5bx5tdua",user_id="abcdefgh-123456789",source="sin-prod-catalyst-3.lp-playback.studio",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}1795`, + `mist_sessions{stream="video+51b13mqy7sgw520w",user_id="hgfedcba-987654321",sessType="viewers",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}5`, + `mist_latency{stream="video+51b13mqy7sgw520w",user_id="hgfedcba-987654321",source="prg-prod-catalyst-0.lp-playback.studio",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}1156`, + `mist_sessions_count{sessType="viewers",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}93`, + `mist_logs{catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"} 14860632`, } for _, exp := range expLines { require.Contains(t, resLines, exp)