Skip to content

Commit

Permalink
Add missing Mist metrics labels (#1380)
Browse files Browse the repository at this point in the history
* Add missing Mist labels

We used to add them with VM agent, but when catalyst-api is run standalone, then they didn't exist

* Fix lint
  • Loading branch information
leszko authored Sep 5, 2024
1 parent 5b29321 commit 0b50934
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 16 deletions.
48 changes: 37 additions & 11 deletions mapic/mistmetrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,7 @@ func (mc *mac) enrichMistMetrics(metrics string) string {
res := strings.Builder{}
lines := strings.Split(metrics, "\n")
for i, line := range lines {
playbackID, ok := mc.parsePlaybackID(line)
if ok {
// Enrich labels for the lines that contains playbackID
oldStr := mc.streamLabel(playbackID)
newStr := mc.enrichLabels(playbackID)
res.WriteString(strings.Replace(line, oldStr, newStr, -1))
} else {
// Do not enrich labels for lines that do not contain playbackID
res.WriteString(line)
}
res.WriteString(mc.enrichLine(line))

// Skip last end of line to preserve the same number of lines as Mist
if i < len(lines)-1 {
Expand All @@ -76,6 +67,42 @@ func (mc *mac) enrichMistMetrics(metrics string) string {
return res.String()
}

func (mc *mac) enrichLine(line string) string {
res := mc.enrichPlaybackSpecificLabels(line)
return mc.enrichConstLabels(res)
}

func (mc *mac) enrichPlaybackSpecificLabels(line string) string {
playbackID, ok := mc.parsePlaybackID(line)
if ok {
// Enrich labels for the lines that contains playbackID
oldStr := mc.streamLabel(playbackID)
newStr := mc.enrichLabels(playbackID)
return strings.Replace(line, oldStr, newStr, 1)
}
return line
}

func (mc *mac) enrichConstLabels(line string) string {
constLabels := fmt.Sprintf(`catalyst="true",catalyst_node="%s"`, mc.nodeID)
if len(line) == 0 || strings.HasPrefix(line, "#") {
// empty lines or comments
return line
}
if strings.Contains(line, "}") {
// metrics with labels
return strings.Replace(line, "}", fmt.Sprintf(",%s}", constLabels), 1)
}
// metrics without labels
lineSplit := strings.Split(line, " ")
if len(lineSplit) < 2 {
// invalid metric, do not enrich
return line
}
metricName := lineSplit[0]
return strings.Replace(line, metricName, fmt.Sprintf("%s{%s}", metricName, constLabels), 1)
}

func (mc *mac) parsePlaybackID(line string) (string, bool) {
match := mc.streamMetricsRe.FindStringSubmatch(line)
if len(match) > 1 {
Expand All @@ -86,7 +113,6 @@ func (mc *mac) parsePlaybackID(line string) (string, bool) {

func (mc *mac) enrichLabels(playbackID string) string {
res := mc.streamLabel(playbackID)
res += `,catalyst="true"`
si, err := mc.getStreamInfo(playbackID)
if err != nil {
glog.Warning("could not enrich Mist metrics for stream=%s err=%v", playbackID, err)
Expand Down
15 changes: 10 additions & 5 deletions mapic/mistmetrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,15 @@ mist_sessions{stream="video+077bh6xx5bx5tdua",sessType="viewers"}1
mist_latency{stream="video+077bh6xx5bx5tdua",source="sin-prod-catalyst-3.lp-playback.studio"}1795
mist_sessions{stream="video+51b13mqy7sgw520w",sessType="viewers"}5
mist_latency{stream="video+51b13mqy7sgw520w",source="prg-prod-catalyst-0.lp-playback.studio"}1156
mist_sessions_count{sessType="viewers"}93
mist_logs 14860632
`

func TestEnrichMistMetrics(t *testing.T) {
// given
mc := mac{
baseStreamName: "video",
nodeID: "fra-staging-staging-catalyst-0.livepeer.monster",
streamInfo: map[string]*streamInfo{
"077bh6xx5bx5tdua": {stream: &api.Stream{UserID: "abcdefgh-123456789"}},
"51b13mqy7sgw520w": {stream: &api.Stream{UserID: "hgfedcba-987654321"}},
Expand All @@ -41,11 +44,13 @@ func TestEnrichMistMetrics(t *testing.T) {
require.Equal(t, len(inLines), len(resLines))

expLines := []string{
`version{app="MistServer",version="729ddd4b42980d0124c72a46f13d8e0697293e94",release="Generic_x86_64"} 1`,
`mist_sessions{stream="video+077bh6xx5bx5tdua",catalyst="true",user_id="abcdefgh-123456789",sessType="viewers"}1`,
`mist_latency{stream="video+077bh6xx5bx5tdua",catalyst="true",user_id="abcdefgh-123456789",source="sin-prod-catalyst-3.lp-playback.studio"}1795`,
`mist_sessions{stream="video+51b13mqy7sgw520w",catalyst="true",user_id="hgfedcba-987654321",sessType="viewers"}5`,
`mist_latency{stream="video+51b13mqy7sgw520w",catalyst="true",user_id="hgfedcba-987654321",source="prg-prod-catalyst-0.lp-playback.studio"}1156`,
`version{app="MistServer",version="729ddd4b42980d0124c72a46f13d8e0697293e94",release="Generic_x86_64",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"} 1`,
`mist_sessions{stream="video+077bh6xx5bx5tdua",user_id="abcdefgh-123456789",sessType="viewers",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}1`,
`mist_latency{stream="video+077bh6xx5bx5tdua",user_id="abcdefgh-123456789",source="sin-prod-catalyst-3.lp-playback.studio",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}1795`,
`mist_sessions{stream="video+51b13mqy7sgw520w",user_id="hgfedcba-987654321",sessType="viewers",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}5`,
`mist_latency{stream="video+51b13mqy7sgw520w",user_id="hgfedcba-987654321",source="prg-prod-catalyst-0.lp-playback.studio",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}1156`,
`mist_sessions_count{sessType="viewers",catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"}93`,
`mist_logs{catalyst="true",catalyst_node="fra-staging-staging-catalyst-0.livepeer.monster"} 14860632`,
}
for _, exp := range expLines {
require.Contains(t, resLines, exp)
Expand Down

0 comments on commit 0b50934

Please sign in to comment.