Skip to content

Commit

Permalink
textparse: Refactored main testing utils for reusability; fixed proto…
Browse files Browse the repository at this point in the history 
… Units. (prometheus#15095)

Signed-off-by: bwplotka <[email protected]>
  • Loading branch information
@bwplotka
bwplotka authored Oct 7, 2024
1 parent 2b4ca98 commit f6e110d
Show file tree
Hide file tree
Showing 6 changed files with 382 additions and 477 deletions.
2 changes: 2 additions & 0 deletions model/textparse/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ type Parser interface {
// CreatedTimestamp returns the created timestamp (in milliseconds) for the
// current sample. It returns nil if it is unknown e.g. if it wasn't set,
// if the scrape protocol or metric type does not support created timestamps.
// Assume the CreatedTimestamp returned pointer is only valid until
// the Next iteration.
CreatedTimestamp() *int64

// Next advances the parser to the next sample.
Expand Down
97 changes: 97 additions & 0 deletions model/textparse/interface_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,18 @@
package textparse

import (
"errors"
"io"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"

"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/util/testutil"
)

func TestNewParser(t *testing.T) {
Expand Down Expand Up @@ -103,3 +110,93 @@ func TestNewParser(t *testing.T) {
})
}
}

// parsedEntry represents data that is parsed for each entry.
type parsedEntry struct {
// In all but EntryComment, EntryInvalid.
m string

// In EntryHistogram.
shs *histogram.Histogram
fhs *histogram.FloatHistogram

// In EntrySeries.
v float64

// In EntrySeries and EntryHistogram.
lset labels.Labels
t *int64
es []exemplar.Exemplar
ct *int64

// In EntryType.
typ model.MetricType
// In EntryHelp.
help string
// In EntryUnit.
unit string
// In EntryComment.
comment string
}

func requireEntries(t *testing.T, exp, got []parsedEntry) {
t.Helper()

testutil.RequireEqualWithOptions(t, exp, got, []cmp.Option{
cmp.AllowUnexported(parsedEntry{}),
})
}

func testParse(t *testing.T, p Parser) (ret []parsedEntry) {
t.Helper()

for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)

var got parsedEntry
var m []byte
switch et {
case EntryInvalid:
t.Fatal("entry invalid not expected")
case EntrySeries, EntryHistogram:
if et == EntrySeries {
m, got.t, got.v = p.Series()
got.m = string(m)
} else {
m, got.t, got.shs, got.fhs = p.Histogram()
got.m = string(m)
}

p.Metric(&got.lset)
for e := (exemplar.Exemplar{}); p.Exemplar(&e); {
got.es = append(got.es, e)
}
// Parser reuses int pointer.
if ct := p.CreatedTimestamp(); ct != nil {
got.ct = int64p(*ct)
}
case EntryType:
m, got.typ = p.Type()
got.m = string(m)

case EntryHelp:
m, h := p.Help()
got.m = string(m)
got.help = string(h)

case EntryUnit:
m, u := p.Unit()
got.m = string(m)
got.unit = string(u)

case EntryComment:
got.comment = string(p.Comment())
}
ret = append(ret, got)
}
return ret
}
124 changes: 55 additions & 69 deletions model/textparse/openmetricsparse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package textparse

import (
"errors"
"io"
"testing"

Expand Down Expand Up @@ -115,7 +114,7 @@ foobar{quantile="0.99"} 150.1`
input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
input += "\n# EOF\n"

exp := []expectedParse{
exp := []parsedEntry{
{
m: "go_gc_duration_seconds",
help: "A summary of the GC invocation durations.",
Expand Down Expand Up @@ -190,38 +189,50 @@ foobar{quantile="0.99"} 150.1`
m: `hhh_bucket{le="+Inf"}`,
v: 1,
lset: labels.FromStrings("__name__", "hhh_bucket", "le", "+Inf"),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "histogram-bucket-test"), Value: 4},
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "histogram-bucket-test"), Value: 4},
},
}, {
m: `hhh_count`,
v: 1,
lset: labels.FromStrings("__name__", "hhh_count"),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "histogram-count-test"), Value: 4},
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "histogram-count-test"), Value: 4},
},
}, {
m: "ggh",
typ: model.MetricTypeGaugeHistogram,
}, {
m: `ggh_bucket{le="+Inf"}`,
v: 1,
lset: labels.FromStrings("__name__", "ggh_bucket", "le", "+Inf"),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "gaugehistogram-bucket-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123},
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "gaugehistogram-bucket-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123},
},
}, {
m: `ggh_count`,
v: 1,
lset: labels.FromStrings("__name__", "ggh_count"),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "gaugehistogram-count-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123},
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "gaugehistogram-count-test", "xx", "yy"), Value: 4, HasTs: true, Ts: 123123},
},
}, {
m: "smr_seconds",
typ: model.MetricTypeSummary,
}, {
m: `smr_seconds_count`,
v: 2,
lset: labels.FromStrings("__name__", "smr_seconds_count"),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "summary-count-test"), Value: 1, HasTs: true, Ts: 123321},
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "summary-count-test"), Value: 1, HasTs: true, Ts: 123321},
},
}, {
m: `smr_seconds_sum`,
v: 42,
lset: labels.FromStrings("__name__", "smr_seconds_sum"),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "summary-sum-test"), Value: 1, HasTs: true, Ts: 123321},
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "summary-sum-test"), Value: 1, HasTs: true, Ts: 123321},
},
}, {
m: "ii",
typ: model.MetricTypeInfo,
Expand Down Expand Up @@ -270,15 +281,19 @@ foobar{quantile="0.99"} 150.1`
v: 17,
lset: labels.FromStrings("__name__", "foo_total"),
t: int64p(1520879607789),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
ct: int64p(1520872607123),
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
},
ct: int64p(1520872607123),
}, {
m: `foo_total{a="b"}`,
v: 17.0,
lset: labels.FromStrings("__name__", "foo_total", "a", "b"),
t: int64p(1520879607789),
e: &exemplar.Exemplar{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
ct: int64p(1520872607123),
es: []exemplar.Exemplar{
{Labels: labels.FromStrings("id", "counter-test"), Value: 5},
},
ct: int64p(1520872607123),
}, {
m: "bar",
help: "Summary with CT at the end, making sure we find CT even if it's multiple lines a far",
Expand Down Expand Up @@ -430,7 +445,8 @@ foobar{quantile="0.99"} 150.1`
}

p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
checkParseResultsWithCT(t, p, exp, true)
got := testParse(t, p)
requireEntries(t, exp, got)
}

func TestUTF8OpenMetricsParse(t *testing.T) {
Expand All @@ -455,7 +471,7 @@ func TestUTF8OpenMetricsParse(t *testing.T) {

input += "\n# EOF\n"

exp := []expectedParse{
exp := []parsedEntry{
{
m: "go.gc_duration_seconds",
help: "A summary of the GC invocation durations.",
Expand Down Expand Up @@ -504,7 +520,8 @@ choices}`, "strange©™\n'quoted' \"name\"", "6"),
}

p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
checkParseResultsWithCT(t, p, exp, true)
got := testParse(t, p)
requireEntries(t, exp, got)
}

func TestOpenMetricsParseErrors(t *testing.T) {
Expand Down Expand Up @@ -878,8 +895,8 @@ func TestOMNullByteHandling(t *testing.T) {
}
}

// While not desirable, there are cases were CT fails to parse and
// these tests show them.
// TestCTParseFailures tests known failure edge cases, we know does not work due
// current OM spec limitations or clients with broken OM format.
// TODO(maniktherana): Make sure OM 1.1/2.0 pass CT via metadata or exemplar-like to avoid this.
func TestCTParseFailures(t *testing.T) {
input := `# HELP thing Histogram with _created as first line
Expand All @@ -892,68 +909,37 @@ thing_bucket{le="+Inf"} 17`

input += "\n# EOF\n"

int64p := func(x int64) *int64 { return &x }

type expectCT struct {
m string
ct *int64
typ model.MetricType
help string
isErr bool
}

exp := []expectCT{
exp := []parsedEntry{
{
m: "thing",
help: "Histogram with _created as first line",
isErr: false,
m: "thing",
help: "Histogram with _created as first line",
}, {
m: "thing",
typ: model.MetricTypeHistogram,
isErr: false,
m: "thing",
typ: model.MetricTypeHistogram,
}, {
m: `thing_count`,
ct: int64p(1520872607123),
isErr: true,
m: `thing_count`,
ct: nil, // Should be int64p(1520872607123).
}, {
m: `thing_sum`,
ct: int64p(1520872607123),
isErr: true,
m: `thing_sum`,
ct: nil, // Should be int64p(1520872607123).
}, {
m: `thing_bucket{le="0.0"}`,
ct: int64p(1520872607123),
isErr: true,
m: `thing_bucket{le="0.0"}`,
ct: nil, // Should be int64p(1520872607123).
}, {
m: `thing_bucket{le="+Inf"}`,
ct: int64p(1520872607123),
isErr: true,
m: `thing_bucket{le="+Inf"}`,
ct: nil, // Should be int64p(1520872607123),
},
}

p := NewOpenMetricsParser([]byte(input), labels.NewSymbolTable(), WithOMParserCTSeriesSkipped())
i := 0

var res labels.Labels
for {
et, err := p.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)

switch et {
case EntrySeries:
p.Metric(&res)
got := testParse(t, p)
resetValAndLset(got) // Keep this test focused on metric, basic entries and CT only.
requireEntries(t, exp, got)
}

if ct := p.CreatedTimestamp(); exp[i].isErr {
require.Nil(t, ct)
} else {
require.Equal(t, *exp[i].ct, *ct)
}
default:
i++
continue
}
i++
func resetValAndLset(e []parsedEntry) {
for i := range e {
e[i].v = 0
e[i].lset = labels.EmptyLabels()
}
}
Loading

0 comments on commit f6e110d

Please sign in to comment.