From 38f05dbf0c0c6fb3047f8caf954f227ece68f6a6 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 24 Jun 2024 16:18:11 +0200 Subject: [PATCH] all: review fixes --- pkg/covermerger/covermerger.go | 64 ++++-------- pkg/covermerger/covermerger_test.go | 1 - pkg/covermerger/deleted_file_merger.go | 2 +- pkg/covermerger/file_line_merger.go | 22 +--- pkg/covermerger/lines_matcher.go | 4 - pkg/covermerger/repos.go | 15 +-- pkg/vcs/fuchsia.go | 9 -- pkg/vcs/git.go | 21 ---- pkg/vcs/vcs.go | 6 -- tools/syz-bq.sh | 19 ++-- tools/syz-covermerger/db.go | 45 +++----- tools/syz-covermerger/syz_covermerger.go | 124 ++++++++++++----------- 12 files changed, 117 insertions(+), 215 deletions(-) diff --git a/pkg/covermerger/covermerger.go b/pkg/covermerger/covermerger.go index e27545d4fa67..be5bec750fcc 100644 --- a/pkg/covermerger/covermerger.go +++ b/pkg/covermerger/covermerger.go @@ -20,7 +20,6 @@ const ( KeyFilePath = "file_path" KeyStartLine = "sl" KeyHitCount = "hit_count" - KeyArch = "arch" ) type FileRecord map[string]string @@ -46,27 +45,23 @@ type Frame struct { EndCol int } -func (fr FileRecord) Frame() Frame { - f := Frame{} +func (fr FileRecord) Frame() (*Frame, error) { + f := &Frame{} var err error if f.StartLine, err = strconv.Atoi(fr[KeyStartLine]); err != nil { - panic(fmt.Sprintf("failed to Atoi(%s)", fr[KeyStartLine])) + return nil, fmt.Errorf("failed to Atoi(%s): %w", fr[KeyStartLine], err) } - return f + return f, nil } -func (fr FileRecord) HitCount() int { +func (fr FileRecord) HitCount() (int, error) { if hitCount, err := strconv.Atoi(fr[KeyHitCount]); err != nil { - panic(fmt.Sprintf("failed to Atoi(%s)", fr[KeyHitCount])) + return 0, fmt.Errorf("failed to Atoi(%s): %w", fr[KeyHitCount], err) } else { - return hitCount + return hitCount, nil } } -func (fr FileRecord) Arch() string { - return fr[KeyArch] -} - type MergeResult struct { HitCounts map[int]int FileExists bool @@ -74,7 +69,7 @@ type MergeResult struct { } type FileCoverageMerger interface { - AddRecord(rbc RepoBranchCommit, arch string, f Frame, hitCount int) + AddRecord(rbc RepoBranchCommit, f *Frame, hitCount int) Result() *MergeResult } @@ -85,9 +80,7 @@ func batchFileData(c *Config, targetFilePath string, records FileRecords, for _, record := range records { repoBranchCommitsMap[record.RepoBranchCommit()] = true } - if c.BaseType == BaseManual { - repoBranchCommitsMap[c.Base] = true - } + repoBranchCommitsMap[c.Base] = true repoBranchCommits := maps.Keys(repoBranchCommitsMap) getFiles := getFileVersions if c.getFileVersionsMock != nil { @@ -97,35 +90,24 @@ func batchFileData(c *Config, targetFilePath string, records FileRecords, if err != nil { return nil, fmt.Errorf("failed to getFileVersions: %w", err) } - base := getBaseRBC(c, targetFilePath, fvs) - merger := makeFileLineCoverMerger(fvs, base) + merger := makeFileLineCoverMerger(fvs, c.Base) for _, record := range records { + var f *Frame + if f, err = record.Frame(); err != nil { + return nil, fmt.Errorf("error parsing %s records: %w", targetFilePath, err) + } + var hitCount int + if hitCount, err = record.HitCount(); err != nil { + return nil, fmt.Errorf("error parsing %s records: %w", targetFilePath, err) + } merger.AddRecord( record.RepoBranchCommit(), - record.Arch(), - record.Frame(), - record.HitCount()) + f, + hitCount) } return merger.Result(), nil } -// getBaseRBC is a base(target) file version selector. -// The easiest strategy is to use some specified commit. -// For the namespace level signals merging we'll select target dynamically. -func getBaseRBC(c *Config, targetFilePath string, fvs fileVersions) RepoBranchCommit { - switch c.BaseType { - case BaseManual: - return c.Base - case BaseLastUpdated: - // If repo is not specifies use the much more expensive approach. - // The base commit is the commit where non-empty target file was last modified. - if res := freshestRBC(fvs); res != nil { - return *res - } - } - panic(fmt.Sprintf("failed searching best RBC for file %s", targetFilePath)) -} - func makeRecord(fields, schema []string) FileRecord { record := make(FileRecord) if len(fields) != len(schema) { @@ -138,15 +120,9 @@ func makeRecord(fields, schema []string) FileRecord { return record } -const ( - BaseManual = iota - BaseLastUpdated -) - type Config struct { Workdir string skipRepoClone bool - BaseType int // BaseManual, BaseLastUpdated. Base RepoBranchCommit // used by BaseManual getFileVersionsMock func(*Config, string, []RepoBranchCommit) (fileVersions, error) } diff --git a/pkg/covermerger/covermerger_test.go b/pkg/covermerger/covermerger_test.go index 9b99d6e3a587..5d99e35df984 100644 --- a/pkg/covermerger/covermerger_test.go +++ b/pkg/covermerger/covermerger_test.go @@ -123,7 +123,6 @@ samp_time,1,360,arch,b1,ci-mock,git://repo,master,commit2,not_changed.c,func1,4, &Config{ Workdir: test.workdir, skipRepoClone: true, - BaseType: BaseManual, Base: RepoBranchCommit{ Repo: test.baseRepo, Branch: test.baseBranch, diff --git a/pkg/covermerger/deleted_file_merger.go b/pkg/covermerger/deleted_file_merger.go index 7313c8dae936..32513ecb1d66 100644 --- a/pkg/covermerger/deleted_file_merger.go +++ b/pkg/covermerger/deleted_file_merger.go @@ -6,7 +6,7 @@ package covermerger type DeletedFileLineMerger struct { } -func (a *DeletedFileLineMerger) AddRecord(RepoBranchCommit, string, Frame, int) { +func (a *DeletedFileLineMerger) AddRecord(RepoBranchCommit, *Frame, int) { } func (a *DeletedFileLineMerger) Result() *MergeResult { diff --git a/pkg/covermerger/file_line_merger.go b/pkg/covermerger/file_line_merger.go index 1ac0e617246f..29fd4200b410 100644 --- a/pkg/covermerger/file_line_merger.go +++ b/pkg/covermerger/file_line_merger.go @@ -3,10 +3,6 @@ package covermerger -import ( - "time" -) - func makeFileLineCoverMerger( fvs fileVersions, base RepoBranchCommit) FileCoverageMerger { baseFile := "" @@ -34,22 +30,6 @@ func makeFileLineCoverMerger( return a } -// freshestRBC returns RepoBranchCommit with the last modified non-empty file. -func freshestRBC(fvs fileVersions) *RepoBranchCommit { - var res *RepoBranchCommit - var resLastUpdated time.Time - for rbc, fv := range fvs { - if fv.content == "" { - continue - } - if res == nil || resLastUpdated.Before(fv.lastUpdated) { - res = &rbc - resLastUpdated = fv.lastUpdated - } - } - return res -} - type FileLineCoverMerger struct { rbcToFile fileVersions baseFile string @@ -58,7 +38,7 @@ type FileLineCoverMerger struct { lostFrames map[RepoBranchCommit]int64 } -func (a *FileLineCoverMerger) AddRecord(rbc RepoBranchCommit, arch string, f Frame, hitCount int) { +func (a *FileLineCoverMerger) AddRecord(rbc RepoBranchCommit, f *Frame, hitCount int) { if a.matchers[rbc] == nil { if hitCount > 0 { a.lostFrames[rbc]++ diff --git a/pkg/covermerger/lines_matcher.go b/pkg/covermerger/lines_matcher.go index ecfaf9425600..b54c049f05fe 100644 --- a/pkg/covermerger/lines_matcher.go +++ b/pkg/covermerger/lines_matcher.go @@ -4,7 +4,6 @@ package covermerger import ( - "log" "strings" dmp "github.com/sergi/go-diff/diffmatchpatch" @@ -44,8 +43,5 @@ type LineToLineMatcher struct { } func (lm *LineToLineMatcher) SameLinePos(line int) int { - if line < 0 || line > len(lm.lineToLine) { - log.Printf("wrong line number %d", line) - } return lm.lineToLine[line] } diff --git a/pkg/covermerger/repos.go b/pkg/covermerger/repos.go index 96d3f36857a9..8b1d191b67ee 100644 --- a/pkg/covermerger/repos.go +++ b/pkg/covermerger/repos.go @@ -14,8 +14,7 @@ import ( ) type fileVersion struct { - content string - lastUpdated time.Time + content string } type fileVersions map[RepoBranchCommit]fileVersion @@ -29,21 +28,13 @@ func getFileVersions(c *Config, targetFilePath string, rbcs []RepoBranchCommit, res := make(fileVersions) for _, rbc := range rbcs { - fileBytes, err := repos[rbc].FileVersion(targetFilePath, rbc.Commit) + fileBytes, err := repos[rbc].Object(targetFilePath, rbc.Commit) // It is ok if some file doesn't exist. It means we have repo FS diff. if err != nil { continue } - var lastUpdated time.Time - if c.BaseType == BaseLastUpdated { - if lastUpdated, err = repos[rbc].FileEditTime(targetFilePath); err != nil { - return nil, fmt.Errorf("failed to get file %s modification date: %w", - targetFilePath, err) - } - } res[rbc] = fileVersion{ - content: string(fileBytes), - lastUpdated: lastUpdated, + content: string(fileBytes), } } return res, nil diff --git a/pkg/vcs/fuchsia.go b/pkg/vcs/fuchsia.go index 0de639de0c31..223c0b8604c2 100644 --- a/pkg/vcs/fuchsia.go +++ b/pkg/vcs/fuchsia.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/google/syzkaller/pkg/osutil" ) @@ -104,11 +103,3 @@ func (ctx *fuchsia) Object(name, commit string) ([]byte, error) { func (ctx *fuchsia) MergeBases(firstCommit, secondCommit string) ([]*Commit, error) { return ctx.repo.MergeBases(firstCommit, secondCommit) } - -func (ctx *fuchsia) FileEditTime(path string) (time.Time, error) { - return time.Time{}, fmt.Errorf("not implemented for fuchsia") -} - -func (ctx *fuchsia) FileVersion(path, commit string) ([]byte, error) { - return nil, fmt.Errorf("not implemented for fuchsia") -} diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go index 160009cd7a3e..840a8592e7d3 100644 --- a/pkg/vcs/git.go +++ b/pkg/vcs/git.go @@ -619,24 +619,3 @@ func (git *git) MergeBases(firstCommit, secondCommit string) ([]*Commit, error) } return ret, nil } - -func (git *git) FileEditTime(path string) (time.Time, error) { - output, err := git.git("log", "-1", "--pretty=format:%cI", path) - if err != nil { - return time.Time{}, fmt.Errorf("failed to get time for %s: %w", path, err) - } - t, err := time.Parse(time.RFC3339, string(output)) - if err != nil { - return time.Time{}, fmt.Errorf("failed to parse datatime %s: %w", output, err) - } - return t, nil -} - -func (git *git) FileVersion(path, commit string) ([]byte, error) { - target := fmt.Sprintf("%s:%s", commit, path) - output, err := git.git("show", target) - if err != nil { - return nil, fmt.Errorf("failed to get %s: %w", target, err) - } - return output, nil -} diff --git a/pkg/vcs/vcs.go b/pkg/vcs/vcs.go index 05dabe4d4a56..faf1cf85dd90 100644 --- a/pkg/vcs/vcs.go +++ b/pkg/vcs/vcs.go @@ -69,12 +69,6 @@ type Repo interface { // MergeBases returns good common ancestors of the two commits. MergeBases(firstCommit, secondCommit string) ([]*Commit, error) - - // FileEditTime returns the last file modification date. - FileEditTime(path string) (time.Time, error) - - // FileVersion "git show" file. It is apx. 3 times slower than cat. - FileVersion(path, commit string) ([]byte, error) } // Bisecter may be optionally implemented by Repo. diff --git a/tools/syz-bq.sh b/tools/syz-bq.sh index 88a466b29035..9e062e075ed4 100755 --- a/tools/syz-bq.sh +++ b/tools/syz-bq.sh @@ -5,12 +5,12 @@ set -e # exit on any problem set -o pipefail -while getopts w:f:t:n:r: option +while getopts w:d:t:n:r: option do case "${option}" in w)workdir=${OPTARG};; - f)from_date=${OPTARG};; + d)duration=${OPTARG};; t)to_date=${OPTARG};; n)namespace=${OPTARG};; r)repo=${OPTARG};; @@ -27,9 +27,9 @@ then echo "-t is required to specify to_date" exit fi -if [ -z "$from_date" ] +if [ -z "$duration" ] then - echo "-f is required to specify from_date" + echo "-d is required to specify duration" exit fi if [ -z "$namespace" ] @@ -52,12 +52,12 @@ CREATE TABLE IF NOT EXISTS "repo" text, "commit" text, "filepath" text, - "datefrom" date, + "duration" bigint, "dateto" date, "instrumented" bigint, "covered" bigint, PRIMARY KEY - (datefrom, dateto, commit, filepath) );') + (duration, dateto, commit, filepath) );') gcloud spanner databases ddl update coverage --instance=syzbot --project=syzkaller \ --ddl="$create_table" @@ -82,6 +82,7 @@ echo The latest commit as of $to_date is $base_commit. # rm -rf $base_dir # echo Temp dir $base_dir deleted. +from_date=$(date -d "$to_date - $duration days" +%Y-%m-%d) sessionID=$(uuidgen) gsURI=$(echo gs://syzbot-temp/bq-exports/${sessionID}/*.csv.gz) echo fetching data from bigquery @@ -98,8 +99,8 @@ AS ( kernel_repo, kernel_branch, kernel_commit, file_path, sl, SUM(hit_count) as hit_count FROM syzkaller.syzbot_coverage.'$namespace' WHERE - TIMESTAMP_TRUNC(timestamp, DAY) >= TIMESTAMP("'$from_date'") AND - TIMESTAMP_TRUNC(timestamp, DAY) <= TIMESTAMP("'$to_date'") AND + TIMESTAMP_TRUNC(timestamp, DAY) >= "'$from_date'" AND + TIMESTAMP_TRUNC(timestamp, DAY) <= "'$to_date'" AND version = 1 GROUP BY file_path, kernel_commit, kernel_repo, kernel_branch, sl ORDER BY file_path @@ -117,7 +118,7 @@ go run ./tools/syz-covermerger/ -workdir $workdir \ -commit $base_commit \ -save-to-spanner true \ -namespace $namespace \ - -date-from $from_date \ + -duration $duration \ -date-to $to_date echo Cleanup diff --git a/tools/syz-covermerger/db.go b/tools/syz-covermerger/db.go index 0d3a790e9790..996e7a6e065f 100644 --- a/tools/syz-covermerger/db.go +++ b/tools/syz-covermerger/db.go @@ -9,54 +9,41 @@ import ( "cloud.google.com/go/civil" "cloud.google.com/go/spanner" - "github.com/google/syzkaller/pkg/covermerger" ) // TODO: move to dashAPI once tested? I'm not sure we'll benefit. -type Coverage struct { +type DBRecord struct { Namespace string - FilePath string Repo string Commit string - DateFrom civil.Date + Duration int64 DateTo civil.Date + FilePath string Instrumented int64 Covered int64 } -func saveToSpanner(c context.Context, mergedCoverage map[string]*covermerger.MergeResult, - repo, commit, ns string, dateFrom, dateTo civil.Date) { - ctx := context.Background() - client, err := spanner.NewClient(ctx, "projects/syzkaller/instances/syzbot/databases/coverage") +func saveToSpanner(ctx context.Context, projectID string, coverage map[string]*Coverage, + template *DBRecord) { + client, err := spanner.NewClient(ctx, "projects/"+projectID+"/instances/syzbot/databases/coverage") if err != nil { panic(fmt.Sprintf("spanner.NewClient() failed: %s", err.Error())) } defer client.Close() mutations := []*spanner.Mutation{} - for fileName, fileStat := range mergedCoverage { - var instrumentedLines int64 - var coveredLines int64 - if !fileStat.FileExists { - continue - } - for _, lineHitCount := range fileStat.HitCounts { - instrumentedLines++ - if lineHitCount > 0 { - coveredLines++ - } - } + for filePath, record := range coverage { var insert *spanner.Mutation - if insert, err = spanner.InsertStruct("files", Coverage{ - Namespace: ns, - Repo: repo, - Commit: commit, - FilePath: fileName, - DateFrom: dateFrom, - DateTo: dateTo, - Instrumented: instrumentedLines, - Covered: coveredLines, + if insert, err = spanner.InsertOrUpdateStruct("files", &DBRecord{ + Namespace: template.Namespace, + Repo: template.Repo, + Commit: template.Commit, + Duration: template.Duration, + DateTo: template.DateTo, + FilePath: filePath, + Instrumented: record.Instrumented, + Covered: record.Covered, }); err != nil { panic(fmt.Sprintf("failed to spanner.InsertStruct(): %s", err.Error())) } diff --git a/tools/syz-covermerger/syz_covermerger.go b/tools/syz-covermerger/syz_covermerger.go index 2b33d0a3f848..0feede97f1a7 100644 --- a/tools/syz-covermerger/syz_covermerger.go +++ b/tools/syz-covermerger/syz_covermerger.go @@ -23,7 +23,7 @@ func baseTypeFromString(name string) (int, error) { case "lastupdated": return covermerger.BaseLastUpdated, nil default: - return -1, fmt.Errorf("unexpected baseType(manual|lasupdated): %s", name) + return -1, fmt.Errorf("unexpected baseType(manual|lastupdated): %s", name) } } @@ -50,21 +50,22 @@ func BaseIsWellSpecifiedOrExit(flagBaseType, flagRepo, flagBranch, flagCommit *s } } -func main() { - flagWorkdir := flag.String("workdir", "workdir-cover-aggregation", +var ( + flagWorkdir = flag.String("workdir", "workdir-cover-aggregation", "[optional] used to clone repos") - flagCleanWorkdir := flag.Bool("clean-workdir", false, - "[optional] cleans workdir before start") - flagBaseType := flag.String("base-type", "manual", + flagBaseType = flag.String("base-type", "manual", "commit to be used as a base. Can be manual/lastupdated and works per fileF") - flagRepo := flag.String("repo", "", "[required] repo to be used as an aggregation point") - flagBranch := flag.String("branch", "", "[required] branch to be used as an aggregation point") - flagCommit := flag.String("commit", "", "[required] commit hash to be used as an aggregation point") - flagNamespace := flag.String("namespace", "upstream", "[optional] target namespace") - flagDateFrom := flag.String("date-from", "", "[optional] used to mark DB records") - flagDateTo := flag.String("date-to", "", "[optional] used to mark DB records") - flagSaveToSpanner := flag.String("save-to-spanner", "", "[optional] save aggregation to spanner") + flagRepo = flag.String("repo", "", "[required] repo to be used as an aggregation point") + flagBranch = flag.String("branch", "", "[required] branch to be used as an aggregation point") + flagCommit = flag.String("commit", "", "[required] commit hash to be used as an aggregation point") + flagNamespace = flag.String("namespace", "upstream", "[optional] target namespace") + flagDuration = flag.Int64("duration", 0, "[optional] used to mark DB records") + flagDateTo = flag.String("date-to", "", "[optional] used to mark DB records") + flagSaveToSpanner = flag.String("save-to-spanner", "", "[optional] save aggregation to spanner") + flagProjectID = flag.String("project-id", "syzkaller", "[optional] target spanner db project") +) +func main() { flag.Parse() BaseIsWellSpecifiedOrExit(flagBaseType, flagRepo, flagBranch, flagCommit) @@ -78,69 +79,46 @@ func main() { Commit: *flagCommit, }, } - - if *flagCleanWorkdir { - if err := os.RemoveAll(*flagWorkdir); err != nil { - panic("failed to clean workdir " + *flagWorkdir) - } + mergeResult, err := covermerger.MergeCSVData(config, os.Stdin) + if err != nil { + panic(err) } - - mergeResult := commandProcessStdin(config, *flagRepo, *flagBranch, *flagCommit) printMergeResult(mergeResult) - if *flagSaveToSpanner != "" { log.Print("saving to spanner") - if *flagDateFrom == "" || *flagDateTo == "" { - panic("date-from and date-to are required to store to DB") - } - var err error - var dateFrom, dateTo civil.Date - if dateFrom, err = civil.ParseDate(*flagDateFrom); err != nil { - panic(fmt.Sprintf("failed to parse time_from: %s", err.Error())) + if *flagDuration == 0 || *flagDateTo == "" { + panic("duration and date-to are required to store to DB") } + var dateTo civil.Date if dateTo, err = civil.ParseDate(*flagDateTo); err != nil { panic(fmt.Sprintf("failed to parse time_to: %s", err.Error())) } - saveToSpanner(context.Background(), mergeResult, *flagRepo, *flagCommit, *flagNamespace, dateFrom, dateTo) + coverage, _, _ := mergeResultsToCoverage(mergeResult) + saveToSpanner(context.Background(), *flagProjectID, coverage, + &DBRecord{ + Namespace: *flagNamespace, + Repo: *flagRepo, + Commit: *flagCommit, + Duration: *flagDuration, + DateTo: dateTo, + }, + ) } } -func commandProcessStdin(config *covermerger.Config, flagRepo, flagBranch, flagCommit string, -) map[string]*covermerger.MergeResult { - mergedCoverage, err := covermerger.MergeCSVData(config, os.Stdin) - if err != nil { - panic(err) - } - return mergedCoverage -} - -func printMergeResult(mergedCoverage map[string]*covermerger.MergeResult) { +func printMergeResult(mergeResult map[string]*covermerger.MergeResult) { totalLostFrames := map[covermerger.RepoBranchCommit]int64{} - totalInstrumentedLines := 0 - totalCoveredLines := 0 - keys := maps.Keys(mergedCoverage) + coverage, totalInstrumentedLines, totalCoveredLines := mergeResultsToCoverage(mergeResult) + keys := maps.Keys(coverage) sort.Strings(keys) for _, fileName := range keys { - lineStat := mergedCoverage[fileName] - instrumentedLines := 0 - coveredLines := 0 - if !lineStat.FileExists { - continue - } - for _, lineHitCount := range lineStat.HitCounts { - instrumentedLines++ - if lineHitCount > 0 { - coveredLines++ - } - } + lineStat := mergeResult[fileName] for rbc, lostFrames := range lineStat.LostFrames { log.Printf("\t[warn] lost %d frames from rbc(%s, %s, %s)", lostFrames, rbc.Repo, rbc.Branch, rbc.Commit) totalLostFrames[rbc] += lostFrames } - printCoverage(fileName, instrumentedLines, coveredLines) - totalInstrumentedLines += instrumentedLines - totalCoveredLines += coveredLines + printCoverage(fileName, coverage[fileName].Instrumented, coverage[fileName].Covered) } printCoverage("total", totalInstrumentedLines, totalCoveredLines) for rbc, lostFrames := range totalLostFrames { @@ -150,7 +128,7 @@ func printMergeResult(mergedCoverage map[string]*covermerger.MergeResult) { } } -func printCoverage(target string, instrumented, covered int) { +func printCoverage(target string, instrumented, covered int64) { coverage := 0.0 if instrumented != 0 { coverage = float64(covered) / float64(instrumented) @@ -158,3 +136,33 @@ func printCoverage(target string, instrumented, covered int) { fmt.Printf("%s,%d,%d,%.2f%%\n", target, instrumented, covered, coverage*100) } + +type Coverage struct { + Instrumented int64 + Covered int64 +} + +func mergeResultsToCoverage(mergedCoverage map[string]*covermerger.MergeResult, +) (map[string]*Coverage, int64, int64) { + res := make(map[string]*Coverage) + var totalInstrumented, totalCovered int64 + for fileName, lineStat := range mergedCoverage { + if !lineStat.FileExists { + continue + } + var instrumented, covered int64 + for _, lineHitCount := range lineStat.HitCounts { + instrumented++ + if lineHitCount > 0 { + covered++ + } + } + res[fileName] = &Coverage{ + Instrumented: instrumented, + Covered: covered, + } + totalInstrumented += instrumented + totalCovered += covered + } + return res, totalInstrumented, totalCovered +}