From 647f67266f35bac85ca6ac506447330d1c71e7fb Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 27 May 2023 18:06:52 -0300 Subject: [PATCH 01/25] =?UTF-8?q?[desafio-07]=20Primeira=20implementa?= =?UTF-8?q?=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- desafio-07/MatMercer/go/tac.go | 70 ++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 desafio-07/MatMercer/go/tac.go diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go new file mode 100644 index 000000000..131e76d49 --- /dev/null +++ b/desafio-07/MatMercer/go/tac.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "io" + "os" +) + +const bufSize = int64(32) + +func check(e error) { + if e != nil { + errAndExit(fmt.Sprintf("tac error: %v", e)) + } +} + +func errAndExit(msg string) { + os.Stderr.WriteString(msg + "\n") + os.Exit(1) +} + +func main() { + args := os.Args[1:] + + if len(args) != 1 { + errAndExit("usage: tac [file]") + } + + fileName := args[0] + + f, err := os.Open(fileName) + check(err) + defer f.Close() + fi, err := f.Stat() + check(err) + + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + out := os.Stdout + b := make([]byte, bufSize) + end := fi.Size() + maxReadIdx := bufSize - 1 + start := end + for start != 0 { + start -= bufSize + if start < 0 { + // prevent over reading if result is less than buf size + maxReadIdx += start + start = 0 + } + + _, err = f.Seek(start, io.SeekStart) + check(err) + _, err = f.Read(b[:maxReadIdx]) + check(err) + + // search until backwards \n and prints it + lastEnd := maxReadIdx + for i := maxReadIdx; i >= 0; i-- { + if b[i] == '\n' { + _, err = out.Write(b[i+1 : lastEnd]) + check(err) + } + + if i == 0 { + _, err = out.Write(b[i:lastEnd]) + check(err) + } + } + } +} From b282aeb779555bf8fa7f959b535dbc8c2832136f Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 27 May 2023 18:35:33 -0300 Subject: [PATCH 02/25] [desafio-07] Arruma buffer pequeno dando erro --- desafio-07/MatMercer/go/tac.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 131e76d49..409357a11 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "io" "os" @@ -38,33 +39,50 @@ func main() { out := os.Stdout b := make([]byte, bufSize) end := fi.Size() - maxReadIdx := bufSize - 1 + maxRead := bufSize start := end + acc := bytes.NewBuffer([]byte{}) for start != 0 { start -= bufSize if start < 0 { // prevent over reading if result is less than buf size - maxReadIdx += start + maxRead += start start = 0 } _, err = f.Seek(start, io.SeekStart) check(err) - _, err = f.Read(b[:maxReadIdx]) + _, err = f.Read(b[:maxRead]) check(err) // search until backwards \n and prints it - lastEnd := maxReadIdx - for i := maxReadIdx; i >= 0; i-- { + lastEnd := maxRead + for i := maxRead - 1; i >= 0; i-- { if b[i] == '\n' { _, err = out.Write(b[i+1 : lastEnd]) check(err) + _, err = out.Write(acc.Bytes()) + check(err) + _, err = out.Write([]byte{'\n'}) + check(err) + acc.Reset() + lastEnd = i } if i == 0 { - _, err = out.Write(b[i:lastEnd]) + newAcc := bytes.NewBuffer([]byte{}) + _, err = newAcc.Write(b[i:lastEnd]) check(err) + _, err = newAcc.Write(acc.Bytes()) + check(err) + acc.Reset() + acc = newAcc } } + if start == 0 { + _, err = out.Write(acc.Bytes()) + check(err) + acc.Reset() + } } } From 3df68ab0d0da29ceb0842967627c3aa80bfa270a Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 27 May 2023 18:46:59 -0300 Subject: [PATCH 03/25] [desafio-07] Code cleanup --- desafio-07/MatMercer/go/tac.go | 39 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 409357a11..cadc841ed 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -35,13 +35,11 @@ func main() { fi, err := f.Stat() check(err) - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - out := os.Stdout b := make([]byte, bufSize) end := fi.Size() maxRead := bufSize start := end - acc := bytes.NewBuffer([]byte{}) + lineAcc := bytes.NewBuffer([]byte{}) for start != 0 { start -= bufSize if start < 0 { @@ -59,30 +57,33 @@ func main() { lastEnd := maxRead for i := maxRead - 1; i >= 0; i-- { if b[i] == '\n' { - _, err = out.Write(b[i+1 : lastEnd]) - check(err) - _, err = out.Write(acc.Bytes()) - check(err) - _, err = out.Write([]byte{'\n'}) - check(err) - acc.Reset() - lastEnd = i + // write everything but '\n', since b[i] == '\n' + out(b[i+1 : lastEnd]) + // need to write accumulated value + out(lineAcc.Bytes()) + lineAcc.Reset() + // +1 here makes '\n' be printed in next iteration + lastEnd = i + 1 } if i == 0 { newAcc := bytes.NewBuffer([]byte{}) _, err = newAcc.Write(b[i:lastEnd]) check(err) - _, err = newAcc.Write(acc.Bytes()) + _, err = newAcc.Write(lineAcc.Bytes()) check(err) - acc.Reset() - acc = newAcc + lineAcc.Reset() + lineAcc = newAcc } } - if start == 0 { - _, err = out.Write(acc.Bytes()) - check(err) - acc.Reset() - } } + // prints last chunk of data + out(lineAcc.Bytes()) +} + +// out outputs to os.Stdout checking for errors +func out(b []byte) { + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err := os.Stdout.Write(b) + check(err) } From 9a1058fdc06e30d99afa89e08851b0f2e0f930a1 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 27 May 2023 20:51:23 -0300 Subject: [PATCH 04/25] =?UTF-8?q?[desafio-07]=20Melhora=20performance=20co?= =?UTF-8?q?m=20arquivos=20pequenos=20e=20limpa=20c=C3=B3digo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- desafio-07/MatMercer/go/tac.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index cadc841ed..65eeda373 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -7,7 +7,12 @@ import ( "os" ) -const bufSize = int64(32) +func main() { + tac() +} + +// in MB size, currently the program uses a max of 2*bufSize iff file size > bufSize +const maxBufSize = int64(250 << (10 * 2)) func check(e error) { if e != nil { @@ -16,11 +21,11 @@ func check(e error) { } func errAndExit(msg string) { - os.Stderr.WriteString(msg + "\n") + _, _ = os.Stderr.WriteString(msg + "\n") os.Exit(1) } -func main() { +func tac() { args := os.Args[1:] if len(args) != 1 { @@ -35,10 +40,11 @@ func main() { fi, err := f.Stat() check(err) + fs := fi.Size() + bufSize := min(fs, maxBufSize) b := make([]byte, bufSize) - end := fi.Size() maxRead := bufSize - start := end + start := fs lineAcc := bytes.NewBuffer([]byte{}) for start != 0 { start -= bufSize @@ -67,9 +73,7 @@ func main() { } if i == 0 { - newAcc := bytes.NewBuffer([]byte{}) - _, err = newAcc.Write(b[i:lastEnd]) - check(err) + newAcc := bytes.NewBuffer(b[i:lastEnd]) _, err = newAcc.Write(lineAcc.Bytes()) check(err) lineAcc.Reset() @@ -87,3 +91,10 @@ func out(b []byte) { _, err := os.Stdout.Write(b) check(err) } + +func min(x int64, y int64) int64 { + if x > y { + return y + } + return x +} From d46824ccbd92b83f91111e138fd5e9347598dc49 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sun, 28 May 2023 15:41:05 -0300 Subject: [PATCH 05/25] [desafio-07] Add reverse reader --- desafio-07/MatMercer/go/tac.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 65eeda373..a1bb5533a 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -46,6 +46,7 @@ func tac() { maxRead := bufSize start := fs lineAcc := bytes.NewBuffer([]byte{}) + r := NewReverseReader(f, fs) for start != 0 { start -= bufSize if start < 0 { @@ -54,9 +55,7 @@ func tac() { start = 0 } - _, err = f.Seek(start, io.SeekStart) - check(err) - _, err = f.Read(b[:maxRead]) + _, err = r.Read(b[:maxRead]) check(err) // search until backwards \n and prints it @@ -98,3 +97,28 @@ func min(x int64, y int64) int64 { } return x } + +type ReverseReader struct { + f *os.File + offset int64 +} + +func NewReverseReader(f *os.File, size int64) *ReverseReader { + return &ReverseReader{ + f, + size, + } +} + +func (r *ReverseReader) Read(b []byte) (n int, err error) { + r.offset -= int64(len(b)) + _, err = r.f.Seek(r.offset, io.SeekStart) + if err != nil { + return 0, err + } + read, err := r.f.Read(b) + if err != nil { + return 0, err + } + return read, nil +} From bcb27f5e9d258c93f54b7d91c818b10348a3b273 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sun, 28 May 2023 16:30:20 -0300 Subject: [PATCH 06/25] [desafio-07] Corrige problema de ultimo output --- desafio-07/MatMercer/go/tac.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index a1bb5533a..564b87a93 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -72,7 +72,9 @@ func tac() { } if i == 0 { - newAcc := bytes.NewBuffer(b[i:lastEnd]) + newAcc := bytes.NewBuffer(nil) + _, err = newAcc.Write(b[i:lastEnd]) + check(err) _, err = newAcc.Write(lineAcc.Bytes()) check(err) lineAcc.Reset() From 8e1104a6e30c7ea418d36c52b154c9e5b2d5b978 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sun, 28 May 2023 16:37:14 -0300 Subject: [PATCH 07/25] [desafio-07] Melhora performance bufferizando stdout --- desafio-07/MatMercer/go/tac.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 564b87a93..5a8535633 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "fmt" "io" @@ -14,6 +15,8 @@ func main() { // in MB size, currently the program uses a max of 2*bufSize iff file size > bufSize const maxBufSize = int64(250 << (10 * 2)) +var stdout *bufio.Writer + func check(e error) { if e != nil { errAndExit(fmt.Sprintf("tac error: %v", e)) @@ -40,6 +43,10 @@ func tac() { fi, err := f.Stat() check(err) + // buffers stdout by 128kb + stdout = bufio.NewWriterSize(os.Stdout, 128<<(10)) + defer stdout.Flush() + fs := fi.Size() bufSize := min(fs, maxBufSize) b := make([]byte, bufSize) @@ -89,7 +96,7 @@ func tac() { // out outputs to os.Stdout checking for errors func out(b []byte) { // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - _, err := os.Stdout.Write(b) + _, err := stdout.Write(b) check(err) } From a58271cb9d67617a15f01063bdf59d2a55899233 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sun, 28 May 2023 16:45:03 -0300 Subject: [PATCH 08/25] [desafio-07] Tuning --- desafio-07/MatMercer/go/tac.go | 1 - 1 file changed, 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 5a8535633..bd152b2c8 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -84,7 +84,6 @@ func tac() { check(err) _, err = newAcc.Write(lineAcc.Bytes()) check(err) - lineAcc.Reset() lineAcc = newAcc } } From 84e111910230510aa3f43fca26e9c1a7c91d1f2b Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sun, 28 May 2023 16:49:08 -0300 Subject: [PATCH 09/25] [desafio-07] Readme --- desafio-07/MatMercer/go/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 desafio-07/MatMercer/go/README.md diff --git a/desafio-07/MatMercer/go/README.md b/desafio-07/MatMercer/go/README.md new file mode 100644 index 000000000..ae71a4f39 --- /dev/null +++ b/desafio-07/MatMercer/go/README.md @@ -0,0 +1,30 @@ +# Solução desafio 7 unix tac em Go + +Executar: + +```bash +go build tac.go +./tac 1GB.txt > out.txt +md5sum out.txt +2b4fd25f11d75c285ec69ecac420bd07 out.txt +``` + +Para testar os 512MB de memória com control groups v1: + +```bash +# setup cgroup +export CGROUP=osProgramadoresD7 +export CGROUPP="memory/$CGROUP" +sudo cgcreate -t $USER:$USER -a $USER:$USER -g memory:"$CGROUP" +echo 512M > "/sys/fs/cgroup/$CGROUPP/memory.limit_in_bytes" +echo 0 > "/sys/fs/cgroup/$CGROUPP/memory.swappiness" + +# run in cgroup +cgexec -g memory:osProgramadoresD7 ./tac 1GB.txt > out.txt +md5sum out.txt +2b4fd25f11d75c285ec69ecac420bd07 out.txt + +# check max mem used +cat "/sys/fs/cgroup/$CGROUPP/memory.max_usage_in_bytes" | numfmt --to=iec +512M +``` From afa6ca68bb891597b6cd0b9f9ee0388983c596c4 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 14:56:51 -0300 Subject: [PATCH 10/25] [desafio-07] Main final arquivo --- desafio-07/MatMercer/go/tac.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index bd152b2c8..daf0c460d 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -8,10 +8,6 @@ import ( "os" ) -func main() { - tac() -} - // in MB size, currently the program uses a max of 2*bufSize iff file size > bufSize const maxBufSize = int64(250 << (10 * 2)) @@ -130,3 +126,7 @@ func (r *ReverseReader) Read(b []byte) (n int, err error) { } return read, nil } + +func main() { + tac() +} From 21026571c9afe7a6dec5b12e5a83c40c000b27f1 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 14:57:51 -0300 Subject: [PATCH 11/25] [desafio-07] args -> os.Args --- desafio-07/MatMercer/go/tac.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index daf0c460d..2f3a56353 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -25,14 +25,11 @@ func errAndExit(msg string) { } func tac() { - args := os.Args[1:] - - if len(args) != 1 { + if len(os.Args) != 2 { errAndExit("usage: tac [file]") } - fileName := args[0] - + fileName := os.Args[0] f, err := os.Open(fileName) check(err) defer f.Close() From 6088fc671842aca29258e766d5af10ff9efdef6e Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 14:59:13 -0300 Subject: [PATCH 12/25] [desafio-07] Use log.fatalX --- desafio-07/MatMercer/go/tac.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 2f3a56353..0de59b92e 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -3,8 +3,8 @@ package main import ( "bufio" "bytes" - "fmt" "io" + "log" "os" ) @@ -15,18 +15,13 @@ var stdout *bufio.Writer func check(e error) { if e != nil { - errAndExit(fmt.Sprintf("tac error: %v", e)) + log.Fatalf("tac error: %v", e) } } -func errAndExit(msg string) { - _, _ = os.Stderr.WriteString(msg + "\n") - os.Exit(1) -} - func tac() { if len(os.Args) != 2 { - errAndExit("usage: tac [file]") + log.Fatalln("usage: tac [file]") } fileName := os.Args[0] From 1221b984845ff0d026c9af96b15359903ee866f7 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:04:46 -0300 Subject: [PATCH 13/25] [desafio-07] Remove check() --- desafio-07/MatMercer/go/tac.go | 106 ++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 49 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 0de59b92e..a9c4aba4e 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -19,17 +19,62 @@ func check(e error) { } } -func tac() { +// out outputs to os.Stdout checking for errors +func out(b []byte) { + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err := stdout.Write(b) + if err != nil { + log.Fatalf("tac error: %v", err) + } +} + +func min(x int64, y int64) int64 { + if x > y { + return y + } + return x +} + +type ReverseReader struct { + f *os.File + offset int64 +} + +func NewReverseReader(f *os.File, size int64) *ReverseReader { + return &ReverseReader{ + f, + size, + } +} + +func (r *ReverseReader) Read(b []byte) (n int, err error) { + r.offset -= int64(len(b)) + _, err = r.f.Seek(r.offset, io.SeekStart) + if err != nil { + return 0, err + } + read, err := r.f.Read(b) + if err != nil { + return 0, err + } + return read, nil +} + +func main() { if len(os.Args) != 2 { log.Fatalln("usage: tac [file]") } fileName := os.Args[0] f, err := os.Open(fileName) - check(err) + if err != nil { + log.Fatalf("tac error: %v", err) + } defer f.Close() fi, err := f.Stat() - check(err) + if err != nil { + log.Fatalf("tac error: %v", err) + } // buffers stdout by 128kb stdout = bufio.NewWriterSize(os.Stdout, 128<<(10)) @@ -51,7 +96,9 @@ func tac() { } _, err = r.Read(b[:maxRead]) - check(err) + if err != nil { + log.Fatalf("tac error: %v", err) + } // search until backwards \n and prints it lastEnd := maxRead @@ -69,9 +116,13 @@ func tac() { if i == 0 { newAcc := bytes.NewBuffer(nil) _, err = newAcc.Write(b[i:lastEnd]) - check(err) + if err != nil { + log.Fatalf("tac error: %v", err) + } _, err = newAcc.Write(lineAcc.Bytes()) - check(err) + if err != nil { + log.Fatalf("tac error: %v", err) + } lineAcc = newAcc } } @@ -79,46 +130,3 @@ func tac() { // prints last chunk of data out(lineAcc.Bytes()) } - -// out outputs to os.Stdout checking for errors -func out(b []byte) { - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - _, err := stdout.Write(b) - check(err) -} - -func min(x int64, y int64) int64 { - if x > y { - return y - } - return x -} - -type ReverseReader struct { - f *os.File - offset int64 -} - -func NewReverseReader(f *os.File, size int64) *ReverseReader { - return &ReverseReader{ - f, - size, - } -} - -func (r *ReverseReader) Read(b []byte) (n int, err error) { - r.offset -= int64(len(b)) - _, err = r.f.Seek(r.offset, io.SeekStart) - if err != nil { - return 0, err - } - read, err := r.f.Read(b) - if err != nil { - return 0, err - } - return read, nil -} - -func main() { - tac() -} From 1c7628f7d0b95da17810d37ccb28b0fbd34c3a6e Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:08:55 -0300 Subject: [PATCH 14/25] [desafio-07] Remove out() --- desafio-07/MatMercer/go/tac.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index a9c4aba4e..49e75773e 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -19,15 +19,6 @@ func check(e error) { } } -// out outputs to os.Stdout checking for errors -func out(b []byte) { - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - _, err := stdout.Write(b) - if err != nil { - log.Fatalf("tac error: %v", err) - } -} - func min(x int64, y int64) int64 { if x > y { return y @@ -105,9 +96,17 @@ func main() { for i := maxRead - 1; i >= 0; i-- { if b[i] == '\n' { // write everything but '\n', since b[i] == '\n' - out(b[i+1 : lastEnd]) + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err = stdout.Write(b[i+1 : lastEnd]) + if err != nil { + log.Fatalf("tac error: %v", err) + } // need to write accumulated value - out(lineAcc.Bytes()) + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err = stdout.Write(lineAcc.Bytes()) + if err != nil { + log.Fatalf("tac error: %v", err) + } lineAcc.Reset() // +1 here makes '\n' be printed in next iteration lastEnd = i + 1 @@ -128,5 +127,9 @@ func main() { } } // prints last chunk of data - out(lineAcc.Bytes()) + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err = stdout.Write(lineAcc.Bytes()) + if err != nil { + log.Fatalf("tac error: %v", err) + } } From 8b9faa3893669c932b35d97ffff79adfc3adcc90 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:09:50 -0300 Subject: [PATCH 15/25] [desafio-07] Manual flush --- desafio-07/MatMercer/go/tac.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 49e75773e..931b8ae08 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -69,7 +69,6 @@ func main() { // buffers stdout by 128kb stdout = bufio.NewWriterSize(os.Stdout, 128<<(10)) - defer stdout.Flush() fs := fi.Size() bufSize := min(fs, maxBufSize) @@ -132,4 +131,9 @@ func main() { if err != nil { log.Fatalf("tac error: %v", err) } + + err = stdout.Flush() + if err != nil { + log.Fatalf("tac error: %v", err) + } } From a96d742ce4d4ca2f7bd0ab0d45bbe559ab12c511 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:25:58 -0300 Subject: [PATCH 16/25] [desafio-07] Fix filename bug --- desafio-07/MatMercer/go/tac.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 931b8ae08..432120e85 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -56,7 +56,7 @@ func main() { log.Fatalln("usage: tac [file]") } - fileName := os.Args[0] + fileName := os.Args[1] f, err := os.Open(fileName) if err != nil { log.Fatalf("tac error: %v", err) From 75953502ecccfd7af84d06c3d4a982dce923a5e2 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:33:58 -0300 Subject: [PATCH 17/25] [desafio-07] Unbuf tac --- desafio-07/MatMercer/go/tac.go | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 432120e85..8f09ea195 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -1,8 +1,8 @@ package main import ( - "bufio" "bytes" + "fmt" "io" "log" "os" @@ -11,8 +11,6 @@ import ( // in MB size, currently the program uses a max of 2*bufSize iff file size > bufSize const maxBufSize = int64(250 << (10 * 2)) -var stdout *bufio.Writer - func check(e error) { if e != nil { log.Fatalf("tac error: %v", e) @@ -67,9 +65,6 @@ func main() { log.Fatalf("tac error: %v", err) } - // buffers stdout by 128kb - stdout = bufio.NewWriterSize(os.Stdout, 128<<(10)) - fs := fi.Size() bufSize := min(fs, maxBufSize) b := make([]byte, bufSize) @@ -95,17 +90,9 @@ func main() { for i := maxRead - 1; i >= 0; i-- { if b[i] == '\n' { // write everything but '\n', since b[i] == '\n' - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - _, err = stdout.Write(b[i+1 : lastEnd]) - if err != nil { - log.Fatalf("tac error: %v", err) - } + fmt.Print(string(b[i+1 : lastEnd])) // need to write accumulated value - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - _, err = stdout.Write(lineAcc.Bytes()) - if err != nil { - log.Fatalf("tac error: %v", err) - } + fmt.Print(lineAcc.String()) lineAcc.Reset() // +1 here makes '\n' be printed in next iteration lastEnd = i + 1 @@ -126,14 +113,5 @@ func main() { } } // prints last chunk of data - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 - _, err = stdout.Write(lineAcc.Bytes()) - if err != nil { - log.Fatalf("tac error: %v", err) - } - - err = stdout.Flush() - if err != nil { - log.Fatalf("tac error: %v", err) - } + fmt.Print(lineAcc) } From 5eb4085418f772104c37c3ffe2e659d9ded7b36d Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:35:12 -0300 Subject: [PATCH 18/25] Revert "[desafio-07] Unbuf tac" This reverts commit 75953502ecccfd7af84d06c3d4a982dce923a5e2. --- desafio-07/MatMercer/go/tac.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 8f09ea195..432120e85 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -1,8 +1,8 @@ package main import ( + "bufio" "bytes" - "fmt" "io" "log" "os" @@ -11,6 +11,8 @@ import ( // in MB size, currently the program uses a max of 2*bufSize iff file size > bufSize const maxBufSize = int64(250 << (10 * 2)) +var stdout *bufio.Writer + func check(e error) { if e != nil { log.Fatalf("tac error: %v", e) @@ -65,6 +67,9 @@ func main() { log.Fatalf("tac error: %v", err) } + // buffers stdout by 128kb + stdout = bufio.NewWriterSize(os.Stdout, 128<<(10)) + fs := fi.Size() bufSize := min(fs, maxBufSize) b := make([]byte, bufSize) @@ -90,9 +95,17 @@ func main() { for i := maxRead - 1; i >= 0; i-- { if b[i] == '\n' { // write everything but '\n', since b[i] == '\n' - fmt.Print(string(b[i+1 : lastEnd])) + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err = stdout.Write(b[i+1 : lastEnd]) + if err != nil { + log.Fatalf("tac error: %v", err) + } // need to write accumulated value - fmt.Print(lineAcc.String()) + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err = stdout.Write(lineAcc.Bytes()) + if err != nil { + log.Fatalf("tac error: %v", err) + } lineAcc.Reset() // +1 here makes '\n' be printed in next iteration lastEnd = i + 1 @@ -113,5 +126,14 @@ func main() { } } // prints last chunk of data - fmt.Print(lineAcc) + // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + _, err = stdout.Write(lineAcc.Bytes()) + if err != nil { + log.Fatalf("tac error: %v", err) + } + + err = stdout.Flush() + if err != nil { + log.Fatalf("tac error: %v", err) + } } From 14e334be6b4247ba94be2e1269db2c3e4a7bbb9f Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:44:50 -0300 Subject: [PATCH 19/25] [desafio-07] Cleanup --- desafio-07/MatMercer/go/tac.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 432120e85..b5f8b0605 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -13,12 +13,6 @@ const maxBufSize = int64(250 << (10 * 2)) var stdout *bufio.Writer -func check(e error) { - if e != nil { - log.Fatalf("tac error: %v", e) - } -} - func min(x int64, y int64) int64 { if x > y { return y @@ -61,7 +55,6 @@ func main() { if err != nil { log.Fatalf("tac error: %v", err) } - defer f.Close() fi, err := f.Stat() if err != nil { log.Fatalf("tac error: %v", err) @@ -95,22 +88,22 @@ func main() { for i := maxRead - 1; i >= 0; i-- { if b[i] == '\n' { // write everything but '\n', since b[i] == '\n' - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 _, err = stdout.Write(b[i+1 : lastEnd]) if err != nil { log.Fatalf("tac error: %v", err) } - // need to write accumulated value - // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 + // writes accumulated value _, err = stdout.Write(lineAcc.Bytes()) if err != nil { log.Fatalf("tac error: %v", err) } + // reset the accumulator to receive next line lineAcc.Reset() - // +1 here makes '\n' be printed in next iteration + // makes '\n' be printed in next iteration lastEnd = i + 1 } + // on last iteration, create a new accumulator with [max-read][current-accumulator] if i == 0 { newAcc := bytes.NewBuffer(nil) _, err = newAcc.Write(b[i:lastEnd]) @@ -125,6 +118,9 @@ func main() { } } } + // closes the file + _ = f.Close() + // prints last chunk of data // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 _, err = stdout.Write(lineAcc.Bytes()) From 1fa573e46ad7c0013d26a2005dd5ef7c73d366b6 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:50:06 -0300 Subject: [PATCH 20/25] [desafio-07] Stderr logger --- desafio-07/MatMercer/go/tac.go | 41 +++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index b5f8b0605..b8452f1ad 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -13,6 +13,7 @@ const maxBufSize = int64(250 << (10 * 2)) var stdout *bufio.Writer +// min standard math.Min supports only float64 func min(x int64, y int64) int64 { if x > y { return y @@ -45,19 +46,14 @@ func (r *ReverseReader) Read(b []byte) (n int, err error) { return read, nil } -func main() { - if len(os.Args) != 2 { - log.Fatalln("usage: tac [file]") - } - - fileName := os.Args[1] +func tac(fileName string) error { f, err := os.Open(fileName) if err != nil { - log.Fatalf("tac error: %v", err) + return err } fi, err := f.Stat() if err != nil { - log.Fatalf("tac error: %v", err) + return err } // buffers stdout by 128kb @@ -80,7 +76,7 @@ func main() { _, err = r.Read(b[:maxRead]) if err != nil { - log.Fatalf("tac error: %v", err) + return err } // search until backwards \n and prints it @@ -90,12 +86,12 @@ func main() { // write everything but '\n', since b[i] == '\n' _, err = stdout.Write(b[i+1 : lastEnd]) if err != nil { - log.Fatalf("tac error: %v", err) + return err } // writes accumulated value _, err = stdout.Write(lineAcc.Bytes()) if err != nil { - log.Fatalf("tac error: %v", err) + return err } // reset the accumulator to receive next line lineAcc.Reset() @@ -108,11 +104,11 @@ func main() { newAcc := bytes.NewBuffer(nil) _, err = newAcc.Write(b[i:lastEnd]) if err != nil { - log.Fatalf("tac error: %v", err) + return err } _, err = newAcc.Write(lineAcc.Bytes()) if err != nil { - log.Fatalf("tac error: %v", err) + return err } lineAcc = newAcc } @@ -125,11 +121,26 @@ func main() { // fmt.Println is unbuffered https://github.com/golang/go/issues/36619 _, err = stdout.Write(lineAcc.Bytes()) if err != nil { - log.Fatalf("tac error: %v", err) + return err } err = stdout.Flush() if err != nil { - log.Fatalf("tac error: %v", err) + return err + } + + return nil +} + +func main() { + logger := log.New(os.Stderr, "", 0) + if len(os.Args) != 2 { + logger.Fatalln("usage: tac [file]") + } + + fileName := os.Args[1] + err := tac(fileName) + if err != nil { + logger.Fatalf("tac error: %v", err) } } From 41277925923c96fd10f17998cb50980444d4e1b8 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:50:52 -0300 Subject: [PATCH 21/25] [desafio-07] *ReverseReader -> ReverseReader --- desafio-07/MatMercer/go/tac.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index b8452f1ad..2970c3f63 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -33,7 +33,7 @@ func NewReverseReader(f *os.File, size int64) *ReverseReader { } } -func (r *ReverseReader) Read(b []byte) (n int, err error) { +func (r ReverseReader) Read(b []byte) (n int, err error) { r.offset -= int64(len(b)) _, err = r.f.Seek(r.offset, io.SeekStart) if err != nil { From d59346c3a7eb4c2b49e9a2e7f706f05c9a0d694d Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:55:07 -0300 Subject: [PATCH 22/25] Revert "[desafio-07] *ReverseReader -> ReverseReader" This reverts commit 41277925923c96fd10f17998cb50980444d4e1b8. --- desafio-07/MatMercer/go/tac.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index 2970c3f63..b8452f1ad 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -33,7 +33,7 @@ func NewReverseReader(f *os.File, size int64) *ReverseReader { } } -func (r ReverseReader) Read(b []byte) (n int, err error) { +func (r *ReverseReader) Read(b []byte) (n int, err error) { r.offset -= int64(len(b)) _, err = r.f.Seek(r.offset, io.SeekStart) if err != nil { From 135f5eb85942d45f4a2d8bf290e610880717fcb7 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:56:39 -0300 Subject: [PATCH 23/25] [desafio-07] Improve README.md --- desafio-07/MatMercer/go/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/desafio-07/MatMercer/go/README.md b/desafio-07/MatMercer/go/README.md index ae71a4f39..3612413f4 100644 --- a/desafio-07/MatMercer/go/README.md +++ b/desafio-07/MatMercer/go/README.md @@ -12,6 +12,9 @@ md5sum out.txt Para testar os 512MB de memória com control groups v1: ```bash +# install cgroup +sudo apt-get install cgroup-tools + # setup cgroup export CGROUP=osProgramadoresD7 export CGROUPP="memory/$CGROUP" @@ -20,7 +23,7 @@ echo 512M > "/sys/fs/cgroup/$CGROUPP/memory.limit_in_bytes" echo 0 > "/sys/fs/cgroup/$CGROUPP/memory.swappiness" # run in cgroup -cgexec -g memory:osProgramadoresD7 ./tac 1GB.txt > out.txt +cgexec -g memory:$CGROUP ./tac 1GB.txt > out.txt md5sum out.txt 2b4fd25f11d75c285ec69ecac420bd07 out.txt From 51edce4d8f001b7a5ca6802d0836f9adf0da7686 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:58:43 -0300 Subject: [PATCH 24/25] [desafio-07] b -> readBuf --- desafio-07/MatMercer/go/tac.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index b8452f1ad..cbc55e7cd 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -59,13 +59,13 @@ func tac(fileName string) error { // buffers stdout by 128kb stdout = bufio.NewWriterSize(os.Stdout, 128<<(10)) - fs := fi.Size() - bufSize := min(fs, maxBufSize) - b := make([]byte, bufSize) + fileSize := fi.Size() + bufSize := min(fileSize, maxBufSize) + readBuf := make([]byte, bufSize) maxRead := bufSize - start := fs + start := fileSize lineAcc := bytes.NewBuffer([]byte{}) - r := NewReverseReader(f, fs) + r := NewReverseReader(f, fileSize) for start != 0 { start -= bufSize if start < 0 { @@ -74,7 +74,7 @@ func tac(fileName string) error { start = 0 } - _, err = r.Read(b[:maxRead]) + _, err = r.Read(readBuf[:maxRead]) if err != nil { return err } @@ -82,9 +82,9 @@ func tac(fileName string) error { // search until backwards \n and prints it lastEnd := maxRead for i := maxRead - 1; i >= 0; i-- { - if b[i] == '\n' { - // write everything but '\n', since b[i] == '\n' - _, err = stdout.Write(b[i+1 : lastEnd]) + if readBuf[i] == '\n' { + // write everything but '\n', since readBuf[i] == '\n' + _, err = stdout.Write(readBuf[i+1 : lastEnd]) if err != nil { return err } @@ -102,7 +102,7 @@ func tac(fileName string) error { // on last iteration, create a new accumulator with [max-read][current-accumulator] if i == 0 { newAcc := bytes.NewBuffer(nil) - _, err = newAcc.Write(b[i:lastEnd]) + _, err = newAcc.Write(readBuf[i:lastEnd]) if err != nil { return err } From e6de0bdd14cd80ad7e15d8705f700d1d76a3c464 Mon Sep 17 00:00:00 2001 From: MatMercer Date: Sat, 12 Aug 2023 15:59:44 -0300 Subject: [PATCH 25/25] [desafio-07] Remove NewReverseReader --- desafio-07/MatMercer/go/tac.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/desafio-07/MatMercer/go/tac.go b/desafio-07/MatMercer/go/tac.go index cbc55e7cd..1b837ffb6 100644 --- a/desafio-07/MatMercer/go/tac.go +++ b/desafio-07/MatMercer/go/tac.go @@ -26,13 +26,6 @@ type ReverseReader struct { offset int64 } -func NewReverseReader(f *os.File, size int64) *ReverseReader { - return &ReverseReader{ - f, - size, - } -} - func (r *ReverseReader) Read(b []byte) (n int, err error) { r.offset -= int64(len(b)) _, err = r.f.Seek(r.offset, io.SeekStart) @@ -65,7 +58,7 @@ func tac(fileName string) error { maxRead := bufSize start := fileSize lineAcc := bytes.NewBuffer([]byte{}) - r := NewReverseReader(f, fileSize) + r := &ReverseReader{f, fileSize} for start != 0 { start -= bufSize if start < 0 {