From 8166f66e851c8f4b797bb50a5a38aa5b55fa5c58 Mon Sep 17 00:00:00 2001 From: Nathan Rijksen Date: Fri, 8 Sep 2023 09:50:48 -0700 Subject: [PATCH] Update termtest --- go.mod | 2 +- go.sum | 4 +- .../ActiveState/termtest/helpers.go | 4 + .../ActiveState/termtest/helpers_windows.go | 90 ++++++++++++---- .../ActiveState/termtest/outputproducer.go | 102 +++++++++++++----- vendor/modules.txt | 2 +- 6 files changed, 158 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 0080bb8dc0..7ac453726b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ replace cloud.google.com/go => cloud.google.com/go v0.110.0 require ( github.com/99designs/gqlgen v0.17.19 github.com/ActiveState/go-ogle-analytics v0.0.0-20170510030904-9b3f14901527 - github.com/ActiveState/termtest v0.7.3-0.20230905190146-7d53357589c0 + github.com/ActiveState/termtest v0.7.3-0.20230908163501-7dbf9b0e4aa9 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/andygrunwald/go-jira v1.15.1 diff --git a/go.sum b/go.sum index e9bef410e0..3ca8a663cb 100644 --- a/go.sum +++ b/go.sum @@ -345,8 +345,8 @@ github.com/ActiveState/graphql v0.0.0-20230719154233-6949037a6e48 h1:UCx/ObpVRgC github.com/ActiveState/graphql v0.0.0-20230719154233-6949037a6e48/go.mod h1:NhUbNQ8UpfnC6nZvZ8oThqYSCE/G8FQp9JUrK9jXJs0= github.com/ActiveState/pty v0.0.0-20230628221854-6fb90eb08a14 h1:RdhhSiwmgyUaaF2GBNrbqTwE5SM+MaVjwf91Ua+CK8c= github.com/ActiveState/pty v0.0.0-20230628221854-6fb90eb08a14/go.mod h1:5mM6vNRQwshCjlkOnVpwC//4ZpkiC6nmZr8lPOxJdXs= -github.com/ActiveState/termtest v0.7.3-0.20230905190146-7d53357589c0 h1:lwzAOgZDJV1kgwPi2gtrRm3A9QuPdmNgSlIWnhwIH/g= -github.com/ActiveState/termtest v0.7.3-0.20230905190146-7d53357589c0/go.mod h1:RyWp2NaaTrVAa+XjMHpKAqwBFWbL6wE12HQxiZNGAqU= +github.com/ActiveState/termtest v0.7.3-0.20230908163501-7dbf9b0e4aa9 h1:n4fVV71e5FRLTgqLZiKf3YeU/tsI4OdDFZaqFQHOM+I= +github.com/ActiveState/termtest v0.7.3-0.20230908163501-7dbf9b0e4aa9/go.mod h1:RyWp2NaaTrVAa+XjMHpKAqwBFWbL6wE12HQxiZNGAqU= github.com/AlecAivazis/survey/v2 v2.0.5/go.mod h1:WYBhg6f0y/fNYUuesWQc0PKbJcEliGcYHB9sNT3Bg74= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= diff --git a/vendor/github.com/ActiveState/termtest/helpers.go b/vendor/github.com/ActiveState/termtest/helpers.go index acee6a2d27..93fd15dab9 100644 --- a/vendor/github.com/ActiveState/termtest/helpers.go +++ b/vendor/github.com/ActiveState/termtest/helpers.go @@ -105,3 +105,7 @@ func NormalizeLineEnds(v string) string { func NormalizeLineEndsB(v []byte) []byte { return bytes.ReplaceAll(v, []byte(lineSepWindows), []byte(lineSepPosix)) } + +func copyBytes(b []byte) []byte { + return append([]byte{}, b...) +} diff --git a/vendor/github.com/ActiveState/termtest/helpers_windows.go b/vendor/github.com/ActiveState/termtest/helpers_windows.go index 3025dda726..4febeb9129 100644 --- a/vendor/github.com/ActiveState/termtest/helpers_windows.go +++ b/vendor/github.com/ActiveState/termtest/helpers_windows.go @@ -8,38 +8,92 @@ import ( var ERR_ACCESS_DENIED = windows.ERROR_ACCESS_DENIED -func cleanPtySnapshot(b []byte, isPosix bool) []byte { - b = bytes.TrimRight(b, "\x00") +const UnicodeEscapeRune = '\u001B' +const UnicodeBellRune = '\u0007' +const UnicodeBackspaceRune = '\u0008' // Note in the docs this is \u007f, but in actual use we're seeing \u0008. Possibly badly documented. +// cleanPtySnapshot removes windows console escape sequences from the output so we can interpret it plainly. +// Ultimately we want to emulate the windows console here, just like we're doing for v10x on posix. +// The current implementation is geared towards our needs, and won't be able to handle all escape sequences as a result. +// For details on escape sequences see https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +func cleanPtySnapshot(snapshot []byte, isPosix bool) []byte { if isPosix { - return b + return snapshot } - // If non-posix we need to remove virtual escape sequences from the given byte slice - // https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences + // Most escape sequences appear to end on `A-Za-z@` + plainVirtualEscapeSeqEndValues := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@") - // All escape sequences appear to end on `A-Za-z@` - virtualEscapeSeqEndValues := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@") + // Cheaper than converting to ints + numbers := []byte("0123456789") + + // Some escape sequences are more complex, such as window titles + recordingCode := false + escapeSequenceCode := "" inEscapeSequence := false + inTitleEscapeSequence := false - return bytes.Map(func(r rune) rune { + var result []rune + runes := bytes.Runes(snapshot) + for _, r := range runes { + // Reset code recording outside of escape sequence, so we don't have to manually handle this throughout + if !inEscapeSequence { + recordingCode = false + escapeSequenceCode = "" + } switch { - // Detect start of sequence - case !inEscapeSequence && r == 27: + // SEQUENCE START + + // Detect start of escape sequence + case !inEscapeSequence && r == UnicodeEscapeRune: inEscapeSequence = true - return -1 + recordingCode = true + continue + + // Detect start of complex escape sequence + case inEscapeSequence && !inTitleEscapeSequence && (escapeSequenceCode == "0" || escapeSequenceCode == "2"): + inTitleEscapeSequence = true + recordingCode = false + continue + + // SEQUENCE END + + // Detect end of escape sequence + case inEscapeSequence && !inTitleEscapeSequence && bytes.ContainsRune(plainVirtualEscapeSeqEndValues, r): + inEscapeSequence = false + continue - // Detect end of sequence - case inEscapeSequence && bytes.ContainsRune(virtualEscapeSeqEndValues, r): + // Detect end of complex escape sequence + case inTitleEscapeSequence && r == UnicodeBellRune: inEscapeSequence = false - return -1 + inTitleEscapeSequence = false + continue + + // SEQUENCE CONTINUATION + + case inEscapeSequence && recordingCode: + if r == ']' { + continue + } + if !bytes.ContainsRune(numbers, r) { + recordingCode = false + continue + } + escapeSequenceCode += string(r) - // Anything between start and end of escape sequence should also be dropped + // Detect continuation of escape sequence case inEscapeSequence: - return -1 + recordingCode = false + continue + + // OUTSIDE OF ESCAPE SEQUENCE + + case r == UnicodeBackspaceRune && len(result) > 0: + result = result[:len(result)-1] default: - return r + result = append(result, r) } - }, b) + } + return []byte(string(result)) } diff --git a/vendor/github.com/ActiveState/termtest/outputproducer.go b/vendor/github.com/ActiveState/termtest/outputproducer.go index 3e93e7c20f..000af7f602 100644 --- a/vendor/github.com/ActiveState/termtest/outputproducer.go +++ b/vendor/github.com/ActiveState/termtest/outputproducer.go @@ -2,6 +2,7 @@ package termtest import ( "bufio" + "bytes" "errors" "fmt" "io" @@ -19,11 +20,12 @@ const producerBufferSize = 1024 // outputProducer is responsible for keeping track of the output and notifying consumers when new output is produced type outputProducer struct { - output []byte - snapshotPos int - consumers []*outputConsumer - opts *Opts - mutex *sync.Mutex + output []byte + snapshotPos int + cleanUptoPos int + consumers []*outputConsumer + opts *Opts + mutex *sync.Mutex } func newOutputProducer(opts *Opts) *outputProducer { @@ -39,7 +41,7 @@ func (o *outputProducer) Listen(r io.Reader, w io.Writer) error { return o.listen(r, w, o.appendBuffer, producerPollInterval, producerBufferSize) } -func (o *outputProducer) listen(r io.Reader, w io.Writer, appendBuffer func([]byte) error, interval time.Duration, size int) (rerr error) { +func (o *outputProducer) listen(r io.Reader, w io.Writer, appendBuffer func([]byte, bool) error, interval time.Duration, size int) (rerr error) { o.opts.Logger.Println("listen started") defer func() { o.opts.Logger.Printf("listen stopped, err: %v\n", rerr) @@ -64,26 +66,33 @@ func (o *outputProducer) listen(r io.Reader, w io.Writer, appendBuffer func([]by var ptyEOF = errors.New("pty closed") -func (o *outputProducer) processNextRead(r io.Reader, w io.Writer, appendBuffer func([]byte) error, size int) error { +func (o *outputProducer) processNextRead(r io.Reader, w io.Writer, appendBuffer func([]byte, bool) error, size int) error { o.opts.Logger.Printf("processNextRead started with size: %d\n", size) defer o.opts.Logger.Println("processNextRead stopped") snapshot := make([]byte, size) n, errRead := r.Read(snapshot) + + isEOF := false + if errRead != nil { + pathError := &fs.PathError{} + if errors.Is(errRead, fs.ErrClosed) || errors.Is(errRead, io.EOF) || (runtime.GOOS == "linux" && errors.As(errRead, &pathError)) { + isEOF = true + } + } + if n > 0 { o.opts.Logger.Printf("outputProducer read %d bytes from pty, value: %s", n, snapshot[:n]) if _, err := w.Write(snapshot[:n]); err != nil { return fmt.Errorf("could not write: %w", err) } - snapshot = cleanPtySnapshot(snapshot[:n], o.opts.Posix) - if err := appendBuffer(snapshot); err != nil { + if err := appendBuffer(snapshot[:n], isEOF); err != nil { return fmt.Errorf("could not append buffer: %w", err) } } if errRead != nil { - pathError := &fs.PathError{} - if errors.Is(errRead, fs.ErrClosed) || errors.Is(errRead, io.EOF) || (runtime.GOOS == "linux" && errors.As(errRead, &pathError)) { + if isEOF { return errors.Join(errRead, ptyEOF) } return fmt.Errorf("could not read pty output: %w", errRead) @@ -92,23 +101,27 @@ func (o *outputProducer) processNextRead(r io.Reader, w io.Writer, appendBuffer return nil } -func (o *outputProducer) appendBuffer(value []byte) error { - output := append(o.output, value...) - - if o.opts.OutputSanitizer != nil { - v, err := o.opts.OutputSanitizer(output) - if err != nil { - return fmt.Errorf("could not sanitize output: %w", err) - } - output = v - } - +func (o *outputProducer) appendBuffer(value []byte, isFinal bool) error { if o.opts.NormalizedLineEnds { o.opts.Logger.Println("NormalizedLineEnds prior to appendBuffer") - output = NormalizeLineEndsB(output) + value = NormalizeLineEndsB(value) } - o.output = output + output := append(o.output, value...) + + // Clean output + var err error + o.output, o.cleanUptoPos, err = o.processDirtyOutput(output, o.cleanUptoPos, isFinal, func(output []byte) ([]byte, error) { + var err error + output = cleanPtySnapshot(output, o.opts.Posix) + if o.opts.OutputSanitizer != nil { + output, err = o.opts.OutputSanitizer(output) + } + return output, err + }) + if err != nil { + return fmt.Errorf("cleaning output failed: %w", err) + } o.opts.Logger.Printf("flushing %d output consumers", len(o.consumers)) defer o.opts.Logger.Println("flushed output consumers") @@ -120,6 +133,47 @@ func (o *outputProducer) appendBuffer(value []byte) error { return nil } +// processDirtyOutput will sanitize the output received, but we have to be careful not to clean output that hasn't fully arrived +// For example we may be inside an escape sequence and the escape sequence hasn't finished +// So instead we only process new output up to the most recent line break +// In order for this to work properly the invoker must ensure the output and cleanUptoPos are consistent with each other. +func (o *outputProducer) processDirtyOutput(output []byte, cleanUptoPos int, isFinal bool, cleaner func([]byte) ([]byte, error)) ([]byte, int, error) { + alreadyCleanedOutput := copyBytes(output[:cleanUptoPos]) + processedOutput := []byte{} + unprocessedOutput := copyBytes(output[cleanUptoPos:]) + + if isFinal { + // If we've reached the end there's no point looking for the most recent line break as there's no guarantee the + // output will be terminated by a newline. + processedOutput = copyBytes(unprocessedOutput) + unprocessedOutput = []byte{} + } else { + // Find the most recent line break, and only clean until that point. + // Any output after the most recent line break is considered not ready for cleaning as cleaning depends on + // multiple consecutive characters. + lineSepN := bytes.LastIndex(unprocessedOutput, []byte("\n")) + if lineSepN != -1 { + processedOutput = copyBytes(unprocessedOutput[0 : lineSepN+1]) + unprocessedOutput = unprocessedOutput[lineSepN+1:] + } + } + + // Invoke the cleaner now that we have output that can be cleaned + if len(processedOutput) > 0 { + var err error + processedOutput, err = cleaner(processedOutput) + if err != nil { + return processedOutput, cleanUptoPos, fmt.Errorf("cleaner failed: %w", err) + } + } + + // Keep a record of what point we're up to + cleanUptoPos = cleanUptoPos + len(processedOutput) + + // Stitch everything back together + return append(append(alreadyCleanedOutput, processedOutput...), unprocessedOutput...), cleanUptoPos, nil +} + func (o *outputProducer) flushConsumers() error { o.opts.Logger.Println("flushing consumers") defer o.opts.Logger.Println("flushed consumers") diff --git a/vendor/modules.txt b/vendor/modules.txt index d70fcab5fb..76f98e9603 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,7 +19,7 @@ github.com/ActiveState/graphql # github.com/ActiveState/pty v0.0.0-20230628221854-6fb90eb08a14 ## explicit; go 1.13 github.com/ActiveState/pty -# github.com/ActiveState/termtest v0.7.3-0.20230905190146-7d53357589c0 +# github.com/ActiveState/termtest v0.7.3-0.20230908163501-7dbf9b0e4aa9 ## explicit; go 1.18 github.com/ActiveState/termtest # github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78