From e227f83c34f2f61482367ea70104e9fb980768f6 Mon Sep 17 00:00:00 2001
From: Simone Ragusa <hi@interrato.dev>
Date: Mon, 26 Dec 2022 23:16:07 +0100
Subject: [PATCH] cmd/age: fix terminal escape sequences on Windows

If possible, we enable virtual terminal processing, which is necessary
for using terminal escape sequences on instances of the Windows Console.
When enabling virtual terminal processing fails, we completely avoid
using escape sequences to prevent weird characters to be printed to the
console.

Fixes #474
---
 cmd/age/tui.go         | 12 +++++++++++
 cmd/age/tui_windows.go | 48 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 cmd/age/tui_windows.go

diff --git a/cmd/age/tui.go b/cmd/age/tui.go
index 47a13604..ef12a46e 100644
--- a/cmd/age/tui.go
+++ b/cmd/age/tui.go
@@ -67,6 +67,11 @@ func exit(code int) {
 	os.Exit(code)
 }
 
+// When avoidTerminalEscapeSequences is true, we avoid using escape sequences
+// to prevent weird characters to be printed to the console. This will happen
+// on Windows when virtual terminal processing cannot be enabled.
+var avoidTerminalEscapeSequences bool
+
 // clearLine clears the current line on the terminal, or opens a new line if
 // terminal escape codes don't work.
 func clearLine(out io.Writer) {
@@ -76,6 +81,13 @@ func clearLine(out io.Writer) {
 		EL  = CUI + "K" // Erase in Line
 	)
 
+	// We just open a new line when weird characters may be printed as a result
+	// of using escape codes.
+	if avoidTerminalEscapeSequences {
+		fmt.Fprintf(out, "\r\n")
+		return
+	}
+
 	// First, open a new line, which is guaranteed to work everywhere. Then, try
 	// to erase the line above with escape codes.
 	//
diff --git a/cmd/age/tui_windows.go b/cmd/age/tui_windows.go
new file mode 100644
index 00000000..e8cff2ec
--- /dev/null
+++ b/cmd/age/tui_windows.go
@@ -0,0 +1,48 @@
+// Copyright 2022 The age Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/windows"
+)
+
+// Some instances of the Windows Console (e.g., cmd.exe and Windows PowerShell)
+// do not have the virtual terminal processing enabled, which is necessary to
+// make terminal escape sequences work. For this reason the clearLine function
+// may not properly work. Here we enable the virtual terminal processing, if
+// possible.
+//
+// See https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences.
+func init() {
+	const (
+		ENABLE_PROCESSED_OUTPUT            uint32 = 0x1
+		ENABLE_VIRTUAL_TERMINAL_PROCESSING uint32 = 0x4
+	)
+
+	kernel32DLL := windows.NewLazySystemDLL("Kernel32.dll")
+	setConsoleMode := kernel32DLL.NewProc("SetConsoleMode")
+
+	var mode uint32
+	err := syscall.GetConsoleMode(syscall.Stdout, &mode)
+	if err != nil {
+		// TODO: terminal escape sequences may work at this point, even though
+		// we could not get the console mode. It may be worth trying using
+		// them, but more research is needed to avoid the weird characters
+		// printed out.
+		avoidTerminalEscapeSequences = true
+		return
+	}
+
+	mode |= ENABLE_PROCESSED_OUTPUT
+	mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING
+	ret, _, err := setConsoleMode(uintptr(syscall.Stdout), uintptr(mode))
+	// If the SetConsoleMode function fails, the return value is zero.
+	// See https://learn.microsoft.com/en-us/windows/console/setconsolemode#return-value.
+	if ret == 0 {
+		avoidTerminalEscapeSequences = true
+	}
+}