SJIS escape convert for battle/area/tik2/area.c

Jenkinsfile

@@ -40,7 +40,7 @@ pipeline {
  def jp_progress = sh(returnStdout: true, script: "python3 progress.py jp --pr-comment").trim()
  def ique_progress = sh(returnStdout: true, script: "python3 progress.py ique --pr-comment").trim()
  def pal_progress = sh(returnStdout: true, script: "python3 progress.py pal --pr-comment").trim()
- def warnings = sh(returnStdout: true, script: "./tools/warnings_count/check_new_warnings.sh --jenkins").trim()
+ def warnings = sh(returnStdout: true, script: "./tools/warnings_count/compare_warnings.py --pr-message").trim()
  def comment_id = -1
 
  for (comment in pullRequest.comments) {

src/battle/area/tik2/area.c

@@ -24,12 +24,10 @@ Formation A(Formation_02) = {
  ACTOR_BY_POS(A(super_blooper), A(BlooperPos), 10),
 };
 
-// ソ is encoded as 0x83 0x5C, so it needs another backslash to be interpreted as valid escape sequence
-// TODO modify iconv instead of strings in this file
 BattleList A(Formations) = {
- BATTLE(A(Formation_00), A(tik_01), "ゲッソ\ー"),
- BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソ\ー"),
- BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソ\ー　チビゲッソ\ー"),
+ BATTLE(A(Formation_00), A(tik_01), "ゲッソー"),
+ BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソー"),
+ BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソー　チビゲッソー"),
  {},
 };
 

tools/build/configure.py

@@ -62,8 +62,6 @@ def write_ninja_rules(
  cc_egcs = f"{cc_egcs_dir}/gcc"
  cxx = f"{BUILD_TOOLS}/cc/gcc/g++"
 
- ICONV = "iconv --from UTF-8 --to $encoding"
-
  CPPFLAGS_COMMON = (
  "-Iver/$version/include -Iver/$version/build/include -Iinclude -Isrc -Iassets/$version -D_LANGUAGE_C -D_FINALROM "
  "-DVERSION=$version -DF3DEX_GBI_2 -D_MIPS_SZLONG=32"
@@ -141,15 +139,15 @@ def write_ninja_rules(
  ninja.rule(
  "cc",
  description="gcc $in",
- command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc} {cflags} $cflags - -o $out'",
+ command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc} {cflags} $cflags - -o $out'",
  depfile="$out.d",
  deps="gcc",
  )
 
  ninja.rule(
  "cc_modern",
  description="gcc $in",
- command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'",
+ command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'",
  depfile="$out.d",
  deps="gcc",
  )
@@ -175,7 +173,7 @@ def write_ninja_rules(
  ninja.rule(
  "cxx",
  description="cxx $in",
- command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cxx} {cflags} $cflags - -o $out'",
+ command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cxx} {cflags} $cflags - -o $out'",
  depfile="$out.d",
  deps="gcc",
  )
@@ -690,6 +688,12 @@ def build(
  if version == "ique":
  encoding = "EUC-JP"
 
+ iconv = f"iconv --from UTF-8 --to {encoding}"
+
+ # use tools/sjis-escape.py for src/battle/area/tik2/area.c
+ if version != "ique" and seg.dir.parts[-3:] == ("battle", "area", "tik2") and seg.name == "area":
+ iconv += " | tools/sjis-escape.py"
+
  # Dead cod
  if isinstance(seg.parent.yaml, dict) and seg.parent.yaml.get("dead_code", False):
  obj_path = str(entry.object_path)
@@ -701,7 +705,7 @@ def build(
  variables={
  "cflags": cflags,
  "cppflags": f"-DVERSION_{self.version.upper()}",
- "encoding": encoding,
+ "iconv": iconv,
  },
  )
  build(
@@ -724,7 +728,7 @@ def build(
  variables={
  "cflags": cflags,
  "cppflags": f"-DVERSION_{self.version.upper()}",
- "encoding": encoding,
+ "iconv": iconv,
  },
  )
 
@@ -1077,7 +1081,7 @@ def build(
  variables={
  "cflags": "",
  "cppflags": f"-DVERSION_{self.version.upper()}",
- "encoding": "CP932", # similar to SHIFT-JIS, but includes backslash and tilde
+ "iconv": "iconv --from UTF-8 --to CP932", # similar to SHIFT-JIS, but includes backslash and tilde
  },
  )
  build(elf_path, [o_path], "shape_ld")
@@ -1176,7 +1180,7 @@ def build(
  variables={
  "cflags": "",
  "cppflags": f"-DVERSION_{self.version.upper()}",
- "encoding": "CP932", # similar to SHIFT-JIS, but includes backslash and tilde
+ "iconv": "iconv --from UTF-8 --to CP932", # similar to SHIFT-JIS, but includes backslash and tilde
  },
  )
  else:

tools/sjis-escape.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+
+
+def is_sjis_2byte(c):
+ return (c >= 0x81 and c <= 0x9F) or (c >= 0xE0 and c <= 0xFC)
+
+
+parser = argparse.ArgumentParser(description="")
+parser.add_argument("infile", nargs="?", type=argparse.FileType("rb"), default=sys.stdin.buffer)
+parser.add_argument("outfile", nargs="?", type=argparse.FileType("wb"), default=sys.stdout.buffer)
+
+args = parser.parse_args()
+
+
+def main():
+ for line in args.infile:
+ newline = bytearray()
+ i = 0
+ lsize = len(line)
+ while i < lsize:
+ c = line[i]
+ if is_sjis_2byte(c):
+ i += 1
+ c2 = line[i]
+ newline += "\\x{:02x}\\x{:02x}".format(c, c2).encode("ascii")
+ else:
+ newline.append(c)
+ i += 1
+ args.outfile.write(newline)
+
+
+if __name__ == "__main__":
+ main()

tools/warnings_count/compare_warnings.py

@@ -1,33 +1,48 @@
 #!/usr/bin/env python3
 
 import argparse
+import os
+from pathlib import Path
 import sys
 
+import requests
 
-def countFileLines(filename: str) -> int:
- with open(filename) as f:
- return len(f.readlines())
+script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
+root_dir = script_dir / "../.."
 
 
 def main():
  parser = argparse.ArgumentParser()
- parser.add_argument(
- "currentwarnings",
- help="Name of file which contains the current warnings of the repo.",
- )
- parser.add_argument(
- "newwarnings",
- help="Name of file which contains the *new* warnings of the repo.",
- )
  parser.add_argument("--pr-message", action="store_true")
  args = parser.parse_args()
 
- currentLines = countFileLines(args.currentwarnings)
- newLines = countFileLines(args.newwarnings)
- if newLines > currentLines:
+ # Download the latest warnings.txt
+ response = requests.get("https://papermar.io/reports/warnings.txt")
+ current_warnings = response.content.decode("utf-8").strip().split("\n")
+
+ # Write the current warnings to a file
+ with open(script_dir / "warnings.txt", "w") as f:
+ for line in current_warnings:
+ f.write(line + "\n")
+
+ # Get the new warnings from the build log
+ if not (root_dir / "build_log.txt").exists():
+ print("build_log.txt not found. Exiting.")
+ sys.exit(1)
+
+ with open(root_dir / "build_log.txt") as f:
+ new_warnings = [line for line in f.readlines() if "warning" in line]
+
+ # Write the new warnings to a file
+ with open(script_dir / "warnings_new.txt", "w") as f:
+ f.writelines(new_warnings)
+
+ num_current_warnings = len(current_warnings)
+ num_new_warnings = len(new_warnings)
+ if num_new_warnings > num_current_warnings:
  stderr = False
  if args.pr_message:
- delta = newLines - currentLines
+ delta = num_new_warnings - num_current_warnings
 
  if delta == 1:
  print(f"⚠️ This PR introduces a warning:")
@@ -40,25 +55,21 @@ def main():
  else:
  print()
  print("There are more warnings now. Go fix them!")
- print("\tCurrent warnings: " + str(currentLines))
- print("\tNew warnings: " + str(newLines))
+ print("\tCurrent warnings: " + str(num_current_warnings))
+ print("\tNew warnings: " + str(num_new_warnings))
  print()
 
- with open(args.newwarnings) as new:
- new = new.readlines()
- with open(args.currentwarnings) as current:
- current = current.readlines()
- for newLine in new:
- if "warning: previous declaration of" in newLine:
- continue
-
- if newLine not in current:
- if stderr:
- print(newLine.strip(), file=sys.stderr)
- else:
- print("- " + newLine.strip())
- elif newLines < currentLines:
- delta = currentLines - newLines
+ for newLine in new_warnings:
+ if "warning: previous declaration of" in newLine:
+ continue
+
+ if newLine not in current_warnings:
+ if stderr:
+ print(newLine.strip(), file=sys.stderr)
+ else:
+ print("- " + newLine.strip())
+ elif num_new_warnings < num_current_warnings:
+ delta = num_current_warnings - num_new_warnings
 
  if args.pr_message:
  print(f"✅ This PR fixes {delta} warnings!")