From 5cae8cd35f7e6c2dc4235b767edf2bb687349d90 Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:40 -0500 Subject: [PATCH 1/8] genutil.py: Fixup existing doc comments Signed-off-by: Alexander McClain --- pysrc/genutil.py | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index f5c5fd3a..ce53b174 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -39,7 +39,7 @@ import resource def msgerr(s): - "Write to stderr" + """Write to stderr""" sys.stderr.write("%s\n" % s) msgout = sys.stdout @@ -48,16 +48,16 @@ def set_msgs(fp): msgout = fp def msge(s): - "Write to msgout" + """Write to msgout""" msgout.write("%s\n" % s) def msg(s): - "Write to msgout" + """Write to msgout""" msgout.write("%s\n" % s) def msgn(s): - "Write to msgout" + """Write to msgout""" msgout.write(s) def msgb(s,t=''): - "Write to msgout" + """Write to msgout""" msgout.write('[%s] %s\n' % (s,t)) def cond_die(v, cmd, msg): @@ -72,13 +72,13 @@ def activate_debugger(): global _debugging _debugging = True pdb.set_trace() - + def die(m): global _debugging msgerr('[ERROR] ' + m) if _debugging: pdb.set_trace() - else: + else: traceback.print_stack() sys.exit(1) def warn(m): @@ -89,11 +89,11 @@ def check_python_version(argmaj, argmin): tup = sys.version_info major = tup[0] minor = tup[1] - if ( (major > argmaj ) or + if ( (major > argmaj ) or (major == argmaj and minor >= argmin) ): - return + return die('Need Python version %d.%d or later.' % (argmaj, argmin)) - + def make_readable_by_all_writeable_by_owner(fn, errorname=''): try: rwx = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH @@ -146,7 +146,7 @@ def get_memory_usage(): return (dct['VmSize'].strip(), dct['VmRSS'].strip(), dct['VmData'].strip()) except: return (0,0,0) - + def print_resource_usage(i=''): # 2014-05-19: disabled for now. return @@ -155,14 +155,14 @@ def print_resource_usage(i=''): s = format_resource_usage(x) mem = get_memory_usage() msge('RUSAGE: %s %s vmsize: %s' % (str(i), str(s), str(mem[0]))) - + def flatten_sub(retlist,cur_list,rest): if len(rest)==0: retlist.append(cur_list) return - + r0 = rest[0] if type(r0) == list: for v in r0: @@ -172,7 +172,7 @@ def flatten_sub(retlist,cur_list,rest): else: cur_list.append(r0) flatten_sub(retlist,cur_list,rest[1:]) - + def flatten(list_with_sublists): """Take a list with some possible sublists, and return a list of @@ -194,12 +194,12 @@ def flatten_dict_sub(retlist,cur_dict,main_dict_with_lists,rest_keys): for v in rhs: tdict = copy.copy(cur_dict) # change the list-valued entry to a scalar-valued entry - tdict[r0]=v + tdict[r0]=v flatten_dict_sub(retlist,tdict,main_dict_with_lists,rest_keys[1:]) else: cur_dict[r0] = rhs flatten_dict_sub(retlist,cur_dict,main_dict_with_lists,rest_keys[1:]) - + def flatten_dict(dict_with_lists): """Take a dict with some possible sublists, and return a list of @@ -218,7 +218,7 @@ def cmkdir(path_to_dir): def convert_binary_to_hex(b): - "convert a bit string to hex" + """convert a bit string to hex""" decimal = 0 radix = 1 blist = list(b) @@ -229,9 +229,9 @@ def convert_binary_to_hex(b): radix = radix + radix hexnum = hex(decimal) return hexnum - + def decimal_to_binary(i): - "Take a decimal integer, and return a list of bits MSB to LSB" + """Take a decimal integer, and return a list of bits MSB to LSB""" if i == 0: return [ '0' ] rev_out = [] @@ -245,7 +245,7 @@ def decimal_to_binary(i): return rev_out def hex_to_binary(x): - "Take a hex number, no 0x prefix required, and return a list of bits MSB to LSB" + """Take a hex number, no 0x prefix required, and return a list of bits MSB to LSB""" i = int(x,16) return decimal_to_binary(i) @@ -260,13 +260,13 @@ def round_up_power_of_two(x): make_numeric_decimal_pattern = re.compile(r'^[-]?[0-9]+$') make_numeric_hex_pattern = re.compile(r'^0[xX][0-9A-Fa-f]+$') -make_numeric_binary_pattern = re.compile(r'^0b[01_]+$') +make_numeric_binary_pattern = re.compile(r'^0b[01_]+$') make_numeric_old_binary_pattern = re.compile(r"B['](?P[01_]+)") # leading "B'" make_numeric_old_decimal_pattern = re.compile(r'^0m[0-9]+$') # only base 10 numbers def make_binary(bits): - "return a string of 1s and 0s. Could return letter strings as well" + """return a string of 1s and 0s. Could return letter strings as well""" # binary numbers must preserve the number of bits. If we are # doing a conversion, then we just go with the number of bits we get. @@ -310,12 +310,12 @@ def make_numeric(s, restriction_pattern=None): global make_numeric_hex_pattern global make_numeric_binary_pattern global make_numeric_old_binary_pattern - + if type(s) == int: die("Converting integer to integer") elif make_numeric_hex_pattern.match(s): out = int(s,16) - elif make_numeric_binary_pattern.match(s): + elif make_numeric_binary_pattern.match(s): # I thought that I could leave the '0b' prefix. Python >= 2.6 # handles '0b' just fine but Python 2.5 cannot. As of # 2012-06-20 the pin team currently still relies upon python @@ -327,7 +327,7 @@ def make_numeric(s, restriction_pattern=None): elif make_numeric_old_decimal_pattern.match(s): sys.stderr.write("0m should not occur. Rewrite files!") sys.exit(1) - elif make_numeric_old_binary_pattern.match(s): + elif make_numeric_old_binary_pattern.match(s): sys.stderr.write("B' binary specifer should not occur. Rewrite files!") sys.exit(1) else: @@ -424,7 +424,7 @@ def field_check(obj,fld): def generate_lookup_function_basis(gi,state_space): """Return a dictionary whose values are dictionaries of all the values that the operand decider might have""" - argnames = {} # tokens -> list of all values for that token + argnames = {} # tokens -> list of all values for that token for ii in gi.parser_output.instructions: for bt in ii.ipattern.bits: if bt.is_operand_decider(): @@ -457,9 +457,9 @@ def uniqueify(values): def is_stringish(x): - return isinstance(x,bytes) or isinstance(x,str) + return isinstance(x,bytes) or isinstance(x,str) def make_list_of_str(lst): return [ str(x) for x in lst] def open_readlines(fn): return open(fn,'r').readlines() - + From e9e4d0ba95456c922926451064fe5b83e9224eab Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:41 -0500 Subject: [PATCH 2/8] genutil.py: Add types to functions Signed-off-by: Alexander McClain --- pysrc/genutil.py | 99 ++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index ce53b174..931f5a6a 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -26,6 +26,7 @@ import re import stat import platform +from typing import Tuple, Any psystem = platform.system() if (psystem == 'Microsoft' or @@ -38,27 +39,27 @@ if not on_windows: import resource -def msgerr(s): +def msgerr(msg: str): """Write to stderr""" - sys.stderr.write("%s\n" % s) + sys.stderr.write("%s\n" % msg) msgout = sys.stdout def set_msgs(fp): global msgout msgout = fp -def msge(s): +def msge(msg: str): """Write to msgout""" - msgout.write("%s\n" % s) -def msg(s): + msgout.write("%s\n" % msg) +def msg(msg: str): """Write to msgout""" - msgout.write("%s\n" % s) -def msgn(s): + msgout.write("%s\n" % msg) +def msgn(msg: str): """Write to msgout""" - msgout.write(s) -def msgb(s,t=''): + msgout.write(msg) +def msgb(prefix: str, text: str =''): """Write to msgout""" - msgout.write('[%s] %s\n' % (s,t)) + msgout.write('[%s] %s\n' % (prefix, text)) def cond_die(v, cmd, msg): if v != 0: @@ -73,19 +74,19 @@ def activate_debugger(): _debugging = True pdb.set_trace() -def die(m): +def die(msg: str): global _debugging - msgerr('[ERROR] ' + m) + msgerr('[ERROR] ' + msg) if _debugging: pdb.set_trace() else: traceback.print_stack() sys.exit(1) -def warn(m): - msgerr('[WARNING] ' + m) +def warn(msg: str): + msgerr('[WARNING] ' + msg) -def check_python_version(argmaj, argmin): +def check_python_version(argmaj: int, argmin: int): tup = sys.version_info major = tup[0] minor = tup[1] @@ -94,14 +95,14 @@ def check_python_version(argmaj, argmin): return die('Need Python version %d.%d or later.' % (argmaj, argmin)) -def make_readable_by_all_writeable_by_owner(fn, errorname=''): +def make_readable_by_all_writeable_by_owner(file_name: str, errorname: str = ''): try: rwx = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH - os.chmod(fn, rwx) + os.chmod(file_name, rwx) except IOError: - die('Could not chmod: ' + errorname + ' file: [' + fn + ']' ) + die('Could not chmod: ' + errorname + ' file: [' + file_name + ']' ) -def open_for_writing(mode): +def open_for_writing(mode: str) -> bool: # The default mode is 'r'. mode = mode or 'r' for c in mode: @@ -109,13 +110,13 @@ def open_for_writing(mode): return True return False -def base_open_file(fn, rw, errorname=''): +def base_open_file(file_name: str, rw: str, errorname: str =''): try: - fp = open(fn,rw) + fp = open(file_name, rw) except IOError: - die('Could not open: ' + errorname + ' file: [' + fn + ']' ) + die('Could not open: ' + errorname + ' file: [' + file_name + ']' ) if open_for_writing(rw): - make_readable_by_all_writeable_by_owner(fn,errorname) + make_readable_by_all_writeable_by_owner(file_name, errorname) return fp def resource_usage(): @@ -125,7 +126,7 @@ def resource_usage(): x = resource.getrusage(resource.RUSAGE_SELF) return x -def format_resource_usage(x): +def format_resource_usage(x) -> str: s = '' s += 'user: ' + str(x[0]) s += ' sys: ' + str(x[1]) @@ -136,7 +137,7 @@ def format_resource_usage(x): #s += ' maxstk: ' + str(x[5]) return s -def get_memory_usage(): +def get_memory_usage() -> Tuple[int, int, int]: """Return a tuple of (vmsize, vmrss, vmdata) on linux systems with /proc filesystems.""" try: @@ -147,7 +148,7 @@ def get_memory_usage(): except: return (0,0,0) -def print_resource_usage(i=''): +def print_resource_usage(i: str = ''): # 2014-05-19: disabled for now. return @@ -217,11 +218,11 @@ def cmkdir(path_to_dir): -def convert_binary_to_hex(b): +def convert_binary_to_hex(bit_string: str) -> str: """convert a bit string to hex""" decimal = 0 radix = 1 - blist = list(b) + blist = list(bit_string) blist.reverse() for bit in blist: if bit == '1': @@ -230,7 +231,7 @@ def convert_binary_to_hex(b): hexnum = hex(decimal) return hexnum -def decimal_to_binary(i): +def decimal_to_binary(i: int) -> list[str]: """Take a decimal integer, and return a list of bits MSB to LSB""" if i == 0: return [ '0' ] @@ -244,15 +245,15 @@ def decimal_to_binary(i): rev_out.reverse() return rev_out -def hex_to_binary(x): +def hex_to_binary(x: str) -> list[str]: """Take a hex number, no 0x prefix required, and return a list of bits MSB to LSB""" i = int(x,16) return decimal_to_binary(i) -def stringify_list(lst): +def stringify_list(lst: list[Any]) -> str: return ' '.join([ str(x) for x in lst]) -def round_up_power_of_two(x): +def round_up_power_of_two(x: int) -> int: lg = math.ceil(math.log(x,2)) return 1 << int(lg) @@ -265,7 +266,7 @@ def round_up_power_of_two(x): make_numeric_old_binary_pattern = re.compile(r"B['](?P[01_]+)") # leading "B'" make_numeric_old_decimal_pattern = re.compile(r'^0m[0-9]+$') # only base 10 numbers -def make_binary(bits): +def make_binary(bits: str) -> str: """return a string of 1s and 0s. Could return letter strings as well""" # binary numbers must preserve the number of bits. If we are # doing a conversion, then we just go with the number of bits we get. @@ -286,12 +287,12 @@ def make_binary(bits): bits = re.sub('_','',bits) return bits -def is_hex(s): +def is_hex(s: str) -> bool: if make_numeric_hex_pattern.match(s): return True return False -def numeric(s): +def numeric(s: str) -> bool: if make_numeric_decimal_pattern.match(s): return True if make_numeric_hex_pattern.match(s): @@ -300,12 +301,12 @@ def numeric(s): return True return False -def is_binary(s): +def is_binary(s: str) -> bool: if make_numeric_binary_pattern.match(s): return True return False -def make_numeric(s, restriction_pattern=None): +def make_numeric(s: str, restriction_pattern=None) -> int: global make_numeric_old_decimal_pattern global make_numeric_hex_pattern global make_numeric_binary_pattern @@ -337,7 +338,7 @@ def make_numeric(s, restriction_pattern=None): ######################### -def find_runs(blist): +def find_runs(blist: list[str]) -> list[Tuple[str, int]]: """Accept a bit list. Return a list tuples (letter,count) describing bit runs, the same bit repeated n times""" last = None @@ -363,24 +364,24 @@ def print_runs(runs): s.append("(%s,%d)" % (val,count)) msge("Runs: %s" % ' '.join(s) ) -def no_underscores(s): +def no_underscores(s: str) -> str: v = s.replace('_','') # remove underscores return v comment_pattern = re.compile(r'[#].*$') -def no_comments(line): +def no_comments(line: str) -> str: global comment_pattern oline = comment_pattern.sub('',line) oline = oline.strip() return oline -def blank_line(line): +def blank_line(line: str) -> bool: if line == '': return False return True continuation_pattern = re.compile(r'\\$') -def process_continuations(lines): +def process_continuations(lines: list[str]) -> list[str]: global continuation_pattern olines=[] while len(lines) != 0: @@ -410,12 +411,12 @@ def skip_junk(lines): else: break return lines -def field_check(obj,fld): +def field_check(obj: object, field: str) -> bool: "Return true if fld exists in obj" try: # ignore returned value - s = getattr(obj,fld) + s = getattr(obj, field) return True except AttributeError: retval = False @@ -447,7 +448,7 @@ def generate_lookup_function_basis(gi,state_space): die("Bad patten bit (not an operand decider) in " + ii.dump_str()) return argnames -def uniqueify(values): +def uniqueify(values) -> list[Any]: s = {} for a in values: s[a] = True @@ -456,10 +457,10 @@ def uniqueify(values): return k -def is_stringish(x): +def is_stringish(x) -> bool: return isinstance(x,bytes) or isinstance(x,str) -def make_list_of_str(lst): +def make_list_of_str(lst: list[Any]) -> list[str]: return [ str(x) for x in lst] -def open_readlines(fn): - return open(fn,'r').readlines() +def open_readlines(file_name: str) -> list[str]: + return open(file_name,'r').readlines() From 1ac5cb34ba5d71e9f7557ab7850da9ffbc619c4a Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:41 -0500 Subject: [PATCH 3/8] genutil.py: Remove unused functions `skip_junk` and `print_runs` were not used anywhere. Signed-off-by: Alexander McClain --- pysrc/genutil.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index 931f5a6a..593b0737 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -358,12 +358,6 @@ def find_runs(blist: list[str]) -> list[Tuple[str, int]]: output.append( (last, run) ) return output -def print_runs(runs): - s = [] - for (val, count) in runs: - s.append("(%s,%d)" % (val,count)) - msge("Runs: %s" % ' '.join(s) ) - def no_underscores(s: str) -> str: v = s.replace('_','') # remove underscores return v @@ -402,15 +396,6 @@ def process_continuations(lines: list[str]) -> list[str]: del lines return olines -def skip_junk(lines): - while len(lines) != 0: - line = no_comments(lines[0]) - line = line.strip() - if line == '': - lines.pop(0) - else: - break - return lines def field_check(obj: object, field: str) -> bool: "Return true if fld exists in obj" From 878a2debf9d5bb8f6106c1ebe1df257b673e92e0 Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:41 -0500 Subject: [PATCH 4/8] genutil.py: Rename `numeric` to `is_numeric` To follow the same naming convention as the surrounding functions. Signed-off-by: Alexander McClain --- pysrc/genutil.py | 4 ++-- pysrc/ild.py | 2 +- pysrc/map_info_rdr.py | 6 +++--- pysrc/opnds.py | 2 +- pysrc/read-encfile.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index 593b0737..f1d71089 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -280,7 +280,7 @@ def make_binary(bits: str) -> str: # only get 3 bits out. Because this routine is not cognizant of # the field width. - if numeric(bits): + if is_numeric(bits): v = make_numeric(bits) d = decimal_to_binary(v) # a list of bits return ''.join(d) @@ -292,7 +292,7 @@ def is_hex(s: str) -> bool: return True return False -def numeric(s: str) -> bool: +def is_numeric(s: str) -> bool: if make_numeric_decimal_pattern.match(s): return True if make_numeric_hex_pattern.match(s): diff --git a/pysrc/ild.py b/pysrc/ild.py index ba43dc1a..b585497d 100755 --- a/pysrc/ild.py +++ b/pysrc/ild.py @@ -449,7 +449,7 @@ def set_mode(self, ii, mode_space): def parse_opcode(self, op_str): # has side effects of settting self.missing_bits and self.incomplete val = None - if genutil.numeric(op_str): + if genutil.is_numeric(op_str): val = genutil.make_numeric(op_str) # special check for partial binary numbers as opcodes diff --git a/pysrc/map_info_rdr.py b/pysrc/map_info_rdr.py index e6863569..85d9a48f 100755 --- a/pysrc/map_info_rdr.py +++ b/pysrc/map_info_rdr.py @@ -175,7 +175,7 @@ def _parse_map_line(s): if mi.map_id == 'N/A': _die("Bad map description map-id [{}]".format(s)) - elif genutil.numeric(mi.map_id): + elif genutil.is_numeric(mi.map_id): mi.map_id = genutil.make_numeric(mi.map_id) else: mi.map_id_fixup=True @@ -185,7 +185,7 @@ def _parse_map_line(s): _die("Bad map description legacy escape [{}]".format(s)) if mi.legacy_opcode != 'N/A': _die("Bad map description legacy opcode [{}]".format(s)) - if genutil.numeric(mi.map_id): + if genutil.is_numeric(mi.map_id): mi.map_id = genutil.make_numeric(mi.map_id) else: _die("Bad map description map id [{}]".format(s)) @@ -196,7 +196,7 @@ def _parse_map_line(s): _die("Bad map description modrm specifier [{}]".format(s)) if mi.imm not in ['var','0','1','2','4']: _die("Bad map description imm specifier [{}]".format(s)) - if genutil.numeric(mi.opcpos): + if genutil.is_numeric(mi.opcpos): mi.opcpos = genutil.make_numeric(mi.opcpos) else: _die("Bad map description opcode position specifier [{}]".format(s)) diff --git a/pysrc/opnds.py b/pysrc/opnds.py index 4d2bef8d..ad25461f 100644 --- a/pysrc/opnds.py +++ b/pysrc/opnds.py @@ -426,7 +426,7 @@ def parse_one_operand(w, elif enum_pattern.match(rhs): # for storing XED_* enum values as RHS's of operand bindings optype = 'imm_const' - elif (not genutil.numeric(rhs)) and az_cap_pattern.search(rhs): + elif (not genutil.is_numeric(rhs)) and az_cap_pattern.search(rhs): genutil.die("THIS SHOULD NOT HAPPEN: %s" % (rhs)) elif letters_underscore_pattern.match(rhs): rhs = list(rhs.replace('_','')) diff --git a/pysrc/read-encfile.py b/pysrc/read-encfile.py index c50a70d7..ce5afb31 100755 --- a/pysrc/read-encfile.py +++ b/pysrc/read-encfile.py @@ -2231,7 +2231,7 @@ def parse_one_decode_rule(self, iclass, operand_str, pattern_str): # from the instruction decode patterns (MOD[mm] etc.). We # ignore the ones for constant bindings! for (field_name,value) in extra_bindings: - if genutil.numeric(value): + if genutil.is_numeric(value): #msgerr("IGNORING %s %s" % (field_name, value)) pass # we ignore things that are just bits at this point. else: From 64e07c4941b0490b45f6a7dc217dc84feeb9aa2d Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:42 -0500 Subject: [PATCH 5/8] genutil.py: Reformat file The file had many formatting issues, the main ones being: 1. Many functions had 3 space indentation, which is inconsistent with the rest of the file's 4 space indentation. 2. No spaces between list and tuple elements 3. Extra spaces around list and tuple elements 4. Lack of space between function definitions (2 newlines is recommended) Signed-off-by: Alexander McClain --- pysrc/genutil.py | 414 ++++++++++++++++++++++++++--------------------- 1 file changed, 230 insertions(+), 184 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index f1d71089..74464fdc 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -21,7 +21,7 @@ import os import math import traceback -#import types +# import types import copy import re import stat @@ -30,8 +30,8 @@ psystem = platform.system() if (psystem == 'Microsoft' or - psystem == 'Windows' or - psystem.find('CYGWIN') != -1) : + psystem == 'Windows' or + psystem.find('CYGWIN') != -1): on_windows = True else: on_windows = False @@ -39,41 +39,56 @@ if not on_windows: import resource + def msgerr(msg: str): """Write to stderr""" sys.stderr.write("%s\n" % msg) + msgout = sys.stdout + + def set_msgs(fp): global msgout msgout = fp + def msge(msg: str): """Write to msgout""" msgout.write("%s\n" % msg) + + def msg(msg: str): """Write to msgout""" msgout.write("%s\n" % msg) + + def msgn(msg: str): """Write to msgout""" msgout.write(msg) -def msgb(prefix: str, text: str =''): + + +def msgb(prefix: str, text: str = ''): """Write to msgout""" msgout.write('[%s] %s\n' % (prefix, text)) + def cond_die(v, cmd, msg): if v != 0: s = msg + '\n [CMD] ' + cmd die(s) + import pdb _debugging = False + def activate_debugger(): global _debugging _debugging = True pdb.set_trace() + def die(msg: str): global _debugging msgerr('[ERROR] ' + msg) @@ -82,6 +97,8 @@ def die(msg: str): else: traceback.print_stack() sys.exit(1) + + def warn(msg: str): msgerr('[WARNING] ' + msg) @@ -90,42 +107,47 @@ def check_python_version(argmaj: int, argmin: int): tup = sys.version_info major = tup[0] minor = tup[1] - if ( (major > argmaj ) or - (major == argmaj and minor >= argmin) ): + if ((major > argmaj) or + (major == argmaj and minor >= argmin)): return die('Need Python version %d.%d or later.' % (argmaj, argmin)) + def make_readable_by_all_writeable_by_owner(file_name: str, errorname: str = ''): try: - rwx = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH + rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH os.chmod(file_name, rwx) except IOError: - die('Could not chmod: ' + errorname + ' file: [' + file_name + ']' ) + die('Could not chmod: ' + errorname + ' file: [' + file_name + ']') + def open_for_writing(mode: str) -> bool: - # The default mode is 'r'. - mode = mode or 'r' - for c in mode: - if c in ['w', 'a', '+']: - return True - return False - -def base_open_file(file_name: str, rw: str, errorname: str =''): + # The default mode is 'r'. + mode = mode or 'r' + for c in mode: + if c in ['w', 'a', '+']: + return True + return False + + +def base_open_file(file_name: str, rw: str, errorname: str = ''): try: fp = open(file_name, rw) except IOError: - die('Could not open: ' + errorname + ' file: [' + file_name + ']' ) + die('Could not open: ' + errorname + ' file: [' + file_name + ']') if open_for_writing(rw): - make_readable_by_all_writeable_by_owner(file_name, errorname) + make_readable_by_all_writeable_by_owner(file_name, errorname) return fp + def resource_usage(): if on_windows: - x = (0,0,0,0,0,0) + x = (0, 0, 0, 0, 0, 0) else: x = resource.getrusage(resource.RUSAGE_SELF) return x + def format_resource_usage(x) -> str: s = '' s += 'user: ' + str(x[0]) @@ -137,16 +159,18 @@ def format_resource_usage(x) -> str: #s += ' maxstk: ' + str(x[5]) return s + def get_memory_usage() -> Tuple[int, int, int]: """Return a tuple of (vmsize, vmrss, vmdata) on linux systems with /proc filesystems.""" try: - lines = open('/proc/%s/status' % os.getpid(),'r').readlines() - pairs = [ x.split(':') for x in lines] + lines = open('/proc/%s/status' % os.getpid(), 'r').readlines() + pairs = [x.split(':') for x in lines] dct = dict(pairs) - return (dct['VmSize'].strip(), dct['VmRSS'].strip(), dct['VmData'].strip()) + return (dct['VmSize'].strip(), dct['VmRSS'].strip(), dct['VmData'].strip()) except: - return (0,0,0) + return (0, 0, 0) + def print_resource_usage(i: str = ''): # 2014-05-19: disabled for now. @@ -158,9 +182,8 @@ def print_resource_usage(i: str = ''): msge('RUSAGE: %s %s vmsize: %s' % (str(i), str(s), str(mem[0]))) - -def flatten_sub(retlist,cur_list,rest): - if len(rest)==0: +def flatten_sub(retlist, cur_list, rest): + if len(rest) == 0: retlist.append(cur_list) return @@ -169,10 +192,10 @@ def flatten_sub(retlist,cur_list,rest): for v in r0: tlist = copy.copy(cur_list) tlist.append(v) - flatten_sub(retlist,tlist,rest[1:]) + flatten_sub(retlist, tlist, rest[1:]) else: cur_list.append(r0) - flatten_sub(retlist,cur_list,rest[1:]) + flatten_sub(retlist, cur_list, rest[1:]) def flatten(list_with_sublists): @@ -183,8 +206,8 @@ def flatten(list_with_sublists): return retval -def flatten_dict_sub(retlist,cur_dict,main_dict_with_lists,rest_keys): - if len(rest_keys)==0: +def flatten_dict_sub(retlist, cur_dict, main_dict_with_lists, rest_keys): + if len(rest_keys) == 0: retlist.append(cur_dict) return @@ -195,11 +218,11 @@ def flatten_dict_sub(retlist,cur_dict,main_dict_with_lists,rest_keys): for v in rhs: tdict = copy.copy(cur_dict) # change the list-valued entry to a scalar-valued entry - tdict[r0]=v - flatten_dict_sub(retlist,tdict,main_dict_with_lists,rest_keys[1:]) + tdict[r0] = v + flatten_dict_sub(retlist, tdict, main_dict_with_lists, rest_keys[1:]) else: cur_dict[r0] = rhs - flatten_dict_sub(retlist,cur_dict,main_dict_with_lists,rest_keys[1:]) + flatten_dict_sub(retlist, cur_dict, main_dict_with_lists, rest_keys[1:]) def flatten_dict(dict_with_lists): @@ -207,9 +230,10 @@ def flatten_dict(dict_with_lists): dicts where no rhs is a list. All possible combinations""" retval = [] kys = list(dict_with_lists.keys()) - flatten_dict_sub(retval, {}, dict_with_lists,kys) + flatten_dict_sub(retval, {}, dict_with_lists, kys) return retval + def cmkdir(path_to_dir): """Make a directory if it does not exist""" if not os.path.exists(path_to_dir): @@ -217,54 +241,57 @@ def cmkdir(path_to_dir): os.makedirs(path_to_dir) - def convert_binary_to_hex(bit_string: str) -> str: - """convert a bit string to hex""" - decimal = 0 - radix = 1 - blist = list(bit_string) - blist.reverse() - for bit in blist: - if bit == '1': - decimal = decimal + radix - radix = radix + radix - hexnum = hex(decimal) - return hexnum + """convert a bit string to hex""" + decimal = 0 + radix = 1 + blist = list(bit_string) + blist.reverse() + for bit in blist: + if bit == '1': + decimal = decimal + radix + radix = radix + radix + hexnum = hex(decimal) + return hexnum + def decimal_to_binary(i: int) -> list[str]: - """Take a decimal integer, and return a list of bits MSB to LSB""" - if i == 0: - return [ '0' ] - rev_out = [] - while i > 0: - bit = i & 1 - #print hex(i),ig, bit - rev_out.append(str(bit)) - i = i >> 1 - #print str(rev_out) - rev_out.reverse() - return rev_out + """Take a decimal integer, and return a list of bits MSB to LSB""" + if i == 0: + return ['0'] + rev_out = [] + while i > 0: + bit = i & 1 + # print hex(i),ig, bit + rev_out.append(str(bit)) + i = i >> 1 + # print str(rev_out) + rev_out.reverse() + return rev_out + def hex_to_binary(x: str) -> list[str]: - """Take a hex number, no 0x prefix required, and return a list of bits MSB to LSB""" - i = int(x,16) - return decimal_to_binary(i) + """Take a hex number, no 0x prefix required, and return a list of bits MSB to LSB""" + i = int(x, 16) + return decimal_to_binary(i) + def stringify_list(lst: list[Any]) -> str: - return ' '.join([ str(x) for x in lst]) + return ' '.join([str(x) for x in lst]) -def round_up_power_of_two(x: int) -> int: - lg = math.ceil(math.log(x,2)) - return 1 << int(lg) +def round_up_power_of_two(x: int) -> int: + lg = math.ceil(math.log(x, 2)) + return 1 << int(lg) make_numeric_decimal_pattern = re.compile(r'^[-]?[0-9]+$') make_numeric_hex_pattern = re.compile(r'^0[xX][0-9A-Fa-f]+$') make_numeric_binary_pattern = re.compile(r'^0b[01_]+$') -make_numeric_old_binary_pattern = re.compile(r"B['](?P[01_]+)") # leading "B'" -make_numeric_old_decimal_pattern = re.compile(r'^0m[0-9]+$') # only base 10 numbers +make_numeric_old_binary_pattern = re.compile(r"B['](?P[01_]+)") # leading "B'" +make_numeric_old_decimal_pattern = re.compile(r'^0m[0-9]+$') # only base 10 numbers + def make_binary(bits: str) -> str: """return a string of 1s and 0s. Could return letter strings as well""" @@ -273,7 +300,7 @@ def make_binary(bits: str) -> str: if make_numeric_binary_pattern.match(bits): # strip off the 0b prefix - bits = re.sub('_','',bits) + bits = re.sub('_', '', bits) return bits[2:] # this might return fewer than the expected number of binary bits. # for example, if you are in a 4 bit field and use a 5, you will @@ -282,16 +309,18 @@ def make_binary(bits: str) -> str: if is_numeric(bits): v = make_numeric(bits) - d = decimal_to_binary(v) # a list of bits + d = decimal_to_binary(v) # a list of bits return ''.join(d) - bits = re.sub('_','',bits) + bits = re.sub('_', '', bits) return bits + def is_hex(s: str) -> bool: if make_numeric_hex_pattern.match(s): return True return False + def is_numeric(s: str) -> bool: if make_numeric_decimal_pattern.match(s): return True @@ -301,151 +330,168 @@ def is_numeric(s: str) -> bool: return True return False + def is_binary(s: str) -> bool: if make_numeric_binary_pattern.match(s): return True return False + def make_numeric(s: str, restriction_pattern=None) -> int: - global make_numeric_old_decimal_pattern - global make_numeric_hex_pattern - global make_numeric_binary_pattern - global make_numeric_old_binary_pattern - - if type(s) == int: - die("Converting integer to integer") - elif make_numeric_hex_pattern.match(s): - out = int(s,16) - elif make_numeric_binary_pattern.match(s): - # I thought that I could leave the '0b' prefix. Python >= 2.6 - # handles '0b' just fine but Python 2.5 cannot. As of - # 2012-06-20 the pin team currently still relies upon python - # 2.5. - just_bits = s.replace('0b','') - just_bits = just_bits.replace('_','') - out = int(just_bits,2) - #msgb("MAKE BINARY NUMERIC", "%s -> %d" % (s,out)) - elif make_numeric_old_decimal_pattern.match(s): - sys.stderr.write("0m should not occur. Rewrite files!") - sys.exit(1) - elif make_numeric_old_binary_pattern.match(s): - sys.stderr.write("B' binary specifer should not occur. Rewrite files!") - sys.exit(1) - else: - out = int(s) - return out + global make_numeric_old_decimal_pattern + global make_numeric_hex_pattern + global make_numeric_binary_pattern + global make_numeric_old_binary_pattern + + if type(s) == int: + die("Converting integer to integer") + elif make_numeric_hex_pattern.match(s): + out = int(s, 16) + elif make_numeric_binary_pattern.match(s): + # I thought that I could leave the '0b' prefix. Python >= 2.6 + # handles '0b' just fine but Python 2.5 cannot. As of + # 2012-06-20 the pin team currently still relies upon python + # 2.5. + just_bits = s.replace('0b', '') + just_bits = just_bits.replace('_', '') + out = int(just_bits, 2) + # msgb("MAKE BINARY NUMERIC", "%s -> %d" % (s,out)) + elif make_numeric_old_decimal_pattern.match(s): + sys.stderr.write("0m should not occur. Rewrite files!") + sys.exit(1) + elif make_numeric_old_binary_pattern.match(s): + sys.stderr.write("B' binary specifer should not occur. Rewrite files!") + sys.exit(1) + else: + out = int(s) + return out ######################### def find_runs(blist: list[str]) -> list[Tuple[str, int]]: - """Accept a bit list. Return a list tuples (letter,count) + """Accept a bit list. Return a list tuples (letter,count) describing bit runs, the same bit repeated n times""" - last = None - run = 1 - output = [] - if blist == None: - return output - for b in blist: - if last != None: - if b == last: - run = run + 1 - else: - output.append( (last, run) ) - run = 1 - last = b - if last != None: - output.append( (last, run) ) - return output + last = None + run = 1 + output = [] + if blist == None: + return output + for b in blist: + if last != None: + if b == last: + run = run + 1 + else: + output.append((last, run)) + run = 1 + last = b + if last != None: + output.append((last, run)) + return output + def no_underscores(s: str) -> str: - v = s.replace('_','') # remove underscores + v = s.replace('_', '') # remove underscores return v + comment_pattern = re.compile(r'[#].*$') + + def no_comments(line: str) -> str: - global comment_pattern - oline = comment_pattern.sub('',line) - oline = oline.strip() - return oline + global comment_pattern + oline = comment_pattern.sub('', line) + oline = oline.strip() + return oline + def blank_line(line: str) -> bool: - if line == '': - return False - return True + if line == '': + return False + return True + continuation_pattern = re.compile(r'\\$') + + def process_continuations(lines: list[str]) -> list[str]: - global continuation_pattern - olines=[] - while len(lines) != 0: - line = no_comments(lines[0]) - line = line.strip() - lines.pop(0) - if line == '': - continue - if continuation_pattern.search(line): - # combine this line with the next line if the next line exists - line = continuation_pattern.sub('',line) - if len(lines) >= 1: - combined_lines = [ line + lines[0] ] - lines.pop(0) - lines = combined_lines + lines - continue - olines.append(line) - del lines - return olines + global continuation_pattern + olines = [] + while len(lines) != 0: + line = no_comments(lines[0]) + line = line.strip() + lines.pop(0) + if line == '': + continue + if continuation_pattern.search(line): + # combine this line with the next line if the next line exists + line = continuation_pattern.sub('', line) + if len(lines) >= 1: + combined_lines = [line + lines[0]] + lines.pop(0) + lines = combined_lines + lines + continue + olines.append(line) + del lines + return olines + def field_check(obj: object, field: str) -> bool: - "Return true if fld exists in obj" - - try: - # ignore returned value - s = getattr(obj, field) - return True - except AttributeError: - retval = False - - return retval -def generate_lookup_function_basis(gi,state_space): - """Return a dictionary whose values are dictionaries of all the values + "Return true if fld exists in obj" + + try: + # ignore returned value + s = getattr(obj, field) + return True + except AttributeError: + retval = False + + return retval + + +def generate_lookup_function_basis(gi, state_space): + """Return a dictionary whose values are dictionaries of all the values that the operand decider might have""" - argnames = {} # tokens -> list of all values for that token - for ii in gi.parser_output.instructions: - for bt in ii.ipattern.bits: - if bt.is_operand_decider(): - if bt.token not in argnames: - argnames[bt.token] = {} - - if bt.test == 'eq': - argnames[bt.token][bt.requirement]=True - elif bt.test == 'ne': - all_values_for_this_od = state_space[bt.token] - trimmed_vals = list(filter(lambda x: x != bt.requirement, - all_values_for_this_od)) - for tv in trimmed_vals: - argnames[bt.token][tv]=True + argnames = {} # tokens -> list of all values for that token + for ii in gi.parser_output.instructions: + for bt in ii.ipattern.bits: + if bt.is_operand_decider(): + if bt.token not in argnames: + argnames[bt.token] = {} + + if bt.test == 'eq': + argnames[bt.token][bt.requirement] = True + elif bt.test == 'ne': + all_values_for_this_od = state_space[bt.token] + trimmed_vals = list(filter(lambda x: x != bt.requirement, + all_values_for_this_od)) + for tv in trimmed_vals: + argnames[bt.token][tv] = True + else: + die("Bad bit test (not eq or ne) in " + ii.dump_str()) + elif bt.is_nonterminal(): + pass # FIXME make a better test else: - die("Bad bit test (not eq or ne) in " + ii.dump_str()) - elif bt.is_nonterminal(): - pass # FIXME make a better test - else: - die("Bad patten bit (not an operand decider) in " + ii.dump_str()) - return argnames + die("Bad patten bit (not an operand decider) in " + ii.dump_str()) + return argnames + def uniqueify(values) -> list[Any]: - s = {} - for a in values: - s[a] = True - k = list(s.keys()) - k.sort() - return k + s = {} + for a in values: + s[a] = True + k = list(s.keys()) + k.sort() + return k def is_stringish(x) -> bool: - return isinstance(x,bytes) or isinstance(x,str) + return isinstance(x, bytes) or isinstance(x, str) + + def make_list_of_str(lst: list[Any]) -> list[str]: - return [ str(x) for x in lst] -def open_readlines(file_name: str) -> list[str]: - return open(file_name,'r').readlines() + return [str(x) for x in lst] + +def open_readlines(file_name: str) -> list[str]: + return open(file_name, 'r').readlines() From 2f79f3b3e94c458a0e69be5606814ff586aeaf7b Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:42 -0500 Subject: [PATCH 6/8] genutil.py: Use `die` in `make_numeric` There doesn't seem to be a need to manually write to stderr and exit here. Signed-off-by: Alexander McClain --- pysrc/genutil.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index 74464fdc..e388a1d9 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -357,11 +357,9 @@ def make_numeric(s: str, restriction_pattern=None) -> int: out = int(just_bits, 2) # msgb("MAKE BINARY NUMERIC", "%s -> %d" % (s,out)) elif make_numeric_old_decimal_pattern.match(s): - sys.stderr.write("0m should not occur. Rewrite files!") - sys.exit(1) + die("0m should not occur. Rewrite files!") elif make_numeric_old_binary_pattern.match(s): - sys.stderr.write("B' binary specifer should not occur. Rewrite files!") - sys.exit(1) + die("B' binary specifer should not occur. Rewrite files!") else: out = int(s) return out From 4183e7ca414ef56bf026db6c8edd06cc7bb8ce3e Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:42 -0500 Subject: [PATCH 7/8] genutil.py: Add types for remaining functions Signed-off-by: Alexander McClain --- pysrc/genutil.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index e388a1d9..274c0bd9 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -182,7 +182,7 @@ def print_resource_usage(i: str = ''): msge('RUSAGE: %s %s vmsize: %s' % (str(i), str(s), str(mem[0]))) -def flatten_sub(retlist, cur_list, rest): +def flatten_sub(retlist: list[Any], cur_list: list[Any], rest: list[list[Any]]): if len(rest) == 0: retlist.append(cur_list) return @@ -198,7 +198,7 @@ def flatten_sub(retlist, cur_list, rest): flatten_sub(retlist, cur_list, rest[1:]) -def flatten(list_with_sublists): +def flatten(list_with_sublists: list[list[Any]]) -> list[Any]: """Take a list with some possible sublists, and return a list of lists of flat lists. All possible combinations.""" retval = [] @@ -206,7 +206,7 @@ def flatten(list_with_sublists): return retval -def flatten_dict_sub(retlist, cur_dict, main_dict_with_lists, rest_keys): +def flatten_dict_sub(retlist: list[Any], cur_dict: dict[Any, Any], main_dict_with_lists: dict[Any, Any], rest_keys: list[Any]): if len(rest_keys) == 0: retlist.append(cur_dict) return @@ -225,7 +225,7 @@ def flatten_dict_sub(retlist, cur_dict, main_dict_with_lists, rest_keys): flatten_dict_sub(retlist, cur_dict, main_dict_with_lists, rest_keys[1:]) -def flatten_dict(dict_with_lists): +def flatten_dict(dict_with_lists: dict[Any, Any]) -> list[dict[Any, Any]]: """Take a dict with some possible sublists, and return a list of dicts where no rhs is a list. All possible combinations""" retval = [] @@ -234,7 +234,7 @@ def flatten_dict(dict_with_lists): return retval -def cmkdir(path_to_dir): +def cmkdir(path_to_dir: str): """Make a directory if it does not exist""" if not os.path.exists(path_to_dir): msgb("MKDIR", path_to_dir) From 2ced1ec5ce7d25a917f359d77039a76719e90673 Mon Sep 17 00:00:00 2001 From: Serial <69764315+Serial-ATA@users.noreply.github.com> Date: Wed, 15 Nov 2023 00:10:43 -0500 Subject: [PATCH 8/8] genutil.py: Add examples to flatten{_dict} This just makes it easier to visualize what these do. Signed-off-by: Alexander McClain --- pysrc/genutil.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pysrc/genutil.py b/pysrc/genutil.py index 274c0bd9..ce57b1f2 100755 --- a/pysrc/genutil.py +++ b/pysrc/genutil.py @@ -199,8 +199,15 @@ def flatten_sub(retlist: list[Any], cur_list: list[Any], rest: list[list[Any]]): def flatten(list_with_sublists: list[list[Any]]) -> list[Any]: - """Take a list with some possible sublists, and return a list of - lists of flat lists. All possible combinations.""" + """ + Take a list with some possible sublists, and return a list of + lists of flat lists. All possible combinations. + + Example: + >>> l = [ [1, 2], [3, 4] ] + >>> flatten(l) + [ [1, 3], [1, 4], [2, 3], [2, 4] ] + """ retval = [] flatten_sub(retval, [], list_with_sublists) return retval @@ -226,8 +233,15 @@ def flatten_dict_sub(retlist: list[Any], cur_dict: dict[Any, Any], main_dict_wit def flatten_dict(dict_with_lists: dict[Any, Any]) -> list[dict[Any, Any]]: - """Take a dict with some possible sublists, and return a list of - dicts where no rhs is a list. All possible combinations""" + """ + Take a dict with some possible sublists, and return a list of + dicts where no rhs is a list. All possible combinations. + + Example: + >>> d = { 'a': 1, 'b': [1, 2] } + >>> flatten_dict(d) + [{ 'a': 1, 'b': 1 }, { 'a': 1, 'b': 2 }] + """ retval = [] kys = list(dict_with_lists.keys()) flatten_dict_sub(retval, {}, dict_with_lists, kys)