From 369aa7d6754a5be3914014c62e3f5ab4ec266bd2 Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Fri, 11 Oct 2024 17:10:00 +0200 Subject: [PATCH] Updated Franca2IFEX with table-driven translation approach Signed-off-by: Gunnar Andersson --- ifex/model/ifex_ast.py | 4 +- ifex/model/ifex_ast_construction.py | 4 +- other/franca/franca_to_ifex.py | 231 ++++++++++++++++++ other/franca/rule_translator.py | 357 ++++++++++++++++++++++++++++ 4 files changed, 592 insertions(+), 4 deletions(-) create mode 100644 other/franca/franca_to_ifex.py create mode 100644 other/franca/rule_translator.py diff --git a/ifex/model/ifex_ast.py b/ifex/model/ifex_ast.py index a55a3a5..4835caa 100644 --- a/ifex/model/ifex_ast.py +++ b/ifex/model/ifex_ast.py @@ -683,7 +683,7 @@ class FundamentalTypes: ctypes = [ # name, description, min value, max value - ["set", "A set of fundamental unsigned 8-bit integer", "N/A", "N/A"], - ["map", "A key-value mapping type", "N/A", "N/A"], + ["set", "A set (unique values), each of the same type. Format: set", "N/A", "N/A"], + ["map", "A key-value mapping type. Format: map", "N/A", "N/A"], ["opaque", "Indicates a complex type which is not explicitly defined in this context.", "N/A","N/A"] ] diff --git a/ifex/model/ifex_ast_construction.py b/ifex/model/ifex_ast_construction.py index 79d9b08..0d0a466 100644 --- a/ifex/model/ifex_ast_construction.py +++ b/ifex/model/ifex_ast_construction.py @@ -76,7 +76,7 @@ def ifex_ast_to_dict(node, debug_context="") -> OrderedDict: """Given a root node, return a key-value mapping dict (which represents the YAML equivalent). The function is recursive. """ if node is None: - raise TypeError(f"None-value should not be passed to function, {debug_context=}") + raise TypeError(f"None-value should not be passed to function, parent debug: {debug_context=}") # Strings and Ints are directly understood by the YAML output printer so just put them in. if is_simple_type(type(node)): @@ -84,7 +84,7 @@ def ifex_ast_to_dict(node, debug_context="") -> OrderedDict: # In addition to dicts, we might have python lists, which will be output as lists in YAML #if is_list(node) or type(node) == list: - if type(node) == list: + if type(node) is list: ret = [] for listitem in node: ret.append(ifex_ast_to_dict(listitem, debug_context=str(node))) diff --git a/other/franca/franca_to_ifex.py b/other/franca/franca_to_ifex.py new file mode 100644 index 0000000..f4a3911 --- /dev/null +++ b/other/franca/franca_to_ifex.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MBition GmbH. +# SPDX-License-Identifier: MPL-2.0 + +# This file is part of the IFEX project +# vim: tw=120 ts=4 et + +# Have to define a search path to submodule to make this work (might be rearranged later) +import os +import sys +mydir = os.path.dirname(__file__) +for p in ['pyfranca', 'pyfranca/pyfranca']: + if p not in sys.path: + sys.path.append(os.path.join(mydir,p)) + +import ifex.model.ifex_ast as ifex +import other.franca.pyfranca.pyfranca as pyfranca +import other.franca.rule_translator as m2m +import pyfranca.ast as franca +import re + +from ifex.model.ifex_ast_construction import add_constructors_to_ifex_ast_model, ifex_ast_as_yaml +from other.franca.rule_translator import Preparation, Constant, Unsupported + +def translate_type_name(francaitem): + return translate_type(francaitem) + +def concat_comments(list): + return "\n".join(list) + +# If enumerator values are not given, we must use auto-generated values. +# IFEX model requires all enumerators to be given values. +enum_count = -1 +def reset_enumerator_counter(): + #print("***Resetting enum counter") + global enum_count + enum_count = -1 + +def translate_enumerator_value(franca_int_value): + if franca_int_value is None: + global enum_count + enum_count = enum_count + 1 + return enum_count + return translate_simple_constant(franca_int_value) + +# Integer value is represented by an instance of IntegerValue class type, which has a "value" member. +# Same principle for: BooleanValue, FloatValue, DoubleValue, and StringValue: We expect +# that the .value field has the corresponding python type and that this is also what we expect +# in the corresponding field on the IFEX AST. +# FIXME: Confirm that constants in Binary, Hexadecimal, etc. will also translate correctly? +def translate_simple_constant(franca_int_value): + return franca_int_value.value + +mapname = "UNDEF" +def map_name(x): + global mapname + map_name = x + return + +keytype = "UNDEF" +def map_keytype(x): + global keytype + # Input attribute will be a franca.Reference to the actual type + keytype = translate_type_name(x) + +valuetype = "UNDEF" +def map_valuetype(x): + global valuetype + valuetype = translate_type_name(x) + +def assemble_map_type(): + global valuetype, keytype + return f"map<{keytype},{valuetype}>" + + +# Tip: This translation table format is described in more detail in rule_translator.py +franca_to_ifex_mapping = { + + 'global_attribute_map': { + # Franca-name : IFEX-name + 'comments' : 'description', # FIXME allow transform also here, e.g. concat comments + 'extends' : None, # TODO + 'flags' : None, + 'type' : ('datatype', translate_type_name), + }, + + 'type_map': { + (franca.Interface, ifex.Interface) : [ + ('maps', 'typedefs'), + ('manages', Unsupported), + ], + (franca.Package, ifex.Namespace) : [ + # TEMPORARY: Translates only the first interface + ('interfaces', 'interface', lambda x: x[0]), + ('typecollections', 'namespaces'), + ], + (franca.Method, ifex.Method) : [ + ('in_args', 'input'), + ('out_args', 'output'), + ('namespace', None) ], + (franca.Argument, ifex.Argument) : [ + ('type', 'datatype', translate_type_name), ], + (franca.Enumeration, ifex.Enumeration) : [ + Preparation(reset_enumerator_counter), + ('enumerators', 'options'), + ('extends', Unsupported), + + # Franca only describes integer-based Enumerations but IFEX can use any type. + # In the translation we hard-code the enumeration datatype to be int32, which ought to + # normally work. + (Constant('int32'), 'datatype') + ], + (franca.Enumerator, ifex.Option) : [ + ('value', 'value', translate_enumerator_value) + ], + (franca.TypeCollection, ifex.Namespace) : [ + # FIXME - these translations are valid also for Interfaces -> move to global + ('arrays', 'typedefs'), + ('maps', 'typedefs'), + ('enumerations', 'enumerations'), + ('structs', 'structs'), + ('unions', None), # TODO - use the variant type on IFEX side, need to check its implementation first + ], + (franca.Struct, ifex.Struct) : [ + ('fields', 'members') + ], + (franca.StructField, ifex.Member) : [] , + (franca.Array, ifex.Typedef) : [], + (franca.Typedef, ifex.Typedef) : [], + (franca.Map, ifex.Typedef) : [ + ('key_type', None, map_keytype), + ('value_type', None, map_valuetype), + (assemble_map_type, 'datatype') + ], + (franca.Attribute, ifex.Property) : [], + (franca.Import, ifex.Include) : [], + } + } + +# --- Map fundamental/built-in types --- + +type_translation = { + franca.Boolean : "boolean", + franca.ByteBuffer : "uint8[]", + franca.Double : "double", + franca.Float : "float", + franca.Int8 : "int8", + franca.Int16 : "int16", + franca.Int16 : "int16", + franca.Int32 : "int32", + franca.Int64 : "int64", + franca.String : "string", + franca.UInt8 : "uint8", + franca.UInt16 : "uint16", + franca.UInt32 : "uint32", + franca.UInt64 : "uint64", + franca.ComplexType : "opaque", # FIXME this is actually a struct reference? + franca.Map : "opaque", # FIXME maps shall be supported +} + +# ---------------------------------------------------------------------------- +# HELPER FUNCTIONS +# ---------------------------------------------------------------------------- + +def translate_type(t): + + # Special case: For references to complex types we want to refer to the original type name, since IFEX supports + # named type definitions as well. The type ought to be translated into a named typedef (or enum or struct, etc.) in + # the IFEX representation also. We don't need to de-reference it once more and end up with the inner definition of + # the typedef, since IFEX supports named type definitions as well -> so don't recurse to figure out the inner type. + if type(t) is franca.Reference and type(t.reference) in [franca.Array, franca.Struct, franca.Enumeration, + franca.Typedef, franca.Map]: + return t.reference.name + + # Other references in a Franca AST are just some level of indirection -> Recurse on the actual type. + # TODO: Check if this case still happens, considering previous lines? + if type(t) is franca.Reference: + return translate_type(t.reference) + + # This case can happen for arrays for plain-types that are defined directly, without a named typedef. + # -> Translate the array's inner simple type, and add array to it. + if type(t) is franca.Array: + return translate_type(t.type) + '[]' + + # TODO This case is now probably redundant but let's come back to the comment about how to use qualified names + if type(t) is franca.Enumeration: + return t.name # FIXME use qualified name _, or change in the other place + + # Plain type -> use lookup table and if that fails return the original for now + t2 = type_translation.get(type(t)) + return t2 if t2 is not None else t + + +# Rename fidl to ifex, for imports +def ifex_import_ref_from_fidl(fidl_file): + return re.sub('.fidl$', '.ifex', fidl_file) + + +# Calling the pyfranca parser to build the Franca AST +def parse_franca(fidl_file): + processor = pyfranca.Processor() + return processor.import_file(fidl_file) # This returns the top level package + + +# --- Script entry point --- + +if __name__ == '__main__': + + if len(sys.argv) != 2: + print(f"Usage: python {os.path.basename(__file__)} ") + sys.exit(1) + + # Add the type-checking constructor mixin + # FIXME Add this back later for strict checking + #add_constructors_to_ifex_ast_model() + + try: + # Parse franca input and create franca AST (top node is the Package definition) + franca_package = parse_franca(sys.argv[1]) + + # Convert Franca AST to IFEX AST + ifex_ast = m2m.transform(franca_to_ifex_mapping, franca_package) + final_ast = ifex.AST(namespaces = [ifex_ast]) + + # Output as YAML + print(ifex_ast_as_yaml(final_ast)) + + except FileNotFoundError: + log("ERROR: File not found") + except Exception as e: + raise(e) + log("ERROR: An unexpected error occurred: {}".format(e)) diff --git a/other/franca/rule_translator.py b/other/franca/rule_translator.py new file mode 100644 index 0000000..33d6fa8 --- /dev/null +++ b/other/franca/rule_translator.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MBition GmbH. +# SPDX-License-Identifier: MPL-2.0 + +# This file is part of the IFEX project +# vim: tw=120 ts=4 et + +""" +## rule_translator.py + +rule_translator implements a generic model-to-model translation function used to copy-and-transform values from one +hierarchical AST representation to another. It is driven by an input data structure that describes the translation +rules to follow, and implemented by a generic function that can be used for many types of transformation. + +## Translation definition + +- The data structure (table) describes the mapping from the types (classes) of the input AST to the output AST +- Every type that is found in the input AST *should* have a mapping. There is not yet perfect error + reporting if something is missing, but it might be improved. +- For each class, the equivalent member variable that need to be mapped is listed. +- Member variable mappings are optional because any variable with Equal Name on each object + will be copied automatically (with possible transformation, *if* the input data is listed as a + complex type). +- Each attribute mapping can also optionally state the name of a transformation function (or lambda) + If no function is stated, the value will be mapped directly. Mapping means to follow the transformation + rules of the type-mapping table *if* the value is an instance of an AST class, and in other + cases simly copy the value as it is (typically a string, integer, etc.) +- In addition, it is possible to define global name-translations for attributes that are + equivalent but have different name in the two AST class families. +- To *ignore* an attribute, map it to the None value. + +See example table in rule_translator.py source for more information, or any of the implemented programs. + +""" + +from collections import OrderedDict +from dataclasses import dataclass +import os +import re +import sys + + +# ----------------------------------------------------------------------------- +# Translation Table Helper-objects +# ----------------------------------------------------------------------------- +# +# These classes help the table definition by defining something akin to a small DSL (Domain Specific Language) that aids +# us in expressing the translation table with things like "List Of" a certain type, or "Constant" when a given value is +# always supposed to be used, etc. Some ideas of representing the rules using python builtin primitives do not work. +# For example, using '[SomeClass]' to represent and array (list) of SomeClass is a valid statement in general, but +# does not work in our current translation table format because it is a key in a dict. Plain arrays are not a hashable +# value and therefore can't be used as a key. Similarly list(SomeClass) -> a type is not hashable. + +# (Use frozen dataclasses to make them hashable. The attributes are given values at construction time only.) + +# Map to Unsupported to make a node type unsupported +@dataclass(frozen=True) +class Unsupported: + pass + +# To insert the same value for all translations +@dataclass(frozen=True) +class Constant: + const_value: int # Temporary, might be reassigned another type + +# To wrap a function that will be called at this stage in the attribute mapping +@dataclass(frozen=True) +class Preparation: + func: callable + pass + + +# ----------------------------------------------------------------------------- +# Translation Table - Example, not used. The table be provided instead by the program +# that uses this module) +# ----------------------------------------------------------------------------- + +example = """ +example_mapping = { + + # global_attribute_map: Attributes with identical name are normally automatically mapped. If attributes have + # different names we can avoid repeated mapping definitions by still defining them as equivalent in the + # global_attribute_map. Attributes defined here ill be mapped in *all* classes. Note: To ignore an attribute, + # map it to None! + + 'global_attribute_map': { + # (Attribute in FROM-class on the left, attribute in TO-class on the right) + 'this' : 'that', + 'something_not_needed' : None + }, + + # type_map: Here follows Type (class) mappings with optional local attribute mappings + 'type_map': { + (inmodule.ASTClass1, outmodule.MyASTClass) : + # followed by array of attribute mapping + + # Here an array of tuples is used. (Format is subject to change) + # (FROM-class on the left, TO-class on the right) + # *optional* transformation function as third argument + # Special case: Preparation(myfunc), which calls any function at that point in the list + [ + ('thiss', 'thatt'), + ('name', 'thename', capitalize_name_string), + Preparation(pre_counter_init), + ('zero_based_counter', 'one_based_counter', lambda x : x + 1), + ('thing', None) + ] + + # Equally named types have no magic - it is still required to + # define that they shall be transformed/copied over. + (inmodule.AnotherType, outmodule.Anothertype), [ + # Use a Constant object to always set the same value in target attribute + (Constant('int32'), 'datatype') + ], + # ListOf and other features are not yet implemented -> when the need arises + } +} +""" + +# ---------------------------------------------------------------------------- +# HELPER FUNCTIONS +# ---------------------------------------------------------------------------- + +# TODO - add better logging +def _log_if(condition, level, string): + if condition: + _log(level, string) + +def _log(level, string): + #if level == "DEBUG": + # print(f"{level}: {string}") + #if level == "INFO": + # print(f"{level}: {string}") + #if level == "WARN": + # print(f"{level}: {string}") + if level == "ERROR": + print(f"{level}: {string}") + #pass + + +def is_builtin(x): + return x.__class__.__module__ == 'builtins' + +# This function is used by the general translation to handle multiple mappings with the same target attribute. +# We don't want to overwrite and destroy the previous value with a new one, and if the target is a list +# then it is useful to be able to add to that list at multiple occasions -> append to it. +def set_attr(attrs_dict, attr_key, attr_value): + if attr_key in attrs_dict: + value = attrs_dict[attr_key] + + # If it's a list, we want to add to it instead of overwriting: + if isinstance(value, list): + # We don't have lists in lists, but it can happen that we get more than one list + if isinstance(attr_value, list): + value = value + attr_value + else: + value.append(attr_value) + attrs_dict[attr_key] = value + return + # If it's set to None by an earlier step -> just overwrite + elif value is None: + attrs_dict[attr_key] = attr_value + else: + _log("ERROR", f"""Attribute {attr_key} already has a scalar (non-list) value. Check for multiple translations + mapping to this one. We should not overwrite it, and since it is not a list type, we can't append.""") + _log("DEBUG", f"Target value {value} was ignored.") + return + + # If it's a new assignment, go ahead + attrs_dict[attr_key] = attr_value + +# ---------------------------------------------------------------------------- +# --- MAIN conversion function --- +# ---------------------------------------------------------------------------- + +# Here we use a helper function to allow one, two, or three values in the tuple. +# With normal decomposition of a tuple, only the last can be optional, like this: +# first, second, *maybe_more = (...tuple...) +# But the table allows a single item, like Preparation() which is not a tuple, +# so let's add some logic. +# This function always returns a full 4-value tuple, for processing in the +# main loop: +# Returns: (preparation_function, input_arg, output_arg, field_transform) + +def eval_mapping(type_map_entry): + if isinstance(type_map_entry, Preparation): + # Return the function that is wrapped inside Preparation() + return (type_map_entry.func, None, None, None) + else: # 3 or 4-value tuple is expected (field_transform is optional) + input_arg, output_arg, *field_transform = type_map_entry + # Unwrap array and use identity-function if no transformation required + field_transform = field_transform[0] if field_transform else lambda _ : _ + return (None, input_arg, output_arg, field_transform) + + +# Additional named helpers to make logic very visible. +# (We're at this time not concerned with performance hit of calling some extra functions) +def dataclass_has_field(_class, attr): + return attr in _class.__dataclass_fields__ + +# The following two functions are mutually recursive (transform -> transform_value_common -> transform) +# but you can think of it primarily as the main function, transform(), calling itself as it +# descends down the tree of nodes/values that neeed converting. +# This _common function is here only to avoid repeated code for the type-specific handling +def transform_value_common(mapping_table, value, field_transform): + + # OrderedDict is used at least by Franca AST -> return a list of transformed items + if isinstance(value, OrderedDict): + if len(value.items()) == 0: + name = "" + try: + name = value.name + except: + pass + _log("DEBUG", f"Empty OrderedDict for {value=} {name=} {field_transform=}") + value = [] + else: + value = field_transform([transform(mapping_table, item) for name, item in value.items()]) + + # A list in input yields a list in output, transforming each item + # (not used by Franca parser, but others might) + elif isinstance(value, list): + if len(value) == 0: + log("DEBUG", f"Empty list: {value=}") + value = None + else: + log("DEBUG", f"Non-empty list: {value=}") + value = field_transform([transform(mapping_table, item) for item in value]) + + # Plain attribute -> use transformation function if it was defined + else: + value = field_transform(value) + + return value + + +def transform(mapping_table, input_obj): + + # Builtin types (str, int, ...) are assumed to be just values that shall be copied without any change + if is_builtin(input_obj): + return input_obj + + # Find a translation rule in the metadata + for (from_class, to_class), mappings in mapping_table['type_map'].items(): + + # Use linear-search in mapping table until we find something matching input object. + # Since the translation table is reasonably short, it should be OK for now. + if from_class != input_obj.__class__: + continue + + # Continuing here with a matching mapping definition... + _log("INFO", f"Type mapping found: {from_class=} -> {to_class=}") + + # Comment: Here we might create an empty instance of the class and fill it with values using setattr(), but + # that won't work since the redesign using dataclasses. The AST classes now have a default constructor that + # requires all mandatory fields to be specified when an instance is created. Therefore we are forced to + # follow this approach: Gather all attributes in a dict and pass it into the constructor at the end using + # python's dict to keyword-arguments capability. + + attributes = {} + + # To remember the args we have converted + done_attrs = set() + + # First loop: Perform explicitly defined attribute conversions listed in each entry + + for preparation_function, input_attr, output_attr, field_transform in [eval_mapping(m) for m in mappings]: + _log("INFO", f"Attribute mapping found: {input_attr=} -> {output_attr=} with {field_transform=}") + + # TODO: It should be possible to let the preparation_function be a closure, with predefined parameters. + # Also to be investigated: Consider if it's better to go back to eval_mapping returning the + # function-wrapper object (Preparation) and not just the function. + + # Call preparation function, if given. + if preparation_function is not None: + preparation_function() + # preparation_function function always has its own line in mapping table, so skip to next line + continue + + # In the case the rule is set to output_attr==None, then no output_attr is written. + # But if field_transform function is defined, the function is still called with the input_attr value. This can be + # used to store/manipulate that value for later use. There is no return value copied to any output_attr. + if output_attr is None: + field_transform(getattr(input_obj, input_attr)) + _log("DEBUG", f"{input_attr=} for {type(input_obj)} was mapped to None") + continue + + if output_attr is Unsupported: + value = getattr(input_obj, input_attr) + _log_if(value is not None, "ERROR", f"{type(input_obj)}:{input_obj.name} has an attribute for '{input_attr}' but that feature is unsupported. ({value=})") + continue + + # If the input_attr is set to a function instead of an attribute name, then the result of the function is + # copied to the output_attr. + if callable(input_attr): + set_attr(attributes, output_attr, input_attr()) # <- note input_attr called as a function + continue + + # If Constant-object, copy the value + if isinstance(input_attr, Constant): + set_attr(attributes, output_attr, input_attr.const_value) + continue + + # (else: Get input value and copy it, after transforming as necessary) + set_attr(attributes, output_attr, transform_value_common(mapping_table, getattr(input_obj, input_attr), + field_transform)) + + # Mark this attribute as handled + done_attrs.add(input_attr) + + + # Second loop: Any attributes that have the _same name_ in the input and output classes are assumed to be + # mappable to each other. Thus, identical names do not need to be listed in the type-specific translation table + # unless they need a custom transformation. So here we can find all matching names and map them (with recursive + # transformation, as needed), but we of course skip all attributes that have been handled already by a + # type-specific rule (done_attrs). + + # In addition: global_attribute_map defines which attribute names shall be considered the same in all objects. + + global_attribute_map = mapping_table['global_attribute_map'] + + # Checking all fields in input object, except fields that were handled and stored in done_attrs + for attr, value in vars(input_obj).items(): + + if attr in done_attrs: + continue + + glob_transform = lambda _ : _ # Default is no-op + + # Translate attribute name according to global rules, if defined. + # ... and extract the transform function if it's given + if attr in global_attribute_map: + attr = global_attribute_map.get(attr) + if isinstance(attr, tuple): + # Tuple with transformation function + glob_transform = attr[1] + attr = attr[0] + + if dataclass_has_field(to_class, attr): + _log("DEBUG", f"Performing global or same-name auto-conversion for {attr=} from {from_class.__name__} to {to_class.__name__}\n") + set_attr(attributes, attr, transform_value_common(mapping_table, value, glob_transform)) + continue + + _log_if(attr is not None, "WARN", f"Attribute '{attr}' from Input AST:{input_obj.__class__.__name__} was not used in IFEX:{to_class.__name__}") + + + # Both loops done. Attributes now filled with key/values. Instantiate "to_class" object and return it. + _log("DEBUG", f"Creating and returning object of type {to_class} with {attributes=}") + try: + obj = to_class(**attributes) + return obj + except Exception as e: + _log("ERROR", f"Could not create object of type {to_class} with {attributes=}.\n(Was mapped from type {from_class=}). ") + + + no_rule = f"no translation rule found for object {input_obj} of class {input_obj.__class__.__name__}" + _log("ERROR:", no_rule) + raise TypeError(no_rule)