Merge pull request #684 from binpash/future

New PaSh release

.github/workflows/tests.yaml

@@ -81,7 +81,9 @@ jobs:
  runs-on: ubuntu-latest
  if: github.event.pull_request.draft == false
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v3 
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
 
  - uses: ludeeus/action-shellcheck@master
  env:

compiler/annotations_utils/util_cmd_invocations.py

@@ -96,8 +96,8 @@ def construct_property_container_from_list_of_properties(list_properties):
 # this function is needed to wrap a node in `r_wrap`
 def to_arg_from_cmd_inv_with_io_vars_without_streaming_inputs_or_outputs_for_wrapping(cmd_inv, edges):
  # we already expand here
- whole_cmd = Arg(string_to_argument("\'"))
- arg_cmd_name = Arg(string_to_argument(cmd_inv.cmd_name))
+ whole_cmd = Arg.string_to_arg("\'")
+ arg_cmd_name = Arg.string_to_arg(cmd_inv.cmd_name)
  arg_flagoptions = []
  for flagoption in cmd_inv.flag_option_list:
  arg_flagoptions += to_arg_flagoption(flagoption, edges)
@@ -107,14 +107,14 @@ def to_arg_from_cmd_inv_with_io_vars_without_streaming_inputs_or_outputs_for_wra
  all_cmd_parts_arg.extend(arg_operands)
  for part in all_cmd_parts_arg:
  whole_cmd.concatenate(part)
- whole_cmd.concatenate(Arg(string_to_argument("\'")))
+ whole_cmd.concatenate(Arg.string_to_arg("\'"))
  return whole_cmd
 
 def to_arg_flagoption(flagoption, edges):
  if isinstance(flagoption, Flag):
- return [Arg(string_to_argument(flagoption.get_name()))]
+ return [Arg.string_to_arg(flagoption.get_name())]
  elif isinstance(flagoption, OptionWithIO):
- opt_name_arg = Arg(string_to_argument(flagoption.get_name()))
+ opt_name_arg = Arg.string_to_arg(flagoption.get_name())
  opt_arg_arg = translate_io_var_to_arg_if_applicable(flagoption.get_arg(), edges)
  return [opt_name_arg, opt_arg_arg]
 

compiler/ast_to_ir.py

@@ -1,12 +1,12 @@
+import subprocess
+
+from shasta.ast_node import *
+from sh_expand.expand import expand_command, ExpansionState
+
 from shell_ast.ast_util import *
 from ir import *
-from shell_ast.ast_node import *
-from shell_ast.ast_node_c import *
 from util import *
 from parse import from_ast_objects_to_shell
-from shell_ast.expand import *
-import subprocess
-import config
 
 ## TODO: Separate the ir stuff to the bare minimum and 
 ## try to move this to the shell_ast folder.
@@ -40,22 +40,25 @@
  lambda ast_node: compile_node_redir_subshell(ast_node, fileIdGen, config)),
  "Background": (lambda fileIdGen, config:
  lambda ast_node: compile_node_background(ast_node, fileIdGen, config)),
- "Defun": (lambda fileIdGen, config:
- lambda ast_node: compile_node_defun(ast_node, fileIdGen, config)),
  "For": (lambda fileIdGen, config:
  lambda ast_node: compile_node_for(ast_node, fileIdGen, config))
  }
 
 
-def compile_asts(ast_objects, fileIdGen, config):
+def compile_asts(ast_objects: "list[AstNode]", fileIdGen, config):
  compiled_asts = []
  acc_ir = None
  for i, ast_object in enumerate(ast_objects):
  # log("Compiling AST {}".format(i))
  # log(ast_object)
+ assert(isinstance(ast_object, AstNode))
 
  ## Compile subtrees of the AST to out intermediate representation
- expanded_ast = expand_command(ast_object, config)
+ ## KK 2023-05-25: Would we ever want to pass this state to the expansion 
+ ## of the next object? I don't think so.
+ exp_state = ExpansionState(config['shell_variables'])
+ expanded_ast = expand_command(ast_object, exp_state)
+ # log("Expanded:", expanded_ast)
  compiled_ast = compile_node(expanded_ast, fileIdGen, config)
 
  # log("Compiled AST:")
@@ -112,14 +115,12 @@ def compile_node_pipe(ast_node, fileIdGen, config):
  ## pipeline) the compiled_pipe_nodes should always
  ## be one IR
  compiled_ir = compiled_pipe_nodes[0]
- ## Note: Save the old ast for the end-to-end prototype
- old_ast_node = make_kv(ast_node.construct.value, [ast_node.is_background, ast_node.items])
- compiled_ir.set_ast(old_ast_node)
+ ## Save the old ast for the end-to-end prototype
+ old_untyped_ast_node = ast_node.json()
+ compiled_ir.set_ast(old_untyped_ast_node)
  ## Set the IR background so that it can be parallelized with
  ## the next command if the pipeline was run in background
  compiled_ir.set_background(ast_node.is_background)
- ## TODO: If the pipeline is in background, I also have to
- ## redirect its stdin, stdout
  compiled_ast = compiled_ir
  return compiled_ast
 
@@ -145,70 +146,39 @@ def combine_pipe(ast_nodes):
  return [combined_nodes]
 
 def compile_node_command(ast_node, fileIdGen, config):
- construct_str = ast_node.construct.value
- old_ast_node = make_kv(construct_str, [ast_node.line_number,
- ast_node.assignments, ast_node.arguments, ast_node.redir_list])
-
- ## TODO: Do we need the line number?
-
  ## Compile assignments and redirection list
  compiled_assignments = compile_assignments(ast_node.assignments, fileIdGen, config)
  compiled_redirections = compile_redirections(ast_node.redir_list, fileIdGen, config)
 
- ## If there are no arguments, the command is just an
- ## assignment
- ##
- ## TODO: The if-branch of this conditional should never be possible since the preprocessor
- ## wouldn't replace a call without arguments (simple assignment).
- ##
- ## Also the return is not in the correct indentation so probably it never gets called
- ## in our tests.
- ##
- ## We should remove it and add the following assert:
- ## assert len(ast_node.arguments) > 0
- if(len(ast_node.arguments) == 0):
- ## Just compile the assignments. Specifically compile the
- ## assigned values, because they might have command
- ## substitutions etc..
- compiled_ast = make_kv(construct_str, [ast_node.line_number] +
- [compiled_assignments] + [ast_node.arguments, compiled_redirections])
- else:
- arguments = ast_node.arguments
- command_name = arguments[0]
- options = compile_command_arguments(arguments[1:], fileIdGen, config)
+ ## This should never be possible since the preprocessor
+ ## wouldn't replace a call without arguments (simple assignment).
+ assert len(ast_node.arguments) > 0
+
+ arguments = ast_node.arguments
+ command_name = arguments[0]
+ options = compile_command_arguments(arguments[1:], fileIdGen, config)
+
+ try:
+ ## If the command is not compileable to a DFG the following call will fail
+ ir = compile_command_to_DFG(fileIdGen,
+ command_name,
+ options,
+ redirections=compiled_redirections)
+ compiled_ast = ir
+ except ValueError as err:
+ log("Command not compiled to DFG:", err)
+ ## TODO: Maybe we want to fail here instead of waiting for later?
+ ## Is there any case where a non-compiled command is fine?
+ # log(traceback.format_exc())
+ compiled_arguments = compile_command_arguments(arguments, fileIdGen, config)
+ compiled_ast = make_kv(type(ast_node).NodeName,
+ [ast_node.line_number, compiled_assignments,
+ compiled_arguments, compiled_redirections])
 
- ## Question: Should we return the command in an IR if one of
- ## its arguments is a command substitution? Meaning that we
- ## will have to wait for its command to execute first?
- ##
- ## ANSWER: Kind of. If a command has a command substitution or
- ## anything that evaluates we should add it to the IR, but we
- ## should also make sure that its category is set to the most
- ## general one. That means that it can be executed
- ## concurrently with other commands, but it cannot be
- ## parallelized.
- try:
- ## If the command is not compileable to a DFG the following call will fail
- ir = compile_command_to_DFG(fileIdGen,
- command_name,
- options,
- redirections=compiled_redirections)
- compiled_ast = ir
- except ValueError as err:
- ## TODO: Delete this log from here
- log(err)
- ## TODO: Maybe we want to fail here instead of waiting for later?
- ## Is there any case where a non-compiled command is fine?
- # log(traceback.format_exc())
- compiled_arguments = compile_command_arguments(arguments, fileIdGen, config)
- compiled_ast = make_kv(construct_str,
- [ast_node.line_number, compiled_assignments,
- compiled_arguments, compiled_redirections])
-
- return compiled_ast
+ return compiled_ast
 
 def compile_node_and_or_semi(ast_node, fileIdGen, config):
- compiled_ast = make_kv(ast_node.construct.value,
+ compiled_ast = make_kv(type(ast_node).NodeName,
  [compile_node(ast_node.left_operand, fileIdGen, config),
  compile_node(ast_node.right_operand, fileIdGen, config)])
  return compiled_ast
@@ -221,7 +191,7 @@ def compile_node_redir_subshell(ast_node, fileIdGen, config):
  ## the IR accordingly
  compiled_ast = compiled_node
  else:
- compiled_ast = make_kv(ast_node.construct.value, [ast_node.line_number,
+ compiled_ast = make_kv(type(ast_node).NodeName, [ast_node.line_number,
  compiled_node, ast_node.redir_list])
 
  return compiled_ast
@@ -248,24 +218,10 @@ def compile_node_background(ast_node, fileIdGen, config):
 
  return compiled_ast
 
-def compile_node_defun(ast_node, fileIdGen, config):
- ## It is not clear how we should handle functions.
- ##
- ## - Should we transform their body to IR?
- ## - Should we handle calls to the functions as commands?
- ##
- ## It seems that we should do both. But we have to think if
- ## this introduces any possible problem.
-
- ## TODO: Investigate whether it is fine to just compile the
- ## body of functions.
- compiled_body = compile_node(ast_node.body, fileIdGen, config)
- return make_kv(construct, [ast_node.line_number, ast_node.name, compiled_body])
-
 def compile_node_for(ast_node, fileIdGen, config):
  ## TODO: Investigate what kind of check could we do to make a for
  ## loop parallel
- compiled_ast = make_kv(ast_node.construct.value,
+ compiled_ast = make_kv(type(ast_node).NodeName,
  [ast_node.line_number,
  compile_command_argument(ast_node.argument, fileIdGen, config),
  compile_node(ast_node.body, fileIdGen, config),
@@ -352,25 +308,25 @@ def expand_command_argument(argument, config):
 ## This function compiles an arg char by recursing if it contains quotes or command substitution.
 ##
 ## It is currently being extended to also expand any arguments that are safe to expand.
-def compile_arg_char(arg_char, fileIdGen, config):
+def compile_arg_char(arg_char: ArgChar, fileIdGen, config):
  ## Compile the arg char
- key, val = get_kv(arg_char)
- if (key in ['C', # Single character
-  'E']): # Escape
+ if isinstance(arg_char, CArgChar) \
+  or isinstance(arg_char, EArgChar):
+ # Single character or escape
  return arg_char
- elif (key == 'B'):
+ elif isinstance(arg_char, BArgChar):
  ## TODO: I probably have to redirect the input of the compiled
  ## node (IR) to be closed, and the output to be
  ## redirected to some file that we will use to write to
  ## the command argument to complete the command
  ## substitution.
- compiled_node = compile_node(val, fileIdGen, config)
- return [key, compiled_node]
- elif (key == 'Q'):
- compiled_val = compile_command_argument(val, fileIdGen, config)
- return [key, compiled_val]
+ arg_char.node = compile_node(arg_char.node, fileIdGen, config)
+ return arg_char
+ elif isinstance(arg_char, QArgChar):
+ arg_char.arg = compile_command_argument(arg_char.arg, fileIdGen, config)
+ return arg_char
  else:
- log("Unknown arg_char:", arg_char)
+ log(f'Unknown arg_char: {arg_char}')
  ## TODO: Complete this
  return arg_char
 
@@ -390,11 +346,9 @@ def compile_assignments(assignments, fileIdGen, config):
  return compiled_assignments
 
 def compile_redirection(redirection, fileIdGen, config):
- redir_type = redirection[0]
- redir_subtype = redirection[1][0]
- stream_id = redirection[1][1]
- file_arg = compile_command_argument(redirection[1][2], fileIdGen, config)
- return [redir_type, [redir_subtype, stream_id, file_arg]]
+ file_arg = compile_command_argument(redirection.arg, fileIdGen, config)
+ redirection.arg = file_arg
+ return redirection
 
 def compile_redirections(redirections, fileIdGen, config):
  compiled_redirections = [compile_redirection(redirection, fileIdGen, config)