diff --git a/common/util/BUILD b/common/util/BUILD index 1235f7f06..dceeb89af 100644 --- a/common/util/BUILD +++ b/common/util/BUILD @@ -41,6 +41,16 @@ cc_library( deps = ["@com_google_absl//absl/base:core_headers"], ) +cc_library( + name = "cmd_positional_arguments", + srcs = ["cmd_positional_arguments.cc"], + hdrs = ["cmd_positional_arguments.h"], + deps = [ + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "subcommand", srcs = ["subcommand.cc"], diff --git a/common/util/cmd_positional_arguments.cc b/common/util/cmd_positional_arguments.cc new file mode 100644 index 000000000..8f82579e2 --- /dev/null +++ b/common/util/cmd_positional_arguments.cc @@ -0,0 +1,127 @@ +// Copyright 2017-2020 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/util/cmd_positional_arguments.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" + +namespace verible { + +// Sets a file argument. +absl::Status CmdPositionalArguments::SetFile(absl::string_view file) { + files_.push_back(file); + return absl::OkStatus(); +} + +// Sets a define arguments. +absl::Status CmdPositionalArguments::SetDefine( + std::pair define) { + defines_.push_back(define); + return absl::OkStatus(); +} + +// Sets a include directory argument. +absl::Status CmdPositionalArguments::SetIncludeDir( + absl::string_view include_dir) { + include_dirs_.push_back(include_dir); + return absl::OkStatus(); +} + +// Gets include directories arguments. +std::vector CmdPositionalArguments::GetIncludeDirs() const { + return include_dirs_; +} + +// Gets macro defines arguments. +std::vector> +CmdPositionalArguments::GetDefines() const { + return defines_; +} + +// Gets SV files arguments. +std::vector CmdPositionalArguments::GetFiles() const { + return files_; +} + +// Main function that parses arguments and add them to the correct data memeber. +absl::Status CmdPositionalArguments::ParseArgs() { + // Positional arguments types: + // 1) + // 2) +define+[=] + // 3) +incdir+ + + int argument_index = 0; + for (absl::string_view argument : all_args_) { + if (!argument_index) { + argument_index++; + continue; + } // all_args_[0] is the tool's name, which can be skipped. + if (argument[0] != '+') { // the argument is a SV file name. + if (auto status = SetFile(argument); !status.ok()) + return status; // add it to the file arguments. + } else { // it should be either a define or incdir. + std::vector argument_plus_splitted = + absl::StrSplit(argument, absl::ByChar('+'), absl::SkipEmpty()); + if (argument_plus_splitted.size() < 2) { + // Unknown argument. + return absl::InvalidArgumentError("Unkown argument."); + } + absl::string_view plus_argument_type = argument_plus_splitted[0]; + if (absl::StrContains(plus_argument_type, "define")) { + // define argument. + int define_argument_index = 0; + for (absl::string_view define_argument : argument_plus_splitted) { + if (!define_argument_index) { + define_argument_index++; + continue; + } + // define_argument is something like =. + std::pair macro_pair = + absl::StrSplit( + define_argument, absl::ByChar('='), + absl::SkipEmpty()); // parse the macro name and value. + if (auto status = CmdPositionalArguments::SetDefine(macro_pair); + !status.ok()) + return status; // add the define argument. + define_argument_index++; + } + } else if (absl::StrContains(plus_argument_type, "incdir")) { + // incdir argument. + int incdir_argument_index = 0; + for (absl::string_view incdir_argument : argument_plus_splitted) { + if (!incdir_argument_index) { + incdir_argument_index++; + continue; + } + if (auto status = + CmdPositionalArguments::SetIncludeDir(incdir_argument); + !status.ok()) + return status; // add file argument. + incdir_argument_index++; + } + } else { + return absl::InvalidArgumentError("Unkown argument."); + } + } + argument_index++; + } + return absl::OkStatus(); +} + +} // namespace verible diff --git a/common/util/cmd_positional_arguments.h b/common/util/cmd_positional_arguments.h new file mode 100644 index 000000000..33b6962d1 --- /dev/null +++ b/common/util/cmd_positional_arguments.h @@ -0,0 +1,68 @@ +// Copyright 2017-2022 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VERIBLE_COMMON_UTIL_CMD_POSITIONAL_ARGUMENTS_H_ +#define VERIBLE_COMMON_UTIL_CMD_POSITIONAL_ARGUMENTS_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" + +namespace verible { + +class CmdPositionalArguments { + public: + explicit CmdPositionalArguments(const std::vector &args) + : all_args_(args){}; + + // Main function that parses arguments. + absl::Status ParseArgs(); + + // Gets include directories arguments. + std::vector GetIncludeDirs() const; + + // Gets macro defines arguments. + std::vector> GetDefines() + const; + + // Gets SV files arguments. + std::vector GetFiles() const; + + private: + // Sets a include directory argument. + absl::Status SetIncludeDir(absl::string_view include_dir); + + // Sets a define arguments. + absl::Status SetDefine( + std::pair define); + + // Sets a file argument. + absl::Status SetFile(absl::string_view file); + + std::vector + all_args_; // contains all arguments (tool's name is included). + std::vector + include_dirs_; // contains all arugments that follows +incdir+. + std::vector + files_; // contains all SV files passed to the tool. + std::vector> + defines_; // contains all arguments that follow +define+[=] + // as a pair. +}; // class CmdPositionalArguments + +} // namespace verible + +#endif // VERIBLE_COMMON_UTIL_CMD_POSITIONAL_ARGUMENTS_H_ diff --git a/verilog/analysis/BUILD b/verilog/analysis/BUILD index bc4b35f5a..131e35e38 100644 --- a/verilog/analysis/BUILD +++ b/verilog/analysis/BUILD @@ -74,6 +74,18 @@ cc_library( ], ) +cc_library( + name = "flow_tree", + srcs = ["flow_tree.cc"], + hdrs = ["flow_tree.h"], + deps = [ + "//common/lexer:token_stream_adapter", + "//verilog/parser:verilog_token_enum", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/status", + ], +) + cc_library( name = "lint_rule_registry", srcs = ["lint_rule_registry.cc"], diff --git a/verilog/analysis/flow_tree.cc b/verilog/analysis/flow_tree.cc new file mode 100644 index 000000000..b9d0d4ca0 --- /dev/null +++ b/verilog/analysis/flow_tree.cc @@ -0,0 +1,147 @@ +// Copyright 2017-2020 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance wedge_from_iteratorh the +// License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in wredge_from_iteratoring, +// software distributed under the License is distributed on an "AS IS" BASIS, +// Wedge_from_iteratorHOUT WARRANTIES OR CONDedge_from_iteratorIONS OF ANY KIND, +// eedge_from_iteratorher express or implied. See the License for the specific +// language governing permissions and limedge_from_iteratorations under the +// License. + +#include "verilog/analysis/flow_tree.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "common/lexer/token_stream_adapter.h" +#include "verilog/parser/verilog_token_enum.h" + +namespace verilog { + +// Constructs the control flow tree, which determines the edge from each node +// (token index) to the next possible childs, And save edge_from_iterator in +// edges_. +absl::Status FlowTree::GenerateControlFlowTree() { + // Adding edges for if blocks. + int token_index = 0; + int current_token_enum = 0; + for (auto current_token : source_sequence_) { + current_token_enum = current_token.token_enum(); + + if (current_token_enum == PP_ifdef || current_token_enum == PP_ifndef) { + ifs_.push_back(token_index); // add index of `ifdef and `ifndef to ifs_. + elses_[ifs_.back()].push_back( + token_index); // also add edge_from_iterator to elses_ as if will + // help marking edges later on. + + } else if (current_token_enum == PP_else || + current_token_enum == PP_elsif || + current_token_enum == PP_endif) { + elses_[ifs_.back()].push_back( + token_index); // add index of `elsif, `else and `endif to else_ of + // the last recent if block. + if (current_token_enum == + PP_endif) { // if the current token is an `endif, then we are ready + // to create edges for this if block. + auto& current_if_block = + elses_[ifs_.back()]; // current_if_block contains all indexes of + // ifs and elses in the latest block. + + // Adding edges for each index in the if block using a nested loop. + for (auto edge_from_iterator = current_if_block.begin(); + edge_from_iterator != current_if_block.end(); + edge_from_iterator++) { + for (auto edge_to_iterator = edge_from_iterator + 1; + edge_to_iterator != current_if_block.end(); edge_to_iterator++) { + if (edge_from_iterator == current_if_block.begin() && + edge_to_iterator == current_if_block.end() - 1 && + current_if_block.size() > 2) + continue; // skip edges from `if to `endif if there is an else in + // this bloc wheneven there is an else in this block. + edges_[*edge_from_iterator].push_back( + *edge_to_iterator + + (edge_to_iterator != + current_if_block.end() - 1)); // add the possible edge. + } + } + ifs_.pop_back(); // the if block edges were added, ready to pop it. + } + } + token_index++; // increment the token index. + } + + // Adding edges for non-if blocks. + token_index = 0; + for (auto current_token : source_sequence_) { + current_token_enum = current_token.token_enum(); + if (current_token_enum != PP_else && current_token_enum != PP_elsif) { + if (token_index > 0) + edges_[token_index - 1].push_back( + token_index); // edges from a token to the one coming after it + // directly. + } else { + if (token_index > 0) + edges_[token_index - 1].push_back( + edges_[token_index] + .back()); // edges from the last token in `ifdef/`ifndef body + // to `endif from the same if block. + } + token_index++; // increment the token index. + } + + return absl::OkStatus(); +} + +// Traveses the control flow tree in a depth first manner, appending the visited +// tokens to current_sequence_, then adding current_sequence_ to variants_ upon +// completing. +absl::Status FlowTree::DepthFirstSearch(int current_node_index) { + // skips preprocessor directives so that current_sequence_ doesn't contain + // any. + const auto& current_token = source_sequence_[current_node_index]; + if (current_token.token_enum() != PP_Identifier && + current_token.token_enum() != PP_ifndef && + current_token.token_enum() != PP_ifdef && + current_token.token_enum() != PP_define && + current_token.token_enum() != PP_define_body && + current_token.token_enum() != PP_elsif && + current_token.token_enum() != PP_else && + current_token.token_enum() != PP_endif) + current_sequence_.push_back(current_token); + + // do recursive search through every possible edge. + for (auto next_node : edges_[current_node_index]) { + if (auto status = FlowTree::DepthFirstSearch(next_node); !status.ok()) { + std::cerr << "ERROR: DepthFirstSearch fails\n"; + return status; + } + } + if (current_node_index == + int(source_sequence_.size()) - + 1) { // if the current node is the last one, push the completed + // current_sequence_ to variants_. + variants_.push_back(current_sequence_); + } + if (current_token.token_enum() != PP_Identifier && + current_token.token_enum() != PP_ifndef && + current_token.token_enum() != PP_ifdef && + current_token.token_enum() != PP_define && + current_token.token_enum() != PP_define_body && + current_token.token_enum() != PP_elsif && + current_token.token_enum() != PP_else && + current_token.token_enum() != PP_endif) + current_sequence_ + .pop_back(); // remove tokens to back track into other variants. + return absl::OkStatus(); +} + +} // namespace verilog diff --git a/verilog/analysis/flow_tree.h b/verilog/analysis/flow_tree.h new file mode 100644 index 000000000..3de93c2bb --- /dev/null +++ b/verilog/analysis/flow_tree.h @@ -0,0 +1,56 @@ +// Copyright 2017-2020 The Verible Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VERIBLE_VERILOG_FLOW_TREE_H_ +#define VERIBLE_VERILOG_FLOW_TREE_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "common/lexer/token_stream_adapter.h" +#include "verilog/parser/verilog_token_enum.h" + +namespace verilog { + +class FlowTree { + public: + explicit FlowTree(verible::TokenSequence source_sequence) + : source_sequence_(std::move(source_sequence)){}; + + absl::Status GenerateControlFlowTree(); // constructs the control flow tree. + absl::Status DepthFirstSearch( + int index); // travese the tree in a depth first manner. + std::vector + variants_; // a memory for all variants generated. + + private: + std::vector + ifs_; // vector of all `ifdef/`ifndef indexes in source_sequence_. + std::map> elses_; // indexes of `elsif/`else indexes in + // source_sequence_ of each if block. + std::map> + edges_; // the tree edges which defines the possible next childs of each + // token in source_sequence_. + verible::TokenSequence + source_sequence_; // the original source code lexed token seqouence. + verible::TokenSequence + current_sequence_; // the variant's token sequence currrently being built + // by DepthFirstSearch. +}; + +} // namespace verilog + +#endif // VERIBLE_VERILOG_FLOW_TREE_H_ diff --git a/verilog/preprocessor/BUILD b/verilog/preprocessor/BUILD index 4ecdaca08..b110f0870 100644 --- a/verilog/preprocessor/BUILD +++ b/verilog/preprocessor/BUILD @@ -21,6 +21,7 @@ cc_library( "//common/text:token_stream_view", "//common/util:container_util", "//common/util:logging", + "//common/util:file_util", "//verilog/parser:verilog_parser", "//verilog/parser:verilog_token_enum", "//verilog/parser:verilog_lexer", diff --git a/verilog/preprocessor/verilog_preprocess.cc b/verilog/preprocessor/verilog_preprocess.cc index 2946f3c23..019a468a1 100644 --- a/verilog/preprocessor/verilog_preprocess.cc +++ b/verilog/preprocessor/verilog_preprocess.cc @@ -30,6 +30,7 @@ #include "common/text/token_info.h" #include "common/text/token_stream_view.h" #include "common/util/container_util.h" +#include "common/util/file_util.h" #include "common/util/logging.h" #include "verilog/parser/verilog_lexer.h" #include "verilog/parser/verilog_parser.h" // for verilog_symbol_name() @@ -42,6 +43,16 @@ using verible::TokenStreamView; using verible::container::FindOrNull; using verible::container::InsertOrUpdate; +TokenStreamView::const_iterator VerilogPreprocess::GenerateBypassWhiteSpaces( + const StreamIteratorGenerator& generator) { + auto iterator = + generator(); // iterator should be pointing to a non-whitespace token; + while (verilog::VerilogLexer::KeepSyntaxTreeTokens(**iterator) == 0) { + iterator = generator(); + } + return iterator; +} + VerilogPreprocess::VerilogPreprocess(const Config& config) : config_(config) { // To avoid having to check at every place if the stack is empty, we always // place a toplevel 'conditional' that is always selected. @@ -54,7 +65,8 @@ VerilogPreprocess::VerilogPreprocess(const Config& config) : config_(config) { absl::StatusOr VerilogPreprocess::ExtractMacroName(const StreamIteratorGenerator& generator) { // Next token to expect is macro definition name. - TokenStreamView::const_iterator token_iter = generator(); + TokenStreamView::const_iterator token_iter = + GenerateBypassWhiteSpaces(generator); if ((*token_iter)->isEOF()) { preprocess_data_.errors.push_back( {**token_iter, "unexpected EOF where expecting macro name"}); @@ -85,7 +97,7 @@ absl::Status VerilogPreprocess::ConsumeMacroDefinition( // Everything else covers macro parameters and the definition body. TokenStreamView::const_iterator token_iter; do { - token_iter = generator(); + token_iter = GenerateBypassWhiteSpaces(generator); if ((*token_iter)->isEOF()) { // Diagnose unexpected EOF downstream instead of erroring here. // Other subroutines can give better context about the parsing state. @@ -219,10 +231,11 @@ absl::Status VerilogPreprocess::ConsumeAndParseMacroCall( macro_call->has_parameters = 1; // Parsing parameters. - TokenStreamView::const_iterator token_iter = generator(); + TokenStreamView::const_iterator token_iter = + GenerateBypassWhiteSpaces(generator); int parameters_size = macro_definition.Parameters().size(); if ((*token_iter)->text() == "(") { - token_iter = generator(); // skip the "(" + token_iter = GenerateBypassWhiteSpaces(generator); // skip the "(" } else { return absl::InvalidArgumentError( "Error it is illegal to call a callable macro without ()."); @@ -231,14 +244,15 @@ absl::Status VerilogPreprocess::ConsumeAndParseMacroCall( while (parameters_size > 0) { if ((*token_iter)->token_enum() == MacroArg) { macro_call->positional_arguments.emplace_back(**token_iter); - token_iter = generator(); - if ((*token_iter)->text() == ",") token_iter = generator(); + token_iter = GenerateBypassWhiteSpaces(generator); + if ((*token_iter)->text() == ",") + token_iter = GenerateBypassWhiteSpaces(generator); parameters_size--; continue; } else if ((*token_iter)->text() == ",") { macro_call->positional_arguments.emplace_back( verible::DefaultTokenInfo()); - token_iter = generator(); + token_iter = GenerateBypassWhiteSpaces(generator); parameters_size--; continue; } else if ((*token_iter)->text() == ")") @@ -453,8 +467,10 @@ absl::Status VerilogPreprocess::HandleDefine( RegisterMacroDefinition(macro_definition); // For now, forward all definition tokens. - for (const auto& token : define_tokens) { - preprocess_data_.preprocessed_token_stream.push_back(token); + if (config_.forward_define) { + for (const auto& token : define_tokens) { + preprocess_data_.preprocessed_token_stream.push_back(token); + } } } @@ -552,6 +568,135 @@ absl::Status VerilogPreprocess::HandleEndif( return absl::OkStatus(); } +// Handle `include directives. +absl::Status VerilogPreprocess::HandleInclude( + TokenStreamView::const_iterator iter, + const StreamIteratorGenerator& generator) { + // karimtera(TODO): how to differentiate between and "file"?? both are + // the same token, need to edit the lexer. + TokenStreamView::const_iterator token_iter = GenerateBypassWhiteSpaces( + generator); // token_iter should now be pointing to a token containing + // the file path. + auto file_token_iter = *token_iter; + if (file_token_iter->token_enum() != TK_StringLiteral) { + preprocess_data_.errors.push_back( + {**token_iter, "Expected a path to a SV file."}); + return absl::InvalidArgumentError("Expected a path to a SV file."); + } + // currently the file path looks like "path", we need to remove "". + const auto& token_text = file_token_iter->text(); + absl::string_view file_path( + token_text.begin() + 1, + token_text.size() - 2); // removed 2 " chars surrounding the path. + + bool is_absolute = !file_path.find_first_of("/"); + + std::string file_path_to_search = std::string(file_path); + + if (auto status = verible::file::FileExists(file_path_to_search); + !status.ok()) { // file doesn't exist in the current directory. + // if it's absolute then it's an error. + if (is_absolute) { + preprocess_data_.errors.push_back({**token_iter, "no such file found."}); + return absl::InvalidArgumentError("ERROR: included file can't be found."); + } + + bool found = 0; + for (const auto& base_dir : search_paths_) { + file_path_to_search = absl::StrCat(base_dir, "/", file_path); + auto search_status = verible::file::FileExists(file_path_to_search); + if (search_status.ok()) { + found = 1; + break; + } + } + if (!found) { + preprocess_data_.errors.push_back({**token_iter, "no such file found."}); + return absl::InvalidArgumentError("ERROR: included file can't be found."); + } + } + + // actually open the file. + std::string source_contents; + if (auto status = + verible::file::GetContents(file_path_to_search, &source_contents); + !status.ok()) { + preprocess_data_.errors.push_back( + {**token_iter, "ERROR: passed file can't be open"}); + return absl::InvalidArgumentError("ERROR: passed file can't be open."); + } + + verilog::VerilogPreprocess child_preprocessor(config_); + // karimtera(TODO): share the macros defined with child pp. + // karimtera(TODO): limit number of nested pps. + + verilog::VerilogLexer lexer(source_contents); + verible::TokenSequence lexed_sequence; + for (lexer.DoNextToken(); !lexer.GetLastToken().isEOF(); + lexer.DoNextToken()) { + // For now we will store the syntax tree tokens only, ignoring all the + // white-space characters. however that should be stored to output the + // source code just like it was, but with conditionals filtered. + /* if (verilog::VerilogLexer::KeepSyntaxTreeTokens(lexer.GetLastToken())) */ + lexed_sequence.push_back(lexer.GetLastToken()); + } + + verible::TokenStreamView lexed_streamview; + // Initializing the lexed token stream view. + InitTokenStreamView(lexed_sequence, &lexed_streamview); + verilog::VerilogPreprocessData child_preprocessed_data = + child_preprocessor.ScanStream(lexed_streamview); + auto& child_preprocessed_stream = + child_preprocessed_data.preprocessed_token_stream; + + ////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////// + /// A very un-efficient work-around to not lose ownership of the child + /// included tokens. Steps: + /// 1) Own the TokenInfo's content by copying into a + /// pair of int, and string. + /// + /// 2) Push it back inside the parent's + /// preprocess_data_.child_preprocessed_stream which is a memory containing + /// sequences of these pairs(not tokens). + /// + /// 3) Create a TokenSequence out of the + /// pair sequence. + /// + /// 4) Push it into preprocess_data_.lexed_macros_backup which + /// is a memory backup created for expanded macros (might change its name). + /// + /// 5) Push it into preprocess_data_.preprocessed_token_stream safely. + std::vector> owner; + for (auto u : child_preprocessed_stream) { + auto child_token = *u; + owner.push_back( + {child_token.token_enum(), std::string(child_token.text())}); + } + preprocess_data_.child_included_content.emplace_back(owner); + + const auto& child_preprocessed_pair_sequence = + preprocess_data_.child_included_content.back(); + + std::vector child_sequence; + for (const auto& u : child_preprocessed_pair_sequence) { + verible::TokenInfo child_sequence_token(u.first, u.second); + child_sequence.push_back(child_sequence_token); + } + ////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////// + + preprocess_data_.lexed_macros_backup.push_back(child_sequence); + auto& sequence_to_add = preprocess_data_.lexed_macros_backup.back(); + auto iter_generator = verible::MakeConstIteratorStreamer(sequence_to_add); + + const auto it_end = sequence_to_add.end(); + for (auto it = iter_generator(); it != it_end; it++) { + preprocess_data_.preprocessed_token_stream.push_back(it); + } + + return absl::OkStatus(); +} // Interprets preprocessor tokens as directives that act on this preprocessor // object and possibly transform the input token stream. absl::Status VerilogPreprocess::HandleTokenIterator( @@ -572,12 +717,16 @@ absl::Status VerilogPreprocess::HandleTokenIterator( case PP_endif: return HandleEndif(iter); } + if (config_.expand_macros && ((*iter)->token_enum() == MacroIdentifier || (*iter)->token_enum() == MacroIdItem || (*iter)->token_enum() == MacroCallId)) { return HandleMacroIdentifier(iter, generator); } + if (config_.include_files && (*iter)->token_enum() == PP_include) + return HandleInclude(iter, generator); + // If not return'ed above, any other tokens are passed through unmodified // unless filtered by a branch. if (conditional_block_.top().InSelectedBranch()) { @@ -586,6 +735,29 @@ absl::Status VerilogPreprocess::HandleTokenIterator( return absl::OkStatus(); } +// Add search directory paths passed to the tool with +incdir+. +absl::Status VerilogPreprocess::AddIncludeDirFromCmdLine( + absl::string_view include_dir_path) { + search_paths_.push_back(std::string(include_dir_path)); + return absl::OkStatus(); +} + +// Add defines passed to the tool with +define+[=]. +absl::Status VerilogPreprocess::AddDefineFromCmdLine( + std::pair define) { + // manually create the tokens to save them into a MacroDefinition. + verible::TokenInfo macro_directive(PP_define, "`define"); + verible::TokenInfo macro_name(PP_Identifier, define.first); + verible::TokenInfo macro_body(PP_define_body, define.second); + verible::MacroDefinition macro_definition(macro_directive, macro_name); + macro_definition.SetDefinitionText(macro_body); + + // add the macro definition to memeory. + RegisterMacroDefinition(macro_definition); + + return absl::OkStatus(); +} + VerilogPreprocessData VerilogPreprocess::ScanStream( const TokenStreamView& token_stream) { preprocess_data_.preprocessed_token_stream.reserve(token_stream.size()); diff --git a/verilog/preprocessor/verilog_preprocess.h b/verilog/preprocessor/verilog_preprocess.h index 7e091123d..2bfea6bc3 100644 --- a/verilog/preprocessor/verilog_preprocess.h +++ b/verilog/preprocessor/verilog_preprocess.h @@ -74,6 +74,10 @@ struct VerilogPreprocessData { verible::TokenStreamView preprocessed_token_stream; std::vector lexed_macros_backup; + // A backup memory of pairs sequence, which owns the content text unlike a + // TokenInfo or a TokenSequence. + std::vector>> child_included_content; + // Map of defined macros. MacroDefinitionRegistry macro_definitions; @@ -103,9 +107,15 @@ class VerilogPreprocess { // want to emit all tokens. bool filter_branches = false; + // Inlude files with `include. + bool include_files = false; + // Expand macro definition bodies, this will relexes the macro body. bool expand_macros = false; // TODO(hzeller): Provide a map of command-line provided +define+'s + + // Foward `define statements + bool forward_define = true; }; explicit VerilogPreprocess(const Config& config); @@ -123,6 +133,13 @@ class VerilogPreprocess { // TODO(fangism): ExpandMacro, ExpandMacroCall // TODO(b/111544845): ExpandEvalStringLiteral + // Add defines passed to the tool with +define+[=]. + absl::Status AddDefineFromCmdLine( + std::pair define); + + // Add search directory paths passed to the tool with +incdir+. + absl::Status AddIncludeDirFromCmdLine(absl::string_view include_dir_path); + private: using StreamIteratorGenerator = std::function; @@ -168,6 +185,12 @@ class VerilogPreprocess { absl::Status ExpandText(const absl::string_view&); absl::Status ExpandMacro(const verible::MacroCall&, const verible::MacroDefinition*); + absl::Status HandleInclude(TokenStreamView::const_iterator, + const StreamIteratorGenerator&); + + // Generate a const_iterator to a non-whitespace token. + static TokenStreamView::const_iterator GenerateBypassWhiteSpaces( + const StreamIteratorGenerator&); const Config config_; @@ -219,6 +242,9 @@ class VerilogPreprocess { bool current_branch_condition_met_; }; + // Paths to search into during handling `include + std::vector search_paths_; + // State of nested conditional blocks. For code simplicity, this always has // a toplevel branch that is selected. std::stack conditional_block_; diff --git a/verilog/tools/preprocessor/BUILD b/verilog/tools/preprocessor/BUILD index 4216fe9e8..5d2bedf05 100644 --- a/verilog/tools/preprocessor/BUILD +++ b/verilog/tools/preprocessor/BUILD @@ -13,7 +13,12 @@ cc_binary( "//common/util:file_util", "//common/util:init_command_line", "//common/util:subcommand", + "//common/util:cmd_positional_arguments", "//verilog/transform:strip_comments", + "//verilog/preprocessor:verilog_preprocess", + "//verilog/parser:verilog_lexer", + "//verilog/analysis:verilog_analyzer", + "//verilog/analysis:flow_tree", "@com_google_absl//absl/flags:usage", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", diff --git a/verilog/tools/preprocessor/verilog_preprocessor.cc b/verilog/tools/preprocessor/verilog_preprocessor.cc index f42a4f68d..ceaa9d8aa 100644 --- a/verilog/tools/preprocessor/verilog_preprocessor.cc +++ b/verilog/tools/preprocessor/verilog_preprocessor.cc @@ -16,29 +16,31 @@ #include #include +#include "absl/flags/flag.h" #include "absl/flags/usage.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "common/lexer/token_stream_adapter.h" +#include "common/util/cmd_positional_arguments.h" #include "common/util/file_util.h" #include "common/util/init_command_line.h" #include "common/util/subcommand.h" +#include "verilog/analysis/flow_tree.h" +#include "verilog/analysis/verilog_analyzer.h" +#include "verilog/parser/verilog_lexer.h" +#include "verilog/parser/verilog_token_enum.h" +#include "verilog/preprocessor/verilog_preprocess.h" #include "verilog/transform/strip_comments.h" using verible::SubcommandArgsRange; -using verible::SubcommandEntry; -static absl::Status StripComments(const SubcommandArgsRange& args, - std::istream&, std::ostream& outs, - std::ostream&) { - if (args.empty()) { - return absl::InvalidArgumentError( - "Missing file argument. Use '-' for stdin."); - } - const char* source_file = args[0]; +static absl::Status StripComments(absl::string_view source_file, std::istream&, + std::ostream& outs, std::ostream&) { std::string source_contents; if (auto status = verible::file::GetContents(source_file, &source_contents); !status.ok()) { + std::cerr << "ERROR: passed file can't be open\n."; return status; } @@ -49,53 +51,206 @@ static absl::Status StripComments(const SubcommandArgsRange& args, return absl::OkStatus(); } -static const std::pair kCommands[] = { - {"strip-comments", // - {&StripComments, // - R"(strip-comments file - -Input: - 'file' is a Verilog or SystemVerilog source file. - Use '-' to read from stdin. +static absl::Status MultipleCU( + absl::string_view source_file, std::istream&, std::ostream& outs, + std::ostream&, + const std::vector>& defines, + const std::vector& include_dirs) { + std::string source_contents; + if (auto status = verible::file::GetContents(source_file, &source_contents); + !status.ok()) { + std::cerr << "ERROR: passed file can't be open\n."; + return status; + } + verilog::VerilogPreprocess::Config config; + config.filter_branches = 1; + config.expand_macros = 1; + config.include_files = 1; + config.forward_define = 0; + verilog::VerilogPreprocess preprocessor(config); -Output: (stdout) - Contents of original file with // and /**/ comments removed. -)"}}, -}; + // Add defines passed with +define+ to the tool. + for (auto define : defines) { + if (auto status = preprocessor.AddDefineFromCmdLine(define); !status.ok()) { + std::cerr << "ERROR: couldn't add macros passed to tool.\n"; + return status; + } + } -int main(int argc, char* argv[]) { - // Create a registry of subcommands (locally, rather than as a static global). - verible::SubcommandRegistry commands; - for (const auto& entry : kCommands) { - const auto status = commands.RegisterCommand(entry.first, entry.second); - if (!status.ok()) { - std::cerr << status.message() << std::endl; - return 2; // fatal error + // Add search paths for `includes. + for (auto path : include_dirs) { + if (auto status = preprocessor.AddIncludeDirFromCmdLine(path); + !status.ok()) { + std::cerr << "ERROR: couldn't add include search paths to tool.\n"; + return status; } } - const std::string usage = absl::StrCat("usage: ", argv[0], - " command args...\n" - "available commands:\n", - commands.ListCommands()); + verilog::VerilogLexer lexer(source_contents); + verible::TokenSequence lexed_sequence; + for (lexer.DoNextToken(); !lexer.GetLastToken().isEOF(); + lexer.DoNextToken()) { + // For now we will store the syntax tree tokens only, ignoring all the + // white-space characters. however that should be stored to output the + // source code just like it was, but with conditionals filtered. + lexed_sequence.push_back(lexer.GetLastToken()); + } + verible::TokenStreamView lexed_streamview; + + // Tyring to create a string_view between 2 consecutive tokens to preserve the + // white-spaces as suggested by fangism. It worked but it solves a different + // problem, where I have a non-whitespace token stream, and want to preserve + // the whitespaces in between. + /* auto it_end=lexed_sequence.end(); */ + /* auto it_prv=lexed_sequence.begin(); */ + /* for(auto it=lexed_sequence.begin();it!=it_end;it++){ */ + /* if(it==it_prv) continue; */ + /* std::cout<text(); */ + /* absl::string_view + * white_space(it_prv->text().end(),std::distance(it_prv->text().end(),it->text().begin())); + */ + /* std::cout<text(); // output the preprocessed tokens. + /* outs << *u << '\n'; // output the preprocessed tokens. */ + for (auto& u : preprocessed_data.errors) + outs << u.error_message << '\n'; // for debugging. + // parsing just as a trial + std::string post_preproc; + for (auto u : preprocessed_stream) post_preproc += std::string{u->text()}; + std::string source_view{post_preproc}; + verilog::VerilogAnalyzer analyzer(source_view, "file1", config); + auto analyze_status = analyzer.Analyze(); + /* const auto& mydata = analyzer.Data().Contents(); */ + /* outs<...]\n"); // Process invocation args. const auto args = verible::InitCommandLine(usage, &argc, &argv); - if (args.size() == 1) { + if (args.empty()) { std::cerr << absl::ProgramUsageMessage() << std::endl; return 1; } - // args[0] is the program name - // args[1] is the subcommand - // subcommand args start at [2] - const SubcommandArgsRange command_args(args.cbegin() + 2, args.cend()); - - const auto& sub = commands.GetSubcommandEntry(args[1]); - // Run the subcommand. - const auto status = sub.main(command_args, std::cin, std::cout, std::cerr); - if (!status.ok()) { - std::cerr << status.message() << std::endl; + + // Get flags. + bool strip_comments_flag = absl::GetFlag(FLAGS_strip_comments); + bool multiple_cu_flag = absl::GetFlag(FLAGS_multiple_cu); + bool generate_variants_flag = absl::GetFlag(FLAGS_generate_variants); + + // Check for flags and argument illegal usage. + if (strip_comments_flag && generate_variants_flag) { // Illegal usage. + std::cerr << "ERROR: the flags passed shouldn't be used together.\n\n" + << absl::ProgramUsageMessage() << std::endl; + return 1; + } + if (args.size() == + 1) { // No Files are passed (files are passed as positional arguments). + std::cerr << "ERROR: No System-Verilog files were passed.\n\n" + << absl::ProgramUsageMessage() << std::endl; + return 1; + } + + verible::CmdPositionalArguments positional_arguments(args); + if (auto status = positional_arguments.ParseArgs(); !status.ok()) { + std::cerr << "ERROR: parsing positional arguments failed.\n"; return 1; } + + auto include_dirs = positional_arguments.GetIncludeDirs(); + auto defines = positional_arguments.GetDefines(); + auto files = positional_arguments.GetFiles(); + + // Select the operation mode and execute it. + if (strip_comments_flag) { + for (auto filename : files) { + if (auto status = StripComments(filename, std::cin, std::cout, std::cerr); + !status.ok()) { + std::cerr << "ERROR: stripping comments failed.\n"; + return 1; + } + } + } else if (generate_variants_flag) { + for (auto filename : files) { + if (auto status = + GenerateVariants(filename, std::cin, std::cout, std::cerr); + !status.ok()) { + std::cerr << "ERROR: generating variants failed.\n"; + return 1; + } + } + } else if (multiple_cu_flag) { + for (auto filename : files) { + if (auto status = MultipleCU(filename, std::cin, std::cout, std::cerr, + defines, include_dirs); + !status.ok()) { + std::cerr + << "ERROR: preprocessing in multiple comiplation units failed.\n"; + return 1; + } + } + } + return 0; } diff --git a/verilog/tools/preprocessor/verilog_preprocessor_test.sh b/verilog/tools/preprocessor/verilog_preprocessor_test.sh index f92d2cc0a..701cf5701 100755 --- a/verilog/tools/preprocessor/verilog_preprocessor_test.sh +++ b/verilog/tools/preprocessor/verilog_preprocessor_test.sh @@ -29,59 +29,59 @@ readonly MY_EXPECT_FILE preprocessor="$(rlocation ${TEST_WORKSPACE}/$1)" ################################################################################ -echo "=== Test no command." +# echo "=== Test no command." -"$preprocessor" > "$MY_OUTPUT_FILE" 2>&1 +# "$preprocessor" > "$MY_OUTPUT_FILE" 2>&1 -status="$?" -[[ $status == 1 ]] || { - "Expected exit code 1, but got $status" - exit 1 -} +# status="$?" +# [[ $status == 1 ]] || { +# "Expected exit code 1, but got $status" +# exit 1 +# } ################################################################################ -echo "=== Test invalid command." +# echo "=== Test invalid command." -"$preprocessor" bad-subcommand > "$MY_OUTPUT_FILE" 2>&1 +# "$preprocessor" bad-subcommand > "$MY_OUTPUT_FILE" 2>&1 -status="$?" -[[ $status == 1 ]] || { - "Expected exit code 1, but got $status" - exit 1 -} +# status="$?" +# [[ $status == 1 ]] || { +# "Expected exit code 1, but got $status" +# exit 1 +# } ################################################################################ echo "=== Test the 'help' command." -"$preprocessor" help > "$MY_OUTPUT_FILE" 2>&1 +"$preprocessor" --helpfull > "$MY_OUTPUT_FILE" 2>&1 -status="$?" -[[ $status == 0 ]] || { - "Expected exit code 0, but got $status" - exit 1 -} +# status="$?" +# [[ $status == 0 ]] || { +# "Expected exit code 0, but got $status" +# exit 1 +# } -grep -q "strip-comments" "$MY_OUTPUT_FILE" || { - echo "Expected \"strip-comments\" in $MY_OUTPUT_FILE but didn't find it. Got:" +grep -q "strip_comments" "$MY_OUTPUT_FILE" || { + echo "Expected \"strip_comments\" in $MY_OUTPUT_FILE but didn't find it. Got:" cat "$MY_OUTPUT_FILE" exit 1 } ################################################################################ -echo "=== Test 'help' on a specific command." +# echo "=== Test 'help' on a specific command." -"$preprocessor" help help > "$MY_OUTPUT_FILE" 2>&1 +# "$preprocessor" help help > "$MY_OUTPUT_FILE" 2>&1 -status="$?" -[[ $status == 0 ]] || { - "Expected exit code 0, but got $status" - exit 1 -} +# status="$?" +# [[ $status == 0 ]] || { +# "Expected exit code 0, but got $status" +# exit 1 +# } ################################################################################ -echo "=== Test strip-comments: missing file argument." +echo "=== Test -strip_comments: missing file argument." -"$preprocessor" strip-comments > /dev/null +"$preprocessor" -strip_comments > /dev/null status="$?" [[ $status == 1 ]] || { @@ -90,7 +90,7 @@ status="$?" } ################################################################################ -echo "=== Test strip-comments: white out comments" +echo "=== Test -strip_comments: white out comments" cat > "$MY_INPUT_FILE" < "$MY_OUTPUT_FILE" +"$preprocessor" -strip_comments "$MY_INPUT_FILE" > "$MY_OUTPUT_FILE" status="$?" [[ $status == 0 ]] || { @@ -132,7 +132,7 @@ diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { # Same but piped into stdin. -"$preprocessor" strip-comments - < "$MY_INPUT_FILE" > "$MY_OUTPUT_FILE" +"$preprocessor" -strip_comments - < "$MY_INPUT_FILE" > "$MY_OUTPUT_FILE" status="$?" [[ $status == 0 ]] || { @@ -145,7 +145,7 @@ diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { } ################################################################################ -echo "=== Test strip-comments: on a lexically invalid source file" +echo "=== Test -strip_comments: on a lexically invalid source file" cat > "$MY_INPUT_FILE" < "$MY_EXPECT_FILE" < "$MY_OUTPUT_FILE" +"$preprocessor" -strip_comments "$MY_INPUT_FILE" > "$MY_OUTPUT_FILE" status="$?" [[ $status == 0 ]] || { @@ -169,9 +169,9 @@ diff --strip-trailing-cr -u "$MY_EXPECT_FILE" "$MY_OUTPUT_FILE" || { ################################################################################ -# Test strip-comments: reading a nonexistent source file. +# Test -strip_comments: reading a nonexistent source file. -"$preprocessor" strip-comments "$MY_INPUT_FILE.does.not.exist" > /dev/null +"$preprocessor" -strip_comments "$MY_INPUT_FILE.does.not.exist" > /dev/null status="$?" [[ $status == 1 ]] || {