Merge pull request #53 from erezsh/lark_cache

Use Lark with its cache feature, instead of creating a standalone parser

.gitignore

@@ -120,3 +120,4 @@ node_modules/
 
 # Don't commit the generated parser
 lark_parser.py
+.lark_cache.bin

hcl2/api.py

@@ -15,4 +15,7 @@ def loads(text: str) -> dict:
  # Lark doesn't support a EOF token so our grammar can't look for "new line or end of file"
  # This means that all blocks must end in a new line even if the file ends
  # Append a new line as a temporary fix
- return hcl2.parse(text + "\n")
+ # Ignoring type as the type-annotation of Lark.parse() claims that it always returns a Tree,
+ # but in the docs of the parse() said that it returns whatever the supplied transformer returns.
+ # We supply DictTransformer so the return type is Dict.
+ return hcl2.parse(text + "\n") # type: ignore

hcl2/parser.py

@@ -1,42 +1,18 @@
 """A parser for HCL2 implemented using the Lark parser"""
 from pathlib import Path
 
-from hcl2.transformer import DictTransformer
-
-THIS_DIR = Path(__file__).absolute().resolve().parent
-PARSER_FILE = THIS_DIR / "lark_parser.py"
-
+from lark import Lark
 
-def create_parser_file(parser_file: Path = PARSER_FILE) -> None:
- """
- Parsing the Lark grammar takes about 0.5 seconds. In order to improve performance we can cache the parser
- file. The below code caches the entire python file which is generated by Lark's standalone parser feature
- See: https://github.com/lark-parser/lark/blob/master/lark/tools/standalone.py
-
- Lark also supports serializing the parser config but the deserialize function did not work for me.
- The lark state contains dicts with numbers as keys which is not supported by json so the serialized
- state can't be written to a json file. Exporting to other file types would have required
- adding additional dependencies or writing a lot more code. Lark's standalone parser
- feature works great but it expects to be run as a separate shell command
- The below code copies some of the standalone parser generator code in a way that we can use
- """
- # This function is needed only if standalone parser is not yet compiled.
- # pylint: disable=import-outside-toplevel
- from lark import Lark
- from lark.tools.standalone import gen_standalone
-
- lark_file = THIS_DIR / "hcl2.lark"
- with open(parser_file, "w", encoding="utf-8") as parser_file_stream:
- lark_inst = Lark(lark_file.read_text(), parser="lalr", lexer="contextual")
- parser_file_stream.write("# mypy: ignore-errors\n")
- gen_standalone(lark_inst, out=parser_file_stream)
+from hcl2.transformer import DictTransformer
 
 
-if not PARSER_FILE.exists():
- create_parser_file(PARSER_FILE)
+PARSER_FILE = Path(__file__).absolute().resolve().parent / ".lark_cache.bin"
 
-# pylint: disable=wrong-import-position
-# Lark_StandAlone needs to be imported after the above block of code because lark_parser.py might not exist
-from hcl2.lark_parser import Lark_StandAlone
 
-hcl2 = Lark_StandAlone(transformer=DictTransformer())
+hcl2 = Lark.open(
+ "hcl2.lark",
+ parser="lalr",
+ cache=str(PARSER_FILE), # Disable/Delete file to effect changes to the grammar
+ rel_to=__file__,
+ transformer=DictTransformer(),
+)

test/unit/test_load.py

@@ -1,10 +1,11 @@
 """ Test parsing a variety of hcl files"""
+
 import json
 from pathlib import Path
 from unittest import TestCase
 
+from hcl2.parser import PARSER_FILE
 import hcl2
-from hcl2.parser import PARSER_FILE, create_parser_file
 
 
 HELPERS_DIR = Path(__file__).absolute().parent.parent / "helpers"
@@ -18,9 +19,8 @@ class TestLoad(TestCase):
 
  def test_load_terraform(self):
  """Test parsing a set of hcl2 files and force recreating the parser file"""
- parser_file = Path(hcl2.__file__).absolute().parent / PARSER_FILE
- parser_file.unlink()
- create_parser_file()
+ # delete the parser file to force it to be recreated
+ PARSER_FILE.unlink()
  for hcl_path in HCL2_FILES:
  yield self.check_terraform, hcl_path