From 6a64d3d26dee4ccd9b5cd662141c54200a7096bb Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Thu, 11 May 2023 15:02:58 +0100 Subject: [PATCH 1/8] #412 put back original fifo handling for include files [skip ci] --- src/fparser/common/readfortran.py | 27 +++++++++++++----- src/fparser/common/tests/test_readfortran.py | 1 + src/fparser/two/tests/test_utils.py | 30 ++++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/fparser/common/readfortran.py b/src/fparser/common/readfortran.py index 16480fd0..a541a686 100644 --- a/src/fparser/common/readfortran.py +++ b/src/fparser/common/readfortran.py @@ -780,13 +780,26 @@ def next(self, ignore_comments=None): if self.reader is not None: # inside INCLUDE statement try: - # Return a line from the include. - return self.reader.next(ignore_comments) - except StopIteration: - # There is nothing left in the include - # file. Setting reader to None indicates that - # we should now read from the main reader. - self.reader = None + # Manually check to see if something has not + # matched and has been placed in the fifo. We + # can't use _next() as this method is associated + # with the include reader (self.reader._next()), + # not this reader (self._next()). + item = self.fifo_item.pop(0) + #if ignore_comments: + # while isinstance(item, Comment): + # item = self.fifo_item.pop(0) + return item + except IndexError: + # There is nothing in the fifo buffer. + try: + # Return a line from the include. + return self.reader.next(ignore_comments) + except StopIteration: + # There is nothing left in the include + # file. Setting reader to None indicates that + # we should now read from the main reader. + self.reader = None item = self._next(ignore_comments) if isinstance(item, Line) and _IS_INCLUDE_LINE(item.line): # catch INCLUDE statement and create a new FortranReader diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py index a3b7f5ab..886be354 100644 --- a/src/fparser/common/tests/test_readfortran.py +++ b/src/fparser/common/tests/test_readfortran.py @@ -416,6 +416,7 @@ def check_include_works( reader = FortranFileReader(fortran_filename, ignore_comments=ignore_comments) for orig_line in expected.split("\n"): new_line = reader.next().line + print(new_line+"\n====\n"+orig_line) assert new_line == orig_line with pytest.raises(StopIteration): reader.next() diff --git a/src/fparser/two/tests/test_utils.py b/src/fparser/two/tests/test_utils.py index 2803daf1..2462cd51 100644 --- a/src/fparser/two/tests/test_utils.py +++ b/src/fparser/two/tests/test_utils.py @@ -37,8 +37,11 @@ """ +import os + import pytest from fparser.api import get_reader +from fparser.common.readfortran import FortranFileReader from fparser.two import Fortran2003, utils @@ -141,3 +144,30 @@ def test_endstmtbase_match(): require_stmt_type=True, ) assert result == ("SUBROUTINE", Fortran2003.Name("sub")) + + +@pytest.mark.usefixtures("f2003_create") +def test_base_include_handling(tmpdir): + """Test parsing of code that involves an INCLUDE. This is beyond what + is tested in test_readfortran as it exercises the mechanisms used when + a match fails and content is given back to the reader.""" + fortran_code = ( + "module include_test\n include 'test_include.h'\nend module include_test" + ) + include_code = ( + "interface mpi_sizeof\n" + "subroutine simple()\n" + "end subroutine simple\n" + "end interface mpi_sizeof" + ) + with open(os.path.join(tmpdir, "test_prog.f90"), "w", encoding="utf-8") as cfile: + cfile.write(fortran_code) + with open(os.path.join(tmpdir, "test_include.h"), "w", encoding="utf-8") as cfile: + cfile.write(include_code) + reader = FortranFileReader( + os.path.join(tmpdir, "test_prog.f90"), include_dirs=[tmpdir] + ) + prog = Fortran2003.Program(reader) + assert isinstance(prog, Fortran2003.Program) + nodes = utils.walk(prog, Fortran2003.Interface_Stmt) + assert len(nodes) == 1 From a37cb7f359689925d70044537af47f9295451c73 Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Thu, 11 May 2023 15:14:30 +0100 Subject: [PATCH 2/8] #412 make put_item() aware of tree of reader objects --- src/fparser/common/readfortran.py | 39 ++++++++++++++----------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/src/fparser/common/readfortran.py b/src/fparser/common/readfortran.py index a541a686..320a1f7e 100644 --- a/src/fparser/common/readfortran.py +++ b/src/fparser/common/readfortran.py @@ -745,8 +745,17 @@ def get_item(self, ignore_comments=None): return item def put_item(self, item): - """Insert item to FIFO item buffer.""" - self.fifo_item.insert(0, item) + """Insert item into FIFO buffer of 'innermost' reader object. + + :param item: + :type item: + """ + if self.reader: + # We are reading an INCLUDE file so put this item in the FIFO + # of the corresponding reader. + self.reader.put_item(item) + else: + self.fifo_item.insert(0, item) # Iterator methods: @@ -780,26 +789,12 @@ def next(self, ignore_comments=None): if self.reader is not None: # inside INCLUDE statement try: - # Manually check to see if something has not - # matched and has been placed in the fifo. We - # can't use _next() as this method is associated - # with the include reader (self.reader._next()), - # not this reader (self._next()). - item = self.fifo_item.pop(0) - #if ignore_comments: - # while isinstance(item, Comment): - # item = self.fifo_item.pop(0) - return item - except IndexError: - # There is nothing in the fifo buffer. - try: - # Return a line from the include. - return self.reader.next(ignore_comments) - except StopIteration: - # There is nothing left in the include - # file. Setting reader to None indicates that - # we should now read from the main reader. - self.reader = None + return self.reader.next(ignore_comments) + except StopIteration: + # There is nothing left in the include + # file. Setting reader to None indicates that + # we should now read from the main reader. + self.reader = None item = self._next(ignore_comments) if isinstance(item, Line) and _IS_INCLUDE_LINE(item.line): # catch INCLUDE statement and create a new FortranReader From 23bf7c06b408c36c2f171fe9ed9497c2d7e2d706 Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Thu, 11 May 2023 16:54:37 +0100 Subject: [PATCH 3/8] #412 tidy put_item docstring --- src/fparser/common/readfortran.py | 8 +++++--- src/fparser/common/tests/test_readfortran.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/fparser/common/readfortran.py b/src/fparser/common/readfortran.py index 320a1f7e..a5f886d4 100644 --- a/src/fparser/common/readfortran.py +++ b/src/fparser/common/readfortran.py @@ -747,8 +747,10 @@ def get_item(self, ignore_comments=None): def put_item(self, item): """Insert item into FIFO buffer of 'innermost' reader object. - :param item: - :type item: + :param item: the item to insert into the FIFO. + :type item: :py:class:`fparser.common.readfortran.Line` | \ + :py:class:`fparser.common.readfortran.MultiLine` | \ + :py:class:`fparser.common.readfortran.Comment` """ if self.reader: # We are reading an INCLUDE file so put this item in the FIFO @@ -776,7 +778,7 @@ def next(self, ignore_comments=None): value. :returns: the next line item. This can be from a local fifo \ - buffer, from an include reader or from this reader. + buffer, from an include reader or from this reader. :rtype: py:class:`fparser.common.readfortran.Line` :raises StopIteration: if no more lines are found. diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py index 886be354..a3b7f5ab 100644 --- a/src/fparser/common/tests/test_readfortran.py +++ b/src/fparser/common/tests/test_readfortran.py @@ -416,7 +416,6 @@ def check_include_works( reader = FortranFileReader(fortran_filename, ignore_comments=ignore_comments) for orig_line in expected.split("\n"): new_line = reader.next().line - print(new_line+"\n====\n"+orig_line) assert new_line == orig_line with pytest.raises(StopIteration): reader.next() From 11a78bfb888d9a90a506974830f2675b38273a8c Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Mon, 15 May 2023 13:39:50 +0100 Subject: [PATCH 4/8] #414 fix existing test so that it really does exercise FIFO with include file --- src/fparser/common/tests/test_readfortran.py | 35 ++++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py index a3b7f5ab..17d4aa1f 100644 --- a/src/fparser/common/tests/test_readfortran.py +++ b/src/fparser/common/tests/test_readfortran.py @@ -680,20 +680,41 @@ def test_put_item(ignore_comments): assert fifo_line == orig_line -def test_put_item_include(ignore_comments): +def test_put_item_include(ignore_comments, tmpdir): """Check that when a line that has been included via an include statement is consumed it can be pushed back so it can be consumed again. Test with and without ignoring comments. """ - reader = FortranStringReader(FORTRAN_CODE, ignore_comments=ignore_comments) + _ = tmpdir.chdir() + cwd = str(tmpdir) + include_code = """ + var1 = 1 + var2 = 2 + var3 = 3 +""" + with open(os.path.join(cwd, "my_include.h"), "w") as cfile: + cfile.write(include_code) + code = """ +program my_prog + integer :: var1, var2, var3 + include 'my_include.h' +end program my_prog +""" + reader = FortranStringReader(code, ignore_comments=ignore_comments) + lines = [] while True: - orig_line = reader.get_item() - if not orig_line: + lines.append(reader.get_item()) + if "var3 =" in lines[-1].line: + # Stop reading while we're still in the INCLUDE file. break - reader.put_item(orig_line) - fifo_line = reader.get_item() - assert fifo_line == orig_line + # Put all the lines back in the same order that we saw them. + for line in reversed(lines): + reader.put_item(line) + # Check that the reader returns all those lines in the same order that + # we saw them originally. + for line in lines: + assert reader.get_item().line == line.line def test_multi_put_item(ignore_comments): From 26eed1d6a8762db76a0c6b806157566aa301be54 Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Mon, 15 May 2023 15:47:37 +0100 Subject: [PATCH 5/8] #414 reproduce failure in reader-only test --- src/fparser/common/tests/test_readfortran.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py index 17d4aa1f..5f571335 100644 --- a/src/fparser/common/tests/test_readfortran.py +++ b/src/fparser/common/tests/test_readfortran.py @@ -705,6 +705,9 @@ def test_put_item_include(ignore_comments, tmpdir): lines = [] while True: lines.append(reader.get_item()) + # Try immediately putting the line back and then requesting it again. + reader.put_item(lines[-1]) + assert reader.get_item().line == lines[-1].line if "var3 =" in lines[-1].line: # Stop reading while we're still in the INCLUDE file. break From 06de07b193858c68ab5bd16cd21e89d8a7ff733d Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Mon, 15 May 2023 15:50:29 +0100 Subject: [PATCH 6/8] #414 improve comments in test --- src/fparser/common/tests/test_readfortran.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fparser/common/tests/test_readfortran.py b/src/fparser/common/tests/test_readfortran.py index 5f571335..05d48bb9 100644 --- a/src/fparser/common/tests/test_readfortran.py +++ b/src/fparser/common/tests/test_readfortran.py @@ -706,10 +706,12 @@ def test_put_item_include(ignore_comments, tmpdir): while True: lines.append(reader.get_item()) # Try immediately putting the line back and then requesting it again. + # This checks that the correct FIFO buffer is being used. reader.put_item(lines[-1]) assert reader.get_item().line == lines[-1].line if "var3 =" in lines[-1].line: - # Stop reading while we're still in the INCLUDE file. + # Stop reading while we're still in the INCLUDE file so that we + # have a stack of readers when calling `put_item` below. break # Put all the lines back in the same order that we saw them. for line in reversed(lines): From 0ac4cc8ce14174fdbf924955f2e46a31acf94a3e Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Mon, 15 May 2023 16:00:24 +0100 Subject: [PATCH 7/8] #414 revert test file under two/ --- src/fparser/two/tests/test_utils.py | 30 ----------------------------- 1 file changed, 30 deletions(-) diff --git a/src/fparser/two/tests/test_utils.py b/src/fparser/two/tests/test_utils.py index 2462cd51..2803daf1 100644 --- a/src/fparser/two/tests/test_utils.py +++ b/src/fparser/two/tests/test_utils.py @@ -37,11 +37,8 @@ """ -import os - import pytest from fparser.api import get_reader -from fparser.common.readfortran import FortranFileReader from fparser.two import Fortran2003, utils @@ -144,30 +141,3 @@ def test_endstmtbase_match(): require_stmt_type=True, ) assert result == ("SUBROUTINE", Fortran2003.Name("sub")) - - -@pytest.mark.usefixtures("f2003_create") -def test_base_include_handling(tmpdir): - """Test parsing of code that involves an INCLUDE. This is beyond what - is tested in test_readfortran as it exercises the mechanisms used when - a match fails and content is given back to the reader.""" - fortran_code = ( - "module include_test\n include 'test_include.h'\nend module include_test" - ) - include_code = ( - "interface mpi_sizeof\n" - "subroutine simple()\n" - "end subroutine simple\n" - "end interface mpi_sizeof" - ) - with open(os.path.join(tmpdir, "test_prog.f90"), "w", encoding="utf-8") as cfile: - cfile.write(fortran_code) - with open(os.path.join(tmpdir, "test_include.h"), "w", encoding="utf-8") as cfile: - cfile.write(include_code) - reader = FortranFileReader( - os.path.join(tmpdir, "test_prog.f90"), include_dirs=[tmpdir] - ) - prog = Fortran2003.Program(reader) - assert isinstance(prog, Fortran2003.Program) - nodes = utils.walk(prog, Fortran2003.Interface_Stmt) - assert len(nodes) == 1 From e2b499b69cf893e12d505c1ae0374eeba5421e0e Mon Sep 17 00:00:00 2001 From: rupertford Date: Tue, 16 May 2023 10:14:23 +0100 Subject: [PATCH 8/8] pr #414. Updated changelog ready for merge to master. --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7f83b3..151f365a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,11 @@ Modifications by (in alphabetical order): * P. Vitt, University of Siegen, Germany * A. Voysey, UK Met Office -15/05/2023 PR #415 for #165. Bug fix for spurious matching of 'NAMELIST' in - certain contexts. +16/05/2023 PR #414 for #412. Bug fix for disappearing line when parsing + include files. + +15/05/2023 PR #415 for #165. Bug fix for code aborting when trying to match + 'NAMELIST' in certain contexts. 15/05/2023 PR #408 for #403. Add support for the F2008 DO CONCURRENT.