Skip to content

Commit

Permalink
Refactored the cpython integration test
Browse files Browse the repository at this point in the history
  • Loading branch information
bugdea1er committed Oct 24, 2023
1 parent 9245a3c commit 00c945b
Showing 1 changed file with 44 additions and 31 deletions.
Original file line number Diff line number Diff line change
@@ -1,28 +1,35 @@
package org.polystat.py2eo.parser

import org.junit.jupiter.api.Assertions.fail
import org.junit.jupiter.api.{AfterEach, Test}
import org.junit.jupiter.api.{AfterAll, Assertions, Test}
import org.polystat.py2eo.parser.ParserPrinterIT.blacklisted

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Awaitable, Future}
import scala.reflect.io.Directory
import scala.reflect.io.{Directory, File}
import scala.sys.process.Process
import scala.util.Properties

/** Parse and re-print tests from cpython repo and test them out */
final class ParserPrinterIT {
object ParserPrinterIT {

/** Temporary directory where to conduct tests */
private val directory = Directory.makeTemp(prefix = "org.polystat.py2eo.").toAbsolute

/** Delete the directory by hand since Scala has some problems with it */
@AfterAll def cleanup(): Unit = ParserPrinterIT.directory.deleteRecursively

private val cpythonLink = "https://github.com/python/cpython"
private val directory = Directory.makeTemp(prefix = "org.polystat.py2eo.")
private val availableProcessors = sys.runtime.availableProcessors
/** Repository with python tests */
private val repo = "https://github.com/python/cpython"

/** Blacklisted test names; do not update them */
private val blacklisted = Set(
// these are excluded because of some encoding problems in the lexer
"test_unicode_identifiers.py", "test_source_encoding.py",
"badsyntax_3131.py", "badsyntax_pep3120.py",
"module_koi8_r.py", "module_iso_8859_1.py",
// most of these are excluded because they do tests by comparing stack traces as strings
// but code before parser-printer has different line numbers than code after => traces are always different =>
// most of these are excluded because they do tests by comparing
// stack traces as strings but code before parser-printer has different
// line numbers than code after => traces are always different =>
// those tests cannot possibly pass
"test_traceback.py", "test_dis.py", "test_zipfile.py",
"test_multiprocessing_fork.py", "test_sys.py",
Expand All @@ -36,40 +43,46 @@ final class ParserPrinterIT {

"test_grammar.py", "test_headerregistry.py"
)
}

@Test def apply(): Unit = {
val cpython = Directory(directory / "cpython")
/** Parse and re-print tests from cpython repo and test them out */
final class ParserPrinterIT {

Process(s"git clone $cpythonLink ${cpython.name}", directory.jfile).!!
Process("git checkout v3.8.10", cpython.jfile).!!
@Test def apply(): Unit = {
val cpython = Directory(ParserPrinterIT.directory / "cpython")

val testsDirectory = Directory(cpython / "Lib" / "test")
val tests = testsDirectory.deepFiles.filter(file => (file.extension == "py") && (file.name.startsWith("test")))
Process(s"git clone --depth=1 --branch v3.8.10 ${ParserPrinterIT.repo} $cpython").!!

val futures = for {test <- tests if !blacklisted(test.name)} yield Future {
println(s"parsed ${test.name}")
Parse(test).map(PrintPython.print).fold(fail())(test writeAll _)
}
val tests = Directory(cpython / "Lib" / "test")
.deepFiles
.toList
.filter(_.extension == "py")
.filter(_.name.startsWith("test"))
.filterNot(file => blacklisted(file.name))
.map(reprint)

futures foreach await
tests foreach await

println(s"Total of ${futures.length} files transpiled")
println(s"Total of ${tests.length} files transpiled")

assume(Properties.isMac || Properties.isLinux)
Process("./configure", cpython.jfile).!!
Process(s"make -j ${availableProcessors + 2}", cpython.jfile).!!
Process(s"make -j ${sys.runtime.availableProcessors}", cpython.jfile).!!
Process("make test", cpython.jfile).!!
}

// @AfterEach def cleanup(): Unit = {
// directory.deleteRecursively
// }
/** Parses and reprints the given test and calls [[Assertions.fail]] if failed to parse it */
private def reprint(test: File): Future[Unit] = Future {
Parse(test).map(PrintPython.print).fold(fail(test))(test writeAll _)
}

/** Prints the failed test name and calls [[Assertions.fail]] */
private def fail(test: File): Unit = {
println(s"failed on ${test.name}")
Assertions.fail()
}

/**
* Await and return the result (of type `T`) of an [[Awaitable]]
*
* @param awaitable the [[Awaitable]] to be awaited
*/
/** Awaits and returns the result of an [[Awaitable]] */
private def await[T](awaitable: Awaitable[T]): T = {
Await.result(awaitable, Duration.Inf)
}
Expand Down

0 comments on commit 00c945b

Please sign in to comment.