tools: add script for automatic conda tool revision tests

Signed-off-by: Piotr Binkowski <[email protected]>

tools/compare-conda.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import os
+import csv
+import json
+import argparse
+import subprocess
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("count")
+
+args = parser.parse_args()
+
+tools = [
+    'odin_ii', 'tree-sitter-verilog', 'zachjs-sv2v', 'verilator', 'iverilog',
+    'yosys', 'slang'
+]
+
+packages = {}
+
+versions = {}
+
+for tool in tools:
+    cmd = 'conda search {} -c symbiflow --json'.format(tool)
+    proc = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
+    log, _ = proc.communicate()
+    packages[tool] = json.loads(log)
+
+for p in packages:
+    versions[p] = []
+    for r in packages[p][p]:
+        if r['version'] not in versions[p]:
+            versions[p].append(r['version'])
+
+os.system('rm -rf ./tests/generated')
+os.system('make generate-tests')
+
+last_ver = {}
+
+used_versions = []
+
+for i in range(1, int(args.count) + 1):
+    os.system('rm -rf ./out')
+
+    for tool in tools:
+        try:
+            ver = versions[tool][-i]
+        except IndexError:
+            ver = last_ver[tool]
+
+        last_ver[tool] = ver
+
+        os.system('conda install -y -c symbiflow {}={}'.format(tool, ver))
+
+        print('{}={} installed'.format(tool, ver))
+
+    used_versions.append(last_ver.copy())
+
+    os.system('make -j12')
+    os.system('cp ./out/report/report.csv report-{}.csv'.format(i))
+
+results = []
+
+for i in range(int(args.count)):
+    with open('report-{}.csv'.format(i+1), newline='') as csvfile:
+        reader = csv.DictReader(csvfile)
+        runners = reader.fieldnames[3:]
+        res = {}
+        for runner in runners:
+            res[runner] = 0
+        for row in reader:
+            for runner in runners:
+                if row[runner] == 'True':
+                    res[runner] += 1
+
+        results.append((res, used_versions[0]))
+
+
+with open('out.csv', 'w', newline='') as csvfile:
+    writer = csv.writer(csvfile)
+    for res in results:
+        writer.writerow(*res[0], *res[1])