Merge pull request #92 from weiya711/master_refactor_scripts

Master refactor scripts

.gitignore

@@ -5,8 +5,13 @@ __pycache__/
 # Temporary output files
 *.swp
 *.swo
+*.swn
 *.txt
 *.out
+*.log
+
+# Generated Python Package files
+*.egg-info/
 
 # Files for MacOS and IDEs
 .DS_store
@@ -26,6 +31,13 @@ cmake-build-*/
 *.csv
 *.json
 
+# Tensor files
+*.mtx
+*.tns
+
+# Generated formatted tensor files
+tensor_*_mode_*
+
 # Generated folders
 build/
 logs/
@@ -35,22 +47,16 @@ suitesparse-bench/
 sam-outputs/
 compiler/benchmark/
 
-# Generated SAM simulator tests
-*/sim/test/apps/test_*.py
-
-*.gv
-
-*.mtx
-*.tns
-
-# Temporary matrices
+# Temporary or generated tensor directories 
 tmp_mat*/
+tiles/
+synthetic/
 
-# Jupyter notebook checkpoints
-.ipynb_checkpoints/
+# Generated SAM simulator tests
+*/sim/test/apps/test_*.py
 
-# Generated formatted tensor files
-tensor_*_mode_*
+# Temporary scripts
+download_suitesparse_partial.sh
 
-# Tensor files
-tiles/
+# Network Filesystem 
+.nfs*

Makefile

@@ -57,16 +57,16 @@ guard-%:
 
 # ---- Run SAM python simulator stuff ----
 csv: 
-	scripts/pytest_suitesparse_with_benchmarks.sh
+	scripts/run_sam_sim/pytest_suitesparse_with_benchmarks.sh
 run-final: submodules
-	./scripts/run_suitesparse_final.sh $(realpath ./scripts/tensor_names/$(TENSOR_TXT)) 
+	./scripts/run_sam_sim/run_suitesparse_final.sh $(realpath ./scripts/tensor_names/$(TENSOR_TXT)) 
 
 run-gen: submodules
-	./scripts/run_suitesparse_generated.sh $(realpath ./scripts/tensor_names/$(TENSOR_TXT)) 
+	./scripts/run_sam_sim/run_suitesparse_generated.sh $(realpath ./scripts/tensor_names/$(TENSOR_TXT)) 
 
 tests: sam 
 	mkdir -p sam/sim/test/apps
-	python scripts/test_generating_code.py
+	python scripts/gen_sam_apps/test_generating_code.py
 
 # ---- Build taco and make sam graphs ----
 .PHONY: sam
@@ -109,10 +109,10 @@ endif
 # ---- Setup proper environment stuff ----
 suitesparse-formats: guard-SUITESPARSE_FORMATTED_PATH guard-SUITESPARSE_PATH
 	rm -rf ${SUITESPARSE_FORMATTED_PATH}/*
-	set -e && ./scripts/generate_suitesparse_formats.sh
+	set -e && ./scripts/formatting/generate_suitesparse_formats.sh
 
 frostt-formats: taco/build guard-FROSTT_FORMATTED_PATH guard-FROSTT_PATH
-	./scripts/generate_frostt_formats.sh
+	./scripts/formatting/generate_frostt_formats.sh
 
 .PHONY: env
 env:

requirements.txt

@@ -25,5 +25,6 @@ PyYAML==6.0
 requests==2.28.2
 scipy==1.10.0
 six==1.16.0
+sparse==0.13.0
 tomli==2.0.1
 tqdm==4.64.1

sam/util.py

@@ -7,7 +7,7 @@
 import shutil
 import numpy as np
 import math
-import pydata
+import sparse
 
 from pathlib import Path
 from dataclasses import dataclass
@@ -142,7 +142,7 @@ def load(self, path):
 
 
 # PydataSparseTensorLoader loads a sparse tensor from a file into
-# a pydata.sparse tensor.
+# a sparse tensor.
 # class PydataSparseTensorLoader:
 #     def __init__(self):
 #         self.loader = TnsFileLoader()
@@ -547,7 +547,7 @@ def load(self, tensor, suiteSparse, cast, format_str):
 
 
 # PydataMatrixMarketTensorLoader loads tensors in the matrix market format
-# into pydata.sparse matrices.
+# into sparse matrices.
 # class PydataMatrixMarketTensorLoader:
 #     def __init__(self):
 #         pass
@@ -601,7 +601,7 @@ def safeCastPydataTensorToInts(tensor):
         # else:
         #     data[i] = int(tensor.data[i])
         data[i] = round_sparse(tensor.data[i])
-    return pydata.sparse.COO(tensor.coords, data, tensor.shape)
+    return sparse.COO(tensor.coords, data, tensor.shape)
 
 
 def parse_taco_format(infilename, outdir, tensorname, format_str):

scripts/README.md

@@ -1,4 +1,21 @@
-----------------------------------
-| File Name | Usage | Description |
------------------------------------
-| 
+# SAM Scripts
+
+| Folder Name 		| Description 									|
+| ------------		| ----------------------- 							|
+| `artifact/` 		| Scripts used solely for The Sparse Abstract Machine ASPLOS 2023 AE 		| 
+| `formatting/` 	| Scripts used to format tensors from .mtx or .tns to other compressed formats 	|
+| `gen_sam_apps/` 	| Scripts used to generate SAM apps from the compiler 				| 
+| `get_data/` 		| Scripts used to download and unpack datasets  				| 
+| `logs/` 		| Saved log files 								| 
+| `run_cpu/` 		| Scripts used to run the CPU baseline code 					| 
+| `run_onyx/` 		| Scripts used to run Onyx simulations 						| 
+| `run_sam_sim/` 	| Scripts used to run SAM simulations via Pytest 				| 
+| `stats/` 		| Scripts used to compute statistics on data (for DSE/development) 		| 
+| `tensor_names` 	| Scripts and files that have all the dataset tensor names 			| 
+| `tiling/` 		| Scripts used to generate, run, and check tiled simulations 			| 
+| `util/` 		| Utility code used by scripts under the `scripts/` directory 			| 
+
+
+Each folder has a README with a description of the corresponding scripts in
+that directory.  Also For each individual script, see the first line comment for how
+to run the command. 

scripts/artifact/README.md

@@ -0,0 +1,10 @@
+# Artifact Scripts
+
+The `scripts/artifact/` folder contains scripts used only for artifact evaluation of "The Sparse Abstract Machine" ASPLOS 2023. 
+
+1. `artifact_docker_copy.py` - Python script that copies figures from docker to local machine for viewing.  
+2. `collect_node_counts.py` - Python script that counts and prints the number of each SAM primitive for Table 1 in "The Sparse Abstract Machine". 
+3. `plot_memory_model.py` - Python script that plots the memory model graph (Figure  
+4. `plot_stream_overhead.py` - 
+5. `stream_overhead.sh` - 
+

scripts/formatting/README.md

@@ -0,0 +1,15 @@
+# Formatting
+
+The `scripts/formatting/` folder contains scripts used to format the datasets from mtx/tns files to the seg/crd/vals arrays for CSF. The CSF files are expected by both the Onyx AHA flow and the SAM simulator.
+
+1. `datastructure_suitesparse.py` - Python script used by
+   `generate_suitesparse_formats.sh` to format from mtx to CSF files. 
+2. `datastructure_tns.py` - Python script used by
+   `generate_frostt_formats.sh` to format from tns to CSF files. 
+3. `download_unpack_format_suitesparse.sh` - Script that downloads, unpacks,
+   and formats a list of suitesparse matrices. To download and unpack it
+   calls scripts in `scripts/get_data`.
+4. `generate_frostt_formats.sh` - Formats all FROSTT datasets. FIXME: This file needs fixing as it uses the old CSF formatting (e.g. `matrix_name/B_seg0`) instead of the new one (e.g. `app/tensor_B_mode_0_seg`)  
+5. `generate_suitesparse_formats.sh` - Formats all SuiteSparse datasets 
+
+Formatted CSF files should reside in `$SUITESPARSE_FORMATTED_PATH` for SuiteSparse matrices. 

scripts/datastructure_suitesparse.py → scripts/formatting/datastructure_suitesparse.py

@@ -6,7 +6,7 @@
 
 from pathlib import Path
 
-from util import FormatWriter, SuiteSparseTensor, InputCacheSuiteSparse
+from scripts.util.util import FormatWriter, SuiteSparseTensor, InputCacheSuiteSparse
 from sam.util import SUITESPARSE_FORMATTED_PATH, ScipyTensorShifter
 
 all_formats = ["coo", "cooT", "csr", "dcsr", "dcsc", "csc", "dense", "denseT"]

scripts/datastructure_tns.py → scripts/formatting/datastructure_tns.py

@@ -1,7 +1,7 @@
 import argparse
 import os
 from pathlib import Path
-from util import parse_taco_format
+from sam.util import parse_taco_format
 
 cwd = os.getcwd()