Moved example scripts into proper integration tests

Also adjusted makefiles to allow easy invoking of the tests.

Adjusted and updated CI to invoke tests correctly.

Fixed #141

.github/workflows/continuous-integration-v2.yml

@@ -14,48 +14,73 @@ on:
       - v2
   workflow_dispatch:
 
-#CI workflows using a matrix
+#CI workflows using the matrix strategy, skipping GDB if it's not supported for the platform
 jobs:
   run-test-cases:
     continue-on-error: true
     strategy:
       matrix:
         platforms:
-          - { os: ubuntu-latest, preinstall: sudo apt-get install gdb, gdb_skip: false }
-          - { os: windows-latest, preinstall: , gdb_skip: false }
-          - { os: macos-latest, preinstall: , gdb_skip: true }
+          - { os: ubuntu-latest, preinstall: sudo apt-get install gdb, gdb_enabled: true }
+          - { os: windows-latest, preinstall: , gdb_enabled: true }
+          - { os: macos-latest, preinstall: , gdb_enabled: false }
         commands:
-          - { exec: make tests, gdb: false }
-          - { exec: make tests-gdb, gdb: true }
+          - { exec: make test-cases, gdb: false }
+          - { exec: make test-cases-gdb, gdb: true }
 
     runs-on: ${{ matrix.platforms.os }}
     steps:
       - uses: actions/checkout@v4
       - name: Preinstall dependencies
+        if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
         run: ${{ matrix.platforms.preinstall }}
-      - name: run the test cases
-        if: matrix.commands.gdb == false || matrix.platforms.gdb_skip == false
+      - name: run the tests
+        if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
         run: ${{ matrix.commands.exec }}
-
-  #TODO: hook this up to real script files, preferably in the test section
-  run-test-repl-scripts:
-    continue-on-error: true
+
+  run-test-integrations:
     needs: run-test-cases
+    continue-on-error: true
+    strategy:
+      matrix:
+        platforms:
+          - { os: ubuntu-latest, preinstall: sudo apt-get install gdb, gdb_enabled: true }
+          - { os: windows-latest, preinstall: , gdb_enabled: true }
+          - { os: macos-latest, preinstall: , gdb_enabled: false }
+        commands:
+          - { exec: make test-integrations, gdb: false }
+          - { exec: make test-integrations-gdb, gdb: true }
+
+    runs-on: ${{ matrix.platforms.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Preinstall dependencies
+        if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
+        run: ${{ matrix.platforms.preinstall }}
+      - name: run the tests
+        if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
+        run: ${{ matrix.commands.exec }}
+
+  run-test-benchmarks:
+    if: false #Not ready yet
+    needs: run-test-integrations
+    continue-on-error: true
     strategy:
       matrix:
         platforms:
-          - { os: ubuntu-latest }
-          - { os: windows-latest }
-          - { os: macos-latest }
+          - { os: ubuntu-latest, preinstall: sudo apt-get install gdb, gdb_enabled: true }
+          - { os: windows-latest, preinstall: , gdb_enabled: true }
+          - { os: macos-latest, preinstall: , gdb_enabled: false }
         commands:
-          - { build: make repl, run: out/repl.exe -f '../scripts/example.toy' }
-          - { build: make repl, run: out/repl.exe -f '../scripts/example-print.toy' }
-          - { build: make repl, run: out/repl.exe -f '../scripts/example-variables.toy' }
+          - { exec: make test-benchmarks, gdb: false }
+          - { exec: make test-benchmarks-gdb, gdb: true }
 
     runs-on: ${{ matrix.platforms.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: compile the repl
-        run: ${{ matrix.commands.build }}
-      - name: run the repl scripts
-        run: ${{ matrix.commands.run }}
+      - name: Preinstall dependencies
+        if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
+        run: ${{ matrix.platforms.preinstall }}
+      - name: run the tests
+        if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
+        run: ${{ matrix.commands.exec }}

CONTRIBUTING.md

@@ -16,7 +16,7 @@ v2 is a ground-up rewrite, with additions, changes and deletions to the language
 
 The [Issue Tracker](https://github.com/Ratstail91/Toy/issues) is a good place to see what tasks and issues are currently waiting to be addressed. The [toy.h](https://github.com/Ratstail91/Toy/blob/v2/source/toy.h) source file is a quick way to see what building blocks are available in the source code. There are also a number of comments prepended with `TODO` scattered throughout the source code, as reminders of planned features.
 
-The [test cases](https://github.com/Ratstail91/Toy/tree/v2/tests/cases), which test individual parts of the code in isolation, can be a good way to see how those parts are used. Likewise, the [REPL](https://github.com/Ratstail91/Toy/tree/v2/repl) shows a practical usage of Toy.
+The [tests directory](https://github.com/Ratstail91/Toy/tree/v2/tests), which holds a collection of automated tests for the CI pipeline, can be a good way to see how those parts are used. Likewise, the [REPL](https://github.com/Ratstail91/Toy/tree/v2/repl) shows a practical usage of Toy.
 
 *v2 is under heavy development, and as such may not be in a working state yet. Your patience and feedback can help, but missing features such as a documentation website are coming, eventually.*
 
@@ -30,10 +30,10 @@ graph TB
     Toy_Value ---> Toy_String
     Toy_Value ---> Toy_Stack
     Toy_Value ---> Toy_Table
-    Toy_Array
+    Toy_Value ---> Toy_Array
 ```
 
-In addition, [toy_common.h](https://github.com/Ratstail91/Toy/blob/v2/source/toy_common.h) grants platform portability and version info, while [toy_console_colors.h](https://github.com/Ratstail91/Toy/blob/v2/source/toy_console_colors.h) provides string constants as macros that help with console output (where supported).
+In addition, [toy_common.h](https://github.com/Ratstail91/Toy/blob/v2/source/toy_common.h) grants platform portability and version info, while [toy_console_colors.h](https://github.com/Ratstail91/Toy/blob/v2/source/toy_console_colors.h) provides string constants as macros that can help with console output (where supported).
 
 # Coding Habits
 
@@ -43,15 +43,29 @@ Here's a few coding habits that I use to keep the source code consistent. While
 
 When adding a new piece of code, it must be thoroughly tested via a [test case](https://github.com/Ratstail91/Toy/tree/v2/tests/cases). If it has multiple features, they should be tested individually, and in combination with each other. Any kind of corner case which can cause an issue on any supported platform must be resolved (I'm happy to help with this, if needed).
 
+Once a feature has been tested on its own, it can be added to or expanded in the [integration tests](https://github.com/Ratstail91/Toy/tree/v2/tests/integrations).
+
 This is probably the most important habit listed here. While I'm not too fussy as to how the tests are written, they do need to prove that the code works flawlessly. Toy is intended to be used by others (potentially many others), so please write simple and straight forward tests to ensure correctness.
 
 ## Tabs, 4 Characters Wide
 
-I use tabs over spaces, with a width of 4. I don't have a linter, please don't make me use one.
+I use tabs over spaces, with a width of 4. I don't have a linter, please don't make me use one. For those who care, here's my `.vimrc`:
+
+```bash
+" Load the defaults
+runtime defaults.vim
+
+" my custom stuff
+set tabstop=4
+set shiftwidth=4
+
+set autoindent
+set smartindent
+```
 
 ## Error Messages
 
-Fatal errors have this general format:
+Fatal errors in the source code have this general format:
 
 ```c
 fprintf(stderr, TOY_CC_ERROR "ERROR: [Info]\n" TOY_CC_RESET);
@@ -60,7 +74,7 @@ exit(-1);
 
 The use of `fprintf()` will ensure the error is written to the console, and allows extra information to be printed - just replace `[Info]` with the relevant output. These kinds of fatal errors are intended to catch issues with the language itself, rather than errors in the Toy scripts.
 
-In the test cases, the `exit(-1)` is instead replaced with `return -1` to allow `main()` to clean up that test set, and run others if needed.
+In the test cases, the `exit(-1)` is instead replaced with `return -1` to allow `main()` to clean up that test case, and run others if needed.
 
 ## Naming Things
 
@@ -124,5 +138,5 @@ The directories in the repository's root have certain intended uses. If you find
 | scripts | Storage for various example scripts written in Toy that can be loaded and executed by the repl. |
 | source | The source directory for the core of the Toy programming language. |
 | tests | The source directory for the testing systems. Within, `cases/` is used for test cases, `benchmarks/` for benchmarking, etc. |
-| tools | The source directory for various external tools. |
+| tools | The source directory for various standalone tools. |
 

README.md

@@ -15,7 +15,7 @@ This repository holds the reference implementation for Toy version 2.x, written
 * Simple C-like syntax
 * Intermediate AST representation
 * Strong, but optional type system
-* First-class functions
+* First-class functions and types
 * Extensible via external libraries
 * Can re-direct output, error and assertion failure messages
 * Open source under the zlib license
@@ -39,14 +39,13 @@ var foobar = 42;
 
 Supported platforms are: `linux-latest`, `windows-latest`, `macos-latest`, using [GitHub's standard runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories).
 
-To build the library, run `make source`.  
-To build the library and repl, run `make repl`.  
-To build and run the test cases, run `make tests`.  
-To build and run the test cases under gdb, run `make tests-gdb`.  
+To build the shared library, run `make source`.  
+To build the shared library and repl, run `make repl`.  
+To build and run the standard available tests, run `make tests`.  
 
 # Tools
 
-*Coming Soon.*
+*Coming Soon, see #126 for details.*
 
 # License
 
@@ -56,7 +55,7 @@ This source code is covered by the zlib license (see [LICENSE.md](LICENSE.md)).
 
 For a guide on how you can contribute, see [CONTRIBUTING.md](CONTRIBUTING.md).
 
-@8051Enthusiast - `fixAlignment()` trick
+@8051Enthusiast - `fixAlignment()` trick  
 @hiperiondev - v1 Disassembler, v1 porting support and feedback  
 @add00 - v1 Library support  
 @gruelingpine185 - Unofficial v1 MacOS support  

makefile

@@ -6,11 +6,15 @@
 
 #directories
 export TOY_SOURCEDIR=source
+export TOY_REPLDIR=repl
+export TOY_CASESDIR=tests/cases
+export TOY_INTEGRATIONSDIR=tests/integrations
+export TOY_BENCHMARKSDIR=tests/benchmarks
 export TOY_OUTDIR=out
 export TOY_OBJDIR=obj
 
 #targets
-all:
+#all:
 
 .PHONY: source
 source:
@@ -20,13 +24,39 @@ source:
 repl: source
 	$(MAKE) -C repl -k
 
+#various kinds of available tests
 .PHONY: tests
-tests: clean
-	$(MAKE) -C tests -k
+tests: clean test-cases test-integrations test-benchmarks
 
-.PHONY: tests-gdb
-tests-gdb: clean
-	$(MAKE) -C tests all-gdb -k
+.PHONY: test-cases
+test-cases:
+	$(MAKE) -C $(TOY_CASESDIR) -k
+
+.PHONY: test-integrations
+test-integrations:
+	$(MAKE) -C $(TOY_INTEGRATIONSDIR) -k
+
+.PHONY: test-benchmarks
+test-benchmarks:
+	$(MAKE) -C $(TOY_BENCHMARKSDIR) -k
+
+#same as above, but with GDB
+#.PHONY: gdb
+#gdb: clean test-cases-gdb test-integrations-gdb test-benchmarks-gdb
+
+.PHONY: test-cases-gdb
+test-cases-gdb:
+	$(MAKE) -C $(TOY_CASESDIR) gdb -k
+
+.PHONY: test-integrations-gdb
+test-integrations-gdb:
+	$(MAKE) -C $(TOY_INTEGRATIONSDIR) gdb -k
+
+.PHONY: test-benchmarks-gdb
+test-benchmarks-gdb:
+	$(MAKE) -C $(TOY_BENCHMARKSDIR) gdb -k
+
+#TODO: mustfail tests
 
 #util targets
 $(TOY_OUTDIR):

repl/main.c

@@ -26,22 +26,22 @@ unsigned char* readFile(char* path, int* size) {
 		return NULL;
 	}
 
-	//
+	//read the file
 	if (fread(buffer, sizeof(unsigned char), *size, file) < *size) {
 		fclose(file);
 		*size = -2; //singal a read error
 		return NULL;
 	}
 
-	fclose(file);
-
 	buffer[(*size)++] = '\0';
+
+	//clean up and return
+	fclose(file);
 	return buffer;
 }
 
 int getDirPath(char* dest, const char* src) {
 	//extract the directory from src, and store it in dest
-
 #if defined(_WIN32) || defined(_WIN64)
 	char* p = strrchr(src, '\\');
 #else
@@ -57,7 +57,6 @@ int getDirPath(char* dest, const char* src) {
 
 int getFileName(char* dest, const char* src) {
 	//extract the directory from src, and store it in dest
-
 #if defined(_WIN32) || defined(_WIN64)
 	char* p = strrchr(src, '\\') + 1;
 #else
@@ -96,6 +95,8 @@ void usageCmdLine(int argc, const char* argv[]) {
 void helpCmdLine(int argc, const char* argv[]) {
 	usageCmdLine(argc, argv);
 
+	printf("The Toy Programming Language, leave arguments blank for an interactive REPL.\n\n");
+
 	printf("  -h, --help\t\t\tShow this help then exit.\n");
 	printf("  -v, --version\t\t\tShow version and copyright information then exit.\n");
 	printf("  -f, --file infile\t\tParse, compile and execute the source file then exit.\n");

repl/makefile

@@ -49,34 +49,3 @@ ifeq ($(shell uname),Darwin) #dylib fix
 	install_name_tool -change ../out/libToy.dylib @executable_path/libToy.dylib $@
 	otool -L $@
 endif
-
-#util commands
-.PHONY: clean
-clean:
-ifeq ($(shell uname),Linux)
-	find . -type f -name '*.o' -delete
-	find . -type f -name '*.a' -delete
-	find . -type f -name '*.exe' -delete
-	find . -type f -name '*.dll' -delete
-	find . -type f -name '*.lib' -delete
-	find . -type f -name '*.so' -delete
-	find . -type f -name '*.dylib' -delete
-	find . -type d -name 'out' -delete
-	find . -type d -name 'obj' -delete
-else ifeq ($(OS),Windows_NT)
-	$(RM) *.o *.a *.exe *.dll *.lib *.so *.dylib
-	$(RM) out
-	$(RM) obj
-else ifeq ($(shell uname),Darwin)
-	find . -type f -name '*.o' -delete
-	find . -type f -name '*.a' -delete
-	find . -type f -name '*.exe' -delete
-	find . -type f -name '*.dll' -delete
-	find . -type f -name '*.lib' -delete
-	find . -type f -name '*.so' -delete
-	find . -type f -name '*.dylib' -delete
-	find . -type d -name 'out' -delete
-	find . -type d -name 'obj' -delete
-else
-	@echo "Deletion failed - what platform is this?"
-endif