lua4jvm: Add ipairs() for tables, fix array-table bugs

lua4jvm/src/main/java/fi/benjami/code4jvm/lua/runtime/LuaTable.java

@@ -239,7 +239,8 @@ private void enlargeTables() {
 	private void enlargeArray() {
 		shapeChanged();
 
-		var newCapacity = Math.max(4, (int) (arrayCapacity * 1.3f));
+		// TODO don't always double capacity; but make sure this always adds at least 3 slots (to account for gaps)!
+		var newCapacity = Math.max(4, arrayCapacity * 2);
 		var newTable = new Object[newCapacity + keys.length];
 
 		// Copy old array part
@@ -360,7 +361,15 @@ public Object[] next(Object prevKey) {
 	 * @return A table iterator.
 	 */
 	public Iterator iterator() {
-		return new Iterator();
+		return new Iterator(false);
+	}
+
+	/**
+	 * Creates a stateful table iterator that only iterates through the array part.
+	 * @return A table iterator.
+	 */
+	public Iterator arrayIterator() {
+		return new Iterator(true);
 	}
 
 	/**
@@ -378,10 +387,12 @@ public Iterator iterator() {
 	 */
 	public class Iterator {
 
+		private final boolean arrayOnly;
 		private boolean array;
 		private int index;
 
-		private Iterator() {
+		private Iterator(boolean arrayOnly) {
+			this.arrayOnly = arrayOnly;
 			this.array = true;
 			this.index = 0;
 		}
@@ -392,8 +403,15 @@ public boolean next() {
 
 		private boolean nextArray() {
 			index++;
-			if (index >= arraySize) {
+			if (index >= arraySize || table[index] == null) {
 				// Reached array end
+				// ... but if there were previously gaps, some hash table entries
+				// might also need to be visible to array iterators
+				var nextEntry = getRaw((double) index);
+				if (nextEntry != null) {
+					return true;
+				} // else: REALLY reached array end
+
 				array = false;
 				index = arrayCapacity - 1; // Jump over possible unused array space
 				return nextTable(); // Table might or might not have entries
@@ -402,6 +420,10 @@ private boolean nextArray() {
 		}
 
 		private boolean nextTable() {
+			if (arrayOnly) {
+				return false; // ipairs() like behavior
+			}
+
 			// Iterate over empty space until we find next entry
 			for (var i = index + 1; i < table.length; i++) {
 				if (table[i] != null) {

lua4jvm/src/main/java/fi/benjami/code4jvm/lua/stdlib/BasicLib.java

@@ -194,7 +194,8 @@ public static String type(Object value) {
 	}
 
 	private static final JavaFunction INTRINSIC_ITERATOR = InternalLib.FUNCTIONS.get("intrinsicIterator"),
-			TABLE_ITERATOR = InternalLib.FUNCTIONS.get("tableIterator");
+			TABLE_ITERATOR = InternalLib.FUNCTIONS.get("tableIterator"),
+			ARRAY_ITERATOR = InternalLib.FUNCTIONS.get("arrayIterator");
 
 	@LuaExport("pairs")
 	@LuaIntrinsic("iteratorFor")
@@ -228,6 +229,38 @@ private static Object[] pairs(@Inject LuaVm vm, Object iterable) throws Throwabl
 		}
 	}
 
+	@LuaExport("ipairs")
+	@LuaIntrinsic("iteratorFor")
+	private static Object[] ipairsStateful(@Inject LuaVm vm, Object iterable) throws Throwable {
+		if (iterable instanceof LuaTable table
+				&& (table.metatable() == null || table.metatable().get("__ipairs") == null)) {
+			// Normal Lua table; let's cheat a bit and use a stateful table iterator (intrinsic path)
+			return new Object[] {INTRINSIC_ITERATOR, table.arrayIterator()};
+		}
+
+		// Aside of the above fast path, delegate to normal pairs
+		return pairs(vm, iterable);
+	}
+
+	@LuaExport("ipairs")
+	private static Object[] ipairs(@Inject LuaVm vm, Object iterable) throws Throwable {
+		if (iterable instanceof LuaTable table) {
+			if (table.metatable() != null) {
+				var metamethod = table.metatable().get("__ipairs");
+				if (metamethod != null) {
+					// Call __pairs and use whatever it returns as an iterator
+					var target = LuaLinker.linkCall(new LuaCallSite(null, CallSiteOptions.nonFunction(vm, LuaType.TABLE)),
+							metamethod, table);
+					return (Object[]) target.target().invoke(metamethod, table);
+				}
+			}
+			// No __pairs, just iterate over the table normally (non-intrinsic path)
+			return new Object[] {ARRAY_ITERATOR, table};
+		} else {
+			throw new LuaException("value not iterable");
+		}
+	}
+
 	@LuaExport("next")
 	public static Object[] next(LuaTable table, Object prevKey) {
 		return table.next(prevKey);

lua4jvm/src/main/java/fi/benjami/code4jvm/lua/stdlib/InternalLib.java

@@ -41,4 +41,11 @@ private static Object[] intrinsicIterator(LuaTable.Iterator iterator) {
 	private static Object[] tableIterator(LuaTable table, String prevKey) {
 		return table.next(prevKey);
 	}
+
+	@LuaExport("arrayIterator")
+	private static Object[] arrayIterator(LuaTable table, Object prevKey) {
+		int index = prevKey == null ? 1 : ((Number) prevKey).intValue();
+		index++;
+		return new Object[] {(double) index, table.get(index)};
+	}
 }

lua4jvm/src/test/java/fi/benjami/code4jvm/lua/test/BasicLibTest.java

@@ -11,7 +11,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import fi.benjami.code4jvm.lua.LuaVm;
@@ -152,4 +151,21 @@ public void next() throws Throwable {
 				""");
 		assertArrayEquals(new Object[] {"foo", "bar", null, null}, results);
 	}
+
+	@Test
+	public void ipairs() throws Throwable {
+		vm.execute("""
+				tbl = {"foo", "bar", "baz", test = 1, second = 2}
+				out = {}
+				for k,v in ipairs(tbl) do
+					out[#out+1] = v
+				end
+				""");
+		var out = (LuaTable) vm.globals().get("out");
+		assertEquals("foo", out.get(1d));
+		assertEquals("bar", out.get(2d));
+		assertEquals("baz", out.get(3d));
+		assertEquals(null, out.get("test"));
+		assertEquals(null, out.get("second"));
+	}
 }

lua4jvm/src/test/java/fi/benjami/code4jvm/lua/test/TableTest.java

@@ -1,8 +1,10 @@
 package fi.benjami.code4jvm.lua.test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -275,4 +277,98 @@ public void tableIterators() {
 		assertEquals(keys, nextKeys);
 		assertEquals(values, nextVals);
 	}
+
+	@Test
+	public void arrayIterator() {
+		var table = new LuaTable();
+		table.set(1d, "test");
+		table.set(2d, "second");
+		table.set(4d, "third"); // This is after gap, shouldn't be visible!
+		table.set("foo", 1d);
+		table.set("bar", 2d);
+		table.set("baz", 3d);
+
+		{
+			var itKeys = new HashSet<>();
+			var itVals = new HashSet<>();
+
+			var it = table.arrayIterator();
+			while (it.next()) {
+				itKeys.add(it.key());
+				itVals.add(it.value());
+			}
+
+			var keys = Set.of(1d, 2d);
+			var values = Set.of("test", "second");
+
+			assertEquals(keys, itKeys);
+			assertEquals(values, itVals);
+		}
+
+		table.set(3d, "later!"); // This should close the gap
+
+		{
+			var itKeys = new HashSet<>();
+			var itVals = new HashSet<>();
+
+			var it = table.arrayIterator();
+			while (it.next()) {
+				itKeys.add(it.key());
+				itVals.add(it.value());
+			}
+
+			var keys = Set.of(1d, 2d, 3d, 4d);
+			var values = Set.of("test", "second", "later!", "third");
+
+			assertEquals(keys, itKeys);
+			assertEquals(values, itVals);
+		}
+	}
+
+	@Test
+	public void fakeArray() {
+		// If there were too big gaps in table for it to be array,
+		// filling those gaps should still make the entries visible
+		var table = new LuaTable();
+		table.set(1d, "test");
+		table.set(10d, "second");
+		table.set(100d, "third");
+
+		{
+			var it = table.arrayIterator();
+			assertTrue(it.next());
+			assertEquals(1d, it.key());
+			assertEquals("test", "test");
+			assertFalse(it.next());
+		}
+
+		for (double i = 2; i < 10; i++) {
+			table.set(i, i);
+		}
+
+		{
+			var itKeys = new HashSet<>();
+			var itVals = new HashSet<>();
+
+			var it = table.arrayIterator();
+			while (it.next()) {
+				itKeys.add(it.key());
+				itVals.add(it.value());
+			}
+
+			var keys = Set.of(1d, 2d, 3d, 4d, 5d, 6d, 7d, 8d, 9d, 10d);
+			var values = Set.of("test", "second", 2d, 3d, 4d, 5d, 6d, 7d, 8d, 9d);
+
+			assertEquals(keys, itKeys);
+			assertEquals(values, itVals);
+		}
+	}
+
+	@Test
+	public void enlargeArray() {
+		// Wrong multiplier for array size may cause AOOBE in edge cases like this
+		var table = new LuaTable();
+		table.set(1d, "test");
+		table.set(5d, "second");
+	}
 }