From b3808473c3ba767339d4d57180369bc74267f1b5 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 25 Jul 2024 09:01:57 +0200 Subject: [PATCH 1/7] Enabling allocate lots of resources test again with more debugging messages --- test/Base_Tests/src/Runtime/GC_Example.enso | 6 +++--- test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/Base_Tests/src/Runtime/GC_Example.enso b/test/Base_Tests/src/Runtime/GC_Example.enso index 72507f7cb933..97a1c61b58d1 100644 --- a/test/Base_Tests/src/Runtime/GC_Example.enso +++ b/test/Base_Tests/src/Runtime/GC_Example.enso @@ -15,11 +15,11 @@ type My_Resource close_resource resource = resource.close -repeat_cleanup_until_done counter = +repeat_cleanup_until_done counter println = go i = if counter.get == 0 then Nothing else if i % 100 == 0 then - IO.println "Still "+counter.get.to_text+" resources to clean up..." + println "Still "+counter.get.to_text+" resources to clean up..." Runtime.gc @Tail_Call go i+1 go 1 @@ -32,7 +32,7 @@ perform_test n:Integer println = My_Resource.allocate counter println "Cleaning up..." - repeat_cleanup_until_done counter + repeat_cleanup_until_done counter println println "All cleaned up! Remaining: "+counter.get.to_text counter.get diff --git a/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso b/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso index 748a311f3e67..7ab051b7d9ed 100644 --- a/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso +++ b/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso @@ -58,8 +58,8 @@ add_specs suite_builder = suite_builder.group "Managed_Resource" group_builder-> r_3 = Panic.recover Any <| Managed_Resource.bracket 42 (_-> Nothing) (_-> Panic.throw "action") r_3.catch . should_equal "action" - group_builder.specify "allocate lots of resources at once" pending=(if Environment.get "CI" . is_nothing . not then "GC Example is marked as pending as it was causing problems on the CI.") <| - remaining = GC_Example.perform_test 100000 (_->Nothing) + group_builder.specify "allocate lots of resources at once" <| + remaining = GC_Example.perform_test 100000 IO.println remaining . should_equal 0 main filter=Nothing = From 647ed1e7ddbfa15dff58cbaef631db4696ea05b4 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 25 Jul 2024 09:55:12 +0200 Subject: [PATCH 2/7] Use request state to select a choosen Thread --- .../interpreter/runtime/ResourceManager.java | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/ResourceManager.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/ResourceManager.java index d78746647ba5..5a9bbaad6274 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/ResourceManager.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/ResourceManager.java @@ -177,29 +177,34 @@ private final class ProcessItems extends ThreadLocalAction implements Runnable { */ @Override protected void perform(ThreadLocalAction.Access access) { + var isMyThreadChoosen = false; for (; ; ) { Item[] toProcess; synchronized (pendingItems) { - request.cancel(false); + if (!isMyThreadChoosen) { + if (request == null || request.isCancelled()) { + // some thread is already handing the request + return; + } else { + // I am choosen and I will loop and process pendingItems + // until they are available + isMyThreadChoosen = true; + // signal others this request has choosen thread + request.cancel(false); + } + } if (pendingItems.isEmpty()) { - // nothing to process, - // signal request is finished + // nothing to process anymore, + // signal request is finished and new one shall be scheduled request = null; return; } toProcess = pendingItems.toArray(Item[]::new); - // mark as being processed - pendingItems.set(0, null); + pendingItems.clear(); } - try { - for (var it : toProcess) { - it.finalizeNow(context); - removeFromItems(it); - } - } finally { - synchronized (pendingItems) { - pendingItems.subList(0, toProcess.length).clear(); - } + for (var it : toProcess) { + it.finalizeNow(context); + removeFromItems(it); } } } From 51df8a64a09afe0f58a03b9cea83b94d3ac719e8 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 25 Jul 2024 17:27:24 +0200 Subject: [PATCH 3/7] Don't capture Base_Tests output --- build/build/src/engine/context.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs index ce0ee683d58b..84a1f684b8f7 100644 --- a/build/build/src/engine/context.rs +++ b/build/build/src/engine/context.rs @@ -661,12 +661,10 @@ pub async fn runner_sanity_test( .args(["--run", repo_root.test.join("Base_Tests").as_str()]) .set_env_opt(ENSO_JAVA, enso_java)? .set_env(ENSO_DATA_DIRECTORY, engine_package)? - .run_stdout() - .await?; - ensure!( - test_base.contains("0 tests failed."), - "All tests shall succeed. Output:\n{test_base}", - ); + .run_ok() + .await; + test_base + } else { + Ok(()) } - Ok(()) } From 3c934304a4b9f3db4aa332e2584912602cafc4fd Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 26 Jul 2024 10:33:47 +0200 Subject: [PATCH 4/7] Allocate resources first and only then clean them --- test/Base_Tests/src/Runtime/GC_Example.enso | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/Base_Tests/src/Runtime/GC_Example.enso b/test/Base_Tests/src/Runtime/GC_Example.enso index 97a1c61b58d1..b7c01a2feb75 100644 --- a/test/Base_Tests/src/Runtime/GC_Example.enso +++ b/test/Base_Tests/src/Runtime/GC_Example.enso @@ -15,9 +15,12 @@ type My_Resource close_resource resource = resource.close -repeat_cleanup_until_done counter println = +repeat_cleanup_until_done holder counter println = go i = if counter.get == 0 then Nothing else + if i % 100 == 3 then + println "Releasing holder with resources: "+holder.get.length.to_text + holder.put [] if i % 100 == 0 then println "Still "+counter.get.to_text+" resources to clean up..." Runtime.gc @@ -25,14 +28,18 @@ repeat_cleanup_until_done counter println = go 1 perform_test n:Integer println = - + resource_holder = Ref.new Nothing counter = Ref.new 0 - println "Allocating "+n.to_text+" resources..." - 0.up_to n . each _-> - My_Resource.allocate counter - println "Cleaning up..." - repeat_cleanup_until_done counter println + println "Allocating "+n.to_text+" resources..." + allocate_resources ref = + all = 0.up_to n . map _-> + My_Resource.allocate counter + ref.put all + allocate_resources resource_holder + + println "Cleaning up... "+resource_holder.get.length.to_text+" resources" + repeat_cleanup_until_done resource_holder counter println println "All cleaned up! Remaining: "+counter.get.to_text counter.get From 89cb205d1584e7264f543edbb8fd994b52e649ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 26 Jul 2024 13:58:39 +0200 Subject: [PATCH 5/7] remove 3 'empty' GC runs --- test/Base_Tests/src/Runtime/GC_Example.enso | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/Base_Tests/src/Runtime/GC_Example.enso b/test/Base_Tests/src/Runtime/GC_Example.enso index b7c01a2feb75..72570c161a31 100644 --- a/test/Base_Tests/src/Runtime/GC_Example.enso +++ b/test/Base_Tests/src/Runtime/GC_Example.enso @@ -15,12 +15,9 @@ type My_Resource close_resource resource = resource.close -repeat_cleanup_until_done holder counter println = +repeat_cleanup_until_done counter println = go i = if counter.get == 0 then Nothing else - if i % 100 == 3 then - println "Releasing holder with resources: "+holder.get.length.to_text - holder.put [] if i % 100 == 0 then println "Still "+counter.get.to_text+" resources to clean up..." Runtime.gc @@ -39,10 +36,12 @@ perform_test n:Integer println = allocate_resources resource_holder println "Cleaning up... "+resource_holder.get.length.to_text+" resources" - repeat_cleanup_until_done resource_holder counter println + # We replace the vector with Nothing to make the resources inaccessible and schedule them for GC + resource_holder.put Nothing + + repeat_cleanup_until_done counter println println "All cleaned up! Remaining: "+counter.get.to_text counter.get main n=1000000 = perform_test n IO.println - From 189bb1eb7bbcd526eb17798869f8bef41078b629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 26 Jul 2024 14:06:29 +0200 Subject: [PATCH 6/7] add timeout to test --- test/Base_Tests/src/Runtime/GC_Example.enso | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/Base_Tests/src/Runtime/GC_Example.enso b/test/Base_Tests/src/Runtime/GC_Example.enso index 72570c161a31..64b7353820b0 100644 --- a/test/Base_Tests/src/Runtime/GC_Example.enso +++ b/test/Base_Tests/src/Runtime/GC_Example.enso @@ -1,4 +1,5 @@ from Standard.Base import all +import Standard.Base.Errors.Illegal_State.Illegal_State import Standard.Base.Runtime.Managed_Resource.Managed_Resource import Standard.Base.Runtime.Ref.Ref @@ -16,8 +17,13 @@ type My_Resource close_resource resource = resource.close repeat_cleanup_until_done counter println = + deadline = Date_Time.now + (Duration.new minutes=5) go i = if counter.get == 0 then Nothing else + if Date_Time.now > deadline then + message = "Timeout has been reached, but there are "+counter.get.to_text+" resources remaining that were not cleaned up." + println message + Panic.throw (Illegal_State.Error message) if i % 100 == 0 then println "Still "+counter.get.to_text+" resources to clean up..." Runtime.gc @@ -35,7 +41,7 @@ perform_test n:Integer println = ref.put all allocate_resources resource_holder - println "Cleaning up... "+resource_holder.get.length.to_text+" resources" + println "Cleaning up "+resource_holder.get.length.to_text+" resources" # We replace the vector with Nothing to make the resources inaccessible and schedule them for GC resource_holder.put Nothing From d96d6154d536193582aaaaf485f1fb4dac8c0541 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 26 Jul 2024 15:55:49 +0200 Subject: [PATCH 7/7] Don't print any messages when the test is successful --- test/Base_Tests/src/Runtime/GC_Example.enso | 28 +++++++++++-------- .../src/Runtime/Managed_Resource_Spec.enso | 10 +++++-- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/test/Base_Tests/src/Runtime/GC_Example.enso b/test/Base_Tests/src/Runtime/GC_Example.enso index 64b7353820b0..8df17ab0fdec 100644 --- a/test/Base_Tests/src/Runtime/GC_Example.enso +++ b/test/Base_Tests/src/Runtime/GC_Example.enso @@ -18,17 +18,21 @@ close_resource resource = resource.close repeat_cleanup_until_done counter println = deadline = Date_Time.now + (Duration.new minutes=5) - go i = - if counter.get == 0 then Nothing else - if Date_Time.now > deadline then - message = "Timeout has been reached, but there are "+counter.get.to_text+" resources remaining that were not cleaned up." - println message - Panic.throw (Illegal_State.Error message) - if i % 100 == 0 then - println "Still "+counter.get.to_text+" resources to clean up..." - Runtime.gc - @Tail_Call go i+1 - go 1 + while_counter_not_zero i = if counter.get != 0 then + if i % 10 == 0 then + println "Still "+counter.get.to_text+" resources to clean up..." + + Runtime.gc + + deadline_not_reached at = if at < deadline then True else + message = "Timeout has been reached, but there are "+counter.get.to_text+" resources remaining that were not cleaned up." + println message + False + + if deadline_not_reached Date_Time.now then + @Tail_Call while_counter_not_zero i+1 + + while_counter_not_zero 1 perform_test n:Integer println = resource_holder = Ref.new Nothing @@ -46,7 +50,7 @@ perform_test n:Integer println = resource_holder.put Nothing repeat_cleanup_until_done counter println - println "All cleaned up! Remaining: "+counter.get.to_text + println "Cleaning finished! Remaining: "+counter.get.to_text counter.get main n=1000000 = diff --git a/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso b/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso index 7ab051b7d9ed..14f3cd6730cc 100644 --- a/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso +++ b/test/Base_Tests/src/Runtime/Managed_Resource_Spec.enso @@ -59,8 +59,14 @@ add_specs suite_builder = suite_builder.group "Managed_Resource" group_builder-> r_3.catch . should_equal "action" group_builder.specify "allocate lots of resources at once" <| - remaining = GC_Example.perform_test 100000 IO.println - remaining . should_equal 0 + messages = Vector.build builder-> + builder.append '\n' + remaining = GC_Example.perform_test 100000 builder.append + if remaining == 0 then + builder.append "OK" + + if messages.last != "OK" then + Test.fail (messages.join '\n') main filter=Nothing = suite = Test.build suite_builder->