Merge branch 'main' into mine

chains-project · Aug 2, 2023 · 1346b1d · 1346b1d
2 parents acede92 + 861a44c
commit 1346b1d
Show file tree

Hide file tree

Showing 38 changed files with 128 additions and 161 deletions.
diff --git a/.github/workflows/validate-json-files.yml b/.github/workflows/validate-json-files.yml
@@ -20,18 +20,28 @@ jobs:
           refSchemasMap: 
             '{"https://github.com/chains-project/breaking-updates/blob/main/schemas/successful-reproduction-schemas/breaking-update-dependency.schema.json": "schemas/successful-reproduction-schemas/breaking-update-dependency.schema.json"}'
 
-      - name: Validate not reproduced breaking update JSON
-        uses: cardinalby/schema-validator-action@v3
-        with:
-          schema: "schemas/not-attempted-reproduction-schemas/breaking-update.schema.json"
-          file: "data/not-reproduced-data/*.json"
-          refSchemasMap:
-            '{"https://github.com/chains-project/breaking-updates/blob/main/schemas/not-attempted-reproduction-schemas/breaking-update-dependency.schema.json": "schemas/not-attempted-reproduction-schemas/breaking-update-dependency.schema.json"}'
-
       - name: Validate reproduction failed breaking update JSON
         uses: cardinalby/schema-validator-action@v3
         with:
           schema: "schemas/unsuccessful-reproduction-schemas/breaking-update.schema.json"
           file: "data/unsuccessful-reproductions/*.json"
           refSchemasMap:
             '{"https://github.com/chains-project/breaking-updates/blob/main/schemas/unsuccessful-reproduction-schemas/breaking-update-dependency.schema.json": "schemas/unsuccessful-reproduction-schemas/breaking-update-dependency.schema.json"}'
+
+      - name: Check if JSON files exist
+        id: file_check
+        run: |
+          if find data/not-reproduced-data -name "*.json" -print -quit | grep -q .; then
+            echo "check_result=true" >> $GITHUB_OUTPUT
+          else
+            echo "check_result=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Validate not reproduced breaking update JSON
+        if: steps.file_check.outputs.check_result == 'true'
+        uses: cardinalby/schema-validator-action@v3
+        with:
+          schema: "schemas/not-attempted-reproduction-schemas/breaking-update.schema.json"
+          file: "data/not-reproduced-data/*.json"
+          refSchemasMap:
+            '{"https://github.com/chains-project/breaking-updates/blob/main/schemas/not-attempted-reproduction-schemas/breaking-update-dependency.schema.json": "schemas/not-attempted-reproduction-schemas/breaking-update-dependency.schema.json"}'
diff --git a/README.md b/README.md
@@ -101,11 +101,11 @@ java -jar target/BreakingUpdateReproducer.jar --help
 
 ## Stats
 As of Aug 2 2023:
-  * The benchmark consists of 185 reproducible breaking updates.
-    - Of these breaking updates, 59 (31.89%) fail compilation with the updated dependency.
-    - 16 (8.65%) fail tests with the updated dependency.
-    - 1 (0.54%) have dependency resolution failures with the updated dependency.
-    - 44 (23.78%) fail after updating the dependency due to maven enforcer failures.
-    - 65 (35.14%) fail due to unknown failures after updating the dependency.
-  * Overall, reproduction has been attempted for 1462 breaking updates, and 1277 (87.35%) could not be locally reproduced.
-  * For 640 potential breaking updates, reproduction has not been attempted yet.
+  * The benchmark consists of 181 reproducible breaking updates.
+    - Of these breaking updates, 55 (30.39%) fail compilation with the updated dependency.
+    - 30 (16.57%) fail tests with the updated dependency.
+    - 1 (0.55%) have dependency resolution failures with the updated dependency.
+    - 44 (24.31%) fail after updating the dependency due to maven enforcer failures.
+    - 51 (28.18%) fail due to unknown failures after updating the dependency.
+  * Overall, reproduction has been attempted for 1458 breaking updates, and 1277 (87.59%) could not be locally reproduced.
+  * For 4 potential breaking updates, reproduction has not been attempted yet.
diff --git a/data/benchmark/097d93fbb2f9998ef1390ad10f92d1b70fda8b90.json b/data/benchmark/097d93fbb2f9998ef1390ad10f92d1b70fda8b90.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:097d93fbb2f9998ef1390ad10f92d1b70fda8b90-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:097d93fbb2f9998ef1390ad10f92d1b70fda8b90-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/0a62cebef8b3dd4c544e8b70ce1b96b33390f4c0.json b/data/benchmark/0a62cebef8b3dd4c544e8b70ce1b96b33390f4c0.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0a62cebef8b3dd4c544e8b70ce1b96b33390f4c0-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0a62cebef8b3dd4c544e8b70ce1b96b33390f4c0-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/12684ee6cfd293b27c08495f97900bcd849b452c.json b/data/benchmark/12684ee6cfd293b27c08495f97900bcd849b452c.json
@@ -11,7 +11,7 @@
     "previousVersion" : "2.7",
     "newVersion" : "2.11.0",
     "dependencyScope" : "compile",
-    "versionUpdateType" : "other",
+    "versionUpdateType" : "minor",
     "githubCompareLink" : "A GitHub repository could not be found for the updated dependency.",
     "mavenSourceLinkPre" : "https://repo1.maven.org/maven2/commons-io/commons-io/2.7/commons-io-2.7-sources.jar",
     "mavenSourceLinkBreaking" : "https://repo1.maven.org/maven2/commons-io/commons-io/2.11.0/commons-io-2.11.0-sources.jar",

diff --git a/data/benchmark/30caf72b737bd40b6b4a16c6b25ba574e3048810.json b/data/benchmark/30caf72b737bd40b6b4a16c6b25ba574e3048810.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:30caf72b737bd40b6b4a16c6b25ba574e3048810-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:30caf72b737bd40b6b4a16c6b25ba574e3048810-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/44a15fa526c10b0c39d8d7337b833d42448fcca4.json b/data/benchmark/44a15fa526c10b0c39d8d7337b833d42448fcca4.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:44a15fa526c10b0c39d8d7337b833d42448fcca4-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:44a15fa526c10b0c39d8d7337b833d42448fcca4-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/4507371bb061f3add726d3ab7527bebb8c41960e.json b/data/benchmark/4507371bb061f3add726d3ab7527bebb8c41960e.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:4507371bb061f3add726d3ab7527bebb8c41960e-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:4507371bb061f3add726d3ab7527bebb8c41960e-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/4c7b193ce573e31766da82268ffc9ba51412faa6.json b/data/benchmark/4c7b193ce573e31766da82268ffc9ba51412faa6.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:4c7b193ce573e31766da82268ffc9ba51412faa6-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:4c7b193ce573e31766da82268ffc9ba51412faa6-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/51e103fc80aec2526ab5dd3bcc2d4d5ce4177842.json b/data/benchmark/51e103fc80aec2526ab5dd3bcc2d4d5ce4177842.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:51e103fc80aec2526ab5dd3bcc2d4d5ce4177842-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:51e103fc80aec2526ab5dd3bcc2d4d5ce4177842-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/5287fc631fa78e7f11d39983824cdd4215b9a03b.json b/data/benchmark/5287fc631fa78e7f11d39983824cdd4215b9a03b.json
diff --git a/data/benchmark/5743793a5d54e43d9acf27e46fdb3a257ee0196f.json b/data/benchmark/5743793a5d54e43d9acf27e46fdb3a257ee0196f.json
@@ -11,7 +11,7 @@
     "previousVersion" : "0.6",
     "newVersion" : "0.7.1",
     "dependencyScope" : "compile",
-    "versionUpdateType" : "other",
+    "versionUpdateType" : "minor",
     "githubCompareLink" : "A GitHub repository could not be found for the updated dependency.",
     "mavenSourceLinkPre" : "https://repo1.maven.org/maven2/com/google/flogger/google-extensions/0.6/google-extensions-0.6-sources.jar",
     "mavenSourceLinkBreaking" : "https://repo1.maven.org/maven2/com/google/flogger/google-extensions/0.7.1/google-extensions-0.7.1-sources.jar",

diff --git a/data/benchmark/5769bdad76925da568294cb8a40e7d4469699ac3.json b/data/benchmark/5769bdad76925da568294cb8a40e7d4469699ac3.json
@@ -11,7 +11,7 @@
     "previousVersion" : "1.93",
     "newVersion" : "1.313",
     "dependencyScope" : "compile",
-    "versionUpdateType" : "other",
+    "versionUpdateType" : "minor",
     "githubCompareLink" : "https://github.com/kohsuke/github-api/compare/github-api-1.93...github-api-1.313",
     "mavenSourceLinkPre" : "https://repo1.maven.org/maven2/org/kohsuke/github-api/1.93/github-api-1.93-sources.jar",
     "mavenSourceLinkBreaking" : "https://repo1.maven.org/maven2/org/kohsuke/github-api/1.313/github-api-1.313-sources.jar",

diff --git a/data/benchmark/65b5a75419981118d5a65e295abfd2069dfce0b0.json b/data/benchmark/65b5a75419981118d5a65e295abfd2069dfce0b0.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:65b5a75419981118d5a65e295abfd2069dfce0b0-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:65b5a75419981118d5a65e295abfd2069dfce0b0-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/8f16da085185a09deb5f3d810cc66f6e73348a22.json b/data/benchmark/8f16da085185a09deb5f3d810cc66f6e73348a22.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:8f16da085185a09deb5f3d810cc66f6e73348a22-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:8f16da085185a09deb5f3d810cc66f6e73348a22-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/90df9018630b8f8248f332e747d88254d99da1c9.json b/data/benchmark/90df9018630b8f8248f332e747d88254d99da1c9.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:90df9018630b8f8248f332e747d88254d99da1c9-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:90df9018630b8f8248f332e747d88254d99da1c9-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/99d4dfc8ca5b2db281f154ca9db3ad953b792b4d.json b/data/benchmark/99d4dfc8ca5b2db281f154ca9db3ad953b792b4d.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:99d4dfc8ca5b2db281f154ca9db3ad953b792b4d-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:99d4dfc8ca5b2db281f154ca9db3ad953b792b4d-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/9d450e88e8744638ba37037d6f41e65d9067aa3b.json b/data/benchmark/9d450e88e8744638ba37037d6f41e65d9067aa3b.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:9d450e88e8744638ba37037d6f41e65d9067aa3b-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:9d450e88e8744638ba37037d6f41e65d9067aa3b-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/ae16b526695fe275ab5e6a1992916875d26da860.json b/data/benchmark/ae16b526695fe275ab5e6a1992916875d26da860.json
diff --git a/data/benchmark/b209fe48162aad0f97eec1ddbb10fae165af4791.json b/data/benchmark/b209fe48162aad0f97eec1ddbb10fae165af4791.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:b209fe48162aad0f97eec1ddbb10fae165af4791-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:b209fe48162aad0f97eec1ddbb10fae165af4791-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/c09896887acf0fe59320e01145a7034cd8d4e326.json b/data/benchmark/c09896887acf0fe59320e01145a7034cd8d4e326.json
@@ -11,7 +11,7 @@
     "previousVersion" : "1.93",
     "newVersion" : "1.314",
     "dependencyScope" : "compile",
-    "versionUpdateType" : "other",
+    "versionUpdateType" : "minor",
     "githubCompareLink" : "https://github.com/kohsuke/github-api/compare/github-api-1.93...github-api-1.314",
     "mavenSourceLinkPre" : "https://repo1.maven.org/maven2/org/kohsuke/github-api/1.93/github-api-1.93-sources.jar",
     "mavenSourceLinkBreaking" : "https://repo1.maven.org/maven2/org/kohsuke/github-api/1.314/github-api-1.314-sources.jar",

diff --git a/data/benchmark/cb541fd65c7b9bbc3424ea927f1dab223261d156.json b/data/benchmark/cb541fd65c7b9bbc3424ea927f1dab223261d156.json
@@ -11,7 +11,7 @@
     "previousVersion" : "1.3",
     "newVersion" : "2.2",
     "dependencyScope" : "provided",
-    "versionUpdateType" : "other",
+    "versionUpdateType" : "major",
     "githubCompareLink" : "A GitHub repository could not be found for the updated dependency.",
     "mavenSourceLinkPre" : "https://repo1.maven.org/maven2/org/hamcrest/hamcrest-library/1.3/hamcrest-library-1.3-sources.jar",
     "mavenSourceLinkBreaking" : "https://repo1.maven.org/maven2/org/hamcrest/hamcrest-library/2.2/hamcrest-library-2.2-sources.jar",

diff --git a/data/benchmark/d3af06df4613be146bb9f8034e1a8a3098050c82.json b/data/benchmark/d3af06df4613be146bb9f8034e1a8a3098050c82.json
diff --git a/data/benchmark/dde0743630a540bff1ba3033b7016fcfc345122f.json b/data/benchmark/dde0743630a540bff1ba3033b7016fcfc345122f.json
@@ -20,5 +20,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:dde0743630a540bff1ba3033b7016fcfc345122f-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:dde0743630a540bff1ba3033b7016fcfc345122f-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }
diff --git a/data/benchmark/f26cd85b97b24c07a2e446f43ac8793619fa0724.json b/data/benchmark/f26cd85b97b24c07a2e446f43ac8793619fa0724.json
diff --git a/data/not-reproduced-data/5287fc631fa78e7f11d39983824cdd4215b9a03b.json b/data/not-reproduced-data/5287fc631fa78e7f11d39983824cdd4215b9a03b.json
@@ -0,0 +1,16 @@
+{
+  "url" : "https://github.com/jadler-mocking/jadler/pull/323",
+  "project" : "jadler",
+  "breakingCommit" : "5287fc631fa78e7f11d39983824cdd4215b9a03b",
+  "prAuthor" : "bot",
+  "preCommitAuthor" : "human",
+  "breakingCommitAuthor" : "bot",
+  "updatedDependency" : {
+    "dependencyGroupID" : "org.eclipse.jetty",
+    "dependencyArtifactID" : "jetty-server",
+    "previousVersion" : "8.1.11.v20130520",
+    "newVersion" : "11.0.14",
+    "dependencyScope" : "compile",
+    "versionUpdateType" : "other"
+  }
+}
diff --git a/data/not-reproduced-data/ae16b526695fe275ab5e6a1992916875d26da860.json b/data/not-reproduced-data/ae16b526695fe275ab5e6a1992916875d26da860.json
@@ -0,0 +1,16 @@
+{
+  "url" : "https://github.com/jadler-mocking/jadler/pull/160",
+  "project" : "jadler",
+  "breakingCommit" : "ae16b526695fe275ab5e6a1992916875d26da860",
+  "prAuthor" : "bot",
+  "preCommitAuthor" : "human",
+  "breakingCommitAuthor" : "bot",
+  "updatedDependency" : {
+    "dependencyGroupID" : "org.eclipse.jetty",
+    "dependencyArtifactID" : "jetty-server",
+    "previousVersion" : "8.1.11.v20130520",
+    "newVersion" : "9.4.41.v20210516",
+    "dependencyScope" : "compile",
+    "versionUpdateType" : "other"
+  }
+}
diff --git a/data/not-reproduced-data/d3af06df4613be146bb9f8034e1a8a3098050c82.json b/data/not-reproduced-data/d3af06df4613be146bb9f8034e1a8a3098050c82.json
@@ -0,0 +1,16 @@
+{
+  "url" : "https://github.com/jadler-mocking/jadler/pull/207",
+  "project" : "jadler",
+  "breakingCommit" : "d3af06df4613be146bb9f8034e1a8a3098050c82",
+  "prAuthor" : "bot",
+  "preCommitAuthor" : "human",
+  "breakingCommitAuthor" : "bot",
+  "updatedDependency" : {
+    "dependencyGroupID" : "org.eclipse.jetty",
+    "dependencyArtifactID" : "jetty-server",
+    "previousVersion" : "8.1.11.v20130520",
+    "newVersion" : "11.0.8",
+    "dependencyScope" : "compile",
+    "versionUpdateType" : "other"
+  }
+}
diff --git a/data/not-reproduced-data/f26cd85b97b24c07a2e446f43ac8793619fa0724.json b/data/not-reproduced-data/f26cd85b97b24c07a2e446f43ac8793619fa0724.json
@@ -0,0 +1,16 @@
+{
+  "url" : "https://github.com/jadler-mocking/jadler/pull/230",
+  "project" : "jadler",
+  "breakingCommit" : "f26cd85b97b24c07a2e446f43ac8793619fa0724",
+  "prAuthor" : "bot",
+  "preCommitAuthor" : "human",
+  "breakingCommitAuthor" : "bot",
+  "updatedDependency" : {
+    "dependencyGroupID" : "org.eclipse.jetty",
+    "dependencyArtifactID" : "jetty-server",
+    "previousVersion" : "8.1.11.v20130520",
+    "newVersion" : "11.0.9",
+    "dependencyScope" : "compile",
+    "versionUpdateType" : "other"
+  }
+}