Update benchmark by adding GitHub org and new failure categories (#126)

README.md

@@ -100,12 +100,12 @@ java -jar target/BreakingUpdateReproducer.jar --help
 ```
 
 ## Stats
-As of Aug 24 2023:
+As of Aug 30 2023:
   * The benchmark consists of 625 reproducible breaking updates.
-    - Of these breaking updates, 257 (41.12%) fail compilation with the updated dependency.
-    - 203 (32.48%) fail tests with the updated dependency.
-    - 2 (0.32%) have dependency resolution failures with the updated dependency.
+    - Of these breaking updates, 261 (41.76%) fail compilation with the updated dependency.
+    - 211 (33.76%) fail tests with the updated dependency.
+    - 5 (0.80%) have dependency resolution failures with the updated dependency.
     - 69 (11.04%) fail after updating the dependency due to maven enforcer failures.
-    - 94 (15.04%) fail due to unknown failures after updating the dependency.
+    - 3 (0.48%) fail due to unknown failures after updating the dependency.
   * Overall, reproduction has been attempted for 5364 breaking updates, and 4739 (88.35%) could not be locally reproduced.
   * For 249 potential breaking updates, reproduction has not been attempted yet.

data/benchmark/00a7cc31784ac4a9cc27d506a73ae589d6df36d6.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/quickfix-j/quickfixj/pull/502",
   "project" : "quickfixj",
+  "projectOrganisation" : "quickfix-j",
   "breakingCommit" : "00a7cc31784ac4a9cc27d506a73ae589d6df36d6",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/00c78c1273efbc6aeae852963ed328a5cb57092e.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jenkinsci/publish-over-ssh-plugin/pull/282",
   "project" : "publish-over-ssh-plugin",
+  "projectOrganisation" : "jenkinsci",
   "breakingCommit" : "00c78c1273efbc6aeae852963ed328a5cb57092e",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",
@@ -20,5 +21,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:00c78c1273efbc6aeae852963ed328a5cb57092e-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:00c78c1273efbc6aeae852963ed328a5cb57092e-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "JENKINS_PLUGIN_FAILURE"
 }

data/benchmark/01075f5aedd9a75448599e3724483afaff95cb93.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/lukas-krecan/future-converter/pull/93",
   "project" : "future-converter",
+  "projectOrganisation" : "lukas-krecan",
   "breakingCommit" : "01075f5aedd9a75448599e3724483afaff95cb93",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/01609f962f32cc2ba04fa574c79429e86b372c6b.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/International-Data-Spaces-Association/IDS-Messaging-Services/pull/729",
   "project" : "IDS-Messaging-Services",
+  "projectOrganisation" : "International-Data-Spaces-Association",
   "breakingCommit" : "01609f962f32cc2ba04fa574c79429e86b372c6b",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/01737a7815069eef6fd035f077f5cc6f0a79b635.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/googleapis/java-storage-nio/pull/1206",
   "project" : "java-storage-nio",
+  "projectOrganisation" : "googleapis",
   "breakingCommit" : "01737a7815069eef6fd035f077f5cc6f0a79b635",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/02363207c365b3befd6426f6dda5c8eac904d760.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/retest/recheck/pull/1041",
   "project" : "recheck",
+  "projectOrganisation" : "retest",
   "breakingCommit" : "02363207c365b3befd6426f6dda5c8eac904d760",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/02eedffd33cc1fe94b59372a1c1bfd91b2fc2ced.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/International-Data-Spaces-Association/IDS-Messaging-Services/pull/741",
   "project" : "IDS-Messaging-Services",
+  "projectOrganisation" : "International-Data-Spaces-Association",
   "breakingCommit" : "02eedffd33cc1fe94b59372a1c1bfd91b2fc2ced",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0305beafdecb0b28f7c94264ed20cdc4e41ff067.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/feedzai/pdb/pull/342",
   "project" : "pdb",
+  "projectOrganisation" : "feedzai",
   "breakingCommit" : "0305beafdecb0b28f7c94264ed20cdc4e41ff067",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/04c07b066b60a2e9d4f797b98dcd439a3a42f0b9.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/feedzai/pdb/pull/364",
   "project" : "pdb",
+  "projectOrganisation" : "feedzai",
   "breakingCommit" : "04c07b066b60a2e9d4f797b98dcd439a3a42f0b9",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/04f776fa9bed9155776378daee788c7c166d3d35.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/snowflakedb/snowflake-jdbc/pull/1441",
   "project" : "snowflake-jdbc",
+  "projectOrganisation" : "snowflakedb",
   "breakingCommit" : "04f776fa9bed9155776378daee788c7c166d3d35",
   "prAuthor" : "human",
   "preCommitAuthor" : "human",

data/benchmark/05d8c1f9e83555617c2e3f5a6dbe8273642eb4a0.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/premium-minds/billy/pull/295",
   "project" : "billy",
+  "projectOrganisation" : "premium-minds",
   "breakingCommit" : "05d8c1f9e83555617c2e3f5a6dbe8273642eb4a0",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",
@@ -20,5 +21,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:05d8c1f9e83555617c2e3f5a6dbe8273642eb4a0-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:05d8c1f9e83555617c2e3f5a6dbe8273642eb4a0-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "JAXB_FAILURE"
 }

data/benchmark/063cf862c045e0a9e401a539497e34ddaf8332b3.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/versly/wsdoc/pull/75",
   "project" : "wsdoc",
+  "projectOrganisation" : "versly",
   "breakingCommit" : "063cf862c045e0a9e401a539497e34ddaf8332b3",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0671a30e3673989830ed7f4aa345b867c29547f0.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/International-Data-Spaces-Association/IDS-Messaging-Services/pull/492",
   "project" : "IDS-Messaging-Services",
+  "projectOrganisation" : "International-Data-Spaces-Association",
   "breakingCommit" : "0671a30e3673989830ed7f4aa345b867c29547f0",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",
@@ -20,5 +21,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0671a30e3673989830ed7f4aa345b867c29547f0-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0671a30e3673989830ed7f4aa345b867c29547f0-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "TEST_FAILURE"
 }

data/benchmark/067f5d2c81ff87c90755f4ed48f62eb5faa8ecf9.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/pinterest/singer/pull/166",
   "project" : "singer",
+  "projectOrganisation" : "pinterest",
   "breakingCommit" : "067f5d2c81ff87c90755f4ed48f62eb5faa8ecf9",
   "prAuthor" : "human",
   "preCommitAuthor" : "human",

data/benchmark/06c5386831e97e94d9b9fd155d3ea4aa8711c4e7.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jenkinsci/code-coverage-api-plugin/pull/645",
   "project" : "code-coverage-api-plugin",
+  "projectOrganisation" : "jenkinsci",
   "breakingCommit" : "06c5386831e97e94d9b9fd155d3ea4aa8711c4e7",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/072528ee5e678feabeaa1e2962725134564bdd3c.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/s4u/sign-maven-plugin/pull/135",
   "project" : "sign-maven-plugin",
+  "projectOrganisation" : "s4u",
   "breakingCommit" : "072528ee5e678feabeaa1e2962725134564bdd3c",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0771fe8d060aec9dc22344b8946e25ba8500afb9.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/apache/opennlp/pull/413",
   "project" : "opennlp",
+  "projectOrganisation" : "apache",
   "breakingCommit" : "0771fe8d060aec9dc22344b8946e25ba8500afb9",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",
@@ -20,5 +21,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0771fe8d060aec9dc22344b8946e25ba8500afb9-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0771fe8d060aec9dc22344b8946e25ba8500afb9-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "CHECKSTYLE_FAILURE"
 }

data/benchmark/07e4b2894bc68cd3bb1892beaa13ec353564dcf1.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jenkinsci/code-coverage-api-plugin/pull/707",
   "project" : "code-coverage-api-plugin",
+  "projectOrganisation" : "jenkinsci",
   "breakingCommit" : "07e4b2894bc68cd3bb1892beaa13ec353564dcf1",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/07fad972bb884e9fa6143b4f870d08305811607d.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/alphagov/pay-adminusers/pull/1594",
   "project" : "pay-adminusers",
+  "projectOrganisation" : "alphagov",
   "breakingCommit" : "07fad972bb884e9fa6143b4f870d08305811607d",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/07ff1a34661db6c7f0ca03156ff5d8936b5123f5.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/camunda-community-hub/camunda-platform-7-mockito/pull/274",
   "project" : "camunda-platform-7-mockito",
+  "projectOrganisation" : "camunda-community-hub",
   "breakingCommit" : "07ff1a34661db6c7f0ca03156ff5d8936b5123f5",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/08e33c7bbc45ceac64564dadb9abbae0ed481ca5.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/International-Data-Spaces-Association/IDS-Messaging-Services/pull/705",
   "project" : "IDS-Messaging-Services",
+  "projectOrganisation" : "International-Data-Spaces-Association",
   "breakingCommit" : "08e33c7bbc45ceac64564dadb9abbae0ed481ca5",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0968864d08e0fce1e5e1caaf89afddd2cc1b2569.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/assertj/assertj-guava/pull/98",
   "project" : "assertj-guava",
+  "projectOrganisation" : "assertj",
   "breakingCommit" : "0968864d08e0fce1e5e1caaf89afddd2cc1b2569",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/097d93fbb2f9998ef1390ad10f92d1b70fda8b90.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/dadoonet/fscrawler/pull/1296",
   "project" : "fscrawler",
+  "projectOrganisation" : "dadoonet",
   "breakingCommit" : "097d93fbb2f9998ef1390ad10f92d1b70fda8b90",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/09a8774fc7d35a4bff24b55363fb6d2b94ff09e9.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jenkinsci/pull-request-monitoring-plugin/pull/178",
   "project" : "pull-request-monitoring-plugin",
+  "projectOrganisation" : "jenkinsci",
   "breakingCommit" : "09a8774fc7d35a4bff24b55363fb6d2b94ff09e9",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",
@@ -20,5 +21,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:09a8774fc7d35a4bff24b55363fb6d2b94ff09e9-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:09a8774fc7d35a4bff24b55363fb6d2b94ff09e9-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "JENKINS_PLUGIN_FAILURE"
 }

data/benchmark/0a11c04038eae517540051dbf51f7f26b7221f20.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/simplelocalize/simplelocalize-cli/pull/74",
   "project" : "simplelocalize-cli",
+  "projectOrganisation" : "simplelocalize",
   "breakingCommit" : "0a11c04038eae517540051dbf51f7f26b7221f20",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0a62cebef8b3dd4c544e8b70ce1b96b33390f4c0.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/dadoonet/fscrawler/pull/1295",
   "project" : "fscrawler",
+  "projectOrganisation" : "dadoonet",
   "breakingCommit" : "0a62cebef8b3dd4c544e8b70ce1b96b33390f4c0",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0abf7148300f40a1da0538ab060552bca4a2f1d8.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/xdev-software/biapi/pull/69",
   "project" : "biapi",
+  "projectOrganisation" : "xdev-software",
   "breakingCommit" : "0abf7148300f40a1da0538ab060552bca4a2f1d8",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0b776fa1446ee6c34e234e0a3c6103d84274e395.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jenkinsci/webhook-step-plugin/pull/121",
   "project" : "webhook-step-plugin",
+  "projectOrganisation" : "jenkinsci",
   "breakingCommit" : "0b776fa1446ee6c34e234e0a3c6103d84274e395",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",
@@ -20,5 +21,5 @@
   "preCommitReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0b776fa1446ee6c34e234e0a3c6103d84274e395-pre",
   "breakingUpdateReproductionCommand" : "docker run ghcr.io/chains-project/breaking-updates:0b776fa1446ee6c34e234e0a3c6103d84274e395-breaking",
   "javaVersionUsedForReproduction" : "11",
-  "failureCategory" : "UNKNOWN_FAILURE"
+  "failureCategory" : "JENKINS_PLUGIN_FAILURE"
 }

data/benchmark/0c088ff435224b168f93d2a20b829af940bb67dd.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/fmtn/a/pull/219",
   "project" : "a",
+  "projectOrganisation" : "fmtn",
   "breakingCommit" : "0c088ff435224b168f93d2a20b829af940bb67dd",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0c60d0b08c999769313bfe2335fa792efcfb0300.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/International-Data-Spaces-Association/IDS-Messaging-Services/pull/725",
   "project" : "IDS-Messaging-Services",
+  "projectOrganisation" : "International-Data-Spaces-Association",
   "breakingCommit" : "0c60d0b08c999769313bfe2335fa792efcfb0300",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0c748afce24c983e8f330bc2435837c70b0fcde6.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/pac4j/dropwizard-pac4j/pull/344",
   "project" : "dropwizard-pac4j",
+  "projectOrganisation" : "pac4j",
   "breakingCommit" : "0c748afce24c983e8f330bc2435837c70b0fcde6",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0c9a9c80287e739424508b4afd3e7b73697733ae.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/googleapis/java-pubsub-group-kafka-connector/pull/257",
   "project" : "java-pubsub-group-kafka-connector",
+  "projectOrganisation" : "googleapis",
   "breakingCommit" : "0c9a9c80287e739424508b4afd3e7b73697733ae",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0cdcc1f1319311f383676a89808c9b8eb190145c.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/Samsung/LPVS/pull/100",
   "project" : "LPVS",
+  "projectOrganisation" : "Samsung",
   "breakingCommit" : "0cdcc1f1319311f383676a89808c9b8eb190145c",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0ddd0efa29634a4783358cba727d0851236aa579.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/International-Data-Spaces-Association/IDS-Messaging-Services/pull/733",
   "project" : "IDS-Messaging-Services",
+  "projectOrganisation" : "International-Data-Spaces-Association",
   "breakingCommit" : "0ddd0efa29634a4783358cba727d0851236aa579",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0e8625f492854a78c0e1ceff67b2abd7e081d42b.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jadler-mocking/jadler/pull/253",
   "project" : "jadler",
+  "projectOrganisation" : "jadler-mocking",
   "breakingCommit" : "0e8625f492854a78c0e1ceff67b2abd7e081d42b",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0ead9a6c759b422a5b8db197bb52d1cc0cabce78.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jnidzwetzki/bitfinex-v2-wss-api-java/pull/323",
   "project" : "bitfinex-v2-wss-api-java",
+  "projectOrganisation" : "jnidzwetzki",
   "breakingCommit" : "0ead9a6c759b422a5b8db197bb52d1cc0cabce78",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0ec1ab7e205e0ea1bafc0e0ab7be056716dd560b.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/apache/causeway/pull/970",
   "project" : "causeway",
+  "projectOrganisation" : "apache",
   "breakingCommit" : "0ec1ab7e205e0ea1bafc0e0ab7be056716dd560b",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/0ed34fa61c9e31bd4ebb7e4bf12ce792aec96091.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/camunda-community-hub/camunda-platform-7-mockito/pull/319",
   "project" : "camunda-platform-7-mockito",
+  "projectOrganisation" : "camunda-community-hub",
   "breakingCommit" : "0ed34fa61c9e31bd4ebb7e4bf12ce792aec96091",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/0ee8b9376b967938e8efd89a0959214a30d1b3fb.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/pholser/junit-quickcheck/pull/483",
   "project" : "junit-quickcheck",
+  "projectOrganisation" : "pholser",
   "breakingCommit" : "0ee8b9376b967938e8efd89a0959214a30d1b3fb",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",

data/benchmark/1053033eef680f0199bf25ec6e3db52cc13ef3da.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/jcabi/jcabi-maven-plugin/pull/145",
   "project" : "jcabi-maven-plugin",
+  "projectOrganisation" : "jcabi",
   "breakingCommit" : "1053033eef680f0199bf25ec6e3db52cc13ef3da",
   "prAuthor" : "bot",
   "preCommitAuthor" : "bot",

data/benchmark/10d7545c5771b03dd9f6122bd5973a759eb2cd03.json

@@ -1,6 +1,7 @@
 {
   "url" : "https://github.com/wireapp/lithium/pull/98",
   "project" : "lithium",
+  "projectOrganisation" : "wireapp",
   "breakingCommit" : "10d7545c5771b03dd9f6122bd5973a759eb2cd03",
   "prAuthor" : "bot",
   "preCommitAuthor" : "human",