diff --git a/readme.md b/readme.md index b1554e6..acc2f26 100644 --- a/readme.md +++ b/readme.md @@ -41,10 +41,10 @@ The program will first download the osv.dev dataset and create a folder called " If you already have downloaded this dataset and you don't want to update it, you can add the "noUpdate" argument on the java -jar command. Example: -> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar .\target\goblinWeaver-2.0.0.jar +> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar goblinWeaver-2.0.0.jar -> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar .\target\goblinWeaver-2.0.0.jar noUpdate +> java -Dneo4jUri="bolt://localhost:7687/" -Dneo4jUser="neo4j" -Dneo4jPassword="Password1" -jar goblinWeaver-2.0.0.jar noUpdate ## Use the API Pre-designed requests are available, but you can also send your own Cypher requests directly to the API. diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java index e5203e6..85fb1da 100644 --- a/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/controllers/GraphController.java @@ -1,11 +1,13 @@ package com.cifre.sap.su.goblinWeaver.api.controllers; +import com.cifre.sap.su.goblinWeaver.api.entities.GraphTraversingQuery; import com.cifre.sap.su.goblinWeaver.api.entities.ReleaseQueryList; +import com.cifre.sap.su.goblinWeaver.api.entities.enums.FilterEnum; import com.cifre.sap.su.goblinWeaver.graphDatabase.GraphDatabaseSingleton; import com.cifre.sap.su.goblinWeaver.graphEntities.InternGraph; import com.cifre.sap.su.goblinWeaver.graphEntities.edges.DependencyEdge; +import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.ArtifactNode; import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeObject; -import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.NodeType; import com.cifre.sap.su.goblinWeaver.graphEntities.nodes.ReleaseNode; import com.cifre.sap.su.goblinWeaver.weaver.Weaver; import io.swagger.v3.oas.annotations.Operation; @@ -15,9 +17,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -26,33 +26,85 @@ public class GraphController { @Operation( - description = "Get the project rooted graph", - summary = "Get the project rooted all graph from releases dependencies list" + description = "Creates a graph by traversing from a root. Start from a set of releases and add their dependencies", + summary = "Creates a graph by traversing from a root" ) - @PostMapping("/graph/rootedGraph") - public JSONObject getRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + @PostMapping("/graph/traversing") + public JSONObject traversingGraph(@RequestBody GraphTraversingQuery graphTraversingQuery) { + // Create root node and its dependencies InternGraph resultGraph = new InternGraph(); resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); - for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { - resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); + for (String releaseGav : graphTraversingQuery.getStartReleasesGav()) { + String[] splitedGav = releaseGav.split(":"); + resultGraph.addEdge(new DependencyEdge("ROOT", splitedGav[0]+":"+splitedGav[1], splitedGav[2], "compile")); } - resultGraph.mergeGraph( - GraphDatabaseSingleton.getInstance() - .getRootedGraph( - releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGav).collect(Collectors.toSet() - ) - ) - ); - Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); + Set releasesToTreat = new HashSet<>(graphTraversingQuery.getStartReleasesGav()); + Set librariesToExpends = new HashSet<>(graphTraversingQuery.getLibToExpendsGa()); + Set visitedReleases = new HashSet<>(); + Set visitedLibrary = new HashSet<>(); + boolean expendsNewLibs = searchAndRemoveAllKeyWord(librariesToExpends); + while(!releasesToTreat.isEmpty()) { + // Step 1: for each release, get parent lib, release, lib dependencies, lib target release: + // (lib:a)-[versions]->(release:a1)-[:dependency]->(lib:b)-[versions]->(release:b1) + for(String releaseGav : new HashSet<>(releasesToTreat)) { + InternGraph releaseGraph = GraphDatabaseSingleton.getInstance().getReleaseWithLibAndDependencies(releaseGav); + releaseGraph.clearValueNodes(); + resultGraph.mergeGraph(releaseGraph); + visitedReleases.add(releaseGav); + releasesToTreat.addAll(releaseGraph.getGraphNodes().stream().filter(ReleaseNode.class::isInstance).map(NodeObject::getId).collect(Collectors.toSet())); + if(expendsNewLibs){ + librariesToExpends.addAll(releaseGraph.getGraphNodes().stream().filter(ArtifactNode.class::isInstance).map(NodeObject::getId).collect(Collectors.toSet())); + } + } + librariesToExpends.removeAll(visitedLibrary); + // Step 2: for each libraryToExpends, get all releases + // (lib:a)-[versions]->(release:a1) + for(String libraryGa : librariesToExpends){ + InternGraph artifactGraph; + if(graphTraversingQuery.getFilters().contains(FilterEnum.MORE_RECENT)) { + Long timestamp = resultGraph.getGraphNodes().stream() + .filter(ReleaseNode.class::isInstance) + .map(ReleaseNode.class::cast) + .filter(release -> release.getGa().equals(libraryGa)) + .map(ReleaseNode::getTimestamp) + .min(Long::compare) + .orElse(null); + artifactGraph = (timestamp != null) + ? GraphDatabaseSingleton.getInstance().getArtifactNewReleasesGraph(libraryGa, timestamp) + : GraphDatabaseSingleton.getInstance().getArtifactReleasesGraph(libraryGa); + } else { + artifactGraph = GraphDatabaseSingleton.getInstance().getArtifactReleasesGraph(libraryGa); + } + resultGraph.mergeGraph(artifactGraph); + visitedLibrary.add(libraryGa); + releasesToTreat.addAll(artifactGraph.getGraphNodes().stream().filter(ReleaseNode.class::isInstance).map(NodeObject::getId).collect(Collectors.toSet())); + } + releasesToTreat.removeAll(visitedReleases); + } + Weaver.weaveGraph(resultGraph, graphTraversingQuery.getAddedValues()); return resultGraph.getJsonGraph(); } + private boolean searchAndRemoveAllKeyWord(Set setString){ + boolean found = false; + Set toRemove = new HashSet<>(); + for (String str : setString) { + if (str.equalsIgnoreCase("all")) { + toRemove.add(str); + found = true; + } + } + setString.removeAll(toRemove); + return found; + } + + @Operation( - description = "Get the project rooted all possibilities graph", - summary = "Get the project rooted all possibilities graph from releases dependencies list" + description = "Get the project rooted graph", + summary = "Get the project rooted all graph from releases dependencies list" ) - @PostMapping("/graph/allPossibilitiesRooted") - public JSONObject getAllPossibilitiesRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { + @PostMapping("/graph/rootedGraph") + public JSONObject getRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { InternGraph resultGraph = new InternGraph(); resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { @@ -60,8 +112,8 @@ public JSONObject getAllPossibilitiesRootedGraph(@RequestBody ReleaseQueryList r } resultGraph.mergeGraph( GraphDatabaseSingleton.getInstance() - .getAllPossibilitiesGraph( - releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGa).collect(Collectors.toSet() + .getRootedGraph( + releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGav).collect(Collectors.toSet() ) ) ); @@ -91,71 +143,4 @@ public JSONObject getDirectPossibilitiesRootedGraph(@RequestBody ReleaseQueryLis return resultGraph.getJsonGraph(); } - @Operation( - description = "Get the project rooted direct dependencies possibilities graph with transitive", - summary = "Get the project rooted direct dependencies possibilities graph with transitive version from releases dependencies list" - ) - @PostMapping("/graph/directPossibilitiesWithTransitiveRooted") - public JSONObject getDirectPossibilitiesWithTransitiveRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { - InternGraph resultGraph = new InternGraph(); - resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); - for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { - resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); - } - // Get direct all possibilities - InternGraph directAllPossibilities = GraphDatabaseSingleton.getInstance() - .getDirectPossibilitiesGraph(releaseQueryList.getReleases().stream().map(ReleaseQueryList.Release::getGa).collect(Collectors.toSet())); - resultGraph.mergeGraph(directAllPossibilities); - // Get Releases dependencies - Map parameters = new HashMap<>(); - Set visitedReleases = new HashSet<>(); - Set releasesToTreat = directAllPossibilities.getGraphNodes().stream().filter(n -> n.getType().equals(NodeType.RELEASE)).map(NodeObject::getId).collect(Collectors.toSet()); - while (!releasesToTreat.isEmpty()){ - parameters.put("releaseIdList",releasesToTreat); - InternGraph queryResult = GraphDatabaseSingleton.getInstance().executeQueryWithParameters(GraphDatabaseSingleton.getInstance().getQueryDictionary().getDependencyGraphFromReleaseIdListParameter(), parameters); - resultGraph.mergeGraph(queryResult); - visitedReleases.addAll(releasesToTreat); - Set newReleaseToTreat = resultGraph.getGraphNodes().stream().filter(node -> node instanceof ReleaseNode).map(NodeObject::getId).collect(Collectors.toSet()); - newReleaseToTreat.removeAll(visitedReleases); - releasesToTreat.clear(); - releasesToTreat.addAll(newReleaseToTreat); - } - - Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); - return resultGraph.getJsonGraph(); - } - - @Operation( - description = "Get the project rooted direct dependencies possibilities graph with transitive", - summary = "Get the project rooted direct dependencies possibilities graph with transitive version from releases dependencies list" - ) - @PostMapping("/graph/directNewPossibilitiesWithTransitiveRooted") - public JSONObject getDirectNewPossibilitiesWithTransitiveRootedGraph(@RequestBody ReleaseQueryList releaseQueryList) { - InternGraph resultGraph = new InternGraph(); - resultGraph.addNode(new ReleaseNode("ROOT", "ROOT", 0, "")); - for (ReleaseQueryList.Release release : releaseQueryList.getReleases()) { - resultGraph.addEdge(new DependencyEdge("ROOT", release.getGa(), release.getVersion(), "compile")); - } - // Get direct all possibilities - InternGraph directAllPossibilities = GraphDatabaseSingleton.getInstance() - .getDirectNewPossibilitiesGraph(releaseQueryList.getReleases()); - resultGraph.mergeGraph(directAllPossibilities); - // Get Releases dependencies - Map parameters = new HashMap<>(); - Set visitedReleases = new HashSet<>(); - Set releasesToTreat = directAllPossibilities.getGraphNodes().stream().filter(n -> n.getType().equals(NodeType.RELEASE)).map(NodeObject::getId).collect(Collectors.toSet()); - while (!releasesToTreat.isEmpty()){ - parameters.put("releaseIdList",releasesToTreat); - InternGraph queryResult = GraphDatabaseSingleton.getInstance().executeQueryWithParameters(GraphDatabaseSingleton.getInstance().getQueryDictionary().getDependencyGraphFromReleaseIdListParameter(), parameters); - resultGraph.mergeGraph(queryResult); - visitedReleases.addAll(releasesToTreat); - Set newReleaseToTreat = resultGraph.getGraphNodes().stream().filter(node -> node instanceof ReleaseNode).map(NodeObject::getId).collect(Collectors.toSet()); - newReleaseToTreat.removeAll(visitedReleases); - releasesToTreat.clear(); - releasesToTreat.addAll(newReleaseToTreat); - } - - Weaver.weaveGraph(resultGraph, releaseQueryList.getAddedValues()); - return resultGraph.getJsonGraph(); - } } diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/GraphTraversingQuery.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/GraphTraversingQuery.java new file mode 100644 index 0000000..4f2377b --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/GraphTraversingQuery.java @@ -0,0 +1,46 @@ +package com.cifre.sap.su.goblinWeaver.api.entities; + +import com.cifre.sap.su.goblinWeaver.api.entities.enums.FilterEnum; +import com.cifre.sap.su.goblinWeaver.weaver.addedValue.AddedValueEnum; + +import java.util.Set; + +public class GraphTraversingQuery { + private Set startReleasesGav; + private Set libToExpendsGa; + private Set filters; + private Set addedValues; + + public Set getStartReleasesGav() { + return startReleasesGav; + } + + public void setStartReleasesGav(Set startReleasesGav) { + this.startReleasesGav = startReleasesGav; + } + + public Set getLibToExpendsGa() { + return libToExpendsGa; + } + + public void setLibToExpendsGa(Set libToExpendsGa) { + this.libToExpendsGa = libToExpendsGa; + } + + public Set getFilters() { + return filters; + } + + public void setFilters(Set filters) { + this.filters = filters; + } + + public Set getAddedValues() { + return addedValues; + } + + public void setAddedValues(Set addedValues) { + this.addedValues = addedValues; + } +} + diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/enums/FilterEnum.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/enums/FilterEnum.java new file mode 100644 index 0000000..e781dba --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/enums/FilterEnum.java @@ -0,0 +1,8 @@ +package com.cifre.sap.su.goblinWeaver.api.entities.enums; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(using = FilterEnumDeserializer.class) +public enum FilterEnum { + MORE_RECENT +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/enums/FilterEnumDeserializer.java b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/enums/FilterEnumDeserializer.java new file mode 100644 index 0000000..c51660c --- /dev/null +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/api/entities/enums/FilterEnumDeserializer.java @@ -0,0 +1,18 @@ +package com.cifre.sap.su.goblinWeaver.api.entities.enums; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; + +public class FilterEnumDeserializer extends StdDeserializer { + protected FilterEnumDeserializer() { + super(FilterEnum.class); + } + + @Override + public FilterEnum deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return FilterEnum.valueOf(p.getText().toUpperCase()); + } +} diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java index ba57a9b..da7e4d2 100644 --- a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/GraphDatabaseInterface.java @@ -22,4 +22,8 @@ public interface GraphDatabaseInterface { InternGraph getAllPossibilitiesGraph(Set artifactIdList); InternGraph getDirectPossibilitiesGraph(Set artifactIdList); InternGraph getDirectNewPossibilitiesGraph(Set artifactIdList); + InternGraph getReleaseWithLibAndDependencies(String artifactId); + InternGraph getArtifactReleasesGraph(String artifactId); + InternGraph getArtifactSpecificReleasesGraph(String releaseId); + InternGraph getArtifactNewReleasesGraph(String artifactId, long timestamp); } diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java index 61f17fc..76bff5e 100644 --- a/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphDatabase/neo4j/Neo4jGraphDatabase.java @@ -223,6 +223,51 @@ public InternGraph getRootedGraph(Set releaseIdList){ return rootedGraph; } + @Override + public InternGraph getReleaseWithLibAndDependencies(String releaseId){ + Map parameters = new HashMap<>(); + String[] splitedGav = releaseId.split(":"); + parameters.put("releaseId",releaseId); + parameters.put("artifactId",splitedGav[0]+":"+splitedGav[1]); + String query = "MATCH (a:Artifact)-[re:relationship_AR]->(r:Release) " + + "WHERE a.id = $artifactId AND r.id = $releaseId " + + "OPTIONAL MATCH (r)-[d:dependency]->(a2:Artifact)-[re2:relationship_AR]->(target:Release) " + + "WHERE d.scope = 'compile' AND target.version = d.targetVersion " + + "RETURN a, re, r, d, a2, re2, target"; + return executeQueryWithParameters(query, parameters); + } + + @Override + public InternGraph getArtifactReleasesGraph(String artifactId){ + Map parameters = new HashMap<>(); + parameters.put("artifactId",artifactId); + String query = "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id = $artifactId " + + "RETURN a,e,r"; + return executeQueryWithParameters(query, parameters); + } + + @Override + public InternGraph getArtifactSpecificReleasesGraph(String releaseId){ + Map parameters = new HashMap<>(); + parameters.put("releaseId",releaseId); + String query = "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE r.id = $releaseId " + + "RETURN a,e,r"; + return executeQueryWithParameters(query, parameters); + } + + @Override + public InternGraph getArtifactNewReleasesGraph(String artifactId, long timestamp){ + Map parameters = new HashMap<>(); + parameters.put("artifactId",artifactId); + parameters.put("timestamp",timestamp); + String query = "MATCH (a:Artifact)-[e:relationship_AR]->(r:Release) " + + "WHERE a.id = $artifactId AND r.timestamp >= $timestamp " + + "RETURN a,e,r"; + return executeQueryWithParameters(query, parameters); + } + @Override public InternGraph getAllPossibilitiesGraph(Set artifactIdList){ InternGraph graphAllPossibilities = new InternGraph(); diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java index 92c08f1..9751625 100644 --- a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/InternGraph.java @@ -10,7 +10,7 @@ public class InternGraph { private final Set graphNodes = new HashSet<>(); private final Set graphEdges = new HashSet<>(); - private final Set graphValues = new HashSet<>(); + private Set graphValues = new HashSet<>(); public InternGraph() { } @@ -54,6 +54,10 @@ public void mergeGraph(InternGraph graph){ this.graphValues.addAll(graph.getGraphValues()); } + public void clearValueNodes(){ + this.graphValues = new HashSet<>(); + } + private void addObjectsToJSONArray(Set objects, String key, JSONObject graphJSON) { if (!objects.isEmpty()) { JSONArray array = new JSONArray(); diff --git a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java index 4f38d02..576701a 100644 --- a/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java +++ b/src/main/java/com/cifre/sap/su/goblinWeaver/graphEntities/nodes/ReleaseNode.java @@ -12,6 +12,14 @@ public ReleaseNode(String neo4jId, String id, long timestamp, String version) { this.version = version; } + public String getGa() { + return getId().equals("ROOT") ? "ROOT" : getId().substring(0, getId().lastIndexOf(':')); + } + + public long getTimestamp() { + return timestamp; + } + @Override public JSONObject getJsonObject() { JSONObject jsonObject = super.getJsonObject();