-
Notifications
You must be signed in to change notification settings - Fork 64
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Comprehensions of Lists, Sets and Maps and generator expressions (#…
…1786) * Add comprehension expression * Initial python translation for listcomp * SetComp and DictComp in python frontend * First simple tests * test in main * Try to add DFG edges * Fix not implemented error * Fix more bugs * Also handle GeneratorExp, add some documentation. * Extract nested class to own file * Fix bug, aggregate predicates * Remove unnecessary changes * Specify idea for EOG * Fake higher test coverage * More testing * More tests * Fix error from renaming * Handle the comprehension expression in the control flow sensitive DFG * Adding alternatives to EOG for collection comprehension and fixing syntax error in comprehension expression * Adding alternative that properly depicts generator behavior * Small fix * Alternative for ComprehensionExpression * Fix * Adding EOG handling for ComprehensionExpression and CollectionComprehension * Add test and fix EOG pass implementation * Allow th addition to something that holds arguments and something that holds statements * Remove useless stuff from ControlflowSensitiveDFGPass * Make non-optional things non-optional * Fix test * Remove condition to reduce code which needs coverage * More tests * Update stuff * review * review * generator type --------- Co-authored-by: Konrad Weiss <[email protected]>
- Loading branch information
Showing
19 changed files
with
1,241 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
118 changes: 118 additions & 0 deletions
118
...in/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| /* | ||
| * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| * $$$$$$\ $$$$$$$\ $$$$$$\ | ||
| * $$ __$$\ $$ __$$\ $$ __$$\ | ||
| * $$ / \__|$$ | $$ |$$ / \__| | ||
| * $$ | $$$$$$$ |$$ |$$$$\ | ||
| * $$ | $$ ____/ $$ |\_$$ | | ||
| * $$ | $$\ $$ | $$ | $$ | | ||
| * \$$$$$ |$$ | \$$$$$ | | ||
| * \______/ \__| \______/ | ||
| * | ||
| */ | ||
| package de.fraunhofer.aisec.cpg.graph.statements.expressions | ||
|
|
||
| import de.fraunhofer.aisec.cpg.graph.* | ||
| import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf | ||
| import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf | ||
| import de.fraunhofer.aisec.cpg.graph.edges.unwrapping | ||
| import de.fraunhofer.aisec.cpg.graph.statements.Statement | ||
| import java.util.Objects | ||
| import org.apache.commons.lang3.builder.ToStringBuilder | ||
| import org.neo4j.ogm.annotation.Relationship | ||
|
|
||
| /** | ||
| * Represent a list/set/map comprehension or similar expression. It contains four major components: | ||
| * The statement, the variable, the iterable and a predicate which are combined to something like | ||
| * `[statement(variable) : variable in iterable if predicate(variable)]`. | ||
| * | ||
| * Some languages provide a way to have multiple variables, iterables and predicates. For this | ||
| * reason, we represent the `variable, iterable and predicate in its own class | ||
| * [ComprehensionExpression]. | ||
| */ | ||
| class CollectionComprehension : Expression(), ArgumentHolder { | ||
|
|
||
| @Relationship("COMPREHENSION_EXPRESSIONS") | ||
| var comprehensionExpressionEdges = astEdgesOf<ComprehensionExpression>() | ||
| /** | ||
| * This field contains one or multiple [ComprehensionExpression]s. | ||
| * | ||
| * Note: Instead of having a list here, we could also enforce that the frontend nests the | ||
| * expressions in a meaningful way (in particular this would help us to satisfy dependencies | ||
| * between the comprehensions' variables). | ||
| */ | ||
| var comprehensionExpressions by | ||
| unwrapping(CollectionComprehension::comprehensionExpressionEdges) | ||
|
|
||
| @Relationship("STATEMENT") | ||
| var statementEdge = | ||
| astEdgeOf<Statement>( | ||
| ProblemExpression("No statement provided but is required in ${this::class}") | ||
| ) | ||
| /** | ||
| * This field contains the statement which is applied to each element of the input for which the | ||
| * predicate returned `true`. | ||
| */ | ||
| var statement by unwrapping(CollectionComprehension::statementEdge) | ||
|
|
||
| override fun toString() = | ||
| ToStringBuilder(this, TO_STRING_STYLE) | ||
| .appendSuper(super.toString()) | ||
| .append("statement", statement) | ||
| .append("comprehensions", comprehensionExpressions) | ||
| .toString() | ||
|
|
||
| override fun equals(other: Any?): Boolean { | ||
| if (this === other) return true | ||
| if (other !is CollectionComprehension) return false | ||
| return super.equals(other) && | ||
| statement == other.statement && | ||
| comprehensionExpressions == other.comprehensionExpressions | ||
| } | ||
|
|
||
| override fun hashCode() = Objects.hash(super.hashCode(), statement, comprehensionExpressions) | ||
|
|
||
| override fun addArgument(expression: Expression) { | ||
| if (this.statement is ProblemExpression) { | ||
| this.statement = expression | ||
| } else if (expression is ComprehensionExpression) { | ||
| this.comprehensionExpressions += expression | ||
| } | ||
| } | ||
|
|
||
| override fun replaceArgument(old: Expression, new: Expression): Boolean { | ||
| if (this.statement == old) { | ||
| this.statement = new | ||
| return true | ||
| } | ||
| if (new !is ComprehensionExpression) return false | ||
| var changedSomething = false | ||
| val newCompExp = | ||
| this.comprehensionExpressions.map { | ||
| if (it == old) { | ||
| changedSomething = true | ||
| new | ||
| } else it | ||
| } | ||
| this.comprehensionExpressions.clear() | ||
| this.comprehensionExpressions.addAll(newCompExp) | ||
| return changedSomething | ||
| } | ||
|
|
||
| override fun hasArgument(expression: Expression): Boolean { | ||
| return this.statement == expression || expression in this.comprehensionExpressions | ||
| } | ||
| } |
125 changes: 125 additions & 0 deletions
125
...in/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| /* | ||
| * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| * $$$$$$\ $$$$$$$\ $$$$$$\ | ||
| * $$ __$$\ $$ __$$\ $$ __$$\ | ||
| * $$ / \__|$$ | $$ |$$ / \__| | ||
| * $$ | $$$$$$$ |$$ |$$$$\ | ||
| * $$ | $$ ____/ $$ |\_$$ | | ||
| * $$ | $$\ $$ | $$ | $$ | | ||
| * \$$$$$ |$$ | \$$$$$ | | ||
| * \______/ \__| \______/ | ||
| * | ||
| */ | ||
| package de.fraunhofer.aisec.cpg.graph.statements.expressions | ||
|
|
||
| import de.fraunhofer.aisec.cpg.graph.AccessValues | ||
| import de.fraunhofer.aisec.cpg.graph.ArgumentHolder | ||
| import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf | ||
| import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf | ||
| import de.fraunhofer.aisec.cpg.graph.edges.unwrapping | ||
| import de.fraunhofer.aisec.cpg.graph.statements.Statement | ||
| import java.util.Objects | ||
| import org.apache.commons.lang3.builder.ToStringBuilder | ||
| import org.neo4j.ogm.annotation.Relationship | ||
|
|
||
| /** This class holds the variable, iterable and predicate of the [CollectionComprehension]. */ | ||
| class ComprehensionExpression : Expression(), ArgumentHolder { | ||
| @Relationship("VARIABLE") | ||
| var variableEdge = | ||
| astEdgeOf<Statement>( | ||
| of = ProblemExpression("Missing variableEdge in ${this::class}"), | ||
| onChanged = { _, new -> | ||
| val end = new?.end | ||
| if (end is Reference) { | ||
| end.access = AccessValues.WRITE | ||
| } | ||
| } | ||
| ) | ||
|
|
||
| /** | ||
| * This field contains the iteration variable of the comprehension. It can be either a new | ||
| * variable declaration or a reference (probably to a new variable). | ||
| */ | ||
| var variable by unwrapping(ComprehensionExpression::variableEdge) | ||
|
|
||
| @Relationship("ITERABLE") | ||
| var iterableEdge = | ||
| astEdgeOf<Expression>(ProblemExpression("Missing iterable in ${this::class}")) | ||
|
|
||
| /** This field contains the iteration subject of the loop. */ | ||
| var iterable by unwrapping(ComprehensionExpression::iterableEdge) | ||
|
|
||
| @Relationship("PREDICATE") var predicateEdge = astOptionalEdgeOf<Statement>() | ||
|
|
||
| /** | ||
| * This field contains the predicate which has to hold to evaluate `statement(variable)` and | ||
| * include it in the result. | ||
| */ | ||
| var predicate by unwrapping(ComprehensionExpression::predicateEdge) | ||
|
|
||
| override fun toString() = | ||
| ToStringBuilder(this, TO_STRING_STYLE) | ||
| .appendSuper(super.toString()) | ||
| .append("variable", variable) | ||
| .append("iterable", iterable) | ||
| .append("predicate", predicate) | ||
| .toString() | ||
|
|
||
| override fun equals(other: Any?): Boolean { | ||
| if (this === other) return true | ||
| if (other !is ComprehensionExpression) return false | ||
| return super.equals(other) && | ||
| variable == other.variable && | ||
| iterable == other.iterable && | ||
| predicate == other.predicate | ||
| } | ||
|
|
||
| override fun hashCode() = Objects.hash(super.hashCode(), variable, iterable, predicate) | ||
|
|
||
| override fun addArgument(expression: Expression) { | ||
| if (this.variable is ProblemExpression) { | ||
| this.variable = expression | ||
| } else if (this.iterable is ProblemExpression) { | ||
| this.iterable = expression | ||
| } else { | ||
| this.predicate = expression | ||
| } | ||
| } | ||
|
|
||
| override fun replaceArgument(old: Expression, new: Expression): Boolean { | ||
| if (this.variable == old) { | ||
| this.variable = new | ||
| return true | ||
| } | ||
|
|
||
| if (this.iterable == old) { | ||
| this.iterable = new | ||
| return true | ||
| } | ||
|
|
||
| if (this.predicate == old) { | ||
| this.predicate = new | ||
| return true | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| override fun hasArgument(expression: Expression): Boolean { | ||
| return this.variable == expression || | ||
| this.iterable == expression || | ||
| expression == this.predicate | ||
| } | ||
| } |
Oops, something went wrong.