Merge branch 'main' into feature/automated-graph-doc

.github/workflows/build.yml

@@ -9,6 +9,7 @@ on:
       - v*.**
     paths-ignore:
       - "docs/**"
+  merge_group:
   pull_request:
     types: [opened, synchronize, reopened]
     paths-ignore:
@@ -18,24 +19,25 @@ jobs:
   build-cpgo-osx:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version-file: cpg-language-go/src/main/golang/go.mod
+          cache-dependency-path: cpg-language-go/src/main/golang/go.sum
       - name: Build
         run: |
           cd cpg-language-go/src/main/golang
           ./build.sh
       - name: Archive cpgo library (amd64)
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: libcpgo-amd64.dylib
           path: cpg-language-go/src/main/resources/libcpgo-amd64.dylib
       - name: Archive cpgo library (arm64)
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: libcpgo-arm64.dylib
           path: cpg-language-go/src/main/resources/libcpgo-arm64.dylib
@@ -44,30 +46,30 @@ jobs:
     runs-on: [self-hosted, linux, x64, faster]
     needs: build-cpgo-osx
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of SonarQube analysis
       - run: |
           cp gradle.properties.example gradle.properties
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: "17"
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: "18"
       - name: Setup neo4j
         run: |
           docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.18
+          go-version: 1.21
       - name: Cache SonarCloud packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
@@ -92,17 +94,11 @@ jobs:
           if [ -d "/opt/hostedtoolcache/Python" ]; then
             find /opt/hostedtoolcache/Python/ -name libjep.so -exec sudo cp '{}' /usr/lib/ \;
           fi
-      - name: Install pycodestyle
-        run: |
-          pip3 install pycodestyle
-      - name: Run pycodestyle
-        run: |
-          find cpg-language-python/src/main/python -iname "*.py" -print0 | xargs -n 1 -0 pycodestyle
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: libcpgo-arm64.dylib
           path: cpg-language-go/src/main/resources/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: libcpgo-amd64.dylib
           path: cpg-language-go/src/main/resources/
@@ -126,10 +122,10 @@ jobs:
       - name: Prepare test and coverage reports
         if: ${{ always() }}
         run: |
-          zip reports.zip **/build/reports/**/** || true
+          zip -r reports.zip **/build/reports/**/** || true
       - name: Archive test and coverage reports
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: reports
           path: reports.zip
@@ -148,7 +144,7 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
         run: |
           export ORG_GRADLE_PROJECT_signingKey=`echo ${{ secrets.GPG_PRIVATE_KEY }} | base64 -d`
-          ./gradlew --no-daemon -Dorg.gradle.internal.publish.checksums.insecure=true --parallel -Pversion=$VERSION -PenableJavaFrontend=true  -PenableCXXFrontend=true -PenableGoFrontend=true -PenablePythonFrontend=true -PenableLLVMFrontend=true -PenableTypeScriptFrontend=true publishToSonatype closeSonatypeStagingRepository
+          ./gradlew --no-daemon -Dorg.gradle.internal.publish.checksums.insecure=true --parallel -Pversion=$VERSION -PenableJavaFrontend=true  -PenableCXXFrontend=true -PenableGoFrontend=true -PenablePythonFrontend=true -PenableLLVMFrontend=true -PenableTypeScriptFrontend=true -PenableRubyFrontend=true publishToSonatype closeSonatypeStagingRepository
         env:
           VERSION: ${{ env.version }}
           ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.GPG_PASSWORD }}

.github/workflows/docs.yml

@@ -18,13 +18,13 @@ jobs:
     if: github.event.repository.fork == false
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install python3
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           key: ${{ github.ref }}
           path: .cache

README.md

@@ -34,10 +34,11 @@ In order to get familiar with the graph itself, you can use the subproject [cpg-
 ### As Library
 
 The most recent version is being published to Maven central and can be used as a simple dependency, either using Maven or Gradle. Since Eclipse CDT is not published on maven central, it is necessary to add a repository with a custom layout to find the released CDT files. For example, using Gradle's Kotlin syntax:
-```
+
+```kotlin
 repositories {
     ivy {
-        setUrl("https://download.eclipse.org/tools/cdt/releases/11.0/cdt-11.0.0/plugins")
+        setUrl("https://download.eclipse.org/tools/cdt/releases/11.3/cdt-11.3.1/plugins")
         metadataSources {
             artifact()
         }
@@ -48,11 +49,11 @@ repositories {
 }
 
 dependencies {
-    var cpgVersion = "5.1.0" 
-    
+    val cpgVersion = "8.0.0"
+
     // if you want to include all published cpg modules
     implementation("de.fraunhofer.aisec", "cpg", cpgVersion)
-    
+
     // if you only want to use some of the cpg modules
     // use the 'cpg-core' module
     // and then add the needed extra modules, such as Go and Python
@@ -115,20 +116,19 @@ val translationConfig = TranslationConfiguration
 ### Experimental Languages
 
 Some languages, such as Golang are experimental and depend on other native libraries. Therefore, they are not included in the `cpg-core` module but have separate gradle submodules.
-C/CPP and Java are currently required by some of the modules (e.g. `cpg-analysis`) and thus, disabling them can lead to compile errors!
 To include the desired submodules simply toggle them on in your local `gradle.properties` file by setting the value of the properties to `true` e.g., (`enableGoFrontend=true`).
 We provide a sample file with all languages switched on [here](./gradle.properties.example).
 Instead of manually editing the `gradle.properties` file, you can also use the `configure_frontends.sh` script, which edits the properties for you.
 
 #### Golang
 
-In the case of Golang, the necessary native code can be found in the `src/main/golang` folder of the `cpg-language-go` submodule. Gradle should automatically find JNI headers and stores the finished library in the `src/main/golang` folder. This currently only works for Linux and macOS. In order to use it in an external project, the resulting library needs to be placed somewhere in `java.library.path`.
+In the case of Golang, the necessary native code can be found in the `src/main/golang` folder of the `cpg-language-go` submodule. Gradle should automatically store the finished library in the `src/main/golang` folder. This currently only works for Linux and macOS.
 
 #### Python
 
 You need to install [jep](https://github.com/ninia/jep/). This can either be system-wide or in a virtual environment. Your jep version has to match the version used by the CPG (see [version catalog](./gradle/libs.versions.toml)).
 
-Currently, only Python 3.{9,10,11,12} is supported.
+Currently, only Python 3.{9,10,11,12,13} is supported.
 
 ##### System Wide
 
@@ -146,7 +146,6 @@ Through the `JepSingleton`, the CPG library will look for well known paths on Li
 
 For parsing TypeScript, the necessary NodeJS-based code can be found in the `src/main/nodejs` directory of the `cpg-language-typescript` submodule. Gradle should build the script automatically, provided NodeJS (>=16) is installed. The bundles script will be placed inside the jar's resources and should work out of the box.
 
-
 ### Code Style
 
 We use [Google Java Style](https://github.com/google/google-java-format) as a formatting. Please install the appropriate plugin for your IDE, such as the [google-java-format IntelliJ plugin](https://plugins.jetbrains.com/plugin/8527-google-java-format) or [google-java-format Eclipse plugin](https://github.com/google/google-java-format/releases/download/google-java-format-1.6/google-java-format-eclipse-plugin_1.6.0.jar).
@@ -184,8 +183,7 @@ The following authors have contributed to this project (in alphabetical order):
 
 ## Contributing
 
-We are currently discussing the implementation of a Contributor License Agreement (CLA). Unfortunately,
-we cannot merge external pull requests until this issue is resolved.
+Before accepting external contributions, you need to sign our [CLA](https://cla-assistant.io/Fraunhofer-AISEC/cpg). Our CLA assistent will check, whether you already signed the CLA when you open your first pull request.
 
 ## Further reading
 

build.gradle.kts

@@ -1,7 +1,3 @@
-import org.jetbrains.dokka.gradle.DokkaMultiModuleTask
-import java.io.BufferedOutputStream
-import java.io.ByteArrayOutputStream
-
 /*
  * Copyright (c) 2019-2021, Fraunhofer AISEC. All rights reserved.
  *
@@ -76,7 +72,7 @@ fun generateDokkaWithVersionTag(dokkaMultiModuleTask: org.jetbrains.dokka.gradle
     val config = """{ "version": "$tag", "olderVersionsDir":"${oldOutputPath.path}" }"""
     val mapOf = mapOf(id to config)
 
-    dokkaMultiModuleTask.outputDirectory.set(file(buildDir.resolve("dokkaCustomMultiModuleOutput").resolve(tag)))
+    dokkaMultiModuleTask.outputDirectory.set(file(layout.buildDirectory.asFile.get().resolve("dokkaCustomMultiModuleOutput").resolve(tag)))
     dokkaMultiModuleTask.pluginsMapConfiguration.set(mapOf)
 }
 
@@ -148,3 +144,9 @@ val enableTypeScriptFrontend: Boolean by extra {
     enableTypeScriptFrontend.toBoolean()
 }
 project.logger.lifecycle("TypeScript frontend is ${if (enableTypeScriptFrontend) "enabled" else "disabled"}")
+
+val enableRubyFrontend: Boolean by extra {
+    val enableRubyFrontend: String? by project
+    enableRubyFrontend.toBoolean()
+}
+project.logger.lifecycle("Ruby frontend is ${if (enableRubyFrontend) "enabled" else "disabled"}")

buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts

@@ -23,7 +23,7 @@ repositories {
     mavenCentral()
 
     ivy {
-        setUrl("https://download.eclipse.org/tools/cdt/releases/11.0/cdt-11.0.0/plugins")
+        setUrl("https://download.eclipse.org/tools/cdt/releases/11.3/cdt-11.3.1/plugins")
         metadataSources {
             artifact()
         }
@@ -112,7 +112,11 @@ tasks.withType<KotlinCompile> {
 // common testing configuration
 //
 tasks.test {
-    useJUnitPlatform()
+    useJUnitPlatform() {
+        if (!project.hasProperty("integration")) {
+            excludeTags("integration")
+        }
+    }
     maxHeapSize = "4048m"
 }
 

buildSrc/src/main/kotlin/cpg.formatting-conventions.gradle.kts

@@ -122,21 +122,6 @@ spotless {
         licenseHeader(headerWithStars).yearSeparator(" - ")
     }
 
-    python {
-        targetExclude(
-            fileTree(project.projectDir) {
-                include("**/node_modules")
-            }
-        )
-        target("src/main/**/*.py")
-        targetExclude(
-            fileTree(project.projectDir) {
-                include("src/main/nodejs/node_modules")
-            }
-        )
-        licenseHeader(headerWithHashes, "from").yearSeparator(" - ")
-    }
-
     format("golang") {
         target("src/main/golang/**/*.go")
         licenseHeader(headerWithSlashes, "package").yearSeparator(" - ")

buildSrc/src/main/kotlin/cpg.frontend-dependency-conventions.gradle.kts

@@ -9,6 +9,7 @@ val enableGoFrontend: Boolean by rootProject.extra
 val enablePythonFrontend: Boolean by rootProject.extra
 val enableLLVMFrontend: Boolean by rootProject.extra
 val enableTypeScriptFrontend: Boolean by rootProject.extra
+val enableRubyFrontend: Boolean by rootProject.extra
 
 dependencies {
     if (enableJavaFrontend) api(project(":cpg-language-java"))
@@ -17,4 +18,5 @@ dependencies {
     if (enablePythonFrontend) api(project(":cpg-language-python"))
     if (enableLLVMFrontend) api(project(":cpg-language-llvm"))
     if (enableTypeScriptFrontend) api(project(":cpg-language-typescript"))
+    if (enableRubyFrontend) api(project(":cpg-language-ruby"))
 }

configure_frontends.sh

@@ -56,3 +56,5 @@ answerLLVM=$(ask "Do you want to enable the LLVM frontend? (currently $(getPrope
 setProperty "enableLLVMFrontend" $answerLLVM
 answerTypescript=$(ask "Do you want to enable the TypeScript frontend? (currently $(getProperty "enableTypeScriptFrontend"))")
 setProperty "enableTypeScriptFrontend" $answerTypescript
+answerRuby=$(ask "Do you want to enable the Ruby frontend? (currently $(getProperty "enableRubyFrontend"))")
+setProperty "enableRubyFrontend" $answerRuby

cpg-analysis/build.gradle.kts

@@ -42,8 +42,6 @@ publishing {
 
 dependencies {
     api(projects.cpgCore)
-    testImplementation(projects.cpgLanguageJava)
-    testImplementation(projects.cpgLanguageCxx)
 
     testImplementation(testFixtures(projects.cpgCore))
 }

cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt

@@ -172,8 +172,8 @@ class MultiValueEvaluator : ValueEvaluator() {
         val result = mutableSetOf<Any?>()
         val elseResult = evaluateInternal(expr.elseExpression, depth + 1)
         val thenResult = evaluateInternal(expr.thenExpression, depth + 1)
-        if (thenResult is Collection<*>) result.addAll(thenResult) else result.add(thenResult)
-        if (elseResult is Collection<*>) result.addAll(elseResult) else result.add(elseResult)
+        result.addAnything(thenResult)
+        result.addAnything(elseResult)
         return result
     }
 
@@ -226,7 +226,7 @@ class MultiValueEvaluator : ValueEvaluator() {
         if (prevDFG.size == 1) {
             // There's only one incoming DFG edge, so we follow this one.
             val internalRes = evaluateInternal(prevDFG.first(), depth + 1)
-            return if (internalRes is Collection<*>) internalRes else mutableSetOf(internalRes)
+            return (internalRes as? Collection<*>) ?: mutableSetOf(internalRes)
         }
 
         if (prevDFG.size == 2 && prevDFG.all(::isSimpleForLoop)) {
@@ -239,25 +239,21 @@ class MultiValueEvaluator : ValueEvaluator() {
             val decl = prevDFG.filterIsInstance<VariableDeclaration>()
             for (declaration in decl) {
                 val res = evaluateInternal(declaration, depth + 1)
-                if (res is Collection<*>) {
-                    result.addAll(res)
-                } else {
-                    result.add(res)
-                }
+                result.addAnything(res)
             }
         }
 
         for (expression in prevDFG) {
             val res = evaluateInternal(expression, depth + 1)
-            if (res is Collection<*>) {
-                result.addAll(res)
-            } else {
-                result.add(res)
-            }
+            result.addAnything(res)
         }
         return result
     }
 
+    private fun MutableSet<Any?>.addAnything(element: Any?) {
+        if (element is Collection<*>) this.addAll(element) else this.add(element)
+    }
+
     private fun isSimpleForLoop(node: Node): Boolean {
         // Are we in the for statement somehow?
         var forStatement = node.astParent as? ForStatement
@@ -281,8 +277,11 @@ class MultiValueEvaluator : ValueEvaluator() {
 
     private fun handleSimpleLoopVariable(expr: Reference, depth: Int): Collection<Any?> {
         val loop =
-            expr.prevDFG.firstOrNull { e -> e.astParent is ForStatement }?.astParent
-                as? ForStatement
+            expr.prevDFG.firstOrNull { it.astParent is ForStatement }?.astParent as? ForStatement
+                ?: expr.prevDFG
+                    .firstOrNull { it.astParent?.astParent is ForStatement }
+                    ?.astParent
+                    ?.astParent as? ForStatement
         if (loop == null || loop.condition !is BinaryOperator) return setOf()
 
         var loopVar: Any? =