Merge branch 'openvinotoolkit:master' into tokenizer-fix-decode

.ci/azure/linux_cuda.yml

@@ -103,7 +103,7 @@ jobs:
     displayName: 'Install dependencies'
 
   - script: >
-      sudo docker pull openvino.azurecr.io/openvino_ci/cuda-ubuntu2004:2022.1 && 
+      sudo docker pull openvino.azurecr.io/openvino_ci/cuda-ubuntu2004:2022.1 &&
       sudo docker run --volume $(REPO_DIR)/../:/root/repos --volume $(WORK_DIR):/root/w
       openvino.azurecr.io/openvino_ci/cuda-ubuntu2004:2022.1
       bash -c "
@@ -124,8 +124,9 @@ jobs:
       -DENABLE_PYTHON=OFF
       -DENABLE_NVIDIA=ON
       -DENABLE_TESTS=ON
+      -DENABLE_DATA=OFF
       /root/repos/openvino &&
-      /root/w/ninja -v CudaFuncTests CudaUnitTests"
+      /root/w/ninja -v ov_nvidia_func_tests ov_nvidia_unit_tests"
     workingDirectory: $(WORK_DIR)
     displayName: 'Docker build Lin'
 

.ci/azure/mac.yml

@@ -113,7 +113,6 @@ jobs:
         -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 \
         -DCMAKE_OSX_ARCHITECTURES=$(CMAKE_OSX_ARCHITECTURES) \
         -DENABLE_OV_PADDLE_FRONTEND=OFF \
-        -DENABLE_OV_ONNX_FRONTEND=OFF \
         -DENABLE_OV_TF_FRONTEND=OFF \
         -DENABLE_OV_TF_LITE_FRONTEND=OFF \
         -DENABLE_OV_PYTORCH_FRONTEND=OFF \

.github/CODEOWNERS

@@ -12,5 +12,8 @@
 # OpenVINO NVIDIA plugin:
 /modules/nvidia_plugin/ @openvinotoolkit/openvino_contrib-nvidia_plugin-maintainters
 
+# OpenVINO VS Code extension:
+/modules/openvino_code/ @openvinotoolkit/openvino_contrib-vscode-extension-maintainters
+
 # AI extension for OVMS:
 /modules/ovms_ai_extension/ @openvinotoolkit/openvino_contrib-ovms_ai_extension-maintainers

.github/workflows/sanitizer_cuda.yml

@@ -29,6 +29,6 @@ jobs:
     - name: cmake
       run: env -C ~/runner ./configure_cuda_sanitizer.sh
     - name: ninja
-      run: echo 'all:; +NINJA_STATUS="[%u/%r/%f] " ninja -C ~/runner/build_cuda_sanitizer -v CudaFuncTests CudaUnitTests' | make -j`nproc` -f-
+      run: echo 'all:; +NINJA_STATUS="[%u/%r/%f] " ninja -C ~/runner/build_cuda_sanitizer -v ov_nvidia_func_tests ov_nvidia_unit_tests' | make -j`nproc` -f-
     - name: functional tests
-      run: env -C ~/runner PATH=/usr/local/cuda-11.2/bin:/usr/local/cuda-11.4/bin:/usr/local/cuda-11.7/bin:$PATH ./openvino_contrib/modules/nvidia_plugin/utils/cuda-sanitizer.sh ./build_cuda_sanitizer/bin/intel64/CudaFuncTests --gtest_filter=*smoke*:-*dynamic*:*Dynamic*:smoke_GRU*:smoke_LSTM*:smoke_TensorIterator*:*ConvBiasFusion*:*smoke*OVExecGraphImportExportTest.importExportedIENetwork*:*smoke*OVClassBasicTestP.registerNewPluginNoThrows*:*smoke*OVHoldersTest.Orders*:*smoke*IEClassBasicTestP.registerNewPluginNoThrows*:*smoke*IEClassBasicTestP.smoke_registerPluginsXMLUnicodePath*
+      run: env -C ~/runner PATH=/usr/local/cuda-11.2/bin:/usr/local/cuda-11.4/bin:/usr/local/cuda-11.7/bin:$PATH ./openvino_contrib/modules/nvidia_plugin/utils/cuda-sanitizer.sh ./build_cuda_sanitizer/bin/intel64/ov_nvidia_func_tests --gtest_filter=*smoke*:-*dynamic*:*Dynamic*:smoke_GRU*:smoke_LSTM*:smoke_TensorIterator*:*ConvBiasFusion*:*smoke*OVExecGraphImportExportTest.importExportedIENetwork*:*smoke*OVClassBasicTestP.registerNewPluginNoThrows*:*smoke*OVHoldersTest.Orders*:*smoke*IEClassBasicTestP.registerNewPluginNoThrows*:*smoke*IEClassBasicTestP.smoke_registerPluginsXMLUnicodePath*

.github/workflows/test_cuda.yml

@@ -29,19 +29,19 @@ jobs:
     - name: cmake
       run: env -C ~/runner ./configrelease.sh
     - name: ninja
-      run: echo 'all:; +NINJA_STATUS="[%u/%r/%f] " ninja -C ~/runner/buildrelease -v CudaFuncTests CudaUnitTests benchmark_app' | make -j`nproc` -f-
+      run: echo 'all:; +NINJA_STATUS="[%u/%r/%f] " ninja -C ~/runner/buildrelease -v ov_nvidia_func_tests ov_nvidia_unit_tests benchmark_app' | make -j`nproc` -f-
     - name: functional tests
-      run: source ~/runner/sanitizer-env.sh; ~/runner/buildrelease/bin/intel64/CudaFuncTests --gtest_filter=*smoke*:-*dynamic*:*Dynamic*
+      run: source ~/runner/sanitizer-env.sh; ~/runner/buildrelease/bin/intel64/ov_nvidia_func_tests --gtest_filter=*smoke*:-*dynamic*:*Dynamic*
     - name: unit tests
-      run: source ~/runner/sanitizer-env.sh; ~/runner/buildrelease/bin/intel64/CudaUnitTests
+      run: source ~/runner/sanitizer-env.sh; ~/runner/buildrelease/bin/intel64/ov_nvidia_unit_tests
     - name: networks regression
       run: source ~/runner/sanitizer-env.sh; ~/runner/run_all_models.sh
     - name: cmake 440
       run: ~/docker/e440.sh ./configrelease-440.sh
     - name: ninja 440
-      run: ~/docker/e440.sh ./buildrelease-440.sh CudaUnitTests
+      run: ~/docker/e440.sh ./buildrelease-440.sh ov_nvidia_unit_tests
     - name: unit tests 440
-      run: ~/docker/e440.sh buildrelease-440/bin/intel64/CudaUnitTests
+      run: ~/docker/e440.sh buildrelease-440/bin/intel64/ov_nvidia_unit_tests
 # many functional tests use missing operations, just skip everything for now
 #    - name: functional tests
-#      run: ~/docker/e440.sh buildrelease-440/bin/intel64/CudaFuncTests --gtest_filter=*smoke*:-*dynamic*:*Dynamic*
+#      run: ~/docker/e440.sh buildrelease-440/bin/intel64/ov_nvidia_func_tests --gtest_filter=*smoke*:-*dynamic*:*Dynamic*

.github/workflows/token_merging.yml

@@ -0,0 +1,39 @@
+name: Token Merging - Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  Precommit:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8]
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Create and start a virtual environment
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+    - name: Install dependencies
+      run: |
+        source venv/bin/activate
+        python -m pip install --upgrade pip
+        pip install modules/token_merging/[tests]
+    - name: Run test
+      run: |
+        source venv/bin/activate
+        python -m pytest modules/token_merging/tests/

.gitignore

@@ -3,3 +3,5 @@
 
 **/*.png
 **/*.jar
+
+__pycache__/

.readthedocs.yaml

@@ -1,6 +1,8 @@
 version: 2
 
 build:
-  image: latest
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
   apt_packages:
     - default-jdk

README.md

@@ -8,11 +8,11 @@ So, all the new modules should be developed separately, and published in the `op
 
 This list gives an overview of all modules available inside the contrib repository.
 
-* [**arm_plugin**](./modules/arm_plugin): ARM CPU Plugin -- allows to perform deep neural networks inference on ARM CPUs, using OpenVINO API.
 * [**nvidia_plugin**](./modules/nvidia_plugin): NVIDIA GPU Plugin -- allows to perform deep neural networks inference on NVIDIA GPUs using CUDA, using OpenVINO API.
 * [**java_api**](./modules/java_api): Inference Engine Java API -- provides Java wrappers for Inference Engine public API.
-* [**Azure Video Analyzer**](./modules/ovms_ai_extension/): Azure Video Analyzer Extension -- enables exchange of video frames and inference results between [Azure Video Analyzer (AVA)](https://docs.microsoft.com/en-us/azure/azure-video-analyzer/video-analyzer-docs/overview) and OpenVINO™ Model Server.
 * [**custom_operations**](./modules/custom_operations/): Collection of Custom Operations -- implement Custom Operations with OpenVINO Extensibility Mechanism.
+* [**Token Merging**](./modules/token_merging/): adaptation of [Token Merging method](https://arxiv.org/abs/2210.09461) for OpenVINO.
+* [**OpenVINO Code**](./modules/openvino_code): VSCode extension for AI code completion with OpenVINO.
 
 ## How to build OpenVINO with extra modules
 You can build OpenVINO, so it will include the modules from this repository. Contrib modules are under constant development and it is recommended to use them alongside the master branch or latest releases of OpenVINO.
@@ -34,7 +34,6 @@ $ cmake -DOPENVINO_EXTRA_MODULES=<openvino_contrib>/modules -DBUILD_java_api=OFF
 Additional build instructions are available for the following modules:
 
 * [**nvidia_plugin**](./modules/nvidia_plugin/README.md)
-* [**ovms_ai_extension**](./modules/ovms_ai_extension/README.md)
 * [**custom_operations**](./modules/custom_operations/README.md)
 
 ## Update the repository documentation

SECURITY.md

@@ -0,0 +1,12 @@
+# Security Policy
+
+## Report a Vulnerability
+
+Please report security issues or vulnerabilities to the [Intel® Security Center].
+
+For more information on how Intel® works to resolve security issues, see
+[Vulnerability Handling Guidelines].
+
+[Intel® Security Center]:https://www.intel.com/security
+
+[Vulnerability Handling Guidelines]:https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html

modules/custom_operations/tests/run_tests.py

@@ -4,9 +4,7 @@
 from openvino.runtime import Core
 from openvino.tools.mo import convert_model
 
-import subprocess
 import pytest
-from pathlib import Path
 import numpy as np
 import os
 

modules/custom_operations/user_ie_extensions/grid_sample.cpp

@@ -64,13 +64,13 @@ bool GridSample::evaluate(ov::TensorVector& outputs, const ov::TensorVector& inp
                 const float* inp_row1 = (0 <= y1 && y1 < inpHeight) ? inp + y1 * inpWidth : zeros;
                 float* out = outData + d * channels * outPlane;
                 if ((x1 < 0 || inpWidth <= x1) && (x0 < 0 || inpWidth <= x0)) {
-                    for (int c = 0; c < channels; ++c) {
+                    for (size_t c = 0; c < channels; ++c) {
                         out[offset] = 0;
                         out += outPlane;
                     }
                 }
                 else if (x1 < 0 || inpWidth <= x1) {
-                    for (int c = 0; c < channels; ++c) {
+                    for (size_t c = 0; c < channels; ++c) {
                         out[offset] = inp_row0[x0] +
                             (input_y - y0) * (inp_row1[x0] - inp_row0[x0]) +
                             (input_x - x0) * (-inp_row0[x0] +
@@ -81,15 +81,15 @@ bool GridSample::evaluate(ov::TensorVector& outputs, const ov::TensorVector& inp
                     }
                 }
                 else if (x0 < 0 || inpWidth <= x0) {
-                    for (int c = 0; c < channels; ++c) {
+                    for (size_t c = 0; c < channels; ++c) {
                         out[offset] =
                             (input_x - x0) * (inp_row0[x1] + (input_y - y0) * (inp_row1[x1] - inp_row0[x1]));
                         out += outPlane;
                         inp_row0 += inpPlane;
                         inp_row1 += inpPlane;
                     }
                 } else {
-                    for (int c = 0; c < channels; ++c) {
+                    for (size_t c = 0; c < channels; ++c) {
                         out[offset] = inp_row0[x0] +
                             (input_y - y0) * (inp_row1[x0] - inp_row0[x0]) +
                             (input_x - x0) * (inp_row0[x1] - inp_row0[x0] +

modules/java_api/.gitignore

@@ -0,0 +1,2 @@
+.gradle
+**/build/

modules/java_api/CMakeLists.txt

@@ -18,13 +18,18 @@ if(NOT OpenVINODeveloperPackage_FOUND)
     find_package(OpenVINO REQUIRED)
 endif()
 
-file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/cpp/*.cpp)
+if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    # 'argument': conversion from 'size_t' to 'int', possible loss of data
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4267 /wd4244")
+endif()
+
+file(GLOB_RECURSE sources ${CMAKE_CURRENT_SOURCE_DIR}/src/main/cpp/*.cpp)
 
 add_library(${PROJECT_NAME} SHARED ${sources})
 
 target_link_libraries(${PROJECT_NAME} PRIVATE openvino::runtime)
 target_include_directories(${PROJECT_NAME} PRIVATE ${JNI_INCLUDE_DIRS})
-target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cpp)
+target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/main/cpp)
 
 install(TARGETS ${PROJECT_NAME}
         RUNTIME DESTINATION ${IE_CPACK_RUNTIME_PATH} COMPONENT java_api

modules/java_api/README.md

@@ -9,13 +9,20 @@
 To install OpenJDK:
 
 * Ubuntu systems:
-```bash
-sudo apt-get install -y default-jdk
-```
+    ```bash
+    sudo apt-get install -y default-jdk
+    ```
 
-### Build
+## Build
+
+To build OpenVINO so that it includes this module, use the following CMake command:
+```shell
+cd <openvino_build>
+cmake -DBUILD_java_api=ON -DOPENVINO_EXTRA_MODULES=<openvino_contrib>/modules <openvino_source_directory>
+cmake --build . -j8
+```
 
-Set environment OpenVINO variables:
+Set OpenVINO environment variables:
 ```bash
 source <openvino_install>/setupvars.sh
 ```
@@ -26,6 +33,20 @@ cd openvino_contrib/modules/java_api
 gradle build
 ```
 
-### Import
+## Import
 
 Use `import org.intel.openvino.*;` for OpenVINO Java API 2.0 or `import org.intel.openvino.compatibility.*;` for deprecated API.
+
+## Set up the development environment
+
+### Import to IntelliJ IDEA
+
+- Install and enable the **Gradle** IntelliJ Plugin by navigating to **Settings** > **Plugins**. Search for the
+  Gradle plugin and install it, if not already installed.
+- Clone the repository
+  ```shell
+  git clone https://github.com/openvinotoolkit/openvino_contrib.git
+  ```
+- To import the project into IntelliJ IDEA, select **File** > **Open** and locate the  java api module in `<openvino_contrib>/modules/java_api`.
+
+See [here](src/test/README.md) for instructions on running tests.

modules/java_api/build.gradle

@@ -3,29 +3,53 @@ plugins {
 }
 
 import org.gradle.internal.os.OperatingSystem
+
 println 'Operating system: ' + OperatingSystem.current()
+def arch = System.getProperty("os.arch")
+def ov_arch = arch;
+println 'CPU architecture: ' + arch
+
 
 def nativesCPP;
-def openvinoVersion = "2022.1"
+def openvinoVersion = "2023.0"
+
+def native_resources = []
+def tbb_dir = System.getenv('TBB_DIR')
+if (tbb_dir) {
+    if (new File(tbb_dir + "/../lib/").exists()) {
+        native_resources.add(tbb_dir + "/../lib/");
+    } else {
+        native_resources.add(tbb_dir + "/../../");
+    }
+}
+
+if (arch == "x86_64" || arch == "amd64" || arch == "x64" || arch == "x86-64") {
+    arch = "x86_64";
+    ov_arch = "intel64";
+} else if (arch == "aarch64" || arch == "arm64" || arch == "arm-v8") {
+    arch = "arm64";
+    ov_arch = "aarch64";
+} else if (arch == "arm" || arch == "arm-v7" || arch == "armv7" || arch == "arm32") {
+    arch = "arm32";
+    ov_arch = "armhf";
+}
 
 if (OperatingSystem.current().isMacOsX()) {
-    nativesCPP = 'macosx-x86_64'
+    nativesCPP = 'macosx-'
+    native_resources.add(System.getenv('INTEL_OPENVINO_DIR') + "/runtime/lib/" + ov_arch + "/Release");
 } else if (OperatingSystem.current().isLinux()) {
-    nativesCPP = 'linux-x86_64'
+    nativesCPP = 'linux-'
+    native_resources.add(System.getenv('INTEL_OPENVINO_DIR') + "/runtime/lib/" + ov_arch);
 } else if (OperatingSystem.current().isWindows()) {
-    nativesCPP = 'windows-x86_64'
+    nativesCPP = 'windows-'
+    native_resources.add(System.getenv('INTEL_OPENVINO_DIR') + "/runtime/bin/" + ov_arch + "/Release");
 } else {
     logger.warn('Unknown operating system!')
 }
+nativesCPP += arch;
 
 project.version = "${openvinoVersion}-${nativesCPP}"
 
-def native_resources = [
-    System.getenv('INTEL_OPENVINO_DIR') + "/runtime/lib/intel64",  // UNIX
-    System.getenv('INTEL_OPENVINO_DIR') + "/runtime/lib/intel64/Release",  // Mac
-    System.getenv('INTEL_OPENVINO_DIR') + "/runtime/bin/intel64/Release",  // Windows
-    System.getenv('TBB_DIR') + "/../lib/",
-]
 def resources_list = ""
 native_resources.each {
     def tree = fileTree(it) {
@@ -37,26 +61,26 @@ native_resources.each {
         }
     }
 }
-file('native').mkdirs()
-file('native/resources_list.txt').text = "${resources_list}"
+file('src/main/native').mkdirs()
+file('src/main/native/resources_list.txt').text = "${resources_list}"
 sourceSets {
     main {
         java {
-            srcDirs = ["org"]
+            srcDirs = ["src/main/java"]
         }
         resources {
             srcDirs = native_resources
-            srcDir 'native'
+            srcDir 'src/main/native'
         }
     }
     test {
         java {
-            srcDirs = ["."]
-            include "tests/compatibility/*.java", "tests/*.java"
+            srcDirs = ["src/test/java"]
+            include "org/intel/openvino/compatibility/*.java", "org/intel/openvino/*.java"
         }
     }
 }
-repositories{
+repositories {
     mavenCentral()
 }
 dependencies {
@@ -67,4 +91,4 @@ test {
     systemProperty 'MODELS_PATH', System.getProperty('MODELS_PATH')
     systemProperty 'device', System.getProperty('device')
 }
-test.onlyIf { project.hasProperty('run_tests') }
+test.onlyIf { project.hasProperty('run_tests') }

modules/java_api/docs/conf.py

@@ -52,7 +52,7 @@
 
 html_title = 'OpenVINO for Java'
 
-import subprocess
-subprocess.call('javadoc ../org/intel/openvino/*.java', shell=True)
+import subprocess # nosec
+subprocess.call('javadoc ../src/main/java/org/intel/openvino/*.java', shell=True)
 
 html_extra_path = ['.']