Merge branch 'main' into java/dev_lotjonat_modules

Bit-Quill · Apr 3, 2024 · 182a35f · 182a35f
2 parents 93a388d + a422b24
commit 182a35f
Show file tree

Hide file tree

Showing 104 changed files with 8,632 additions and 17,689 deletions.
diff --git a/.editorconfig b/.editorconfig
diff --git a/.github/workflows/install-redis-modules/action.yml b/.github/workflows/install-redis-modules/action.yml
@@ -21,17 +21,6 @@ inputs:
 runs:
     using: "composite"
     steps:
-        - name: Cache RediSearch Dependencies
-          if: inputs.modules == 'all' || inputs.modules == 'search'
-          id: cache-dependencies-redisearch
-          uses: actions/cache@v3
-          with:
-            path: |
-              ./redisearch/.conan
-              ./cmake
-              ./redisearch/bin 
-            key: ${{ runner.os }}-${{ inputs.redis-version }}-redisearch
-
         - name: Cache RedisJSON Dependencies
           if: inputs.modules == 'all' || inputs.modules == 'json'
           id: cache-dependencies-redisjson
@@ -42,14 +31,6 @@ runs:
               ./redisjson/bin 
             key: ${{ runner.os }}-${{ inputs.redis-version }}-redisjson
 
-        - name: Checkout RediSearch Repository
-          if: steps.cache-dependencies-redisearch.outputs.cache-hit != 'true' && (inputs.modules == 'all' || inputs.modules == 'search')
-          uses: actions/checkout@v4
-          with: 
-              repository: "RediSearch/RediSearch"
-              path: "./redisearch" 
-              ref: ${{ startsWith(inputs.redis-version, '6') && '2.6' || '' }}
-              submodules: recursive
 
         - name: Install CMake
           if: steps.cache-dependencies-redisearch.outputs.cache-hit != 'true' || steps.cache-dependencies-redisjson.outputs.cache-hit != 'true'
@@ -60,23 +41,6 @@ runs:
             sudo apt-get install -y cmake
             cp /usr/bin/cmake ./cmake
           
-        - name: Build RediSearch
-          if: steps.cache-dependencies-redisearch.outputs.cache-hit != 'true' && (inputs.modules == 'all' || inputs.modules == 'search')
-          shell: bash  
-          working-directory: ./redisearch
-          run: |
-            set -x
-            echo "Building RediSearch..."
-            sudo pip install --upgrade conan
-            make
-
-        - name: Copy redisearch.so
-          if: inputs.modules == 'all' || inputs.modules == 'search'
-          shell: bash  
-          run: |
-            set -x
-            echo "Copying RediSearch..."
-            cp $GITHUB_WORKSPACE/redisearch/bin/linux-x64-release/search/redisearch.so $GITHUB_WORKSPACE/redisearch.so
 
         - name: Checkout RedisJSON Repository
           if: steps.cache-dependencies-redisjson.outputs.cache-hit != 'true' && (inputs.modules == 'all' || inputs.modules == 'json')

diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml
@@ -85,7 +85,7 @@ jobs:
               working-directory: ./node/hybrid-node-tests/ecmascript-test
 
             - name: test redis modules
-              run: npm run test-modules -- --load-module=$GITHUB_WORKSPACE/redisearch.so --load-module=$GITHUB_WORKSPACE/redisjson.so
+              run: npm run test-modules -- --load-module=$GITHUB_WORKSPACE/redisjson.so
               working-directory: ./node
 
             - uses: ./.github/workflows/test-benchmark

diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml
@@ -7,6 +7,7 @@ on:
       paths:
         - .github/workflows/npm-cd.yml
         - .github/workflows/build-node-wrapper/action.yml
+        - .github/workflows/start-self-hosted-runner/action.yml
     push:
         tags:
             - "v*.*"
@@ -16,7 +17,21 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
+    start-self-hosted-runner:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v4
+        - name: Start self hosted EC2 runner
+          uses: ./.github/workflows/start-self-hosted-runner
+          with:
+              aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }}
+              aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }}
+              aws-region: ${{ secrets.AWS_REGION }}
+              ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}
+
     publish-binaries:
+        needs: start-self-hosted-runner
         if: github.repository_owner == 'aws'
         name: Publish packages to NPM
         runs-on: ${{ matrix.build.RUNNER }}
@@ -54,6 +69,10 @@ jobs:
                           TARGET: aarch64-apple-darwin,
                       }
         steps:
+            - name: Setup self-hosted runner access
+              if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }}
+              run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/glide-for-redis
+
             - name: Checkout
               uses: actions/checkout@v4
               with:

diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml
@@ -32,14 +32,20 @@ jobs:
             - name: Set the release version
               shell: bash
               run: |
-                  export version=`if ${{ github.event_name == 'schedule' || github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${{ github.event.inputs.version }}; fi`
+                  export version=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo '255.255.255'; else echo "$INPUT_VERSION"; fi`
                   echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV
+              env:
+                EVENT_NAME: ${{ github.event_name }}
+                INPUT_VERSION: ${{ github.event.inputs.version }}
 
             - name: Set the base branch
               run: |
-                export BASE_BRANCH=`if ${{ github.event_name == 'schedule'}}; then echo 'main'; elif ${{ github.event_name == 'workflow_dispatch'}}; then echo ${{ github.event.inputs.branch }}; else echo "";fi`
+                export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi`
                 echo "Base branch is: ${BASE_BRANCH}"
                 echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV
+              env:
+                EVENT_NAME: ${{ github.event_name }}
+                INPUT_BRANCH: ${{ github.event.inputs.branch }}
 
             - name: Checkout
               uses: actions/checkout@v4
@@ -184,16 +190,18 @@ jobs:
             - name: Create pull request
               if: ${{ env.FOUND_DIFF  == 'true' && github.event_name != 'pull_request' }}
               run: |
-                export BRANCH_NAME=`if ${{ github.event_name == 'schedule' || github.event_name == 'pull_request' }}; then echo "scheduled-ort"; else echo "ort-v${{ github.event.inputs.version }}"; fi`
+                export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi`
                 echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}"
                 git config --global user.email "[email protected]"
                 git config --global user.name "ort-bot"
                 git checkout -b ${BRANCH_NAME}
-
                 git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS
                 git commit -m "Updated attribution files"
                 git push --set-upstream origin ${BRANCH_NAME} -f
                 title="Updated attribution files for ${BRANCH_NAME}"
                 gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}'
               env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                EVENT_NAME: ${{ github.event_name }}
+                INPUT_VERSION: ${{ github.event.inputs.version }}
+
diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml
@@ -7,6 +7,7 @@ on:
       paths:
         - .github/workflows/pypi-cd.yml
         - .github/workflows/build-python-wrapper/action.yml
+        - .github/workflows/start-self-hosted-runner/action.yml
     push:
         tags:
             - "v*.*"
@@ -16,7 +17,21 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
+    start-self-hosted-runner:
+        runs-on: ubuntu-latest
+        steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+          - name: Start self hosted EC2 runner
+            uses: ./.github/workflows/start-self-hosted-runner
+            with:
+                aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }}
+                aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }}
+                aws-region: ${{ secrets.AWS_REGION }}
+                ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}
+
     publish-binaries:
+        needs: start-self-hosted-runner
         if: github.repository_owner == 'aws'
         name: Publish packages to PyPi
         runs-on: ${{ matrix.build.RUNNER }}
@@ -55,6 +70,10 @@ jobs:
                           TARGET: aarch64-apple-darwin,
                       }
         steps:
+            - name: Setup self-hosted runner access
+              if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }}
+              run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/glide-for-redis
+
             - name: Checkout
               uses: actions/checkout@v4
               with:

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -109,7 +109,7 @@ jobs:
               run: |
                   source .env/bin/activate
                   cd python/tests/
-                  pytest --asyncio-mode=auto --override-ini=addopts= --load-module=$GITHUB_WORKSPACE/redisearch.so --load-module=$GITHUB_WORKSPACE/redisjson.so
+                  pytest --asyncio-mode=auto --override-ini=addopts= --load-module=$GITHUB_WORKSPACE/redisjson.so
 
             - uses: ./.github/workflows/test-benchmark
               with:

diff --git a/.github/workflows/start-self-hosted-runner/action.yml b/.github/workflows/start-self-hosted-runner/action.yml
@@ -0,0 +1,32 @@
+name: Start self hosted EC2 runner
+
+inputs:
+    aws-region:
+        description: AWS Region, e.g. us-east-1
+        required: true
+    aws-access-key-id:
+        description: AWS Access Key ID. Provide this key if you want to assume a role using access keys rather than a web identity token.
+        required: true
+    aws-secret-access-key:
+        description: AWS Secret Access Key. Required if aws-access-key-id is provided.
+        required: true
+    ec2-instance-id:
+        description: AWS EC2 instance ID for the self hosted runner
+        required: true
+
+runs:
+    using: "composite"
+    steps:
+        - name: Configure AWS Credentials
+          uses: aws-actions/configure-aws-credentials@v4
+          with:
+              aws-access-key-id: ${{ inputs.aws-access-key-id }}
+              aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+              aws-region: ${{ inputs.aws-region }}
+        - name: Start EC2 self hosted runner
+          shell: bash
+          run: |
+              sudo apt update
+              sudo apt install awscli -y
+              aws ssm send-command --instance-ids ${{ inputs.ec2-instance-id }} --document-name  StartGithubSelfHostedRunner --output text
+              aws ssm list-command-invocations 
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+#### Changes
+* Python: Added JSON.DEL JSON.FORGET commands  ([#1146](https://github.com/aws/glide-for-redis/pull/1146))
+* 
+#### Fixes
+* Python: Fix typing error "‘type’ object is not subscriptable" ([#1203](https://github.com/aws/glide-for-redis/pull/1203))
+
+## 0.3.3 (2024-03-28)
+
+#### Fixes
+
+* Node: Fix issue with dual usage, `CommonJS` and `ECMAScript` modules. ([#1199](https://github.com/aws/glide-for-redis/pull/1199))
+
+## 0.3.0 (2024-03-25)
+
 #### Changes
 
 * Python Node: Allow routing Cluster requests by address. ([#1021](https://github.com/aws/glide-for-redis/pull/1021))
@@ -21,16 +35,16 @@
 * Python: Added json module and JSON.SET JSON.GET commands  ([#1056](https://github.com/aws/glide-for-redis/pull/1056))
 * Python, Node: Added Time command ([#1147](https://github.com/aws/glide-for-redis/pull/1147)), ([#1114](https://github.com/aws/glide-for-redis/pull/1114))
 * Python, Node: Added LINDEX command ([#1058](https://github.com/aws/glide-for-redis/pull/1058), [#999](https://github.com/aws/glide-for-redis/pull/999))
-* Python: Added ZRANK command ([#1065](https://github.com/aws/glide-for-redis/pull/1065))
+* Python, Node: Added ZRANK command ([#1065](https://github.com/aws/glide-for-redis/pull/1065), [#1149](https://github.com/aws/glide-for-redis/pull/1149))
 * Core: Enabled Cluster Mode periodic checks by default ([#1089](https://github.com/aws/glide-for-redis/pull/1089))
 * Node: Added Rename command. ([#1124](https://github.com/aws/glide-for-redis/pull/1124))
+* Python: Added JSON.TOGGLE command ([#1184](https://github.com/aws/glide-for-redis/pull/1184))
 
 #### Features
 
 * Python: Allow chaining function calls on transaction. ([#987](https://github.com/aws/glide-for-redis/pull/987))
 * Node: Adding support for GLIDE's usage in projects based on either `CommonJS` or `ECMAScript` modules. ([#1132](https://github.com/aws/glide-for-redis/pull/1132))
-* Python: Added Cluster Mode configuration for periodic checks interval ([#1089](https://github.com/aws/glide-for-redis/pull/1089))
-
+* Python, Node: Added Cluster Mode configuration for periodic checks interval ([#1089](https://github.com/aws/glide-for-redis/pull/1089), [#1158](https://github.com/aws/glide-for-redis/pull/1158))
 
 ## 0.2.0 (2024-02-11)
 
@@ -68,4 +82,4 @@
 
 Preview release of **GLIDE for Redis** a Polyglot Redis client.
 
-See the [README](README.md) for additional information.
+See the [README](README.md) for additional information.
diff --git a/benchmarks/csharp/.editorconfig b/benchmarks/csharp/.editorconfig
@@ -0,0 +1 @@
+../../csharp/.editorconfig
diff --git a/benchmarks/csharp/csharp_benchmark.csproj b/benchmarks/csharp/csharp_benchmark.csproj
@@ -1,20 +1,21 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\csharp\lib\glide.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="LinqStatistics" Version="2.3.0" />
-    <PackageReference Include="StackExchange.Redis" Version="2.6.48" />
-  </ItemGroup>
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\csharp\lib\glide.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="LinqStatistics" Version="2.3.0" />
+    <PackageReference Include="StackExchange.Redis" Version="2.6.48" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+  </PropertyGroup>
+
+</Project>
diff --git a/benchmarks/install_and_test.sh b/benchmarks/install_and_test.sh
@@ -68,7 +68,6 @@ function runCSharpBenchmark(){
   dotnet clean
   dotnet build --configuration Release /warnaserror
   dotnet run --framework net6.0 --configuration Release --resultsFile=../$1 --dataSize $2 --concurrentTasks $concurrentTasks --clients $chosenClients --host $host --clientCount $clientCount $tlsFlag $portFlag $minimalFlag
-  dotnet run --framework net8.0 --configuration Release --resultsFile=../$1 --dataSize $2 --concurrentTasks $concurrentTasks --clients $chosenClients --host $host --clientCount $clientCount $tlsFlag $portFlag $minimalFlag
 }
 
 function runJavaBenchmark(){

diff --git a/benchmarks/node/package.json b/benchmarks/node/package.json
@@ -15,9 +15,9 @@
         "@types/stats-lite": "^2.2.0",
         "command-line-args": "^5.2.1",
         "glide-for-redis": "file:../../node",
-        "ioredis": "^5.3.2",
+        "ioredis": "5.3.2",
         "percentile": "^1.6.0",
-        "redis": "^4.6.2",
+        "redis": "4.6.13",
         "stats-lite": "^2.2.0"
     },
     "devDependencies": {

diff --git a/benchmarks/python/requirements.txt b/benchmarks/python/requirements.txt
@@ -2,4 +2,4 @@ hiredis
 numpy
 
 # redis-py
-redis>=5.0.1
+redis==5.0.3