chore: merge with main

.github/workflows/integration_test.yml

@@ -24,6 +24,7 @@ env:
   LOCALHOST_URL: http://localhost
   LOCALHOST_WS: ws://localhost/ws/v1
   APPFLOWY_REDIS_URI: redis://redis:6379
+  APPFLOWY_AI_REDIS_URL: redis://redis:6379
   LOCALHOST_GOTRUE: http://localhost/gotrue
   POSTGRES_PASSWORD: password
   DATABASE_URL: postgres://postgres:password@localhost:5432/postgres
@@ -105,6 +106,12 @@ jobs:
         # the wasm-pack headless tests will run on random ports, so we need to allow all origins
         run: sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1/g' nginx/nginx.conf
 
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
       - name: Run Docker-Compose
         run: |
           export APPFLOWY_WORKER_VERSION=${GITHUB_SHA}
@@ -113,29 +120,20 @@ jobs:
           docker compose -f docker-compose-ci.yml up -d
           docker ps -a
 
-          container_id=$(docker ps --filter name=appflowy-cloud-ai-1 -q)
-          if [ -n "$container_id" ]; then
-          echo "Displaying logs for the AppFlowy-AI container..."
-            docker logs "$container_id"
-          else
-            echo "No running container found to display logs."
-          fi
-
       - name: Install prerequisites
         run: |
           sudo apt-get update
-          sudo apt-get install protobuf-compiler
+          sudo apt-get install -y protobuf-compiler
 
       - name: Run Tests
         run: |
           echo "Running tests for ${{ matrix.test_service }} with flags: ${{ matrix.test_cmd }}"
           RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }}
 
-      - name: Run Tests from main branch
+      - name: Docker Logs
+        if: always()
         run: |
-          git fetch origin main
-          git checkout main
-          RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }}
+          docker logs appflowy-cloud-ai-1
 
   cleanup:
     name: Cleanup Docker Images

.github/workflows/push_latest_docker.yml

@@ -95,6 +95,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           provenance: false
           build-args: |
+            PROFILE=release
             FEATURES=
 
       - name: Logout from Docker Hub

Cargo.toml

@@ -301,6 +301,11 @@ codegen-units = 1
 inherits = "release"
 debug = true
 
+[profile.ci]
+inherits = "release"
+opt-level = 2
+lto = false # Disable Link-Time Optimization
+
 [patch.crates-io]
 # It's diffcult to resovle different version with the same crate used in AppFlowy Frontend and the Client-API crate.
 # So using patch to workaround this issue.
@@ -314,4 +319,5 @@ collab-importer = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev
 
 [features]
 history = []
+# Some AI test features are not available for self-hosted AppFlowy Cloud. Therefore, AI testing is disabled by default.
 ai-test-enabled = ["client-api-test/ai-test-enabled"]

Dockerfile

@@ -16,16 +16,19 @@ RUN apt update && apt install -y protobuf-compiler lld clang
 
 # Specify a default value for FEATURES; it could be an empty string if no features are enabled by default
 ARG FEATURES=""
+ARG PROFILE="release"
 
 COPY --from=planner /app/recipe.json recipe.json
 # Build our project dependencies
+ENV CARGO_BUILD_JOBS=4
 RUN cargo chef cook --release --recipe-path recipe.json
+
 COPY . .
 ENV SQLX_OFFLINE true
 
 # Build the project
-RUN echo "Building with features: ${FEATURES}"
-RUN cargo build --profile=release --features "${FEATURES}" --bin appflowy_cloud
+RUN echo "Building with profile: ${PROFILE}, features: ${FEATURES}, "
+RUN cargo build --profile=${PROFILE} --features "${FEATURES}" --bin appflowy_cloud
 
 FROM debian:bookworm-slim AS runtime
 WORKDIR /app

admin_frontend/src/web_api.rs

@@ -15,8 +15,8 @@ use axum::extract::{Path, Query};
 use axum::http::{status, HeaderMap, StatusCode};
 use axum::response::{IntoResponse, Redirect, Result};
 use axum::routing::{delete, get};
-use axum::{Form, Json};
 use axum::{extract::State, routing::post, Router};
+use axum::{Form, Json};
 use axum_extra::extract::cookie::Cookie;
 use axum_extra::extract::CookieJar;
 use base64::engine::Engine;

deny.toml

@@ -1,2 +1,2 @@
 [advisories]
-ignore = ["RUSTSEC-2024-0370"]
+ignore = ["RUSTSEC-2024-0370", "RUSTSEC-2024-0384"]

deploy.env

@@ -146,6 +146,7 @@ APPFLOWY_AI_OPENAI_API_KEY=
 APPFLOWY_AI_SERVER_PORT=5001
 APPFLOWY_AI_SERVER_HOST=ai
 APPFLOWY_AI_DATABASE_URL=postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
+APPFLOWY_AI_REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}
 APPFLOWY_LOCAL_AI_TEST_ENABLED=false
 
 # AppFlowy Indexer

dev.env

@@ -4,6 +4,7 @@ APPFLOWY_DATABASE_URL=postgres://postgres:password@localhost:5432/postgres
 APPFLOWY_ACCESS_CONTROL=true
 APPFLOWY_WEBSOCKET_MAILBOX_SIZE=6000
 APPFLOWY_DATABASE_MAX_CONNECTIONS=40
+APPFLOWY_DOCUMENT_CONTENT_SPLIT_LEN=8000
 
 # This file is used to set the environment variables for local development
 # Copy this file to .env and change the values as needed
@@ -112,6 +113,7 @@ APPFLOWY_AI_OPENAI_API_KEY=
 APPFLOWY_AI_SERVER_PORT=5001
 APPFLOWY_AI_SERVER_HOST=localhost
 APPFLOWY_AI_DATABASE_URL=postgresql+psycopg://postgres:password@postgres:5432/postgres
+APPFLOWY_AI_REDIS_URL=redis://redis:6379
 APPFLOWY_LOCAL_AI_TEST_ENABLED=false
 
 # AppFlowy Indexer

docker-compose-ci.yml

@@ -120,6 +120,7 @@ services:
       dockerfile: Dockerfile
       args:
         FEATURES: ""
+        PROFILE: ci
     image: appflowyinc/appflowy_cloud:${APPFLOWY_CLOUD_VERSION:-latest}
 
   admin_frontend:
@@ -138,7 +139,7 @@ services:
 
   ai:
     restart: on-failure
-    image: appflowyinc/appflowy_ai:${APPFLOWY_AI_VERSION:-latest}
+    image: appflowyinc/appflowy_ai_premium:${APPFLOWY_AI_VERSION:-latest}
     ports:
       - "5001:5001"
     environment:
@@ -147,6 +148,7 @@ services:
       - LOCAL_AI_AWS_SECRET_ACCESS_KEY=${LOCAL_AI_AWS_SECRET_ACCESS_KEY}
       - APPFLOWY_AI_SERVER_PORT=${APPFLOWY_AI_SERVER_PORT}
       - APPFLOWY_AI_DATABASE_URL=${APPFLOWY_AI_DATABASE_URL}
+      - APPFLOWY_AI_REDIS_URL=${APPFLOWY_AI_REDIS_URL}
 
   appflowy_worker:
     restart: on-failure

docker-compose-dev.yml

@@ -122,6 +122,7 @@ services:
       - OPENAI_API_KEY=${APPFLOWY_AI_OPENAI_API_KEY}
       - APPFLOWY_AI_SERVER_PORT=${APPFLOWY_AI_SERVER_PORT}
       - APPFLOWY_AI_DATABASE_URL=${APPFLOWY_AI_DATABASE_URL}
+      - APPFLOWY_AI_REDIS_URL=${APPFLOWY_AI_REDIS_URL}
 
 volumes:
   postgres_data:

docker-compose.yml

@@ -140,6 +140,7 @@ services:
       - OPENAI_API_KEY=${APPFLOWY_AI_OPENAI_API_KEY}
       - APPFLOWY_AI_SERVER_PORT=${APPFLOWY_AI_SERVER_PORT}
       - APPFLOWY_AI_DATABASE_URL=${APPFLOWY_AI_DATABASE_URL}
+      - APPFLOWY_AI_REDIS_URL=${APPFLOWY_AI_REDIS_URL}
 
   appflowy_worker:
     restart: on-failure