Merge branch 'main' into main

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos, baidu-obs
+# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos, baidu-obs, supabase
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -99,6 +99,11 @@ VOLCENGINE_TOS_ACCESS_KEY=your-access-key
 VOLCENGINE_TOS_SECRET_KEY=your-secret-key
 VOLCENGINE_TOS_REGION=your-region
 
+# Supabase Storage Configuration
+SUPABASE_BUCKET_NAME=your-bucket-name
+SUPABASE_API_KEY=your-access-key
+SUPABASE_URL=your-server-url
+
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*

api/configs/middleware/__init__.py

@@ -12,6 +12,7 @@
 from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
 from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
 from configs.middleware.storage.oci_storage_config import OCIStorageConfig
+from configs.middleware.storage.supabase_storage_config import SupabaseStorageConfig
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
 from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
@@ -191,6 +192,22 @@ def BROKER_USE_SSL(self) -> bool:
  return self.CELERY_BROKER_URL.startswith("rediss://") if self.CELERY_BROKER_URL else False
 
 
+class InternalTestConfig(BaseSettings):
+ """
+ Configuration settings for Internal Test
+ """
+
+ AWS_SECRET_ACCESS_KEY: Optional[str] = Field(
+ description="Internal test AWS secret access key",
+ default=None,
+ )
+
+ AWS_ACCESS_KEY_ID: Optional[str] = Field(
+ description="Internal test AWS access key ID",
+ default=None,
+ )
+
+
 class MiddlewareConfig(
  # place the configs in alphabet order
  CeleryConfig,
@@ -206,6 +223,7 @@ class MiddlewareConfig(
  HuaweiCloudOBSStorageConfig,
  OCIStorageConfig,
  S3StorageConfig,
+ SupabaseStorageConfig,
  TencentCloudCOSStorageConfig,
  VolcengineTOSStorageConfig,
  # configs of vdb and vdb providers
@@ -224,5 +242,6 @@ class MiddlewareConfig(
  TiDBVectorConfig,
  WeaviateConfig,
  ElasticsearchConfig,
+ InternalTestConfig,
 ):
  pass

api/configs/middleware/storage/supabase_storage_config.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class SupabaseStorageConfig(BaseModel):
+ """
+ Configuration settings for Supabase Object Storage Service
+ """
+
+ SUPABASE_BUCKET_NAME: Optional[str] = Field(
+ description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
+ default=None,
+ )
+
+ SUPABASE_API_KEY: Optional[str] = Field(
+ description="API KEY for authenticating with Supabase",
+ default=None,
+ )
+
+ SUPABASE_URL: Optional[str] = Field(
+ description="URL of the Supabase",
+ default=None,
+ )

api/controllers/console/datasets/external.py

@@ -13,6 +13,7 @@
 from services.dataset_service import DatasetService
 from services.external_knowledge_service import ExternalDatasetService
 from services.hit_testing_service import HitTestingService
+from services.knowledge_service import ExternalDatasetTestService
 
 
 def _validate_name(name):
@@ -232,8 +233,31 @@ def post(self, dataset_id):
  raise InternalServerError(str(e))
 
 
+class BedrockRetrievalApi(Resource):
+ # this api is only for internal testing
+ def post(self):
+ parser = reqparse.RequestParser()
+ parser.add_argument("retrieval_setting", nullable=False, required=True, type=dict, location="json")
+ parser.add_argument(
+ "query",
+ nullable=False,
+ required=True,
+ type=str,
+ )
+ parser.add_argument("knowledge_id", nullable=False, required=True, type=str)
+ args = parser.parse_args()
+
+ # Call the knowledge retrieval service
+ result = ExternalDatasetTestService.knowledge_retrieval(
+ args["retrieval_setting"], args["query"], args["knowledge_id"]
+ )
+ return result, 200
+
+
 api.add_resource(ExternalKnowledgeHitTestingApi, "/datasets/<uuid:dataset_id>/external-hit-testing")
 api.add_resource(ExternalDatasetCreateApi, "/datasets/external")
 api.add_resource(ExternalApiTemplateListApi, "/datasets/external-knowledge-api")
 api.add_resource(ExternalApiTemplateApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
 api.add_resource(ExternalApiUseCheckApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>/use-check")
+# this api is only for internal test
+api.add_resource(BedrockRetrievalApi, "/test/retrieval")

api/controllers/console/workspace/model_providers.py

@@ -126,13 +126,12 @@ class ModelProviderIconApi(Resource):
  Get model provider icon
  """
 
- @setup_required
- @login_required
- @account_initialization_required
  def get(self, provider: str, icon_type: str, lang: str):
  model_provider_service = ModelProviderService()
  icon, mimetype = model_provider_service.get_model_provider_icon(
- provider=provider, icon_type=icon_type, lang=lang
+ provider=provider,
+ icon_type=icon_type,
+ lang=lang,
  )
 
  return send_file(io.BytesIO(icon), mimetype=mimetype)

api/core/model_runtime/model_providers/siliconflow/llm/llm.py

@@ -1,8 +1,18 @@
 from collections.abc import Generator
 from typing import Optional, Union
 
-from core.model_runtime.entities.llm_entities import LLMResult
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult
 from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
+from core.model_runtime.entities.model_entities import (
+ AIModelEntity,
+ FetchFrom,
+ ModelFeature,
+ ModelPropertyKey,
+ ModelType,
+ ParameterRule,
+ ParameterType,
+)
 from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAIAPICompatLargeLanguageModel
 
 
@@ -29,3 +39,53 @@ def validate_credentials(self, model: str, credentials: dict) -> None:
  def _add_custom_parameters(cls, credentials: dict) -> None:
  credentials["mode"] = "chat"
  credentials["endpoint_url"] = "https://api.siliconflow.cn/v1"
+
+ def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None:
+ return AIModelEntity(
+ model=model,
+ label=I18nObject(en_US=model, zh_Hans=model),
+ model_type=ModelType.LLM,
+ features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL, ModelFeature.STREAM_TOOL_CALL]
+ if credentials.get("function_calling_type") == "tool_call"
+ else [],
+ fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+ model_properties={
+ ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 8000)),
+ ModelPropertyKey.MODE: LLMMode.CHAT.value,
+ },
+ parameter_rules=[
+ ParameterRule(
+ name="temperature",
+ use_template="temperature",
+ label=I18nObject(en_US="Temperature", zh_Hans="温度"),
+ type=ParameterType.FLOAT,
+ ),
+ ParameterRule(
+ name="max_tokens",
+ use_template="max_tokens",
+ default=512,
+ min=1,
+ max=int(credentials.get("max_tokens", 1024)),
+ label=I18nObject(en_US="Max Tokens", zh_Hans="最大标记"),
+ type=ParameterType.INT,
+ ),
+ ParameterRule(
+ name="top_p",
+ use_template="top_p",
+ label=I18nObject(en_US="Top P", zh_Hans="Top P"),
+ type=ParameterType.FLOAT,
+ ),
+ ParameterRule(
+ name="top_k",
+ use_template="top_k",
+ label=I18nObject(en_US="Top K", zh_Hans="Top K"),
+ type=ParameterType.FLOAT,
+ ),
+ ParameterRule(
+ name="frequency_penalty",
+ use_template="frequency_penalty",
+ label=I18nObject(en_US="Frequency Penalty", zh_Hans="重复惩罚"),
+ type=ParameterType.FLOAT,
+ ),
+ ],
+ )

api/core/model_runtime/model_providers/siliconflow/siliconflow.yaml

@@ -20,6 +20,7 @@ supported_model_types:
  - speech2text
 configurate_methods:
  - predefined-model
+ - customizable-model
 provider_credential_schema:
  credential_form_schemas:
  - variable: api_key
@@ -30,3 +31,57 @@ provider_credential_schema:
  placeholder:
  zh_Hans: 在此输入您的 API Key
  en_US: Enter your API Key
+model_credential_schema:
+ model:
+ label:
+ en_US: Model Name
+ zh_Hans: 模型名称
+ placeholder:
+ en_US: Enter your model name
+ zh_Hans: 输入模型名称
+ credential_form_schemas:
+ - variable: api_key
+ label:
+ en_US: API Key
+ type: secret-input
+ required: true
+ placeholder:
+ zh_Hans: 在此输入您的 API Key
+ en_US: Enter your API Key
+ - variable: context_size
+ label:
+ zh_Hans: 模型上下文长度
+ en_US: Model context size
+ required: true
+ type: text-input
+ default: '4096'
+ placeholder:
+ zh_Hans: 在此输入您的模型上下文长度
+ en_US: Enter your Model context size
+ - variable: max_tokens
+ label:
+ zh_Hans: 最大 token 上限
+ en_US: Upper bound for max tokens
+ default: '4096'
+ type: text-input
+ show_on:
+ - variable: __model_type
+ value: llm
+ - variable: function_calling_type
+ label:
+ en_US: Function calling
+ type: select
+ required: false
+ default: no_call
+ options:
+ - value: no_call
+ label:
+ en_US: Not Support
+ zh_Hans: 不支持
+ - value: function_call
+ label:
+ en_US: Support
+ zh_Hans: 支持
+ show_on:
+ - variable: __model_type
+ value: llm

api/core/model_runtime/model_providers/tongyi/rerank/_position.yaml

@@ -0,0 +1 @@
+- gte-rerank

api/core/model_runtime/model_providers/tongyi/rerank/gte-rerank.yaml

@@ -0,0 +1,4 @@
+model: gte-rerank
+model_type: rerank
+model_properties:
+ context_size: 4000