From 1430ae41472b9f426383eff60356fe4455519220 Mon Sep 17 00:00:00 2001 From: Benjamin Ironside Goldstein Date: Wed, 29 May 2024 16:17:23 -0700 Subject: [PATCH] Adds LLM performance matrix doc to ESS & serverless --- docs/assistant/llm-performance-matrix.asciidoc | 14 ++++++++++++++ docs/assistant/security-assistant.asciidoc | 1 + .../assistant/llm-performance-matrix.mdx | 17 +++++++++++++++++ docs/serverless/serverless-security.docnav.json | 3 +++ 4 files changed, 35 insertions(+) create mode 100644 docs/assistant/llm-performance-matrix.asciidoc create mode 100644 docs/serverless/assistant/llm-performance-matrix.mdx diff --git a/docs/assistant/llm-performance-matrix.asciidoc b/docs/assistant/llm-performance-matrix.asciidoc new file mode 100644 index 0000000000..ed2964493d --- /dev/null +++ b/docs/assistant/llm-performance-matrix.asciidoc @@ -0,0 +1,14 @@ +[[llm-performance-matrix]] += Large language model performance matrix + +This table describes the performance of various LLMs for different use-cases in {elastic-sec}, based on our internal testing. To learn more about how to implement these use-cases, refer to <> or <>. + +[cols="1,1,1,1,1,1,1", options="header"] +|=== +| Feature/Model | Claude 3: Opus | Claude 3: Sonnet | Claude 3: Haiku | GPT-4o | GPT-4 Turbo | GPT-4 32K + +| Assistant - General | Excellent | Excellent | Excellent | Excellent | Excellent | Excellent +| Assistant - {{esql}} generation | Great | Great | Poor | Excellent | Poor | Excellent +| Assistant - Alert questions | Excellent | Excellent | Excellent | Excellent | Poor | Good (limited context) +| Attack Discovery | Excellent | Great | Poor | Poor | Good | Good (limited context) +|=== diff --git a/docs/assistant/security-assistant.asciidoc b/docs/assistant/security-assistant.asciidoc index 9f0066833c..7c94ff23db 100644 --- a/docs/assistant/security-assistant.asciidoc +++ b/docs/assistant/security-assistant.asciidoc @@ -194,6 +194,7 @@ In addition to practical advice, AI Assistant can offer conceptual advice, tips, include::ai-alert-triage.asciidoc[leveloffset=+1] +include::llm-performance-matrix.asciidoc[leveloffset=+1] include::azure-openai-setup.asciidoc[leveloffset=+1] include::connect-to-openai.asciidoc[leveloffset=+1] include::connect-to-bedrock.asciidoc[leveloffset=+1] diff --git a/docs/serverless/assistant/llm-performance-matrix.mdx b/docs/serverless/assistant/llm-performance-matrix.mdx new file mode 100644 index 0000000000..7b51834a09 --- /dev/null +++ b/docs/serverless/assistant/llm-performance-matrix.mdx @@ -0,0 +1,17 @@ +--- +id: llm-performance-matrix +slug: /serverless/security/llm-performance-matrix +title: Large language model performance matrix +description: Learn how different models perform on different tasks in ((elastic-sec)). +tags: ["security", "overview", "get-started"] +status: in review +--- + +This table describes the performance of various LLMs for different use-cases in ((elastic-sec)), based on our internal testing. To learn more about how to implement these use-cases, refer to or . + +| Feature/Model | Claude 3: Opus | Claude 3: Sonnet | Claude 3: Haiku | GPT-4o | GPT-4 Turbo | GPT-4 32K | +|------------------------------|-----------------|-------------------|------------------|---------|--------------|-----------| +| Assistant - General | Excellent | Excellent | Excellent | Excellent| Excellent | Excellent | +| Assistant - ((esql)) generation| Great | Great | Poor | Excellent| Poor | Excellent | +| Assistant - Alert questions | Excellent | Excellent | Excellent | Excellent| Poor | Good (limited context) | +| Attack Discovery | Excellent | Great | Poor | Poor | Good | Good (limited context) | diff --git a/docs/serverless/serverless-security.docnav.json b/docs/serverless/serverless-security.docnav.json index e2b0da2e04..fbc65af420 100644 --- a/docs/serverless/serverless-security.docnav.json +++ b/docs/serverless/serverless-security.docnav.json @@ -30,6 +30,9 @@ { "id":"serverlessSecurityAssistantAlertTriage" }, + { + "id":"llm-performance-matrix" + }, { "id": "serverlessSecurityConnectBedrock" },