From 3de97b92d71be564de5d9260319ea7877581300e Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 15:36:09 -0700 Subject: [PATCH 1/9] add rules to promptmixing --- lib/shared/src/prompt/prompt-mixin.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index 2ea45f582b94..0f1a9ea51d64 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -27,7 +27,8 @@ export class PromptMixin { * Prepends all mixins to `humanMessage`. Modifies and returns `humanMessage`. */ public static mixInto(humanMessage: InteractionMessage): InteractionMessage { - const mixins = [...this.mixins, ...this.customMixin].map(mixin => mixin.prompt).join('\n\n') + const rule = newPromptMixin(rules) + const mixins = [rule, ...this.customMixin, ...this.mixins].map(mixin => mixin.prompt).join('\n\n') if (mixins) { // Stuff the prompt mixins at the start of the human text. // Note we do not reflect them in displayText. @@ -48,10 +49,13 @@ export class PromptMixin { */ export function languagePromptMixin(languageCode: string): PromptMixin { return new PromptMixin( - `(Reply as Cody developed by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}" unless instructed) Hi, I need your help.` + `(Reply as Cody developed by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}".)Hi, I need your help.` ) } export function newPromptMixin(text: string): PromptMixin { return new PromptMixin(text) } + +const rules = + 'Rules: Answer questions only if certain. Reference only verified file names/paths. Provide full code where applicable. State if unsure. Do not guess or fabricate information. Think step-by-step.' From c392971502c6c2f2f21cddcc45f634b972ba8dc1 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 15:37:41 -0700 Subject: [PATCH 2/9] add create --- lib/shared/src/prompt/prompt-mixin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index 0f1a9ea51d64..9f2bb577788c 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -49,7 +49,7 @@ export class PromptMixin { */ export function languagePromptMixin(languageCode: string): PromptMixin { return new PromptMixin( - `(Reply as Cody developed by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}".)Hi, I need your help.` + `(Reply as Cody created and developed by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}".)Hi, I need your help.` ) } From 12aef4fb0639ade541d0513f8e43aadf6ffad73a Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 18:09:20 -0700 Subject: [PATCH 3/9] fix array structure --- lib/shared/src/prompt/prompt-mixin.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index 9f2bb577788c..cf01650ef794 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -27,8 +27,7 @@ export class PromptMixin { * Prepends all mixins to `humanMessage`. Modifies and returns `humanMessage`. */ public static mixInto(humanMessage: InteractionMessage): InteractionMessage { - const rule = newPromptMixin(rules) - const mixins = [rule, ...this.customMixin, ...this.mixins].map(mixin => mixin.prompt).join('\n\n') + const mixins = [...this.mixins, ...this.customMixin].map(mixin => mixin.prompt).join('\n\n') if (mixins) { // Stuff the prompt mixins at the start of the human text. // Note we do not reflect them in displayText. @@ -40,7 +39,9 @@ export class PromptMixin { /** * Creates a mixin with the given, fixed prompt to insert. */ - constructor(private readonly prompt: string) {} + constructor(private readonly prompt: string) { + PromptMixin.mixins.push(new PromptMixin(rules)) + } } /** @@ -49,7 +50,7 @@ export class PromptMixin { */ export function languagePromptMixin(languageCode: string): PromptMixin { return new PromptMixin( - `(Reply as Cody created and developed by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}".)Hi, I need your help.` + `(Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}".)Hi, I need your help.` ) } @@ -58,4 +59,4 @@ export function newPromptMixin(text: string): PromptMixin { } const rules = - 'Rules: Answer questions only if certain. Reference only verified file names/paths. Provide full code where applicable. State if unsure. Do not guess or fabricate information. Think step-by-step.' + 'Rules: Answer questions only if certain. Reference only verified file names/paths. Provide full workable code where applicable. State if unsure. Do not guess or fabricate information. Think step-by-step.' From 28d9fb8e4b4d33255830b610c3b6dff9c42b5720 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 18:33:17 -0700 Subject: [PATCH 4/9] update prompt --- lib/shared/src/prompt/prompt-mixin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index cf01650ef794..8eb9427d1cfb 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -50,7 +50,7 @@ export class PromptMixin { */ export function languagePromptMixin(languageCode: string): PromptMixin { return new PromptMixin( - `(Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}".)Hi, I need your help.` + `(Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}") Hi, I need your help.` ) } @@ -59,4 +59,4 @@ export function newPromptMixin(text: string): PromptMixin { } const rules = - 'Rules: Answer questions only if certain. Reference only verified file names/paths. Provide full workable code where applicable. State if unsure. Do not guess or fabricate information. Think step-by-step.' + "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Answer only if certain or tell me you don't know. Think step-by-step." From fc23b33f5f5b55d88c8334951d2b9636820389a2 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 18:59:29 -0700 Subject: [PATCH 5/9] fix test --- lib/shared/src/prompt/prompt-mixin.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index 8eb9427d1cfb..9ce8f8106ca4 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -1,11 +1,14 @@ import { InteractionMessage } from '../chat/transcript/messages' +const rules = + "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Think step-by-step. Answer only if certain or tell me you don't know." + /** * Prompt mixins elaborate every prompt presented to the LLM. * Add a prompt mixin to prompt for cross-cutting concerns relevant to multiple recipes. */ export class PromptMixin { - private static mixins: PromptMixin[] = [] + private static mixins: PromptMixin[] = [newPromptMixin(rules)] private static customMixin: PromptMixin[] = [] /** @@ -39,9 +42,7 @@ export class PromptMixin { /** * Creates a mixin with the given, fixed prompt to insert. */ - constructor(private readonly prompt: string) { - PromptMixin.mixins.push(new PromptMixin(rules)) - } + constructor(private readonly prompt: string) {} } /** @@ -57,6 +58,3 @@ export function languagePromptMixin(languageCode: string): PromptMixin { export function newPromptMixin(text: string): PromptMixin { return new PromptMixin(text) } - -const rules = - "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Answer only if certain or tell me you don't know. Think step-by-step." From 7aae30a71a456168d03c4eb63220ee99ce9d8711 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 19:05:53 -0700 Subject: [PATCH 6/9] woops --- lib/shared/src/prompt/prompt-mixin.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index 9ce8f8106ca4..da4f25018102 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -8,7 +8,7 @@ const rules = * Add a prompt mixin to prompt for cross-cutting concerns relevant to multiple recipes. */ export class PromptMixin { - private static mixins: PromptMixin[] = [newPromptMixin(rules)] + private static mixins: PromptMixin[] = [] private static customMixin: PromptMixin[] = [] /** @@ -30,6 +30,7 @@ export class PromptMixin { * Prepends all mixins to `humanMessage`. Modifies and returns `humanMessage`. */ public static mixInto(humanMessage: InteractionMessage): InteractionMessage { + this.mixins.push(newPromptMixin(rules)) const mixins = [...this.mixins, ...this.customMixin].map(mixin => mixin.prompt).join('\n\n') if (mixins) { // Stuff the prompt mixins at the start of the human text. From 1aedb1d706dd4b554db68653c18360b4d2e5bffb Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 19:22:55 -0700 Subject: [PATCH 7/9] add rules to languagePromptMixin prompt --- lib/shared/src/prompt/prompt-mixin.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index da4f25018102..df654434c46d 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -1,8 +1,5 @@ import { InteractionMessage } from '../chat/transcript/messages' -const rules = - "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Think step-by-step. Answer only if certain or tell me you don't know." - /** * Prompt mixins elaborate every prompt presented to the LLM. * Add a prompt mixin to prompt for cross-cutting concerns relevant to multiple recipes. @@ -30,7 +27,6 @@ export class PromptMixin { * Prepends all mixins to `humanMessage`. Modifies and returns `humanMessage`. */ public static mixInto(humanMessage: InteractionMessage): InteractionMessage { - this.mixins.push(newPromptMixin(rules)) const mixins = [...this.mixins, ...this.customMixin].map(mixin => mixin.prompt).join('\n\n') if (mixins) { // Stuff the prompt mixins at the start of the human text. @@ -51,8 +47,10 @@ export class PromptMixin { * End with a new statement to redirect Cody to the next prompt. This prevents Cody from responding to the language prompt. */ export function languagePromptMixin(languageCode: string): PromptMixin { + const rules = + "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Think step-by-step. Answer only if certain or tell me you don't know." return new PromptMixin( - `(Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}") Hi, I need your help.` + `${rules} (Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}") Hi, I need your help.` ) } From 504900a1dff228d4810778320f46a80e08be1a65 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 19:25:39 -0700 Subject: [PATCH 8/9] second thought, add rules to output text instead --- lib/shared/src/prompt/prompt-mixin.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index df654434c46d..e01c3247580c 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -27,11 +27,13 @@ export class PromptMixin { * Prepends all mixins to `humanMessage`. Modifies and returns `humanMessage`. */ public static mixInto(humanMessage: InteractionMessage): InteractionMessage { + const rules = + "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Think step-by-step. Answer only if certain or tell me you don't know." const mixins = [...this.mixins, ...this.customMixin].map(mixin => mixin.prompt).join('\n\n') if (mixins) { // Stuff the prompt mixins at the start of the human text. // Note we do not reflect them in displayText. - return { ...humanMessage, text: `${mixins}\n\n${humanMessage.text}` } + return { ...humanMessage, text: `${rules}\n\n${mixins}\n\n${humanMessage.text}` } } return humanMessage } @@ -47,10 +49,8 @@ export class PromptMixin { * End with a new statement to redirect Cody to the next prompt. This prevents Cody from responding to the language prompt. */ export function languagePromptMixin(languageCode: string): PromptMixin { - const rules = - "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Think step-by-step. Answer only if certain or tell me you don't know." return new PromptMixin( - `${rules} (Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}") Hi, I need your help.` + `(Reply as Cody created by Sourcegraph in the language with RFC5646/ISO language code "${languageCode}") Hi, I need your help.` ) } From 457a3a22d7774f46a9b7b1612761bfb9f2d9a9f6 Mon Sep 17 00:00:00 2001 From: Beatrix Date: Thu, 27 Jul 2023 21:10:21 -0700 Subject: [PATCH 9/9] update prompt --- lib/shared/src/prompt/prompt-mixin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/shared/src/prompt/prompt-mixin.ts b/lib/shared/src/prompt/prompt-mixin.ts index e01c3247580c..f114b7e82d6d 100644 --- a/lib/shared/src/prompt/prompt-mixin.ts +++ b/lib/shared/src/prompt/prompt-mixin.ts @@ -28,7 +28,7 @@ export class PromptMixin { */ public static mixInto(humanMessage: InteractionMessage): InteractionMessage { const rules = - "Rules: Provide full workable code as code snippets. Reference only verified file names/paths. Don't make assumptions or fabricate information. Think step-by-step. Answer only if certain or tell me you don't know." + "Answer questions using the shared context without making assumptions or fabricating additional details. If my request requires you to generate new code snippets, make sure to include complete workable code, not fragments. If you do not have enough information to answer, tell me you don't know." const mixins = [...this.mixins, ...this.customMixin].map(mixin => mixin.prompt).join('\n\n') if (mixins) { // Stuff the prompt mixins at the start of the human text.