-
Notifications
You must be signed in to change notification settings - Fork 354
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: move llm & callback manager to core module (#1026)
- Loading branch information
Showing
75 changed files
with
641 additions
and
983 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,12 @@ | ||
export { Settings } from "./settings"; | ||
export { CallbackManager } from "./settings/callback-manager"; | ||
export type { | ||
BaseEvent, | ||
LLMEndEvent, | ||
LLMStartEvent, | ||
LLMStreamEvent, | ||
LLMToolCallEvent, | ||
LLMToolResultEvent, | ||
LlamaIndexEventMaps, | ||
} from "./settings/callback-manager"; | ||
export type { JSONArray, JSONObject, JSONValue } from "./type"; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
import { AsyncLocalStorage, CustomEvent } from "@llamaindex/env"; | ||
import type { | ||
ChatMessage, | ||
ChatResponse, | ||
ChatResponseChunk, | ||
ToolCall, | ||
ToolOutput, | ||
} from "../../llms"; | ||
import type { UUID } from "../type"; | ||
|
||
export type BaseEvent<Payload> = CustomEvent<{ | ||
payload: Readonly<Payload>; | ||
}>; | ||
|
||
export type LLMStartEvent = BaseEvent<{ | ||
id: UUID; | ||
messages: ChatMessage[]; | ||
}>; | ||
export type LLMToolCallEvent = BaseEvent<{ | ||
toolCall: ToolCall; | ||
}>; | ||
export type LLMToolResultEvent = BaseEvent<{ | ||
toolCall: ToolCall; | ||
toolResult: ToolOutput; | ||
}>; | ||
export type LLMEndEvent = BaseEvent<{ | ||
id: UUID; | ||
response: ChatResponse; | ||
}>; | ||
export type LLMStreamEvent = BaseEvent<{ | ||
id: UUID; | ||
chunk: ChatResponseChunk; | ||
}>; | ||
|
||
export interface LlamaIndexEventMaps { | ||
"llm-start": LLMStartEvent; | ||
"llm-end": LLMEndEvent; | ||
"llm-tool-call": LLMToolCallEvent; | ||
"llm-tool-result": LLMToolResultEvent; | ||
"llm-stream": LLMStreamEvent; | ||
} | ||
|
||
export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> { | ||
private constructor(event: string, options?: CustomEventInit) { | ||
super(event, options); | ||
} | ||
|
||
static fromEvent<Type extends keyof LlamaIndexEventMaps>( | ||
type: Type, | ||
detail: LlamaIndexEventMaps[Type]["detail"], | ||
) { | ||
return new LlamaIndexCustomEvent(type, { | ||
detail: detail, | ||
}); | ||
} | ||
} | ||
|
||
type EventHandler<Event> = (event: Event) => void; | ||
|
||
export class CallbackManager { | ||
#handlers = new Map<keyof LlamaIndexEventMaps, EventHandler<CustomEvent>[]>(); | ||
|
||
on<K extends keyof LlamaIndexEventMaps>( | ||
event: K, | ||
handler: EventHandler<LlamaIndexEventMaps[K]>, | ||
) { | ||
if (!this.#handlers.has(event)) { | ||
this.#handlers.set(event, []); | ||
} | ||
this.#handlers.get(event)!.push(handler); | ||
return this; | ||
} | ||
|
||
off<K extends keyof LlamaIndexEventMaps>( | ||
event: K, | ||
handler: EventHandler<LlamaIndexEventMaps[K]>, | ||
) { | ||
if (!this.#handlers.has(event)) { | ||
return this; | ||
} | ||
const cbs = this.#handlers.get(event)!; | ||
const index = cbs.indexOf(handler); | ||
if (index > -1) { | ||
cbs.splice(index, 1); | ||
} | ||
return this; | ||
} | ||
|
||
dispatchEvent<K extends keyof LlamaIndexEventMaps>( | ||
event: K, | ||
detail: LlamaIndexEventMaps[K]["detail"], | ||
) { | ||
const cbs = this.#handlers.get(event); | ||
if (!cbs) { | ||
return; | ||
} | ||
queueMicrotask(() => { | ||
cbs.forEach((handler) => | ||
handler( | ||
LlamaIndexCustomEvent.fromEvent(event, structuredClone(detail)), | ||
), | ||
); | ||
}); | ||
} | ||
} | ||
|
||
export const globalCallbackManager = new CallbackManager(); | ||
|
||
const callbackManagerAsyncLocalStorage = | ||
new AsyncLocalStorage<CallbackManager>(); | ||
|
||
let currentCallbackManager: CallbackManager | null = null; | ||
|
||
export function getCallbackManager(): CallbackManager { | ||
return ( | ||
callbackManagerAsyncLocalStorage.getStore() ?? | ||
currentCallbackManager ?? | ||
globalCallbackManager | ||
); | ||
} | ||
|
||
export function setCallbackManager(callbackManager: CallbackManager) { | ||
currentCallbackManager = callbackManager; | ||
} | ||
|
||
export function withCallbackManager<Result>( | ||
callbackManager: CallbackManager, | ||
fn: () => Result, | ||
): Result { | ||
return callbackManagerAsyncLocalStorage.run(callbackManager, fn); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
export { BaseLLM, ToolCallLLM } from "./base"; | ||
export type { | ||
BaseTool, | ||
BaseToolWithCall, | ||
|
11 changes: 10 additions & 1 deletion
11
...index/src/internal/context/EventCaller.ts → packages/core/src/utils/event-caller.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.