feat: add AI explanations

package.json

@@ -51,6 +51,7 @@
     "monaco-editor": "^0.48.0",
     "svelte-fa": "^4.0.2",
     "svelte-i18n": "^4.0.0",
+    "svelte-markdown": "^0.4.1",
     "vite-plugin-static-copy": "1.0.4",
     "web-serial-polyfill": "^1.0.15"
   }

src/assets/translations/en.json

@@ -118,5 +118,8 @@
 	"UPLOADING_FIRMWARE": "Uploading firmware",
 	"INSTALLING_LIBRARIES": "Installing libraries",
 	"NOT_CONNECTED": "Not connected",
-	"NOT_CONNECTED_DESC": "You won't see new output"
+	"NOT_CONNECTED_DESC": "You won't see new output",
+	"EXPLANATION": "Explanation",
+	"EXPLAIN_BLOCK": "Explain Block ✨",
+	"META_ATTRIBUTION": "Built with Meta Llama 3"
 }

src/assets/translations/nl.json

@@ -118,5 +118,8 @@
 	"UPLOADING_FIRMWARE": "Firmware uploaden",
 	"INSTALLING_LIBRARIES": "Bibliotheken installeren",
 	"NOT_CONNECTED": "Niet verbonden",
-	"NOT_CONNECTED_DESC": "Je ziet geen nieuwe output"
+	"NOT_CONNECTED_DESC": "Je ziet geen nieuwe output",
+	"EXPLANATION": "Uitleg",
+	"EXPLAIN_BLOCK": "Leg blok uit ✨",
+	"META_ATTRIBUTION": "Gemaakt met Meta Llama 3"
 }

src/lib/components/core/popups/Popup.svelte

@@ -2,7 +2,6 @@
 import type { PopupState } from "$state/popup.svelte";
 import { setContext } from "svelte";
 import { writable } from "svelte/store";
-import Windowed from "./Windowed.svelte";
 
 interface Props {
 	state: PopupState;
@@ -21,6 +20,7 @@ $effect(() => {
     <div class="localRoot">
         <div
             class="popup"
+			style:translate="{popupState.anchor}"
             style:left={`${popupState.position.x}px`}
             style:top={`${popupState.position.y}px`}
         >
@@ -35,7 +35,6 @@ $effect(() => {
 <style>
     .popup {
         position: absolute;
-        translate: -50% -50%;
         background: var(--background);
         border-radius: 6px;
         overflow: hidden;

src/lib/components/core/popups/popups/Explanation.svelte

@@ -0,0 +1,83 @@
+<script lang="ts">
+	import SvelteMarkdown from 'svelte-markdown'
+	import {getContext, onDestroy, onMount} from "svelte";
+	import type {Writable} from "svelte/store";
+	import {popups, type PopupState} from "$state/popup.svelte";
+	import { _ } from 'svelte-i18n'
+
+	interface Props {
+		explanation: Promise<string>
+	}
+
+	let { explanation }: Props = $props();
+
+	const popupState = getContext<Writable<PopupState>>("state");
+
+	let element: HTMLDivElement
+	function click(event: MouseEvent) {
+		if (element.contains(event.target as HTMLElement)) return
+
+		popups.close($popupState.id);
+	}
+
+	onMount(() => {
+		document.body.addEventListener('click', click)
+	})
+	onDestroy(() => {
+		document.body.removeEventListener('click', click)
+	})
+</script>
+
+<div class="content" bind:this={element}>
+	<h2>{$_("EXPLANATION")}</h2>
+	{#await explanation}
+		<div class="container">
+			<div class="loading"></div>
+			<div class="loading"></div>
+			<div class="loading"></div>
+			<div class="loading"></div>
+		</div>
+	{:then explanation}
+		<SvelteMarkdown source="{explanation}" />
+	{/await}
+	<div class="footer">{$_("META_ATTRIBUTION")}</div>
+</div>
+
+<style>
+	.content {
+		width: 400px;
+		padding: 20px;
+	}
+
+	.footer {
+		color: var(--on-secondary-muted);
+	}
+
+	.container {
+		display: flex;
+		flex-direction: column;
+		gap: 5px;
+		margin: 16px 0;
+	}
+
+	.loading {
+		background: linear-gradient(
+			100deg,
+			rgba(255, 255, 255, 0) 40%,
+			rgba(255, 255, 255, .5) 50%,
+			rgba(255, 255, 255, 0) 60%
+		) #a5a5a550;
+		background-size: 200% 100%;
+		background-position-x: 180%;
+		animation: 1s loading ease-in-out infinite;
+
+		height: 14px;
+		width: 100%;
+	}
+
+	@keyframes loading {
+		to {
+			background-position-x: -20%;
+		}
+	}
+</style>

src/lib/components/ui/Dropper.svelte

@@ -0,0 +1,94 @@
+<script lang="ts">
+import Fa from "svelte-fa";
+import {
+	faMagicWandSparkles,
+} from "@fortawesome/free-solid-svg-icons";
+import * as Blockly from "blockly";
+import {get} from "svelte/store";
+import {workspace} from "$state/blockly.svelte";
+import type {WorkspaceSvg} from "blockly";
+import {onDestroy, onMount} from "svelte";
+import {explain} from "$domain/blockly/blockly";
+
+function getBlockForPosition(event: PointerEvent) {
+	const space = get(workspace) as WorkspaceSvg
+	if (!space) return
+
+	return space.getAllBlocks(true).reverse().find(block => {
+		const bounding = block.pathObject.svgPath.getBoundingClientRect()
+
+		if (bounding.x > event.clientX || bounding.y > event.clientY) return false
+		return !(bounding.x + bounding.width < event.clientX || bounding.y + bounding.height < event.clientY)
+	})
+}
+
+let previous: Blockly.BlockSvg = null
+let selecting = false
+function pointerMove(event: PointerEvent) {
+	if (!selecting) return
+
+	const block =getBlockForPosition(event)
+	previous?.setHighlighted(false)
+	block?.setHighlighted(true)
+
+	previous = block
+}
+
+async function select(event: PointerEvent) {
+	if (!selecting) return
+
+	previous?.setHighlighted(false)
+	selecting = false
+
+	const block = getBlockForPosition(event)
+	if (!block) return
+
+	await explain(block)
+}
+
+function onclick() {
+	selecting = true
+}
+
+onMount(() => {
+	document.body.addEventListener('pointermove', pointerMove)
+	document.body.addEventListener('pointerup', select)
+})
+onDestroy(() => {
+	document.body.removeEventListener('pointermove', pointerMove)
+	document.body.removeEventListener('pointerup', select)
+})
+</script>
+
+<button class="dropper" class:selecting {onclick}>
+	<Fa icon="{faMagicWandSparkles}" />
+</button>
+{#if selecting}
+	<div class="noInteract"></div>
+{/if}
+
+<style>
+	.dropper {
+		all: unset;
+		position: fixed;
+		bottom: 35px;
+		right: 116px;
+		color: rgba(136, 136, 136, 0.5);
+		font-size: 45px;
+		cursor: pointer;
+	}
+
+	.selecting {
+		color: rgba(84, 93, 149, 0.5);
+	}
+
+	.noInteract {
+		position: fixed;
+		left: 0;
+		top: 0;
+		width: 100vw;
+		height: 100vh;
+		z-index: 9999999999;
+		cursor: crosshair;
+	}
+</style>

src/lib/components/workspace/blocks/Blocks.svelte

@@ -11,10 +11,11 @@ import {
 } from "$state/blockly.svelte";
 import { code, robot } from "$state/workspace.svelte";
 import { arduino } from "@leaphy-robotics/leaphy-blocks";
-import { Events, WorkspaceSvg, serialization } from "blockly";
+import {Events, WorkspaceSvg, serialization, Block, ContextMenuRegistry} from "blockly";
 import { onMount } from "svelte";
 import { locale } from "svelte-i18n";
 import { get } from "svelte/store";
+import Dropper from "$components/ui/Dropper.svelte";
 
 let backgroundX = $state(0);
 
@@ -83,6 +84,7 @@ theme.subscribe((theme) => {
 		<img class="background" src="{$robot.background}" alt="{$robot.name}" style:left={`${backgroundX}px`}>
 	{/if}
     <div class="blockly" bind:this={element}></div>
+	<Dropper />
 </div>
 
 <style>

src/lib/domain/blockly/blockly.ts

@@ -3,17 +3,18 @@ import "@blockly/field-bitmap";
 import Prompt from "$components/core/popups/popups/Prompt.svelte";
 import { type RobotDevice, inFilter } from "$domain/robots";
 import { RobotType } from "$domain/robots.types";
-import { audio } from "$state/blockly.svelte";
-import { popups } from "$state/popup.svelte";
+import {audio, workspace} from "$state/blockly.svelte";
+import {Anchor, popups} from "$state/popup.svelte";
 import { BackpackChange } from "@blockly/workspace-backpack";
+import { _ } from 'svelte-i18n'
 import {
 	CATEGORIES,
 	ProcedureSerializer,
 	blocks,
 	registerExtensions,
 	translations,
 } from "@leaphy-robotics/leaphy-blocks";
-import { serialization } from "blockly";
+import {Block, ContextMenu, ContextMenuRegistry, serialization, type WorkspaceSvg} from "blockly";
 import type { Workspace } from "blockly";
 import type {
 	CategoryInfo,
@@ -24,6 +25,10 @@ import { LeaphyCategory } from "./category-ui/category";
 import { LeaphyToolbox } from "./category-ui/toolbox";
 import PinSelectorField from "./fields";
 import toolbox from "./toolbox";
+import {get} from "svelte/store";
+import Groq from "groq-sdk";
+import {locale} from "svelte-i18n";
+import Explanation from "$components/core/popups/popups/Explanation.svelte";
 
 Blockly.defineBlocksWithJsonArray(blocks);
 Blockly.fieldRegistry.register("field_pin_selector", PinSelectorField);
@@ -185,3 +190,108 @@ export function setupWorkspace(
 
 	return workspace;
 }
+
+const groq = new Groq({
+	apiKey: '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',
+	dangerouslyAllowBrowser: true
+})
+
+function serializeBlock(block: Block, selected?: Block, indent = 0, inline = false) {
+	if (!block) return ''
+
+	const indentValue = '   '.repeat(indent)
+	let value = indentValue
+	if (selected?.id === block.id) value += `**BEGIN_SELECT**${inline ? '' : `\n${indentValue}`}`
+
+	value += block.inputList.map(input => {
+		const result = []
+
+		result.push(...input.fieldRow.map(field => field.getText()))
+		if (input.connection) {
+			switch (input.connection.type) {
+				case 1: {
+					result.push(`(${serializeBlock(input.connection.targetBlock(), selected, 0, true)})`)
+					break
+				}
+				case 3: {
+					result.push(`{\n${serializeBlock(input.connection.targetBlock(), selected, indent + 1)}\n${indentValue}}`)
+					break
+				}
+			}
+		}
+
+		return result
+	}).flat().join(' ')
+
+	if (selected?.id === block.id) value += `${inline ? '' : `\n${indentValue}`}**END_SELECT**`
+	if (block.getNextBlock()) {
+		value += `\n${serializeBlock(block.getNextBlock(), selected, indent)}`
+	}
+
+	return value
+}
+
+function pseudo(workspace: Workspace, selected?: Block) {
+	const blocks = workspace.getTopBlocks()
+
+	return blocks.map(block => serializeBlock(block, selected)).join('\n\n')
+}
+
+export async function explain(block: Blockly.BlockSvg) {
+	const workspace = block.workspace
+	const code = pseudo(workspace, block)
+
+	const locales = {
+		'en': 'English',
+		'nl': 'Dutch'
+	}
+
+
+	const position = block.pathObject.svgPath.getBoundingClientRect()
+	await popups.open({
+		component: Explanation,
+		data: {
+			explanation: fetch('http://localhost:8000/ai/generate', {
+				method: 'post',
+				headers: {
+					'content-type': 'application/json'
+				},
+				body: JSON.stringify({
+					messages: [
+						{
+							role: 'system',
+							content: `explain the selected portion of the following pseudo code (SELECT_BEGIN - SELECT_END) in simple terms, the pseudo code is directly generated from a blockly environment to program robots called Leaphy EasyBloqs, you must do this in ${locales[get(locale)]}`
+						},
+						{
+							role: 'user',
+							content: `\`\`\`\n${code}\n\`\`\``
+						},
+						{
+							role: 'system',
+							content: 'please only return the explanation for the given set of code in simple terms, like you\'re explaining it to someone who has never touched code before, do not explain the code around the given set of code unless directly related, do not talk about or reference the pseudo code directly, you are talking about the selected code almost exclusively, so you do not have to include the **begin_select** and **end_select** tokens in your response, only include your explanation in the response'
+						}
+					],
+					model: 'Llama3-70b-8192',
+				})
+			}).then(async res => JSON.parse(await res.text()))
+		},
+		allowInteraction: true
+	}, {
+		position: { x: position.x + position.width + 10 - window.innerWidth / 2, y: position.y + 10 - window.innerHeight / 2 },
+		anchor: Anchor.TopLeft
+	})
+}
+
+const explainBlockOption: ContextMenuRegistry.RegistryItem = {
+	id: 'explain_block',
+	scopeType: ContextMenuRegistry.ScopeType.BLOCK,
+	displayText: () => get(_)('EXPLAIN_BLOCK'),
+	weight: -1,
+	preconditionFn() {
+		return 'enabled'
+	},
+	async callback(scope) {
+		await explain(scope.block)
+	}
+}
+ContextMenuRegistry.registry.register(explainBlockOption);