Fix retraining for RunMe (#124)

* Fix training on RunMe logs #110 
* We need to allow Foyle to be configured with the directory of the
RunMe logs
   * Add LearnerConfig to Foyle config with the logs directory
* Fix a bug in analyze preventing BlockLogs from being updated with the
executed block when using RunMe
* The bug was that when looking for the last execution trace; we weren't
considering RunMe traces

* Update the RunMe developer guide

* Update the docs

app/cmd/logs.go

@@ -25,7 +25,7 @@ func NewLogsCmd() *cobra.Command {
 
 // NewLogsProcessCmd returns a command to process the assets
 func NewLogsProcessCmd() *cobra.Command {
- var logsDir string
+ logDirs := []string{}
  var outDir string
  cmd := &cobra.Command{
  Use: "process",
@@ -42,8 +42,12 @@ func NewLogsProcessCmd() *cobra.Command {
 
  logVersion()
 
- if logsDir == "" {
- logsDir = app.Config.GetRawLogDir()
+ if len(logDirs) == 0 {
+ logDirs = append(logDirs, app.Config.GetRawLogDir())
+
+ if app.Config.Learner != nil {
+ logDirs = append(logDirs, app.Config.Learner.LogDirs...)
+ }
  }
 
  a, err := analyze.NewAnalyzer()
@@ -52,12 +56,12 @@ func NewLogsProcessCmd() *cobra.Command {
  }
 
  log := zapr.NewLogger(zap.L())
- log.Info("Processing logs", "logs", logsDir)
+ log.Info("Processing logs", "logDirs", logDirs)
 
- if err := a.Analyze(context.Background(), logsDir, app.Config.GetTracesDBDir(), app.Config.GetBlocksDBDir()); err != nil {
+ if err := a.Analyze(context.Background(), logDirs, app.Config.GetTracesDBDir(), app.Config.GetBlocksDBDir()); err != nil {
  return err
  }
- log.Info("Processed logs", "logs", logsDir, "traces", app.Config.GetTracesDBDir(), "blocks", app.Config.GetBlocksDBDir())
+ log.Info("Processed logs", "logs", logDirs, "traces", app.Config.GetTracesDBDir(), "blocks", app.Config.GetBlocksDBDir())
  return nil
  }()
 
@@ -68,7 +72,7 @@ func NewLogsProcessCmd() *cobra.Command {
  },
  }
 
- cmd.Flags().StringVarP(&logsDir, "logs", "", "", "(Optional) Directory containing logs to process")
+ cmd.Flags().StringArrayVarP(&logDirs, "logs", "", []string{}, "(Optional) Directories containing logs to process")
  cmd.Flags().StringVarP(&outDir, "out", "", "", "(Optional) Directory to write the output to")
  return cmd
 }

app/pkg/analyze/analyzer.go

@@ -50,9 +50,12 @@ type ResultFiles struct {
 // logsDir - Is the directory containing the logs
 // tracesDBDir - Is the directory containing the traces pebble database
 // blocksDBDir - Is the directory containing the blocks pebble database
-func (a *Analyzer) Analyze(ctx context.Context, logsDir string, tracesDBDir string, blocksDBDir string) error {
+//
+// TODO(https://github.com/jlewi/foyle/issues/126): I think we need to pass the DBs in because we can't
+// have them be multi process; so we probably want to have a single DB per process.
+func (a *Analyzer) Analyze(ctx context.Context, logDirs []string, tracesDBDir string, blocksDBDir string) error {
  log := logs.FromContext(ctx)
- log.Info("Analyzing logs", "logsDir", logsDir, "tracesDBDir", tracesDBDir, "blocksDBDir", blocksDBDir)
+ log.Info("Analyzing logs", "logDirs", logDirs, "tracesDBDir", tracesDBDir, "blocksDBDir", blocksDBDir)
 
  log.Info("Opening traces database", "database", tracesDBDir)
  tracesDB, err := pebble.Open(tracesDBDir, &pebble.Options{})
@@ -70,9 +73,14 @@ func (a *Analyzer) Analyze(ctx context.Context, logsDir string, tracesDBDir stri
 
  defer helpers.DeferIgnoreError(blocksDB.Close)
 
- jsonFiles, err := findLogFiles(ctx, logsDir)
- if err != nil {
- return err
+ jsonFiles := make([]string, 0, 100)
+ for _, logsDir := range logDirs {
+ newFiles, err := findLogFiles(ctx, logsDir)
+ if err != nil {
+ return err
+ }
+ log.Info("Found logs", "numFiles", len(newFiles), "logsDir", logsDir)
+ jsonFiles = append(jsonFiles, newFiles...)
  }
 
  if err := buildTraces(ctx, jsonFiles, tracesDB, blocksDB); err != nil {
@@ -327,8 +335,8 @@ func buildBlockLog(ctx context.Context, block *logspb.BlockLog, tracesDB *pebble
  return
  }
 
- if _, ok := trace.Data.(*logspb.Trace_Execute); !ok {
- log.Error(errors.New("Invalid ExecuteTrace for traceId"), "Error getting execute trace", "execTraceId", tid)
+ if trace.GetExecute() == nil && trace.GetRunMe() == nil {
+ log.Error(errors.New("Invalid execution trace for traceId"), "Error getting execute trace", "execTraceId", tid)
  return
  }
  if lastTrace == nil {

app/pkg/analyze/analyzer_test.go

@@ -298,7 +298,7 @@ func Test_Analyzer(t *testing.T) {
  tracesDBDir := filepath.Join(oDir, "traces")
  blocksDBDir := filepath.Join(oDir, "blocks")
 
- if err := a.Analyze(context.Background(), testDir, tracesDBDir, blocksDBDir); err != nil {
+ if err := a.Analyze(context.Background(), []string{testDir}, tracesDBDir, blocksDBDir); err != nil {
  t.Fatalf("Analyze failed: %v", err)
  }
  t.Logf("Output written to: %s", oDir)

app/pkg/config/config.go

@@ -55,6 +55,13 @@ type Config struct {
 
  // TODO(jeremy): Should we move this into the experiment?
  Eval *EvalConfig `json:"eval,omitempty" yaml:"eval,omitempty"`
+
+ Learner *LearnerConfig `json:"learner,omitempty" yaml:"learner,omitempty"`
+}
+
+type LearnerConfig struct {
+ // LogDirs is an additional list of directories to search for logs.
+ LogDirs []string `json:"logDirs" yaml:"logDirs"`
 }
 
 type EvalConfig struct {

app/pkg/learn/learner.go

@@ -142,6 +142,8 @@ func (l *Learner) reconcileExamples(ctx context.Context, blocksDB *pebble.DB) er
  continue
  }
 
+ log.Info("Found new training example", "blockId", b.GetId())
+
  // TODO(jeremy): Should we take into account execution status when looking for mistakes?
 
  // Deep copy the original message

app/pkg/runme/proxy.go

@@ -64,5 +64,6 @@ func (p *Proxy) GenerateCells(ctx context.Context, req *aiv1alpha1.GenerateCells
  resp := &aiv1alpha1.GenerateCellsResponse{
  Cells: cells,
  }
+
  return resp, nil
 }

data/eval/runme/logs.md

@@ -0,0 +1,5 @@
+Check the runme logs for an execution for the block 01HYZXS2Q5XYX7P3PT1KH5Q881
+
+```sh {"id":"01HYZY4V5NNQN5Q2RDZ5G15JY8"}
+grep -r 01HYZXS2Q5XYX7P3PT1KH5Q881 "/Users/jlewi/Library/Application Support/runme/logs/"
+```

developer_guides/runme-01HYZVHZ41CAN75B22CRC9D9J6.md

@@ -0,0 +1,80 @@
+---
+runme:
+ document:
+ relativePath: runme.md
+ session:
+ id: 01HYZVHZ41CAN75B22CRC9D9J6
+ updated: 2024-05-28 14:57:56-07:00
+---
+
+# Developing the Runme Extension
+
+* The Runme extension is in the [vscode-runme](ht************************************me) repository
+* The service is defined in [runme/pkg/api/proto/runme/ai](ht**************************************************************ai)
+* Follow [RunMe's vscode contributing.md](ht**************************************************************md)
+* If you need nvm you can brew install it
+
+```sh {"id":"01HY2569DM0SR533BT4ZJTD2WV"}
+brew install nvm
+```
+
+* The command inside Runme's contributing guide assumed vscode's binary was on the path; for me it wasn't so I had to execut
+ the command using the full path.
+
+```sh {"id":"01HY2584G3Q0A89TK1NRWVH0ZN"}
+jq -r ".recommendations[]" .vscode/extensions.json | xargs -n 1 /Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension
+```
+
+## Building and installing the extension from source
+
+* [VSCode Extension Packaging & Publishing](ht**************************************************************************on)
+* It looks like the package has a `bundle` command that will build the extension and package it into a `.vsix` file
+
+```sh {"id":"01HY25HEG7CR7QCGJSERF3BB4K"}
+cd ~/git_vscode-runme
+npm run bundle
+```
+
+```sh {"id":"01HY25KVHCN2P1W9NV0ECD1TW0"}
+ls -la ~/git_vscode-runme/
+```
+
+Now we can install the extension using the vscode binary
+
+* I had to uninstall the RunMe extension first before installing the new one
+* If you search for the extension in the extensions view, you can click on the arrow next to the update button and uncheck auto-update
+ if you don't do that it may continue to auto update
+* The exact file will be named `runme-X.Y.Z.vsix` so it will change as the version changes
+* You can bump the version in `package.json` to something like `"version": "3.*******.0",` and then do the build and install
+ * **Note**: Your version number should be higher than whats in the vscode marketplace otherwise vscode
+ does some odd version magic
+ * The advantage of this is that then you can tell which version of the extension you have installed
+ * It also seemed like when I didn't bump the version I might have actually been using an old version of the extension
+
+```bash {"id":"01HYZVG8KZKYSTFS4R1RJZDS7P"}
+/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension ~/gi************me/ru************ix
+```
+
+```sh {"id":"01HY264KZTS4J9NHJASJT1GYJ7"}
+ls -la ~/git_vscode-runme/dist
+```
+
+So it looks like my runme install is messed up
+Lets try installing and reinstalling it
+
+* Now everything is working; I can generate completions using Foyle
+
+```bash {"id":"01HY74YTEZDZVJYPMB0VMCE84S"}
+/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --uninstall-extension stateful.runme
+
+```
+
+```bash {"id":"01HY75KYKE3SFAM5EXMDAVJDTQ"}
+echo "hello world"
+```
+
+## Debugging the Runme Extension in vscode
+
+* It seems like you may need to run `yarn build` for changes to get picked up; running `F5` doesn't always seem to work
+* Console logs will show up in the `debug console` in the development workspace; not the instance of vscode that gets launched to run
+ your extension

developer_guides/runme-01HZ0KBGBKTSRH95B2E3TWH16S.md

@@ -0,0 +1,80 @@
+---
+runme:
+ document:
+ relativePath: runme.md
+ session:
+ id: 01HZ0KBGBKTSRH95B2E3TWH16S
+ updated: 2024-05-28 14:57:56-07:00
+---
+
+# Developing the Runme Extension
+
+* The Runme extension is in the [vscode-runme](ht************************************me) repository
+* The service is defined in [runme/pkg/api/proto/runme/ai](ht**************************************************************ai)
+* Follow [RunMe's vscode contributing.md](ht**************************************************************md)
+* If you need nvm you can brew install it
+
+```sh {"id":"01HY2569DM0SR533BT4ZJTD2WV"}
+brew install nvm
+```
+
+* The command inside Runme's contributing guide assumed vscode's binary was on the path; for me it wasn't so I had to execut
+ the command using the full path.
+
+```sh {"id":"01HY2584G3Q0A89TK1NRWVH0ZN"}
+jq -r ".recommendations[]" .vscode/extensions.json | xargs -n 1 /Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension
+```
+
+## Building and installing the extension from source
+
+* [VSCode Extension Packaging & Publishing](ht**************************************************************************on)
+* It looks like the package has a `bundle` command that will build the extension and package it into a `.vsix` file
+
+```sh {"id":"01HY25HEG7CR7QCGJSERF3BB4K"}
+cd ~/git_vscode-runme
+npm run bundle
+```
+
+```sh {"id":"01HY25KVHCN2P1W9NV0ECD1TW0"}
+ls -la ~/git_vscode-runme/
+```
+
+Now we can install the extension using the vscode binary
+
+* I had to uninstall the RunMe extension first before installing the new one
+* If you search for the extension in the extensions view, you can click on the arrow next to the update button and uncheck auto-update
+ if you don't do that it may continue to auto update
+* The exact file will be named `runme-X.Y.Z.vsix` so it will change as the version changes
+* You can bump the version in `package.json` to something like `"version": "3.*******.0",` and then do the build and install
+ * **Note**: Your version number should be higher than whats in the vscode marketplace otherwise vscode
+ does some odd version magic
+ * The advantage of this is that then you can tell which version of the extension you have installed
+ * It also seemed like when I didn't bump the version I might have actually been using an old version of the extension
+
+```bash {"id":"01HYZVG8KZKYSTFS4R1RJZDS7P"}
+/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension ~/gi************me/ru************ix
+```
+
+```sh {"id":"01HY264KZTS4J9NHJASJT1GYJ7"}
+ls -la ~/git_vscode-runme/dist
+```
+
+So it looks like my runme install is messed up
+Lets try installing and reinstalling it
+
+* Now everything is working; I can generate completions using Foyle
+
+```bash {"id":"01HY74YTEZDZVJYPMB0VMCE84S"}
+/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --uninstall-extension stateful.runme
+
+```
+
+```bash {"id":"01HY75KYKE3SFAM5EXMDAVJDTQ"}
+echo "hello world"
+```
+
+## Debugging the Runme Extension in vscode
+
+* It seems like you may need to run `yarn build` for changes to get picked up; running `F5` doesn't always seem to work
+* Console logs will show up in the `debug console` in the development workspace; not the instance of vscode that gets launched to run
+ your extension

developer_guides/runme.md

@@ -34,10 +34,16 @@ Now we can install the extension using the vscode binary
 
 * I had to uninstall the RunMe extension first before installing the new one
 * If you search for the extension in the extensions view, you can click on the arrow next to the update button and uncheck auto-update
- if you don't do that it may continue to auto update
-
-```sh {"id":"01HY25NW7H5RRC50HJBJJ0XYDM"}
-/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension ~/git_vscode-runme/runme-3.5.3.vsix
+ if you don't do that it may continue to auto update
+* The exact file will be named `runme-X.Y.Z.vsix` so it will change as the version changes
+* You can bump the version in `package.json` to something like `"version": "3.5.7-dev.0",` and then do the build and install
+ * **Note**: Your version number should be higher than whats in the vscode marketplace otherwise vscode
+ does some odd version magic
+ * The advantage of this is that then you can tell which version of the extension you have installed
+ * It also seemed like when I didn't bump the version I might have actually been using an old version of the extension
+
+```bash {"id":"01HYZVG8KZKYSTFS4R1RJZDS7P"}
+/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension ~/git_vscode-runme/runme-3.5.9.vsix
 ```
 
 ```sh {"id":"01HY264KZTS4J9NHJASJT1GYJ7"}
@@ -62,4 +68,4 @@ echo "hello world"
 
 * It seems like you may need to run `yarn build` for changes to get picked up; running `F5` doesn't always seem to work
 * Console logs will show up in the `debug console` in the development workspace; not the instance of vscode that gets launched to run
- your extension
+  your extension

docs/content/en/docs/learning/_index.md

@@ -17,6 +17,47 @@ foyle config set agent.rag.enabled=true
 foyle config set agent.rag.maxResults=3
 ```
 
+## Enabling Logging In RunMe
+
+If you are using [RunMe.dev](https://runme.dev/) as the frontend for Foyle
+then you need to configure RunMe to enable the AI logging experiment
+
+1. Inside vscode open the settings panel
+2. Enable the option `Runme › Experiments: Ai Logs`
+
+Now that logging is enabled. You can verify that the logs are being written and identify the location
+of the logs.
+
+Inside VSCode open an output window and select the **RunMe** output channel. 
+Scroll to the top of the messages and look for a `Logger initialized` message like the one below
+
+```
+[2024-05-28T22:12:20.681Z] INFO Runme(RunmeServer): {"level":"info","ts":1716934340.6789708,"caller":"cmd/common.go:190","msg":"Logger initialized","devMode":false,"aiLogs":true,"aiLogFile":"/Users/jlewi/Library/Application Support/runme/logs/logs.2024-05-28T15:12:20.json"}
+```
+
+The field `aiLogs` will contain the file that the current instance of RunMe is using for the JSON logs.
+
+By default, on MacOs RunMe will use the directory
+
+```bash
+/Users/${USER}/Library/Application Support/runme/logs/
+```
+
+## Configuring Learning
+
+If you are using [RunMe.dev](https://runme.dev/) as the frontend for Foyle
+then you need to configure Foyle with the location of the **directory** of RunMe's logs.
+Refer to the previous section for instructions on how to locate the file
+where `RunMe` is writing the logs. Then remove the filename to get the directory
+where the logs are being written.
+
+Once you know the directory run the following command to configure Foyle to use that
+directory
+
+```bash
+foyle config set learner.logDirs=${RUNME_LOGS_DIR}
+```
+
 ## Learning from past mistakes
 
 To learn from past mistakes you should periodically run the command