Fix issues with incorrect serialization of LocalStorageDB items (#5969)

## Changes

To avoid further doubts I adjusted behaviour and interface of
`LocalStorageDB`to be more in sunc with `vscode.Memento`, that is:
* setting key to `null` or `undefined` removes key from the store
* get on non-existing key returns an `undefined`

Note that it means setting key to `null` and then doing get on it will
result in `undefined` value, not a `null`.

I also added exception handling in the `LocalStorageDB::get` to make
sure we won't fail on previously incorrectly serialised values.

## Test plan

Unit tests which should cover all corner cases were added.

agent/src/global-state/AgentGlobalState.test.ts

@@ -1,6 +1,7 @@
+import os from 'node:os'
 import { CodyIDE } from '@sourcegraph/cody-shared'
-import { beforeEach, describe, expect, it } from 'vitest'
-import { AgentGlobalState } from './AgentGlobalState'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { AgentGlobalState, LocalStorageDB } from './AgentGlobalState'
 
 describe('AgentGlobalState', () => {
     let globalState: AgentGlobalState
@@ -38,3 +39,77 @@ describe('AgentGlobalState', () => {
         expect(globalState.get('nonExistentKey', 'defaultValue')).toBe('defaultValue')
     })
 })
+
+describe('LocalStorageDB', () => {
+    let localStorageDB: LocalStorageDB
+
+    beforeEach(() => {
+        localStorageDB = new LocalStorageDB('testIDE', os.tmpdir())
+    })
+
+    afterEach(() => {
+        localStorageDB.clear()
+    })
+
+    it('should set and get values correctly', () => {
+        localStorageDB.set('testKey', 'testValue')
+        expect(localStorageDB.get('testKey')).toBe('testValue')
+    })
+
+    it('should handle complex objects', () => {
+        const complexObject = { a: 1, b: { c: 'test' }, d: [1, 2, 3] }
+        localStorageDB.set('complexKey', complexObject)
+        expect(localStorageDB.get('complexKey')).toEqual(complexObject)
+    })
+
+    it('should return undefined for non-existent keys', () => {
+        expect(localStorageDB.get('nonExistentKey')).toBeUndefined()
+    })
+
+    it('should overwrite existing values', () => {
+        localStorageDB.set('overwriteKey', 'initialValue')
+        localStorageDB.set('overwriteKey', 'newValue')
+        expect(localStorageDB.get('overwriteKey')).toBe('newValue')
+    })
+
+    it('should clear all stored values', () => {
+        localStorageDB.set('key1', 'value1')
+        localStorageDB.set('key2', 'value2')
+        localStorageDB.clear()
+        expect(localStorageDB.get('key1')).toBeUndefined()
+        expect(localStorageDB.get('key2')).toBeUndefined()
+    })
+
+    it('should return all keys', () => {
+        localStorageDB.set('key1', 'value1')
+        localStorageDB.set('key2', 'value2')
+        const keys = localStorageDB.keys()
+        expect(keys).toContain('key1')
+        expect(keys).toContain('key2')
+        expect(keys.length).toBe(2)
+    })
+
+    it('should handle different data types', () => {
+        localStorageDB.set('numberKey', 42)
+        localStorageDB.set('booleanKey', true)
+        localStorageDB.set('nullKey', null)
+        expect(localStorageDB.get('numberKey')).toBe(42)
+        expect(localStorageDB.get('booleanKey')).toBe(true)
+        expect(localStorageDB.get('nullKey')).toBeUndefined()
+    })
+
+    it('should threat setting null values as removing the key', () => {
+        localStorageDB.set('nullKey', null)
+        expect(localStorageDB.get('nullKey')).toBeUndefined()
+    })
+
+    it('should threat setting undefined values as removing the key', () => {
+        localStorageDB.set('undefinedKey', undefined)
+        expect(localStorageDB.get('undefinedKey')).toBeUndefined()
+    })
+
+    it('should handle empty string value', () => {
+        localStorageDB.set('emptyStringKey', '')
+        expect(localStorageDB.get('emptyStringKey')).toBe('')
+    })
+})

agent/src/global-state/AgentGlobalState.ts

@@ -58,17 +58,17 @@ export class AgentGlobalState implements vscode.Memento {
         return [localStorage.LAST_USED_ENDPOINT, localStorage.ANONYMOUS_USER_ID_KEY, ...this.db.keys()]
     }
 
-    public get<T>(key: string, defaultValue?: unknown): any {
+    public get<T>(key: string, defaultValue?: T): T {
         if (this.manager === 'server') {
             return this.db.get(key) ?? defaultValue
         }
         switch (key) {
             case localStorage.LAST_USED_ENDPOINT:
-                return vscode_shim.extensionConfiguration?.serverEndpoint
+                return vscode_shim.extensionConfiguration?.serverEndpoint as T
             case localStorage.ANONYMOUS_USER_ID_KEY:
                 // biome-ignore lint/suspicious/noFallthroughSwitchClause: This is intentional
                 if (vscode_shim.extensionConfiguration?.anonymousUserID) {
-                    return vscode_shim.extensionConfiguration?.anonymousUserID
+                    return vscode_shim.extensionConfiguration?.anonymousUserID as T
                 }
             default:
                 return this.db.get(key) ?? defaultValue
@@ -86,7 +86,7 @@ export class AgentGlobalState implements vscode.Memento {
 }
 
 interface DB {
-    get(key: string): any
+    get(key: string): any | undefined
     set(key: string, value: any): void
     keys(): readonly string[]
     clear(): void
@@ -95,7 +95,7 @@ interface DB {
 class InMemoryDB implements DB {
     private store = new Map<string, any>()
 
-    get(key: string): any {
+    get(key: string): any | undefined {
         return this.store.get(key)
     }
 
@@ -112,7 +112,7 @@ class InMemoryDB implements DB {
     }
 }
 
-class LocalStorageDB implements DB {
+export class LocalStorageDB implements DB {
     storage: LocalStorage
 
     constructor(ide: string, dir: string) {
@@ -123,12 +123,20 @@ class LocalStorageDB implements DB {
         this.storage.clear()
     }
 
-    get(key: string): any {
+    get(key: string): any | undefined {
         const item = this.storage.getItem(key)
-        return item ? JSON.parse(item) : undefined
+        try {
+            return item ? JSON.parse(item) : undefined
+        } catch (error) {
+            return undefined
+        }
     }
     set(key: string, value: any): void {
-        this.storage.setItem(key, JSON.stringify(value))
+        if (value !== null && value !== undefined) {
+            this.storage.setItem(key, JSON.stringify(value))
+        } else {
+            this.storage.removeItem(key)
+        }
     }
     keys(): readonly string[] {
         const keys = []