Skip to content

Commit

Permalink
problem: missing events
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsovereignty
gsovereignty committed Apr 19, 2024
1 parent 518e134 commit 6a24891
Show file tree
Hide file tree
Showing 7 changed files with 200 additions and 69 deletions.
73 changes: 73 additions & 0 deletions src/lib/snort_workers/fetch_events.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { NostrSystem, RequestBuilder } from '@snort/system';
import type { NostrEvent } from 'nostr-tools';
import { writable } from 'svelte/store';
import type { Command } from './types';
import { seedRelays } from '@/snort_workers/seed_relays';

let events = new Map<string, NostrEvent>();
let eventStore = writable(events);
let eventIDSet = new Set<string>()

onmessage = (m: MessageEvent<Command>) => {
if (m.data.command == 'fetch_events') {
if (m.data.events && m.data.events.length > 0) start(m.data.events);
eventStore.subscribe((evs) => {
postMessage(evs)
//postMessage(evs.size)
// for (let [id, e] of evs) {
// postMessage(e);
// }
});
}
};
let total = 0
let count = 0
async function start(eventIDs: string[]) {
return new Promise((resolve, reject) => {
const sys = new NostrSystem({
checkSigs: false //todo: check sigs JIT on rendering
// automaticOutboxModel: true,
// buildFollowGraph: true,
});
(async () => {
await sys.Init();
seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false }));

const rb = new RequestBuilder('permasub');

rb.withFilter().ids(eventIDs);
rb.withOptions({ leaveOpen: false });
const q = sys.Query(rb);

q.on('event', (evs): void => {
total += evs.length
count ++
//console.log(count, evs.length, total)
//console.log(42, evs.length)
//let dirty = false;
evs.forEach(e=>{
events.set(e.id, e)
})
// evs.forEach(e=>{
// if (!events.has(e.id)) {
// dirty = true;
// let eNt = getNostrEvent(e);
// events.set(e.id, eNt);
// }
// })
// for (let e of evs) {
// if (!events.has(e.id)) {
// dirty = true;
// let eNt = getNostrEvent(e);
// events.set(e.id, eNt);
// }
// }
// if (true) {
eventStore.update((d) => {
return d;
});
// }
});
})();
});
}
130 changes: 93 additions & 37 deletions src/lib/snort_workers/master_worker.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@

import { NostrSystem, RequestBuilder } from '@snort/system';
import { NostrSystem, RequestBuilder, type QueryLike } from '@snort/system';
import { derived, writable } from 'svelte/store';
import { Command, FrontendData, WorkerData } from './types';
import { followsFromKind3, getNostrEvent, tagSplits } from './utils';
import type { NostrEvent } from '@nostr-dev-kit/ndk';
import WorkerVite from "./live_subs?worker"
import WorkerPubkeys from './live_subs?worker';
import WorkerEvents from './fetch_events?worker';
import { seedRelays } from '@/snort_workers/seed_relays';

let workerData = new WorkerData();
Expand All @@ -14,17 +14,17 @@ workerDataStore.subscribe((data) => {
let fed = new FrontendData();
fed.basePubkey = data.ourPubkey();
fed.baseFollows = data._ourFollows;
let roots:NostrEvent[] = []
let roots: NostrEvent[] = [];
for (let r of data.roots) {
let re = data.events.get(r)
let re = data.events.get(r);
if (!r) {
throw new Error("missing event, this should not happen, bug!")
throw new Error('missing event, this should not happen, bug!');
}
roots.push(re!)
roots.push(re!);
}
fed.roots = roots.toSorted((a,b)=>{
let a_replies = data.replies.get(a.id!)
let b_replies = data.replies.get(b.id!)
fed.roots = roots.toSorted((a, b) => {
let a_replies = data.replies.get(a.id!);
let b_replies = data.replies.get(b.id!);
if (a_replies && b_replies) {
return b_replies.size - a_replies.size;
}
Expand All @@ -35,9 +35,9 @@ workerDataStore.subscribe((data) => {
return -1;
}
return 0;
})
fed.replies = data.replies
fed.rawEvents = data.events
});
fed.replies = data.replies;
fed.rawEvents = data.events;
//console.log(data)
postMessage(fed);
});
Expand All @@ -47,7 +47,7 @@ let lengthOfFollows = derived(workerDataStore, ($wds) => {
});

lengthOfFollows.subscribe((x) => {
console.log("follows updated");
console.log('follows updated');
if (x > 0) {
PermaSub([...workerData._ourFollows]);
}
Expand All @@ -73,7 +73,7 @@ async function start(pubkey?: string, pubkeys?: string[]) {
pubkey = workerData.ourPubkey();
}
const sys = new NostrSystem({
checkSigs: true
checkSigs: false
// automaticOutboxModel: true,
// buildFollowGraph: true,
});
Expand Down Expand Up @@ -151,18 +151,16 @@ async function PermaSub(pubkeys: string[]) {
if (permaSub) {
permaSub.terminate();
}
//const w = await import('./live_subs.ts?worker');
//permaSub = new w.default();
permaSub = new WorkerVite()
permaSub = new WorkerPubkeys();
permaSub.onmessage = (x: MessageEvent<Map<string, NostrEvent>>) => {
workerDataStore.update(current=>{
current.events = new Map([...x.data, ...current.events])
workerDataStore.update((current) => {
current.events = new Map([...x.data, ...current.events]);
//console.log(current.events.size)
let printed = 0
let printedID = new Set<string>()
let printed = 0;
let printedID = new Set<string>();
for (let [id, e] of current.events) {
current.missingEvents.delete(id)
let tagsForEvent = new tagSplits(e)
current.missingEvents.delete(id);
let tagsForEvent = new tagSplits(e);
if (tagsForEvent.unknown.size > 0) {
//tell user that there's an unhandled tag
if (printed < 20 && !printedID.has(tagsForEvent.id)) {
Expand All @@ -171,29 +169,29 @@ async function PermaSub(pubkeys: string[]) {
console.log('unknown tag detected', printed, tagsForEvent.rawEvent);
}
}
tagsForEvent.roots.forEach(r=>{
tagsForEvent.roots.forEach((r) => {
if (!current.events.has(r)) {
current.missingEvents.add(r)
current.missingEvents.add(r);
} else {
current.roots.add(r)
current.roots.add(r);
}
})
});
if (
(tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) ||
tagsForEvent.replies.size > 1
) {
//we don't know which tag is the _real_ reply (parent), let's try and find out
let possibleParents = new Map<string, NostrEvent>();
let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]);
let numMissing = 0
let numMissing = 0;
for (let _id of possibleReplyTags) {
let _event = current.events.get(_id);
if (_event) {
possibleParents.set(_id, _event);
}
if (!_event) {
current.missingEvents.add(_id);
numMissing++
numMissing++;
}
}
if (numMissing == 0 && possibleParents.size > 0) {
Expand Down Expand Up @@ -224,20 +222,78 @@ async function PermaSub(pubkeys: string[]) {
if (tagsForEvent.replies.size == 1) {
let existing = current.replies.get([...tagsForEvent.replies][0]);
if (!existing) {
existing = new Set()
existing = new Set();
}
existing.add(tagsForEvent.id);
current.replies.set([...tagsForEvent.replies][0], existing);
}
}
return current
})

return current;
});
};
let cmd = new Command('sub_to_pubkeys')
cmd.pubkeys = pubkeys
let cmd = new Command('sub_to_pubkeys');
cmd.pubkeys = pubkeys;
permaSub.postMessage(cmd);
}
}

export default{}
let numberOfMissingEvents = derived(workerDataStore, ($wds) => {
return $wds.missingEvents.size;
});

//let fetchEventsWorker: Worker | undefined = undefined;

const missingEventSys = new NostrSystem({
checkSigs: false
// automaticOutboxModel: true,
// buildFollowGraph: true,
});


let q: QueryLike



let fmeStarted = false;
function fmeStart() {
if (!fmeStarted) {
fmeStarted = true;
seedRelays.forEach((r) => missingEventSys.ConnectToRelay(r, { read: true, write: false }));
}
}

numberOfMissingEvents.subscribe((n) => {
if (n > 0) {
fmeStart()
const rb = new RequestBuilder('fetch-missing-events');
rb.withFilter().ids([...workerData.missingEvents])
rb.withOptions({ leaveOpen: false });
if (q) {q.cancel()}
q = missingEventSys.Query(rb);
q.on('event', (evs): void => {
evs.forEach(e=>{
workerData.events.set(e.id, getNostrEvent(e))
})
})

// console.log(248, n)
// if (fetchEventsWorker) {
// fetchEventsWorker.terminate();
// }
// fetchEventsWorker = new WorkerEvents();
// fetchEventsWorker.onmessage = (x: MessageEvent<Map<string, NostrEvent>>) => {
// console.log(253, x.data.size);
// for (let [_, e] of x.data) {
// workerData.events.set(e.id!, e);
// }
// workerDataStore.update((x) => {
// return x;
// });
// };
// let cmd = new Command('sub_to_pubkeys');
// cmd.events = [...workerData.missingEvents];
// fetchEventsWorker.postMessage(cmd);
}
});

export default {};
43 changes: 19 additions & 24 deletions src/lib/snort_workers/seed_relays.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
export let seedRelays = [
// 'wss://purplepag.es',
// 'wss://relay.nostr.band',
// // "wss://nos.lol",
// // "wss://relay.wikifreedia.xyz",
// 'wss://relay.nostrocket.org',
// 'wss://search.nos.today',
// 'wss://relay.damus.io',
// // "wss://relay.nostr.bg",
// 'wss://relay.snort.social',
// // "wss://offchain.pub",
// 'wss://relay.primal.net'
// // "wss://pyramid.fiatjaf.com",
export let seedRelays = Array.from(new Set([
'wss://purplepag.es',
'wss://relay.nostr.band',
// "wss://nos.lol",
// "wss://relay.wikifreedia.xyz",
'wss://relay.nostrocket.org',
'wss://search.nos.today',
'wss://relay.damus.io',
// "wss://relay.nostr.bg",
'wss://relay.snort.social',
// "wss://offchain.pub",
'wss://relay.primal.net',
// "wss://pyramid.fiatjaf.com",
"wss://nostr.wine",
"wss://nostr-01.yakihonne.com",
"wss://nostr-02.dorafactory.org",
Expand Down Expand Up @@ -46,8 +46,6 @@ export let seedRelays = [
"wss://eden.nostr.land",
"wss://nostr-pub.wellorder.net/",
"wss://140.f7z.io",
"wss://filter.nostr.wine/npub1rtlqca8r6auyaw5n5h3l5422dm4sry5dzfee4696fqe8s6qgudks7djtfs?broadcast=true",
"wss://nostr.wine/",
"wss://nos.lol/",
"wss://relay.mostr.pub/",
"wss://relay.snort.social/",
Expand Down Expand Up @@ -112,8 +110,6 @@ export let seedRelays = [
"wss://nostr3.actn.io",
"wss://nostr.noones.com",
"wss://nostr.massmux.com",
"wss://filter.nostr.wine/npub1clk6vc9xhjp8q5cws262wuf2eh4zuvwupft03hy4ttqqnm7e0jrq3upup9?broadcast=true",
"wss://filter.nostr.wine/npub1lelkh3hhxw9hdwlcpk6q9t0xt9f7yze0y0nxazvzqjmre3p98x3sthkvyz?broadcast=true",
"wss://nostr.mutinywallet.com",
"wss://relay.bitcoinpark.com",
"wss://public.relaying.io",
Expand All @@ -123,24 +119,23 @@ export let seedRelays = [
"wss://nostr.fractalized.net",
"wss://welcome.nostr.wine",
"wss://relay.plebstr.com",
"wss://Nostr.wine",
"wss://relay.utxo.one",
"wss://nostr.bitcoiner.social/",
"wss://nostr.huszonegy.world/",
"wss://relay.nostr.bg/",
"wss://relayable.org/",
"wss://bitcoinmaximalists.online"
]
"wss://relayable.org/",]))

export let reducedRelays = [
export let short = [
"wss://relay.nostr.band",
"wss://relay.current.fyi",
'wss://purplepag.es',
'wss://relay.nostr.band',
// "wss://nos.lol",
"wss://nos.lol",
// "wss://relay.wikifreedia.xyz",
'wss://relay.nostrocket.org',
'wss://search.nos.today',
'wss://relay.damus.io',
// "wss://relay.nostr.bg",
"wss://relay.nostr.bg",
'wss://relay.snort.social',
// "wss://offchain.pub",
'wss://relay.primal.net'
Expand Down
5 changes: 3 additions & 2 deletions src/lib/snort_workers/types.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import type { NostrEvent } from 'nostr-tools';

export class Command {
command: 'start' | 'sub_to_pubkeys';
command: 'start' | 'sub_to_pubkeys' | 'fetch_events';
pubkey?: string;
pubkeys?: string[];
constructor(command: 'start' | 'sub_to_pubkeys') {
events?: string[];
constructor(command: 'start' | 'sub_to_pubkeys' | 'fetch_events') {
this.command = command;
}
}
Expand Down
Loading

0 comments on commit 6a24891

Please sign in to comment.