feat(subm): 1 BLD request status google sheet

subm/app.yaml

@@ -31,7 +31,10 @@ includes:
 #   # Discord
 #   DISCORD_API_TOKEN: sekretsekret
 #   DISCORD_GUILD_ID: 585...
-#   # DISCORD_USER_ID: 358096357862408195
+#   # validator-1-bld
+#   CHANNEL_ID: 946...
+#   # mod1bld
+#   REVIEWER_ROLE_ID: 946...
 #   DISCORD_CLIENT_ID: 874...
 #   DISCORD_CLIENT_SECRET: sekretsekret
 

subm/package.json

@@ -9,6 +9,7 @@
     "lint:eslint": "eslint '**/*.js'",
     "lint:types": "tsc -p jsconfig.json",
     "start": "node src/subm.js",
+    "upsert": "node src/request1bld.js",
     "deploy": "gcloud app deploy",
     "test": "echo \"Error: no test specified\" && exit 1"
   },

subm/src/discordGuild.js

@@ -35,11 +35,18 @@ function getContent(host, path, headers, { get }) {
   });
 }
 
+const { keys } = Object;
+const query = opts =>
+  keys(opts).length > 0 ? `?${new URLSearchParams(opts).toString()}` : '';
+
 /**
  * Discord API (a small slice of it, anyway)
  *
  * @param {string} token
- * @param {{ get: typeof import('https').get }} io
+ * @param {{
+ *   get: typeof import('https').get,
+ *   setTimeout: typeof setTimeout,
+ * }} io
  *
  * // https://discordapp.com/developers/docs/resources/user
  * @typedef {{
@@ -76,21 +83,63 @@ function getContent(host, path, headers, { get }) {
  * }} UserObject
  * @typedef { string } Snowflake 64 bit numeral
  * @typedef { string } TimeStamp ISO8601 format
+ *
+ * https://discord.com/developers/docs/resources/channel#message-object
+ * @typedef {{
+ *   id: Snowflake,
+ *   author: UserObject,
+ *   content: string,
+ *   timestamp: TimeStamp
+ * }} MessageObject
  */
-function DiscordAPI(token, { get }) {
+function DiscordAPI(token, { get, setTimeout }) {
   // cribbed from rchain-dbr/o2r/gateway/server/main.js
   const host = 'discordapp.com';
   const api = '/api/v6';
   const headers = { Authorization: `Bot ${token}` };
 
+  /**
+   * @param {string} path
+   * @returns {Promise<any>}
+   */
   const getJSON = async path => {
+    console.log('getJSON', { path });
     const body = await getContent(host, path, headers, { get });
     const data = JSON.parse(body);
     // console.log('Discord done:', Object.keys(data));
+    if ('retry_after' in data) {
+      await new Promise(r => setTimeout(r, data.retry_after));
+      return getJSON(path);
+    }
     return data;
   };
 
   return freeze({
+    /** @param { Snowflake } channelID */
+    channels: channelID => {
+      return freeze({
+        /**
+         * @param {Record<string,unknown>} opts
+         * @returns {Promise<MessageObject[]>}
+         */
+        getMessages: (opts = {}) =>
+          getJSON(`${api}/channels/${channelID}/messages${query(opts)}`),
+        /** @param { Snowflake } messageID */
+        messages: messageID =>
+          freeze({
+            /**
+             * @param {string} emoji
+             * @returns {Promise<UserObject[]>}
+             */
+            reactions: emoji =>
+              getJSON(
+                `${api}/channels/${channelID}/messages/${messageID}/reactions/${encodeURIComponent(
+                  emoji,
+                )}`,
+              ),
+          }),
+      });
+    },
     /**
      * @param { string } userID
      * @returns { Promise<UserObject> }
@@ -120,8 +169,7 @@ function DiscordAPI(token, { get }) {
           const opts = after
             ? { limit: `${limit}`, after }
             : { limit: `${limit}` };
-          const query = new URLSearchParams(opts).toString();
-          return getJSON(`${api}/guilds/${guildID}/members?${query}`);
+          return getJSON(`${api}/guilds/${guildID}/members${query(opts)}`);
         },
       });
     },
@@ -136,6 +184,27 @@ function avatar(user) {
   return `${avatarBase}/${user.id}/${user.avatar}.png`;
 }
 
+/**
+ * @param {(opts: any) => Promise<T[]>} fn
+ * @param {number} [limit]
+ * @template {{ id: string }} T
+ */
+async function paged(fn, limit = 100) {
+  /** @type {T[][]} */
+  const pages = [];
+  /** @type { string | undefined } */
+  let before;
+  do {
+    console.error('getting page', pages.length, before);
+    // eslint-disable-next-line no-await-in-loop
+    const page = await fn(before ? { limit, before } : { limit });
+    if (!page.length) break;
+    before = page.slice(-1)[0].id;
+    pages.push(page);
+  } while (before);
+  return pages.flat();
+}
+
 /**
  * @param {ReturnType<ReturnType<DiscordAPI>['guilds']>} guild
  */
@@ -163,11 +232,12 @@ async function pagedMembers(guild) {
  * @param {{
  *   get: typeof import('https').get,
  *   stdout: typeof import('process').stdout
+ *   setTimeout: typeof setTimeout,
  * }} io
  */
-async function main(env, { stdout, get }) {
+async function integrationTest(env, { stdout, get, setTimeout }) {
   const config = makeConfig(env);
-  const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get });
+  const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get, setTimeout });
   const guild = discordAPI.guilds(config`DISCORD_GUILD_ID`);
 
   const roles = await guild.roles();
@@ -179,12 +249,15 @@ async function main(env, { stdout, get }) {
 
 /* global require, process */
 if (require.main === module) {
-  main(process.env, {
+  integrationTest(process.env, {
     stdout: process.stdout,
     // eslint-disable-next-line global-require
     get: require('https').get,
+    // @ts-ignore
+    // eslint-disable-next-line no-undef
+    setTimeout,
   }).catch(err => console.error(err));
 }
 
 /* global module */
-module.exports = { DiscordAPI, avatar };
+module.exports = { DiscordAPI, avatar, getContent, paged };

subm/src/request1bld.js

@@ -0,0 +1,220 @@
+/* eslint-disable no-await-in-loop */
+// See https://github.com/Agoric/validator-profiles/wiki/Request-1-BLD
+
+const { DiscordAPI, getContent, paged } = require('./discordGuild.js');
+const { upsert } = require('./sheetAccess');
+const { searchBySender, transfers } = require('./tendermintRPC');
+
+const config = {
+  host: 'rpc-agoric.nodes.guru',
+  address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h',
+  quorum: 2,
+};
+
+const fail = () => {
+  throw Error();
+};
+
+/**
+ * @param {ReturnType<ReturnType<typeof DiscordAPI>['channels']>} channel
+ * @param {ReturnType<ReturnType<typeof DiscordAPI>['guilds']>} guild
+ * @param {Snowflake} role
+ * @param {number} quorum
+ * @yields {{ message: Message, endorsers: User[] }}
+ * @typedef {import('./discordGuild').Snowflake} Snowflake
+ */
+async function* authorizedRequests(channel, guild, role, quorum) {
+  /** @type {Map<Snowflake, import('./discordGuild').GuildMember>} */
+  const memberDetail = new Map();
+  /** @param {Snowflake} id */
+  const getMemberDetail = async id => {
+    if (memberDetail.has(id)) {
+      return memberDetail.get(id) || fail();
+    }
+    const detail = await guild.members(id);
+    // console.log(detail);
+    memberDetail.set(id, detail);
+    return detail;
+  };
+
+  const messages = await paged(channel.getMessages);
+  const hasAddr = messages.filter(msg => msg.content.match(/agoric1/));
+  if (!hasAddr) return;
+  const hasChecks = hasAddr.filter(msg => {
+    const [checks] = (msg.reactions || []).filter(r => r.emoji.name === '✅');
+    return (checks || {}).count >= quorum;
+  });
+  if (!hasChecks) return;
+
+  for (const msg of hasChecks) {
+    const endorsements = await channel.messages(msg.id).reactions('✅');
+    const endorsers = [];
+    for (const endorsement of endorsements) {
+      const detail = await getMemberDetail(endorsement.id);
+      if (detail && detail.roles && detail.roles.includes(role)) {
+        endorsers.push(detail);
+      }
+    }
+    if (endorsers.length >= quorum) {
+      const [_, address] = msg.content.match(/(agoric1\S+)/);
+      if (typeof address !== 'string') throw TypeError(address);
+      yield { message: msg, address, endorsers };
+    }
+  }
+}
+
+/**
+ * @param {ReturnType<ReturnType<typeof DiscordAPI>['channels']>} channel
+ * @param {ReturnType<ReturnType<typeof DiscordAPI>['guilds']>} guild
+ * @param {string} roleID
+ * @param {{
+ *   get: typeof import('https').get,
+ * }} io
+ */
+async function requestStatus(channel, guild, roleID, { get }) {
+  const txs = await getContent(
+    config.host,
+    searchBySender(config.address),
+    {},
+    { get },
+  ).then(txt => JSON.parse(txt).result.txs);
+
+  const txfrs = transfers(txs);
+  // console.log(txfrs);
+  const byRecipient = new Map(txfrs.map(txfr => [txfr.recipient, txfr]));
+  // console.log(byRecipient.keys());
+
+  const result = [];
+  const eachRequest = authorizedRequests(channel, guild, roleID, config.quorum);
+  for await (const { message, address, endorsers } of eachRequest) {
+    const hash = byRecipient.has(address)
+      ? byRecipient.get(address).hash
+      : undefined;
+    result.push({
+      message,
+      address,
+      endorsers,
+      hash,
+    });
+  }
+  return result;
+}
+
+const label = user => `${user.username}#${user.discriminator}`;
+
+/**
+ * @param {Record<string, string | undefined>} env
+ * @param {{
+ *   get: typeof import('https').get,
+ *   setTimeout: typeof setTimeout,
+ * }} io
+ */
+async function main(env, { get, setTimeout }) {
+  const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout });
+  const guild = discordAPI.guilds(env.DISCORD_GUILD_ID);
+
+  // to get mod-1-bld role id:
+  // console.log(await guild.roles());
+
+  const channel = discordAPI.channels(env.CHANNEL_ID);
+
+  const txs = await getContent(
+    config.host,
+    searchBySender(config.address),
+    {},
+    { get },
+  ).then(txt => JSON.parse(txt).result.txs);
+
+  const txfrs = transfers(txs);
+  // console.log(txfrs);
+  const byRecipient = new Map(txfrs.map(txfr => [txfr.recipient, txfr]));
+  // console.log(byRecipient.keys());
+
+  const header = [
+    'timestamp',
+    'msgID',
+    'requestor',
+    'address',
+    'endorsers',
+    'hash',
+  ];
+  console.log(header.join(','));
+  for await (const { message: msg, address, endorsers } of authorizedRequests(
+    channel,
+    guild,
+    env.REVIEWER_ROLE_ID,
+    2,
+  )) {
+    const ok = endorsers.map(u => label(u.user)).join(' ');
+    const hash = byRecipient.has(address) ? byRecipient.get(address).hash : '';
+    console.log(
+      `${msg.timestamp},${msg.id},${label(
+        msg.author,
+      )},${address},${ok},${hash}`,
+    );
+  }
+}
+
+/**
+ * @param {string[]} args
+ * @param {Record<string, string | undefined>} env
+ * @param {Object} io
+ * @param {typeof import('google-spreadsheet').GoogleSpreadsheet} io.GoogleSpreadsheet
+ * @param {typeof import('https').get} io.get
+ * @param {typeof setTimeout} io.setTimeout
+ */
+const main2 = async (args, env, { get, setTimeout, GoogleSpreadsheet }) => {
+  const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout });
+  const guild = discordAPI.guilds(env.DISCORD_GUILD_ID);
+
+  // to get mod-1-bld role id:
+  // console.log(await guild.roles());
+
+  const channel = discordAPI.channels(env.CHANNEL_ID);
+
+  const creds = {
+    client_email: env.GOOGLE_SERVICES_EMAIL,
+    private_key: env.GCS_PRIVATE_KEY,
+  };
+  // Initialize the sheet - doc ID is the long id in the sheets URL
+  const doc = new GoogleSpreadsheet(env.SHEET_1BLD_ID);
+  // Initialize Auth - see https://theoephraim.github.io/node-google-spreadsheet/#/getting-started/authentication
+  await doc.useServiceAccountAuth(creds);
+  await doc.loadInfo(); // loads document properties and worksheets
+  console.log(doc.title);
+  const sheet = doc.sheetsByIndex[0];
+
+  const memberLabel = mem => mem.nick || label(mem.user);
+  const eachStatus = await requestStatus(channel, guild, env.REVIEWER_ROLE_ID, {
+    get,
+  });
+  for (const { message: msg, address, endorsers, hash } of eachStatus) {
+    const record = {
+      To: address,
+      Link: `https://discord.com/channels/${env.DISCORD_GUILD_ID}/${env.CHANNEL_ID}/${msg.id}`,
+      At: msg.timestamp.slice(0, '1999-01-01T12:59'.length).replace('T', ' '),
+      By: label(msg.author),
+      Reviewers: endorsers.map(memberLabel).join(','),
+      Tx: hash && `https://agoric.bigdipper.live/transactions/${hash}`,
+      Request: msg.content,
+    };
+    await upsert(sheet, address, record); // do not interleave upserts!
+  }
+};
+
+/* global require, process, module */
+if (require.main === module) {
+  main2(
+    process.argv.slice(2),
+    { ...process.env },
+    {
+      // eslint-disable-next-line global-require
+      get: require('https').get,
+      setTimeout,
+      // eslint-disable-next-line global-require
+      GoogleSpreadsheet: require('google-spreadsheet').GoogleSpreadsheet, // please excuse CJS
+    },
+  ).catch(err => console.error(err));
+}
+
+module.exports = { requestStatus };