Create data feed update loop (#37)

* Create update data feed loop

* Small implementation changes, add tests

* Fix lint

* Fix tests and update configs

* Add documentation

* Update test configuration

* Make deviationThresholdCoefficient optional

* Rename startUpdateFeedsLoops

* Fix tests, remove unnecessary flag

.github/workflows/main.yml

@@ -41,8 +41,8 @@ jobs:
       - name: Lint
         run: pnpm run prettier:check && pnpm run eslint:check
       - name: Test
-        run: pnpm run test
-  
+        run: pnpm run test --bail
+
   test-e2e:
     name: Test e2e
     runs-on: ubuntu-latest
@@ -62,7 +62,7 @@ jobs:
       - name: Start Hardhat
         run: pnpm dev:eth-node&
       - name: Test E2E
-        run: pnpm test:e2e
+        run: pnpm test:e2e --bail
 
   required-checks-passed:
     name: All required checks passed

README.md

@@ -191,14 +191,24 @@ The number of minutes used to calculate the scaling multiplier if a pending tran
 
 The maximum scaling multiplier used when the pending transaction lag exceeds the `scalingWindow`.
 
-#### `deviationThresholdCoefficient`
+#### `deviationThresholdCoefficient` _(optional)_
 
 The global coefficient applied to all deviation checks. Used to differentiate alternate deployments. For example:
 
 ```jsonc
 "deviationThresholdCoefficient": 1,
 ```
 
+Defaults to `1`.
+
+#### `dataFeedUpdateInterval`
+
+The interval specifying how often to run the data feed update loop. In seconds.
+
+#### `dataFeedBatchSize`
+
+The batch size of active dAPIs that are to be fetched in a single RPC call.
+
 ## Docker
 
 ### Build

config/airseeker.example.json

@@ -30,7 +30,9 @@
         "sanitizationPercentile": 80,
         "scalingWindow": 2,
         "maxScalingMultiplier": 2
-      }
+      },
+      "dataFeedUpdateInterval": 60,
+      "dataFeedBatchSize": 10
     }
   },
   "deviationThresholdCoefficient": 1,

jest-e2e.config.js

@@ -1,7 +1,19 @@
-const config = require('./jest.config');
+const { join } = require('node:path');
 
+/**
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/configuration
+ * @type {import('jest').Config}
+ */
 module.exports = {
-  ...config,
-  displayName: 'e2e',
+  collectCoverage: false, // It doesn't make sense to collect coverage for e2e tests because they target high level features and interaction with other services.
+  maxWorkers: 1, // We don't want to run tests in parallel because they might interfere with each other. This option is the same as --runInBand. See: https://stackoverflow.com/a/46489246.
+
+  preset: 'ts-jest',
+  restoreMocks: true,
+  setupFiles: [join(__dirname, './jest.setup.js')],
+  testEnvironment: 'jest-environment-node',
   testMatch: ['**/?(*.)+(feature).[t]s?(x)'],
+  testPathIgnorePatterns: ['<rootDir>/.build', '<rootDir>/dist/', '<rootDir>/build/'],
+  verbose: true,
 };

jest-unit.config.js

@@ -1,7 +1,21 @@
-const config = require('./jest.config');
+const { join } = require('node:path');
 
+/**
+ * For a detailed explanation regarding each configuration property and type check, visit:
+ * https://jestjs.io/docs/configuration
+ * @type {import('jest').Config}
+ */
 module.exports = {
-  ...config,
-  displayName: 'unit',
+  collectCoverage: true,
+  coverageDirectory: 'coverage',
+  coveragePathIgnorePatterns: ['node_modules', '<rootDir>/typechain-types'], // Coverage is collected for all files imported by the tests. We want to exclude files generated by Typechain.
+  coverageProvider: 'v8',
+
+  preset: 'ts-jest',
+  restoreMocks: true,
+  setupFiles: [join(__dirname, './jest.setup.js')],
+  testEnvironment: 'jest-environment-node',
   testMatch: ['**/?(*.)+(spec|test).[t]s?(x)'],
+  testPathIgnorePatterns: ['<rootDir>/.build', '<rootDir>/dist/', '<rootDir>/build/'],
+  verbose: true,
 };

package.json

@@ -19,8 +19,8 @@
     "eslint:fix": "pnpm run eslint:check --fix",
     "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"",
     "prettier:fix": "prettier --write \"./**/*.{js,ts,md,json}\"",
-    "test": "jest --selectProjects unit --verbose --runInBand --bail --detectOpenHandles --silent",
-    "test:e2e": "jest --selectProjects e2e --runInBand",
+    "test": "jest --config=jest-unit.config.js",
+    "test:e2e": "jest ---config=jest-e2e.config.js",
     "tsc": "tsc --project .",
     "docker:build": "docker build -t api3/airseekerv2:latest -f docker/Dockerfile .",
     "docker:run": "docker run -it --rm api3/airseekerv2:latest",

src/config/schema.test.ts

@@ -59,6 +59,8 @@ describe('chains schema', () => {
           activeDapiNames: [],
         },
         gasSettings,
+        dataFeedBatchSize: 10,
+        dataFeedUpdateInterval: 60,
       },
     };
 
@@ -83,6 +85,8 @@ describe('chains schema', () => {
           activeDapiNames: [],
         },
         gasSettings,
+        dataFeedBatchSize: 10,
+        dataFeedUpdateInterval: 60,
       },
     };
 
@@ -107,6 +111,8 @@ describe('chains schema', () => {
           activeDapiNames: [],
         },
         gasSettings,
+        dataFeedBatchSize: 10,
+        dataFeedUpdateInterval: 60,
       },
     };
 
@@ -138,6 +144,8 @@ describe('chains schema', () => {
           activeDapiNames: [],
         },
         gasSettings,
+        dataFeedBatchSize: 10,
+        dataFeedUpdateInterval: 60,
       },
     };
 
@@ -180,6 +188,8 @@ describe('chains schema', () => {
           activeDapiNames: [],
         },
         gasSettings,
+        dataFeedBatchSize: 10,
+        dataFeedUpdateInterval: 60,
       },
     };
 

src/config/schema.ts

@@ -66,6 +66,8 @@ export const optionalChainSchema = z
     __Temporary__DapiDataRegistry: temporaryDapiDataRegistrySchema,
     contracts: optionalContractsSchema.optional(),
     gasSettings: gasSettingsSchema,
+    dataFeedUpdateInterval: z.number().positive(),
+    dataFeedBatchSize: z.number().positive(),
   })
   .strict();
 
@@ -133,8 +135,8 @@ export const configSchema = z
   .object({
     sponsorWalletMnemonic: z.string().refine((mnemonic) => ethers.utils.isValidMnemonic(mnemonic), 'Invalid mnemonic'),
     chains: chainsSchema,
-    deviationThresholdCoefficient: z.number().optional().default(1),
-    fetchInterval: z.number().positive(),
+    fetchInterval: z.number().positive(), // TODO: Rename to signedDataFetchInterval
+    deviationThresholdCoefficient: z.number().positive().optional().default(1), // Explicitly agreed to make this optional. See: https://github.com/api3dao/airseeker-v2/pull/20#issuecomment-1750856113.
   })
   .strict();
 

src/signed-data-store/signed-data-store.test.ts

@@ -1,10 +1,10 @@
 import { BigNumber, ethers } from 'ethers';
 
-import { generateRandomBytes32, signData } from '../../test/utils/evm';
+import { generateRandomBytes32, signData } from '../../test/utils';
 import type { SignedData } from '../types';
 
-import * as localDataStore from './signed-data-store';
 import { verifySignedDataIntegrity } from './signed-data-store';
+import * as localDataStore from './signed-data-store';
 
 describe('datastore', () => {
   let testDataPoint: SignedData;

src/update-feeds/index.ts

@@ -0,0 +1 @@
+export * from './update-feeds';

src/update-feeds/temporary-contract-mock.ts

@@ -0,0 +1,18 @@
+// NOTE: The function is currently returning static data, because the contract is not yet finalized, but we mark it as
+// async in advance.
+//
+// eslint-disable-next-line @typescript-eslint/require-await
+export const getStaticActiveDapis = async (_offset: number, _limit: number) => {
+  return {
+    totalCount: 1,
+    dapiNames: ['MOCK_FEED'],
+    dataFeedIds: ['0xebba8507d616ed80766292d200a3598fdba656d9938cecc392765d4a284a69a4'],
+    updateParameters: [{ deviationThresholdInPercentage: 0.5, deviationReference: 0.5, heartbeatInterval: 100 }],
+    // NOTE: We will need to decode this from the contract, because it will store the template IDs as encoded bytes.
+    dataFeedTemplateIds: [['0xcc35bd1800c06c12856a87311dd95bfcbb3add875844021d59a929d79f3c99bd']],
+    signedApiUrls: [['http://localhost:8080']],
+    airnodeAddresses: ['0xbF3137b0a7574563a23a8fC8badC6537F98197CC'],
+  };
+};
+
+export type ActiveDapisBatch = Awaited<ReturnType<typeof getStaticActiveDapis>>;