From 1ffff4d7ede0b497ecba63452139b8df9ea1cb31 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Fri, 8 Nov 2024 15:49:48 +0100 Subject: [PATCH 01/18] chore: removed ping in favor of keepalive --- README.md | 16 +-- lib/client.js | 2 +- lib/db.js | 39 +------- lib/db_ping.js | 106 -------------------- test/db.clustermode.tests.js | 4 +- test/db.standalonemode.tests.js | 172 +------------------------------- test/db.tests.js | 73 +++++++++++++- 7 files changed, 81 insertions(+), 331 deletions(-) delete mode 100644 lib/db_ping.js diff --git a/README.md b/README.md index dc5e939..cc8e47d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ It's a fork from [LimitDB](https://github.com/limitd/limitdb). - [Configure](#configure) - [Options available](#options-available) - [Buckets](#buckets) - - [Ping](#ping) - [Overrides](#overrides) - [ERL (Elevated Rate Limits)](#erl-elevated-rate-limits) - [Prerequisites](#prerequisites) @@ -55,11 +54,6 @@ const limitd = new Limitd({ } }, prefix: 'test:', - ping: { - interval: 1000, - maxFailedAttempts: 5, - reconnectIfFailed: true - }, username: 'username', password: 'password' }); @@ -71,9 +65,9 @@ const limitd = new Limitd({ - `nodes` (array): [Redis Cluster Configuration](https://github.com/luin/ioredis#cluster). - `buckets` (object): Setup your bucket types. - `prefix` (string): Prefix keys in Redis. -- `ping` (object): Configure ping to Redis DB. - `username` (string): Redis username. This is ignored if not in Cluster mode. Needs Redis >= v6. - `password` (string): Redis password. +- `keepAlive` (number): TCP KeepAlive on the socket expressed in milliseconds. Set to a non-number value to disable keepAlive ### Buckets: @@ -91,14 +85,6 @@ If you omit `size`, limitdb assumes that `size` is the value of `per_interval`. If you don't specify a filling rate with `per_interval` or any other `per_x`, the bucket is fixed and you have to manually reset it using `PUT`. -### Ping: - -- `interval` (number): represents the time between two consecutive pings. Default: 3000. -- `maxFailedAttempts` (number): is the allowed number of failed pings before declaring the connection as dead. Default: 5. -- `reconnectIfFailed` (boolean): indicates whether we should try to reconnect is the connection is declared dead. Default: true. - - - ## Overrides You can also define `overrides` inside your type definitions as follows: diff --git a/lib/client.js b/lib/client.js index 0fe058e..14cb573 100644 --- a/lib/client.js +++ b/lib/client.js @@ -31,7 +31,7 @@ class LimitdRedis extends EventEmitter { this.db = new LimitDBRedis(_.pick(params, [ 'uri', 'nodes', 'buckets', 'prefix', 'slotsRefreshTimeout', 'slotsRefreshInterval', - 'username', 'password', 'tls', 'dnsLookup', 'globalTTL', 'cacheSize', 'ping'])); + 'username', 'password', 'tls', 'dnsLookup', 'globalTTL', 'cacheSize'])); this.db.on('error', (err) => { this.emit('error', err); diff --git a/lib/db.js b/lib/db.js index e594798..fdcd689 100644 --- a/lib/db.js +++ b/lib/db.js @@ -6,7 +6,6 @@ const LRU = require('lru-cache'); const utils = require('./utils'); const Redis = require('ioredis'); const { validateParams, validateERLParams } = require('./validation'); -const DBPing = require("./db_ping"); const { calculateQuotaExpiration, resolveElevatedParams, isFixedWindowEnabled, removeHashtag } = require('./utils'); const EventEmitter = require('events').EventEmitter; @@ -14,27 +13,10 @@ const TAKE_LUA = fs.readFileSync(`${__dirname}/take.lua`, "utf8"); const TAKE_ELEVATED_LUA = fs.readFileSync(`${__dirname}/take_elevated.lua`, "utf8"); const PUT_LUA = fs.readFileSync(`${__dirname}/put.lua`, "utf8"); -const PING_SUCCESS = "successful"; -const PING_ERROR = "error"; -const PING_RECONNECT = "reconnect"; -const PING_RECONNECT_DRY_RUN = "reconnect-dry-run"; - const DEFAULT_COMMAND_TIMEOUT = 125; // Milliseconds +const DEFAULT_KEEPALIVE = 10000; // Milliseconds class LimitDBRedis extends EventEmitter { - static get PING_SUCCESS() { - return PING_SUCCESS; - } - static get PING_ERROR() { - return PING_ERROR; - } - static get PING_RECONNECT() { - return PING_RECONNECT; - } - static get PING_RECONNECT_DRY_RUN() { - return PING_RECONNECT_DRY_RUN; - } - /** * Creates an instance of LimitDB client for Redis. * @param {params} params - The configuration for the database and client. @@ -63,6 +45,7 @@ class LimitDBRedis extends EventEmitter { keyPrefix: config.prefix, password: config.password, tls: config.tls, + keepAlive: config.keepAlive || DEFAULT_KEEPALIVE, reconnectOnError: (err) => { // will force a reconnect when error starts with `READONLY` // this code is only triggered when auto-failover is disabled @@ -88,7 +71,6 @@ class LimitDBRedis extends EventEmitter { this.redis = new Redis.Cluster(config.nodes, clusterOptions); } else { this.redis = new Redis(config.uri, redisOptions); - this.#setupPing(config); } this.redis.defineCommand('take', { @@ -120,24 +102,7 @@ class LimitDBRedis extends EventEmitter { } - #setupPing(config) { - this.redis.on("ready", () => this.#startPing(config)); - this.redis.on("close", () => this.#stopPing()); - } - - #startPing(config) { - this.#stopPing(); - this.ping = new DBPing(config.ping, this.redis); - this.ping.on("ping", (data) => this.emit("ping", data)); - } - - #stopPing() { - this.ping?.stop(); - this.ping?.removeAllListeners(); - } - close(callback) { - this.#stopPing(); this.redis.quit(callback); } diff --git a/lib/db_ping.js b/lib/db_ping.js deleted file mode 100644 index f564d61..0000000 --- a/lib/db_ping.js +++ /dev/null @@ -1,106 +0,0 @@ -const cbControl = require('./cb'); -const utils = require("./utils"); -const EventEmitter = require("events").EventEmitter; - -const PING_SUCCESS = "successful"; -const PING_ERROR = "error"; -const PING_RECONNECT = "reconnect"; -const PING_RECONNECT_DRY_RUN = "reconnect-dry-run"; - -const DEFAULT_PING_INTERVAL = 3000; // Milliseconds - -class DBPing extends EventEmitter { - constructor(config, redis) { - super(); - - this.redis = redis; - this.config = { - commandTimeout: 125, - enabled: config ? true : false, - interval: config?.interval || DEFAULT_PING_INTERVAL, - maxFailedAttempts: config?.maxFailedAttempts || 5, - reconnectIfFailed: - utils.functionOrFalse(config?.reconnectIfFailed) || (() => false), - }; - - this.failedPings = 0; - - this.#start(); - } - - #start() { - const doPing = () => { - if (!this.config.enabled) { - return; - } - - let start = Date.now(); - this.redis.ping(cbControl((err) => { - let duration = Date.now() - start; - err - ? this.#pingKO(triggerLoop, err, duration) - : this.#pingOK(triggerLoop, duration); - }).timeout(this.config.commandTimeout)); - }; - - const triggerLoop = () => setTimeout(doPing, this.config.interval); - - doPing(); - } - - stop() { - this.enabled = false; - } - - #pingOK(callback, duration) { - this.reconnecting = false; - this.failedPings = 0; - this.#emitPingResult(PING_SUCCESS, undefined, duration, 0); - callback(); - } - - #pingKO(callback, err, duration) { - this.failedPings++; - this.#emitPingResult(PING_ERROR, err, duration, this.failedPings); - - if (this.failedPings < this.config.maxFailedAttempts) { - return callback(); - } - - if (!this.config.reconnectIfFailed()) { - return this.#emitPingResult( - PING_RECONNECT_DRY_RUN, - undefined, - 0, - this.failedPings - ); - } - - this.#retryStrategy(() => { - this.#emitPingResult(PING_RECONNECT, undefined, 0, this.failedPings); - this.redis.disconnect(true); - }); - } - - #emitPingResult(status, err, duration, failedPings) { - const result = { - status: status, - duration: duration, - error: err, - failedPings: failedPings, - }; - this.emit("ping", result); - } - - #retryStrategy(callback) { - //jitter between 0% and 10% of the total wait time needed to reconnect - //i.e. if interval = 100 and maxFailedAttempts = 3 => it'll randomly jitter between 0 and 30 ms - const deviation = - utils.randomBetween(0, 0.1) * - this.config.interval * - this.config.maxFailedAttempts; - setTimeout(callback, deviation); - } -} - -module.exports = DBPing; diff --git a/test/db.clustermode.tests.js b/test/db.clustermode.tests.js index 1dd614a..56eb4a4 100644 --- a/test/db.clustermode.tests.js +++ b/test/db.clustermode.tests.js @@ -12,7 +12,7 @@ describe('when using LimitDB', () => { return new LimitDB({ nodes: clusterNodes, buckets: {}, prefix: 'tests:', ..._.omit(params, ['uri']) }); }; - dbTests(clientCreator); + dbTests(clientCreator, { clusterNodes }); describe('when using the clustered #constructor', () => { it('should allow setting username and password', (done) => { @@ -28,4 +28,4 @@ describe('when using LimitDB', () => { }); }); }) -}); \ No newline at end of file +}); diff --git a/test/db.standalonemode.tests.js b/test/db.standalonemode.tests.js index fb8784e..68db900 100644 --- a/test/db.standalonemode.tests.js +++ b/test/db.standalonemode.tests.js @@ -10,10 +10,10 @@ const crypto = require('crypto'); describe('when using LimitDB', () => { describe('in standalone mode', () => { const clientCreator = (params) => { - return new LimitDB({ uri: 'localhost', buckets: {}, prefix: 'tests:', ..._.omit(params, ['nodes']) }); + return new LimitDB({ uri: 'localhost:6379', buckets: {}, prefix: 'tests:', ..._.omit(params, ['nodes']) }); }; - dbTests(clientCreator); + dbTests(clientCreator, { uri: 'localhost:6379' } ); describe('when using the standalone #constructor', () => { it('should emit error on failure to connect to redis', (done) => { @@ -27,171 +27,5 @@ describe('when using LimitDB', () => { }); }); }); - - describe('LimitDBRedis Ping', () => { - - let ping = { - enabled: () => true, - interval: 10, - maxFailedAttempts: 3, - reconnectIfFailed: () => true, - maxFailedAttemptsToRetryReconnect: 10 - }; - - let config = { - uri: 'localhost:22222', - buckets, - prefix: 'tests:', - ping, - }; - - let redisProxy; - let toxiproxy; - let db; - - beforeEach((done) => { - toxiproxy = new Toxiproxy('http://localhost:8474'); - proxyBody = { - listen: '0.0.0.0:22222', - name: crypto.randomUUID(), //randomize name to avoid concurrency issues - upstream: 'redis:6379' - }; - toxiproxy.createProxy(proxyBody) - .then((proxy) => { - redisProxy = proxy; - done(); - }); - - }); - - afterEach((done) => { - redisProxy.remove().then(() => - db.close((err) => { - // Can't close DB if it was never open - if (err?.message.indexOf('enableOfflineQueue') > 0 || err?.message.indexOf('Connection is closed') >= 0) { - err = undefined; - } - done(err); - }) - ); - }); - - it('should emit ping success', (done) => { - db = createDB({ uri: 'localhost:22222', buckets, prefix: 'tests:', ping }, done); - db.once(('ping'), (result) => { - if (result.status === LimitDB.PING_SUCCESS) { - done(); - } - }); - }); - - it('should emit "ping - error" when redis stops responding pings', (done) => { - let called = false; - - db = createDB(config, done); - db.once(('ready'), () => addLatencyToxic(redisProxy, 20000, noop)); - db.on(('ping'), (result) => { - if (result.status === LimitDB.PING_ERROR && !called) { - called = true; - db.removeAllListeners('ping'); - done(); - } - }); - }); - - it('should emit "ping - reconnect" when redis stops responding pings and client is configured to reconnect', (done) => { - let called = false; - db = createDB(config, done); - db.once(('ready'), () => addLatencyToxic(redisProxy, 20000, noop)); - db.on(('ping'), (result) => { - if (result.status === LimitDB.PING_RECONNECT && !called) { - called = true; - db.removeAllListeners('ping'); - done(); - } - }); - }); - - it('should emit "ping - reconnect dry run" when redis stops responding pings and client is NOT configured to reconnect', (done) => { - let called = false; - db = createDB({ ...config, ping: { ...ping, reconnectIfFailed: () => false } }, done); - db.once(('ready'), () => addLatencyToxic(redisProxy, 20000, noop)); - db.on(('ping'), (result) => { - if (result.status === LimitDB.PING_RECONNECT_DRY_RUN && !called) { - called = true; - db.removeAllListeners('ping'); - done(); - } - }); - }); - - it(`should NOT emit ping events when config.ping is not set`, (done) => { - db = createDB({ ...config, ping: undefined }, done); - - db.once(('ping'), (result) => { - done(new Error(`unexpected ping event emitted ${result}`)); - }); - - //If after 100ms there are no interactions, we mark the test as passed. - setTimeout(done, 100); - }); - - it('should recover from a connection loss', (done) => { - let pingResponded = false; - let reconnected = false; - let toxic = undefined; - let timeoutId; - db = createDB({ ...config, ping: { ...ping, interval: 50 } }, done); - - db.on(('ping'), (result) => { - if (result.status === LimitDB.PING_SUCCESS) { - if (!pingResponded) { - pingResponded = true; - toxic = addLatencyToxic(redisProxy, 20000, (t) => toxic = t); - } else if (reconnected) { - clearTimeout(timeoutId); - db.removeAllListeners('ping'); - done(); - } - } else if (result.status === LimitDB.PING_RECONNECT) { - if (pingResponded && !reconnected) { - reconnected = true; - toxic.remove(); - } - } - }); - - timeoutId = setTimeout(() => done(new Error('Not reconnected')), 1800); - }); - - const createDB = (config, done) => { - let tmpDB = new LimitDB(config); - - tmpDB.on(('error'), (err) => { - //As we actively close the connection, there might be network-related errors while attempting to reconnect - if (err?.message.indexOf('enableOfflineQueue') > 0 || err?.message.indexOf('Command timed out') >= 0) { - err = undefined; - } - - if (err) { - done(err); - } - }); - - return tmpDB; - }; - - const addLatencyToxic = (proxy, latency, callback) => { - let toxic = new Toxic( - proxy, - { type: 'latency', attributes: { latency: latency } } - ); - proxy.addToxic(toxic).then(callback); - }; - - - const noop = () => { - }; - }); }) -}); \ No newline at end of file +}); diff --git a/test/db.tests.js b/test/db.tests.js index 163aaea..228de83 100644 --- a/test/db.tests.js +++ b/test/db.tests.js @@ -5,6 +5,7 @@ const _ = require('lodash'); const assert = require('chai').assert; const { endOfMonthTimestamp, replicateHashtag } = require('../lib/utils'); const sinon = require('sinon'); +const { exec } = require('child_process'); const buckets = { ip: { @@ -120,7 +121,7 @@ const elevatedBuckets = { module.exports.buckets = buckets; module.exports.elevatedBuckets = elevatedBuckets; -module.exports.tests = (clientCreator) => { +module.exports.tests = (clientCreator, opts) => { describe('LimitDBRedis', () => { let db; const prefix = 'tests:' @@ -143,6 +144,76 @@ module.exports.tests = (clientCreator) => { }); }); + describe('KeepAlive', () => { + const checkSocketOption = (portPairs, option, expectedValue, delta, done) => { + const pid = process.pid; + const command = `lsof -a -p ${pid} -i 4 -T f`; + + exec(command, (error, stdout, stderr) => { + if (error) { + return done(error); + } + if (stderr) { + return done(new Error(stderr)); + } + + const keepAliveOption = stdout + .split('\n') + .find(line => + portPairs.some(portPair => + line.includes(`:${portPair.localPort}`) + && line.includes(`:${portPair.remotePort}`) + ) + ); + assert.isNotNull(keepAliveOption, `no entry found for port ${portPairs}: ${stdout}`); + assert.notEqual(keepAliveOption, undefined, `no entry found for port ${portPairs}: ${stdout}`); + assert.include(keepAliveOption, option, `${option} option not found: ${keepAliveOption}`); + + const keepAliveValue = parseInt(keepAliveOption.match(new RegExp(`${option}=(\\d+)`))[1], 10); + assert.isAtLeast(keepAliveValue, expectedValue - delta, `${option} is lesser than expected`); + assert.isAtMost(keepAliveValue, expectedValue + delta, `${option} is greater than expected`); + + done(); + }); + }; + + it('should set SO=KEEPALIVE to 10000 by default', (done) => { + const ports = []; + if (opts.clusterNodes) { + Object.values(db.redis.connectionPool.nodes.all).forEach(node => { + ports.push({ localPort: node.stream.localPort, remotePort: node.stream.remotePort }); + }); + } else { + ports.push({ localPort: db.redis.stream.localPort, remotePort: db.redis.stream.remotePort }); + } + checkSocketOption(ports, 'SO=KEEPALIVE', 10000, 10, done); + }); + + describe('when setting keepAlive option', () => { + const keepAliveValue = 5000; + beforeEach((done) => { + db.close(() => { + db = clientCreator({ buckets, prefix: prefix, keepAlive: keepAliveValue }); + db.once('ready', () => done()); + }); + }); + + it('should set SO=KEEPALIVE to the value specified in the constructor config', (done) => { + const ports = []; + if (opts.clusterNodes) { + Object.values(db.redis.connectionPool.nodes.all) + .filter(node => node.status === 'ready') + .forEach(node => { + ports.push({ localPort: node.stream.localPort, remotePort: node.stream.remotePort }); + }); + } else { + ports.push({ localPort: db.redis.stream.localPort, remotePort: db.redis.stream.remotePort }); + } + checkSocketOption(ports, 'SO=KEEPALIVE', keepAliveValue, 10, done); + }); + }); + }); + describe('#constructor', () => { it('should throw an when missing redis information', () => { assert.throws(() => clientCreator({ From b14048c512937e5774f179ef51c68ce3f2505e85 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Mon, 11 Nov 2024 12:21:01 +0100 Subject: [PATCH 02/18] bumped minor version --- .github/workflows/ci.yml | 2 ++ package.json | 3 +- test/db.tests.js | 68 +++++++++++++++------------------------- test/testutils.js | 54 +++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 44 deletions(-) create mode 100644 test/testutils.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16da035..c2e71b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,8 @@ on: jobs: test: + env: + CI: true timeout-minutes: 10 runs-on: ubuntu-latest strategy: diff --git a/package.json b/package.json index 8ede626..2f09ae5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "limitd-redis", - "version": "8.3.1", + "version": "8.4.0", "description": "A database client for limits on top of redis", "main": "index.js", "repository": { @@ -30,6 +30,7 @@ "mockdate": "^3.0.5", "nyc": "^14.1.1", "sinon": "^19.0.2", + "sockopt": "^2.0.1", "toxiproxy-node-client": "^2.0.6" } } diff --git a/test/db.tests.js b/test/db.tests.js index 228de83..1bd5fa4 100644 --- a/test/db.tests.js +++ b/test/db.tests.js @@ -6,6 +6,7 @@ const assert = require('chai').assert; const { endOfMonthTimestamp, replicateHashtag } = require('../lib/utils'); const sinon = require('sinon'); const { exec } = require('child_process'); +const { getSockOptValue } = require('./testutils'); const buckets = { ip: { @@ -145,48 +146,28 @@ module.exports.tests = (clientCreator, opts) => { }); describe('KeepAlive', () => { - const checkSocketOption = (portPairs, option, expectedValue, delta, done) => { - const pid = process.pid; - const command = `lsof -a -p ${pid} -i 4 -T f`; - - exec(command, (error, stdout, stderr) => { - if (error) { - return done(error); - } - if (stderr) { - return done(new Error(stderr)); - } - - const keepAliveOption = stdout - .split('\n') - .find(line => - portPairs.some(portPair => - line.includes(`:${portPair.localPort}`) - && line.includes(`:${portPair.remotePort}`) - ) - ); - assert.isNotNull(keepAliveOption, `no entry found for port ${portPairs}: ${stdout}`); - assert.notEqual(keepAliveOption, undefined, `no entry found for port ${portPairs}: ${stdout}`); - assert.include(keepAliveOption, option, `${option} option not found: ${keepAliveOption}`); - - const keepAliveValue = parseInt(keepAliveOption.match(new RegExp(`${option}=(\\d+)`))[1], 10); - assert.isAtLeast(keepAliveValue, expectedValue - delta, `${option} is lesser than expected`); - assert.isAtMost(keepAliveValue, expectedValue + delta, `${option} is greater than expected`); - - done(); - }); + const assertSockOpt = (expected, delta, done) => (err, value) => { + if (err) { + done(err); + } + assert.isAtLeast(value, expected - delta, 'SO=KEEPALIVE is lesser than expected'); + assert.isAtMost(value, expected + delta, 'SO=KEEPALIVE is greater than expected'); + done(); }; + it('should set SO=KEEPALIVE to 10000 by default', (done) => { - const ports = []; + let socket; if (opts.clusterNodes) { - Object.values(db.redis.connectionPool.nodes.all).forEach(node => { - ports.push({ localPort: node.stream.localPort, remotePort: node.stream.remotePort }); - }); + const node = Object.values(db.redis.connectionPool.nodes.all).find(node => node.status === 'ready'); + if (!node) { + return done(new Error('No ready node found')); + } + socket = node.stream; } else { - ports.push({ localPort: db.redis.stream.localPort, remotePort: db.redis.stream.remotePort }); + socket = db.redis.stream; } - checkSocketOption(ports, 'SO=KEEPALIVE', 10000, 10, done); + getSockOptValue(socket, 'SO=KEEPALIVE', assertSockOpt(10000, 10, done) ); }); describe('when setting keepAlive option', () => { @@ -200,16 +181,17 @@ module.exports.tests = (clientCreator, opts) => { it('should set SO=KEEPALIVE to the value specified in the constructor config', (done) => { const ports = []; + let socket; if (opts.clusterNodes) { - Object.values(db.redis.connectionPool.nodes.all) - .filter(node => node.status === 'ready') - .forEach(node => { - ports.push({ localPort: node.stream.localPort, remotePort: node.stream.remotePort }); - }); + const node = Object.values(db.redis.connectionPool.nodes.all).find(node => node.status === 'ready'); + if (!node) { + return done(new Error('No ready node found')); + } + socket = node.stream; } else { - ports.push({ localPort: db.redis.stream.localPort, remotePort: db.redis.stream.remotePort }); + socket = db.redis.stream; } - checkSocketOption(ports, 'SO=KEEPALIVE', keepAliveValue, 10, done); + getSockOptValue(socket, 'SO=KEEPALIVE', assertSockOpt(5000, 10, done) ); }); }); }); diff --git a/test/testutils.js b/test/testutils.js new file mode 100644 index 0000000..af0b3c6 --- /dev/null +++ b/test/testutils.js @@ -0,0 +1,54 @@ +const { exec } = require('child_process'); +const { getsockopt } = require('sockopt'); + +const getSockOptValue = (socket, opt, cb) => { + if (process.env.CI !== undefined) { + return linuxGetSockOptValue(socket, opt, cb); + } + + return macosGetSockOptValue(socket, opt, cb); +}; + +const macosGetSockOptValue = (socket, opt, cb) => { + const pid = process.pid; + exec(`lsof -a -p ${pid} -i 4 -T f`, (error, stdout, stderr) => { + if (error) { + return cb(error); + } + if (stderr) { + return cb(new Error(stderr)); + } + + const keepAliveOption = stdout + .split('\n') + .find(line => line.includes(`:${socket.localPort}`) && line.includes(`:${socket.remotePort}`)); + + if (!keepAliveOption) { + cb(new Error(`no entry found for local port ${socket.localPort}, and remote port ${socket.remotePort}`)); + } + + if (!keepAliveOption.includes(opt)) { + cb(new Error(`${opt} option not found: ${keepAliveOption}`)); + } + + const keepAliveValue = parseInt(keepAliveOption.match(new RegExp(`${opt}=(\\d+)`))[1], 10); + cb(null, keepAliveValue); + }); +}; + +const linuxGetSockOptValue = (socket, opt, cb) => { + const SOL_SOCKET = 0xffff; + const SOCK_OPTS = { + 'SO=KEEPALIVE': 0x0008, + }; + if (!SOCK_OPTS[opt]) { + return cb(new Error(`Unknown socket option: ${opt}`)); + } + + const keepAliveValue = getsockopt(socket, SOL_SOCKET, SOCK_OPTS[opt]); + cb(null, keepAliveValue); +}; + +module.exports = { + getSockOptValue +}; From 8f724a62502d5f687bee700423fbbee239af001a Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Tue, 12 Nov 2024 12:50:52 +0100 Subject: [PATCH 03/18] added tests --- package.json | 1 + test/db.tests.js | 78 ++++++++----------- test/getsockopt_cli | Bin 0 -> 33808 bytes test/getsockopt_cli.c | 47 +++++++++++ test/testutils.js | 176 ++++++++++++++++++++++++++++++++++-------- 5 files changed, 224 insertions(+), 78 deletions(-) create mode 100755 test/getsockopt_cli create mode 100644 test/getsockopt_cli.c diff --git a/package.json b/package.json index 2f09ae5..fb961a6 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "eslint": "^6.1.0", "mocha": "^5.2.0", "mockdate": "^3.0.5", + "net-keepalive": "^4.0.10", "nyc": "^14.1.1", "sinon": "^19.0.2", "sockopt": "^2.0.1", diff --git a/test/db.tests.js b/test/db.tests.js index 1bd5fa4..175014b 100644 --- a/test/db.tests.js +++ b/test/db.tests.js @@ -6,7 +6,7 @@ const assert = require('chai').assert; const { endOfMonthTimestamp, replicateHashtag } = require('../lib/utils'); const sinon = require('sinon'); const { exec } = require('child_process'); -const { getSockOptValue } = require('./testutils'); +const { getSockOptValue, assertIsNear, dropPackets } = require('./testutils'); const buckets = { ip: { @@ -146,53 +146,43 @@ module.exports.tests = (clientCreator, opts) => { }); describe('KeepAlive', () => { - const assertSockOpt = (expected, delta, done) => (err, value) => { - if (err) { - done(err); - } - assert.isAtLeast(value, expected - delta, 'SO=KEEPALIVE is lesser than expected'); - assert.isAtMost(value, expected + delta, 'SO=KEEPALIVE is greater than expected'); - done(); - }; - - - it('should set SO=KEEPALIVE to 10000 by default', (done) => { - let socket; - if (opts.clusterNodes) { - const node = Object.values(db.redis.connectionPool.nodes.all).find(node => node.status === 'ready'); - if (!node) { - return done(new Error('No ready node found')); - } - socket = node.stream; - } else { - socket = db.redis.stream; - } - getSockOptValue(socket, 'SO=KEEPALIVE', assertSockOpt(10000, 10, done) ); - }); + describe('when keepalive is not specified and the socket stops responding', () => { + it('should try to reconnect after 10 seconds', (done) => { + let dropTime, reconnectTime; + db.redis.on('connect', () => { + if (reconnectTime) { + done(); + } + }); - describe('when setting keepAlive option', () => { - const keepAliveValue = 5000; - beforeEach((done) => { - db.close(() => { - db = clientCreator({ buckets, prefix: prefix, keepAlive: keepAliveValue }); - db.once('ready', () => done()); + db.redis.on('reconnecting', () => { + reconnectTime = Date.now(); + assertIsNear(reconnectTime - dropTime, 10000, 100); }); - }); - it('should set SO=KEEPALIVE to the value specified in the constructor config', (done) => { - const ports = []; - let socket; - if (opts.clusterNodes) { - const node = Object.values(db.redis.connectionPool.nodes.all).find(node => node.status === 'ready'); - if (!node) { - return done(new Error('No ready node found')); + dropPackets(db.redis.options.host); + dropTime = Date.now(); + }).timeout(15000); + }); + + describe('when keepalive is specific and the socket stops responding', () => { + it('should try to reconnect after the specified time', (done) => { + db = clientCreator({ buckets, prefix: prefix, keepalive: 5000 }); + let dropTime, reconnectTime; + db.redis.on('connect', () => { + if (reconnectTime) { + done(); } - socket = node.stream; - } else { - socket = db.redis.stream; - } - getSockOptValue(socket, 'SO=KEEPALIVE', assertSockOpt(5000, 10, done) ); - }); + }); + + db.redis.on('reconnecting', () => { + reconnectTime = Date.now(); + assertIsNear(reconnectTime - dropTime, 10000, 100); + }); + + dropPackets(db.redis.options.host); + dropTime = Date.now(); + }).timeout(10000); }); }); diff --git a/test/getsockopt_cli b/test/getsockopt_cli new file mode 100755 index 0000000000000000000000000000000000000000..ead056fb52c81208cc7e0c40e016ca76420b8510 GIT binary patch literal 33808 zcmeI5e{2**6vt=pTKa=RD^$Q1=wgDY#1v2jVt;t09MDkP($-oDGPJ#GxzOIVcU$FW zJX;Zk#BhqFDoJY~*dmg~kWh?BlNv)J2~-m>#t{5Nb3{!D!K6p1_4{_W+};7fsDGLF zlDF^2%)XiZ%-o+dcj4CWf2Ikc2#5{(5L8VNVz*G?3GoPY4OGg?;?>Ue&Z-K^#)|%O zZ0fFv>b!}LQr0-@YGRx9_Z_i0VaAC_lVVD`NU6EEYGwcvUoTi-e9Ub67zo9tYa|jx z&~S{WlmV@yA#f)ZU+++nQE!vkgJNsFWV0R^>LO)hK=XUN&3EG1@4ao-`^0Rg*ecfA zl$Wx@`-;cwk{)l9Pmg2k^*w9Mug)w&aeNK4NjJ0(-B2l47S|NZl8W+bV>KM+CV^4(caO;VZ3ifViKI>i2ydb&&o>6*E(00PVA9j+`JHzWSAR* z=A-W)JuAc#EIG}OR{0#vQ)YY)@gl^;W)@VuCd6czQP?(E@++ZMp}wRk)?}PNfcNP) z6E9PdP@%Lh7Ip;u3tK!53tih%Bp2UDb!eYkg~nYDr8fOLT*deqDbcxLKGoX-n!7b`MIPqm zX)xY?42J5D295Q3+>D7A$s8!Pw{;59YfO^TEKGn2FaajO1egF5U;<2l2`~XBzyz28 z6JP>NfC(@GCcp%k025#WOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz28 z6JP>NfC(@GCcp%k!2gKAc$YFVwj;5BtV9XnJm^4jq9YQXE+S`#a|V06at1%L=ZyAs z$-(3l(H@@Jlhvn)WjJ=28J_Nl2*nZUv&+$B*lTtX%2h-JZTsh4&kyYq;(#iGRUNR@ zN1!gw&mQO!*ld8ej4wovMHTahr7hcdbHnntjQu$f_56K7FHi? za@<(o7lc-!jmE;!;RF5nhOj;5*u?i^YwQTshNIT8>fdB|g3+QX;r!6r#WW}=Qe3Rnld%G}{g*KY+G}!*G zd(5|>C}h+LR*mg&_G3?khV1eZ&Y2Em*s#vF9K%nw_){(ZG>bpO;%8a>xfcIX!+$2e1e{hl_>=PybXrQVn8TRbk6TGVI@bx#%LK25Dx1HQ&BZq3N7 zx~rQjRa94$$?A%dwayxJ(qp;S>8va+E3I`JnP#^}ReWt`nd*vY4;2&Wp!wlV{S5GI z^LKBxbheky{yJbu>3lkgLw`mebbdGuRWkHSdfq%iO@2nuILez5qNfC(@GCcp%k025#W zOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#W zOn?b60Vco%m;e*_#{^WoSqa?=rT;oN?gD@Vv0M^h{BOFEHhc$e18|rL7ED6TX>w?p@u{iT#{zu)J_KgY{}=7QfQWWDC|2zU3To)H_ii?3) zUlY2If$n0Udl={r23j{<6CtRb?rxxUyI&tP2^HM$#v^q+|Mk+)coTB#O4<+ze2930 zX(n=*wz5aqFl(Fs3C(CH6eRex&NkJC>nr}2*l5W}l<1C;jV&JWT;9diyKX#v@#lwo zj(zjnsXady9X&ej#G(9~o30NX`p)_Or>`FWtj_oKwWgtj%OfLOY!!8h<2z68+`jvV zGyB%O^~A>yYX2gm;rN;Mbtf0B=nlSqGymMdy*G}`uKlt$d+EhL&PVdk=AE$n#%h<% zTK?4wj_QQkUpgjr`?^~P_FM=Do02LrPd%vqzisbz9CmJ8n%(zO?v-^5%9~`t%EedC Ue~^9f*LQBe{YJy-eLA$i0InNo-2eap literal 0 HcmV?d00001 diff --git a/test/getsockopt_cli.c b/test/getsockopt_cli.c new file mode 100644 index 0000000..b265708 --- /dev/null +++ b/test/getsockopt_cli.c @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + if (argc != 2) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(EXIT_FAILURE); + } + + int sockfd = atoi(argv[1]); + int optval; + socklen_t optlen = sizeof(optval); + + // Check if the file descriptor is valid + if (fcntl(sockfd, F_GETFD) == -1) { + fprintf(stderr, "Invalid file descriptor: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + // Check if the file descriptor is a socket + int type; + socklen_t length = sizeof(type); + if (getsockopt(sockfd, SOL_SOCKET, SO_TYPE, &type, &length) == -1) { + fprintf(stderr, "Not a socket: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + printf("File descriptor: %d\n", sockfd); + printf("SOL_SOCKET: %d\n", SOL_SOCKET); + printf("SO_KEEPALIVE: %d\n", SO_KEEPALIVE); + + // Get the value of SO_KEEPALIVE + if (getsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen) < 0) { + fprintf(stderr, "getsockopt: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + printf("SO_KEEPALIVE value: %d\n", optval); + return 0; +} diff --git a/test/testutils.js b/test/testutils.js index af0b3c6..cd490a4 100644 --- a/test/testutils.js +++ b/test/testutils.js @@ -1,54 +1,162 @@ const { exec } = require('child_process'); -const { getsockopt } = require('sockopt'); +const os = require('os'); +const assert = require('chai').assert; -const getSockOptValue = (socket, opt, cb) => { - if (process.env.CI !== undefined) { - return linuxGetSockOptValue(socket, opt, cb); - } - - return macosGetSockOptValue(socket, opt, cb); +// Linux-specific functions +const dropPacketsLinux = (remoteIp) => { + exec(`sudo iptables -A OUTPUT -d ${remoteIp} -j DROP`, (error, stdout, stderr) => { + if (error) { + console.error(`Error dropping packets: ${error.message}`); + return; + } + if (stderr) { + console.error(`stderr: ${stderr}`); + return; + } + console.log(`Packets to ${remoteIp} dropped`); + }); }; -const macosGetSockOptValue = (socket, opt, cb) => { - const pid = process.pid; - exec(`lsof -a -p ${pid} -i 4 -T f`, (error, stdout, stderr) => { +const restorePacketsLinux = (remoteIp) => { + exec(`sudo iptables -D OUTPUT -d ${remoteIp} -j DROP`, (error, stdout, stderr) => { if (error) { - return cb(error); + console.error(`Error restoring packets: ${error.message}`); + return; } if (stderr) { - return cb(new Error(stderr)); + console.error(`stderr: ${stderr}`); + return; } + console.log(`Packets to ${remoteIp} restored`); + }); +}; - const keepAliveOption = stdout - .split('\n') - .find(line => line.includes(`:${socket.localPort}`) && line.includes(`:${socket.remotePort}`)); - - if (!keepAliveOption) { - cb(new Error(`no entry found for local port ${socket.localPort}, and remote port ${socket.remotePort}`)); +// macOS-specific functions +const dropPacketsMacOS = (remoteIp) => { + const pfRule = `block drop out from any to ${remoteIp}`; + exec(`echo "${pfRule}" | sudo pfctl -ef -`, (error, stdout, stderr) => { + if (error) { + console.error(`Error dropping packets: ${error.message}`); + return; } - - if (!keepAliveOption.includes(opt)) { - cb(new Error(`${opt} option not found: ${keepAliveOption}`)); + if (stderr) { + console.error(`stderr: ${stderr}`); + return; } + console.log(`Packets to ${remoteIp} dropped`); + }); +}; - const keepAliveValue = parseInt(keepAliveOption.match(new RegExp(`${opt}=(\\d+)`))[1], 10); - cb(null, keepAliveValue); +const restorePacketsMacOS = (remoteIp) => { + exec(`sudo pfctl -F all -f /etc/pf.conf`, (error, stdout, stderr) => { + if (error) { + console.error(`Error restoring packets: ${error.message}`); + return; + } + if (stderr) { + console.error(`stderr: ${stderr}`); + return; + } + console.log(`Packets to ${remoteIp} restored`); }); }; -const linuxGetSockOptValue = (socket, opt, cb) => { - const SOL_SOCKET = 0xffff; - const SOCK_OPTS = { - 'SO=KEEPALIVE': 0x0008, - }; - if (!SOCK_OPTS[opt]) { - return cb(new Error(`Unknown socket option: ${opt}`)); +// Main functions +const dropPackets = (remoteIp) => { + if (os.platform() === 'linux') { + dropPacketsLinux(remoteIp); + } else if (os.platform() === 'darwin') { + dropPacketsMacOS(remoteIp); + } else { + console.error('Unsupported OS'); } +}; - const keepAliveValue = getsockopt(socket, SOL_SOCKET, SOCK_OPTS[opt]); - cb(null, keepAliveValue); +const restorePackets = (remoteIp) => { + if (os.platform() === 'linux') { + restorePacketsLinux(remoteIp); + } else if (os.platform() === 'darwin') { + restorePacketsMacOS(remoteIp); + } else { + console.error('Unsupported OS'); + } }; +const assertIsNear = (actual, expected, delta) => { + assert.isAtLeast(actual, expected - delta, `Expected ${actual} to be at least ${expected - delta}`); + assert.isAtMost(actual, expected + delta, `Expected ${actual} to be at most ${expected + delta}`); +} + module.exports = { - getSockOptValue -}; + dropPackets, + restorePackets, + assertIsNear +} + +// // Example usage +// const remoteRedisIp = '192.168.1.100'; +// dropPackets(remoteRedisIp); +// +// // Wait for 5 seconds before restoring packet flow +// setTimeout(() => { +// restorePackets(remoteRedisIp); +// }, 5000); + + + +// const getSockOptValue = (socket, opt, cb) => { +// // if (process.env.CI !== undefined) { +// // return linuxGetSockOptValue(socket, opt, cb); +// // } +// // +// // return macosGetSockOptValue(socket, opt, cb); +// cb(NetKeepAlive.getKeepAliveInterval(socket)); +// }; +// +// const macosGetSockOptValue = (socket, opt, cb) => { +// const pid = process.pid; +// exec(`lsof -a -p ${pid} -i 4 -T f`, (error, stdout, stderr) => { +// if (error) { +// return cb(error); +// } +// if (stderr) { +// return cb(new Error(stderr)); +// } +// +// const keepAliveOption = stdout +// .split('\n') +// .find(line => line.includes(`:${socket.localPort}`) && line.includes(`:${socket.remotePort}`)); +// +// if (!keepAliveOption) { +// cb(new Error(`no entry found for local port ${socket.localPort}, and remote port ${socket.remotePort}`)); +// } +// +// if (!keepAliveOption.includes(opt)) { +// cb(new Error(`${opt} option not found: ${keepAliveOption}`)); +// } +// +// const keepAliveValue = parseInt(keepAliveOption.match(new RegExp(`${opt}=(\\d+)`))[1], 10); +// cb(null, keepAliveValue); +// }); +// }; +// +// const linuxGetSockOptValue = (socket, opt, cb) => { +// const SOL_SOCKET = 0xffff; +// const SOCK_OPTS = { +// 'SO=KEEPALIVE': 0x0008, +// }; +// if (!SOCK_OPTS[opt]) { +// return cb(new Error(`Unknown socket option: ${opt}`)); +// } +// +// const keepAliveValue = getsockopt(socket, SOL_SOCKET, SOCK_OPTS[opt]); +// cb(null, keepAliveValue); +// }; +// +// module.exports = { +// getSockOptValue +// }; +// +// +// +// const { exec } = require('child_process'); From 7452c84ef941eaa3029172cb131cac50a6917ba2 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 09:32:50 +0100 Subject: [PATCH 04/18] wip --- .github/workflows/ci.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2e71b3..ab424ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,12 +16,33 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [14.x, 16.x, 18.x, 20.x] + node-version: [18.x] + kernel: [6.5.0-1025-azure] steps: - name: Checkout uses: actions/checkout@v4 + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential linux-headers-$(uname -r) wget unzip libelf-dev + + - name: Download kernel sources + run: wget https://cl.archive.ubuntu.com/ubuntu/pool/main/l/linux-azure-6.5/linux-azure-6.5_6.5.0.orig.tar.gz | tar xvz + + - name: Configure kernel sources + run: | + cp /boot/config-$(uname -r) linux-6.5/.config + yes "" | make -C linux-6.5 oldconfig + make -C linux-6.5 modules_prepare + + - name: Install knetstat + run: | + wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip + unzip master.zip + cd knetstat-master + make clean + make KSRC=linux-6.5.0-1025-azure + - name: Install node uses: actions/setup-node@v4 with: @@ -30,6 +51,7 @@ jobs: - name: install redis-cli run: sudo apt-get install redis-tools + - name: Install dependencies run: npm install From 563280d3d73a529d8e70b483337bfbe77d900a12 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 10:39:15 +0100 Subject: [PATCH 05/18] wip --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab424ad..af5950e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y build-essential linux-headers-$(uname -r) wget unzip libelf-dev - name: Download kernel sources - run: wget https://cl.archive.ubuntu.com/ubuntu/pool/main/l/linux-azure-6.5/linux-azure-6.5_6.5.0.orig.tar.gz | tar xvz + run: wget https://cl.archive.ubuntu.com/ubuntu/pool/main/l/linux-azure-6.5/linux-azure-6.5_6.5.0.orig.tar.gz && tar xvzf linux-azure-6.5_6.5.0.orig.tar.gz - name: Configure kernel sources run: | From c7a55725fd45964d7785a21553bb3fd639ba2065 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 10:56:14 +0100 Subject: [PATCH 06/18] wip --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af5950e..573e36f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: unzip master.zip cd knetstat-master make clean - make KSRC=linux-6.5.0-1025-azure + make KSRC=linux-6.5 - name: Install node uses: actions/setup-node@v4 From b5675d56425f941e147935e28387f61eb5bd217d Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 10:58:31 +0100 Subject: [PATCH 07/18] wip --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 573e36f..1a6bd9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: unzip master.zip cd knetstat-master make clean - make KSRC=linux-6.5 + make - name: Install node uses: actions/setup-node@v4 From cb9ecf78963b1c622c56feeb4580d26af46028cb Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:02:52 +0100 Subject: [PATCH 08/18] wip --- .github/workflows/ci.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a6bd9d..5b1b653 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [18.x] - kernel: [6.5.0-1025-azure] + kernel: [6.5] steps: - name: Checkout @@ -27,21 +27,20 @@ jobs: run: sudo apt-get update && sudo apt-get install -y build-essential linux-headers-$(uname -r) wget unzip libelf-dev - name: Download kernel sources - run: wget https://cl.archive.ubuntu.com/ubuntu/pool/main/l/linux-azure-6.5/linux-azure-6.5_6.5.0.orig.tar.gz && tar xvzf linux-azure-6.5_6.5.0.orig.tar.gz + run: curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ - name: Configure kernel sources run: | - cp /boot/config-$(uname -r) linux-6.5/.config - yes "" | make -C linux-6.5 oldconfig - make -C linux-6.5 modules_prepare + cp /boot/config-$(uname -r) linux-${{ matrix.kernel }}/.config + yes "" | make -C linux-${{ matrix.kernel }} oldconfig + make -C linux-${{ matrix.kernel }} modules_prepare - name: Install knetstat run: | wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip unzip master.zip cd knetstat-master - make clean - make + make KSRC=linux-${{ matrix.kernel }} - name: Install node uses: actions/setup-node@v4 From 9b28fbb8bb4f0395c079dc0993b1da4d7b7df6fe Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:04:58 +0100 Subject: [PATCH 09/18] wip --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b1b653..5f9f699 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip unzip master.zip cd knetstat-master - make KSRC=linux-${{ matrix.kernel }} + make - name: Install node uses: actions/setup-node@v4 From 7dbe15ee2f98e0ebd83be923e899af7520b597b5 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:13:32 +0100 Subject: [PATCH 10/18] wip --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f9f699..b88ad73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: - name: Configure kernel sources run: | cp /boot/config-$(uname -r) linux-${{ matrix.kernel }}/.config + make -C linux-${{ matrix.kernel }} clean yes "" | make -C linux-${{ matrix.kernel }} oldconfig make -C linux-${{ matrix.kernel }} modules_prepare From dec3292671c5f361538acf49386aebb63f07dec1 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:17:03 +0100 Subject: [PATCH 11/18] wip --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b88ad73..5ce82ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [18.x] - kernel: [6.5] + kernel: [5.15] steps: - name: Checkout From cb3df4887e6cd628fdc686b1fca264f5f861c308 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:21:36 +0100 Subject: [PATCH 12/18] wip --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ce82ba..c2fe0fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,13 +35,10 @@ jobs: make -C linux-${{ matrix.kernel }} clean yes "" | make -C linux-${{ matrix.kernel }} oldconfig make -C linux-${{ matrix.kernel }} modules_prepare - - - name: Install knetstat - run: | wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip unzip master.zip cd knetstat-master - make + make KSRC=linux-${{ matrix.kernel }} - name: Install node uses: actions/setup-node@v4 From 0ba8a9175ad15369335a472c5d27139cc68f519a Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:25:08 +0100 Subject: [PATCH 13/18] wip --- .github/workflows/ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2fe0fa..1446e96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,18 +26,16 @@ jobs: - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y build-essential linux-headers-$(uname -r) wget unzip libelf-dev - - name: Download kernel sources - run: curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ - - name: Configure kernel sources run: | + wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip + unzip master.zip + cd knetstat-master + curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ cp /boot/config-$(uname -r) linux-${{ matrix.kernel }}/.config make -C linux-${{ matrix.kernel }} clean yes "" | make -C linux-${{ matrix.kernel }} oldconfig make -C linux-${{ matrix.kernel }} modules_prepare - wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip - unzip master.zip - cd knetstat-master make KSRC=linux-${{ matrix.kernel }} - name: Install node From 4cf1621f635861d773f65d063bf425b19ebd2251 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:30:42 +0100 Subject: [PATCH 14/18] wip --- .github/workflows/ci.yml | 71 +++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1446e96..e42d929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,36 +37,41 @@ jobs: yes "" | make -C linux-${{ matrix.kernel }} oldconfig make -C linux-${{ matrix.kernel }} modules_prepare make KSRC=linux-${{ matrix.kernel }} - - - name: Install node - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: install redis-cli - run: sudo apt-get install redis-tools - - - - name: Install dependencies - run: npm install - - - name: Setup Standalone Tests - run: make test-standalone-setup - - - name: Run Standalone tests - run: make test-standalone - - - name: Teardown Standalone Tests - run: make test-standalone-teardown - - - name: Setup Clustered Tests - run: make test-cluster-setup - - - name: Check Redis Cluster - run: timeout 60 bash <<< "until redis-cli -c -p 16371 cluster info | grep 'cluster_state:ok'; do sleep 1; done" - - - name: Run Clustered tests - run: make test-cluster - - - name: Teardown Clustered Tests - run: make test-cluster-teardown + insmod knetstat.ko + lsmod | grep knetstat + + - name: run knetstat + run: cat /proc/net/tcpstat + +# - name: Install node +# uses: actions/setup-node@v4 +# with: +# node-version: ${{ matrix.node-version }} +# +# - name: install redis-cli +# run: sudo apt-get install redis-tools +# +# +# - name: Install dependencies +# run: npm install +# +# - name: Setup Standalone Tests +# run: make test-standalone-setup +# +# - name: Run Standalone tests +# run: make test-standalone +# +# - name: Teardown Standalone Tests +# run: make test-standalone-teardown +# +# - name: Setup Clustered Tests +# run: make test-cluster-setup +# +# - name: Check Redis Cluster +# run: timeout 60 bash <<< "until redis-cli -c -p 16371 cluster info | grep 'cluster_state:ok'; do sleep 1; done" +# +# - name: Run Clustered tests +# run: make test-cluster +# +# - name: Teardown Clustered Tests +# run: make test-cluster-teardown From 8653602cf686c67d5207ad64997aabffc8ca797f Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:33:00 +0100 Subject: [PATCH 15/18] wip --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e42d929..f4ebed3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [18.x] - kernel: [5.15] + kernel: [6.5] steps: - name: Checkout @@ -37,8 +37,8 @@ jobs: yes "" | make -C linux-${{ matrix.kernel }} oldconfig make -C linux-${{ matrix.kernel }} modules_prepare make KSRC=linux-${{ matrix.kernel }} - insmod knetstat.ko - lsmod | grep knetstat + sudo insmod knetstat.ko + sudo lsmod | grep knetstat - name: run knetstat run: cat /proc/net/tcpstat From aa932e83b7dcb23393aca119dbf2466d504fafb7 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:41:54 +0100 Subject: [PATCH 16/18] wip --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4ebed3..365eeab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [18.x] - kernel: [6.5] + kernel: [5.15] steps: - name: Checkout @@ -31,7 +31,9 @@ jobs: wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip unzip master.zip cd knetstat-master - curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ +# curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ + apt install linux-source-${{ matrix.kernel }} + tar -C ./ -xvf /usr/src/linux-source-${{ matrix.kernel }}.tar.xz cp /boot/config-$(uname -r) linux-${{ matrix.kernel }}/.config make -C linux-${{ matrix.kernel }} clean yes "" | make -C linux-${{ matrix.kernel }} oldconfig From ae629da6b013e547bfda261eb7f626b24c16daa0 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 11:43:00 +0100 Subject: [PATCH 17/18] wip --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 365eeab..bd8398e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [18.x] - kernel: [5.15] + kernel: [6.1] steps: - name: Checkout @@ -31,9 +31,7 @@ jobs: wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip unzip master.zip cd knetstat-master -# curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ - apt install linux-source-${{ matrix.kernel }} - tar -C ./ -xvf /usr/src/linux-source-${{ matrix.kernel }}.tar.xz + curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ cp /boot/config-$(uname -r) linux-${{ matrix.kernel }}/.config make -C linux-${{ matrix.kernel }} clean yes "" | make -C linux-${{ matrix.kernel }} oldconfig From 59f394d37df8a1ae5e28995f2971c634f9e85f90 Mon Sep 17 00:00:00 2001 From: Pablo Ubal Naveira Date: Wed, 13 Nov 2024 12:11:35 +0100 Subject: [PATCH 18/18] wip --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd8398e..855ee7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,28 +17,48 @@ jobs: strategy: matrix: node-version: [18.x] - kernel: [6.1] + kernel: [6.5] steps: - name: Checkout uses: actions/checkout@v4 + - name: Uninstall all linux-headers packages + run: | + sudo apt-get remove --purge -y 'linux-headers-*' + + - name: Reinstall specific linux-headers package + run: | + sudo apt-get update + sudo apt-get install -y linux-headers-$(uname -r) + - name: Install build dependencies - run: sudo apt-get update && sudo apt-get install -y build-essential linux-headers-$(uname -r) wget unzip libelf-dev + run: | + sudo apt-get update && sudo apt-get install -y build-essential linux-headers-$(uname -r) wget unzip libelf-dev linux-source-6.5.0 + tar jxvf /usr/src/linux-source-6.5.0.tar.bz2 + cd linux-source-6.5.0 - name: Configure kernel sources run: | wget https://github.com/veithen/knetstat/archive/refs/heads/master.zip unzip master.zip cd knetstat-master - curl https://cdn.kernel.org/pub/linux/kernel/$(echo ${{ matrix.kernel }} | sed -E 's/([^.]+)[.].*/v\1.x/')/linux-${{ matrix.kernel }}.tar.xz | tar xJ + # wget https://cl.archive.ubuntu.com/ubuntu/pool/main/l/linux-azure-6.5/linux-azure-6.5_6.5.0.orig.tar.gz && tar xvzf linux-azure-6.5_6.5.0.orig.tar.gz + mv ../linux-source-6.5.0 linux-6.5 cp /boot/config-$(uname -r) linux-${{ matrix.kernel }}/.config make -C linux-${{ matrix.kernel }} clean yes "" | make -C linux-${{ matrix.kernel }} oldconfig make -C linux-${{ matrix.kernel }} modules_prepare - make KSRC=linux-${{ matrix.kernel }} + sed -i '/Module.symvers/d' ./Makefile + make KBUILD_MODPOST_WARN=1 KSRC=linux-${{ matrix.kernel }} sudo insmod knetstat.ko sudo lsmod | grep knetstat + - name: dmesg + if: always() + run: | + sudo dmesg|tail -n 10 + ls -la /lib/modules + - name: run knetstat run: cat /proc/net/tcpstat