break up redis tx inserts into smaller chunks

zerotier · Jul 8, 2024 · 508527f · 508527f
1 parent 34cc261
commit 508527f
Showing 1 changed file with 26 additions and 2 deletions.
diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp
@@ -780,11 +780,25 @@ void PostgreSQL::initializeNetworks()
  fprintf(stderr, "adding networks to redis...\n");
  if (_rc->clusterMode) {
  auto tx = _cluster->transaction(_myAddressStr, true, false);
- tx.sadd(setKey, networkSet.begin(), networkSet.end());
+ uint64_t count = 0;
+ for (std::string nwid : networkSet) {
+ tx.sadd(setKey, nwid);
+ if (++count % 30000 == 0) {
+ tx.exec();
+ tx = _cluster->transaction(_myAddressStr, true, false);
+ }
+ }
  tx.exec();
  } else {
  auto tx = _redis->transaction(true, false);
- tx.sadd(setKey, networkSet.begin(), networkSet.end());
+ uint64_t count = 0;
+ for (std::string nwid : networkSet) {
+ tx.sadd(setKey, nwid);
+ if (++count % 30000 == 0) {
+ tx.exec();
+ tx = _redis->transaction(true, false);
+ }
+ }
  tx.exec();
  }
  fprintf(stderr, "done.\n");
@@ -1005,14 +1019,24 @@ void PostgreSQL::initializeMembers()
  fprintf(stderr, "Load member data into redis...\n");
  if (_rc->clusterMode) {
  auto tx = _cluster->transaction(_myAddressStr, true, false);
+ uint64_t count = 0;
  for (auto it : networkMembers) {
  tx.sadd(it.first, it.second);
+ if (++count % 30000 == 0) {
+ tx.exec();
+ tx = _cluster->transaction(_myAddressStr, true, false);
+ }
  }
  tx.exec();
  } else {
  auto tx = _redis->transaction(true, false);
+ uint64_t count = 0;
  for (auto it : networkMembers) {
  tx.sadd(it.first, it.second);
+ if (++count % 30000 == 0) {
+ tx.exec();
+ tx = _redis->transaction(true, false);
+ }
  }
  tx.exec();
  }