From 508527f7cdcf622b51cedab86d95be79a82d9a37 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 8 Jul 2024 14:08:54 -0700 Subject: [PATCH] break up redis tx inserts into smaller chunks --- controller/PostgreSQL.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 010df3b9d..e4a31ba28 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -780,11 +780,25 @@ void PostgreSQL::initializeNetworks() fprintf(stderr, "adding networks to redis...\n"); if (_rc->clusterMode) { auto tx = _cluster->transaction(_myAddressStr, true, false); - tx.sadd(setKey, networkSet.begin(), networkSet.end()); + uint64_t count = 0; + for (std::string nwid : networkSet) { + tx.sadd(setKey, nwid); + if (++count % 30000 == 0) { + tx.exec(); + tx = _cluster->transaction(_myAddressStr, true, false); + } + } tx.exec(); } else { auto tx = _redis->transaction(true, false); - tx.sadd(setKey, networkSet.begin(), networkSet.end()); + uint64_t count = 0; + for (std::string nwid : networkSet) { + tx.sadd(setKey, nwid); + if (++count % 30000 == 0) { + tx.exec(); + tx = _redis->transaction(true, false); + } + } tx.exec(); } fprintf(stderr, "done.\n"); @@ -1005,14 +1019,24 @@ void PostgreSQL::initializeMembers() fprintf(stderr, "Load member data into redis...\n"); if (_rc->clusterMode) { auto tx = _cluster->transaction(_myAddressStr, true, false); + uint64_t count = 0; for (auto it : networkMembers) { tx.sadd(it.first, it.second); + if (++count % 30000 == 0) { + tx.exec(); + tx = _cluster->transaction(_myAddressStr, true, false); + } } tx.exec(); } else { auto tx = _redis->transaction(true, false); + uint64_t count = 0; for (auto it : networkMembers) { tx.sadd(it.first, it.second); + if (++count % 30000 == 0) { + tx.exec(); + tx = _redis->transaction(true, false); + } } tx.exec(); }