Add prometheus metrics for Central controllers (#1969)

* add header-only prometheus lib to ext

* rename folder

* Undo rename directory

* prometheus simpleapi included on mac & linux

* wip

* wire up some controller stats

* Get windows building with prometheus

* bsd build flags for prometheus

* Fix multiple network join from environment entrypoint.sh.release (#1961)

* _bond_m guards _bond, not _paths_m (#1965)

* Fix: warning: mutex '_aqm_m' is not held on every path through here [-Wthread-safety-analysis] (#1964)

* Serve prom metrics from /metrics endpoint

* Add prom metrics for Central controller specific things

* reorganize metric initialization

* testing out a labled gauge on Networks

* increment error counter on throw

* Consolidate metrics definitions

Put all metric definitions into node/Metrics.hpp.  Accessed as needed
from there.

* Revert "testing out a labled gauge on Networks"

This reverts commit 499ed6d95e.

* still blows up but adding to the record for completeness right now

* Fix runtime issues with metrics

* Add metrics files to visual studio project

* Missed an "extern"

* add copyright headers to new files

* Add metrics for sent/received bytes (total)

* put /metrics endpoint behind auth

* sendto returns int on Win32

---------

Co-authored-by: Leonardo Amaral <leleobhz@users.noreply.github.com>
Co-authored-by: Brenton Bostick <bostick@gmail.com>
This commit is contained in:
Grant Limberg
2023-04-21 12:12:43 -07:00
committed by GitHub
parent 0b03ad9a21
commit 8e6e4ede6d
62 changed files with 4023 additions and 25 deletions

View File

@ -119,6 +119,7 @@ MemberNotificationReceiver::MemberNotificationReceiver(PostgreSQL *p, pqxx::conn
void MemberNotificationReceiver::operator() (const std::string &payload, int packend_pid) {
fprintf(stderr, "Member Notification received: %s\n", payload.c_str());
Metrics::pgsql_mem_notification++;
json tmp(json::parse(payload));
json &ov = tmp["old_val"];
json &nv = tmp["new_val"];
@ -141,6 +142,7 @@ NetworkNotificationReceiver::NetworkNotificationReceiver(PostgreSQL *p, pqxx::co
void NetworkNotificationReceiver::operator() (const std::string &payload, int packend_pid) {
fprintf(stderr, "Network Notification received: %s\n", payload.c_str());
Metrics::pgsql_net_notification++;
json tmp(json::parse(payload));
json &ov = tmp["old_val"];
json &nv = tmp["new_val"];
@ -705,6 +707,8 @@ void PostgreSQL::initializeNetworks()
}
}
Metrics::network_count++;
_networkChanged(empty, config, false);
auto end = std::chrono::high_resolution_clock::now();
@ -925,6 +929,8 @@ void PostgreSQL::initializeMembers()
}
}
Metrics::member_count++;
_memberChanged(empty, config, false);
memberId = "";
@ -1034,7 +1040,6 @@ void PostgreSQL::heartbeat()
w.commit();
} catch (std::exception &e) {
fprintf(stderr, "%s: Heartbeat update failed: %s\n", controllerId, e.what());
w.abort();
_pool->unborrow(c);
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
continue;
@ -1230,6 +1235,7 @@ void PostgreSQL::_networksWatcher_Redis() {
}
lastID = id;
}
Metrics::redis_net_notification++;
}
}
} catch (sw::redis::Error &e) {
@ -1788,6 +1794,7 @@ uint64_t PostgreSQL::_doRedisUpdate(sw::redis::Transaction &tx, std::string &con
.sadd("network-nodes-all:{"+controllerId+"}:"+networkId, memberId)
.hmset("member:{"+controllerId+"}:"+networkId+":"+memberId, record.begin(), record.end());
++count;
Metrics::redis_mem_notification++;
}
// expire records from all-nodes and network-nodes member list