Add prometheus metrics for Central controllers (#1969)

* add header-only prometheus lib to ext

* rename folder

* Undo rename directory

* prometheus simpleapi included on mac & linux

* wip

* wire up some controller stats

* Get windows building with prometheus

* bsd build flags for prometheus

* Fix multiple network join from environment entrypoint.sh.release (#1961)

* _bond_m guards _bond, not _paths_m (#1965)

* Fix: warning: mutex '_aqm_m' is not held on every path through here [-Wthread-safety-analysis] (#1964)

* Serve prom metrics from /metrics endpoint

* Add prom metrics for Central controller specific things

* reorganize metric initialization

* testing out a labled gauge on Networks

* increment error counter on throw

* Consolidate metrics definitions

Put all metric definitions into node/Metrics.hpp.  Accessed as needed
from there.

* Revert "testing out a labled gauge on Networks"

This reverts commit 499ed6d95e.

* still blows up but adding to the record for completeness right now

* Fix runtime issues with metrics

* Add metrics files to visual studio project

* Missed an "extern"

* add copyright headers to new files

* Add metrics for sent/received bytes (total)

* put /metrics endpoint behind auth

* sendto returns int on Win32

---------

Co-authored-by: Leonardo Amaral <leleobhz@users.noreply.github.com>
Co-authored-by: Brenton Bostick <bostick@gmail.com>

ext/prometheus-cpp-lite-1.0/examples/CMakeLists.txt

@@ -0,0 +1,31 @@
+
+add_executable       (original_example               "original_example.cpp" )
+target_link_libraries(original_example               prometheus-cpp-lite-core)
+
+add_executable       (modern_example                 "modern_example.cpp" )
+target_link_libraries(modern_example                 prometheus-cpp-lite-core)
+
+add_executable       (use_counters_in_class_example  "use_counters_in_class_example.cpp" )
+target_link_libraries(use_counters_in_class_example  prometheus-cpp-lite-core)
+
+add_executable       (use_gauge_in_class_example     "use_gauge_in_class_example.cpp" )
+target_link_libraries(use_gauge_in_class_example     prometheus-cpp-lite-core)
+
+add_executable       (use_benchmark_in_class_example "use_benchmark_in_class_example.cpp" )
+target_link_libraries(use_benchmark_in_class_example prometheus-cpp-lite-core)
+
+add_executable       (save_to_file_example           "save_to_file_example.cpp" )
+target_link_libraries(save_to_file_example           prometheus-cpp-lite-core)
+
+add_executable       (push_to_server_example         "push_to_server_example.cpp" )
+target_link_libraries(push_to_server_example         prometheus-cpp-lite-core)
+
+add_executable       (gateway_example                "gateway_example.cpp" )
+target_link_libraries(gateway_example                prometheus-cpp-lite-core)
+
+
+add_executable       (simpleapi_example              "simpleapi_example.cpp")
+target_link_libraries(simpleapi_example              prometheus-cpp-simpleapi)
+
+add_executable       (simpleapi_use_in_class_example "simpleapi_use_in_class_example.cpp")
+target_link_libraries(simpleapi_use_in_class_example prometheus-cpp-simpleapi)

ext/prometheus-cpp-lite-1.0/examples/gateway_example.cpp

@@ -0,0 +1,64 @@
+#include <chrono>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <thread>
+
+#include "prometheus/client_metric.h"
+#include "prometheus/counter.h"
+#include "prometheus/family.h"
+#include "prometheus/gateway.h"
+#include "prometheus/registry.h"
+
+#ifdef _WIN32
+#include <Winsock2.h>
+#else
+#include <unistd.h>
+#endif
+
+static std::string GetHostName() {
+  char hostname[1024];
+
+  if (::gethostname(hostname, sizeof(hostname))) {
+    return {};
+  }
+  return hostname;
+}
+
+int main() {
+  using namespace prometheus;
+
+  // create a push gateway
+  const auto labels = Gateway::GetInstanceLabel(GetHostName());
+
+  Gateway gateway{"127.0.0.1", "9091", "sample_client", labels};
+
+  // create a metrics registry with component=main labels applied to all its
+  // metrics
+  auto registry = std::make_shared<Registry>();
+
+  // add a new counter family to the registry (families combine values with the
+  // same name, but distinct label dimensions)
+  auto& counter_family = BuildCounter()
+                             .Name("time_running_seconds_total")
+                             .Help("How many seconds is this server running?")
+                             .Labels({{"label", "value"}})
+                             .Register(*registry);
+
+  // add a counter to the metric family
+  auto& second_counter = counter_family.Add(
+      {{"another_label", "value"}, {"yet_another_label", "value"}});
+
+  // ask the pusher to push the metrics to the pushgateway
+  gateway.RegisterCollectable(registry);
+
+  for (;;) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    // increment the counter by one (second)
+    second_counter.Increment();
+
+    // push metrics
+    auto returnCode = gateway.Push();
+    std::cout << "returnCode is " << returnCode << std::endl;
+  }
+}

ext/prometheus-cpp-lite-1.0/examples/modern_example.cpp

@@ -0,0 +1,65 @@
+#include <prometheus/registry.h>
+#include <prometheus/counter.h>
+#include <prometheus/text_serializer.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+int main() {
+
+  using namespace prometheus;
+
+  // for clarity, we deduce the required types
+  using IntegerCounter  = Counter<uint64_t>;
+  using FloatingCounter = Counter<double>;
+
+  using IntegerCounterFamily  = CustomFamily<IntegerCounter>;
+  using FloatingCounterFamily = CustomFamily<FloatingCounter>;
+
+  // create a metrics registry
+  // @note it's the users responsibility to keep the object alive
+  Registry registry;
+
+  // add a new counter family to the registry (families combine values with the
+  // same name, but distinct label dimensions)
+  //
+  // @note please follow the metric-naming best-practices:
+  // https://prometheus.io/docs/practices/naming/
+  FloatingCounterFamily& packet_counter{ FloatingCounter::Family::Build(registry, "observed_packets_total", "Number of observed packets") };
+
+  // add and remember dimensional data, incrementing those is very cheap
+  FloatingCounter& tcp_rx_counter{ packet_counter.Add({ {"protocol", "tcp"}, {"direction", "rx"} }) };
+  FloatingCounter& tcp_tx_counter{ packet_counter.Add({ {"protocol", "tcp"}, {"direction", "tx"} }) };
+  FloatingCounter& udp_rx_counter{ packet_counter.Add({ {"protocol", "udp"}, {"direction", "rx"} }) };
+  FloatingCounter& udp_tx_counter{ packet_counter.Add({ {"protocol", "udp"}, {"direction", "tx"} }) };
+
+  // add a counter whose dimensional data is not known at compile time
+  // nevertheless dimensional values should only occur in low cardinality:
+  // https://prometheus.io/docs/practices/naming/#labels
+  IntegerCounterFamily& http_requests_counter = IntegerCounter::Family::Build(registry, "http_requests_total", "Number of HTTP requests");
+
+  for (;; ) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    const int random_value = std::rand();
+
+    if (random_value & 1)   tcp_rx_counter++;
+    if (random_value & 2) ++tcp_tx_counter;
+    if (random_value & 4) udp_rx_counter += 0.5;
+    if (random_value & 8) udp_tx_counter += 0.7;
+
+    const std::array<std::string, 4> methods = { "GET", "PUT", "POST", "HEAD" };
+    const std::string& method = methods.at(static_cast<std::size_t>(random_value) % methods.size());
+
+    // dynamically calling Family<T>.Add() works but is slow and should be avoided
+    http_requests_counter.Add({ {"method", method} }) += 10;
+
+    TextSerializer text_serializer;
+    text_serializer.Serialize(std::cout, registry.Collect());
+
+  }
+}

ext/prometheus-cpp-lite-1.0/examples/original_example.cpp

@@ -0,0 +1,67 @@
+
+#include <prometheus/registry.h>
+#include <prometheus/counter.h>
+#include <prometheus/text_serializer.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+int main() {
+
+  using namespace prometheus;
+
+  // create a metrics registry
+  // @note it's the users responsibility to keep the object alive
+  auto registry = std::make_shared<Registry>();
+
+  // add a new counter family to the registry (families combine values with the
+  // same name, but distinct label dimensions)
+  //
+  // @note please follow the metric-naming best-practices:
+  // https://prometheus.io/docs/practices/naming/
+  auto& packet_counter = BuildCounter()
+                             .Name("observed_packets_total")
+                             .Help("Number of observed packets")
+                             .Register(*registry);
+
+  // add and remember dimensional data, incrementing those is very cheap
+  auto& tcp_rx_counter = packet_counter.Add({ {"protocol", "tcp"}, {"direction", "rx"} });
+  auto& tcp_tx_counter = packet_counter.Add({ {"protocol", "tcp"}, {"direction", "tx"} });
+  auto& udp_rx_counter = packet_counter.Add({ {"protocol", "udp"}, {"direction", "rx"} });
+  auto& udp_tx_counter = packet_counter.Add({ {"protocol", "udp"}, {"direction", "tx"} });
+
+  // add a counter whose dimensional data is not known at compile time
+  // nevertheless dimensional values should only occur in low cardinality:
+  // https://prometheus.io/docs/practices/naming/#labels
+  auto& http_requests_counter = BuildCounter()
+                                    .Name("http_requests_total")
+                                    .Help("Number of HTTP requests")
+                                    .Register(*registry);
+
+  // ask the exposer to scrape the registry on incoming HTTP requests
+  //exposer.RegisterCollectable(registry);
+
+  for ( ;; ) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    const auto random_value = std::rand();
+
+    if (random_value & 1) tcp_rx_counter.Increment();
+    if (random_value & 2) tcp_tx_counter.Increment();
+    if (random_value & 4) udp_rx_counter.Increment(10);
+    if (random_value & 8) udp_tx_counter.Increment(10);
+
+    const std::array<std::string, 4> methods = { "GET", "PUT", "POST", "HEAD" };
+    auto method = methods.at(static_cast<std::size_t>(random_value) % methods.size());
+    // dynamically calling Family<T>.Add() works but is slow and should be avoided
+    http_requests_counter.Add({ {"method", method} }).Increment();
+
+    TextSerializer text_serializer;
+    text_serializer.Serialize(std::cout, registry->Collect());
+
+  }
+}

ext/prometheus-cpp-lite-1.0/examples/push_to_server_example.cpp

@@ -0,0 +1,44 @@
+#include <prometheus/registry.h>
+#include <prometheus/counter.h>
+#include <prometheus/push_to_server.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+int main() {
+
+  using namespace prometheus;
+
+  // for clarity, we deduce the required types
+  using Metric = Counter<uint64_t>;
+
+  using Family = Metric::Family;
+
+  // create a metrics registry
+  // @note it's the users responsibility to keep the object alive
+  std::shared_ptr<Registry> registry_ptr = std::make_shared<Registry>();
+
+  PushToServer pusher(registry_ptr, std::chrono::seconds(5),
+                      std::string("http://127.0.0.1:9091/metrics/job/samples/instance/test") );
+
+  // add a new counter family to the registry (families combine values with the
+  // same name, but distinct label dimensions)
+  //
+  // @note please follow the metric-naming best-practices:
+  // https://prometheus.io/docs/practices/naming/
+  Family& family { Family::Build(*registry_ptr, "our_metric", "some metric") };
+
+  // add and remember dimensional data, incrementing those is very cheap
+  Metric& metric { family.Add({}) };
+
+  for (;; ) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    const int random_value = std::rand();
+    metric += random_value % 10;
+  }
+}

ext/prometheus-cpp-lite-1.0/examples/save_to_file_example.cpp

@@ -0,0 +1,43 @@
+#include <prometheus/registry.h>
+#include <prometheus/counter.h>
+#include <prometheus/save_to_file.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+int main() {
+
+  using namespace prometheus;
+
+  // for clarity, we deduce the required types
+  using Metric = Counter<uint64_t>;
+
+  using Family = Metric::Family;
+
+  // create a metrics registry
+  // @note it's the users responsibility to keep the object alive
+  std::shared_ptr<Registry> registry_ptr = std::make_shared<Registry>();
+
+  SaveToFile saver( registry_ptr, std::chrono::seconds(5), "./metrics.prom" );
+
+  // add a new counter family to the registry (families combine values with the
+  // same name, but distinct label dimensions)
+  //
+  // @note please follow the metric-naming best-practices:
+  // https://prometheus.io/docs/practices/naming/
+  Family& family { Family::Build(*registry_ptr, "our_metric", "some metric") };
+
+  // add and remember dimensional data, incrementing those is very cheap
+  Metric& metric { family.Add({}) };
+
+  for (;; ) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    const int random_value = std::rand();
+    metric += random_value % 10;
+  }
+}

ext/prometheus-cpp-lite-1.0/examples/simpleapi_example.cpp

@@ -0,0 +1,26 @@
+
+#include <prometheus/simpleapi.h>
+
+int main() {
+
+  using namespace prometheus::simpleapi;
+
+  counter_family_t family  { "simple_family", "simple family example" };
+  counter_metric_t metric1 { family.Add({{"name", "counter1"}}) };
+  counter_metric_t metric2 { family.Add({{"name", "counter2"}}) };
+
+  counter_metric_t metric3 { "simple_counter_1", "simple counter 1 without labels example" };
+  counter_metric_t metric4 { "simple_counter_2", "simple counter 2 without labels example" };
+
+  for (;; ) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    const int random_value = std::rand();
+    if (random_value & 1) metric1++;
+    if (random_value & 2) metric2++;
+    if (random_value & 4) metric3++;
+    if (random_value & 8) metric4++;
+  }
+
+  //return 0;
+
+}

ext/prometheus-cpp-lite-1.0/examples/simpleapi_use_in_class_example.cpp

@@ -0,0 +1,57 @@
+
+#include <prometheus/simpleapi.h>
+
+// use prometheus namespace
+using namespace prometheus::simpleapi;
+
+class MyClass {
+
+  counter_family_t metric_family { "simple_family", "simple family example" };
+  counter_metric_t metric1 { metric_family.Add({{"name", "counter1"}}) };
+  counter_metric_t metric2 { metric_family.Add({{"name", "counter2"}}) };
+
+  counter_metric_t metric3 { "simple_counter_1", "simple counter 1 without labels example" };
+  counter_metric_t metric4 { "simple_counter_2", "simple counter 2 without labels example" };
+
+  benchmark_family_t benchmark_family { "simple_benchmark_family", "simple benchmark family example" };
+  benchmark_metric_t benchmark1 { benchmark_family.Add({{"benchmark", "1"}}) };
+  benchmark_metric_t benchmark2 { benchmark_family.Add({{"benchmark", "2"}}) };
+
+public:
+
+  MyClass() = default;
+
+  void member_to_do_something() {
+
+    benchmark1.start();
+    const int random_value = std::rand();
+    benchmark1.stop();
+
+    benchmark2.start();
+    if (random_value & 1)  metric1++;
+    if (random_value & 2)  metric2++;
+    if (random_value & 4)  metric3++;
+    if (random_value & 8)  metric4++;
+    benchmark2.stop();
+
+  }
+
+};
+
+int main() {
+
+  MyClass myClass;
+  benchmark_metric_t benchmark { "simple_benchmark", "simple benchmark example" };
+
+  for (;; ) {
+
+    benchmark.start();
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    benchmark.stop();
+
+    myClass.member_to_do_something();
+
+  }
+
+}
+

ext/prometheus-cpp-lite-1.0/examples/use_benchmark_in_class_example.cpp

@@ -0,0 +1,59 @@
+#include <prometheus/registry.h>
+#include <prometheus/benchmark.h>
+#include <prometheus/text_serializer.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+// use prometheus namespace
+using namespace prometheus;
+
+// create global registry for use it from our classes
+static Registry globalRegistry;
+
+class MyClass {
+
+  Benchmark::Family& benchmarkFamily { Benchmark::Family::Build(globalRegistry,
+                                         "benchmark_family", "family for check benchmark functionality") };
+
+  Benchmark& benchmark1 { benchmarkFamily.Add({{"number", "1"}}) };
+  Benchmark& benchmark2 { benchmarkFamily.Add({{"number", "2"}}) };
+
+public:
+
+  MyClass() = default;
+
+  void member_to_do_something() {
+
+    benchmark1.start();
+    benchmark2.start();
+    std::this_thread::sleep_for(std::chrono::milliseconds(500));
+    benchmark1.stop();
+    std::this_thread::sleep_for(std::chrono::milliseconds(500));
+    benchmark2.stop();
+
+  }
+
+};
+
+int main() {
+
+  MyClass myClass;
+
+  for (;; ) {
+
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+
+    myClass.member_to_do_something();
+
+    TextSerializer text_serializer;
+    text_serializer.Serialize(std::cout, globalRegistry.Collect());
+
+  }
+}
+

ext/prometheus-cpp-lite-1.0/examples/use_counters_in_class_example.cpp

@@ -0,0 +1,79 @@
+#include <prometheus/registry.h>
+#include <prometheus/counter.h>
+#include <prometheus/text_serializer.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+// use prometheus namespace
+using namespace prometheus;
+
+// for clarity, we deduce the required types
+using IntegerCounter  = Counter<uint64_t>;
+using FloatingCounter = Counter<double>;
+
+using IntegerCounterFamily  = CustomFamily<IntegerCounter>;
+using FloatingCounterFamily = CustomFamily<FloatingCounter>;
+
+// create global registry for use it from our classes
+static Registry globalRegistry;
+
+class MyClass {
+
+  IntegerCounterFamily& counterFamily1 { IntegerCounter::Family::Build(globalRegistry,
+                                         "counter_family_1", "counter for check integer functionality",
+                                         {{"type","integer"}} ) };
+  
+  IntegerCounter& counter11{ counterFamily1.Add({{"number", "1"}}) };
+  IntegerCounter& counter12{ counterFamily1.Add({{"number", "2"}}) };
+  IntegerCounter& counter13{ counterFamily1.Add({{"number", "3"}}) };
+  
+  
+  FloatingCounterFamily& counterFamily2 { FloatingCounter::Family::Build(globalRegistry,
+                                          "counter_family_2", "counter for check floating functionality",
+                                          {{"type","float"}} ) };
+  
+  FloatingCounter& counter21{ counterFamily2.Add({{"number", "1"}}) };
+  FloatingCounter& counter22{ counterFamily2.Add({{"number", "2"}}) };
+  FloatingCounter& counter23{ counterFamily2.Add({{"number", "3"}}) };
+
+public:
+
+  MyClass() = default;
+
+  void member_to_do_something() {
+
+    const int random_value = std::rand();
+
+    if (random_value & 1)  counter11++;
+    if (random_value & 2)  counter12++;
+    if (random_value & 4)  counter13++;
+    if (random_value & 8)  counter21++;
+    if (random_value & 16) counter22++;
+    if (random_value & 32) counter23++;
+
+  }
+
+};
+
+int main() {
+
+  MyClass myClass;
+
+  for (;; ) {
+
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+
+    myClass.member_to_do_something();
+
+    TextSerializer text_serializer;
+    text_serializer.Serialize(std::cout, globalRegistry.Collect());
+
+  }
+}
+

ext/prometheus-cpp-lite-1.0/examples/use_gauge_in_class_example.cpp

@@ -0,0 +1,79 @@
+#include <prometheus/registry.h>
+#include <prometheus/gauge.h>
+#include <prometheus/text_serializer.h>
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <thread>
+#include <iostream>
+
+// use prometheus namespace
+using namespace prometheus;
+
+// for clarity, we deduce the required types
+using IntegerGauge  = Gauge<int64_t>;
+using FloatingGauge = Gauge<double>;
+
+using IntegerGaugeFamily  = CustomFamily<IntegerGauge>;
+using FloatingGaugeFamily = CustomFamily<FloatingGauge>;
+
+// create global registry for use it from our classes
+static Registry globalRegistry;
+
+class MyClass {
+
+  IntegerGaugeFamily& gaugeFamily1  { IntegerGauge::Family::Build(globalRegistry,
+                                      "gauge_family_1", "gauge for check integer functionality",
+                                      {{"type","integer"}} ) };
+  
+  IntegerGauge& gauge11{ gaugeFamily1.Add({{"number", "1"}}) };
+  IntegerGauge& gauge12{ gaugeFamily1.Add({{"number", "2"}}) };
+  IntegerGauge& gauge13{ gaugeFamily1.Add({{"number", "3"}}) };
+  
+  
+  FloatingGaugeFamily& gaugeFamily2 { FloatingGauge::Family::Build(globalRegistry,
+                                      "gauge_family_2", "gauge for check floating functionality",
+                                      {{"type","float"}} ) };
+  
+  FloatingGauge& gauge21{ gaugeFamily2.Add({{"number", "1"}}) };
+  FloatingGauge& gauge22{ gaugeFamily2.Add({{"number", "2"}}) };
+  FloatingGauge& gauge23{ gaugeFamily2.Add({{"number", "3"}}) };
+
+public:
+
+  MyClass() = default;
+
+  void member_to_do_something() {
+
+    const int random_value = std::rand();
+
+    if (random_value &  1      ) gauge11++; else gauge11--;
+    if (random_value & (1 << 1)) gauge12++; else gauge12--;
+    if (random_value & (1 << 2)) gauge13++; else gauge13--;
+    if (random_value & (1 << 3)) gauge21++; else gauge21--;
+    if (random_value & (1 << 4)) gauge22++; else gauge22--;
+    if (random_value & (1 << 5)) gauge23++; else gauge23--;
+
+  }
+
+};
+
+int main() {
+
+  MyClass myClass;
+
+  for (;; ) {
+
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+
+    myClass.member_to_do_something();
+
+    TextSerializer text_serializer;
+    text_serializer.Serialize(std::cout, globalRegistry.Collect());
+
+  }
+}
+