21 #include "metrics_thread.h"
23 #include "metrics_processor.h"
25 #include <core/threading/mutex_locker.h>
26 #include <interfaces/MetricCounterInterface.h>
27 #include <interfaces/MetricGaugeInterface.h>
28 #include <interfaces/MetricHistogramInterface.h>
29 #include <interfaces/MetricUntypedInterface.h>
30 #include <utils/misc/string_split.h>
31 #include <webview/url_manager.h>
39 #define CFG_PREFIX "/metrics/"
40 #define URL_PREFIX "/metrics"
70 std::list<MetricFamilyInterface *> ifaces =
73 for (
auto &i : ifaces) {
76 MetricFamilyBB mfbb{.metric_family = i, .metric_type = i->metric_type()};
77 metric_bbs_[i->id()] = mfbb;
79 if (!conditional_open(i->id(), metric_bbs_[i->id()])) {
88 imf_loop_count_ = std::make_shared<io::prometheus::client::MetricFamily>();
89 imf_loop_count_->set_name(
"fawkes_loop_count");
90 imf_loop_count_->set_help(
"Number of Fawkes main loop iterations");
91 imf_loop_count_->set_type(io::prometheus::client::COUNTER);
92 imf_loop_count_->add_metric();
93 internal_metrics_.push_back(imf_loop_count_);
95 imf_metrics_requests_ = std::make_shared<io::prometheus::client::MetricFamily>();
96 imf_metrics_requests_->set_name(
"fawkes_metrics_requests");
97 imf_metrics_requests_->set_help(
"Number of requests for metrics");
98 imf_metrics_requests_->set_type(io::prometheus::client::COUNTER);
99 imf_metrics_requests_->add_metric();
100 internal_metrics_.push_back(imf_metrics_requests_);
103 std::vector<float> buckets_le =
106 if (!buckets_le.empty()) {
107 std::sort(buckets_le.begin(), buckets_le.end());
109 imf_metrics_proctime_ = std::make_shared<io::prometheus::client::MetricFamily>();
110 imf_metrics_proctime_->set_name(
"fawkes_metrics_proctime");
111 imf_metrics_proctime_->set_help(
"Time required to process metrics");
112 imf_metrics_proctime_->set_type(io::prometheus::client::HISTOGRAM);
113 auto m = imf_metrics_proctime_->add_metric();
114 auto h = m->mutable_histogram();
115 for (
float &b : buckets_le) {
116 h->add_bucket()->set_upper_bound(b);
118 internal_metrics_.push_back(imf_metrics_proctime_);
122 "Internal metric metrics_proctime bucket bounds not configured, disabling");
125 metrics_suppliers_.push_back(
this);
132 std::placeholders::_1));
145 imf_loop_count_->mutable_metric(0)->mutable_counter()->set_value(
146 imf_loop_count_->metric(0).counter().value() + 1);
150 MetricsThread::bb_interface_created(
const char *type,
const char *
id)
throw()
153 MetricFamilyInterface *mfi;
156 logger->
log_info(name(),
"Opened %s:%s", type, id);
164 bbil_add_reader_interface(mfi);
165 bbil_add_writer_interface(mfi);
166 bbil_add_data_interface(mfi);
169 logger->
log_warn(name(),
"Failed to register for %s:%s: %s", type,
id, e.
what());
171 bbil_remove_reader_interface(mfi);
172 bbil_remove_writer_interface(mfi);
174 blackboard->
close(mfi);
177 name(),
"Failed to deregister %s:%s during error recovery: %s", type,
id, e.
what());
181 MetricFamilyBB mfbb{.metric_family = mfi, .metric_type = mfi->metric_type()};
182 metric_bbs_[id] = mfbb;
185 std::list<io::prometheus::client::MetricFamily>
186 MetricsThread::metrics()
188 std::chrono::high_resolution_clock::time_point proc_start =
189 std::chrono::high_resolution_clock::now();
191 imf_metrics_requests_->mutable_metric(0)->mutable_counter()->set_value(
192 imf_metrics_requests_->metric(0).counter().value() + 1);
194 std::list<io::prometheus::client::MetricFamily> rv;
197 for (
auto &mbbp : metric_bbs_) {
198 auto &mfbb = mbbp.second;
200 io::prometheus::client::MetricFamily mf;
201 mfbb.metric_family->read();
202 mf.set_name(mfbb.metric_family->name());
203 mf.set_help(mfbb.metric_family->help());
205 switch (mfbb.metric_type) {
206 case MetricFamilyInterface::COUNTER:
207 mf.set_type(io::prometheus::client::COUNTER);
208 for (
const auto &d : mfbb.data) {
210 io::prometheus::client::Metric *m = mf.add_metric();
211 parse_labels(d.counter->labels(), m);
212 m->mutable_counter()->set_value(d.counter->value());
216 case MetricFamilyInterface::GAUGE:
217 mf.set_type(io::prometheus::client::GAUGE);
218 for (
const auto &d : mfbb.data) {
220 io::prometheus::client::Metric *m = mf.add_metric();
221 parse_labels(d.gauge->labels(), m);
222 m->mutable_gauge()->set_value(d.gauge->value());
226 case MetricFamilyInterface::UNTYPED:
227 mf.set_type(io::prometheus::client::UNTYPED);
228 for (
const auto &d : mfbb.data) {
230 io::prometheus::client::Metric *m = mf.add_metric();
231 parse_labels(d.untyped->labels(), m);
232 m->mutable_untyped()->set_value(d.untyped->value());
236 case MetricFamilyInterface::HISTOGRAM:
237 mf.set_type(io::prometheus::client::HISTOGRAM);
238 for (
const auto &d : mfbb.data) {
240 io::prometheus::client::Metric *m = mf.add_metric();
241 parse_labels(d.histogram->labels(), m);
242 io::prometheus::client::Histogram *h = m->mutable_histogram();
243 h->set_sample_count(d.histogram->sample_count());
244 h->set_sample_sum(d.histogram->sample_sum());
245 for (
unsigned int i = 0; i < d.histogram->bucket_count(); ++i) {
246 io::prometheus::client::Bucket *b = h->add_bucket();
247 b->set_cumulative_count(d.histogram->bucket_cumulative_count(i));
248 b->set_upper_bound(d.histogram->bucket_upper_bound(i));
253 case MetricFamilyInterface::NOT_INITIALIZED:
257 rv.push_back(std::move(mf));
260 if (imf_metrics_proctime_) {
261 std::chrono::high_resolution_clock::time_point proc_end =
262 std::chrono::high_resolution_clock::now();
263 const std::chrono::duration<double> proc_diff = proc_end - proc_start;
264 for (
int i = 0; i < imf_metrics_proctime_->metric(0).histogram().bucket_size(); ++i) {
265 io::prometheus::client::Histogram *h =
266 imf_metrics_proctime_->mutable_metric(0)->mutable_histogram();
267 if (proc_diff.count() < h->bucket(i).upper_bound()) {
268 io::prometheus::client::Bucket *b = h->mutable_bucket(i);
269 b->set_cumulative_count(b->cumulative_count() + 1);
270 h->set_sample_count(h->sample_count() + 1);
271 h->set_sample_sum(h->sample_sum() + proc_diff.count());
277 for (
auto &im : internal_metrics_) {
278 rv.push_back(std::move(*im));
284 std::list<io::prometheus::client::MetricFamily>
285 MetricsThread::all_metrics()
287 std::list<io::prometheus::client::MetricFamily> metrics;
289 for (
auto &s : metrics_suppliers_) {
290 metrics.splice(metrics.begin(), std::move(s->metrics()));
300 auto i = std::find(metrics_suppliers_.begin(), metrics_suppliers_.end(), supplier);
301 if (i == metrics_suppliers_.end()) {
302 metrics_suppliers_.push_back(supplier);
310 auto i = std::find(metrics_suppliers_.begin(), metrics_suppliers_.end(), supplier);
311 if (i != metrics_suppliers_.end()) {
312 metrics_suppliers_.erase(i);
317 MetricsThread::metrics_suppliers()
const
319 return metrics_suppliers_;
323 MetricsThread::parse_labels(
const std::string &labels, io::prometheus::client::Metric *m)
325 std::vector<std::string> labelv = str_split(labels,
',');
326 for (
const std::string &l : labelv) {
327 std::vector<std::string> key_value = str_split(l,
'=');
328 if (key_value.size() == 2) {
329 io::prometheus::client::LabelPair *lp = m->add_label();
330 lp->set_name(key_value[0]);
331 lp->set_value(key_value[1]);
340 unsigned int instance_serial)
throw()
342 conditional_close(interface);
347 unsigned int instance_serial)
throw()
349 conditional_close(interface);
355 MetricFamilyInterface *mfi =
dynamic_cast<MetricFamilyInterface *
>(interface);
358 if (!mfi->has_writer())
362 if (mfi->metric_type() == MetricFamilyInterface::NOT_INITIALIZED) {
364 "Got data changed event for %s which is not yet initialized",
370 if (metric_bbs_.find(mfi->id()) == metric_bbs_.end()) {
371 logger->
log_warn(name(),
"Got data changed event for %s which is not registered", mfi->uid());
375 metric_bbs_[mfi->id()].metric_type = mfi->metric_type();
376 if (conditional_open(mfi->id(), metric_bbs_[mfi->id()])) {
377 bbil_remove_data_interface(mfi);
383 MetricsThread::conditional_open(
const std::string &
id, MetricFamilyBB &mfbb)
385 mfbb.metric_family->read();
387 std::string data_id_pattern =
id +
"/*";
389 switch (mfbb.metric_type) {
390 case MetricFamilyInterface::COUNTER: {
391 std::list<MetricCounterInterface *> ifaces =
395 std::transform(ifaces.begin(),
397 std::back_inserter(mfbb.data),
398 [](MetricCounterInterface *iface) {
399 return MetricFamilyData{.counter = iface};
403 case MetricFamilyInterface::GAUGE: {
404 std::list<MetricGaugeInterface *> ifaces =
408 std::transform(ifaces.begin(),
410 std::back_inserter(mfbb.data),
411 [](MetricGaugeInterface *iface) { return MetricFamilyData{.gauge = iface}; });
414 case MetricFamilyInterface::UNTYPED: {
415 std::list<MetricUntypedInterface *> ifaces =
419 std::transform(ifaces.begin(),
421 std::back_inserter(mfbb.data),
422 [](MetricUntypedInterface *iface) {
423 return MetricFamilyData{.untyped = iface};
427 case MetricFamilyInterface::HISTOGRAM: {
428 std::list<MetricHistogramInterface *> ifaces =
432 std::transform(ifaces.begin(),
434 std::back_inserter(mfbb.data),
435 [](MetricHistogramInterface *iface) {
436 return MetricFamilyData{.histogram = iface};
440 case MetricFamilyInterface::NOT_INITIALIZED:
441 logger->
log_info(name(),
"Metric family %s not yet initialized",
id.c_str());
445 logger->
log_info(name(),
"Initialized metric %s",
id.c_str());
450 MetricsThread::conditional_close(
Interface *interface)
throw()
452 MetricFamilyInterface *mfi =
dynamic_cast<MetricFamilyInterface *
>(interface);
458 if (metric_bbs_.find(mfi->id()) == metric_bbs_.end()) {
459 logger->
log_warn(name(),
"Called to close %s whic was not opened", mfi->uid());
463 logger->
log_info(name(),
"Last on metric family %s, closing", interface->id());
464 auto &mfbb(metric_bbs_[mfi->id()]);
466 switch (mfbb.metric_type) {
467 case MetricFamilyInterface::COUNTER:
468 std::for_each(mfbb.data.begin(), mfbb.data.end(), [
this](
auto &d) {
469 this->blackboard->close(d.counter);
473 case MetricFamilyInterface::GAUGE:
474 std::for_each(mfbb.data.begin(), mfbb.data.end(), [
this](
auto &d) {
475 this->blackboard->close(d.gauge);
478 case MetricFamilyInterface::UNTYPED:
479 std::for_each(mfbb.data.begin(), mfbb.data.end(), [
this](
auto &d) {
480 this->blackboard->close(d.untyped);
483 case MetricFamilyInterface::HISTOGRAM:
484 std::for_each(mfbb.data.begin(), mfbb.data.end(), [
this](
auto &d) {
485 this->blackboard->close(d.histogram);
488 case MetricFamilyInterface::NOT_INITIALIZED: bbil_remove_data_interface(mfi);
break;
491 metric_bbs_.erase(mfi->id());
494 std::string uid = interface->uid();
496 bbil_remove_reader_interface(interface);
497 bbil_remove_writer_interface(interface);
499 blackboard->
close(interface);
501 logger->
log_error(name(),
"Failed to unregister or close %s: %s", uid.c_str(), e.
what());