diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index 1cdd0db35da..4f298910674 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 0) set(CMT_VERSION_MINOR 9) -set(CMT_VERSION_PATCH 5) +set(CMT_VERSION_PATCH 6) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers @@ -69,12 +69,18 @@ endif() # Configuration options option(CMT_DEV "Enable development mode" No) +option(CMT_DEBUG "Enable debug mode" No) option(CMT_TESTS "Enable unit testing" No) option(CMT_INSTALL_TARGETS "Enable subdirectory library installations" Yes) option(CMT_ENABLE_PROMETHEUS_DECODER "Enable prometheus decoder" Yes) if(CMT_DEV) set(CMT_TESTS Yes) + set(CMT_DEBUG Yes) +endif() + +if(CMT_DEBUG) + set(CMAKE_BUILD_TYPE Debug) endif() set(FLEX_BISON_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/lib/cmetrics/include/cmetrics/cmt_math.h b/lib/cmetrics/include/cmetrics/cmt_math.h index 6f7e3f18fb6..91383818275 100644 --- a/lib/cmetrics/include/cmetrics/cmt_math.h +++ b/lib/cmetrics/include/cmetrics/cmt_math.h @@ -49,4 +49,14 @@ static inline double cmt_math_uint64_to_d64(uint64_t val) return u.d; } +static inline uint64_t cmt_math_sum_native_uint64_as_d64(uint64_t dst, uint64_t src) +{ + double val; + + val = cmt_math_uint64_to_d64(dst); + val += cmt_math_uint64_to_d64(src); + + return cmt_math_d64_to_uint64(val); +} + #endif diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index b2171196086..5ef28d4f831 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -96,9 +96,66 @@ static int copy_label_values(struct cmt_metric *metric, char **out) return i; } -int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) +static inline int cat_histogram_values(struct cmt_metric *metric_dst, struct cmt_histogram *histogram, + struct cmt_metric *metric_src) { int i; + + if (!metric_dst->hist_buckets) { + metric_dst->hist_buckets = calloc(1, sizeof(uint64_t) * (histogram->buckets->count + 1)); + if (!metric_dst->hist_buckets) { + return -1; + } + } + + for (i = 0; i < histogram->buckets->count; i++) { + /* histogram buckets are always integers, no need to convert them */ + metric_dst->hist_buckets[i] += metric_src->hist_buckets[i]; + } + + /* histogram count */ + metric_dst->hist_count = cmt_math_sum_native_uint64_as_d64(metric_dst->hist_count, + metric_src->hist_count); + + /* histoggram sum */ + metric_dst->hist_sum = cmt_math_sum_native_uint64_as_d64(metric_dst->hist_sum, + metric_src->hist_sum); + + return 0; +} + +/* + * For summaries we don't support manual updates through the API, on concatenation we just + * keep the last values reported. + */ +static inline int cat_summary_values(struct cmt_metric *metric_dst, struct cmt_summary *summary, + struct cmt_metric *metric_src) +{ + int i; + + if (!metric_dst->sum_quantiles) { + metric_dst->sum_quantiles = calloc(1, sizeof(uint64_t) * (summary->quantiles_count)); + if (!metric_dst->sum_quantiles) { + return -1; + } + } + + for (i = 0; i < summary->quantiles_count; i++) { + /* summary quantiles are always integers, no need to convert them */ + metric_dst->sum_quantiles[i] = metric_src->sum_quantiles[i]; + } + + metric_dst->sum_quantiles_count = metric_src->sum_quantiles_count; + metric_dst->sum_quantiles_set = metric_src->sum_quantiles_set; + + metric_dst->sum_count = metric_src->sum_count; + metric_dst->sum_sum = metric_src->sum_sum; + + return 0; +} + +int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) +{ int c; int ret; uint64_t ts; @@ -107,6 +164,7 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cfl_list *head; struct cmt_metric *metric_dst; struct cmt_metric *metric_src; + struct cmt_summary *summary; struct cmt_histogram *histogram; /* Handle static metric (no labels case) */ @@ -119,33 +177,17 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map if (src->type == CMT_HISTOGRAM) { histogram = (struct cmt_histogram *) src->parent; - - if (!metric_dst->hist_buckets) { - metric_dst->hist_buckets = calloc(1, sizeof(uint64_t) * (histogram->buckets->count + 1)); - if (!metric_dst->hist_buckets) { - return -1; - } - } - for (i = 0; i < histogram->buckets->count; i++) { - metric_dst->hist_buckets[i] = metric_src->hist_buckets[i]; + ret = cat_histogram_values(metric_dst, histogram, metric_src); + if (ret == -1) { + return -1; } - metric_dst->hist_count = metric_src->hist_count; - metric_dst->hist_sum = metric_src->hist_sum; } else if (src->type == CMT_SUMMARY) { - metric_dst->sum_quantiles_count = metric_src->sum_quantiles_count; - metric_dst->sum_quantiles_set = metric_src->sum_quantiles_set; - if (!metric_dst->sum_quantiles) { - metric_dst->sum_quantiles = calloc(1, sizeof(uint64_t) * (metric_src->sum_quantiles_count)); - if (!metric_dst->sum_quantiles) { - return -1; - } - } - for (i = 0; i < metric_src->sum_quantiles_count; i++) { - metric_dst->sum_quantiles[i] = metric_src->sum_quantiles[i]; + summary = (struct cmt_summary *) src->parent; + ret = cat_summary_values(metric_dst, summary, metric_src); + if (ret == -1) { + return -1; } - metric_dst->sum_count = metric_src->sum_count; - metric_dst->sum_sum = metric_src->sum_sum; } ts = cmt_metric_get_timestamp(metric_src); @@ -173,34 +215,17 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map if (src->type == CMT_HISTOGRAM) { histogram = (struct cmt_histogram *) src->parent; - - if (!metric_dst->hist_buckets) { - metric_dst->hist_buckets = calloc(1, sizeof(uint64_t) * (histogram->buckets->count + 1)); - if (!metric_dst->hist_buckets) { - return -1; - } - } - - for (i = 0; i < histogram->buckets->count; i++) { - metric_dst->hist_buckets[i] = metric_src->hist_buckets[i]; + ret = cat_histogram_values(metric_dst, histogram, metric_src); + if (ret == -1) { + return -1; } - metric_dst->hist_count = metric_src->hist_count; - metric_dst->hist_sum = metric_src->hist_sum; } else if (src->type == CMT_SUMMARY) { - metric_dst->sum_quantiles_count = metric_src->sum_quantiles_count; - metric_dst->sum_quantiles_set = metric_src->sum_quantiles_set; - if (!metric_dst->sum_quantiles) { - metric_dst->sum_quantiles = calloc(1, sizeof(uint64_t) * (metric_src->sum_quantiles_count)); - if (!metric_dst->sum_quantiles) { - return -1; - } + summary = (struct cmt_summary *) src->parent; + ret = cat_summary_values(metric_dst, summary, metric_src); + if (ret == -1) { + return -1; } - for (i = 0; i < metric_src->sum_quantiles_count; i++) { - metric_dst->sum_quantiles[i] = metric_src->sum_quantiles[i]; - } - metric_dst->sum_count = metric_src->sum_count; - metric_dst->sum_sum = metric_src->sum_sum; } ts = cmt_metric_get_timestamp(metric_src); @@ -213,6 +238,88 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } +static inline int cmt_opts_compare(struct cmt_opts *a, struct cmt_opts *b) +{ + int ret; + + ret = strcmp(a->ns, a->ns); + if (ret != 0) { + return ret; + } + + ret = strcmp(a->subsystem, b->subsystem); + if (ret != 0) { + return ret; + } + + ret = strcmp(a->name, b->name); + if (ret != 0) { + return ret; + } + + return strcmp(a->description, b->description); +} + +static struct cmt_counter *counter_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_counter *counter; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + if (cmt_opts_compare(&counter->opts, opts) == 0) { + return counter; + } + } + + return NULL; +} + +static struct cmt_gauge *gauge_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_gauge *gauge; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + if (cmt_opts_compare(&gauge->opts, opts) == 0) { + return gauge; + } + } + + return NULL; +} + +static struct cmt_untyped *untyped_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_untyped *untyped; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + if (cmt_opts_compare(&untyped->opts, opts) == 0) { + return untyped; + } + } + + return NULL; +} + +static struct cmt_histogram *histogram_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_histogram *histogram; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + if (cmt_opts_compare(&histogram->opts, opts) == 0) { + return histogram; + } + } + + return NULL; +} + int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { @@ -230,11 +337,14 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, return -1; } - /* create counter */ - c = cmt_counter_create(cmt, - opts->ns, opts->subsystem, - opts->name, opts->description, - map->label_count, labels); + c = counter_lookup(cmt, opts); + if (!c) { + /* create counter */ + c = cmt_counter_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + map->label_count, labels); + } free(labels); if (!c) { @@ -274,11 +384,15 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, return -1; } - /* create counter */ - g = cmt_gauge_create(cmt, - opts->ns, opts->subsystem, - opts->name, opts->description, - map->label_count, labels); + g = gauge_lookup(cmt, opts); + if (!g) { + /* create counter */ + g = cmt_gauge_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + map->label_count, labels); + } + free(labels); if (!g) { return -1; @@ -317,11 +431,15 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, return -1; } - /* create counter */ - u = cmt_untyped_create(cmt, - opts->ns, opts->subsystem, - opts->name, opts->description, - map->label_count, labels); + u = untyped_lookup(cmt, opts); + if (!u) { + /* create counter */ + u = cmt_untyped_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + map->label_count, labels); + } + free(labels); if (!u) { return -1; @@ -362,16 +480,19 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, return -1; } - buckets_count = histogram->buckets->count; - buckets = cmt_histogram_buckets_create_size(histogram->buckets->upper_bounds, - buckets_count); + hist = histogram_lookup(cmt, opts); + if (!hist) { + buckets_count = histogram->buckets->count; + buckets = cmt_histogram_buckets_create_size(histogram->buckets->upper_bounds, + buckets_count); - /* create histogram */ - hist = cmt_histogram_create(cmt, - opts->ns, opts->subsystem, - opts->name, opts->description, - buckets, - map->label_count, labels); + /* create histogram */ + hist = cmt_histogram_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + buckets, + map->label_count, labels); + } free(labels); if (!hist) { diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index c1bf982e9d2..a41d7945522 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "cmt_tests.h" @@ -232,7 +233,166 @@ void test_cat() cmt_destroy(cmt5); } + +void test_duplicate_metrics() +{ + int i; + int ret; + double val; + struct cmt *cmt1; + struct cmt *cmt2; + struct cmt *final; + struct cmt_counter *c; + struct cmt_gauge *g; + struct cmt_untyped *u; + struct cmt_summary *s; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets1; + struct cmt_histogram_buckets *buckets2; + double sum; + int count; + uint64_t ts; + + cfl_sds_t text; + + /* context 1 */ + cmt1 = cmt_create(); + TEST_CHECK(cmt1 != NULL); + + c = cmt_counter_create(cmt1, "cmetrics", "test", "cat_counter", "first counter", + 2, (char *[]) {"label1", "label2"}); + TEST_CHECK(c != NULL); + cmt_counter_set(c, cfl_time_now(), 10, 0, NULL ); + cmt_counter_inc(c, cfl_time_now(), 2, (char *[]) {"aaa", "bbb"}); + + + g = cmt_gauge_create(cmt1, "cmetrics", "test", "cat_gauge", "first gauge", + 2, (char *[]) {"label3", "label4"}); + TEST_CHECK(g != NULL); + cmt_gauge_inc(g, cfl_time_now(), 2, (char *[]) {"yyy", "xxx"}); + + u = cmt_untyped_create(cmt1, "cmetrics", "test", "cat_untyped", "first untyped", + 2, (char *[]) {"label5", "label6"}); + TEST_CHECK(u != NULL); + cmt_untyped_set(u, cfl_time_now(), 10, 2, (char *[]) {"qwe", "asd"}); + + s = cmt_summary_create(cmt1, + "spring", "kafka_listener", "seconds", "Kafka Listener Timer", + 6, (double[]) {0.1, 0.2, 0.3, 0.4, 0.5, 1.0}, + 3, (char *[]) {"exception", "name", "result"}); + + ts = cfl_time_now(); + + /* Summary + * ------- + */ + /* no quantiles, labels */ + sum = 0.0; + count = 1; + + cmt_summary_set_default(s, ts, NULL, sum, count, + 3, (char *[]) {"ListenerExecutionFailedException", + "org.springframework.kafka.KafkaListenerEndpointContainer#0-0", + "failure"}); + + /* no quantiles, labels */ + sum = 0.1; + count = 2; + cmt_summary_set_default(s, ts, NULL, sum, count, + 3, (char *[]) {"none", + "org.springframework.kafka.KafkaListenerEndpointContainer#0-0", + "success"}); + + /* quantiles, labels */ + sum = 0.2; + count = 3; + cmt_summary_set_default(s, ts, NULL, sum, count, + 3, (char *[]) {"extra test", + "org.springframework.kafka.KafkaListenerEndpointContainer#0-0", + "success"}); + + /* + * Histogram + * --------- + */ + buckets1 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + + h = cmt_histogram_create(cmt1, + "k8s", "network", "load", "Network load", + buckets1, + 1, (char *[]) {"my_label"}); + TEST_CHECK(h != NULL); + + ts = cfl_time_now(); + for (i = 0; i < sizeof(hist_observe_values)/(sizeof(double)); i++) { + val = hist_observe_values[i]; + cmt_histogram_observe(h, ts, val, 1, (char *[]) {"my_label"}); + } + + /* duplicate counter */ + cmt2 = cmt_create(); + TEST_CHECK(cmt2 != NULL); + + c = cmt_counter_create(cmt2, "cmetrics", "test", "cat_counter", "first counter", + 2, (char *[]) {"label1", "label2"}); + TEST_CHECK(c != NULL); + cmt_counter_set(c, cfl_time_now(), 11, 0, NULL ); + cmt_counter_inc(c, cfl_time_now(), 2, (char *[]) {"ddd", "eee"}); + + /* duplicate gauge */ + g = cmt_gauge_create(cmt2, "cmetrics", "test", "cat_gauge", "first gauge", + 2, (char *[]) {"label3", "label4"}); + TEST_CHECK(g != NULL); + cmt_gauge_inc(g, cfl_time_now(), 2, (char *[]) {"zzz", "xxx"}); + + /* duplicate untyped */ + u = cmt_untyped_create(cmt2, "cmetrics", "test", "cat_untyped", "first untyped", + 2, (char *[]) {"label5", "label6"}); + TEST_CHECK(u != NULL); + cmt_untyped_set(u, cfl_time_now(), 20, 2, (char *[]) {"rty", "asd"}); + + buckets2 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + h = cmt_histogram_create(cmt2, + "k8s", "network", "load", "Network load", + buckets2, + 1, (char *[]) {"my_label2"}); + TEST_CHECK(h != NULL); + + ts = cfl_time_now(); + + for (i = 0; i < sizeof(hist_observe_values)/(sizeof(double)); i++) { + val = hist_observe_values[i]; + cmt_histogram_observe(h, ts, val, 1, (char *[]) {"my_label"}); + } + + /* concatenate cmt1 + cmt2 */ + final = cmt_create(); + ret = cmt_cat(final, cmt1); + TEST_CHECK(ret == 0); + + ret = cmt_cat(final, cmt2); + TEST_CHECK(ret == 0); + + /* prometheus format */ + text = cmt_encode_prometheus_create(final, CMT_FALSE); + printf("Prometheus Text====>\n%s\n", text); + cfl_sds_destroy(text); + + + cmt_destroy(cmt1); + cmt_destroy(cmt2); + cmt_destroy(final); + +} + TEST_LIST = { {"cat", test_cat}, + {"duplicate_metrics", test_duplicate_metrics}, { 0 } }; diff --git a/lib/cmetrics/tests/prometheus_parser.c b/lib/cmetrics/tests/prometheus_parser.c index 5fa0c889886..08b3f2dc82d 100644 --- a/lib/cmetrics/tests/prometheus_parser.c +++ b/lib/cmetrics/tests/prometheus_parser.c @@ -1669,7 +1669,6 @@ void test_issue_fluent_bit_9267() { char errbuf[256]; int status; - cfl_sds_t result = NULL; struct cmt *cmt; struct cmt_decode_prometheus_parse_opts opts; memset(&opts, 0, sizeof(opts));