res_prometheus: Add Asterisk channel metrics

This patch adds basic Asterisk channel statistics to the res_prometheus
module. This includes:

* asterisk_calls_sum: A running sum of the total number of
  processed calls

* asterisk_calls_count: The current number of calls

* asterisk_channels_count: The current number of channels

* asterisk_channels_state: The state of any particular channel

* asterisk_channels_duration_seconds: How long a channel has existed,
  in seconds

In all cases, enough information is provided with each channel metric
to determine a unique instance of Asterisk that provided the data, as
well as the name, type, unique ID, and - if present - linked ID of each
channel.

ASTERISK-28403

Change-Id: I0db306ec94205d4f58d1e7fbabfe04b185869f59

configs/samples/prometheus.conf.sample

@@ -3,12 +3,9 @@
 ;
 
 ;
-; Note that this configuration file is consumed by res_prometheus, which
-; provides core functionality for serving up Asterisk statistics to a
-; Prometheus server. By default, this only includes basic information about
-; the Asterisk instance that is running. Additional modules can be loaded to
-; provide specific statistics. In all cases, configuration of said statistics
-; is done through this configuration file.
+; This configuration file is consumed by res_prometheus, which
+; provides the functionality for serving up Asterisk statistics to a
+; Prometheus server.
 ;
 ; Because Prometheus scrapes statistics from HTTP servers, this module requires
 ; Asterisk's built-in HTTP server to be enabled and configured properly.

include/asterisk/res_prometheus.h

@@ -83,6 +83,36 @@ struct prometheus_general_config {
 	);
 };
 
+/*!
+ * \brief A function table for a metrics provider
+ *
+ * \details
+ * It's generally nice to separate out things that provide metrics
+ * from the core of this module. For those that want to be notified
+ * when things happen in the core module, they can provide an instance
+ * of this function table using \c prometheus_metrics_provider_register
+ * and be notified when module affecting changes occur.
+ */
+struct prometheus_metrics_provider {
+	/*!
+	 * \brief Handy name of the provider for debugging purposes
+	 */
+	const char *name;
+	/*!
+	 * \brief Reload callback
+	 *
+	 * \param config The reloaded config
+	 *
+	 * \retval 0 success
+	 * \retval -1 error
+	 */
+	int (* const reload_cb)(struct prometheus_general_config *config);
+	/*!
+	 * \brief Unload callback.
+	 */
+	void (* const unload_cb)(void);
+};
+
 /*!
  * \brief Prometheus metric type
  *
@@ -437,6 +467,13 @@ int prometheus_callback_register(struct prometheus_callback *callback);
  */
 void prometheus_callback_unregister(struct prometheus_callback *callback);
 
+/*!
+ * \brief Register a metrics provider
+ *
+ * \param provider The provider function table to register
+ */
+void prometheus_metrics_provider_register(const struct prometheus_metrics_provider *provider);
+
 /*!
  * \brief Retrieve the current configuration of the module
  *

res/Makefile

@@ -66,6 +66,7 @@ $(call MOD_ADD_C,res_stasis,$(wildcard stasis/*.c))
 $(call MOD_ADD_C,res_snmp,snmp/agent.c)
 $(call MOD_ADD_C,res_parking,$(wildcard parking/*.c))
 $(call MOD_ADD_C,res_pjsip,$(wildcard res_pjsip/*.c))
+$(call MOD_ADD_C,res_prometheus,$(wildcard prometheus/*.c))
 $(call MOD_ADD_C,res_ari,ari/cli.c ari/config.c ari/ari_websockets.c)
 $(call MOD_ADD_C,res_ari_model,ari/ari_model_validators.c)
 $(call MOD_ADD_C,res_stasis_recording,stasis_recording/stored.c)

res/prometheus/channels.c

@@ -0,0 +1,236 @@
+
+/*
+ * Asterisk -- An open source telephony toolkit.
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+/*!
+ * \file
+ * \brief Prometheus Channel Metrics
+ *
+ * \author Matt Jordan <mjordan@digium.com>
+ *
+ */
+
+#include "asterisk.h"
+#include "asterisk/res_prometheus.h"
+#include "asterisk/stasis_channels.h"
+#include "asterisk/pbx.h"
+#include "prometheus_internal.h"
+
+#define CHANNELS_STATE_HELP "Individual channel states. 0=down; 1=reserved; 2=offhook; 3=dialing; 4=ring; 5=ringing; 6=up; 7=busy; 8=dialing_offhook; 9=prering."
+
+#define CHANNELS_DURATION_HELP "Individual channel durations (in seconds)."
+
+/*!
+ * \internal
+ * \brief Callback function to get a channel's current state
+ *
+ * \param metric The metric to populate
+ * \snapshot Channel snapshot
+ */
+static void get_channel_state(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot)
+{
+	snprintf(metric->value, sizeof(metric->value), "%d", snapshot->state);
+}
+
+/*!
+ * \internal
+ * \brief Callback function to get a channel's current duration
+ *
+ * \param metric The metric to populate
+ * \param snapshot Channel snapshot
+ */
+static void get_channel_duration(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot)
+{
+	struct timeval now = ast_tvnow();
+	int64_t duration = ast_tvdiff_sec(now, snapshot->base->creationtime);
+
+	snprintf(metric->value, sizeof(metric->value), "%" PRIu64, duration);
+}
+
+/*!
+ * \internal
+ * \brief Helper struct for generating individual channel stats
+ */
+struct channel_metric_defs {
+	/*!
+	 * \brief Help text to display
+	 */
+	const char *help;
+	/*!
+	 * \brief Name of the metric
+	 */
+	const char *name;
+	/*!
+	 * \brief Callback function to generate a metric value for a given channel
+	 */
+	void (* const get_value)(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot);
+} channel_metric_defs[] = {
+	{
+		.help = CHANNELS_STATE_HELP,
+		.name = "asterisk_channels_state",
+		.get_value = get_channel_state,
+	},
+	{
+		.help = CHANNELS_DURATION_HELP,
+		.name = "asterisk_channels_duration_seconds",
+		.get_value = get_channel_duration,
+	},
+};
+
+static void get_total_call_count(struct prometheus_metric *metric)
+{
+	snprintf(metric->value, sizeof(metric->value), "%d", ast_processed_calls());
+}
+
+static void get_current_call_count(struct prometheus_metric *metric)
+{
+	snprintf(metric->value, sizeof(metric->value), "%d", ast_active_calls());
+}
+
+/*!
+ * \internal
+ * \brief Channel based metrics that are always available
+ */
+static struct prometheus_metric global_channel_metrics[] = {
+	PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+		PROMETHEUS_METRIC_COUNTER,
+		"asterisk_calls_sum",
+		"Total call count.",
+		&get_total_call_count
+	),
+	PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+		PROMETHEUS_METRIC_GAUGE,
+		"asterisk_calls_count",
+		"Current call count.",
+		&get_current_call_count
+	),
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when Prometheus scrapes the server
+ *
+ * \param response The response to populate with formatted metrics
+ */
+static void channels_scrape_cb(struct ast_str **response)
+{
+	struct ao2_container *channels;
+	struct ao2_iterator it_chans;
+	struct ast_channel_snapshot *snapshot;
+	struct prometheus_metric *channel_metrics;
+	char eid_str[32];
+	int num_channels;
+	int i, j;
+	struct prometheus_metric channel_count = PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+		PROMETHEUS_METRIC_GAUGE,
+		"asterisk_channels_count",
+		"Current channel count.",
+		NULL
+	);
+
+	ast_eid_to_str(eid_str, sizeof(eid_str), &ast_eid_default);
+
+	channels = ast_channel_cache_all();
+	num_channels = ao2_container_count(channels);
+
+	/* Channel count */
+	PROMETHEUS_METRIC_SET_LABEL(&channel_count, 0, "eid", eid_str);
+	snprintf(channel_count.value, sizeof(channel_count.value), "%d", num_channels);
+	prometheus_metric_to_string(&channel_count, response);
+
+	/* Global call values */
+	for (i = 0; i < ARRAY_LEN(global_channel_metrics); i++) {
+		PROMETHEUS_METRIC_SET_LABEL(&global_channel_metrics[i], 0, "eid", eid_str);
+		global_channel_metrics[i].get_metric_value(&global_channel_metrics[i]);
+		prometheus_metric_to_string(&global_channel_metrics[i], response);
+	}
+
+	if (num_channels == 0) {
+		ao2_ref(channels, -1);
+		return;
+	}
+
+	/* Channel dependent values */
+	channel_metrics = ast_calloc(ARRAY_LEN(channel_metric_defs) * num_channels, sizeof(*channel_metrics));
+	if (!channel_metrics) {
+		ao2_ref(channels, -1);
+		return;
+	}
+
+	it_chans = ao2_iterator_init(channels, 0);
+	for (i = 0; (snapshot = ao2_iterator_next(&it_chans)); ao2_ref(snapshot, -1), i++) {
+		for (j = 0; j < ARRAY_LEN(channel_metric_defs); j++) {
+			int index = i * ARRAY_LEN(channel_metric_defs) + j;
+
+			channel_metrics[index].type = PROMETHEUS_METRIC_GAUGE;
+			ast_copy_string(channel_metrics[index].name, channel_metric_defs[j].name, sizeof(channel_metrics[index].name));
+			channel_metrics[index].help = channel_metric_defs[j].help;
+			PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 0, "eid", eid_str);
+			PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 1, "name", (snapshot->base->name));
+			PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 2, "id", (snapshot->base->uniqueid));
+			PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 3, "type", (snapshot->base->type));
+			if (snapshot->peer) {
+				PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 4, "linkedid", (snapshot->peer->linkedid));
+			}
+			channel_metric_defs[j].get_value(&channel_metrics[index], snapshot);
+
+			if (i > 0) {
+				AST_LIST_INSERT_TAIL(&channel_metrics[j].children, &channel_metrics[index], entry);
+			}
+		}
+	}
+	ao2_iterator_destroy(&it_chans);
+
+	for (j = 0; j < ARRAY_LEN(channel_metric_defs); j++) {
+		prometheus_metric_to_string(&channel_metrics[j], response);
+	}
+
+	ast_free(channel_metrics);
+	ao2_ref(channels, -1);
+}
+
+struct prometheus_callback channels_callback = {
+	.name = "Channels callback",
+	.callback_fn = channels_scrape_cb,
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when the core module is unloaded
+ */
+static void channel_metrics_unload_cb(void)
+{
+	prometheus_callback_unregister(&channels_callback);
+}
+
+/*!
+ * \internal
+ * \brief Metrics provider definition
+ */
+static struct prometheus_metrics_provider provider = {
+	.name = "channels",
+	.unload_cb = channel_metrics_unload_cb,
+};
+
+int channel_metrics_init(void)
+{
+	prometheus_metrics_provider_register(&provider);
+	prometheus_callback_register(&channels_callback);
+
+	return 0;
+}

res/prometheus/prometheus_internal.h

@@ -0,0 +1,41 @@
+/*
+ * Prometheus Internal API
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+#ifndef PROMETHEUS_INTERNAL_H__
+#define PROMETHEUS_INTERNAL_H__
+
+/*!
+ * \file prometheus_internal
+ *
+ * \brief Prometheus Metric Internal API
+ *
+ * This module provides internal APIs for \file res_prometheus.
+ * It should not be used outsize of that module, and should
+ * typically only provide intialization functions for units that
+ * want to register metrics / handlers with the core API.
+ */
+
+/*!
+ * \brief Initialize channel metrics
+ *
+ * \retval 0 success
+ * \retval -1 error
+ */
+int channel_metrics_init(void);
+
+#endif /* #define PROMETHEUS_INTERNAL_H__ */

res/res_prometheus.c

@@ -128,6 +128,8 @@
 #include "asterisk/buildinfo.h"
 #include "asterisk/res_prometheus.h"
 
+#include "prometheus/prometheus_internal.h"
+
 /*! \brief Lock that protects data structures during an HTTP scrape */
 AST_MUTEX_DEFINE_STATIC(scrape_lock);
 
@@ -135,6 +137,8 @@ AST_VECTOR(, struct prometheus_metric *) metrics;
 
 AST_VECTOR(, struct prometheus_callback *) callbacks;
 
+AST_VECTOR(, const struct prometheus_metrics_provider *) providers;
+
 /*! \brief The actual module config */
 struct module_config {
 	/*! \brief General settings */
@@ -812,6 +816,11 @@ static void prometheus_config_post_apply(void)
 	}
 }
 
+void prometheus_metrics_provider_register(const struct prometheus_metrics_provider *provider)
+{
+	AST_VECTOR_APPEND(&providers, provider);
+}
+
 static int unload_module(void)
 {
 	SCOPED_MUTEX(lock, &scrape_lock);
@@ -819,6 +828,16 @@ static int unload_module(void)
 
 	ast_http_uri_unlink(&prometheus_uri);
 
+	for (i = 0; i < AST_VECTOR_SIZE(&providers); i++) {
+		const struct prometheus_metrics_provider *provider = AST_VECTOR_GET(&providers, i);
+
+		if (!provider->unload_cb) {
+			continue;
+		}
+
+		provider->unload_cb();
+	}
+
 	for (i = 0; i < AST_VECTOR_SIZE(&metrics); i++) {
 		struct prometheus_metric *metric = AST_VECTOR_GET(&metrics, i);
 
@@ -827,7 +846,7 @@ static int unload_module(void)
 	AST_VECTOR_FREE(&metrics);
 
 	AST_VECTOR_FREE(&callbacks);
-
+	AST_VECTOR_FREE(&providers);
 	aco_info_destroy(&cfg_info);
 	ao2_global_obj_release(global_config);
 
@@ -836,11 +855,31 @@ static int unload_module(void)
 
 static int reload_module(void) {
 	SCOPED_MUTEX(lock, &scrape_lock);
+	int i;
+	struct prometheus_general_config *general_config;
 
 	ast_http_uri_unlink(&prometheus_uri);
 	if (aco_process_config(&cfg_info, 1) == ACO_PROCESS_ERROR) {
 		return -1;
 	}
+
+	/* Our config should be all reloaded now */
+	general_config = prometheus_general_config_get();
+	for (i = 0; i < AST_VECTOR_SIZE(&providers); i++) {
+		const struct prometheus_metrics_provider *provider = AST_VECTOR_GET(&providers, i);
+
+		if (!provider->reload_cb) {
+			continue;
+		}
+
+		if (provider->reload_cb(general_config)) {
+			ast_log(AST_LOG_WARNING, "Failed to reload metrics provider %s\n", provider->name);
+			ao2_ref(general_config, -1);
+			return -1;
+		}
+	}
+	ao2_ref(general_config, -1);
+
 	if (ast_http_uri_link(&prometheus_uri)) {
 		ast_log(AST_LOG_WARNING, "Failed to re-register Prometheus Metrics URI during reload\n");
 		return -1;
@@ -861,6 +900,10 @@ static int load_module(void)
 		goto cleanup;
 	}
 
+	if (AST_VECTOR_INIT(&providers, 8)) {
+		goto cleanup;
+	}
+
 	if (aco_info_init(&cfg_info)) {
 		goto cleanup;
 	}
@@ -874,6 +917,10 @@ static int load_module(void)
 		goto cleanup;
 	}
 
+	if (channel_metrics_init()) {
+		goto cleanup;
+	}
+
 	if (ast_http_uri_link(&prometheus_uri)) {
 		goto cleanup;
 	}
@@ -885,6 +932,7 @@ cleanup:
 	aco_info_destroy(&cfg_info);
 	AST_VECTOR_FREE(&metrics);
 	AST_VECTOR_FREE(&callbacks);
+	AST_VECTOR_FREE(&providers);
 
 	return AST_MODULE_LOAD_DECLINE;
 }

tests/test_res_prometheus.c

@@ -173,13 +173,13 @@ AST_TEST_DEFINE(metric_values)
 	}
 
 	ast_test_status_update(test, " -> Retrieved: %s\n", ast_str_buffer(buffer));
-	ast_test_validate_cleanup(test, strcmp(ast_str_buffer(buffer),
+	ast_test_validate_cleanup(test, strstr(ast_str_buffer(buffer),
 		"# HELP test_counter_one A test counter\n"
 		"# TYPE test_counter_one counter\n"
 		"test_counter_one 1\n"
 		"# HELP test_counter_two A test counter\n"
 		"# TYPE test_counter_two counter\n"
-		"test_counter_two 2\n") == 0, result, metric_values_cleanup);
+		"test_counter_two 2\n") != NULL, result, metric_values_cleanup);
 
 metric_values_cleanup:
 	prometheus_metric_unregister(&test_counter_one);
@@ -247,10 +247,10 @@ AST_TEST_DEFINE(metric_callback_register)
 	}
 
 	ast_test_status_update(test, " -> Retrieved: %s\n", ast_str_buffer(buffer));
-	ast_test_validate(test, strcmp(ast_str_buffer(buffer),
+	ast_test_validate(test, strstr(ast_str_buffer(buffer),
 		"# HELP test_counter A test counter\n"
 		"# TYPE test_counter counter\n"
-		"test_counter 0\n") == 0);
+		"test_counter 0\n") != NULL);
 
 	prometheus_callback_unregister(&callback);