res_prometheus: Add CLI commands

This patch adds a few CLI commands to the res_prometheus module to aid
system administrators setting up and configuring the module. This includes:

* prometheus show status: Display basic statistics about the Prometheus
  module, including its essential configuration, when it was last scraped,
  and how long the scrape took. The last two bits of information are useful
  when Prometheus isn't generating metrics appropriately, as it will at
  least tell you if Asterisk has had its HTTP route hit by the remote
  server.

* prometheus show metrics: Dump the current metrics to the CLI. Useful for
  system administrators to see what metrics are currently available without
  having to cURL or go to Prometheus itself.

ASTERISK-28403

Change-Id: Ic09813e5e14b901571c5c96ebeae2a02566c5172

res/prometheus/bridges.c

@@ -145,7 +145,6 @@ static void bridges_scrape_cb(struct ast_str **response)
 		prometheus_metric_to_string(&bridge_metrics[j], response);
 	}
 
-
 	ast_free(bridge_metrics);
 	ao2_ref(bridges, -1);
 }

res/prometheus/cli.c

@@ -0,0 +1,143 @@
+/*
+ * Asterisk -- An open source telephony toolkit.
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+/*!
+ * \file
+ * \brief Prometheus CLI Commands
+ *
+ * \author Matt Jordan <mjordan@digium.com>
+ *
+ */
+#include "asterisk.h"
+
+#include "asterisk/cli.h"
+#include "asterisk/localtime.h"
+#include "asterisk/res_prometheus.h"
+#include "prometheus_internal.h"
+
+static char *prometheus_show_metrics(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
+{
+	struct ast_str *response;
+
+	if (cmd == CLI_INIT) {
+		e->command = "prometheus show metrics";
+		e->usage =
+			"Usage: prometheus show metrics\n"
+			"       Displays the current metrics and their values,\n"
+			"       without counting as an actual scrape.\n";
+			return NULL;
+	} else if (cmd == CLI_GENERATE) {
+		return NULL;
+	}
+
+	if (a->argc != 3) {
+		return CLI_SHOWUSAGE;
+	}
+
+	response = prometheus_scrape_to_string();
+	if (!response) {
+		ast_cli(a->fd, "Egads! An unknown error occurred getting the metrics\n");
+		return CLI_FAILURE;
+	}
+	ast_cli(a->fd, "%s\n", ast_str_buffer(response));
+	ast_free(response);
+
+	return CLI_SUCCESS;
+}
+
+static char *prometheus_show_status(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
+{
+	struct prometheus_general_config *config;
+	char time_buffer[64];
+	struct ast_tm last_scrape_local;
+	struct timeval last_scrape_time;
+	int64_t scrape_duration;
+
+	if (cmd == CLI_INIT) {
+		e->command = "prometheus show status";
+		e->usage =
+			"Usage: prometheus show status\n"
+			"       Displays the status of metrics collection.\n";
+		return NULL;
+	} else if (cmd == CLI_GENERATE) {
+		return NULL;
+	}
+
+	if (a->argc != 3) {
+		return CLI_SHOWUSAGE;
+	}
+
+	config = prometheus_general_config_get();
+
+	ast_cli(a->fd, "Prometheus Metrics Status:\n");
+	ast_cli(a->fd, "\tEnabled: %s\n", config->enabled ? "Yes" : "No");
+	ast_cli(a->fd, "\tURI: %s\n", config->uri);
+	ast_cli(a->fd, "\tBasic Auth: %s\n", ast_strlen_zero(config->auth_username) ? "No": "Yes");
+	ast_cli(a->fd, "\tLast Scrape Time: ");
+	last_scrape_time = prometheus_last_scrape_time_get();
+	if (last_scrape_time.tv_sec == 0 && last_scrape_time.tv_usec == 0) {
+		snprintf(time_buffer, sizeof(time_buffer), "%s", "(N/A)");
+	} else {
+		ast_localtime(&last_scrape_time, &last_scrape_local, NULL);
+		ast_strftime(time_buffer, sizeof(time_buffer), "%Y-%m-%d %H:%M:%S", &last_scrape_local);
+	}
+	ast_cli(a->fd, "%s\n", time_buffer);
+
+	ast_cli(a->fd, "\tLast Scrape Duration: ");
+	scrape_duration = prometheus_last_scrape_duration_get();
+	if (scrape_duration < 0) {
+		ast_cli(a->fd, "(N/A)\n");
+	} else {
+		ast_cli(a->fd, "%" PRIu64 " ms\n", scrape_duration);
+	}
+
+	ao2_ref(config, -1);
+
+	return CLI_SUCCESS;
+}
+
+static struct ast_cli_entry cli_prometheus[] = {
+	AST_CLI_DEFINE(prometheus_show_metrics, "Display the current metrics and their values"),
+	AST_CLI_DEFINE(prometheus_show_status, "Display the status of Prometheus metrics collection"),
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when the core module is unloaded
+ */
+static void cli_unload_cb(void)
+{
+	ast_cli_unregister_multiple(cli_prometheus, ARRAY_LEN(cli_prometheus));
+}
+
+/*!
+ * \internal
+ * \brief Provider definition
+ */
+static struct prometheus_metrics_provider provider = {
+	.name = "cli",
+	.unload_cb = cli_unload_cb,
+};
+
+int cli_init(void)
+{
+	prometheus_metrics_provider_register(&provider);
+	ast_cli_register_multiple(cli_prometheus, ARRAY_LEN(cli_prometheus));
+
+	return 0;
+}

res/prometheus/endpoints.c

@@ -154,7 +154,6 @@ static void endpoints_scrape_cb(struct ast_str **response)
 				AST_LIST_INSERT_TAIL(&endpoint_metrics[j].children, &endpoint_metrics[index], entry);
 			}
 		}
-		ao2_ref(snapshot, -1);
 	}
 	ao2_iterator_destroy(&it_endpoints);
 

res/prometheus/prometheus_internal.h

@@ -30,6 +30,43 @@
  * want to register metrics / handlers with the core API.
  */
 
+/*!
+ * \brief Retrieve the amount of time it took to perform the last scrape
+ *
+ * \details Time returned is in milliseconds
+ *
+ * \retval The scrape duration, in milliseconds
+ */
+int64_t prometheus_last_scrape_duration_get(void);
+
+/*!
+ * \brief Retrieve the timestamp when the last scrape occurred
+ *
+ * \retval The time when the last scrape occurred
+ */
+struct timeval prometheus_last_scrape_time_get(void);
+
+/*!
+ * \brief Get the raw output of what a scrape would produce
+ *
+ * \details
+ * It can be useful to dump what a scrape will look like.
+ * This function returns the raw string representation
+ * of the metrics.
+ *
+ * \retval NULL on error
+ * \retval Malloc'd ast_str on success
+ */
+struct ast_str *prometheus_scrape_to_string(void);
+
+/*!
+ * \brief Initialize CLI command
+ *
+ * \retval 0 success
+ * \retval -1 error
+ */
+int cli_init(void);
+
 /*!
  * \brief Initialize channel metrics
  *

res/res_prometheus.c

@@ -139,6 +139,8 @@ AST_VECTOR(, struct prometheus_callback *) callbacks;
 
 AST_VECTOR(, const struct prometheus_metrics_provider *) providers;
 
+static struct timeval last_scrape;
+
 /*! \brief The actual module config */
 struct module_config {
 	/*! \brief General settings */
@@ -556,6 +558,36 @@ void prometheus_callback_unregister(struct prometheus_callback *callback)
 	}
 }
 
+static void scrape_metrics(struct ast_str **response)
+{
+	int i;
+
+	for (i = 0; i < AST_VECTOR_SIZE(&callbacks); i++) {
+		struct prometheus_callback *callback = AST_VECTOR_GET(&callbacks, i);
+
+		if (!callback) {
+			continue;
+		}
+
+		callback->callback_fn(response);
+	}
+
+	for (i = 0; i < AST_VECTOR_SIZE(&metrics); i++) {
+		struct prometheus_metric *metric = AST_VECTOR_GET(&metrics, i);
+
+		if (!metric) {
+			continue;
+		}
+
+		ast_mutex_lock(&metric->lock);
+		if (metric->get_metric_value) {
+			metric->get_metric_value(metric);
+		}
+		prometheus_metric_to_string(metric, response);
+		ast_mutex_unlock(&metric->lock);
+	}
+}
+
 static int http_callback(struct ast_tcptls_session_instance *ser,
 	const struct ast_http_uri *urih, const char *uri, enum ast_http_method method,
 	struct ast_variable *get_params, struct ast_variable *headers)
@@ -564,7 +596,6 @@ static int http_callback(struct ast_tcptls_session_instance *ser,
 	struct ast_str *response = NULL;
 	struct timeval start;
 	struct timeval end;
-	int i;
 
 	/* If there is no module config or we're not enabled, we can't handle requests */
 	if (!mod_cfg || !mod_cfg->general->enabled) {
@@ -599,27 +630,12 @@ static int http_callback(struct ast_tcptls_session_instance *ser,
 		goto err500;
 	}
 
-	if (mod_cfg->general->core_metrics_enabled) {
-		start = ast_tvnow();
-	}
+	start = ast_tvnow();
 
 	ast_mutex_lock(&scrape_lock);
-	for (i = 0; i < AST_VECTOR_SIZE(&callbacks); i++) {
-		struct prometheus_callback *callback = AST_VECTOR_GET(&callbacks, i);
-
-		callback->callback_fn(&response);
-	}
-
-	for (i = 0; i < AST_VECTOR_SIZE(&metrics); i++) {
-		struct prometheus_metric *metric = AST_VECTOR_GET(&metrics, i);
 
-		ast_mutex_lock(&metric->lock);
-		if (metric->get_metric_value) {
-			metric->get_metric_value(metric);
-		}
-		prometheus_metric_to_string(metric, &response);
-		ast_mutex_unlock(&metric->lock);
-	}
+	last_scrape = start;
+	scrape_metrics(&response);
 
 	if (mod_cfg->general->core_metrics_enabled) {
 		int64_t duration;
@@ -664,6 +680,40 @@ err500:
 	return 0;
 }
 
+struct ast_str *prometheus_scrape_to_string(void)
+{
+	struct ast_str *response;
+
+	response = ast_str_create(512);
+	if (!response) {
+		return NULL;
+	}
+
+	ast_mutex_lock(&scrape_lock);
+	scrape_metrics(&response);
+	ast_mutex_unlock(&scrape_lock);
+
+	return response;
+}
+
+int64_t prometheus_last_scrape_duration_get(void)
+{
+	int64_t duration;
+
+	if (sscanf(core_scrape_metric.value, "%" PRIu64, &duration) != 1) {
+		return -1;
+	}
+
+	return duration;
+}
+
+struct timeval prometheus_last_scrape_time_get(void)
+{
+	SCOPED_MUTEX(lock, &scrape_lock);
+
+	return last_scrape;
+}
+
 static void prometheus_general_config_dtor(void *obj)
 {
 	struct prometheus_general_config *config = obj;
@@ -846,7 +896,9 @@ static int unload_module(void)
 	AST_VECTOR_FREE(&metrics);
 
 	AST_VECTOR_FREE(&callbacks);
+
 	AST_VECTOR_FREE(&providers);
+
 	aco_info_destroy(&cfg_info);
 	ao2_global_obj_release(global_config);
 
@@ -917,7 +969,8 @@ static int load_module(void)
 		goto cleanup;
 	}
 
-	if (channel_metrics_init()
+	if (cli_init()
+		|| channel_metrics_init()
 		|| endpoint_metrics_init()
 		|| bridge_metrics_init()) {
 		goto cleanup;