TT#71030 Add Prometheus support

Change-Id: I7812836036d8b1a7341af390d2a34016af47d236

debian/control

@@ -19,7 +19,7 @@ Depends:
  ${misc:Depends},
 Suggests:
  asterisk,
- influxdb-client,
+ influxdb-client | prometheus,
  mariadb-server,
  snmp,
  sqlite3,

templates/410_process-ngcp.yaml.tt2

@@ -60,7 +60,35 @@ process:
     running: {{if and [% is_proxy ? 1 : 0 %] .Vars.NODE_ACTIVE}} true {{else}} false {{end}}
 
   influxd:
-    running: true
+    running: [% monitoring.backend == 'influxdb' || monitoring.influxdb_migrated == 'no' ? 'true' : 'false' %]
+
+  prometheus:
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: prometheus-mysqld-exporter.
+  prometheus-mysq:
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: prometheus-nginx-exporter.
+  prometheus-ngin:
+    running: [% monitoring.backend == 'prometheus' && (is_mgmt || is_lb || is_li_dist) && nginx.enable ? 'true' : 'false' %]
+
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: prometheus-node-exporter.
+  prometheus-node:
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: prometheus-process-exporter.
+  prometheus-proc:
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: prometheus-redis-exporter.
+  prometheus-redi:
+    running: [% monitoring.backend == 'prometheus' && (is_mgmt || is_lb || is_proxy || is_db || is_rtp) && redis.enable ? 'true' : 'false' %]
 
   kamailio:
     running: {{if and [% is_lb || is_proxy ? 1 : 0 %] .Vars.NODE_ACTIVE}} true {{else}} false {{end}}
@@ -95,4 +123,9 @@ process:
     running: {{if and [% is_rtp && rtpproxy.recording.enable == "yes" ? 1 : 0 %] .Vars.NODE_ACTIVE}} true {{else}} false {{end}}
 
   telegraf:
-    running: true
+    running: [% monitoring.backend == 'influxdb' ? 'true' : 'false' %]
+
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: victoria-metrics.
+  victoria-metric:
+    running: [% monitoring.backend == 'prometheus' && is_mgmt ? 'true' : 'false' %]

templates/411_process-ngcp-pro.yaml.tt2

@@ -49,7 +49,7 @@ process:
     running: [% ha.crm == 'pacemaker' ? 'true' : 'false' %]
 
   influxdb-relay:
-    running: true
+    running: [% monitoring.backend == 'influxdb' ? 'true' : 'false' %]
 
   monit:
     running: [% general.process_handler == 'monit' ? 'true' : 'false' %]
@@ -66,6 +66,11 @@ process:
   ngcp-voisniff:
     running: {{if and [% (is_proxy || is_lb || is_rtp) && voisniff.daemon.start == 'yes' ? 1 : 0 %] .Vars.NODE_ACTIVE}} true {{else}} false {{end}}
 
+  # We have to use a truncated process name because that's the COMM limit
+  # on Linux (15 characters). Real name: prometheus-haproxy-exporter.
+  prometheus-hapr:
+    running: [% monitoring.backend == 'prometheus' && is_lb && haproxy.enable && general.ngcp_type == 'carrier' ? 'true' : 'false' %]
+
   snmpd:
     running: true
 

templates/500_init-daemons-dabian.yaml.tt2

@@ -1,3 +1,13 @@
+[%
+  hostname = ngcp.get_hostname();
+
+  is_db = ngcp.has_role(hostname, 'db');
+  is_lb = ngcp.has_role(hostname, 'lb');
+  is_li_dist = ngcp.has_role(hostname, 'li_dist');
+  is_mgmt = ngcp.has_role(hostname, 'mgmt');
+  is_proxy = ngcp.has_role(hostname, 'proxy');
+  is_rtp = ngcp.has_role(hostname, 'rtp');
+-%]
 file:
   /run/systemd/generator.late:
     filetype: directory
@@ -30,8 +40,8 @@ service:
     running: true
 
   influxdb.service:
-    enabled: true
-    running: true
+    enabled: [% monitoring.backend == 'influxdb' || monitoring.influxdb_migrated == 'no' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'influxdb' || monitoring.influxdb_migrated == 'no' ? 'true' : 'false' %]
 
   motd.service:
     enabled: [% general.init_system == 'systemd' ? 'false' : 'true' %]
@@ -49,6 +59,30 @@ service:
     enabled: true
     running: true
 
+  prometheus.service:
+    enabled: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  prometheus-mysqld-exporter.service:
+    enabled: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  prometheus-nginx-exporter.service:
+    enabled: [% monitoring.backend == 'prometheus' && (is_mgmt || is_lb || is_li_dist) && nginx.enable ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' && (is_mgmt || is_lb || is_li_dist) && nginx.enable ? 'true' : 'false' %]
+
+  prometheus-node-exporter.service:
+    enabled: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  prometheus-process-exporter.service:
+    enabled: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' ? 'true' : 'false' %]
+
+  prometheus-redis-exporter.service:
+    enabled: [% monitoring.backend == 'prometheus' && (is_mgmt || is_lb || is_proxy || is_db || is_rtp) && redis.enable ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' && (is_mgmt || is_lb || is_proxy || is_db || is_rtp) && redis.enable ? 'true' : 'false' %]
+
   ssh.service:
     enabled: true
     running: true
@@ -58,5 +92,9 @@ service:
     running: true
 
   telegraf.service:
-    enabled: true
-    running: true
+    enabled: [% monitoring.backend == 'influxdb' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'influxdb' ? 'true' : 'false' %]
+
+  victoria-metrics.service:
+    enabled: [% monitoring.backend == 'prometheus' && is_mgmt ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'prometheus' && is_mgmt ? 'true' : 'false' %]

templates/510_init-daemons-ngcp.yaml.tt2

@@ -116,8 +116,8 @@ service:
     running: {{if and [% is_proxy && faxserver.enable == "yes" && general.ngcp_type != 'spce' ? 1 : 0 %] .Vars.NODE_ACTIVE}} true {{else}} false {{end}}
 
   influxdb-relay.service:
-    enabled: [% general.ngcp_type != 'spce' ? 'true' : 'false' %]
-    running: [% general.ngcp_type != 'spce' ? 'true' : 'false' %]
+    enabled: [% monitoring.backend == 'influxdb' && general.ngcp_type != 'spce' ? 'true' : 'false' %]
+    running: [% monitoring.backend == 'influxdb' && general.ngcp_type != 'spce' ? 'true' : 'false' %]
 
   snmpd.service:
     enabled: [% general.ngcp_type != 'spce' ? 'true' : 'false' %]

templates/822_network-ports-ngcp-pro.yaml.tt2

@@ -20,7 +20,7 @@ port:
 
   # InfluxDB Relay
   tcp:9096:
-    listening: true
+    listening: [% monitoring.backend == 'influxdb' && general.ngcp_type != 'spce' ? 'true' : 'false' %]
     ip: ["127.0.0.1"]
 
   # Corosync

templates/910_service-influxdb.yaml.tt2

@@ -1,12 +0,0 @@
-command:
-
-  influx -execute 'show databases':
-    exit-status: 0
-    stdout:
-    - "/^name: databases$/"
-    - "/^_internal$/"
-    - "/^ngcp$/"
-    - "/^telegraf$/"
-    stderr: []
-    timeout: 10000
-

templates/910_service-monitoring.yaml.tt2

@@ -0,0 +1,35 @@
+[%
+  hostname = ngcp.get_hostname();
+
+  argv.host = hostname;
+  argv.type = 'ha_int';
+  PROCESS '/usr/lib/ngcp-ngcpcfg/get_all_shared_ips_for_host';
+  ha_ips = out;
+  PROCESS '/usr/lib/ngcp-ngcpcfg/get_all_ips_for_host';
+  ha_ips = out.merge(ha_ips);
+-%]
+command:
+
+[% IF monitoring.backend == 'influxdb' -%]
+  influx -execute 'show databases':
+    exit-status: 0
+    stdout:
+    - "/^name: databases$/"
+    - "/^_internal$/"
+    - "/^ngcp$/"
+    - "/^telegraf$/"
+    stderr: []
+    timeout: 10000
+[% ELSE -%]
+  promtool query instant http://[% ha_ips.0 || 'localhost' %]:[% prometheus.port || 9090 %] '{__name__=~".+"}':
+    exit-status: 0
+    stdout:
+    - "/^go_/"
+    - "/^namedprocess_/"
+    - "/^ngcp_/"
+    - "/^node_/"
+    - "/^process_/"
+    - "/^redis_/"
+    stderr: []
+    timeout: 10000
+[% END -%]

templates/goss.yaml.tt2

@@ -39,7 +39,7 @@ gossfile:
 [% END -%]
   830_network-interfaces-legacy.yaml: {}
   900_service-ngcp-api.yaml: {}
-  910_service-influxdb.yaml: {}
+  910_service-monitoring.yaml: {}
   911_service-grafana.yaml: {}
 [% IF general.ngcp_type != 'spce' -%]
   915_service-snmp.yaml: {}