Added Prometheus metrics

Makefile.in

@@ -30,8 +30,8 @@ HIREDIS_MODS = src/apps/common/hiredis_libevent2.c
 USERDB_HEADERS = src/apps/relay/dbdrivers/dbdriver.h src/apps/relay/dbdrivers/dbd_sqlite.h src/apps/relay/dbdrivers/dbd_pgsql.h src/apps/relay/dbdrivers/dbd_mysql.h src/apps/relay/dbdrivers/dbd_mongo.h src/apps/relay/dbdrivers/dbd_redis.h
 USERDB_MODS = src/apps/relay/dbdrivers/dbdriver.c src/apps/relay/dbdrivers/dbd_sqlite.c src/apps/relay/dbdrivers/dbd_pgsql.c src/apps/relay/dbdrivers/dbd_mysql.c src/apps/relay/dbdrivers/dbd_mongo.c src/apps/relay/dbdrivers/dbd_redis.c
 
-SERVERAPP_HEADERS = src/apps/relay/userdb.h src/apps/relay/tls_listener.h src/apps/relay/mainrelay.h src/apps/relay/turn_admin_server.h src/apps/relay/dtls_listener.h src/apps/relay/libtelnet.h ${HIREDIS_HEADERS} ${USERDB_HEADERS}
-SERVERAPP_MODS = src/apps/relay/mainrelay.c src/apps/relay/netengine.c src/apps/relay/libtelnet.c src/apps/relay/turn_admin_server.c src/apps/relay/userdb.c src/apps/relay/tls_listener.c src/apps/relay/dtls_listener.c ${HIREDIS_MODS} ${USERDB_MODS}
+SERVERAPP_HEADERS = src/apps/relay/userdb.h src/apps/relay/tls_listener.h src/apps/relay/mainrelay.h src/apps/relay/turn_admin_server.h src/apps/relay/dtls_listener.h src/apps/relay/libtelnet.h src/apps/relay/prom_server.h ${HIREDIS_HEADERS} ${USERDB_HEADERS}
+SERVERAPP_MODS = src/apps/relay/mainrelay.c src/apps/relay/netengine.c src/apps/relay/libtelnet.c src/apps/relay/turn_admin_server.c src/apps/relay/userdb.c src/apps/relay/tls_listener.c src/apps/relay/dtls_listener.c src/apps/relay/prom_server.c ${HIREDIS_MODS} ${USERDB_MODS}
 SERVERAPP_DEPS = ${SERVERTURN_MODS} ${SERVERTURN_DEPS} ${SERVERAPP_MODS} ${SERVERAPP_HEADERS} ${COMMON_DEPS} ${IMPL_DEPS} lib/libturnclient.a
 
 TURN_BUILD_RESULTS = bin/turnutils_oauth bin/turnutils_natdiscovery bin/turnutils_stunclient bin/turnutils_rfc5769check bin/turnutils_uclient bin/turnserver bin/turnutils_peer lib/libturnclient.a include/turn/ns_turn_defs.h sqlite_empty_db

README.md

@@ -70,6 +70,8 @@ Supported user databases (for user repository, with passwords or keys, if authen
 
 Redis can also be used for status and statistics storage and notification.
 
+By default a [prometheus](https://prometheus.io/) exporter endpoint is enabled on port 9641 under path /metrics
+
 Supported message integrity digest algorithms:
 
   * HMAC-SHA1, with MD5-hashed keys (as required by STUN and TURN standards)

README.turnserver

@@ -265,6 +265,9 @@ Flags:
 			check: across the session, all requests must have the same
 			main ORIGIN attribute value (if the ORIGIN was
 			initially used by the session).
+ --no-prometheus	Disable prometheus metrics. By default it is
+			enabled and listening on port 9121 unther the path /metrics
+            also the path / on this port can be used as a health check
 
 -h			Help.
 

configure

@@ -1051,6 +1051,47 @@ else
     fi
 fi
 
+###########################
+# Test Prometheus
+###########################
+
+if [ -z "${TURN_NO_PROMETHEUS}" ] ; then
+
+	testlib prom
+	ER=$?
+	if ! [ ${ER} -eq 0 ] ; then
+		${ECHO_CMD} "Prometheus lib found."
+		testlib promhttp
+		ER=$?
+		if ! [ ${ER} -eq 0 ] ; then
+			${ECHO_CMD} "Prometheus http lib found."
+			testlib microhttpd
+			ER=$?
+			if ! [ ${ER} -eq 0 ] ; then
+				${ECHO_CMD} "Microhttpd lib found."
+			else
+				${ECHO_CMD} "ERROR: microhttpd development libraries are not installed properly in required location."
+				${ECHO_CMD} "Prometheus support will be disabled."
+				${ECHO_CMD} "See the INSTALL file."
+				OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS"
+			fi
+		else
+			${ECHO_CMD} "ERROR: Libpromhttp development libraries are not installed properly in required location."
+			${ECHO_CMD} "Prometheus support will be disabled."
+        	${ECHO_CMD} "See the INSTALL file."
+			OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS"
+		fi
+	else
+        ${ECHO_CMD} "ERROR: Libprom development libraries are not installed properly in required location."
+		${ECHO_CMD} "Prometheus support will be disabled."
+        ${ECHO_CMD} "See the INSTALL file."
+		OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS"
+	fi
+
+else
+	OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS"
+fi
+
 ###########################
 # Test SQLite setup
 ###########################

examples/etc/turnserver.conf

@@ -186,6 +186,18 @@
 #
 #no-auth
 
+# Disable prometheus exporter
+# By default the turnserver will expose an endpoint with stats on a prometheus format
+# this endpoint is on a different port to conflict with other configurations.
+#
+# You can simply run the turnserver and access the port 9641 and path /metrics
+#
+# For mor info on the prometheus exporter and metrics
+# https://prometheus.io/docs/introduction/overview/
+# https://prometheus.io/docs/concepts/data_model/
+#
+# no-prometheus
+
 # TURN REST API flag.
 # (Time Limited Long Term Credential)
 # Flag that sets a special authorization option that is based upon authentication secret.

src/apps/relay/mainrelay.c

@@ -31,6 +31,11 @@
 #include "mainrelay.h"
 #include "dbdrivers/dbdriver.h"
 
+#if !defined(TURN_NO_PROMETHEUS)
+#include "prom_server.h"
+#endif
+
+
 #if (defined LIBRESSL_VERSION_NUMBER && OPENSSL_VERSION_NUMBER == 0x20000000L)
 #undef OPENSSL_VERSION_NUMBER
 #define OPENSSL_VERSION_NUMBER 0x1000107FL
@@ -150,6 +155,7 @@ TURN_CREDENTIALS_NONE, /* ct */
 0, /* bps_capacity_allocated */
 0, /* total_quota */
 0, /* user_quota */
+1, /* prometheus enabled by default */
 ///////////// Users DB //////////////
 { (TURN_USERDB_TYPE)0, {"\0"}, {0,NULL, {NULL,0}} },
 ///////////// CPUs //////////////////
@@ -528,6 +534,10 @@ static char Usage[] = "Usage: turnserver [options]\n"
 "		                                and delivering traffic and allocation event notifications.\n"
 "						The connection string has the same parameters as redis-userdb connection string.\n"
 #endif
+#if !defined(TURN_NO_PROMETHEUS)
+" --no-prometheus				Disable prometheus metrics. By default it is enabled and listening on port 9641 unther the path /metrics\n"
+"						also the path / on this port can be used as a health check\n"
+#endif
 " --use-auth-secret				TURN REST API flag.\n"
 "						Flag that sets a special authorization option that is based upon authentication secret\n"
 "						(TURN Server REST API, see TURNServerRESTAPI.pdf). This option is used with timestamp.\n"
@@ -738,6 +748,7 @@ enum EXTRA_OPTS {
 	MAX_ALLOCATE_LIFETIME_OPT,
 	CHANNEL_LIFETIME_OPT,
 	PERMISSION_LIFETIME_OPT,
+	NO_PROMETHEUS_OPT,
 	AUTH_SECRET_OPT,
 	NO_AUTH_PINGS_OPT,
 	NO_DYNAMIC_IP_LIST_OPT,
@@ -843,6 +854,9 @@ static const struct myoption long_options[] = {
 #if !defined(TURN_NO_HIREDIS)
 				{ "redis-userdb", required_argument, NULL, 'N' },
 				{ "redis-statsdb", required_argument, NULL, 'O' },
+#endif
+#if !defined(TURN_NO_PROMETHEUS)
+				{ "no-prometheus", optional_argument, NULL, NO_PROMETHEUS_OPT },
 #endif
 				{ "use-auth-secret", optional_argument, NULL, AUTH_SECRET_OPT },
 				{ "static-auth-secret", required_argument, NULL, STATIC_AUTH_SECRET_VAL_OPT },
@@ -1443,6 +1457,11 @@ static void set_option(int c, char *value)
 		STRCPY(turn_params.redis_statsdb, value);
 		turn_params.use_redis_statsdb = 1;
 		break;
+#endif
+#if !defined(TURN_NO_PROMETHEUS)
+	case NO_PROMETHEUS_OPT:
+		turn_params.prometheus = 0;
+		break;
 #endif
 	case AUTH_SECRET_OPT:
 		turn_params.use_auth_secret_with_timestamp = 1;
@@ -2463,6 +2482,10 @@ int main(int argc, char **argv)
 	event_add(ev, NULL);
 
 	drop_privileges();
+#if !defined(TURN_NO_PROMETHEUS)
+	start_prometheus_server();
+#endif
+
 
 	run_listener_server(&(turn_params.listener));
 

src/apps/relay/mainrelay.h

@@ -311,6 +311,10 @@ typedef struct _turn_params_ {
   band_limit_t bps_capacity_allocated;
   vint total_quota;
   vint user_quota;
+  #if !defined(TURN_NO_PROMETHEUS)
+  int  prometheus;
+  #endif
+
 
 /////// Users DB ///////////
 

src/apps/relay/ns_ioalib_engine_impl.c

@@ -38,6 +38,10 @@
 
 #include "ns_ioalib_impl.h"
 
+#if !defined(TURN_NO_PROMETHEUS)
+#include "prom_server.h"
+#endif
+
 #if TLS_SUPPORTED
 #include <event2/bufferevent_ssl.h>
 #endif
@@ -3607,6 +3611,16 @@ void turn_report_allocation_set(void *a, turn_time_t lifetime, int refresh)
 					send_message_to_redis(e->rch, "set", key, "%s lifetime=%lu", status, (unsigned long)lifetime);
 					send_message_to_redis(e->rch, "publish", key, "%s lifetime=%lu", status, (unsigned long)lifetime);
 				}
+#endif
+#if !defined(TURN_NO_PROMETHEUS)
+				{
+					// Set status on prometheus metric
+					if(ss->realm_options.name[0]) {
+						prom_set_status(ss->realm_options.name, (const char*)ss->username, (unsigned long long)ss->id, status, (unsigned long)lifetime);
+					} else {
+						prom_set_status(NULL, (const char*)ss->username, (unsigned long long)ss->id, status, (unsigned long)lifetime);
+					}
+				}
 #endif
 			}
 		}
@@ -3650,6 +3664,25 @@ void turn_report_allocation_delete(void *a)
 					}
 					send_message_to_redis(e->rch, "publish", key, "rcvp=%lu, rcvb=%lu, sentp=%lu, sentb=%lu", (unsigned long)(ss->t_peer_received_packets), (unsigned long)(ss->t_peer_received_bytes), (unsigned long)(ss->t_peer_sent_packets), (unsigned long)(ss->t_peer_sent_bytes));
 				}
+#endif
+#if !defined(TURN_NO_PROMETHEUS)
+				{
+					if(ss->realm_options.name[0]){
+						// Set prometheus del metric and update status
+						prom_del_status(ss->realm_options.name, (const char*)ss->username, (unsigned long long)ss->id, (const char *)"deleted");
+
+						// Set prometheus total traffic metrics
+						prom_set_total_traffic(ss->realm_options.name, (const char*)ss->username, (unsigned long long)ss->id, (unsigned long)(ss->t_received_packets), (unsigned long)(ss->t_received_bytes), (unsigned long)(ss->t_sent_packets), (unsigned long)(ss->t_sent_bytes), true);
+						prom_set_total_traffic(ss->realm_options.name, (const char*)ss->username, (unsigned long long)ss->id, (unsigned long)(ss->t_peer_received_packets), (unsigned long)(ss->t_peer_received_bytes), (unsigned long)(ss->t_peer_sent_packets), (unsigned long)(ss->t_peer_sent_bytes), true);
+					} else {
+						// Set prometheus del metric and update status
+						prom_del_status(NULL, (const char*)ss->username, (unsigned long long)ss->id,  (const char *)"deleted");
+
+						// Set prometheus total traffic metrics
+						prom_set_total_traffic(NULL, (const char*)ss->username, (unsigned long long)ss->id, (unsigned long)(ss->t_received_packets), (unsigned long)(ss->t_received_bytes), (unsigned long)(ss->t_sent_packets), (unsigned long)(ss->t_sent_bytes), true);
+						prom_set_total_traffic(NULL, (const char*)ss->username, (unsigned long long)ss->id, (unsigned long)(ss->t_peer_received_packets), (unsigned long)(ss->t_peer_received_bytes), (unsigned long)(ss->t_peer_sent_packets), (unsigned long)(ss->t_peer_sent_bytes), true);
+					}
+				}
 #endif
 			}
 		}
@@ -3686,6 +3719,19 @@ void turn_report_session_usage(void *session, int force_invalid)
 					send_message_to_redis(e->rch, "publish", key, "rcvp=%lu, rcvb=%lu, sentp=%lu, sentb=%lu", (unsigned long)(ss->peer_received_packets), (unsigned long)(ss->peer_received_bytes), (unsigned long)(ss->peer_sent_packets), (unsigned long)(ss->peer_sent_bytes));
 				}
 #endif
+#if !defined(TURN_NO_PROMETHEUS)
+				{
+					// Set prometheus traffic metrics
+					if(ss->realm_options.name[0]){
+						prom_set_traffic(ss->realm_options.name, (const char *)ss->username, (unsigned long long)(ss->id), (unsigned long)(ss->received_packets), (unsigned long)(ss->received_bytes), (unsigned long)(ss->sent_packets), (unsigned long)(ss->sent_bytes), false);
+						prom_set_traffic(ss->realm_options.name, (const char *)ss->username, (unsigned long long)(ss->id), (unsigned long)(ss->peer_received_packets), (unsigned long)(ss->peer_received_bytes), (unsigned long)(ss->peer_sent_packets), (unsigned long)(ss->peer_sent_bytes), true);
+					} else {
+						prom_set_traffic(NULL, (const char *)ss->username, (unsigned long long)(ss->id), (unsigned long)(ss->received_packets), (unsigned long)(ss->received_bytes), (unsigned long)(ss->sent_packets), (unsigned long)(ss->sent_bytes), false);
+						prom_set_traffic(NULL, (const char *)ss->username, (unsigned long long)(ss->id), (unsigned long)(ss->peer_received_packets), (unsigned long)(ss->peer_received_bytes), (unsigned long)(ss->peer_sent_packets), (unsigned long)(ss->peer_sent_bytes), true);
+					}
+				}
+#endif
+
 				ss->t_received_packets += ss->received_packets;
 				ss->t_received_bytes += ss->received_bytes;
 				ss->t_sent_packets += ss->sent_packets;

src/apps/relay/prom_server.c

@@ -0,0 +1,107 @@
+#if !defined(TURN_NO_PROMETHEUS)
+
+#include "mainrelay.h"
+#include "prom_server.h"
+
+int start_prometheus_server(void){
+  if (turn_params.prometheus == 0){
+    return 0;
+  }
+  prom_collector_registry_default_init();
+  // Create status gauge metric
+  turn_status = prom_collector_registry_must_register_metric(prom_gauge_new("turn_status", "Represents status", 5, (const char *[]) {"realm", "user", "allocation", "status", "lifetime" }));
+
+  // Create traffic gauge metrics
+  turn_traffic_rcvp = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_rcvp", "Represents received packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_traffic_rcvb = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_rcvb", "Represents received bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_traffic_sentp = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_sentp", "Represents sent packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_traffic_sentb = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_sentb", "Represents sent bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+
+  // Create traffic for peers gauge metrics
+  turn_traffic_peer_rcvp = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_peer_rcvp", "Represents peer received packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_traffic_peer_rcvb = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_peer_rcvb", "Represents peer received bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_traffic_peer_sentp = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_peer_sentp", "Represents peer sent packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_traffic_peer_sentb = prom_collector_registry_must_register_metric(prom_counter_new("turn_traffic_peer_sentb", "Represents peer sent bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+
+  // Create total traffic gauge metrics
+  turn_total_traffic_rcvp = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_rcvp", "Represents total received packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_total_traffic_rcvb = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_rcvb", "Represents total received bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_total_traffic_sentp = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_sentp", "Represents total sent packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_total_traffic_sentb = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_sentb", "Represents total sent bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+
+  // Create tota traffic for peers gauge metrics
+  turn_total_traffic_peer_rcvp = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_peer_rcvp", "Represents total peer received packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_total_traffic_peer_rcvb = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_peer_rcvb", "Represents total peer received bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_total_traffic_peer_sentp = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_peer_sentp", "Represents total peer sent packets", 3, (const char *[]) {"realm", "user", "allocation" }));
+  turn_total_traffic_peer_sentb = prom_collector_registry_must_register_metric(prom_counter_new("turn_total_traffic_peer_sentb", "Represents total peer sent bytes", 3, (const char *[]) {"realm", "user", "allocation" }));
+
+  promhttp_set_active_collector_registry(NULL);
+
+
+  struct MHD_Daemon *daemon = promhttp_start_daemon(MHD_USE_SELECT_INTERNALLY, DEFAULT_PROM_SERVER_PORT, NULL, NULL);
+  if (daemon == NULL) {
+    return 1;
+  }
+  return 0;
+}
+
+void prom_set_status(const char* realm, const char* user, unsigned long long allocation, const char* status, unsigned long lifetime){
+  if (turn_params.prometheus == 1){
+    char allocation_chars[1024];
+    char lifetime_chars[1024];
+
+    snprintf(allocation_chars, sizeof(allocation_chars), "%018llu", allocation);
+    snprintf(lifetime_chars, sizeof(lifetime_chars), "%lu", lifetime);
+
+    prom_gauge_add(turn_status, 1, (const char *[]) { realm , user, allocation_chars, status, lifetime_chars });
+  }
+}
+
+void prom_del_status(const char* realm, const char* user, unsigned long long allocation, const char* status){
+  if (turn_params.prometheus == 0){
+    char allocation_chars[1024];
+    snprintf(allocation_chars, sizeof(allocation_chars), "%018llu", allocation);
+
+    prom_gauge_sub(turn_status, 1, (const char *[]) { realm , user, allocation_chars, (char *)"new", (char *)"600" });
+    prom_gauge_add(turn_status, 1, (const char *[]) { realm , user, allocation_chars, status, NULL });
+  }
+}
+void prom_set_traffic(const char* realm, const char* user, unsigned long long allocation, unsigned long rsvp, unsigned long rsvb, unsigned long sentp, unsigned long sentb, bool peer){
+  if (turn_params.prometheus == 1){
+    char allocation_chars[1024];
+    snprintf(allocation_chars, sizeof(allocation_chars), "%018llu", allocation);
+
+    if (peer){
+      prom_counter_add(turn_traffic_peer_rcvp, rsvp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_traffic_peer_rcvb, rsvb, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_traffic_peer_sentp, sentp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_traffic_peer_sentb, sentb, (const char *[]) { realm , user, allocation_chars });
+    } else {
+      prom_counter_add(turn_traffic_rcvp, rsvp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_traffic_rcvb, rsvb, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_traffic_sentp, sentp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_traffic_sentb, sentb, (const char *[]) { realm , user, allocation_chars });
+    }
+  }
+}
+
+void prom_set_total_traffic(const char* realm, const char* user, unsigned long long allocation, unsigned long rsvp, unsigned long rsvb, unsigned long sentp, unsigned long sentb, bool peer){
+  if (turn_params.prometheus == 1){
+    char allocation_chars[1024];
+    snprintf(allocation_chars, sizeof(allocation_chars), "%018llu", allocation);
+
+    if (peer){
+      prom_counter_add(turn_total_traffic_peer_rcvp, rsvp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_total_traffic_peer_rcvb, rsvb, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_total_traffic_peer_sentp, sentp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_total_traffic_peer_sentb, sentb, (const char *[]) { realm , user, allocation_chars });
+    } else {
+      prom_counter_add(turn_total_traffic_rcvp, rsvp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_total_traffic_rcvb, rsvb, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_total_traffic_sentp, sentp, (const char *[]) { realm , user, allocation_chars });
+      prom_counter_add(turn_total_traffic_sentb, sentb, (const char *[]) { realm , user, allocation_chars });
+    }
+  }
+}
+
+#endif /* TURN_NO_PROMETHEUS */