TT#36302 support scheduling settings

Change-Id: I490fde10a5ae44d25b95e4368af70003716040b7

README.md

@@ -228,6 +228,10 @@ option and which are reproduced below:
 	  --recording-format=raw|eth       PCAP file format for recorded calls.
 	  --iptables-chain=STRING          Add explicit firewall rules to this iptables chain
 	  --codecs                         Print a list of supported codecs and exit
+	  --scheduling=default|...         Thread scheduling policy
+	  --priority=INT                   Thread scheduling priority
+	  --idle-scheduling=default|...    Idle thread scheduling policy
+	  --idle-priority=INT              Idle thread scheduling priority
 
 Most of these options are indeed optional, with two exceptions. It's mandatory to specify at least one local
 IP address through `--interface`, and at least one of the `--listen-...` options must be given.
@@ -646,6 +650,29 @@ The options are described in more detail below.
 	and its API implementation in the Linux kernel. In such a case, it is recommended to
 	add a static iptables rule for the entire media port range instead, and not use this option.
 
+* --scheduling, --priority, --idle-scheduling, --idle-priority
+
+	These options control various thread scheduling parameters. The `scheduling` and `priority`
+	settings are applied to the main worker threads, while the `idle-` versions of these
+	settings are applied to various lower priority threads, such as timer runs.
+
+	The `scheduling` settings take the name of one of the supported scheduler policies. Setting
+	it to `default` or `none` is equivalent to not setting the option at all and leaves the
+	system default in place. The strings `fifo` and `rr` refer to realtime scheduling policies.
+	`other` is the Linux default scheduling policy. `batch` is similar to `other` except for
+	a small wake-up scheduling penalty. `idle` is an extremely low priority scheduling policy.
+	The Linux-specific `deadline` policy is not supported by *rtpengine*. Not all systems
+	necessarily supports all scheduling policies; refer to your system's `sched(7)` man page
+	for details.
+
+	The `priority` settings correspond to the scheduling priority for realtime (`fifo` or `rr`)
+	scheduling policies and must be in the range of 1 (low) through 99 (high). For all other
+	scheduling policies (including no policy specified), the `priority` settings correspond
+	to the `nice` value and should be in the range of -20 (high) through 19 (low). Not all systems
+	support thread-specific `nice` values; on such a system, using these settings might have
+	unexpected results. (Linux does support thread-specific `nice values.)
+	Refer to your system's `sched(7)` man page.
+
 A typical command line (enabling both UDP and NG protocols) thus may look like:
 
 	/usr/sbin/rtpengine --table=0 --interface=10.64.73.31 --interface=2001:db8::4f3:3d \

daemon/aux.c

@@ -5,6 +5,10 @@
 #include <pcre.h>
 #include <stdlib.h>
 #include <pthread.h>
+#include <sys/resource.h>
+#include <errno.h>
+#include <unistd.h>
+#include "log.h"
 
 
 
@@ -20,10 +24,17 @@
 struct detach_thread {
 	void		(*func)(void *);
 	void		*data;
+	const char	*scheduler;
+	int		priority;
 };
 struct thread_buf {
 	char buf[THREAD_BUF_SIZE];
 };
+struct scheduler {
+	const char *name;
+	int num;
+	int nice;
+};
 
 
 static mutex_t threads_lists_lock = MUTEX_STATIC_INIT;
@@ -41,6 +52,29 @@ volatile int rtpe_shutdown;
 mutex_t __atomic64_mutex = MUTEX_STATIC_INIT;
 #endif
 
+static const struct scheduler schedulers[] = {
+	{ "default",	-1,		1 },
+	{ "none",	-1,		1 },
+#ifdef SCHED_FIFO
+	{ "fifo",	SCHED_FIFO,	0 },
+#endif
+#ifdef SCHED_RR
+	{ "rr",		SCHED_RR,	0 },
+#endif
+//#ifdef SCHED_DEADLINE
+//	{ "deadline",	SCHED_DEADLINE,	0 },
+//#endif
+#ifdef SCHED_OTHER
+	{ "other",	SCHED_OTHER,	1 },
+#endif
+#ifdef SCHED_BATCH
+	{ "batch",	SCHED_BATCH,	1 },
+#endif
+#ifdef SCHED_IDLE
+	{ "idle",	SCHED_IDLE,	0 },
+#endif
+};
+
 
 
 GList *g_list_link(GList *list, GList *el) {
@@ -140,6 +174,38 @@ static void *thread_detach_func(void *d) {
 	threads_running = g_list_prepend(threads_running, t);
 	mutex_unlock(&threads_lists_lock);
 
+	const struct scheduler *scheduler = NULL;
+
+	if (dt->scheduler) {
+		for (int i = 0; i < G_N_ELEMENTS(schedulers); i++) {
+			if (!strcmp(dt->scheduler, schedulers[i].name)) {
+				scheduler = &schedulers[i];
+				break;
+			}
+		}
+
+		if (!scheduler)
+			ilog(LOG_ERR, "Specified scheduler policy '%s' not found", dt->scheduler);
+		else {
+			struct sched_param param = { 0 };
+
+			if (!scheduler->nice)
+				param.sched_priority = dt->priority;
+
+			if (pthread_setschedparam(*t, scheduler->num, &param))
+				ilog(LOG_ERR, "Failed to set thread scheduling paramaters to '%s' (%i) / %i: %s",
+						dt->scheduler, scheduler->num, param.sched_priority,
+						strerror(errno));
+
+		}
+	}
+
+	if ((!scheduler && dt->priority) || (scheduler && scheduler->nice)) {
+		if (setpriority(PRIO_PROCESS, 0, dt->priority))
+			ilog(LOG_ERR, "Failed to set thread nice value: %s",
+					strerror(errno));
+	}
+
 	dt->func(dt->data);
 	g_slice_free1(sizeof(*dt), dt);
 	thread_join_me();
@@ -165,12 +231,14 @@ static int thread_create(void *(*func)(void *), void *arg, int joinable, pthread
 	return 0;
 }
 
-void thread_create_detach(void (*f)(void *), void *d) {
+void thread_create_detach_prio(void (*f)(void *), void *d, const char *scheduler, int priority) {
 	struct detach_thread *dt;
 
 	dt = g_slice_alloc(sizeof(*dt));
 	dt->func = f;
 	dt->data = d;
+	dt->scheduler = scheduler;
+	dt->priority = priority;
 
 	if (thread_create(thread_detach_func, dt, 1, NULL))
 		abort();

daemon/call.c

@@ -468,7 +468,8 @@ destroy:
 	}
 
 	if (xh)
-		thread_create_detach(xmlrpc_kill_calls, xh);
+		thread_create_detach_prio(xmlrpc_kill_calls, xh, rtpe_config.idle_scheduling,
+				rtpe_config.idle_priority);
 }
 
 

daemon/main.c

@@ -305,6 +305,10 @@ static void options(int *argc, char ***argv) {
 		{ "iptables-chain",0,0,	G_OPTION_ARG_STRING,	&rtpe_config.iptables_chain,"Add explicit firewall rules to this iptables chain","STRING" },
 #endif
 		{ "codecs",	0, 0,	G_OPTION_ARG_NONE,	&codecs,		"Print a list of supported codecs and exit",	NULL },
+		{ "scheduling",	0, 0,	G_OPTION_ARG_STRING,	&rtpe_config.scheduling,"Thread scheduling policy",	"default|none|fifo|rr|other|batch|idle" },
+		{ "priority",	0, 0,	G_OPTION_ARG_INT,	&rtpe_config.priority,	"Thread scheduling priority",	"INT" },
+		{ "idle-scheduling",0, 0,G_OPTION_ARG_STRING,	&rtpe_config.idle_scheduling,"Idle thread scheduling policy",	"default|none|fifo|rr|other|batch|idle" },
+		{ "idle-priority",0, 0,	G_OPTION_ARG_INT,	&rtpe_config.idle_priority,"Idle thread scheduling priority",	"INT" },
 		{ NULL, }
 	};
 
@@ -327,7 +331,7 @@ static void options(int *argc, char ***argv) {
 			die("Invalid interface specification: %s", *iter);
 		g_queue_push_tail(&rtpe_config.interfaces, ifa);
 	}
-	if (!&rtpe_config.interfaces)
+	if (!rtpe_config.interfaces.length)
 		die("Cannot start without any configured interfaces");
 
 	if (ks_a) {
@@ -725,8 +729,8 @@ int main(int argc, char **argv) {
 	ilog(LOG_INFO, "Startup complete, version %s", RTPENGINE_VERSION);
 
 	thread_create_detach(sighandler, NULL);
-	thread_create_detach(poller_timer_loop, rtpe_poller);
-	thread_create_detach(load_thread, NULL);
+	thread_create_detach_prio(poller_timer_loop, rtpe_poller, rtpe_config.idle_scheduling, rtpe_config.idle_priority);
+	thread_create_detach_prio(load_thread, NULL, rtpe_config.idle_scheduling, rtpe_config.idle_priority);
 
 	if (!is_addr_unspecified(&rtpe_config.redis_ep.address))
 		thread_create_detach(redis_notify_loop, NULL);
@@ -745,7 +749,7 @@ int main(int argc, char **argv) {
 	}
 
 	for (;idx<rtpe_config.num_threads;++idx) {
-		thread_create_detach(poller_loop, rtpe_poller);
+		thread_create_detach_prio(poller_loop, rtpe_poller, rtpe_config.scheduling, rtpe_config.priority);
 	}
 
 	while (!rtpe_shutdown) {

daemon/main.h

@@ -71,6 +71,10 @@ struct rtpengine_config {
 	int			load_limit;
 	int			cpu_limit;
 	uint64_t		bw_limit;
+	char			*scheduling;
+	int			priority;
+	char			*idle_scheduling;
+	int			idle_priority;
 };
 
 

include/aux.h

@@ -401,7 +401,10 @@ INLINE int __debug_rwlock_unlock_w(rwlock_t *m, const char *file, unsigned int l
 /*** THREAD HELPERS ***/
 
 void threads_join_all(int);
-void thread_create_detach(void (*)(void *), void *);
+void thread_create_detach_prio(void (*)(void *), void *, const char *, int);
+INLINE void thread_create_detach(void (*f)(void *), void *a) {
+	thread_create_detach_prio(f, a, NULL, 0);
+}