MT#61822 support removing unused cache data

Change-Id: I99fef5dd779e1b505aa2ad17ff68cd2eef6e050d

daemon/main.c

@@ -675,11 +675,15 @@ static void options(int *argc, char ***argv, GHashTable *templates) {
 		{ "kernel-player-media",0,0,G_OPTION_ARG_INT,	&rtpe_config.kernel_player_media,"Max number of kernel media files","INT"},
 		{ "preload-media-files",0,0,G_OPTION_ARG_FILENAME_ARRAY,&rtpe_config.preload_media_files,"Preload media file(s) for playback into memory","FILE"},
 		{ "media-files-reload",0,0,G_OPTION_ARG_INT,	&rtpe_config.media_refresh,"Refresh/reload preloaded media files at a certain interval","SECONDS"},
+		{ "media-files-expire",0,0,G_OPTION_ARG_INT,	&rtpe_config.media_expire,"Maximum age of unused cached media files","SECONDS"},
+		{ "expiry-timer",0,0,G_OPTION_ARG_INT,		&rtpe_config.expiry_timer,"How often to check for expired media cache entries","SECONDS"},
 		{ "preload-db-media",0,0,G_OPTION_ARG_STRING_ARRAY,&rtpe_config.preload_db_media,"Preload media from database for playback into memory","INT"},
 		{ "db-media-reload",0,0,G_OPTION_ARG_INT,	&rtpe_config.db_refresh,"Reload preloaded media from DB at a certain interval","SECONDS"},
+		{ "db-media-expire",0,0,G_OPTION_ARG_INT,	&rtpe_config.db_expire,"Maximum age of unused cached DB media entries","SECONDS"},
 		{ "db-media-cache",0,0,	G_OPTION_ARG_FILENAME,	&rtpe_config.db_media_cache,"Directory to store media loaded from database","PATH"},
 		{ "preload-db-cache",0,0,G_OPTION_ARG_STRING_ARRAY,&rtpe_config.preload_db_cache,"Preload media from database for playback into file cache","INT"},
 		{ "db-cache-reload",0,0,G_OPTION_ARG_INT,	&rtpe_config.cache_refresh,"Refresh/reload cached media from DB at a certain interval","SECONDS"},
+		{ "db-cache-expire",0,0,G_OPTION_ARG_INT,	&rtpe_config.cache_expire,"Maximum age of unused cached DB entries in files","SECONDS"},
 		{ "audio-buffer-length",0,0,	G_OPTION_ARG_INT,&rtpe_config.audio_buffer_length,"Length in milliseconds of audio buffer","INT"},
 		{ "audio-buffer-delay",0,0,	G_OPTION_ARG_INT,&rtpe_config.audio_buffer_delay,"Initial delay in milliseconds for buffered audio","INT"},
 		{ "audio-player",0,0,	G_OPTION_ARG_STRING,	&use_audio_player,	"When to enable the internal audio player","on-demand|play-media|transcoding|always"},
@@ -1604,14 +1608,21 @@ int main(int argc, char **argv) {
 	/* thread to refresh DTLS certificate */
 	dtls_timer();
 
-	thread_create_looper(media_player_refresh_timer, rtpe_config.idle_scheduling,
-			rtpe_config.idle_priority, "media refresh", rtpe_config.media_refresh * 1000000LL);
+	if (rtpe_config.media_refresh > 0)
+		thread_create_looper(media_player_refresh_timer, rtpe_config.idle_scheduling,
+				rtpe_config.idle_priority, "media refresh", rtpe_config.media_refresh * 1000000LL);
 
-	thread_create_looper(media_player_refresh_db, rtpe_config.idle_scheduling,
-			rtpe_config.idle_priority, "db refresh", rtpe_config.db_refresh * 1000000LL);
+	if (rtpe_config.db_refresh > 0)
+		thread_create_looper(media_player_refresh_db, rtpe_config.idle_scheduling,
+				rtpe_config.idle_priority, "db refresh", rtpe_config.db_refresh * 1000000LL);
 
-	thread_create_looper(media_player_refresh_cache, rtpe_config.idle_scheduling,
-			rtpe_config.idle_priority, "cache refresh", rtpe_config.cache_refresh * 1000000LL);
+	if (rtpe_config.cache_refresh > 0)
+		thread_create_looper(media_player_refresh_cache, rtpe_config.idle_scheduling,
+				rtpe_config.idle_priority, "cache refresh", rtpe_config.cache_refresh * 1000000LL);
+
+	if (rtpe_config.expiry_timer > 0)
+		thread_create_looper(media_player_expire, rtpe_config.idle_scheduling,
+				rtpe_config.idle_priority, "cache expiry", rtpe_config.expiry_timer * 1000000LL);
 
 	if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async)
 		thread_create_detach(redis_delete_async_loop, NULL, "redis async");

daemon/media_player.c

@@ -2339,29 +2339,17 @@ unsigned int media_player_reload_db_medias(void) {
 }
 
 enum thread_looper_action media_player_refresh_timer(void) {
-	if (rtpe_config.media_refresh <= 0)
-		return TLA_BREAK;
-
 	media_player_reload_files();
-
 	return TLA_CONTINUE;
 }
 
 enum thread_looper_action media_player_refresh_db(void) {
-	if (rtpe_config.db_refresh <= 0)
-		return TLA_BREAK;
-
 	media_player_reload_db_medias();
-
 	return TLA_CONTINUE;
 }
 
 enum thread_looper_action media_player_refresh_cache(void) {
-	if (rtpe_config.cache_refresh <= 0)
-		return TLA_BREAK;
-
 	media_player_reload_caches();
-
 	return TLA_CONTINUE;
 }
 
@@ -2727,3 +2715,97 @@ unsigned int media_player_evict_player_caches(void) {
 #endif
 	return ret;
 }
+
+#ifdef WITH_TRANSCODING
+static void media_player_expire_files(void) {
+	if (rtpe_config.media_expire <= 0)
+		return;
+
+	time_t limit = rtpe_now.tv_sec - rtpe_config.media_expire;
+	unsigned int num = 0;
+
+	{
+		RWLOCK_R(&media_player_media_files_names_lock);
+		str_list *next;
+		for (__auto_type l = media_player_media_files_names.head; l; l = next) {
+			next = l->next;
+			g_auto(str) name;
+			{
+				LOCK(&media_player_media_files_lock);
+				__auto_type fo = t_hash_table_lookup(media_player_media_files, l->data);
+				if (!fo)
+					continue;
+				if (fo->atime >= limit)
+					continue;
+				name = str_dup_str(l->data);
+			}
+			if (__media_player_evict_file(&name))
+				num++;
+		}
+	}
+
+	if (num)
+		ilog(LOG_DEBUG, "Removed %u old entries from media cache", num);
+}
+
+static void media_player_expire_dbs(void) {
+	if (rtpe_config.db_expire <= 0)
+		return;
+
+	time_t limit = rtpe_now.tv_sec - rtpe_config.db_expire;
+	unsigned int num = 0;
+
+	{
+		RWLOCK_R(&media_player_db_media_ids_lock);
+		GList *next;
+		for (__auto_type l = media_player_db_media_ids.head; l; l = next) {
+			next = l->next;
+			unsigned long long id;
+			{
+				LOCK(&media_player_db_media_lock);
+				__auto_type fo = t_hash_table_lookup(media_player_db_media, l->data);
+				if (!fo)
+					continue;
+				if (fo->atime >= limit)
+					continue;
+				id = GPOINTER_TO_UINT(l->data);
+			}
+			if (__media_player_evict_db_media(GPOINTER_TO_UINT(id)))
+				num++;
+		}
+	}
+
+	if (num)
+		ilog(LOG_DEBUG, "Removed %u old entries from DB media cache", num);
+}
+
+static void media_player_expire_cache_entry(unsigned long long id, unsigned int *num) {
+	time_t mtime, atime;
+	if (!media_player_get_cache_times(id, &mtime, &atime))
+		return;
+	time_t limit = rtpe_now.tv_sec - rtpe_config.db_expire;
+	if (atime >= limit)
+		return;
+	if (media_player_evict_cache(id))
+		(*num)++;
+}
+
+static void media_player_expire_caches(void) {
+	if (rtpe_config.cache_expire <= 0)
+		return;
+
+	unsigned int ret = 0;
+	media_player_iterate_db_cache(media_player_expire_cache_entry, &ret);
+}
+#endif
+
+enum thread_looper_action media_player_expire(void) {
+#ifdef WITH_TRANSCODING
+	media_player_expire_files();
+	media_player_expire_dbs();
+	media_player_expire_caches();
+	return TLA_CONTINUE;
+#else
+	return TLA_BREAK;
+#endif
+}

docs/rtpengine.md

@@ -1226,6 +1226,23 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
     Similar to the __db-media-reload__ but applicable to media stored in the
     filesystem-backed cache.
 
+- __\-\-expiry-timer=__*SECONDS*
+- __\-\-media-files-expire=__*SECONDS*
+- __\-\-db-media-expire=__*SECONDS*
+- __\-\-db-cache-expire=__*SECONDS*
+
+    These options control the automatic removal of entries from the various
+    media caches when and if the entries go unused for a certain amount of
+    time. By default automatic removal is disabled.
+
+    The option `expiry-timer` must be set to non-zero for any automatic removal
+    to happen. It enables creation of a background thread and controls how
+    often this thread should run to check for expired unused cache entries.
+
+    The other options set the maximum allowed age for entries in the respective
+    caches. If an entry was not used for longer than the given time, it will be
+    removed from the cache.
+
 - __\-\-audio-buffer-length=__*INT*
 
     Set the buffer length used by the audio player (see below) in milliseconds. The

include/main.h

@@ -96,6 +96,10 @@ enum endpoint_learning {
 	X(media_refresh) \
 	X(db_refresh) \
 	X(cache_refresh) \
+	X(expiry_timer) \
+	X(media_expire) \
+	X(db_expire) \
+	X(cache_expire) \
 
 #define RTPE_CONFIG_UINT64_PARAMS \
 	X(bw_limit)

include/media_player.h

@@ -178,6 +178,7 @@ unsigned int media_player_reload_caches(void);
 enum thread_looper_action media_player_refresh_cache(void);
 charp_q media_player_list_player_cache(void);
 unsigned int media_player_evict_player_caches(void);
+enum thread_looper_action media_player_expire(void);
 
 struct send_timer *send_timer_new(struct packet_stream *);
 void send_timer_push(struct send_timer *, struct codec_packet *);