app_queue: Support persisting and loading of long member lists.

Greenlight in #asterisk brought up that he was receiving an error message "Could
not create persistent member string, out of space" when running app_queue in
Asterisk 10.  dump_queue_members() made an assumption that 8K would be enough to
store the generated string, but with queues that have large member lists this is
not always the case.  This patch removes the limitation and uses ast_str instead
of a fixed sized buffer.

The complicating factor comes from the fact that ast_db_get requires a buffer
and buffer size argument, which doesn't let us pull back more than what we pass
in, so I introduced a new ast_db_get_allocated() which returns an ast_strdup()'d
copy of the value from astdb.

As an aside, I did some testing on the maximum size of data that we can store in
the BDB library we distribute and was able to store a 10MB string and retrieve
it with no problems, so I feel this is a safe patch.

Review: https://reviewboard.asterisk.org/r/2136/
........

Merged revisions 374108 from http://svn.asterisk.org/svn/asterisk/branches/1.8
........

Merged revisions 374135 from http://svn.asterisk.org/svn/asterisk/branches/10
........

Merged revisions 374150 from http://svn.asterisk.org/svn/asterisk/branches/11


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@374151 65c4cc65-6c06-0410-ace0-fbb531ad65f3

apps/app_queue.c

@@ -1014,8 +1014,6 @@ static char *app_ql = "QueueLog" ;
 
 /*! \brief Persistent Members astdb family */
 static const char * const pm_family = "Queue/PersistentMembers";
-/* The maximum length of each persistent member queue database entry */
-#define PM_MAX_LEN 8192
 
 /*! \brief queues.conf [general] option */
 static int queue_persistent_members = 0;
@@ -5815,17 +5813,19 @@ static struct member *interface_exists(struct call_queue *q, const char *interfa
 static void dump_queue_members(struct call_queue *pm_queue)
 {
 	struct member *cur_member;
-	char value[PM_MAX_LEN];
+	struct ast_str *value;
-	int value_len = 0;
-	int res;
 	struct ao2_iterator mem_iter;
 
-	memset(value, 0, sizeof(value));
-
 	if (!pm_queue) {
 		return;
 	}
 
+	/* 4K is a reasonable default for most applications, but we grow to
+	 * accommodate more if necessary. */
+	if (!(value = ast_str_create(4096))) {
+		return;
+	}
+
 	mem_iter = ao2_iterator_init(pm_queue->members, 0);
 	while ((cur_member = ao2_iterator_next(&mem_iter))) {
 		if (!cur_member->dynamic) {
@@ -5833,26 +5833,28 @@ static void dump_queue_members(struct call_queue *pm_queue)
 			continue;
 		}
 
-		res = snprintf(value + value_len, sizeof(value) - value_len, "%s%s;%d;%d;%s;%s",
+		ast_str_append(&value, 0, "%s%s;%d;%d;%s;%s",
-			value_len ? "|" : "", cur_member->interface, cur_member->penalty, cur_member->paused, cur_member->membername, cur_member->state_interface);
+			ast_str_strlen(value) ? "|" : "",
+			cur_member->interface,
+			cur_member->penalty,
+			cur_member->paused,
+			cur_member->membername,
+			cur_member->state_interface);
 
 		ao2_ref(cur_member, -1);
-
-		if (res != strlen(value + value_len)) {
-			ast_log(LOG_WARNING, "Could not create persistent member string, out of space\n");
-			break;
-		}
-		value_len += res;
 	}
 	ao2_iterator_destroy(&mem_iter);
-	
+
-	if (value_len && !cur_member) {
+	if (ast_str_strlen(value) && !cur_member) {
-		if (ast_db_put(pm_family, pm_queue->name, value))
+		if (ast_db_put(pm_family, pm_queue->name, ast_str_buffer(value))) {
-		ast_log(LOG_WARNING, "failed to create persistent dynamic entry!\n");
+			ast_log(LOG_WARNING, "failed to create persistent dynamic entry!\n");
+		}
 	} else {
 		/* Delete the entry if the queue is empty or there is an error */
 		ast_db_del(pm_family, pm_queue->name);
 	}
+
+	ast_free(value);
 }
 
 /*! \brief Remove member from queue 
@@ -6356,7 +6358,7 @@ static void reload_queue_members(void)
 	struct ast_db_entry *db_tree;
 	struct ast_db_entry *entry;
 	struct call_queue *cur_queue;
-	char queue_data[PM_MAX_LEN];
+	char *queue_data;
 
 	/* Each key in 'pm_family' is the name of a queue */
 	db_tree = ast_db_gettree(pm_family, NULL);
@@ -6383,7 +6385,7 @@ static void reload_queue_members(void)
 			continue;
 		}
 
-		if (ast_db_get(pm_family, queue_name, queue_data, PM_MAX_LEN)) {
+		if (ast_db_get_allocated(pm_family, queue_name, &queue_data)) {
 			queue_t_unref(cur_queue, "Expire reload reference");
 			continue;
 		}
@@ -6428,6 +6430,7 @@ static void reload_queue_members(void)
 			}
 		}
 		queue_t_unref(cur_queue, "Expire reload reference");
+		ast_free(queue_data);
 	}
 
 	if (db_tree) {

include/asterisk/astdb.h

@@ -36,6 +36,17 @@ struct ast_db_entry {
 /*!\brief Get key value specified by family/key */
 int ast_db_get(const char *family, const char *key, char *out, int outlen);
 
+/*!\brief Get key value specified by family/key as a heap allocated string.
+ *
+ * Given a \a family and \a key, sets \a out to a pointer to a heap
+ * allocated string.  In the event of an error, \a out will be set to
+ * NULL.  The string must be freed by calling ast_free().
+ *
+ * \retval -1 An error occurred
+ * \retval 0 Success
+ */
+int ast_db_get_allocated(const char *family, const char *key, char **out);
+
 /*!\brief Store value addressed by family/key */
 int ast_db_put(const char *family, const char *key, const char *value);
 

main/db.c

@@ -307,7 +307,21 @@ int ast_db_put(const char *family, const char *key, const char *value)
 	return res;
 }
 
-int ast_db_get(const char *family, const char *key, char *value, int valuelen)
+/*!
+ * \internal
+ * \brief Get key value specified by family/key.
+ *
+ * Gets the value associated with the specified \a family and \a key, and
+ * stores it, either into the fixed sized buffer specified by \a buffer
+ * and \a bufferlen, or as a heap allocated string if \a bufferlen is -1.
+ *
+ * \note If \a bufferlen is -1, \a buffer points to heap allocated memory
+ *       and must be freed by calling ast_free().
+ *
+ * \retval -1 An error occurred
+ * \retval 0 Success
+ */
+static int db_get_common(const char *family, const char *key, char **buffer, int bufferlen)
 {
 	const unsigned char *result;
 	char fullkey[MAX_DB_FIELD];
@@ -332,7 +346,13 @@ int ast_db_get(const char *family, const char *key, char *value, int valuelen)
 		ast_log(LOG_WARNING, "Couldn't get value\n");
 		res = -1;
 	} else {
-		ast_copy_string(value, (const char *) result, valuelen);
+		const char *value = (const char *) result;
+
+		if (bufferlen == -1) {
+			*buffer = ast_strdup(value);
+		} else {
+			ast_copy_string(*buffer, value, bufferlen);
+		}
 	}
 	sqlite3_reset(get_stmt);
 	ast_mutex_unlock(&dblock);
@@ -340,6 +360,23 @@ int ast_db_get(const char *family, const char *key, char *value, int valuelen)
 	return res;
 }
 
+int ast_db_get(const char *family, const char *key, char *value, int valuelen)
+{
+	ast_assert(value != NULL);
+
+	/* Make sure we initialize */
+	value[0] = 0;
+
+	return db_get_common(family, key, &value, valuelen);
+}
+
+int ast_db_get_allocated(const char *family, const char *key, char **out)
+{
+	*out = NULL;
+
+	return db_get_common(family, key, out, -1);
+}
+
 int ast_db_del(const char *family, const char *key)
 {
 	char fullkey[MAX_DB_FIELD];

tests/test_db.c

@@ -232,11 +232,70 @@ AST_TEST_DEFINE(perftest)
 
 	return res;
 }
+
+AST_TEST_DEFINE(put_get_long)
+{
+	int res = AST_TEST_PASS;
+	struct ast_str *s;
+	int i, j;
+
+#define STR_FILL_32 "abcdefghijklmnopqrstuvwxyz123456"
+
+	switch (cmd) {
+	case TEST_INIT:
+		info->name = "put_get_long";
+		info->category = "/main/astdb/";
+		info->summary = "ast_db_(put|get_allocated) unit test";
+		info->description =
+			"Ensures that the ast_db_put and ast_db_get_allocated functions work";
+		return AST_TEST_NOT_RUN;
+	case TEST_EXECUTE:
+		break;
+	}
+
+	if (!(s = ast_str_create(4096))) {
+		return AST_TEST_FAIL;
+	}
+
+	for (i = 1024; i <= 1024 * 1024 * 8; i *= 2) {
+		char *out = NULL;
+
+		ast_str_reset(s);
+
+		for (j = 0; j < i; j += sizeof(STR_FILL_32) - 1) {
+			ast_str_append(&s, 0, "%s", STR_FILL_32);
+		}
+
+		if (ast_db_put("astdbtest", "long", ast_str_buffer(s))) {
+			ast_test_status_update(test, "Failed to put value of %zu bytes\n", ast_str_strlen(s));
+			res = AST_TEST_FAIL;
+		} else if (ast_db_get_allocated("astdbtest", "long", &out)) {
+			ast_test_status_update(test, "Failed to get value of %zu bytes\n", ast_str_strlen(s));
+			res = AST_TEST_FAIL;
+		} else if (strcmp(ast_str_buffer(s), out)) {
+			ast_test_status_update(test, "Failed to match value of %zu bytes\n", ast_str_strlen(s));
+			res = AST_TEST_FAIL;
+		} else if (ast_db_del("astdbtest", "long")) {
+			ast_test_status_update(test, "Failed to delete astdbtest/long\n");
+			res = AST_TEST_FAIL;
+		}
+
+		if (out) {
+			ast_free(out);
+		}
+	}
+
+	ast_free(s);
+
+	return res;
+}
+
 static int unload_module(void)
 {
 	AST_TEST_UNREGISTER(put_get_del);
 	AST_TEST_UNREGISTER(gettree_deltree);
 	AST_TEST_UNREGISTER(perftest);
+	AST_TEST_UNREGISTER(put_get_long);
 	return 0;
 }
 
@@ -245,6 +304,7 @@ static int load_module(void)
 	AST_TEST_REGISTER(put_get_del);
 	AST_TEST_REGISTER(gettree_deltree);
 	AST_TEST_REGISTER(perftest);
+	AST_TEST_REGISTER(put_get_long);
 	return AST_MODULE_LOAD_SUCCESS;
 }