db.c: Remove limit on family/key length

Consumers like media_cache have been running into issues with
the previous astdb "/family/key" limit of 253 bytes when needing
to store things like long URIs.  An Amazon S3 URI is a good example
of this.  Now, instead of using a static 256 byte buffer for
"/family/key", we use ast_asprintf() to dynamically create it.

Both test_db.c and test_media_cache.c were also updated to use
keys/URIs over the old 253 character limit.

Resolves: #881

UserNote: The `ast_db_*()` APIs have had the 253 byte limit on
"/family/key" removed and will now accept families and keys with a
total length of up to SQLITE_MAX_LENGTH (currently 1e9!).  This
affects the `DB*` dialplan applications, dialplan functions,
manager actions and `databse` CLI commands.  Since the
media_cache also uses the `ast_db_*()` APIs, you can now store
resources with URIs longer than 253 bytes.
pull/903/head
George Joseph 8 months ago committed by asterisk-org-access-app[bot]
parent 7f2776129d
commit d1d37d447c

@ -116,7 +116,6 @@
</manager>
***/
#define MAX_DB_FIELD 256
AST_MUTEX_DEFINE_STATIC(dblock);
static ast_cond_t dbcond;
static sqlite3 *astdb;
@ -138,7 +137,7 @@ DEFINE_SQL_STATEMENT(deltree_all_stmt, "DELETE FROM astdb")
DEFINE_SQL_STATEMENT(gettree_stmt, "SELECT key, value FROM astdb WHERE key || '/' LIKE ? || '/' || '%' ORDER BY key")
DEFINE_SQL_STATEMENT(gettree_all_stmt, "SELECT key, value FROM astdb ORDER BY key")
DEFINE_SQL_STATEMENT(showkey_stmt, "SELECT key, value FROM astdb WHERE key LIKE '%' || '/' || ? ORDER BY key")
DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key VARCHAR(256), value VARCHAR(256), PRIMARY KEY(key))")
DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key TEXT, value TEXT, PRIMARY KEY(key))")
/* This query begs an explanation:
*
@ -341,17 +340,17 @@ static int ast_db_rollback_transaction(void)
int ast_db_put(const char *family, const char *key, const char *value)
{
char fullkey[MAX_DB_FIELD];
size_t fullkey_len;
char *fullkey;
int fullkey_len;
int res = 0;
if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
if (fullkey_len < 0) {
ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
family, key);
return -1;
}
fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
ast_mutex_lock(&dblock);
if (sqlite3_bind_text(put_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
@ -363,10 +362,10 @@ int ast_db_put(const char *family, const char *key, const char *value)
ast_log(LOG_WARNING, "Couldn't execute statement: %s\n", sqlite3_errmsg(astdb));
res = -1;
}
sqlite3_reset(put_stmt);
db_sync();
ast_mutex_unlock(&dblock);
ast_free(fullkey);
return res;
}
@ -388,17 +387,17 @@ int ast_db_put(const char *family, const char *key, const char *value)
static int db_get_common(const char *family, const char *key, char **buffer, int bufferlen)
{
const unsigned char *result;
char fullkey[MAX_DB_FIELD];
size_t fullkey_len;
char *fullkey;
int fullkey_len;
int res = 0;
if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
if (fullkey_len < 0) {
ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
family, key);
return -1;
}
fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
ast_mutex_lock(&dblock);
if (sqlite3_bind_text(get_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
@ -420,6 +419,7 @@ static int db_get_common(const char *family, const char *key, char **buffer, int
}
sqlite3_reset(get_stmt);
ast_mutex_unlock(&dblock);
ast_free(fullkey);
return res;
}
@ -444,13 +444,14 @@ int ast_db_get_allocated(const char *family, const char *key, char **out)
int ast_db_exists(const char *family, const char *key)
{
int result;
char fullkey[MAX_DB_FIELD];
size_t fullkey_len;
char *fullkey;
int fullkey_len;
int res = 0;
fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
if (fullkey_len >= sizeof(fullkey)) {
ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
if (fullkey_len < 0) {
ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
family, key);
return -1;
}
@ -468,6 +469,7 @@ int ast_db_exists(const char *family, const char *key)
}
sqlite3_reset(exists_stmt);
ast_mutex_unlock(&dblock);
ast_free(fullkey);
return res;
}
@ -475,17 +477,17 @@ int ast_db_exists(const char *family, const char *key)
int ast_db_del(const char *family, const char *key)
{
char fullkey[MAX_DB_FIELD];
size_t fullkey_len;
char *fullkey;
int fullkey_len;
int res = 0;
if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
if (fullkey_len < 0) {
ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
family, key);
return -1;
}
fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
ast_mutex_lock(&dblock);
if (sqlite3_bind_text(del_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
@ -497,24 +499,25 @@ int ast_db_del(const char *family, const char *key)
sqlite3_reset(del_stmt);
db_sync();
ast_mutex_unlock(&dblock);
ast_free(fullkey);
return res;
}
int ast_db_del2(const char *family, const char *key)
{
char fullkey[MAX_DB_FIELD];
char *fullkey;
char tmp[1];
size_t fullkey_len;
int fullkey_len;
int mres, res = 0;
if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
if (fullkey_len < 0) {
ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
family, key);
return -1;
}
fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
ast_mutex_lock(&dblock);
if (ast_db_get(family, key, tmp, sizeof(tmp))) {
ast_log(LOG_WARNING, "AstDB key %s does not exist\n", fullkey);
@ -529,31 +532,58 @@ int ast_db_del2(const char *family, const char *key)
sqlite3_reset(del_stmt);
db_sync();
ast_mutex_unlock(&dblock);
ast_free(fullkey);
return res;
}
int ast_db_deltree(const char *family, const char *keytree)
static char *create_prefix(const char *family, const char *keytree,
int *prefix_len)
{
sqlite3_stmt *stmt = deltree_stmt;
char prefix[MAX_DB_FIELD];
int res = 0;
char *prefix = NULL;
*prefix_len = 0;
if (!ast_strlen_zero(family)) {
if (!ast_strlen_zero(keytree)) {
/* Family and key tree */
snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
*prefix_len = ast_asprintf(&prefix, "/%s/%s", family, keytree);
} else {
/* Family only */
snprintf(prefix, sizeof(prefix), "/%s", family);
*prefix_len = ast_asprintf(&prefix, "/%s", family);
}
if (*prefix_len < 0) {
ast_log(LOG_WARNING, "Unable to allocate memory for family/keytree '/%s%s%s'\n",
S_OR(family, ""), S_COR(keytree, "/", ""), S_OR(keytree, ""));
return NULL;
}
} else {
prefix[0] = '\0';
prefix = ast_strdup("");
}
return prefix;
}
int ast_db_deltree(const char *family, const char *keytree)
{
sqlite3_stmt *stmt = deltree_stmt;
char *prefix = NULL;
int prefix_len = 0;
int res = 0;
if (ast_strlen_zero(family) && !ast_strlen_zero(keytree)) {
ast_log(LOG_WARNING, "Key tree '%s' specified without family\n", keytree);
return -1;
}
prefix = create_prefix(family, keytree, &prefix_len);
if (!prefix) {
return -1;
}
if (prefix_len == 0) {
stmt = deltree_all_stmt;
}
ast_mutex_lock(&dblock);
if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
if (prefix_len && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
ast_log(LOG_WARNING, "Couldn't bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
res = -1;
} else if (sqlite3_step(stmt) != SQLITE_DONE) {
@ -564,6 +594,7 @@ int ast_db_deltree(const char *family, const char *keytree)
sqlite3_reset(stmt);
db_sync();
ast_mutex_unlock(&dblock);
ast_free(prefix);
return res;
}
@ -609,67 +640,60 @@ static struct ast_db_entry *db_gettree_common(sqlite3_stmt *stmt)
struct ast_db_entry *ast_db_gettree(const char *family, const char *keytree)
{
char prefix[MAX_DB_FIELD];
char *prefix = NULL;
int prefix_len = 0;
sqlite3_stmt *stmt = gettree_stmt;
size_t res = 0;
struct ast_db_entry *ret;
if (!ast_strlen_zero(family)) {
if (!ast_strlen_zero(keytree)) {
/* Family and key tree */
res = snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
} else {
/* Family only */
res = snprintf(prefix, sizeof(prefix), "/%s", family);
}
if (res >= sizeof(prefix)) {
ast_log(LOG_WARNING, "Requested prefix is too long: %s\n", keytree);
return NULL;
}
} else {
prefix[0] = '\0';
prefix = create_prefix(family, keytree, &prefix_len);
if (!prefix) {
return NULL;
}
if (prefix_len == 0) {
stmt = gettree_all_stmt;
}
ast_mutex_lock(&dblock);
if (res && (sqlite3_bind_text(stmt, 1, prefix, res, SQLITE_STATIC) != SQLITE_OK)) {
if (prefix_len && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
ast_log(LOG_WARNING, "Could not bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
sqlite3_reset(stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
return NULL;
}
ret = db_gettree_common(stmt);
sqlite3_reset(stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
return ret;
}
struct ast_db_entry *ast_db_gettree_by_prefix(const char *family, const char *key_prefix)
{
char prefix[MAX_DB_FIELD];
size_t res;
char *prefix = NULL;
int prefix_len = 0;
struct ast_db_entry *ret;
res = snprintf(prefix, sizeof(prefix), "/%s/%s", family, key_prefix);
if (res >= sizeof(prefix)) {
ast_log(LOG_WARNING, "Requested key prefix is too long: %s\n", key_prefix);
prefix = create_prefix(family, key_prefix, &prefix_len);
if (!prefix) {
return NULL;
}
ast_mutex_lock(&dblock);
if (sqlite3_bind_text(gettree_prefix_stmt, 1, prefix, res, SQLITE_STATIC) != SQLITE_OK) {
if (sqlite3_bind_text(gettree_prefix_stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK) {
ast_log(LOG_WARNING, "Could not bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
sqlite3_reset(gettree_prefix_stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
return NULL;
}
ret = db_gettree_common(gettree_prefix_stmt);
sqlite3_reset(gettree_prefix_stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
return ret;
}
@ -714,7 +738,7 @@ static char *handle_cli_database_put(struct ast_cli_entry *e, int cmd, struct as
static char *handle_cli_database_get(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
{
int res;
char tmp[MAX_DB_FIELD];
char *tmp = NULL;
switch (cmd) {
case CLI_INIT:
@ -730,12 +754,14 @@ static char *handle_cli_database_get(struct ast_cli_entry *e, int cmd, struct as
if (a->argc != 4)
return CLI_SHOWUSAGE;
res = ast_db_get(a->argv[2], a->argv[3], tmp, sizeof(tmp));
res = ast_db_get_allocated(a->argv[2], a->argv[3], &tmp);
if (res) {
ast_cli(a->fd, "Database entry not found.\n");
} else {
ast_cli(a->fd, "Value: %s\n", tmp);
ast_free(tmp);
}
return CLI_SUCCESS;
}
@ -803,7 +829,10 @@ static char *handle_cli_database_deltree(struct ast_cli_entry *e, int cmd, struc
static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
{
char prefix[MAX_DB_FIELD];
char *prefix = NULL;
int prefix_len = 0;
const char *family = a->argc > 2 ? a->argv[2] : "";
const char *keytree = a->argc > 3 ? a->argv[3] : "";
int counter = 0;
sqlite3_stmt *stmt = gettree_stmt;
@ -821,26 +850,24 @@ static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct a
return NULL;
}
if (a->argc == 4) {
/* Family and key tree */
snprintf(prefix, sizeof(prefix), "/%s/%s", a->argv[2], a->argv[3]);
} else if (a->argc == 3) {
/* Family only */
snprintf(prefix, sizeof(prefix), "/%s", a->argv[2]);
} else if (a->argc == 2) {
/* Neither */
prefix[0] = '\0';
stmt = gettree_all_stmt;
} else {
if (a->argc > 4) {
return CLI_SHOWUSAGE;
}
prefix = create_prefix(family, keytree, &prefix_len);
if (!prefix) {
return NULL;
}
if (prefix_len == 0) {
stmt = gettree_all_stmt;
}
ast_mutex_lock(&dblock);
if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
if (prefix_len && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
ast_log(LOG_WARNING, "Couldn't bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
sqlite3_reset(stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
return NULL;
}
@ -860,6 +887,7 @@ static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct a
sqlite3_reset(stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
ast_cli(a->fd, "%d results found.\n", counter);
return CLI_SUCCESS;
@ -990,7 +1018,7 @@ static int manager_dbget(struct mansession *s, const struct message *m)
char idText[256];
const char *family = astman_get_header(m, "Family");
const char *key = astman_get_header(m, "Key");
char tmp[MAX_DB_FIELD];
char *tmp = NULL;
int res;
if (ast_strlen_zero(family)) {
@ -1006,7 +1034,7 @@ static int manager_dbget(struct mansession *s, const struct message *m)
if (!ast_strlen_zero(id))
snprintf(idText, sizeof(idText) ,"ActionID: %s\r\n", id);
res = ast_db_get(family, key, tmp, sizeof(tmp));
res = ast_db_get_allocated(family, key, &tmp);
if (res) {
astman_send_error(s, m, "Database entry not found");
} else {
@ -1019,7 +1047,7 @@ static int manager_dbget(struct mansession *s, const struct message *m)
"%s"
"\r\n",
family, key, tmp, idText);
ast_free(tmp);
astman_send_list_complete_start(s, m, "DBGetComplete", 1);
astman_send_list_complete_end(s);
}
@ -1028,7 +1056,8 @@ static int manager_dbget(struct mansession *s, const struct message *m)
static int manager_db_tree_get(struct mansession *s, const struct message *m)
{
char prefix[MAX_DB_FIELD];
char *prefix;
int prefix_len = 0;
char idText[256];
const char *id = astman_get_header(m,"ActionID");
const char *family = astman_get_header(m, "Family");
@ -1036,15 +1065,12 @@ static int manager_db_tree_get(struct mansession *s, const struct message *m)
sqlite3_stmt *stmt = gettree_stmt;
int count = 0;
if (!ast_strlen_zero(family) && !ast_strlen_zero(key)) {
/* Family and key tree */
snprintf(prefix, sizeof(prefix), "/%s/%s", family, key);
} else if (!ast_strlen_zero(family)) {
/* Family only */
snprintf(prefix, sizeof(prefix), "/%s", family);
} else {
/* Neither */
prefix[0] = '\0';
prefix = create_prefix(family, key, &prefix_len);
if (!prefix) {
astman_send_error(s, m, "Unable to allocate memory for Family/Key");
return 0;
}
if (prefix_len == 0) {
stmt = gettree_all_stmt;
}
@ -1054,10 +1080,11 @@ static int manager_db_tree_get(struct mansession *s, const struct message *m)
}
ast_mutex_lock(&dblock);
if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
ast_log(LOG_WARNING, "Couldn't bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
sqlite3_reset(stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
astman_send_error(s, m, "Unable to search database");
return 0;
}
@ -1085,6 +1112,7 @@ static int manager_db_tree_get(struct mansession *s, const struct message *m)
sqlite3_reset(stmt);
ast_mutex_unlock(&dblock);
ast_free(prefix);
astman_send_list_complete_start(s, m, "DBGetTreeComplete", count);
astman_send_list_complete_end(s);

@ -235,6 +235,7 @@ AST_TEST_DEFINE(put_get_long)
{
int res = AST_TEST_PASS;
struct ast_str *s;
struct ast_str *key;
int i, j;
#define STR_FILL_32 "abcdefghijklmnopqrstuvwxyz123456"
@ -245,7 +246,7 @@ AST_TEST_DEFINE(put_get_long)
info->category = "/main/astdb/";
info->summary = "ast_db_(put|get_allocated) unit test";
info->description =
"Ensures that the ast_db_put and ast_db_get_allocated functions work";
"Ensures that the ast_db_put and ast_db_get_allocated functions work with log key and long data";
return AST_TEST_NOT_RUN;
case TEST_EXECUTE:
break;
@ -255,25 +256,34 @@ AST_TEST_DEFINE(put_get_long)
return AST_TEST_FAIL;
}
if (!(key = ast_str_create(512))) {
return AST_TEST_FAIL;
}
for (j = 0; j < 512; j += sizeof(STR_FILL_32) - 1) {
ast_str_append(&key, 0, "%s", STR_FILL_32);
}
for (i = 1024; i <= 1024 * 1024 * 8; i *= 2) {
char *out = NULL;
ast_str_reset(s);
ast_str_reset(key);
for (j = 0; j < i; j += sizeof(STR_FILL_32) - 1) {
ast_str_append(&s, 0, "%s", STR_FILL_32);
}
if (ast_db_put("astdbtest", "long", ast_str_buffer(s))) {
if (ast_db_put("astdbtest", ast_str_buffer(key), ast_str_buffer(s))) {
ast_test_status_update(test, "Failed to put value of %zu bytes\n", ast_str_strlen(s));
res = AST_TEST_FAIL;
} else if (ast_db_get_allocated("astdbtest", "long", &out)) {
} else if (ast_db_get_allocated("astdbtest", ast_str_buffer(key), &out)) {
ast_test_status_update(test, "Failed to get value of %zu bytes\n", ast_str_strlen(s));
res = AST_TEST_FAIL;
} else if (strcmp(ast_str_buffer(s), out)) {
ast_test_status_update(test, "Failed to match value of %zu bytes\n", ast_str_strlen(s));
res = AST_TEST_FAIL;
} else if (ast_db_del("astdbtest", "long")) {
} else if (ast_db_del("astdbtest", ast_str_buffer(key))) {
ast_test_status_update(test, "Failed to delete astdbtest/long\n");
res = AST_TEST_FAIL;
}
@ -284,6 +294,7 @@ AST_TEST_DEFINE(put_get_long)
}
ast_free(s);
ast_free(key);
return res;
}

@ -41,8 +41,8 @@
/*! The unit test category */
#define CATEGORY "/main/media_cache/"
/*! A 'valid' resource for the test bucket behind the media cache facade */
#define VALID_RESOURCE "httptest://localhost:8088/test_media_cache/monkeys.wav"
/*! A 'valid' long resource for the test bucket behind the media cache facade */
#define VALID_RESOURCE "httptest://localhost:8088/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/test_media_cache/monkeys.wav"
/*! An 'invalid' resource for the test bucket behind the media cache facade */
#define INVALID_RESOURCE "httptest://localhost:8088/test_media_cache/bad.wav"
@ -109,6 +109,7 @@ static void *bucket_http_test_wizard_retrieve_id(const struct ast_sorcery *sorce
static int bucket_http_test_wizard_delete(const struct ast_sorcery *sorcery, void *data,
void *object)
{
ast_media_cache_delete(VALID_RESOURCE);
if (!strcmp(ast_sorcery_object_get_id(object), VALID_RESOURCE)) {
return 0;
}
@ -250,6 +251,8 @@ AST_TEST_DEFINE(create_update_nominal)
ast_test_validate(test, res == 0);
ast_test_validate(test, strcmp(file_path, tmp_path_two) == 0);
ast_media_cache_delete(VALID_RESOURCE);
unlink(tmp_path_one);
unlink(tmp_path_two);

Loading…
Cancel
Save