From 80ca94663da1287ffc8304291260a85c42f92270 Mon Sep 17 00:00:00 2001 From: Richard Fuchs Date: Tue, 16 Jul 2019 10:39:53 -0400 Subject: [PATCH] TT#63000 db_redis: support update of type key columns If an update changes the value of a type key, we must propagate that updated value to the respective type keys. Change-Id: I7ad5c85aa9e7540e2ac558ca6b8c3125f2dfec9a (cherry picked from commit 51196c6b276df47d599533f88e2757ce38deb3ae) --- debian/patches/series | 1 + .../sipwise/db_redis_update_type_keys.patch | 242 ++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 debian/patches/sipwise/db_redis_update_type_keys.patch diff --git a/debian/patches/series b/debian/patches/series index 3e3f2eef2..98c921079 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -43,3 +43,4 @@ sipwise/registrar_add_path.patch upstream/limit_the_execution_of_dialplan_reload.patch sipwise/db_redis_fix_scan_usage.patch sipwise/db_redis_skip_not_eq_type_keys.patch +sipwise/db_redis_update_type_keys.patch diff --git a/debian/patches/sipwise/db_redis_update_type_keys.patch b/debian/patches/sipwise/db_redis_update_type_keys.patch new file mode 100644 index 000000000..468b953d8 --- /dev/null +++ b/debian/patches/sipwise/db_redis_update_type_keys.patch @@ -0,0 +1,242 @@ +--- a/src/modules/db_redis/redis_dbase.c ++++ b/src/modules/db_redis/redis_dbase.c +@@ -1483,6 +1483,15 @@ + int i; + int j; + size_t col; ++ redis_key_t *all_type_keys = NULL; ++ int all_type_keys_count = 0; ++ db_val_t *db_vals = NULL; ++ db_key_t *db_keys = NULL; ++ redis_key_t *type_keys = NULL; ++ int type_keys_count = 0; ++ redis_key_t *new_type_keys = NULL; ++ int new_type_keys_count = 0; ++ redis_key_t *all_type_key; + + if (!(*keys_count) && do_table_scan) { + LM_WARN("performing full table scan on table '%.*s' while performing update\n", +@@ -1499,10 +1508,27 @@ + } + } + ++ // TODO: this should be moved to redis_connection structure ++ // and be parsed at startup: ++ // ++ // fetch list of keys in all types ++ if (db_redis_get_keys_for_all_types(con, CON_TABLE(_h), ++ &all_type_keys, &all_type_keys_count) != 0) { ++ LM_ERR("failed to get full list of type keys\n"); ++ goto error; ++ } ++ ++ if (db_redis_build_type_keys(con, CON_TABLE(_h), _uk, _uv, _nu, ++ &new_type_keys, &new_type_keys_count) != 0) { ++ LM_ERR("failed to build type keys\n"); ++ goto error; ++ } ++ LM_DBG("%i new type keys\n", new_type_keys_count); ++ + for (key = *keys; key; key = key->next) { + str *keyname = &key->key; + +- LM_DBG("fetching row for '%.*s' from redis\n", keyname->len, keyname->s); ++ LM_DBG("fetching row for '%.*s' from redis for update\n", keyname->len, keyname->s); + + + if (db_redis_key_add_string(&query_v, "EXISTS", 6) != 0) { +@@ -1520,19 +1546,9 @@ + db_redis_key_free(&query_v); + + // construct HMGET query +- if ((*manual_keys_count) == 0) { +- if (db_redis_key_add_string(&query_v, "HGETALL", 7) != 0) { +- LM_ERR("Failed to set hgetall command to pre-update hget query\n"); +- goto error; +- } +- // TODO: actually we wouldn't have to fetch it at all, but then we'd +- // have to mark this key telling to not fetch reply of HMGET after +- // EXISTS returns false! +- } else { +- if (db_redis_key_add_string(&query_v, "HMGET", 5) != 0) { +- LM_ERR("Failed to set hgetall command to pre-update hget query\n"); +- goto error; +- } ++ if (db_redis_key_add_string(&query_v, "HMGET", 5) != 0) { ++ LM_ERR("Failed to set hgetall command to pre-update hget query\n"); ++ goto error; + } + if (db_redis_key_add_str(&query_v, keyname) != 0) { + LM_ERR("Failed to add key name to pre-update exists query\n"); +@@ -1547,6 +1563,13 @@ + goto error; + } + } ++ // add all type keys to query ++ for (all_type_key = all_type_keys; all_type_key; all_type_key = all_type_key->next) { ++ if (db_redis_key_add_str(&query_v, &all_type_key->key) != 0) { ++ LM_ERR("Failed to add type key to pre-update query\n"); ++ goto error; ++ } ++ } + + if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) { + LM_ERR("Failed to append redis command\n"); +@@ -1573,9 +1596,12 @@ + + + for (key = *keys; key; key = key->next) { ++ redis_key_t *tmp = NULL; ++ redis_key_t *type_key; ++ redis_key_t *new_type_key; + int row_match; + +- LM_DBG("fetching replies for '%.*s' from redis\n", key->key.len, key->key.s); ++ LM_DBG("fetching replies for '%.*s' from redis for update\n", key->key.len, key->key.s); + + // get reply for EXISTS query + if (db_redis_get_reply(con, (void**)&reply) != REDIS_OK) { +@@ -1632,13 +1658,50 @@ + } + } + } +- db_redis_free_reply(&reply); + if (!row_match) { + continue; + } else { + LM_DBG("row matches manual filtering, proceed with update\n"); + } + ++ db_keys = (db_key_t*) pkg_malloc(all_type_keys_count * sizeof(db_key_t)); ++ if (!db_keys) { ++ LM_ERR("Failed to allocate memory for db type keys\n"); ++ goto error; ++ } ++ for (j = 0, tmp = all_type_keys; tmp; ++j, tmp = tmp->next) { ++ db_keys[j] = &tmp->key; ++ } ++ ++ db_vals = (db_val_t*) pkg_malloc(all_type_keys_count * sizeof(db_val_t)); ++ if (!db_vals) { ++ LM_ERR("Failed to allocate memory for manual db vals\n"); ++ goto error; ++ } ++ ++ for (j = 0, all_type_key = all_type_keys; all_type_key; ++j, all_type_key = all_type_key->next) { ++ db_val_t *v = &(db_vals[j]); ++ str *key = &all_type_key->key; ++ char *value = reply->element[*manual_keys_count + j]->str; ++ int coltype = db_redis_schema_get_column_type(con, CON_TABLE(_h), key); ++ if (value == NULL) { ++ VAL_NULL(v) = 1; ++ } else if (db_str2val(coltype, v, value, strlen(value), 0) != 0) { ++ LM_ERR("Failed to convert redis reply column to db value\n"); ++ goto error; ++ } ++ } ++ if (db_redis_build_type_keys(con, CON_TABLE(_h), db_keys, db_vals, all_type_keys_count, ++ &type_keys, &type_keys_count) != 0) { ++ LM_ERR("failed to build type keys\n"); ++ goto error; ++ } ++ pkg_free(db_keys); ++ db_keys = NULL; ++ pkg_free(db_vals); ++ db_vals = NULL; ++ db_redis_free_reply(&reply); ++ + if (db_redis_key_add_string(&query_v, "HMSET", 5) != 0) { + LM_ERR("Failed to add hmset command to update query\n"); + goto error; +@@ -1674,6 +1737,72 @@ + } + + db_redis_key_free(&query_v); ++ ++ for (type_key = type_keys; type_key; type_key = type_key->next) { ++ LM_DBG("checking for update of type key '%.*s'\n", ++ type_key->key.len, type_key->key.s); ++ char *prefix = ser_memmem(type_key->key.s, "::", type_key->key.len, 2); ++ if (!prefix || prefix == type_key->key.s) { ++ LM_DBG("Invalid key without :: '%.*s'\n", ++ type_key->key.len, type_key->key.s); ++ goto error; ++ } ++ for (new_type_key = new_type_keys; new_type_key; new_type_key = new_type_key->next) { ++ // compare prefix to see if this is the same key ++ if (memcmp(new_type_key->key.s, type_key->key.s, prefix - type_key->key.s)) ++ continue; ++ LM_DBG("checking for update of type key against '%.*s'\n", ++ new_type_key->key.len, new_type_key->key.s); ++ if (!str_strcmp(&new_type_key->key, &type_key->key)) ++ continue; ++ ++ // add to new set key and delete from old ++ ++ if (db_redis_key_add_string(&query_v, "SADD", 4) != 0) { ++ LM_ERR("Failed to set sadd command to post-update query\n"); ++ goto error; ++ } ++ if (db_redis_key_add_str(&query_v, &new_type_key->key) != 0) { ++ LM_ERR("Failed to add map key to post-update query\n"); ++ goto error; ++ } ++ if (db_redis_key_add_str(&query_v, &key->key) != 0) { ++ LM_ERR("Failed to set entry key to post-update query\n"); ++ goto error; ++ } ++ ++ update_queries++; ++ if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) { ++ LM_ERR("Failed to append redis command\n"); ++ goto error; ++ } ++ ++ db_redis_key_free(&query_v); ++ ++ if (db_redis_key_add_string(&query_v, "SREM", 4) != 0) { ++ LM_ERR("Failed to set sadd command to post-update query\n"); ++ goto error; ++ } ++ if (db_redis_key_add_str(&query_v, &type_key->key) != 0) { ++ LM_ERR("Failed to add map key to post-update query\n"); ++ goto error; ++ } ++ if (db_redis_key_add_str(&query_v, &key->key) != 0) { ++ LM_ERR("Failed to set entry key to post-update query\n"); ++ goto error; ++ } ++ ++ update_queries++; ++ if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) { ++ LM_ERR("Failed to append redis command\n"); ++ goto error; ++ } ++ ++ db_redis_key_free(&query_v); ++ } ++ } ++ ++ db_redis_key_free(&type_keys); + } + + LM_DBG("getting replies for %d queries\n", update_queries); +@@ -1690,6 +1819,8 @@ + + LM_DBG("done performing update\n"); + ++ db_redis_key_free(&all_type_keys); ++ db_redis_key_free(&new_type_keys); + return 0; + + error: +@@ -1697,6 +1828,9 @@ + if (reply) + db_redis_free_reply(&reply); + db_redis_key_free(&query_v); ++ db_redis_key_free(&all_type_keys); ++ db_redis_key_free(&type_keys); ++ db_redis_key_free(&new_type_keys); + return -1; + } +