diff --git a/cdr/cdr_custom.c b/cdr/cdr_custom.c index e520263732..b44d12cef9 100644 --- a/cdr/cdr_custom.c +++ b/cdr/cdr_custom.c @@ -28,6 +28,9 @@ * * Logs in LOG_DIR/cdr_custom * \ingroup cdr_drivers + * + * The logic for this module now resides in res/res_cdrel_custom.c. + * */ /*! \li \ref cdr_custom.c uses the configuration file \ref cdr_custom.conf @@ -40,192 +43,73 @@ */ /*** MODULEINFO + res_cdrel_custom core ***/ #include "asterisk.h" -#include - -#include "asterisk/paths.h" /* use ast_config_AST_LOG_DIR */ -#include "asterisk/channel.h" #include "asterisk/cdr.h" #include "asterisk/module.h" -#include "asterisk/config.h" -#include "asterisk/pbx.h" -#include "asterisk/utils.h" -#include "asterisk/lock.h" -#include "asterisk/threadstorage.h" -#include "asterisk/strings.h" +#include "asterisk/res_cdrel_custom.h" #define CONFIG "cdr_custom.conf" -AST_THREADSTORAGE(custom_buf); +#define CUSTOM_BACKEND_NAME "CDR File custom backend" -static const char name[] = "cdr-custom"; +static struct cdrel_configs *configs; -struct cdr_custom_config { - AST_DECLARE_STRING_FIELDS( - AST_STRING_FIELD(filename); - AST_STRING_FIELD(format); - ); - ast_mutex_t lock; - AST_RWLIST_ENTRY(cdr_custom_config) list; -}; - -static AST_RWLIST_HEAD_STATIC(sinks, cdr_custom_config); - -static void free_config(void) -{ - struct cdr_custom_config *sink; +/*! + * Protects in-flight log transactions from reloads. + */ +static ast_rwlock_t configs_lock; - while ((sink = AST_RWLIST_REMOVE_HEAD(&sinks, list))) { - ast_mutex_destroy(&sink->lock); - ast_string_field_free_memory(sink); - ast_free(sink); - } -} +#define CDREL_RECORD_TYPE cdrel_record_cdr +#define CDREL_BACKEND_TYPE cdrel_backend_text -static int load_config(void) +static int custom_log(struct ast_cdr *cdr) { - struct ast_config *cfg; - struct ast_variable *var; - struct ast_flags config_flags = { 0 }; int res = 0; - cfg = ast_config_load(CONFIG, config_flags); - if (!cfg || cfg == CONFIG_STATUS_FILEINVALID) { - ast_log(LOG_ERROR, "Unable to load " CONFIG ". Not logging custom CSV CDRs.\n"); - return -1; - } - - var = ast_variable_browse(cfg, "mappings"); - while (var) { - if (!ast_strlen_zero(var->name) && !ast_strlen_zero(var->value)) { - struct cdr_custom_config *sink = ast_calloc_with_stringfields(1, struct cdr_custom_config, 1024); - - if (!sink) { - ast_log(LOG_ERROR, "Unable to allocate memory for configuration settings.\n"); - res = -2; - break; - } - - ast_string_field_build(sink, format, "%s\n", var->value); - if (var->name[0] == '/') { - ast_string_field_build(sink, filename, "%s", var->name); - } else { - ast_string_field_build(sink, filename, "%s/%s/%s", ast_config_AST_LOG_DIR, name, var->name); - } - ast_mutex_init(&sink->lock); - - AST_RWLIST_INSERT_TAIL(&sinks, sink, list); - } else { - ast_log(LOG_NOTICE, "Mapping must have both a filename and a format at line %d\n", var->lineno); - } - var = var->next; - } - ast_config_destroy(cfg); + ast_rwlock_rdlock(&configs_lock); + res = cdrel_logger(configs, cdr); + ast_rwlock_unlock(&configs_lock); return res; } -static int custom_log(struct ast_cdr *cdr) -{ - struct ast_channel *dummy; - struct ast_str *str; - struct cdr_custom_config *config; - - /* Batching saves memory management here. Otherwise, it's the same as doing an allocation and free each time. */ - if (!(str = ast_str_thread_get(&custom_buf, 16))) { - return -1; - } - - dummy = ast_dummy_channel_alloc(); - if (!dummy) { - ast_log(LOG_ERROR, "Unable to allocate channel for variable substitution.\n"); - return -1; - } - - /* We need to dup here since the cdr actually belongs to the other channel, - so when we release this channel we don't want the CDR getting cleaned - up prematurely. */ - ast_channel_cdr_set(dummy, ast_cdr_dup(cdr)); - - AST_RWLIST_RDLOCK(&sinks); - - AST_LIST_TRAVERSE(&sinks, config, list) { - FILE *out; - - ast_str_substitute_variables(&str, 0, dummy, config->format); - - /* Even though we have a lock on the list, we could be being chased by - another thread and this lock ensures that we won't step on anyone's - toes. Once each CDR backend gets it's own thread, this lock can be - removed. */ - ast_mutex_lock(&config->lock); - - /* Because of the absolutely unconditional need for the - highest reliability possible in writing billing records, - we open write and close the log file each time */ - if ((out = fopen(config->filename, "a"))) { - fputs(ast_str_buffer(str), out); - fflush(out); /* be particularly anal here */ - fclose(out); - } else { - ast_log(LOG_ERROR, "Unable to re-open master file %s : %s\n", config->filename, strerror(errno)); - } - - ast_mutex_unlock(&config->lock); - } - - AST_RWLIST_UNLOCK(&sinks); - - ast_channel_unref(dummy); - - return 0; -} - static int unload_module(void) { - if (ast_cdr_unregister(name)) { - return -1; - } + int res = 0; - if (AST_RWLIST_WRLOCK(&sinks)) { - ast_cdr_register(name, ast_module_info->description, custom_log); - ast_log(LOG_ERROR, "Unable to lock sink list. Unload failed.\n"); - return -1; + ast_rwlock_wrlock(&configs_lock); + res = cdrel_unload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, configs, CUSTOM_BACKEND_NAME); + ast_rwlock_unlock(&configs_lock); + if (res == 0) { + ast_rwlock_destroy(&configs_lock); } - free_config(); - AST_RWLIST_UNLOCK(&sinks); - return 0; + return res; } static enum ast_module_load_result load_module(void) { - if (AST_RWLIST_WRLOCK(&sinks)) { - ast_log(LOG_ERROR, "Unable to lock sink list. Load failed.\n"); + if (ast_rwlock_init(&configs_lock) != 0) { return AST_MODULE_LOAD_DECLINE; } - load_config(); - AST_RWLIST_UNLOCK(&sinks); - ast_cdr_register(name, ast_module_info->description, custom_log); - return AST_MODULE_LOAD_SUCCESS; + configs = cdrel_load_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, CONFIG, CUSTOM_BACKEND_NAME, custom_log); + + return configs ? AST_MODULE_LOAD_SUCCESS : AST_MODULE_LOAD_DECLINE; } static int reload(void) { - if (AST_RWLIST_WRLOCK(&sinks)) { - ast_log(LOG_ERROR, "Unable to lock sink list. Load failed.\n"); - return AST_MODULE_LOAD_DECLINE; - } - - free_config(); - load_config(); - AST_RWLIST_UNLOCK(&sinks); - return AST_MODULE_LOAD_SUCCESS; + int res = 0; + ast_rwlock_wrlock(&configs_lock); + res = cdrel_reload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, &configs, CONFIG); + ast_rwlock_unlock(&configs_lock); + return res; } AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_LOAD_ORDER, "Customizable Comma Separated Values CDR Backend", @@ -234,5 +118,5 @@ AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_LOAD_ORDER, "Customizable Comma Se .unload = unload_module, .reload = reload, .load_pri = AST_MODPRI_CDR_DRIVER, - .requires = "cdr", + .requires = "cdr,res_cdrel_custom", ); diff --git a/cdr/cdr_sqlite3_custom.c b/cdr/cdr_sqlite3_custom.c index 4df94d4666..1aa7ad742c 100644 --- a/cdr/cdr_sqlite3_custom.c +++ b/cdr/cdr_sqlite3_custom.c @@ -25,14 +25,16 @@ * cdr_mysql_custom by Edward Eastman , * and cdr_sqlite by Holger Schurig * - * * \arg See also \ref AstCDR * - * * \ingroup cdr_drivers + * + * The logic for this module now resides in res/res_cdrel_custom.c. + * */ /*** MODULEINFO + res_cdrel_custom sqlite3 extended ***/ @@ -41,316 +43,66 @@ #include -#include "asterisk/paths.h" /* use ast_config_AST_LOG_DIR */ -#include "asterisk/channel.h" #include "asterisk/cdr.h" #include "asterisk/module.h" -#include "asterisk/config.h" -#include "asterisk/pbx.h" -#include "asterisk/utils.h" -#include "asterisk/cli.h" -#include "asterisk/app.h" - -AST_MUTEX_DEFINE_STATIC(lock); - -static const char config_file[] = "cdr_sqlite3_custom.conf"; - -static const char desc[] = "Customizable SQLite3 CDR Backend"; -static const char name[] = "cdr_sqlite3_custom"; -static sqlite3 *db = NULL; - -static char table[80]; -static char *columns; -static int busy_timeout; - -struct values { - AST_LIST_ENTRY(values) list; - char expression[1]; -}; - -static AST_LIST_HEAD_STATIC(sql_values, values); - -static void free_config(int reload); - -static int load_column_config(const char *tmp) -{ - char *col = NULL; - char *cols = NULL, *save = NULL; - char *escaped = NULL; - struct ast_str *column_string = NULL; - - if (ast_strlen_zero(tmp)) { - ast_log(LOG_WARNING, "Column names not specified. Module not loaded.\n"); - return -1; - } - if (!(column_string = ast_str_create(1024))) { - ast_log(LOG_ERROR, "Out of memory creating temporary buffer for column list for table '%s.'\n", table); - return -1; - } - if (!(save = cols = ast_strdup(tmp))) { - ast_log(LOG_ERROR, "Out of memory creating temporary buffer for column list for table '%s.'\n", table); - ast_free(column_string); - return -1; - } - while ((col = strsep(&cols, ","))) { - col = ast_strip(col); - escaped = sqlite3_mprintf("%q", col); - if (!escaped) { - ast_log(LOG_ERROR, "Out of memory creating entry for column '%s' in table '%s.'\n", col, table); - ast_free(column_string); - ast_free(save); - return -1; - } - ast_str_append(&column_string, 0, "%s%s", ast_str_strlen(column_string) ? "," : "", escaped); - sqlite3_free(escaped); - } - if (!(columns = ast_strdup(ast_str_buffer(column_string)))) { - ast_log(LOG_ERROR, "Out of memory copying columns string for table '%s.'\n", table); - ast_free(column_string); - ast_free(save); - return -1; - } - ast_free(column_string); - ast_free(save); - - return 0; -} - -static int load_values_config(const char *tmp) -{ - char *vals = NULL, *save = NULL; - struct values *value = NULL; - int i; - AST_DECLARE_APP_ARGS(val, - AST_APP_ARG(ues)[200]; /* More than 200 columns in this CDR? Yeah, right... */ - ); - - if (ast_strlen_zero(tmp)) { - ast_log(LOG_WARNING, "Values not specified. Module not loaded.\n"); - return -1; - } - if (!(save = vals = ast_strdup(tmp))) { - ast_log(LOG_ERROR, "Out of memory creating temporary buffer for value '%s'\n", tmp); - return -1; - } - AST_STANDARD_RAW_ARGS(val, vals); - for (i = 0; i < val.argc; i++) { - /* Strip the single quotes off if they are there */ - char *v = ast_strip_quoted(val.ues[i], "'", "'"); - value = ast_calloc(sizeof(char), sizeof(*value) + strlen(v)); - if (!value) { - ast_log(LOG_ERROR, "Out of memory creating entry for value '%s'\n", v); - ast_free(save); - return -1; - } - strcpy(value->expression, v); /* SAFE */ - AST_LIST_INSERT_TAIL(&sql_values, value, list); - } - ast_free(save); - - return 0; -} - -static int load_config(int reload) -{ - struct ast_config *cfg; - struct ast_flags config_flags = { reload ? CONFIG_FLAG_FILEUNCHANGED : 0 }; - const char *tmp; - - if ((cfg = ast_config_load(config_file, config_flags)) == CONFIG_STATUS_FILEMISSING || cfg == CONFIG_STATUS_FILEINVALID) { - ast_log(LOG_WARNING, "Failed to %sload configuration file. %s\n", reload ? "re" : "", reload ? "" : "Module not activated."); - return -1; - } else if (cfg == CONFIG_STATUS_FILEUNCHANGED) { - return 0; - } - - if (reload) { - free_config(1); - } - - if (!ast_variable_browse(cfg, "master")) { - /* Nothing configured */ - ast_config_destroy(cfg); - return -1; - } - - /* Mapping must have a table name */ - if (!ast_strlen_zero(tmp = ast_variable_retrieve(cfg, "master", "table"))) { - ast_copy_string(table, tmp, sizeof(table)); - } else { - ast_log(LOG_WARNING, "Table name not specified. Assuming cdr.\n"); - strcpy(table, "cdr"); - } +#include "asterisk/res_cdrel_custom.h" - /* sqlite3_busy_timeout in miliseconds */ - if ((tmp = ast_variable_retrieve(cfg, "master", "busy_timeout")) != NULL) { - if (ast_parse_arg(tmp, PARSE_INT32|PARSE_DEFAULT, &busy_timeout, 1000) != 0) { - ast_log(LOG_WARNING, "Invalid busy_timeout value '%s' specified. Using 1000 instead.\n", tmp); - } - } else { - busy_timeout = 1000; - } - - /* Columns */ - if (load_column_config(ast_variable_retrieve(cfg, "master", "columns"))) { - ast_config_destroy(cfg); - free_config(0); - return -1; - } - - /* Values */ - if (load_values_config(ast_variable_retrieve(cfg, "master", "values"))) { - ast_config_destroy(cfg); - free_config(0); - return -1; - } +#define CONFIG "cdr_sqlite3_custom.conf" - ast_verb(4, "cdr_sqlite3_custom: Logging CDR records to table '%s' in 'master.db'\n", table); - - ast_config_destroy(cfg); - - return 0; -} - -static void free_config(int reload) -{ - struct values *value; +#define CUSTOM_BACKEND_NAME "CDR sqlite3 custom backend" - if (!reload && db) { - sqlite3_close(db); - db = NULL; - } +static struct cdrel_configs *configs; - if (columns) { - ast_free(columns); - columns = NULL; - } +/*! + * Protects in-flight log transactions from reloads. + */ +static ast_rwlock_t configs_lock; - while ((value = AST_LIST_REMOVE_HEAD(&sql_values, list))) { - ast_free(value); - } -} +#define CDREL_RECORD_TYPE cdrel_record_cdr +#define CDREL_BACKEND_TYPE cdrel_backend_db -static int write_cdr(struct ast_cdr *cdr) +static int custom_log(struct ast_cdr *cdr) { int res = 0; - char *error = NULL; - char *sql = NULL; - - if (db == NULL) { - /* Should not have loaded, but be failsafe. */ - return 0; - } - ast_mutex_lock(&lock); - - { /* Make it obvious that only sql should be used outside of this block */ - char *escaped; - char subst_buf[2048]; - struct values *value; - struct ast_channel *dummy; - struct ast_str *value_string = ast_str_create(1024); - - dummy = ast_dummy_channel_alloc(); - if (!dummy) { - ast_log(LOG_ERROR, "Unable to allocate channel for variable substitution.\n"); - ast_free(value_string); - ast_mutex_unlock(&lock); - return 0; - } - ast_channel_cdr_set(dummy, ast_cdr_dup(cdr)); - AST_LIST_TRAVERSE(&sql_values, value, list) { - pbx_substitute_variables_helper(dummy, value->expression, subst_buf, sizeof(subst_buf) - 1); - escaped = sqlite3_mprintf("%q", subst_buf); - ast_str_append(&value_string, 0, "%s'%s'", ast_str_strlen(value_string) ? "," : "", escaped); - sqlite3_free(escaped); - } - sql = sqlite3_mprintf("INSERT INTO %q (%s) VALUES (%s)", table, columns, ast_str_buffer(value_string)); - ast_debug(1, "About to log: %s\n", sql); - ast_channel_unref(dummy); - ast_free(value_string); - } - - if (sqlite3_exec(db, sql, NULL, NULL, &error) != SQLITE_OK) { - ast_log(LOG_ERROR, "%s. SQL: %s.\n", error, sql); - sqlite3_free(error); - } - - if (sql) { - sqlite3_free(sql); - } - - ast_mutex_unlock(&lock); + ast_rwlock_rdlock(&configs_lock); + res = cdrel_logger(configs, cdr); + ast_rwlock_unlock(&configs_lock); return res; } static int unload_module(void) { - if (ast_cdr_unregister(name)) { - return -1; - } + int res = 0; - free_config(0); + ast_rwlock_wrlock(&configs_lock); + res = cdrel_unload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, configs, CUSTOM_BACKEND_NAME); + ast_rwlock_unlock(&configs_lock); + if (res == 0) { + ast_rwlock_destroy(&configs_lock); + } - return 0; + return res; } -static int load_module(void) +static enum ast_module_load_result load_module(void) { - char *error; - char filename[PATH_MAX]; - int res; - char *sql; - - if (load_config(0)) { + if (ast_rwlock_init(&configs_lock) != 0) { return AST_MODULE_LOAD_DECLINE; } - /* is the database there? */ - snprintf(filename, sizeof(filename), "%s/master.db", ast_config_AST_LOG_DIR); - res = sqlite3_open(filename, &db); - if (res != SQLITE_OK) { - ast_log(LOG_ERROR, "Could not open database %s.\n", filename); - free_config(0); - return AST_MODULE_LOAD_DECLINE; - } - sqlite3_busy_timeout(db, busy_timeout); - /* is the table there? */ - sql = sqlite3_mprintf("SELECT COUNT(AcctId) FROM %q;", table); - res = sqlite3_exec(db, sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (res != SQLITE_OK) { - /* We don't use %q for the column list here since we already escaped when building it */ - sql = sqlite3_mprintf("CREATE TABLE %q (AcctId INTEGER PRIMARY KEY, %s)", table, columns); - res = sqlite3_exec(db, sql, NULL, NULL, &error); - sqlite3_free(sql); - if (res != SQLITE_OK) { - ast_log(LOG_WARNING, "Unable to create table '%s': %s.\n", table, error); - sqlite3_free(error); - free_config(0); - return AST_MODULE_LOAD_DECLINE; - } - } + configs = cdrel_load_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, CONFIG, CUSTOM_BACKEND_NAME, custom_log); - res = ast_cdr_register(name, desc, write_cdr); - if (res) { - ast_log(LOG_ERROR, "Unable to register custom SQLite3 CDR handling\n"); - free_config(0); - return AST_MODULE_LOAD_DECLINE; - } - - return AST_MODULE_LOAD_SUCCESS; + return configs ? AST_MODULE_LOAD_SUCCESS : AST_MODULE_LOAD_DECLINE; } static int reload(void) { int res = 0; - - ast_mutex_lock(&lock); - res = load_config(1); - ast_mutex_unlock(&lock); - + ast_rwlock_wrlock(&configs_lock); + res = cdrel_reload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, &configs, CONFIG); + ast_rwlock_unlock(&configs_lock); return res; } @@ -360,5 +112,5 @@ AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_LOAD_ORDER, "SQLite3 Custom CDR Mo .unload = unload_module, .reload = reload, .load_pri = AST_MODPRI_CDR_DRIVER, - .requires = "cdr", + .requires = "cdr,res_cdrel_custom", ); diff --git a/cel/cel_custom.c b/cel/cel_custom.c index 5f391e1f21..3c158e44f1 100644 --- a/cel/cel_custom.c +++ b/cel/cel_custom.c @@ -24,197 +24,75 @@ * \author Steve Murphy * Logs in LOG_DIR/cel_custom * \ingroup cel_drivers + * + * The logic for this module now resides in res/res_cdrel_custom.c. + * */ /*** MODULEINFO + res_cdrel_custom core ***/ #include "asterisk.h" -#include "asterisk/paths.h" -#include "asterisk/channel.h" #include "asterisk/cel.h" #include "asterisk/module.h" -#include "asterisk/config.h" -#include "asterisk/pbx.h" -#include "asterisk/utils.h" -#include "asterisk/lock.h" -#include "asterisk/threadstorage.h" -#include "asterisk/strings.h" +#include "asterisk/res_cdrel_custom.h" #define CONFIG "cel_custom.conf" -AST_THREADSTORAGE(custom_buf); - -static const char name[] = "cel-custom"; - -struct cel_config { - AST_DECLARE_STRING_FIELDS( - AST_STRING_FIELD(filename); - AST_STRING_FIELD(format); - ); - ast_mutex_t lock; - AST_RWLIST_ENTRY(cel_config) list; -}; - #define CUSTOM_BACKEND_NAME "CEL Custom CSV Logging" -static AST_RWLIST_HEAD_STATIC(sinks, cel_config); - -static void free_config(void) -{ - struct cel_config *sink; - - while ((sink = AST_RWLIST_REMOVE_HEAD(&sinks, list))) { - ast_mutex_destroy(&sink->lock); - ast_string_field_free_memory(sink); - ast_free(sink); - } -} - -static int load_config(void) -{ - struct ast_config *cfg; - struct ast_variable *var; - struct ast_flags config_flags = { 0 }; - int mappings = 0; - int res = 0; - - cfg = ast_config_load(CONFIG, config_flags); - if (!cfg || cfg == CONFIG_STATUS_FILEINVALID) { - ast_log(LOG_ERROR, "Unable to load " CONFIG ". Not logging CEL to custom CSVs.\n"); - return -1; - } - - if (!(var = ast_variable_browse(cfg, "mappings"))) { - ast_log(LOG_NOTICE, "No mappings found in " CONFIG ". Not logging CEL to custom CSVs.\n"); - } +static struct cdrel_configs *configs; - while (var) { - if (!ast_strlen_zero(var->name) && !ast_strlen_zero(var->value)) { - struct cel_config *sink = ast_calloc_with_stringfields(1, struct cel_config, 1024); - - if (!sink) { - ast_log(LOG_ERROR, "Unable to allocate memory for configuration settings.\n"); - res = -2; - break; - } - - ast_string_field_build(sink, format, "%s\n", var->value); - if (var->name[0] == '/') { - ast_string_field_build(sink, filename, "%s", var->name); - } else { - ast_string_field_build(sink, filename, "%s/%s/%s", ast_config_AST_LOG_DIR, name, var->name); - } - ast_mutex_init(&sink->lock); - - ast_verb(3, "Added CEL CSV mapping for '%s'.\n", sink->filename); - mappings += 1; - AST_RWLIST_INSERT_TAIL(&sinks, sink, list); - } else { - ast_log(LOG_NOTICE, "Mapping must have both a filename and a format at line %d\n", var->lineno); - } - var = var->next; - } - ast_config_destroy(cfg); - - ast_verb(1, "Added CEL CSV mapping for %d files.\n", mappings); +/*! + * Protects in-flight log transactions from reloads. + */ +static ast_rwlock_t configs_lock; - return res; -} +#define CDREL_RECORD_TYPE cdrel_record_cel +#define CDREL_BACKEND_TYPE cdrel_backend_text static void custom_log(struct ast_event *event) { - struct ast_channel *dummy; - struct ast_str *str; - struct cel_config *config; - - /* Batching saves memory management here. Otherwise, it's the same as doing an allocation and free each time. */ - if (!(str = ast_str_thread_get(&custom_buf, 16))) { - return; - } - - dummy = ast_cel_fabricate_channel_from_event(event); - if (!dummy) { - ast_log(LOG_ERROR, "Unable to fabricate channel from CEL event.\n"); - return; - } - - AST_RWLIST_RDLOCK(&sinks); - - AST_LIST_TRAVERSE(&sinks, config, list) { - FILE *out; - - ast_str_substitute_variables(&str, 0, dummy, config->format); - - /* Even though we have a lock on the list, we could be being chased by - another thread and this lock ensures that we won't step on anyone's - toes. Once each CEL backend gets it's own thread, this lock can be - removed. */ - ast_mutex_lock(&config->lock); - - /* Because of the absolutely unconditional need for the - highest reliability possible in writing billing records, - we open write and close the log file each time */ - if ((out = fopen(config->filename, "a"))) { - fputs(ast_str_buffer(str), out); - fflush(out); /* be particularly anal here */ - fclose(out); - } else { - ast_log(LOG_ERROR, "Unable to re-open master file %s : %s\n", config->filename, strerror(errno)); - } - - ast_mutex_unlock(&config->lock); - } - - AST_RWLIST_UNLOCK(&sinks); - - ast_channel_unref(dummy); + ast_rwlock_rdlock(&configs_lock); + cdrel_logger(configs, event); + ast_rwlock_unlock(&configs_lock); } static int unload_module(void) { + int res = 0; - if (AST_RWLIST_WRLOCK(&sinks)) { - ast_log(LOG_ERROR, "Unable to lock sink list. Unload failed.\n"); - return -1; + ast_rwlock_wrlock(&configs_lock); + res = cdrel_unload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, configs, CUSTOM_BACKEND_NAME); + ast_rwlock_unlock(&configs_lock); + if (res == 0) { + ast_rwlock_destroy(&configs_lock); } - free_config(); - AST_RWLIST_UNLOCK(&sinks); - ast_cel_backend_unregister(CUSTOM_BACKEND_NAME); - return 0; + return res; } static enum ast_module_load_result load_module(void) { - if (AST_RWLIST_WRLOCK(&sinks)) { - ast_log(LOG_ERROR, "Unable to lock sink list. Load failed.\n"); + if (ast_rwlock_init(&configs_lock) != 0) { return AST_MODULE_LOAD_DECLINE; } - load_config(); - AST_RWLIST_UNLOCK(&sinks); + configs = cdrel_load_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, CONFIG, CUSTOM_BACKEND_NAME, custom_log); - if (ast_cel_backend_register(CUSTOM_BACKEND_NAME, custom_log)) { - free_config(); - return AST_MODULE_LOAD_DECLINE; - } - return AST_MODULE_LOAD_SUCCESS; + return configs ? AST_MODULE_LOAD_SUCCESS : AST_MODULE_LOAD_DECLINE; } static int reload(void) { - if (AST_RWLIST_WRLOCK(&sinks)) { - ast_log(LOG_ERROR, "Unable to lock sink list. Load failed.\n"); - return AST_MODULE_LOAD_DECLINE; - } - - free_config(); - load_config(); - AST_RWLIST_UNLOCK(&sinks); - return AST_MODULE_LOAD_SUCCESS; + int res = 0; + ast_rwlock_wrlock(&configs_lock); + res = cdrel_reload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, &configs, CONFIG); + ast_rwlock_unlock(&configs_lock); + return res; } AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_LOAD_ORDER, "Customizable Comma Separated Values CEL Backend", @@ -223,5 +101,5 @@ AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_LOAD_ORDER, "Customizable Comma Se .unload = unload_module, .reload = reload, .load_pri = AST_MODPRI_CDR_DRIVER, - .requires = "cel", + .requires = "cel,res_cdrel_custom", ); diff --git a/cel/cel_sqlite3_custom.c b/cel/cel_sqlite3_custom.c index 23a5b43b03..a5acf774e1 100644 --- a/cel/cel_sqlite3_custom.c +++ b/cel/cel_sqlite3_custom.c @@ -27,9 +27,13 @@ * cdr_mysql_custom by Edward Eastman , * and cdr_sqlite by Holger Schurig * \ingroup cel_drivers + * + * The logic for this module now resides in res/res_cdrel_custom.c. + * */ /*** MODULEINFO + res_cdrel_custom sqlite3 extended ***/ @@ -38,314 +42,62 @@ #include -#include "asterisk/paths.h" -#include "asterisk/channel.h" #include "asterisk/cel.h" #include "asterisk/module.h" -#include "asterisk/config.h" -#include "asterisk/pbx.h" -#include "asterisk/logger.h" -#include "asterisk/utils.h" -#include "asterisk/cli.h" -#include "asterisk/options.h" -#include "asterisk/stringfields.h" - -#define SQLITE_BACKEND_NAME "CEL sqlite3 custom backend" +#include "asterisk/res_cdrel_custom.h" -AST_MUTEX_DEFINE_STATIC(lock); +#define CONFIG "cel_sqlite3_custom.conf" -static const char config_file[] = "cel_sqlite3_custom.conf"; +#define CUSTOM_BACKEND_NAME "CEL sqlite3 custom backend" -static sqlite3 *db = NULL; +static struct cdrel_configs *configs; -static char table[80]; /*! - * \bug Handling of this var is crash prone on reloads + * Protects in-flight log transactions from reloads. */ -static char *columns; -static int busy_timeout; - -struct values { - char *expression; - AST_LIST_ENTRY(values) list; -}; - -static AST_LIST_HEAD_STATIC(sql_values, values); +static ast_rwlock_t configs_lock; -static void free_config(void); +#define CDREL_RECORD_TYPE cdrel_record_cel +#define CDREL_BACKEND_TYPE cdrel_backend_db -static int load_column_config(const char *tmp) +static void custom_log(struct ast_event *event) { - char *col = NULL; - char *cols = NULL, *save = NULL; - char *escaped = NULL; - struct ast_str *column_string = NULL; - - if (ast_strlen_zero(tmp)) { - ast_log(LOG_WARNING, "Column names not specified. Module not loaded.\n"); - return -1; - } - if (!(column_string = ast_str_create(1024))) { - ast_log(LOG_ERROR, "Out of memory creating temporary buffer for column list for table '%s.'\n", table); - return -1; - } - if (!(save = cols = ast_strdup(tmp))) { - ast_log(LOG_ERROR, "Out of memory creating temporary buffer for column list for table '%s.'\n", table); - ast_free(column_string); - return -1; - } - while ((col = strsep(&cols, ","))) { - col = ast_strip(col); - escaped = sqlite3_mprintf("%q", col); - if (!escaped) { - ast_log(LOG_ERROR, "Out of memory creating entry for column '%s' in table '%s.'\n", col, table); - ast_free(column_string); - ast_free(save); - return -1; - } - ast_str_append(&column_string, 0, "%s%s", ast_str_strlen(column_string) ? "," : "", escaped); - sqlite3_free(escaped); - } - if (!(columns = ast_strdup(ast_str_buffer(column_string)))) { - ast_log(LOG_ERROR, "Out of memory copying columns string for table '%s.'\n", table); - ast_free(column_string); - ast_free(save); - return -1; - } - ast_free(column_string); - ast_free(save); - - return 0; -} - -static int load_values_config(const char *tmp) -{ - char *val = NULL; - char *vals = NULL, *save = NULL; - struct values *value = NULL; - - if (ast_strlen_zero(tmp)) { - ast_log(LOG_WARNING, "Values not specified. Module not loaded.\n"); - return -1; - } - if (!(save = vals = ast_strdup(tmp))) { - ast_log(LOG_ERROR, "Out of memory creating temporary buffer for value '%s'\n", tmp); - return -1; - } - while ((val = strsep(&vals, ","))) { - /* Strip the single quotes off if they are there */ - val = ast_strip_quoted(val, "'", "'"); - value = ast_calloc(sizeof(char), sizeof(*value) + strlen(val) + 1); - if (!value) { - ast_log(LOG_ERROR, "Out of memory creating entry for value '%s'\n", val); - ast_free(save); - return -1; - } - value->expression = (char *) value + sizeof(*value); - ast_copy_string(value->expression, val, strlen(val) + 1); - AST_LIST_INSERT_TAIL(&sql_values, value, list); - } - ast_free(save); - - return 0; -} - -static int load_config(int reload) -{ - struct ast_config *cfg; - struct ast_flags config_flags = { reload ? CONFIG_FLAG_FILEUNCHANGED : 0 }; - struct ast_variable *mappingvar; - const char *tmp; - - if ((cfg = ast_config_load(config_file, config_flags)) == CONFIG_STATUS_FILEMISSING || cfg == CONFIG_STATUS_FILEINVALID) { - ast_log(LOG_WARNING, "Failed to %sload configuration file. %s\n", - reload ? "re" : "", reload ? "" : "Module not activated."); - return -1; - } else if (cfg == CONFIG_STATUS_FILEUNCHANGED) { - return 0; - } - - if (reload) { - free_config(); - } - - if (!(mappingvar = ast_variable_browse(cfg, "master"))) { - /* Nothing configured */ - ast_config_destroy(cfg); - return -1; - } - - /* Mapping must have a table name */ - if (!ast_strlen_zero(tmp = ast_variable_retrieve(cfg, "master", "table"))) { - ast_copy_string(table, tmp, sizeof(table)); - } else { - ast_log(LOG_WARNING, "Table name not specified. Assuming cel.\n"); - strcpy(table, "cel"); - } - - /* sqlite3_busy_timeout in miliseconds */ - if ((tmp = ast_variable_retrieve(cfg, "master", "busy_timeout")) != NULL) { - if (ast_parse_arg(tmp, PARSE_INT32|PARSE_DEFAULT, &busy_timeout, 1000) != 0) { - ast_log(LOG_WARNING, "Invalid busy_timeout value '%s' specified. Using 1000 instead.\n", tmp); - } - } else { - busy_timeout = 1000; - } - - /* Columns */ - if (load_column_config(ast_variable_retrieve(cfg, "master", "columns"))) { - ast_config_destroy(cfg); - free_config(); - return -1; - } - - /* Values */ - if (load_values_config(ast_variable_retrieve(cfg, "master", "values"))) { - ast_config_destroy(cfg); - free_config(); - return -1; - } - - ast_verb(3, "Logging CEL records to table '%s' in 'master.db'\n", table); - - ast_config_destroy(cfg); - - return 0; -} - -static void free_config(void) -{ - struct values *value; - - if (db) { - sqlite3_close(db); - db = NULL; - } - - if (columns) { - ast_free(columns); - columns = NULL; - } - - while ((value = AST_LIST_REMOVE_HEAD(&sql_values, list))) { - ast_free(value); - } -} - -static void write_cel(struct ast_event *event) -{ - char *error = NULL; - char *sql = NULL; - - if (db == NULL) { - /* Should not have loaded, but be failsafe. */ - return; - } - - ast_mutex_lock(&lock); - - { /* Make it obvious that only sql should be used outside of this block */ - char *escaped; - char subst_buf[2048]; - struct values *value; - struct ast_channel *dummy; - struct ast_str *value_string = ast_str_create(1024); - - dummy = ast_cel_fabricate_channel_from_event(event); - if (!dummy) { - ast_log(LOG_ERROR, "Unable to fabricate channel from CEL event.\n"); - ast_free(value_string); - ast_mutex_unlock(&lock); - return; - } - AST_LIST_TRAVERSE(&sql_values, value, list) { - pbx_substitute_variables_helper(dummy, value->expression, subst_buf, sizeof(subst_buf) - 1); - escaped = sqlite3_mprintf("%q", subst_buf); - ast_str_append(&value_string, 0, "%s'%s'", ast_str_strlen(value_string) ? "," : "", escaped); - sqlite3_free(escaped); - } - sql = sqlite3_mprintf("INSERT INTO %q (%s) VALUES (%s)", table, columns, ast_str_buffer(value_string)); - ast_debug(1, "About to log: %s\n", sql); - dummy = ast_channel_unref(dummy); - ast_free(value_string); - } - - if (sqlite3_exec(db, sql, NULL, NULL, &error) != SQLITE_OK) { - ast_log(LOG_ERROR, "%s. SQL: %s.\n", error, sql); - sqlite3_free(error); - } - - if (sql) { - sqlite3_free(sql); - } - ast_mutex_unlock(&lock); - - return; + ast_rwlock_rdlock(&configs_lock); + cdrel_logger(configs, event); + ast_rwlock_unlock(&configs_lock); } static int unload_module(void) { - ast_cel_backend_unregister(SQLITE_BACKEND_NAME); + int res = 0; - free_config(); + ast_rwlock_wrlock(&configs_lock); + res = cdrel_unload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, configs, CUSTOM_BACKEND_NAME); + ast_rwlock_unlock(&configs_lock); + if (res == 0) { + ast_rwlock_destroy(&configs_lock); + } - return 0; + return res; } -static int load_module(void) +static enum ast_module_load_result load_module(void) { - char *error; - char filename[PATH_MAX]; - int res; - char *sql; - - if (load_config(0)) { + if (ast_rwlock_init(&configs_lock) != 0) { return AST_MODULE_LOAD_DECLINE; } - /* is the database there? */ - snprintf(filename, sizeof(filename), "%s/master.db", ast_config_AST_LOG_DIR); - res = sqlite3_open(filename, &db); - if (res != SQLITE_OK) { - ast_log(LOG_ERROR, "Could not open database %s.\n", filename); - free_config(); - return AST_MODULE_LOAD_DECLINE; - } - sqlite3_busy_timeout(db, busy_timeout); - /* is the table there? */ - sql = sqlite3_mprintf("SELECT COUNT(*) FROM %q;", table); - res = sqlite3_exec(db, sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (res != SQLITE_OK) { - /* We don't use %q for the column list here since we already escaped when building it */ - sql = sqlite3_mprintf("CREATE TABLE %q (AcctId INTEGER PRIMARY KEY, %s)", table, columns); - res = sqlite3_exec(db, sql, NULL, NULL, &error); - sqlite3_free(sql); - if (res != SQLITE_OK) { - ast_log(LOG_WARNING, "Unable to create table '%s': %s.\n", table, error); - sqlite3_free(error); - free_config(); - return AST_MODULE_LOAD_DECLINE; - } - } + configs = cdrel_load_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, CONFIG, CUSTOM_BACKEND_NAME, custom_log); - if (ast_cel_backend_register(SQLITE_BACKEND_NAME, write_cel)) { - ast_log(LOG_ERROR, "Unable to register custom SQLite3 CEL handling\n"); - free_config(); - return AST_MODULE_LOAD_DECLINE; - } - - return AST_MODULE_LOAD_SUCCESS; + return configs ? AST_MODULE_LOAD_SUCCESS : AST_MODULE_LOAD_DECLINE; } static int reload(void) { int res = 0; - - ast_mutex_lock(&lock); - res = load_config(1); - ast_mutex_unlock(&lock); - + ast_rwlock_wrlock(&configs_lock); + res = cdrel_reload_module(CDREL_BACKEND_TYPE, CDREL_RECORD_TYPE, &configs, CONFIG); + ast_rwlock_unlock(&configs_lock); return res; } @@ -355,5 +107,5 @@ AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_LOAD_ORDER, "SQLite3 Custom CEL Mo .unload = unload_module, .reload = reload, .load_pri = AST_MODPRI_CDR_DRIVER, - .requires = "cel", + .requires = "cel,res_cdrel_custom", ); diff --git a/configs/samples/cdr_custom.conf.sample b/configs/samples/cdr_custom.conf.sample index b296e881b6..760c039eba 100644 --- a/configs/samples/cdr_custom.conf.sample +++ b/configs/samples/cdr_custom.conf.sample @@ -1,15 +1,39 @@ ; -; Mappings for custom config file +; Asterisk Call Detail Record Logging (CDR) - Custom DSV Backend ; -; To get your CSV output in a format tailored to your liking, uncomment the -; following lines and look for the output in the cdr-custom directory (usually -; in /var/log/asterisk). Depending on which mapping you uncomment, you may see -; Master.csv, Simple.csv, or both. + +; This is the configuration file for the customizable DSV backend for CDR +; logging. ; -; Alternatively, you can also specify the location of your CSV file using an -; absolute path, e.g.: +; DSV?? Delimiter Separated Values because the field delimiter doesn't have +; to be a comma. ; -; /srv/pbx/cdr/Master.csv => ${CSV_QUOTE(${CDR(clid)})},... +; Legacy vs Advanced Mappings +; +; Legacy mappings are those that are defined using dialplan functions like CDR +; and CSV_QUOTE and require a VERY expensive function replacement process at +; runtime for every record output. +; +; Advanced mappings are those that are defined by a list of field names and +; parameters that define the field separator and quote character you want to use. +; This type of mapping is uses significantly less resources at runtime. +; +; +; Legacy Mappings +; +; Within a legacy mapping, use the CDR() and CSV_QUOTE() functions to retrieve +; values from the CDR. +; +; NOTE: If your legacy mapping uses commas as field separators and only the CSV_QUOTE +; and CDR dialplan functions, the module will attempt to strip the functions and +; and create a much faster advanced mapping for it. However, we urge you to create +; a real advanced mapping and not rely on this process. If the mapping contains +; something not recognized it will go the slower legacy route. +; +; Each entry in the "mappings" category represents a separate output file. +; A filename that starts with a forward-slash '/' will be treated as an absolute +; path name. If it doesn't, it must be a file name with no directory separators +; which will be placed in the /var/log/asterisk/cdr-custom directory. ; ;[mappings] ;Master.csv => ${CSV_QUOTE(${CDR(clid)})},${CSV_QUOTE(${CDR(src)})},${CSV_QUOTE(${CDR(dst)})},${CSV_QUOTE(${CDR(dcontext)})},${CSV_QUOTE(${CDR(channel)})},${CSV_QUOTE(${CDR(dstchannel)})},${CSV_QUOTE(${CDR(lastapp)})},${CSV_QUOTE(${CDR(lastdata)})},${CSV_QUOTE(${CDR(start)})},${CSV_QUOTE(${CDR(answer)})},${CSV_QUOTE(${CDR(end)})},${CSV_QUOTE(${CDR(duration)})},${CSV_QUOTE(${CDR(billsec)})},${CSV_QUOTE(${CDR(disposition)})},${CSV_QUOTE(${CDR(amaflags)})},${CSV_QUOTE(${CDR(accountcode)})},${CSV_QUOTE(${CDR(uniqueid)})},${CSV_QUOTE(${CDR(userfield)})},${CDR(sequence)} @@ -17,3 +41,103 @@ ; High Resolution Time for billsec and duration fields ;Master.csv => ${CSV_QUOTE(${CDR(clid)})},${CSV_QUOTE(${CDR(src)})},${CSV_QUOTE(${CDR(dst)})},${CSV_QUOTE(${CDR(dcontext)})},${CSV_QUOTE(${CDR(channel)})},${CSV_QUOTE(${CDR(dstchannel)})},${CSV_QUOTE(${CDR(lastapp)})},${CSV_QUOTE(${CDR(lastdata)})},${CSV_QUOTE(${CDR(start)})},${CSV_QUOTE(${CDR(answer)})},${CSV_QUOTE(${CDR(end)})},${CSV_QUOTE(${CDR(duration,f)})},${CSV_QUOTE(${CDR(billsec,f)})},${CSV_QUOTE(${CDR(disposition)})},${CSV_QUOTE(${CDR(amaflags)})},${CSV_QUOTE(${CDR(accountcode)})},${CSV_QUOTE(${CDR(uniqueid)})},${CSV_QUOTE(${CDR(userfield)})},${CDR(sequence)} ;Simple.csv => ${CSV_QUOTE(${EPOCH})},${CSV_QUOTE(${CDR(src)})},${CSV_QUOTE(${CDR(dst)})} + + +; +; Advanced Mappings +; +; Each category in this file other than "mappings" represents a separate output file. +; A filename that starts with a forward-slash '/' will be treated as an absolute +; path name. If it doesn't, it must be a file name with no directory separators +; which will be placed in the /var/log/asterisk/cdr-custom directory. +; +;[cdr_master.csv] ; Output file or path name. + +;format = dsv ; Advanced mappings can actually have two types + ; "dsv" or "json". This example uses "dsv", which + ; is the default, but see the example below for more + ; info on the json format. + +;separator_character = , ; The character to use for the field separator, + ; It defaults to a comma but you can use any + ; character you want. For instance, specify + ; \t to separate the fields with a tab. + +;quote_character = " ; The character to use as the quote character. + ; It defaults to the double quote but again, it + ; can be any character you want although the + ; single-quote is the only other one that makes + ; sense. + +;quote_escape_character = " ; The character to use to escape the quote_character + ; should the quote character actually appear in the + ; field output. A good example of this is a caller + ; id string like "My Name" <1000>. For true CSV + ; compatibility, it must be the same as the quote + ; character but a backslash might be acceptable for + ; other formats. + +;quoting_method = all ; Nothing says you _have_ to quote anything. The only + ; time a field MUST be quoted is if it contains the + ; separator character and this is handled automatically. + ; Additionally, the following options are available: + ; "all": Quote all fields. The default. + ; "non_numeric": Quote all non numeric fields. + ; "none": Don't quote any field (unless it contains + ; a separator character). + +;fields = clid,"Some Literal",src, dst, dcontext, channel, dstchannel, lastapp, lastdata, start, answer, end, duration, billsec, disposition, amaflags, accountcode, uniqueid(noquote), userfield, sequence,ds_type(uservar) + ; This is the list of fields to include in the record. The field names are the + ; same as in the legacy mapping but without any enclosing dialplan functions. + ; You can specify literals to be placed in the output record by double-quoting + ; them. There is also some special notation available in the form of "qualifiers". + ; A qualifier is a list of tags, separated by the '^' character and placed + ; directly after the field name and enclosed in parentheses. + ; + ; All fields can accept the "quote" or "noquote" qualifier when you want to exempt + ; a field from the "quoting_method" policy. For example, when quoting_method=all, + ; you can force the uniqueid field to not be quoted with `uniqueid(noquote)`. The + ; example in fields above shows this. + ; + ; If you've added user variables to the CDR using the CDR() dialplan function in + ; extensions.conf, you'll need to retrieve the field using the "uservar" qualifier. + ; For example, if you've added the "ds_type" variable, you'll need to retrieve + ; it with `ds_type(uservar)`. + ; + ; The default output format for the "start", "answer" and "end" timestamp fields + ; is the "%Y-%m-%d %T" strftime string format however you can also format those + ; fields as an int64 or a float: `start(int64),answer(float),end`. + ; + ; The "disposition" and "amaflags" are formatted as their string names like + ; "ANSWERED" and "DOCUMENTATION" by default but if you just want the numbers and + ; not the names... `amaflags(int64),disposition(int64)`. + ; + ; If you need to combine flags, use the caret '^' symbol: `start(int64^noquote)` + ; + ; Final notes about "fields": + ; Field names and qualifiers aren't case sensitive. + ; You MUST use the comma to separate the fields, not the "separator_character". + ; You MUST use the double-quote to indicate literal fields, not the "quote_character". + ; Whitespace in "fields" is ignored except in literals. + +; +; An Advanced JSON example: +; +;[cdr_master.json] +;format = json +;fields = clid,"My: Literal",src, dst, dcontext, channel, dstchannel, lastapp, lastdata, start, answer, end, duration, billsec, disposition, amaflags, accountcode, uniqueid(noquote), userfield, sequence,ds_type(uservar) +; +; In order to ensure valid JSON, the following settings are forced: +; The separator character is always the comma. +; The quote character is always the double-quote. +; The quote escape character is always the backslash. +; The quoting method is always "non_numeric". +; +; Since JSON requires both a name and value, the name is always +; the field name. For literals however you must specify the literal +; as a "name: value" pair as demonstrated above. The output record +; would then look something like: +; {"clid":"\"Alice\" <1000>", "My":"Literal","src":"1000","dst":"18005551212", ...} + + + diff --git a/configs/samples/cdr_sqlite3_custom.conf.sample b/configs/samples/cdr_sqlite3_custom.conf.sample index 4b88d58d48..99f9b96fa6 100644 --- a/configs/samples/cdr_sqlite3_custom.conf.sample +++ b/configs/samples/cdr_sqlite3_custom.conf.sample @@ -1,11 +1,105 @@ ; -; Mappings for custom config file +; Asterisk Call Detail Record Logging (CDR) - Custom Sqlite3 Backend ; -[master] ; currently, only file "master.db" is supported, with only one table at a time. -;table => cdr -;columns => calldate, clid, dcontext, channel, dstchannel, lastapp, lastdata, duration, billsec, disposition, amaflags, accountcode, uniqueid, userfield, test -;values => '${CDR(start)}','${CDR(clid)}','${CDR(dcontext)}','${CDR(channel)}','${CDR(dstchannel)}','${CDR(lastapp)}','${CDR(lastdata)}','${CDR(duration)}','${CDR(billsec)}','${CDR(disposition)}','${CDR(amaflags)}','${CDR(accountcode)}','${CDR(uniqueid)}','${CDR(userfield)}','${CDR(test)}' -;busy_timeout => 1000 -;Enable High Resolution Times for billsec and duration fields -;values => '${CDR(start)}','${CDR(clid)}','${CDR(dcontext)}','${CDR(channel)}','${CDR(dstchannel)}','${CDR(lastapp)}','${CDR(lastdata)}','${CDR(duration,f)}','${CDR(billsec,f)}','${CDR(disposition)}','${CDR(amaflags)}','${CDR(accountcode)}','${CDR(uniqueid)}','${CDR(userfield)}','${CDR(test)}' +; This is the configuration file for the customizable Sqlite3 backend for CDR +; logging. +; +; Legacy vs Advanced Mappings +; +; Legacy mappings are those that are defined using dialplan functions like CDR +; and CSV_QUOTE and require a VERY expensive function replacement process at +; runtime for every record output. +; +; Advanced mappings are those that are defined by a simple list of field names +; which uses significantly less resources at runtime. +; +; +; Each category in this file represents a separate Sqlite3 database file. +; A filename that starts with a forward-slash '/' will be treated as an absolute +; path name. If it doesn't, it must be a file name with no directory separators +; which will be placed in the /var/log/asterisk directory. If the database +; file doesn't already exist, it will be created. +; +; Previous versions of Asterisk limited the output to a single database file +; named "master" which was shared with CDR Sqlite3 logging. That is no longer +; the case. +; +; Legacy Mappings +; +; Within a legacy mapping, use the CDR() and CSV_QUOTE() functions to retrieve +; values from the CDR. +; +; NOTE: If your legacy mapping uses only the CDR dialplan function, the module +; will attempt to strip the functions and create a much faster advanced mapping +; for it. However, we urge you to create a real advanced mapping and not rely +; on this process. If the mapping contains something not recognized it will go +; the slower legacy route. +; +; +;[cdr_master] +;table = cdr ; The name of the table in the database into which records + ; are to be written. If the table doesn't already exist, + ; it will be created. + +;columns = calldate, literal, clid, dcontext, channel, dstchannel, lastapp, lastdata, duration, billsec, disposition, amaflags, accountcode, uniqueid, userfield, ds_store + ; The column names to receive the fields. If the table doesn't already exist, + ; it will be created with these columns. If the table does exist, this list + ; MUST match the existing columns or the config will fail to load. + ; The column names do NOT have to match the field names however. + +;values = '${CDR(start)}','some literal','${CDR(clid)}','${CDR(dcontext)}','${CDR(channel)}','${CDR(dstchannel)}','${CDR(lastapp)}','${CDR(lastdata)}','${CDR(duration)}','${CDR(billsec)}','${CDR(disposition)}','${CDR(amaflags)}','${CDR(accountcode)}','${CDR(uniqueid)}','${CDR(userfield)}','${CDR(ds_store)}' + ; The list of fields to write into the columns. + ; Each field MUST be enclosed in single-quotes and the fields separated + ; by commas. Additionally, the number of fields specified MUST match the + ; number of columns or the config will fail to load. + +;busy_timeout = 1000 ; The number of milliseconds to wait for a database operation + ; to complete before an error is returned. + +; +; Advanced Mappings +; +;[cdr_advanced] +;table = cdr ; The name of the table in the database into which records + ; are to be written. If the table doesn't already exist, + ; it will be created. + +;columns = calldate, literal, clid, dcontext, channel, dstchannel, lastapp, lastdata, duration, billsec, disposition, amaflags, accountcode, uniqueid, userfield, ds_storetest ; The column names to receive the fields. If the table doesn't already exist, + ; it will be created with these columns. If the table does exist, this list + ; MUST match the existing columns or the config will fail to load. + +;fields = start,"some literal",clid,dcontext,channel,dstchannel,lastapp,lastdata,duration,billsec,disposition,amaflags,accountcode,uniqueid,userfield,ds_store(uservar) + ; The "fields" parameter differentiates this mapping as an Advanced one + ; as opposed to "values" used above. + ; + ; This is the list of fields to include in the record. The field names are the + ; same as in the legacy mapping but without any enclosing dialplan functions. + ; You can specify literals to be placed in the output record by double-quoting + ; them. There is also some special notation available in the form of "qualifiers". + ; A qualifier is a list of tags, separated by the '^' character and placed + ; directly after the field name and enclosed in parentheses. + ; + ; If you've added user variables to the CDR using the CDR() dialplan function in + ; extensions.conf, you'll need to retrieve the field using the "uservar" qualifier. + ; For example, if you've added the "ds_type" variable, you'll need to retrieve + ; it with `ds_type(uservar)`. + ; + ; The default output format for the "start", "answer" and "end" timestamp fields + ; is the "%Y-%m-%d %T" strftime string format however you can also format those + ; fields as an int64 or a float: `start(int64),answer(float),end`. + ; + ; The "disposition" and "amaflags" are formatted as their string names like + ; "ANSWERED" and "DOCUMENTATION" by default but if you just want the numbers and + ; not the names... `amaflags(int64),disposition(int64)`. + ; + ; If you need to combine flags, use the caret '^' symbol: `start(int64^noquote)` + ; + ; Final notes about "fields": + ; Field names and qualifiers aren't case sensitive. + ; You MUST use commas to separate the fields. + ; You MUST use double-quotes to indicate literal fields. + ; Whitespace in "fields" is ignored except in literals. + +;busy_timeout = 1000 ; The number of milliseconds to wait for a database operation + ; to complete before an error is returned. diff --git a/configs/samples/cel_custom.conf.sample b/configs/samples/cel_custom.conf.sample index 3d5b978852..03debc6e29 100644 --- a/configs/samples/cel_custom.conf.sample +++ b/configs/samples/cel_custom.conf.sample @@ -1,31 +1,30 @@ ; -; Asterisk Channel Event Logging (CEL) - Custom CSV Backend +; Asterisk Channel Event Logging (CEL) - Custom DSV Backend ; - -; This is the configuration file for the customizable CSV backend for CEL +; This is the configuration file for the customizable DSV backend for CEL ; logging. ; -; In order to create custom CSV logs for CEL, uncomment the template below -; (Master.csv) and start Asterisk. Once CEL events are generated, a file will -; appear in the following location: -; -; /var/log/asterisk/cel-custom/Master.csv +; DSV?? Delimiter Separated Values because the field delimiter doesn't have +; to be a comma. ; -; (Note that /var/log/asterisk is the default and may differ on your system) +; Legacy vs Advanced Mappings ; -; You can also create more than one template if desired. All logs will appear -; in the cel-custom directory under your Asterisk logs directory. +; Legacy mappings are those that are defined using dialplan functions like +; CALLERID and CSV_QUOTE and require a VERY expensive function replacement +; process at runtime for every record output. In performance testing, 38% +; of the CPU instructions executed to handle a call were executed on behalf +; of legacy CEL logging. That's more than the instructions actually used +; to process the call itself. ; -; Alternatively, you can also specify the location of your CSV file using an -; absolute path, e.g.: +; Advanced mappings are those that are defined by a list of field names and +; parameters that define the field separator and quote character you want to +; use. This type of mapping is uses significantly less resources at runtime. +; Performance testing showed advanced CEL logging accounting for less than +; 10% of the CPU instructions executed which is well below the instructions +; needed to process the call itself. ; -; /srv/pbx/cel/Master.csv => ${CSV_QUOTE(${eventtype})},... -; - -; -; Within a mapping, use the CALLERID() and CHANNEL() functions to retrieve -; details from the CEL event. There are also a few variables created by this -; module that can be used in a mapping: +; There are several "special" variables created by this module that can be used +; in a mapping, both legacy and advanced: ; ; eventtype - The name of the CEL event. ; eventtime - The timestamp of the CEL event. @@ -36,5 +35,218 @@ ; BRIDGEPEER - Bridged peer channel name at the time of the CEL event. ; CHANNEL(peer) could also be used. ; -[mappings] +; Legacy Mappings +; +; Within a legacy mapping, use the CALLERID(), CHANNEL() and CSV_QUOTE() +; functions to retrieve values from the CEL event (and pay the price). +; +; NOTE: If your legacy mapping uses commas as field separators and only the +; CSV_QUOTE, CALLERID and CHANNEL dialplan functions or one of the special +; variables, the module will attempt to strip the functions and create a much +; faster advanced mapping for it. However, we urge you to create a real +; advanced mapping and not rely on this process. If the mapping contains +; something not recognized it will go the slower legacy route. +; +; Each entry in the "mappings" category represents a separate output file. +; A filename that starts with a forward-slash '/' will be treated as an absolute +; path name. If it doesn't, it must be a file name with no directory separators +; which will be placed in the /var/log/asterisk/cel-custom directory. +; +;[mappings] ;Master.csv => ${CSV_QUOTE(${eventtype})},${CSV_QUOTE(${eventtime})},${CSV_QUOTE(${CALLERID(name)})},${CSV_QUOTE(${CALLERID(num)})},${CSV_QUOTE(${CALLERID(ANI)})},${CSV_QUOTE(${CALLERID(RDNIS)})},${CSV_QUOTE(${CALLERID(DNID)})},${CSV_QUOTE(${CHANNEL(exten)})},${CSV_QUOTE(${CHANNEL(context)})},${CSV_QUOTE(${CHANNEL(channame)})},${CSV_QUOTE(${CHANNEL(appname)})},${CSV_QUOTE(${CHANNEL(appdata)})},${CSV_QUOTE(${CHANNEL(amaflags)})},${CSV_QUOTE(${CHANNEL(accountcode)})},${CSV_QUOTE(${CHANNEL(uniqueid)})},${CSV_QUOTE(${CHANNEL(linkedid)})},${CSV_QUOTE(${BRIDGEPEER})},${CSV_QUOTE(${CHANNEL(userfield)})},${CSV_QUOTE(${userdeftype})},${CSV_QUOTE(${eventextra})} + +; +; Advanced Mappings +; +; Each category in this file other than "mappings" represents a separate output file. +; A filename that starts with a forward-slash '/' will be treated as an absolute +; path name. If it doesn't, it must be a file name with no directory separators +; which will be placed in the /var/log/asterisk/cel-custom directory. +; +;[cel_master.csv] ; Output file or path name. + +;format = dsv ; Advanced mappings can actually have two types + ; "dsv" or "json". This example uses "dsv", which + ; is the default, but see the example below for more + ; info on the json format. + +;separator_character = , ; The character to use for the field separator, + ; It defaults to a comma but you can use any + ; character you want. For instance, specify + ; \t to separate the fields with a tab. + +;quote_character = " ; The character to use as the quote character. + ; It defaults to the double quote but again, it + ; can be any character you want although the + ; single-quote is the only other one that makes + ; sense. + +;quote_escape_character = " ; The character to use to escape the quote_character + ; should the quote character actually appear in the + ; field output. A good example of this is a caller + ; id string like "My Name" <1000>. For true CSV + ; compatibility, it must be the same as the quote + ; character but a backslash might be acceptable for + ; other formats. + +;quoting_method = all ; Nothing says you _have_ to quote anything. The only + ; time a field MUST be quoted is if it contains the + ; separator character and this is handled automatically. + ; Additionally, the following options are available: + ; "all": Quote all fields. The default. + ; "non_numeric": Quote all non numeric fields. + ; "none": Don't quote any field (unless it contains + ; a separator character). + +;fields = EventType,eventenum,userdeftype,"Some Literal",EventTime,Name,Num,ani,rdnis,dnid,Exten,Context,ChanName,AppName,AppData,AMAFlags(amaflags),AccountCode,UniqueID(noquote),LinkedID(noquote),Peer,PeerAccount,UserField,EventExtra,TenantID + ; This is the list of fields to include in the record. The field names are the + ; same as in the legacy mapping but without any enclosing dialplan functions. + ; You can specify literals to be placed in the output record by double-quoting + ; them. There is also some special notation available in the form of "qualifiers". + ; A qualifier is a list of tags, separated by the '^' character and placed + ; directly after the field name and enclosed in parentheses. + ; + ; All fields can accept the "quote" or "noquote" qualifier when you want to exempt + ; a field from the "quoting_method" policy. For example, when quoting_method=all, + ; you can force the uniqueid field to not be quoted with `uniqueid(noquote)`. The + ; example in fields above shows this. + ; + ; The default output format for the "EventTime" timestamp field is the "%Y-%m-%d %T" + ; strftime string format however you can also format the field as an int64 or a + ; float: `eventtime(int64)` or `eventtime(float)`. + ; + ; Unlike CDRs, the "amaflags" field is output as its numerical value by default + ; for historical reasons. You can output it as its friendly string with + ; `amaflags(amaflags)`. This will print "DOCUMENTATION" instead of "3" for instance. + ; + ; If you need to combine flags, use the caret '^' symbol: `eventtime(int64^noquote)` + ; + ; Final notes about "fields": + ; Field names and qualifiers aren't case sensitive. + ; You MUST use the comma to separate the fields, not the "separator_character". + ; You MUST use the double-quote to indicate literal fields, not the "quote_character". + ; Whitespace in "fields" is ignored except in literals. + +; +; An Advanced JSON example: +; +;[cdr_master.json] +;format = json +;fields = EventType,eventenum,userdeftype,"My: Literal",EventTime(float),Name,Num,ani,rdnis,dnid,Exten,Context,ChanName,AppName,AppData,AMAFlags(amaflags),AccountCode,UniqueID,LinkedID,Peer,PeerAccount,UserField,EventExtra,TenantID +; +; In order to ensure valid JSON, the following settings are forced: +; The separator character is always the comma. +; The quote character is always the double-quote. +; The quote escape character is always the backslash. +; The quoting method is always "non_numeric". +; +; Since JSON requires both a name and value, the name is always +; the field name. For literals however you must specify the literal +; as a "name: value" pair as demonstrated above. The output record +; would then look something like: +; {"eventtype":"HANGUP","eventenum":"HANGUP","userdeftype":"","My":"Literal","eventtime":1771359872.0, ...} + + + + + + + + + + +; +; Advanced Mappings +; +; Advanced mappings use SIGNIFICANTLY less resources than legacy mappings +; because we don't need to use dialplan function replacement. +; +;[cel_master_advanced.csv] ; The destination file name. + ; Can be a name relative to ASTLOGDIR or an + ; absolute path. + +;format = csv ; Sets the output format. The default is "csv" + ; but here "csv" really means "character-separated-values" + ; because the separator doesn't have to be a comma. + ; The other alternative is "json" (see example below). + +;separator_character = , ; Set the character to use between fields. + ; Defaults to a comma but other characters + ; can be used. For example, if you want + ; tab-separated fields, use \t as the separator. + +;quote_character = " ; Set the quoting character. + ; Defaults to double-quote (") but any character + ; can be used although only the double and single + ; quote (') characters make sense. + +;quote_escape_character = " ; Sets the character used to escape quotes that + ; may be in the field values. The default is the + ; same character as the quote_character so an + ; embedded JSON blob would look like this: + ; "{""extra"":""somextratext""}" + ; You could also use a backslash (\) in which case + ; the blob would look like this: + ; "{\"extra\":\"somextratext\"}" + +;quoting_method = all ; Sets what/when to quote. + ; all - Quote all fields. (the default) + ; minimal - Only quote fields that have the + ; separator character in them. + ; non_numeric - Quote all non-numeric fields. + ; none - Don't quote anything. + ; Probably not a good idea but could + ; be useful in special circumstances. + +; The fields to output. These names correspond to the internal CEL event +; field names which is how some of the performance gains are realized. +; Anything not recognized as a field name will be printed as a literal in +; the output. +; +; CEL Event Field Names: +; +; eventtype - Could be a standard event name or a user event name +; eventenum - Will always be a standard event name or 'USER_DEFINED' +; eventtime - Uses the dateformat set in cel.conf. +; usereventname - Will be the user event name if set. +; cidname +; cidnum +; exten +; context +; channame +; appname +; appdata +; amaflags +; acctcode +; uniqueid +; userfield +; cidani +; cidrdnis +; ciddnid +; peer +; linkedid +; peeracct +; extra +; tenantid +; +; You MUST use the comma and double-quote here, not the separator or +; or quote characters specified above. The names are case-insensitive. +; +;fields = EventType,EventEnum,"My Literal",EventTime,UserEventName,CIDName,CIDNum,Exten,Context,ChanName,AppName,AppData,AMAFlags,AcctCode,UniqueID,UserField,CIDani,CIDrdnis,CIDdnid,Peer,LinkedID,PeerAcct,Extra,TenantID + +; +; A tab-separated-value example: +; +;[cel_master.tsv] +;separator_character = \t +;fields = EventType,EventEnum,"My Literal",EventTime,UserEventName,CIDName,CIDNum,Exten,Context,ChanName,AppName,AppData,AMAFlags,AcctCode,UniqueID,UserField,CIDani,CIDrdnis,CIDdnid,Peer,LinkedID,PeerAcct,Extra,TenantID + +; +; A JSON example: +; +; The separator and quoting options don't apply to JSON. +; Literals must be specified as a "name: value" pair or they'll be ignored. +; +;[cel_master.json] +;format = json +;fields = EventType,eventenum,userdeftype,"My: Literal",EventTime,CIDName,CIDNum,CIDani,CIDrdnis,CIDdnid,Exten,Context,ChanName,AppName,AppData,AMAFlags,AcctCode,UniqueID,LinkedID,Peer,PeerAcct,UserField,Extra,TenantID diff --git a/configs/samples/cel_sqlite3_custom.conf.sample b/configs/samples/cel_sqlite3_custom.conf.sample index aa908a4f36..0d549b280a 100644 --- a/configs/samples/cel_sqlite3_custom.conf.sample +++ b/configs/samples/cel_sqlite3_custom.conf.sample @@ -1,13 +1,33 @@ ; -; Asterisk Channel Event Logging (CEL) - SQLite 3 Backend +; Asterisk Channel Event Logging (CEL) - Custom Sqlite3 Backend ; +; This is the configuration file for the customizable Sqlite3 backend for CEL +; logging. ; -; Mappings for sqlite3 config file +; Legacy vs Advanced Mappings ; -; Within a mapping, use the CALLERID() and CHANNEL() functions to retrieve -; details from the CEL event. There are also a few variables created by this -; module that can be used in a mapping: +; Legacy mappings are those that are defined using dialplan functions like +; CALLERID and CHANNEL and require a VERY expensive function replacement +; process at runtime for every record output. +; +; Advanced mappings are those that are defined by a simple list of field names +; and uses significantly less resources at runtime. +; +; +; Each category in this file represents a separate Sqlite3 database file. +; A filename that starts with a forward-slash '/' will be treated as an absolute +; path name. If it doesn't, it must be a file name with no directory separators +; which will be placed in the /var/log/asterisk directory. If the database +; file doesn't already exist, it will be created. +; +; Previous versions of Asterisk limited the output to a single database file +; named "master" which was shared with CDR Sqlite3 logging. That is no longer +; the case. +; +; +; There are several "special" variables created by this module that can be used +; in a mapping, both legacy and advanced: ; ; eventtype - The name of the CEL event. ; eventtime - The timestamp of the CEL event. @@ -18,8 +38,79 @@ ; BRIDGEPEER - Bridged peer channel name at the time of the CEL event. ; CHANNEL(peer) could also be used. ; -;[master] ; currently, only file "master.db" is supported, with only one table at a time. -;table => cel -;columns => eventtype, eventtime, cidname, cidnum, cidani, cidrdnis, ciddnid, context, exten, channame, appname, appdata, amaflags, accountcode, uniqueid, userfield, peer, userdeftype, eventextra -;values => '${eventtype}','${eventtime}','${CALLERID(name)}','${CALLERID(num)}','${CALLERID(ANI)}','${CALLERID(RDNIS)}','${CALLERID(DNID)}','${CHANNEL(context)}','${CHANNEL(exten)}','${CHANNEL(channame)}','${CHANNEL(appname)}','${CHANNEL(appdata)}','${CHANNEL(amaflags)}','${CHANNEL(accountcode)}','${CHANNEL(uniqueid)}','${CHANNEL(userfield)}','${BRIDGEPEER}','${userdeftype}','${eventextra}' -;busy_timeout => 1000 \ No newline at end of file +; Legacy Mappings +; +; Within a legacy mapping, use the CALLERID() and CHANNEL() functions or +; the special variables above to retrieve values from the CEL event. +; +; NOTE: If your legacy mapping uses only those two functions and the special +; variables, the module will attempt to strip the functions and create a much +; faster advanced mapping for it. However, we urge you to create a real advanced +; mapping and not rely on this process. If the mapping contains something not +; recognized it will go the slower legacy route. +; +; +;[cel_master] +;table = cel ; The name of the table in the database into which records + ; are to be written. If the table doesn't already exist, + ; it will be created. + +;columns = eventtype, eventtime, cidname, cidnum, cidani, cidrdnis, ciddnid, context, exten, channame, appname, appdata, amaflags, accountcode, uniqueid, userfield, peer, userdeftype, eventextra + ; The column names to receive the fields. If the table doesn't already exist, + ; it will be created with these columns. If the table does exist, this list + ; MUST match the existing columns or the config will fail to load. + ; The column names do NOT have to match the field names however. + +values = '${eventtype}','${eventtime}','${CALLERID(name)}','${CALLERID(num)}','${CALLERID(ANI)}','${CALLERID(RDNIS)}','${CALLERID(DNID)}','${CHANNEL(context)}','${CHANNEL(exten)}','${CHANNEL(channame)}','${CHANNEL(appname)}','${CHANNEL(appdata)}','${CHANNEL(amaflags)}','${CHANNEL(accountcode)}','${CHANNEL(uniqueid)}','${CHANNEL(userfield)}','${BRIDGEPEER}','${userdeftype}','${eventextra}' + ; The list of fields to write into the columns. + ; Each field MUST be enclosed in single-quotes and the fields separated + ; by commas. Additionally, the number of fields specified MUST match the + ; number of columns or the config will fail to load. + +;busy_timeout = 1000 ; The number of milliseconds to wait for a database operation + ; to complete before an error is returned. + +; +; Advanced Mappings +; +;[cdr_advanced] +;table = cel ; The name of the table in the database into which records + ; are to be written. If the table doesn't already exist, + ; it will be created. + +;columns = eventtype, literal, eventtime, cidname, cidnum, cidani, cidrdnis, ciddnid, context, exten, channame, appname, appdata, amaflags, accountcode, uniqueid, userfield, peer, userdeftype, eventextra + ; The column names to receive the fields. If the table doesn't already exist, + ; it will be created with these columns. If the table does exist, this list + ; MUST match the existing columns or the config will fail to load. + ; The column names do NOT have to match the field names however. + +;fields = eventtype, "some literal", eventtime, name, num, ani, rdnis, dnid, context, exten, channame, appname, appdata, amaflags, accountcode, uniqueid, userfield, peer, userdeftype, eventextra + ; The "fields" parameter differentiates this mapping as an Advanced one + ; as opposed to "values" used above. + ; + ; This is the list of fields to include in the record. The field names are the + ; same as in the legacy mapping but without any enclosing dialplan functions or + ; quotes. You can specify literals to be placed in the output record by + ; double-quoting them. There is also some special notation available in the + ; form of "qualifiers". A qualifier is a list of tags, separated by the '^' + ; character and placed directly after the field name and enclosed in parentheses. + ; + ; The default output format for the "EventTime" timestamp field is the "%Y-%m-%d %T" + ; strftime string format however you can also format the field as an int64 or a + ; float: `eventtime(int64)` or `eventtime(float)`. + ; + ; Unlike CDRs, the "amaflags" field is output as its numerical value by default + ; for historical reasons. You can output it as its friendly string with + ; `amaflags(amaflags)`. This will print "DOCUMENTATION" instead of "3" for instance. + ; + ; If you need to combine flags, use the caret '^' symbol: `eventtime(int64^noquote)` + ; + ; Final notes about "fields": + ; Field names and qualifiers aren't case sensitive. + ; You MUST use commas to separate the fields. + ; You MUST use double-quotes to indicate literal fields. + ; Whitespace in "fields" is ignored except in literals. + +;busy_timeout = 1000 ; The number of milliseconds to wait for a database operation + ; to complete before an error is returned. + diff --git a/include/asterisk/cel.h b/include/asterisk/cel.h index 8a1e8b884f..fd993910af 100644 --- a/include/asterisk/cel.h +++ b/include/asterisk/cel.h @@ -34,6 +34,7 @@ extern "C" { #endif #include "asterisk/event.h" +#include "asterisk/strings.h" /*! * \brief CEL event types @@ -117,6 +118,18 @@ const char *ast_cel_get_type_name(enum ast_cel_event_type type); */ enum ast_cel_event_type ast_cel_str_to_event_type(const char *name); +/*! + * \brief Format an event timeval using dateformat from cel.conf + * + * \param eventtime The timeval to format + * \param timebuf A buffer of at least 30 characters to place the result in + * \param len Length of buffer + + * \retval zero Success + * \retval non-zero Failure + */ +int ast_cel_format_eventtime(struct timeval eventtime, char *timebuf, size_t len); + /*! * \brief Create a fake channel from data in a CEL event * diff --git a/include/asterisk/res_cdrel_custom.h b/include/asterisk/res_cdrel_custom.h new file mode 100644 index 0000000000..8dcc89ba3d --- /dev/null +++ b/include/asterisk/res_cdrel_custom.h @@ -0,0 +1,117 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Protected header for the CDR and CEL Custom Backends + * + * \warning This file should be included only by CDR and CEL backends. + * + */ + +#ifndef _RES_CDREL_CUSTOM_H +#define _RES_CDREL_CUSTOM_H + +/*! \enum Backend Types */ +enum cdrel_backend_type { + cdrel_backend_text = 0, /*!< Text file: DSV or JSON */ + cdrel_backend_db, /*!< Database (currently only sqlite3) */ + cdrel_backend_type_end, /*!< Sentinel */ +}; + +/*! \enum Record Types */ +enum cdrel_record_type { + cdrel_record_cdr = 0, /*!< Call Detail Records */ + cdrel_record_cel, /*!< Channel Event Log records */ + cdrel_record_type_end, /*!< Sentinel */ +}; + +/*! \struct Forward declaration of a configuration */ +struct cdrel_config; + +/*! \struct Vector to hold all configurations in a config file */ +AST_VECTOR(cdrel_configs, struct cdrel_config *); + +/*! + * \brief Perform initial module load. + * + * Needs to be called by each "custom" module + * + * \param backend_type One of \ref cdrel_backend_type. + * \param record_type One of \ref cdrel_record_type. + * \param config_filename The config file name. + * \param backend_name The name to register the backend as. + * \param logging_cb The logging callback to register with CDR or CEL. + * \returns A pointer to a VECTOR or config objects read from the config file. + */ +struct cdrel_configs *cdrel_load_module(enum cdrel_backend_type backend_type, + enum cdrel_record_type record_type, const char *config_filename, + const char *backend_name, void *logging_cb); + +/*! + * \brief Perform module reload. + * + * Needs to be called by each "custom" module + * + * \warning This function MUST be called with the module's config_lock held + * for writing to prevent reloads from happening while we're logging. + * + * \param backend_type One of \ref cdrel_backend_type. + * \param record_type One of \ref cdrel_record_type. + * \param configs A pointer to the VECTOR of config objects returned by \ref cdrel_load_module. + * \param config_filename The config file name. + * \retval AST_MODULE_LOAD_SUCCESS + * \retval AST_MODULE_LOAD_DECLINE + */ +int cdrel_reload_module(enum cdrel_backend_type backend_type, enum cdrel_record_type record_type, + struct cdrel_configs **configs, const char *config_filename); + +/*! + * \brief Perform module unload. + * + * Needs to be called by each "custom" module + * + * \warning This function MUST be called with the module's config_lock held + * for writing to prevent the module from being unloaded while we're logging. + * + * \param backend_type One of \ref cdrel_backend_type. + * \param record_type One of \ref cdrel_record_type. + * \param configs A pointer to the VECTOR of config objects returned by \ref cdrel_load_module. + * \param backend_name The backend name to unregister. + * \retval 0 Success. + * \retval -1 Failure. + */ +int cdrel_unload_module(enum cdrel_backend_type backend_type, enum cdrel_record_type record_type, + struct cdrel_configs *configs, const char *backend_name); + +/*! + * \brief Log a record. The module's \ref logging_cb must call this. + * + * \warning This function MUST be called with the module's config_lock held + * for reading to prevent reloads from happening while we're logging. + * + * \param configs A pointer to the VECTOR of config objects returned by \ref cdrel_load_module. + * \param data A pointer to an ast_cdr or ast_event object to log. + * \retval 0 Success. + * \retval -1 Failure. + */ +int cdrel_logger(struct cdrel_configs *configs, void *data); + +#endif /* _RES_CDREL_CUSTOM_H */ diff --git a/main/cel.c b/main/cel.c index 4e99f63242..251041a90b 100644 --- a/main/cel.c +++ b/main/cel.c @@ -673,6 +673,39 @@ static void check_retire_linkedid(struct ast_channel_snapshot *snapshot, const s ao2_ref(lid, -1); } +static int cel_format_eventtime(struct cel_config *cfg, struct timeval eventtime, char *timebuf, size_t len) +{ + if (!timebuf || len < 30) { + return -1; + } + + if (ast_strlen_zero(cfg->general->date_format)) { + snprintf(timebuf, len, "%ld.%06ld", (long) eventtime.tv_sec, + (long) eventtime.tv_usec); + } else { + struct ast_tm tm; + ast_localtime(&eventtime, &tm, NULL); + ast_strftime(timebuf, len, cfg->general->date_format, &tm); + } + + return 0; +} + +int ast_cel_format_eventtime(struct timeval eventtime, char *timebuf, size_t len) +{ + struct cel_config *cfg = ao2_global_obj_ref(cel_configs); + int res = 0; + + if (!cfg) { + return -1; + } + + res = cel_format_eventtime(cfg, eventtime, timebuf, len); + ao2_cleanup(cfg); + + return res; +} + /* Note that no 'chan_fixup' function is provided for this datastore type, * because the channels that will use it will never be involved in masquerades. */ @@ -719,14 +752,7 @@ struct ast_channel *ast_cel_fabricate_channel_from_event(const struct ast_event AST_LIST_INSERT_HEAD(headp, newvariable, entries); } - if (ast_strlen_zero(cfg->general->date_format)) { - snprintf(timebuf, sizeof(timebuf), "%ld.%06ld", (long) record.event_time.tv_sec, - (long) record.event_time.tv_usec); - } else { - struct ast_tm tm; - ast_localtime(&record.event_time, &tm, NULL); - ast_strftime(timebuf, sizeof(timebuf), cfg->general->date_format, &tm); - } + cel_format_eventtime(cfg, record.event_time, timebuf, sizeof(timebuf)); if ((newvariable = ast_var_assign("eventtime", timebuf))) { AST_LIST_INSERT_HEAD(headp, newvariable, entries); @@ -759,6 +785,7 @@ struct ast_channel *ast_cel_fabricate_channel_from_event(const struct ast_event ast_channel_accountcode_set(tchan, record.account_code); ast_channel_peeraccount_set(tchan, record.peer_account); ast_channel_userfield_set(tchan, record.user_field); + ast_channel_tenantid_set(tchan, record.tenant_id); if ((newvariable = ast_var_assign("BRIDGEPEER", record.peer))) { AST_LIST_INSERT_HEAD(headp, newvariable, entries); diff --git a/res/Makefile b/res/Makefile index 722b93d7db..2bbd0bd689 100644 --- a/res/Makefile +++ b/res/Makefile @@ -69,6 +69,7 @@ $(call MOD_ADD_C,res_stasis_recording,stasis_recording/stored.c) $(call MOD_ADD_C,res_stir_shaken,$(wildcard res_stir_shaken/*.c)) $(call MOD_ADD_C,res_aeap,$(wildcard res_aeap/*.c)) $(call MOD_ADD_C,res_geolocation,$(wildcard res_geolocation/*.c)) +$(call MOD_ADD_C,res_cdrel_custom,$(wildcard cdrel_custom/*.c)) # These are the xml and xslt files to be embedded res_geolocation.so: res_geolocation/pidf_lo_test.o res_geolocation/pidf_to_eprofile.o res_geolocation/eprofile_to_pidf.o diff --git a/res/cdrel_custom/cdrel.h b/res/cdrel_custom/cdrel.h new file mode 100644 index 0000000000..84787840e9 --- /dev/null +++ b/res/cdrel_custom/cdrel.h @@ -0,0 +1,363 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Private header for res_cdrel_custom. + * + */ + +#ifndef _CDREL_H +#define _CDREL_H + +#include + +#include "asterisk.h" +#include "asterisk/cdr.h" +#include "asterisk/cel.h" +#include "asterisk/event.h" +#include "asterisk/lock.h" +#include "asterisk/strings.h" +#include "asterisk/vector.h" +#include "asterisk/res_cdrel_custom.h" + +extern const char *cdrel_record_type_map[]; +#define RECORD_TYPE_STR(_rt) (cdrel_record_type_map[_rt]) + +extern const char *cdrel_module_type_map[]; +#define MODULE_TYPE_STR(_mt) (cdrel_module_type_map[_mt]) + +enum cdrel_text_format_type { + cdrel_format_dsv = 0, + cdrel_format_json, + cdrel_format_sql, + cdrel_format_type_end, +}; + +enum cdrel_config_type { + cdrel_config_legacy = 0, + cdrel_config_advanced, + cdrel_config_type_end, +}; + +enum cdrel_quoting_method { + cdrel_quoting_method_none = 0, + cdrel_quoting_method_all, + cdrel_quoting_method_minimal, + cdrel_quoting_method_non_numeric, + cdrel_quoting_method_end, +}; + +/* + * ORDER IS IMPORTANT! + * The string output data types need to be first. + */ +enum cdrel_data_type { + cdrel_type_string = 0, + cdrel_type_timeval, + cdrel_type_literal, + cdrel_type_amaflags, + cdrel_type_disposition, + cdrel_type_uservar, + cdrel_type_event_type, + cdrel_type_event_enum, + cdrel_data_type_strings_end, + cdrel_type_int32, + cdrel_type_uint32, + cdrel_type_int64, + cdrel_type_uint64, + cdrel_type_float, + cdrel_data_type_end +}; + +extern const char *cdrel_data_type_map[]; +#define DATA_TYPE_STR(_dt) (_dt < cdrel_data_type_end ? cdrel_data_type_map[_dt] : NULL) +enum cdrel_data_type cdrel_data_type_from_str(const char *str); + +#define CDREL_FIELD_FLAG_QUOTE (0) +#define CDREL_FIELD_FLAG_NOQUOTE (1) +#define CDREL_FIELD_FLAG_TYPE_FORCED (2) +#define CDREL_FIELD_FLAG_USERVAR (3) +#define CDREL_FIELD_FLAG_LITERAL (4) +#define CDREL_FIELD_FLAG_FORMAT_SPEC (5) +#define CDREL_FIELD_FLAG_LAST (6) + +enum cdrel_field_flags { + cdrel_flag_quote = (1 << CDREL_FIELD_FLAG_QUOTE), + cdrel_flag_noquote = (1 << CDREL_FIELD_FLAG_NOQUOTE), + cdrel_flag_type_forced = (1 << CDREL_FIELD_FLAG_TYPE_FORCED), + cdrel_flag_uservar = (1 << CDREL_FIELD_FLAG_USERVAR), + cdrel_flag_literal = (1 << CDREL_FIELD_FLAG_LITERAL), + cdrel_flag_format_spec = (1 << CDREL_FIELD_FLAG_FORMAT_SPEC), + cdrel_field_flags_end = (1 << CDREL_FIELD_FLAG_LAST), +}; + +/* + * CEL has a few synthetic fields that aren't defined + * in event.h so we'll define them ourselves after the + * last official id. + */ +#define AST_EVENT_IE_CEL_LITERAL (AST_EVENT_IE_TOTAL + 1) +#define AST_EVENT_IE_CEL_EVENT_ENUM (AST_EVENT_IE_TOTAL + 2) +#define LAST_CEL_ID AST_EVENT_IE_CEL_EVENT_ENUM + +/*! + * While CEL event fields are published as a generic AST_EVENT with + * a field id assigned to each field, CDRs are published as a fixed + * ast_cdr structure. To make it easier to share lower level code, + * we assign pseudo-field-ids to each CDR field that are really the offset + * of the field in the structure. This allows us to generically get + * any field using its id just like we do for CEL. + * + * To avoid conflicts with the existing CEL field ids, we'll start these + * after the last one. + */ +#define CDR_OFFSET_SHIFT (LAST_CEL_ID + 1) +#define CDR_OFFSETOF(_field) (offsetof(struct ast_cdr, _field) + CDR_OFFSET_SHIFT) +#define CDR_FIELD(__cdr, __offset) (((void *)__cdr) + __offset - CDR_OFFSET_SHIFT) + +enum cdr_field_id { + cdr_field_literal = -1, + cdr_field_clid = CDR_OFFSETOF(clid), + cdr_field_src = CDR_OFFSETOF(src), + cdr_field_dst = CDR_OFFSETOF(dst), + cdr_field_dcontext = CDR_OFFSETOF(dcontext), + cdr_field_channel = CDR_OFFSETOF(channel), + cdr_field_dstchannel = CDR_OFFSETOF(dstchannel), + cdr_field_lastapp = CDR_OFFSETOF(lastapp), + cdr_field_lastdata = CDR_OFFSETOF(lastdata), + cdr_field_start = CDR_OFFSETOF(start), + cdr_field_answer = CDR_OFFSETOF(answer), + cdr_field_end = CDR_OFFSETOF(end), + cdr_field_duration = CDR_OFFSETOF(duration), + cdr_field_billsec = CDR_OFFSETOF(billsec), + cdr_field_disposition = CDR_OFFSETOF(disposition), + cdr_field_amaflags = CDR_OFFSETOF(amaflags), + cdr_field_accountcode = CDR_OFFSETOF(accountcode), + cdr_field_peeraccount = CDR_OFFSETOF(peeraccount), + cdr_field_flags = CDR_OFFSETOF(flags), + cdr_field_uniqueid = CDR_OFFSETOF(uniqueid), + cdr_field_linkedid = CDR_OFFSETOF(linkedid), + cdr_field_tenantid = CDR_OFFSETOF(tenantid), + cdr_field_peertenantid = CDR_OFFSETOF(peertenantid), + cdr_field_userfield = CDR_OFFSETOF(userfield), + cdr_field_sequence = CDR_OFFSETOF(sequence), + cdr_field_varshead = CDR_OFFSETOF(varshead), +}; + + +struct cdrel_field; + +/*! + * \internal + * \brief A generic value wrapper structure. + */ +struct cdrel_value { + char *field_name; + enum cdrel_data_type data_type; + int mallocd; + union { + char *string; + int32_t int32; + uint32_t uint32; + int64_t int64; + uint64_t uint64; + struct timeval tv; + float floater; + } values; +}; + +/*! + * \internal + * \brief A vector of cdrel_values. + */ +AST_VECTOR(cdrel_values, struct cdrel_value *); + +/*! + * \internal + * \brief Getter callbacks. + * + * \param record An ast_cdr or ast_event structure. + * \param config Config object. + * \param field Field object. + * \param value A pointer to a cdrel_value structure to populate. + * \retval 0 on success. + * \retval -1 on failure. + */ +typedef int (*cdrel_field_getter)(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value); +/*! + * \internal + * \brief The table of getter callbacks. Populated by getters_cdr.c and getters_cel.c. + * + * Defined in res_cdrel_custom.c + */ +extern cdrel_field_getter cdrel_field_getters[cdrel_record_type_end][cdrel_data_type_end]; + +/*! + * \internal + * \brief Data type formatters. + * + * \param config Config object. + * \param field Field object. + * \param input_value A pointer to a cdrel_value structure with the data to format. + * \param output_value A pointer to a cdrel_value structure to receive the formatted data. + * \retval 0 on success. + * \retval -1 on failure. + */ +typedef int (*cdrel_field_formatter)(struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *input_value, struct cdrel_value *output_value); +/*! + * \internal + * \brief The table of formatter callbacks. Populated by formatters.c. + * + * Defined in res_cdrel_custom.c + */ +extern cdrel_field_formatter cdrel_field_formatters[cdrel_data_type_end]; + +/*! + * \internal + * \brief Backend writers. + * + * \param config Config object. + * \param values A vector of cdrel_values to write. + * \retval 0 on success. + * \retval -1 on failure. + */ +typedef int (*cdrel_backend_writer)(struct cdrel_config *config, struct cdrel_values *values); +/*! + * \internal + * \brief The table of writer callbacks. Populated by writers.c. + * + * Defined in res_cdrel_custom.c + */ +extern cdrel_backend_writer cdrel_backend_writers[cdrel_format_type_end]; + +/*! + * \internal + * \brief Dummy channel allocators. + * + * Legacy configurations use dialplan functions so they need a dummy channel + * to operate on. CDR and CEL each have their own allocator. + * + * \param config Config object. + * \param record An ast_cdr or ast_event structure. + * \return an ast_channel or NULL on failure. + */ +typedef struct ast_channel * (*cdrel_dummy_channel_alloc)(struct cdrel_config *config, void *record); +extern cdrel_dummy_channel_alloc cdrel_dummy_channel_allocators[cdrel_format_type_end]; + + +/*! Represents a field definition */ +struct cdrel_field { + enum cdrel_record_type record_type; /*!< CDR or CEL */ + int field_id; /*!< May be an AST_EVENT_IE_CEL or a cdr_field_id */ + char *name; /*!< The name of the field */ + enum cdrel_data_type input_data_type; /*!< The data type of the field in the source record */ + enum cdrel_data_type output_data_type; + struct ast_flags flags; /*!< Flags used during config parsing */ + char data[1]; /*!< Could be literal data, a user variable name, etc */ +}; + +/*! Represents an output definition from a config file */ +struct cdrel_config { + enum cdrel_record_type record_type; /*!< CDR or CEL */ + AST_DECLARE_STRING_FIELDS( + AST_STRING_FIELD(config_filename); /*!< Input configuration filename */ + AST_STRING_FIELD(output_filename); /*!< Output text file or database */ + AST_STRING_FIELD(template); /*!< Input template */ + AST_STRING_FIELD(db_columns); /*!< List of columns for database backends */ + AST_STRING_FIELD(db_table); /*!< Table name for database backends */ + ); + sqlite3 *db; /*!< sqlite3 database handle */ + sqlite3_stmt *insert; /*!< sqlite3 prepared statement for insert */ + int busy_timeout; /*!< sqlite3 query timeout value */ + cdrel_dummy_channel_alloc dummy_channel_alloc; /*!< Legacy config types need a dummy channel */ + enum cdrel_backend_type backend_type; /*!< Text file or database */ + enum cdrel_config_type config_type; /*!< Legacy or advanced */ + enum cdrel_text_format_type format_type; /*!< For text files, CSV or JSON */ + enum cdrel_quoting_method quoting_method; /*!< When to quote */ + char separator[2]; /*!< For text files, the field separator */ + char quote[2]; /*!< For text files, the quote character */ + char quote_escape[2]; /*!< For text files, character to use to escape embedded quotes */ + AST_VECTOR(, struct cdrel_field *) fields; /*!< Vector of fields for this config */ + ast_mutex_t lock; /*!< Lock that serializes filesystem writes */ +}; + +/*! + * \internal + * \brief Get a cdrel_field structure by record type and field name. + * + * \param record_type The cdrel_record_type to search. + * \param name The field name to search for. + * \returns A pointer to a constant cdrel_field structure or NULL if not found. + * This pointer must never be freed. + */ +const struct cdrel_field *get_registered_field_by_name(enum cdrel_record_type record_type, + const char *name); + +/*! + * \internal + * \brief Write a record to a text file + * + * \param config The configuration object. + * \param record The data to write. + * \retval 0 on success + * \retval -1 on failure + */ +int write_record_to_file(struct cdrel_config *config, struct ast_str *record); + +/*! + * \internal + * \brief Write a record to a database + * + * \param config The configuration object. + * \param values The values to write. + * \retval 0 on success + * \retval -1 on failure + */ +int write_record_to_database(struct cdrel_config *config, struct cdrel_values *values); + +/*! + * \internal + * \brief Return the basename of a path + * + * \param path + * \returns Pointer to last '/' or path if no '/' was found. + */ +const char *cdrel_basename(const char *path); + +/*! + * \internal + * \brief Get a string representing a field's flags + * + * \param flags An ast_flags structure + * \param str Pointer to ast_str* buffer + * \returns A string of field flag names + */ +const char *cdrel_get_field_flags(struct ast_flags *flags, struct ast_str **str); + + +int load_cdr(void); +int load_cel(void); +int load_formatters(void); +int load_writers(void); + +#endif /* _CDREL_H */ diff --git a/res/cdrel_custom/config.c b/res/cdrel_custom/config.c new file mode 100644 index 0000000000..d4655f0792 --- /dev/null +++ b/res/cdrel_custom/config.c @@ -0,0 +1,1458 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Common config file handling for res_cdrel_custom. + * + * This file is a 'bit' complex. The reasoning is that the functions + * do as much work as possible at module load time to reduce the workload + * at run time. + * + */ + +#include "cdrel.h" + +#include "asterisk/config.h" +#include "asterisk/module.h" +#include "asterisk/paths.h" + +/* + * The DSV files get placed in specific subdirectories + * while the SQL databases get placed directly in /var/log/asterisk. + */ +static const char *dirname_map[cdrel_backend_type_end][cdrel_record_type_end] = { + [cdrel_backend_text] = { + [cdrel_record_cdr] = "cdr-custom", + [cdrel_record_cel] = "cel-custom", + }, + [cdrel_backend_db] = { + [cdrel_record_cdr] = NULL, + [cdrel_record_cel] = NULL, + } +}; + +struct field_parse_result { + char *result; + int functions; + int csv_quote; + int cdr; + int is_literal; + int unknown_functions; + int parse_failed; +}; + +/* + * To maximize the possibility that we can put a legacy config through the + * much faster advanced process, we need to ensure that we can handle + * everything in the legacy config. + */ +static const char *allowed_functions[] = { + [cdrel_record_cdr] = "CSV_QUOTE CDR CALLERID CHANNEL", + [cdrel_record_cel] = "CSV_QUOTE CALLERID CHANNEL eventtype eventtime eventenum userdeftype eventextra BRIDGEPEER", +}; + +static const char *special_vars[] = { + [cdrel_record_cdr] = "", + [cdrel_record_cel] = "eventtype eventtime eventenum userdeftype eventextra BRIDGEPEER", +}; + +/*! + * \internal + * \brief Parse a raw legacy field template. + * + * \example + * + * ${CSV_QUOTE(${eventtype})} + * ${CSV_QUOTE(${CALLERID(name)})} + * ${CSV_QUOTE(${CDR(src)})} + * ${CDR(uservar)} + * "some literal" + * ${CSV_QUOTE("some literal")} + * + * \param record_type CDR or CEL + * \param input_field_template The trimmed raw field template + * \return + */ + +static struct field_parse_result parse_field(enum cdrel_record_type record_type, char *input_field_template) +{ + char *tmp_field = NULL; + struct field_parse_result result = { 0, }; + + /* + * If the template starts with a double-quote, it's automatically + * a literal. + */ + if (input_field_template[0] == '"') { + result.result = ast_strdup(ast_strip_quoted(ast_strdupa(input_field_template), "\"", "\"")); + result.csv_quote = 1; + result.is_literal = 1; + return result; + } + + /* + * If it starts with a single quote, it's probably a legacy SQL template + * so we need to force quote it on output. + */ + tmp_field = ast_strip(ast_strdupa(input_field_template)); + + if (tmp_field[0] == '\'') { + result.csv_quote = 1; + } + + /* + * I really hate the fact that ast_strip really trims whitespace + * and ast_strip_quoted will strip anything in pairs. + * Anyway, get rid of any remaining enclosing quotes. + */ + tmp_field = ast_strip(ast_strip_quoted(tmp_field, "\"'", "\"'")); + + /* + * If the template now starts with a '$' it's either a dialplan function + * call or one of the special CEL field names. + * + * Examples: ${CSV_QUOTE(${CALLERID(name)})} + * ${eventtime} + * We're going to iterate over function removal until there's just + * a plain text string left. + * + */ + while (tmp_field[0] == '$') { + char *ptr = NULL; + /* + * A function name longer that 64 characters is highly unlikely but + * we'll check later. + */ + char func_name[65]; + + /* + * Skip over the '$' + * {CSV_QUOTE(${CALLERID(name)})} + * {eventtime} + */ + tmp_field++; + /* + * Remove any enclosing brace-like characters + * CSV_QUOTE(${CALLERID(name)}) + * eventtime + */ + tmp_field = ast_strip(ast_strip_quoted(tmp_field, "[{(", "]})")); + + /* + * Check what's left to see if it matches a special variable. + * If it does (like "eventtime" in the example), we're done. + */ + if (strstr(special_vars[record_type], tmp_field) != NULL) { + result.functions++; + break; + } + + /* + * At this point, it has to be a function name so find the + * openening '('. + * CSV_QUOTE(${CALLERID(name)}) + * ^ + * If we don't find one, it's something we don't recognise + * so bail. + */ + ptr = strchr(tmp_field, '('); + if (!ptr) { + result.parse_failed++; + continue; + } + + /* + * Copy from the beginning to the '(' to func_name, + * not exceeding func_name's size. + * + * CSV_QUOTE(${CALLERID(name)}) + * ^ + * CSV_QUOTE + * + * Then check that it's a function we can handle. + * If not, bail. + */ + ast_copy_string(func_name, tmp_field, MIN(sizeof(func_name), ptr - tmp_field + 1)); + if (strstr(allowed_functions[record_type], func_name) == NULL) { + result.parse_failed++; + result.unknown_functions++; + continue; + } + result.functions++; + /* + * If the function is CSV_QUOTE, we need to set the csv_quote flag. + */ + if (strcmp("CSV_QUOTE", func_name) == 0) { + result.csv_quote = 1; + } else if (strcmp("CDR", func_name) == 0) { + result.cdr = 1; + } + + /* + * ptr still points to the opening '(' so now strip it and the + * matching parens. + * + * ${CALLERID(name)} + * + */ + tmp_field = ast_strip_quoted(ptr, "(", ")"); + if (tmp_field[0] == '"' || tmp_field[0] == '\'') { + result.result = ast_strdup(ast_strip_quoted(tmp_field, "\"'", "\"'")); + result.csv_quote = 1; + result.is_literal = 1; + return result; + } + + /* Repeat the loop until there are no more functions or variables */ + } + + /* + * If the parse failed we'll send back the entire template. + */ + if (result.parse_failed) { + tmp_field = input_field_template; + } else { + /* + * If there were no functions or variables parsed then we'll + * assume it's a literal. + */ + if (result.functions == 0) { + result.is_literal = 1; + } + } + + result.result = ast_strdup(tmp_field); + if (result.result == NULL) { + result.parse_failed = 1; + } + + return result; +} + +/*! + * \internal + * \brief Parse a legacy DSV template string into a vector of individual strings. + * + * The resulting vector will look like it came from an advanced config and will + * be treated as such. + * + * \param record_type CDR or CEL. + * \param config_filename Config filename for logging purposes. + * \param output_filename Output filename for logging purposes. + * \param template The full template. + * \param fields A pointer to a string vector to receive the result. + * \retval 1 on success. + * \retval 0 on failure. + */ +static int parse_legacy_template(enum cdrel_record_type record_type, const char *config_filename, + const char *output_filename, const char *input_template, struct ast_vector_string *fields) +{ + char *template = ast_strdupa(input_template); + char *field_template = NULL; + int res = 0; + + /* + * We have no choice but to assume that a legacy config template uses commas + * as field delimiters. We don't have a reliable way to determine this ourselves. + */ + while((field_template = ast_strsep(&template, ',', AST_STRSEP_TRIM))) { + char *uservar = ""; + char *literal = ""; + /* Try to parse the field. */ + struct field_parse_result result = parse_field(record_type, field_template); + + ast_debug(2, "field: '%s' literal: %d quote: %d cdr: %d failed: %d funcs: %d unknfuncs: %d\n", result.result, + result.is_literal, result.csv_quote, result.cdr, + result.parse_failed, result.functions, result.unknown_functions); + + /* + * If it failed, + */ + if (!result.result || result.parse_failed) { + ast_free(result.result); + return 0; + } + if (result.is_literal) { + literal = "literal^"; + } + + if (!get_registered_field_by_name(record_type, result.result)) { + ast_debug(3, " %s->%s: field '%s' not found\n", cdrel_basename(config_filename), + cdrel_basename(output_filename), result.result); + /* + * If the result was found in a CDR function, treat it as a CDR user variable + * otherwise treat it as a literal. + */ + if (result.cdr) { + uservar = "uservar^"; + } else { + literal = "literal^"; + } + } + res = ast_asprintf(&field_template, "%s(%s%s)", result.result, S_OR(literal,uservar), result.csv_quote ? "quote" : "noquote"); + ast_free(result.result); + + if (!field_template || res < 0) { + ast_free(field_template); + return 0; + } + res = AST_VECTOR_APPEND(fields, field_template); + if (res != 0) { + ast_free(field_template); + return 0; + } + ast_debug(2, " field template: %s\n", field_template); + } + + return 1; +} + +/*! + * \fn Parse an advanced field template and allocate a cdrel_field for it. + * \brief + * + * \param config Config object. + * \param input_field_template Trimmed advanced field template. + * \return + */ +static struct cdrel_field *field_alloc(struct cdrel_config *config, const char *input_field_template) +{ + RAII_VAR(struct cdrel_field *, field, NULL, ast_free); + const struct cdrel_field *registered_field = NULL; + struct cdrel_field *rtn_field = NULL; + char *field_name = NULL; + char *data = NULL; + char *tmp_data = NULL; + char *closeparen = NULL; + char *qualifier = NULL; + enum cdrel_data_type forced_output_data_type = cdrel_data_type_end; + struct ast_flags field_flags = { 0 }; + + /* + * The database fields are specified field-by-field for legacy so we treat them + * as literals containing expressions which will be evaluated record-by-record. + */ + if (config->backend_type == cdrel_backend_db && config->config_type == cdrel_config_legacy) { + registered_field = get_registered_field_by_name(config->record_type, "literal"); + ast_assert(registered_field != NULL); + rtn_field = ast_calloc(1, sizeof(*field) + strlen(input_field_template) + 1); + if (!rtn_field) { + return NULL; + } + memcpy(rtn_field, registered_field, sizeof(*registered_field)); + strcpy(rtn_field->data, input_field_template); /* Safe */ + return rtn_field; + } + + /* + * If the field template is a quoted string, it's a literal. + * We don't check for qualifiers. + */ + if (input_field_template[0] == '"' || input_field_template[0] == '\'') { + data = ast_strip_quoted(ast_strdupa(input_field_template), "\"'", "\"'"); + ast_set_flag(&field_flags, cdrel_flag_literal); + ast_debug(3, " Using qualifier 'literal' for field '%s' flags: %s\n", data, + ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + field_name = "literal"; + } else { + field_name = ast_strdupa(input_field_template); + data = strchr(field_name, '('); + + if (data) { + *data = '\0'; + data++; + closeparen = strchr(data, ')'); + if (closeparen) { + *closeparen = '\0'; + } + } + } + + if (!ast_strlen_zero(data) && !ast_test_flag(&field_flags, cdrel_flag_literal)) { + char *data_swap = NULL; + tmp_data = ast_strdupa(data); + + while((qualifier = ast_strsep(&tmp_data, '^', AST_STRSEP_STRIP | AST_STRSEP_TRIM))) { + enum cdrel_data_type fodt = cdrel_data_type_end; + if (ast_strlen_zero(qualifier)) { + continue; + } + fodt = cdrel_data_type_from_str(qualifier); + if (fodt < cdrel_data_type_end) { + ast_set_flag(&field_flags, cdrel_flag_type_forced); + if (fodt == cdrel_type_uservar) { + ast_set_flag(&field_flags, cdrel_flag_uservar); + ast_debug(3, " Using qualifier '%s' for field '%s' flags: %s\n", qualifier, + field_name, ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + data_swap = ast_strdupa(field_name); + field_name = "uservar"; + } else if (fodt == cdrel_type_literal) { + ast_set_flag(&field_flags, cdrel_flag_literal); + ast_debug(3, " Using qualifier '%s' for field '%s' flags: %s\n", qualifier, + field_name, ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + data_swap = ast_strdupa(field_name); + field_name = "literal"; + } else { + forced_output_data_type = fodt; + ast_debug(3, " Using qualifier '%s' for field '%s' flags: %s\n", qualifier, + field_name, ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + } + continue; + } + if (strcasecmp(qualifier, "quote") == 0) { + ast_set_flag(&field_flags, cdrel_flag_quote); + ast_debug(3, " Using qualifier '%s' for field '%s' flags: %s\n", qualifier, + field_name, ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + continue; + } + if (strcasecmp(qualifier, "noquote") == 0) { + ast_set_flag(&field_flags, cdrel_flag_noquote); + ast_debug(3, " Using qualifier '%s' for field '%s' flags: %s\n", qualifier, + field_name, ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + continue; + } + if (strchr(qualifier, '%') != NULL) { + data_swap = ast_strdupa(qualifier); + ast_set_flag(&field_flags, cdrel_flag_format_spec); + ast_debug(3, " Using qualifier '%s' for field '%s' flags: %s\n", qualifier, + field_name, ast_str_tmp(128, cdrel_get_field_flags(&field_flags, &STR_TMP))); + } + } + if (ast_test_flag(&field_flags, cdrel_flag_quote) && ast_test_flag(&field_flags, cdrel_flag_noquote)) { + ast_log(LOG_WARNING, "%s->%s: Field '%s(%s)' has both quote and noquote\n", + config->config_filename, config->output_filename, field_name, data); + return NULL; + } + data = data_swap; + } + + /* + * Check again for literal. + */ + if (ast_test_flag(&field_flags, cdrel_flag_literal)) { + if (config->format_type == cdrel_format_json && !strchr(data, ':')) { + ast_log(LOG_WARNING, "%s->%s: Literal field '%s' must be formatted as \"name: value\" when using the 'json' format\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + input_field_template); + return NULL; + } + } + + /* + * Now look the field up by just the field name without any data. + */ + registered_field = get_registered_field_by_name(config->record_type, field_name); + if (!registered_field) { + ast_log(LOG_WARNING, "%s->%s: Field '%s' not found\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), field_name); + return NULL; + } + + field = ast_calloc(1, sizeof(*registered_field) + strlen(input_field_template) + 1); + if (!field) { + return NULL; + } + memcpy(field, registered_field, sizeof(*field)); + + if (!ast_strlen_zero(data)) { + strcpy(field->data, data); /* Safe */ + } + + /* + * For user variables, we use the field name from the data + * we set above. + */ + if (field->input_data_type == cdrel_type_uservar) { + field->name = field->data; + } + + if (field->input_data_type == cdrel_type_literal && config->format_type == cdrel_format_json) { + /* + * data should look something like this... lname: lvalue + * We'll need to make field->name point to "lname" and + * field->data point to "lvalue" so that when output the + * json will look like... "lname": "lvalue". + * Since field->data is already long enough to to handle both, + * we'll do this... + * field->data = lvalue\0lname\0 + * field->name = ^ + */ + char *ptr = strchr(data, ':');/* Safe since we checked data for a ':' above */ + *ptr = '\0'; + ptr++; + /* + * data: lname\0 lvalue + * ptr: ^ + */ + strcpy(field->data, ast_strip_quoted(ptr, "\"", "\"")); /* Safe */ + /* + * field->data: lvalue\0 + */ + ptr = field->data + strlen(field->data); + ptr++; + /* + * field->data: lvalue\0 + * ptr: ^ + * data: lname\0 lvalue + */ + strcpy(ptr, data); /* Safe */ + /* + * field->data: lvalue\0lname\0 + */ + field->name = ptr; + /* + * field->data: lvalue\0lname\0 + * field->name: ^ + */ + } + + if (forced_output_data_type < cdrel_data_type_end) { + field->output_data_type = forced_output_data_type; + } + field->flags = field_flags; + + /* + * Unless the field has the 'noquote' flag, we'll set the 'quote' + * flag if quoting method is 'all' or 'non_numeric'. + */ + if (!ast_test_flag(&field->flags, cdrel_flag_noquote)) { + if (config->quoting_method == cdrel_quoting_method_all) { + ast_set_flag(&field->flags, cdrel_flag_quote); + } else if (config->quoting_method == cdrel_quoting_method_non_numeric) { + if (field->output_data_type > cdrel_data_type_strings_end) { + ast_set_flag(&field->flags, cdrel_flag_noquote); + } else { + ast_set_flag(&field->flags, cdrel_flag_quote); + } + } + } + + if (config->quoting_method == cdrel_quoting_method_none) { + ast_clear_flag(&field->flags, cdrel_flag_quote); + ast_set_flag(&field->flags, cdrel_flag_noquote); + } + + ast_debug(2, "%s->%s: Field '%s' processed -> name:'%s' input_type:%s output_type:%s flags:'%s' data:'%s'\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), input_field_template, + field->name, DATA_TYPE_STR(field->input_data_type), + DATA_TYPE_STR(field->output_data_type), + ast_str_tmp(128, cdrel_get_field_flags(&field->flags, &STR_TMP)), + field->data); + + rtn_field = field; + field = NULL; + return rtn_field; +} + +static void field_template_vector_free(struct ast_vector_string *fields) { + AST_VECTOR_RESET(fields, ast_free); + AST_VECTOR_PTR_FREE(fields); +} + +/*! + * \internal + * \brief Load all the fields in the string vector. + * + * \param config Config object + * \param fields String vector. + * \retval 0 on success. + * \retval -1 on failure. + */ +static int load_fields(struct cdrel_config *config, struct ast_vector_string *fields) +{ + int res = 0; + int ix = 0; + + ast_debug(1, "%s->%s: Loading fields\n", cdrel_basename(config->config_filename), + cdrel_basename(config->output_filename)); + + for (ix = 0; ix < AST_VECTOR_SIZE(fields); ix++) { + char *field_name = AST_VECTOR_GET(fields, ix); + struct cdrel_field *field = NULL; + + field = field_alloc(config, field_name); + if (!field) { + res = -1; + continue; + } + + if (AST_VECTOR_APPEND(&config->fields, field) != 0) { + ast_free(field); + return -1; + } + } + + return res; +} + +static void config_free(struct cdrel_config *config) +{ + if (!config) { + return; + } + + if (config->insert) { + sqlite3_finalize(config->insert); + config->insert = NULL; + } + + if (config->db) { + sqlite3_close(config->db); + config->db = NULL; + } + + ast_mutex_destroy(&config->lock); + ast_string_field_free_memory(config); + AST_VECTOR_RESET(&config->fields, ast_free); + AST_VECTOR_FREE(&config->fields); + ast_free(config); +} + +/*! + * \internal + * \brief Allocate a config object. + * + * You should know what these are by now :) + * + * \param record_type + * \param backend_type + * \param config_type + * \param config_filename + * \param output_filename + * \param template + * \return + */ +static struct cdrel_config *config_alloc(enum cdrel_record_type record_type, + enum cdrel_backend_type backend_type, enum cdrel_config_type config_type, + const char *config_filename, const char *output_filename, const char *template) +{ + RAII_VAR(struct cdrel_config *, config, NULL, config_free); + struct cdrel_config *rtn_config = NULL; + const char *file_suffix = ""; + int res = 0; + + ast_debug(1, "%s->%s: Loading\n", cdrel_basename(config_filename), cdrel_basename(output_filename)); + + config = ast_calloc_with_stringfields(1, struct cdrel_config, 1024); + if (!config) { + return NULL; + } + + if (ast_string_field_set(config, config_filename, config_filename) != 0) { + return NULL; + } + + config->record_type = record_type; + config->backend_type = backend_type; + config->dummy_channel_alloc = cdrel_dummy_channel_allocators[record_type]; + config->config_type = config_type; + + /* Set defaults */ + config->format_type = cdrel_format_dsv; + config->separator[0] = ','; + switch(backend_type) { + case cdrel_backend_text: + config->quote[0] = '"'; + config->quoting_method = cdrel_quoting_method_all; + break; + case cdrel_backend_db: + config->quote[0] = '\0'; + config->format_type = cdrel_format_sql; + config->quoting_method = cdrel_quoting_method_none; + if (!ast_ends_with(output_filename, ".db")) { + file_suffix = ".db"; + } + break; + default: + ast_log(LOG_ERROR, "%s->%s: Unknown backend type '%d'\n", cdrel_basename(config_filename), + cdrel_basename(output_filename), backend_type); + break; + } + config->quote_escape[0] = config->quote[0]; + + res = ast_string_field_set(config, template, template); + if (res != 0) { + return NULL; + } + + if (output_filename[0] == '/') { + res = ast_string_field_build(config, output_filename, "%s%s", output_filename, file_suffix); + } else { + const char *subdir = dirname_map[backend_type][record_type]; + res = ast_string_field_build(config, output_filename, "%s/%s%s%s%s", + ast_config_AST_LOG_DIR, S_OR(subdir, ""), ast_strlen_zero(subdir) ? "" : "/", output_filename, file_suffix); + } + if (res != 0) { + return NULL; + } + ast_mutex_init(&config->lock); + + rtn_config = config; + config = NULL; + return rtn_config; +} + +/*! + * \internal + * \brief Load the "columns" parameter from a database config. + * + * \param config Config object + * \param columns The columns parameter. + * \param column_count Set to the count of columns parsed. + * \retval 0 on success. + * \retval -1 on failure. + */ +static int load_database_columns(struct cdrel_config *config, const char *columns, int *column_count) +{ + char *col = NULL; + char *cols = NULL; + RAII_VAR(struct ast_str *, column_string, NULL, ast_free); + + ast_debug(1, "%s->%s: Loading columns\n", cdrel_basename(config->config_filename), + cdrel_basename(config->output_filename)); + + if (!(column_string = ast_str_create(1024))) { + return -1; + } + + cols = ast_strdupa(columns); + *column_count = 0; + + /* We need to trim and remove any single or double quotes from each column name. */ + while((col = ast_strsep(&cols, ',', AST_STRSEP_TRIM))) { + col = ast_strip(ast_strip_quoted(col, "'\"", "'\"")); + if (ast_str_append(&column_string, 0, "%s%s", *column_count ? "," : "", col) <= 0) { + return -1; + } + (*column_count)++; + } + + if (ast_string_field_set(config, db_columns, ast_str_buffer(column_string)) != 0) { + return -1; + } + + return 0; +} + +static char *make_stmt_placeholders(int columns) +{ + char *placeholders = ast_malloc(2 * columns), *c = placeholders; + if (placeholders) { + for (; columns; columns--) { + *c++ = '?'; + *c++ = ','; + } + *(c - 1) = 0; + } + return placeholders; +} + +/*! + * \internal + * \brief Open an sqlite3 database and create the table if needed. + * + * \param config Config object. + * \retval 0 on success. + * \retval -1 on failure. + */ +static int open_database(struct cdrel_config *config) +{ + char *sql = NULL; + int res = 0; + char *placeholders = NULL; + + ast_debug(1, "%s->%s: opening database\n", cdrel_basename(config->config_filename), + cdrel_basename(config->output_filename)); + res = sqlite3_open(config->output_filename, &config->db); + if (res != SQLITE_OK) { + ast_log(LOG_WARNING, "%s->%s: Could not open database\n", cdrel_basename(config->config_filename), + cdrel_basename(config->output_filename)); + return -1; + } + + sqlite3_busy_timeout(config->db, config->busy_timeout); + + /* is the table there? */ + sql = sqlite3_mprintf("SELECT COUNT(*) FROM %q;", config->db_table); + if (!sql) { + return -1; + } + res = sqlite3_exec(config->db, sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (res != SQLITE_OK) { + /* + * Create the table. + * We don't use %q for the column list here since we already escaped when building it + */ + sql = sqlite3_mprintf("CREATE TABLE %q (AcctId INTEGER PRIMARY KEY, %s)", + config->db_table, config->db_columns); + res = sqlite3_exec(config->db, sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (res != SQLITE_OK) { + ast_log(LOG_WARNING, "%s->%s: Unable to create table '%s': %s\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + config->db_table, sqlite3_errmsg(config->db)); + return -1; + } + } else { + /* + * If the table exists, make sure the number of columns + * matches the config. + */ + sqlite3_stmt *get_stmt; + int existing_columns = 0; + int config_columns = AST_VECTOR_SIZE(&config->fields); + + sql = sqlite3_mprintf("SELECT * FROM %q;", config->db_table); + if (!sql) { + return -1; + } + res = sqlite3_prepare_v2(config->db, sql, -1, &get_stmt, NULL); + sqlite3_free(sql); + if (res != SQLITE_OK) { + ast_log(LOG_WARNING, "%s->%s: Unable to get column count for table '%s': %s\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + config->db_table, sqlite3_errmsg(config->db)); + return -1; + } + /* + * prepare figures out the number of columns that would be in a result + * set. We don't need to execute the statement. + */ + existing_columns = sqlite3_column_count(get_stmt); + sqlite3_finalize(get_stmt); + /* config_columns doesn't include the sequence field */ + if ((config_columns + 1) != existing_columns) { + ast_log(LOG_WARNING, "%s->%s: The number of fields in the config (%d) doesn't equal the" + " nummber of data columns (%d) in the existing %s table. This config is disabled.\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + config_columns, existing_columns - 1, config->db_table); + return -1; + } + } + + placeholders = make_stmt_placeholders(AST_VECTOR_SIZE(&config->fields)); + if (!placeholders) { + return -1; + } + + /* Inserting NULL in the ID column still generates an ID */ + sql = sqlite3_mprintf("INSERT INTO %q VALUES (NULL,%s)", config->db_table, placeholders); + ast_free(placeholders); + if (!sql) { + return -1; + } + + res = sqlite3_prepare_v3(config->db, sql, -1, SQLITE_PREPARE_PERSISTENT, &config->insert, NULL); + if (res != SQLITE_OK) { + ast_log(LOG_ERROR, "%s->%s: Unable to prepare INSERT statement '%s': %s\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + sql, sqlite3_errmsg(config->db)); + return -1; + } + + return 0; +} + +/*! + * \internal + * \brief Load a database config from a config file category. + * + * \param record_type CDR or CEL. + * \param category The category (becomes the database file name). + * \param config_filename The config filename for logging purposes. + * \return config or NULL. + */ +static struct cdrel_config *load_database_config(enum cdrel_record_type record_type, + struct ast_category *category, const char *config_filename) +{ + const char *category_name = ast_category_get_name(category); + RAII_VAR(struct cdrel_config *, config, NULL, config_free); + struct cdrel_config *rtn_config = NULL; + int res = 0; + int column_count = 0; + int value_count = 0; + int field_check_passed = 0; + const char *template = ast_variable_find(category, "values"); + enum cdrel_config_type config_type; + const char *value; + char *tmp_fields = NULL; + RAII_VAR(struct ast_vector_string *, field_templates, ast_calloc(1, sizeof(*field_templates)), field_template_vector_free); + + if (!ast_strlen_zero(template)) { + config_type = cdrel_config_legacy; + } else { + template = ast_variable_find(category, "fields"); + if (!ast_strlen_zero(template)) { + config_type = cdrel_config_advanced; + } + } + if (ast_strlen_zero(template)) { + ast_log(LOG_WARNING, "%s->%s: Neither 'values' nor 'fields' specified\n", + cdrel_basename(config_filename), cdrel_basename(category_name)); + return NULL; + } + + res = AST_VECTOR_INIT(field_templates, 25); + if (res != 0) { + return NULL; + } + + /* + * Let's try and and parse a legacy config to see if we can turn + * it into an advanced condig. + */ + if (config_type == cdrel_config_legacy) { + field_check_passed = parse_legacy_template(record_type, config_filename, + category_name, template, field_templates); + if (field_check_passed) { + config_type = cdrel_config_advanced; + ast_log(LOG_NOTICE, "%s->%s: Legacy config upgraded to advanced\n", + cdrel_basename(config_filename), cdrel_basename(category_name)); + } else { + AST_VECTOR_RESET(field_templates, ast_free); + ast_log(LOG_NOTICE, "%s->%s: Unable to upgrade legacy config to advanced. Continuing to process as legacy\n", + cdrel_basename(config_filename), cdrel_basename(category_name)); + } + } + + /* + * If we could, the fields vector will be populated so we don't need to do it again. + * If it was an advanced config or a legacy one we couldn't parse, + * we need to split the string into the vector. + */ + if (AST_VECTOR_SIZE(field_templates) == 0) { + tmp_fields = ast_strdupa(template); + while((value = ast_strsep(&tmp_fields, ',', AST_STRSEP_TRIM))) { + res = AST_VECTOR_APPEND(field_templates, ast_strdup(value)); + if (res != 0) { + return NULL; + } + } + } + + config = config_alloc(record_type, cdrel_backend_db, config_type, + config_filename, category_name, template); + if (!config) { + return NULL; + } + + config->busy_timeout = 1000; + + res = ast_string_field_set(config, db_table, + S_OR(ast_variable_find(category, "table"), config->record_type == cdrel_record_cdr ? "cdr" : "cel")); + if (res != 0) { + return NULL; + } + + /* sqlite3_busy_timeout in miliseconds */ + if ((value = ast_variable_find(category, "busy_timeout")) != NULL) { + if (ast_parse_arg(value, PARSE_INT32|PARSE_DEFAULT, &config->busy_timeout, 1000) != 0) { + ast_log(LOG_WARNING, "%s->%s: Invalid busy_timeout value '%s' specified. Using 1000 instead.\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), value); + } + } + + /* Columns */ + value = ast_variable_find(category, "columns"); + if (ast_strlen_zero(value)) { + ast_log(LOG_WARNING, "%s->%s: The 'columns' parameter is missing", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename)); + return NULL; + } + + if (load_database_columns(config, value, &column_count) != 0) { + return NULL; + } + + if (AST_VECTOR_INIT(&config->fields, AST_VECTOR_SIZE(field_templates)) != 0) { + return NULL; + } + + if (load_fields(config, field_templates) != 0) { + return NULL; + } + + value_count = AST_VECTOR_SIZE(&config->fields); + + if (value_count != column_count) { + ast_log(LOG_WARNING, "%s->%s: There are %d columns but %d values\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + column_count, value_count); + return NULL; + } + + res = open_database(config); + if (res != 0) { + return NULL; + } + + ast_log(LOG_NOTICE, "%s->%s: Logging %s records to table '%s'\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + RECORD_TYPE_STR(config->record_type), + config->db_table); + + rtn_config = config; + config = NULL; + return rtn_config; +} + +/*! + * \internal + * \brief Load all the categories in a database config file. + * + * \param record_type + * \param configs + * \param config_filename + * \param reload + * \retval 0 on success or reload not needed. + * \retval -1 on failure. + */ +static int load_database_config_file(enum cdrel_record_type record_type, struct cdrel_configs *configs, + const char *config_filename, int reload) +{ + struct ast_config *cfg; + struct ast_flags config_flags = { reload ? CONFIG_FLAG_FILEUNCHANGED : 0 }; + struct ast_category *category = NULL; + + ast_debug(1, "%s: %soading\n", config_filename, reload ? "Rel" : "L"); + cfg = ast_config_load(config_filename, config_flags); + if (!cfg || cfg == CONFIG_STATUS_FILEINVALID) { + ast_log(LOG_ERROR, "Unable to load %s. Not logging %ss to custom database\n", + config_filename, RECORD_TYPE_STR(record_type)); + return -1; + } else if (cfg == CONFIG_STATUS_FILEUNCHANGED) { + ast_debug(1, "%s: Config file unchanged, not reloading\n", config_filename); + return 0; + } + + while ((category = ast_category_browse_filtered(cfg, NULL, category, NULL))) { + struct cdrel_config *config = NULL; + + config = load_database_config(record_type, category, config_filename); + if (!config) { + continue; + } + + if (AST_VECTOR_APPEND(configs, config) != 0) { + config_free(config); + break; + } + } + + ast_config_destroy(cfg); + + ast_log(LOG_NOTICE, "%s: Loaded %d configs\n", config_filename, (int)AST_VECTOR_SIZE(configs)); + + /* Only fail if no configs were valid. */ + return AST_VECTOR_SIZE(configs) > 0 ? 0 : -1; +} + +/*! + * \internal + * \brief Load a legacy config from a single entry in the "mappings" castegory. + * + * \param record_type + * \param config_filename + * \param output_filename + * \param template + * \return config or NULL. + */ +static struct cdrel_config *load_text_file_legacy_config(enum cdrel_record_type record_type, + const char *config_filename, const char *output_filename, const char *template) +{ + struct cdrel_config *config = NULL; + int field_check_passed = 0; + int res = 0; + RAII_VAR(struct ast_vector_string *, fields, ast_calloc(1, sizeof(*fields)), field_template_vector_free); + + res = AST_VECTOR_INIT(fields, 25); + if (res != 0) { + return NULL; + } + + /* + * Let's try and and parse a legacy config to see if we can turn + * it into an advanced condig. + */ + field_check_passed = parse_legacy_template(record_type, config_filename, + output_filename, template, fields); + + /* + * If we couldn't, treat as legacy. + */ + if (!field_check_passed) { + config = config_alloc(record_type, cdrel_backend_text, cdrel_config_legacy, + config_filename, output_filename, template); + ast_log(LOG_NOTICE, "%s->%s: Logging legacy %s records\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + RECORD_TYPE_STR(config->record_type)); + return config; + } + + config = config_alloc(record_type, cdrel_backend_text, cdrel_config_advanced, + config_filename, output_filename, template); + if (!config) { + return NULL; + } + config->format_type = cdrel_format_dsv; + config->quote[0] = '"'; + config->quote_escape[0] = '"'; + config->separator[0] = ','; + config->quoting_method = cdrel_quoting_method_all; + + if (AST_VECTOR_INIT(&config->fields, AST_VECTOR_SIZE(fields)) != 0) { + return NULL; + } + + if (load_fields(config, fields) != 0) { + return NULL; + } + + ast_log(LOG_NOTICE, "%s->%s: Logging %s records\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + RECORD_TYPE_STR(config->record_type)); + + return config; +} + +/*! + * \internal + * \brief Load an advanced config from a config file category. + * + * \param record_type + * \param category + * \param config_filename + * \return config or NULL. + */ +static struct cdrel_config *load_text_file_advanced_config(enum cdrel_record_type record_type, + struct ast_category *category, const char *config_filename) +{ + const char *category_name = ast_category_get_name(category); + RAII_VAR(struct cdrel_config *, config, NULL, config_free); + struct cdrel_config *rtn_config = NULL; + const char *value; + int res = 0; + const char *fields_value = ast_variable_find(category, "fields"); + char *tmp_fields = NULL; + RAII_VAR(struct ast_vector_string *, fields, ast_calloc(1, sizeof(*fields)), field_template_vector_free); + + if (ast_strlen_zero(fields_value)) { + ast_log(LOG_WARNING, "%s->%s: Missing 'fields' parameter\n", + cdrel_basename(config_filename), category_name); + return NULL; + } + + config = config_alloc(record_type, cdrel_backend_text, cdrel_config_advanced, + config_filename, category_name, fields_value); + + value = ast_variable_find(category, "format"); + if (!ast_strlen_zero(value)) { + if (ast_strings_equal(value, "json")) { + config->format_type = cdrel_format_json; + config->separator[0] = ','; + config->quote[0] = '"'; + config->quote_escape[0] = '\\'; + config->quoting_method = cdrel_quoting_method_non_numeric; + } else if (ast_strings_equal(value, "dsv")) { + config->format_type = cdrel_format_dsv; + } else { + ast_log(LOG_WARNING, "%s->%s: Invalid format '%s'\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), value); + return NULL; + } + } + + if (config->format_type != cdrel_format_json) { + value = ast_variable_find(category, "separator_character"); + if (!ast_strlen_zero(value)) { + ast_copy_string(config->separator, ast_unescape_c(ast_strdupa(value)), 2); + } + + value = ast_variable_find(category, "quote_character"); + if (!ast_strlen_zero(value)) { + ast_copy_string(config->quote, value, 2); + } + + value = ast_variable_find(category, "quote_escape_character"); + if (!ast_strlen_zero(value)) { + ast_copy_string(config->quote_escape, value, 2); + } + + value = ast_variable_find(category, "quoting_method"); + if (!ast_strlen_zero(value)) { + if (ast_strings_equal(value, "all")) { + config->quoting_method = cdrel_quoting_method_all; + } else if (ast_strings_equal(value, "minimal")) { + config->quoting_method = cdrel_quoting_method_minimal; + } else if (ast_strings_equal(value, "non_numeric")) { + config->quoting_method = cdrel_quoting_method_non_numeric; + } else if (ast_strings_equal(value, "none")) { + config->quoting_method = cdrel_quoting_method_none; + } else { + ast_log(LOG_WARNING, "%s->%s: Invalid quoting method '%s'\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), value); + return NULL; + } + } + } + + res = AST_VECTOR_INIT(fields, 20); + if (res != 0) { + return NULL; + } + tmp_fields = ast_strdupa(fields_value); + while((value = ast_strsep(&tmp_fields, ',', AST_STRSEP_TRIM))) { + res = AST_VECTOR_APPEND(fields, ast_strdup(value)); + if (res != 0) { + return NULL; + } + } + + if (AST_VECTOR_INIT(&config->fields, AST_VECTOR_SIZE(fields)) != 0) { + return NULL; + } + + if (load_fields(config, fields) != 0) { + return NULL; + } + + ast_log(LOG_NOTICE, "%s->%s: Logging %s records\n", + cdrel_basename(config->config_filename), cdrel_basename(config->output_filename), + RECORD_TYPE_STR(config->record_type)); + + rtn_config = config; + config = NULL; + + return rtn_config; +} + +/*! + * \internal + * \brief Load a legacy configs from the "mappings" category. + * + * \param record_type + * \param configs + * \param category + * \param config_filename + * \retval 0 on success. + * \retval -1 on failure. + */ +static int load_text_file_legacy_mappings(enum cdrel_record_type record_type, + struct cdrel_configs *configs, struct ast_category *category, + const char *config_filename) +{ + struct ast_variable *var = NULL; + + for (var = ast_category_first(category); var; var = var->next) { + struct cdrel_config *config = NULL; + + if (ast_strlen_zero(var->name) || ast_strlen_zero(var->value)) { + ast_log(LOG_WARNING, "%s: %s mapping must have both a filename and a template at line %d\n", + cdrel_basename(config_filename), RECORD_TYPE_STR(config->record_type), var->lineno); + continue; + } + + config = load_text_file_legacy_config(record_type, config_filename, var->name, var->value); + if (!config) { + continue; + } + + if (AST_VECTOR_APPEND(configs, config) != 0) { + config_free(config); + return -1; + } + } + + return 0; +} + +/*! + * \internal + * \brief Load all text file backend configs from a config file. + * + * \param record_type + * \param configs + * \param config_filename + * \param reload + * \return + */ +static int load_text_file_config_file(enum cdrel_record_type record_type, + struct cdrel_configs *configs, const char *config_filename, int reload) +{ + struct ast_config *cfg; + struct ast_flags config_flags = { reload ? CONFIG_FLAG_FILEUNCHANGED : 0 }; + struct ast_category *category = NULL; + + ast_debug(1, "%s: %soading\n", config_filename, reload ? "Rel" : "L"); + cfg = ast_config_load(config_filename, config_flags); + if (!cfg || cfg == CONFIG_STATUS_FILEINVALID) { + ast_log(LOG_ERROR, "Unable to load %s. Not logging %ss to custom files\n", + config_filename, RECORD_TYPE_STR(record_type)); + return -1; + } else if (cfg == CONFIG_STATUS_FILEUNCHANGED) { + ast_debug(1, "%s: Config file unchanged, not reloading\n", config_filename); + return 0; + } + + while ((category = ast_category_browse_filtered(cfg, NULL, category, NULL))) { + const char *category_name = ast_category_get_name(category); + + if (ast_strings_equal(category_name, "mappings")) { + load_text_file_legacy_mappings(record_type, configs, category, config_filename); + } else { + struct cdrel_config * config = load_text_file_advanced_config(record_type, category, + config_filename); + if (!config) { + continue; + } + if (AST_VECTOR_APPEND(configs, config) != 0) { + config_free(config); + return -1; + } + } + } + + ast_config_destroy(cfg); + + ast_log(LOG_NOTICE, "%s: Loaded %d configs\n", config_filename, (int)AST_VECTOR_SIZE(configs)); + + /* Only fail if no configs were valid. */ + return AST_VECTOR_SIZE(configs) > 0 ? 0 : -1; +} + +static int register_backend(enum cdrel_record_type record_type, const char *backend_name, void *log_cb) +{ + switch(record_type) { + case cdrel_record_cdr: + return ast_cdr_register(backend_name, "", log_cb); + case cdrel_backend_db: + return ast_cel_backend_register(backend_name, log_cb); + default: + return -1; + } +} + +static int unregister_backend(enum cdrel_record_type record_type, const char *backend_name) +{ + switch(record_type) { + case cdrel_record_cdr: + return ast_cdr_unregister(backend_name); + case cdrel_record_cel: + return ast_cel_backend_unregister(backend_name); + default: + return -1; + } +} + +static int load_config_file(enum cdrel_backend_type output_type, enum cdrel_record_type record_type, + struct cdrel_configs *configs, const char *filename, int reload) +{ + switch(output_type) { + case cdrel_backend_text: + return load_text_file_config_file(record_type, configs, filename, reload); + case cdrel_backend_db: + return load_database_config_file(record_type, configs, filename, reload); + default: + return -1; + } +} + +int cdrel_reload_module(enum cdrel_backend_type output_type, enum cdrel_record_type record_type, + struct cdrel_configs **configs, const char *filename) +{ + int res = 0; + struct cdrel_configs *old_configs = *configs; + struct cdrel_configs *new_configs = NULL; + + /* + * Save new config to a temporary vector to make sure the + * configs are valid before swapping them in. + */ + new_configs = ast_malloc(sizeof(*new_configs)); + if (!new_configs) { + return AST_MODULE_LOAD_DECLINE; + } + + if (AST_VECTOR_INIT(new_configs, AST_VECTOR_SIZE(old_configs)) != 0) { + return AST_MODULE_LOAD_DECLINE; + } + + res = load_config_file(output_type, record_type, new_configs, filename, 1); + if (res != 0) { + AST_VECTOR_RESET(new_configs, config_free); + AST_VECTOR_PTR_FREE(new_configs); + return AST_MODULE_LOAD_DECLINE; + } + + /* Now swap the new ones in. */ + *configs = new_configs; + + /* Free the old ones. */ + AST_VECTOR_RESET(old_configs, config_free); + AST_VECTOR_PTR_FREE(old_configs); + + return AST_MODULE_LOAD_SUCCESS; + + + return -1; +} + +struct cdrel_configs *cdrel_load_module(enum cdrel_backend_type backend_type, + enum cdrel_record_type record_type, const char *filename, const char *backend_name, + void *log_cb) +{ + struct cdrel_configs *configs = ast_calloc(1, sizeof(*configs)); + if (!configs) { + return NULL; + } + ast_debug(1, "Loading %s %s\n", RECORD_TYPE_STR(record_type), MODULE_TYPE_STR(backend_type)); + + if (AST_VECTOR_INIT(configs, 5) != 0) { + cdrel_unload_module(backend_type, record_type, configs, backend_name); + return NULL; + } + + if (load_config_file(backend_type, record_type, configs, filename, 0) != 0) { + cdrel_unload_module(backend_type, record_type, configs, backend_name); + return NULL; + } + + if (register_backend(record_type, backend_name, log_cb)) { + cdrel_unload_module(backend_type, record_type, configs, backend_name); + return NULL; + } + + return configs; +} + +int cdrel_unload_module(enum cdrel_backend_type backend_type, enum cdrel_record_type record_type, + struct cdrel_configs *configs, const char *backend_name) +{ + if (unregister_backend(record_type, backend_name) != 0) { + return -1; + } + + AST_VECTOR_RESET(configs, config_free); + AST_VECTOR_PTR_FREE(configs); + + return 0; +} diff --git a/res/cdrel_custom/formatters.c b/res/cdrel_custom/formatters.c new file mode 100644 index 0000000000..fc4e22a5dc --- /dev/null +++ b/res/cdrel_custom/formatters.c @@ -0,0 +1,194 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Formatters + * + */ + +#include "cdrel.h" +#include "asterisk/json.h" +#include "asterisk/cdr.h" +#include "asterisk/cel.h" + +static char *quote_escaper(const char *value, char quote, char quote_escape, char *qvalue) +{ + char *ptr = qvalue; + const char *dataptr = value; + + if (!qvalue) { + return NULL; + } + + while(*dataptr != '\0') { + if (*dataptr == quote) { + *ptr++ = quote_escape; + } + *ptr++ = *dataptr++; + } + *ptr='\0'; + return qvalue; +} + +static int format_string(struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *input_value, struct cdrel_value *output_value) +{ + int quotes_count = 0; + int needs_quoting = ast_test_flag(&field->flags, cdrel_flag_quote); + int ix = 0; + int input_len = strlen(input_value->values.string ?: ""); + int res = 0; + char *qvalue = NULL; + char *evalue = (char *)input_value->values.string; + + output_value->data_type = cdrel_type_string; + output_value->field_name = input_value->field_name; + + if (input_len == 0) { + output_value->values.string = ast_strdup(needs_quoting ? "\"\"" : ""); + return 0; + } + + for (ix = 0; ix < input_len; ix++) { + char c = input_value->values.string[ix]; + if (c == config->quote[0]) { + quotes_count++; + needs_quoting = 1; + } else if (c == config->separator[0] || c == '\r' || c == '\n') { + needs_quoting = 1; + } + } + + ast_debug(5, "%s: %s=%s %s", cdrel_basename(config->output_filename), input_value->field_name, + input_value->values.string, ast_str_tmp(128, cdrel_get_field_flags(&field->flags, &STR_TMP))); + + if (!needs_quoting) { + output_value->values.string = ast_strdup(input_value->values.string); + return output_value->values.string == NULL ? -1 : 0; + } + + /* For every quote_count, we need an extra byte for the quote escape character. */ + qvalue = ast_alloca(input_len + quotes_count + 1); + if (quotes_count) { + evalue = quote_escaper(input_value->values.string, config->quote[0], config->quote_escape[0], qvalue); + } + res = ast_asprintf(&output_value->values.string, "%s%s%s", config->quote, evalue, config->quote); + return res < 0 ? -1 : 0; +} + +#define DEFINE_FORMATTER(_name, _field, _type, _fmt) \ + static int format_ ## _name (struct cdrel_config *config, \ + struct cdrel_field *field, struct cdrel_value *input_value, struct cdrel_value *output_value) \ + { \ + int res = 0; \ + char *quote = ""; \ + if (input_value->data_type != output_value->data_type) { \ + /* Forward to other formatter */ \ + return cdrel_field_formatters[output_value->data_type](config, field, input_value, output_value); \ + } \ + output_value->field_name = input_value->field_name; \ + if ((config->quoting_method == cdrel_quoting_method_all || ast_test_flag(&field->flags, cdrel_flag_quote)) \ + && !ast_test_flag(&field->flags, cdrel_flag_noquote)) { \ + quote = config->quote; \ + } \ + res = ast_asprintf(&output_value->values.string, "%s" _fmt "%s", quote, input_value->values._field, quote); \ + output_value->data_type = cdrel_type_string; \ + return res < 0 ? -1 : 0; \ + } + +DEFINE_FORMATTER(uint32, uint32, uint32_t, "%u") +DEFINE_FORMATTER(int32, int32, int32_t, "%d") +DEFINE_FORMATTER(uint64, uint64, uint64_t, "%lu") +DEFINE_FORMATTER(int64, int64, int64_t, "%ld") +DEFINE_FORMATTER(float, floater, float, "%.1f") + +static int format_timeval(struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *input_value, struct cdrel_value *output_value) +{ + struct ast_tm tm; + char tempbuf[64] = { 0, }; + int res = 0; + const char *format = "%Y-%m-%d %T"; + + output_value->field_name = input_value->field_name; + + if (field->output_data_type == cdrel_type_int64) { + output_value->data_type = cdrel_type_int64; + output_value->values.int64 = input_value->values.tv.tv_sec; + return format_int64(config, field, output_value, output_value); + } else if (field->output_data_type == cdrel_type_float) { + output_value->data_type = cdrel_type_float; + output_value->values.floater = ((float)input_value->values.tv.tv_sec) + ((float)input_value->values.tv.tv_usec) / 1000000.0; + return format_float(config, field, output_value, output_value); + } else if (!ast_strlen_zero(field->data)) { + format = field->data; + } + + if (input_value->values.tv.tv_sec > 0) { + ast_localtime(&input_value->values.tv, &tm, NULL); + ast_strftime(tempbuf, sizeof(tempbuf), format, &tm); + } + input_value->values.string = tempbuf; + input_value->data_type = cdrel_type_string; + output_value->data_type = cdrel_type_string; + res = format_string(config, field, input_value, output_value); + return res; +} + +static int format_amaflags(struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *input_value, struct cdrel_value *output_value) +{ + int res = 0; + + input_value->values.string = (char *)ast_channel_amaflags2string(input_value->values.int64); + input_value->data_type = cdrel_type_string; + output_value->data_type = cdrel_type_string; + res = format_string(config, field, input_value, output_value); + return res; +} + +static int format_disposition(struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *input_value, struct cdrel_value *output_value) +{ + int res = 0; + + input_value->values.string = (char *)ast_cdr_disp2str(input_value->values.int64); + input_value->data_type = cdrel_type_string; + output_value->data_type = cdrel_type_string; + res = format_string(config, field, input_value, output_value); + return res; +} + +int load_formatters(void) +{ + ast_debug(1, "Loading Formatters\n"); + cdrel_field_formatters[cdrel_type_string] = format_string; + cdrel_field_formatters[cdrel_type_int32] = format_int32; + cdrel_field_formatters[cdrel_type_uint32] = format_uint32; + cdrel_field_formatters[cdrel_type_int64] = format_int64; + cdrel_field_formatters[cdrel_type_uint64] = format_uint64; + cdrel_field_formatters[cdrel_type_timeval] = format_timeval; + cdrel_field_formatters[cdrel_type_float] = format_float; + cdrel_field_formatters[cdrel_type_amaflags] = format_amaflags; + cdrel_field_formatters[cdrel_type_disposition] = format_disposition; + + return 0; +} diff --git a/res/cdrel_custom/getters_cdr.c b/res/cdrel_custom/getters_cdr.c new file mode 100644 index 0000000000..45667db57e --- /dev/null +++ b/res/cdrel_custom/getters_cdr.c @@ -0,0 +1,116 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief CDR Getters + * + */ + +#include "cdrel.h" +#include "asterisk/cdr.h" + +static int cdr_get_string(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_cdr *cdr = record; + value->data_type = field->input_data_type; + value->field_name = field->name; + value->values.string = CDR_FIELD(cdr, field->field_id); + return 0; +} + +#define DEFINE_CDR_GETTER(_sname, _ename, _type) \ +static int cdr_get_ ## _ename (void *record, struct cdrel_config *config, \ + struct cdrel_field *field, struct cdrel_value *value) \ +{ \ + struct ast_cdr *cdr = record; \ + value->data_type = field->input_data_type; \ + value->field_name = field->name; \ + value->values._sname = *(_type *)CDR_FIELD(cdr, field->field_id); \ + return 0; \ +}\ + +DEFINE_CDR_GETTER(int32, int32, int32_t) +DEFINE_CDR_GETTER(uint32, uint32, uint32_t) +DEFINE_CDR_GETTER(int64, int64, int64_t) +DEFINE_CDR_GETTER(uint64, uint64, uint64_t) +DEFINE_CDR_GETTER(tv, timeval, struct timeval) +DEFINE_CDR_GETTER(floater, float, float) + +static int cdr_get_literal(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + value->data_type = cdrel_type_string; + value->field_name = field->name; + value->values.string = field->data; + return 0; +} + +static int cdr_get_uservar(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_cdr *cdr = record; + struct ast_var_t *variables; + const char *rtn = NULL; + + value->data_type = cdrel_type_string; + value->field_name = field->name; + AST_LIST_TRAVERSE(&cdr->varshead, variables, entries) { + if (strcasecmp(field->data, ast_var_name(variables)) == 0) { + rtn = ast_var_value(variables); + } + } + + value->values.string = (char *)S_OR(rtn, ""); + return 0; +} + +static struct ast_channel *dummy_chan_alloc_cdr(struct cdrel_config *config, void *data) +{ + struct ast_cdr *cdr = data; + struct ast_channel *dummy = NULL; + + dummy = ast_dummy_channel_alloc(); + if (!dummy) { + ast_log(LOG_ERROR, "Unable to fabricate channel from CDR for '%s'\n", + config->output_filename); + return NULL; + } + ast_channel_cdr_set(dummy, ast_cdr_dup(cdr)); + return dummy; +} + +int load_cdr(void) +{ + ast_debug(1, "Loading CDR getters\n"); + cdrel_field_getters[cdrel_record_cdr][cdrel_type_string] = cdr_get_string; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_literal] = cdr_get_literal; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_int32] = cdr_get_int32; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_uint32] = cdr_get_uint32; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_int64] = cdr_get_int64; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_uint64] = cdr_get_uint64; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_timeval] = cdr_get_timeval; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_float] = cdr_get_float; + cdrel_field_getters[cdrel_record_cdr][cdrel_type_uservar] = cdr_get_uservar; + cdrel_dummy_channel_allocators[cdrel_record_cdr] = dummy_chan_alloc_cdr; + + return 0; +} diff --git a/res/cdrel_custom/getters_cel.c b/res/cdrel_custom/getters_cel.c new file mode 100644 index 0000000000..d95b6f17a5 --- /dev/null +++ b/res/cdrel_custom/getters_cel.c @@ -0,0 +1,116 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief CEL Getters + * + */ + +#include "cdrel.h" +#include "asterisk/cel.h" + +static int cel_get_string(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_event *event = record; + value->data_type = cdrel_type_string; + value->field_name = field->name; + value->values.string = (char *)ast_event_get_ie_str(event, field->field_id); + return 0; +} + +static int cel_get_literal(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + value->data_type = cdrel_type_string; + value->field_name = field->name; + value->values.string = field->data; + return 0; +} + +static int cel_get_timeval(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_event *event = record; + value->data_type = cdrel_type_timeval; + value->field_name = field->name; + value->values.tv.tv_sec = ast_event_get_ie_uint(event, AST_EVENT_IE_CEL_EVENT_TIME); + value->values.tv.tv_usec = ast_event_get_ie_uint(event, AST_EVENT_IE_CEL_EVENT_TIME_USEC); + return 0; +} + +static int cel_get_uint32(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_event *event = record; + value->data_type = cdrel_type_uint32; + value->field_name = field->name; + value->values.uint32 = ast_event_get_ie_uint(event, field->field_id); + return 0; +} + +static int cel_get_event_type(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_event *event = record; + const char *val = NULL; + value->data_type = cdrel_type_string; + value->field_name = field->name; + + if (ast_event_get_ie_uint(event, AST_EVENT_IE_CEL_EVENT_TYPE) == AST_CEL_USER_DEFINED) { + val = ast_event_get_ie_str(event, AST_EVENT_IE_CEL_USEREVENT_NAME); + } else { + val = ast_cel_get_type_name(ast_event_get_ie_uint(event, AST_EVENT_IE_CEL_EVENT_TYPE)); + } + value->values.string = (char *)val; + return 0; +} + +static int cel_get_event_enum(void *record, struct cdrel_config *config, + struct cdrel_field *field, struct cdrel_value *value) +{ + struct ast_event *event = record; + value->data_type = cdrel_type_string; + value->field_name = field->name; + value->values.string = (char *)ast_cel_get_type_name(ast_event_get_ie_uint(event, AST_EVENT_IE_CEL_EVENT_TYPE)); + return 0; +} + +static struct ast_channel *dummy_chan_alloc_cel(struct cdrel_config *config, void *data) +{ + struct ast_event *event = data; + + return ast_cel_fabricate_channel_from_event(event); +} + +int load_cel(void) +{ + ast_debug(1, "Loading CEL getters\n"); + cdrel_field_getters[cdrel_record_cel][cdrel_type_string] = cel_get_string; + cdrel_field_getters[cdrel_record_cel][cdrel_type_literal] = cel_get_literal; + cdrel_field_getters[cdrel_record_cel][cdrel_type_uint32] = cel_get_uint32; + cdrel_field_getters[cdrel_record_cel][cdrel_type_timeval] = cel_get_timeval; + cdrel_field_getters[cdrel_record_cel][cdrel_type_event_type] = cel_get_event_type; + cdrel_field_getters[cdrel_record_cel][cdrel_type_event_enum] = cel_get_event_enum; + cdrel_dummy_channel_allocators[cdrel_record_cel] = dummy_chan_alloc_cel; + + return 0; +} diff --git a/res/cdrel_custom/loggers.c b/res/cdrel_custom/loggers.c new file mode 100644 index 0000000000..26f9da0262 --- /dev/null +++ b/res/cdrel_custom/loggers.c @@ -0,0 +1,307 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Common log entrypoint from the cdr/cel modules + * + */ + +#include "cdrel.h" +#include "asterisk/pbx.h" +#include "asterisk/vector.h" + +/* + * We can save some time and ast_str memory allocation work by allocating a single + * thread-local buffer and re-using it. + */ +AST_THREADSTORAGE(custom_buf); + +/*! + * \internal + * \brief Free an ast_value object + * + * ... and if the data type is "string", free it as well. + * + * \param data + */ +static void free_value(void *data) +{ + struct cdrel_value *val = data; + if (val->data_type == cdrel_type_string) { + ast_free(val->values.string); + val->values.string = NULL; + } + ast_free(val); +} + +/*! + * \internal + * \brief Free a vector of cdrel_values + * + * \param data + */ +static void free_value_vector(void *data) +{ + struct cdrel_values *values = data; + AST_VECTOR_RESET(values, free_value); + AST_VECTOR_PTR_FREE(values); +} + +/*! + * \internal + * \brief Log a legacy record to a file. + * + * The file legacy format specifies one long string with dialplan functions. We have no idea + * what the separator is so we need to pass the entire string to ast_str_substitute_variables. + * This is where the cycles are spent. We then write the result directly to the backend + * file bypassing all of the advanced processing. + * + * \param config The configuration object. + * \param data The data to write. May be an ast_cdr or an ast_event. + * \retval 0 on success + * \retval -1 on failure + */ +static int log_legacy_dsv_record(struct cdrel_config *config, void *data) +{ + struct ast_channel *dummy = data; + struct ast_str *str; + + if (!(str = ast_str_thread_get(&custom_buf, 1024))) { + return -1; + } + ast_str_reset(str); + + ast_str_substitute_variables(&str, 0, dummy, config->template); + + return write_record_to_file(config, str); +} + +/*! + * \internal + * \brief Log a legacy record to a database. + * + * Unlike the file backends, the legacy database backend configs always use commas + * as field separators but they all still use dialplan functions so we need still + * need to do evaluation and substitution. Since we know the separator however, + * we can iterate over the individual fields. + * + * \param config The configuration object. + * \param data The data to write. May be an ast_cdr or an ast_event. + * \retval 0 on success + * \retval -1 on failure + */ +static int log_legacy_database_record(struct cdrel_config *config, void *data) +{ + struct ast_channel *dummy = data; + int ix = 0; + int res = 0; + char subst_buf[2048]; + size_t field_count = AST_VECTOR_SIZE(&config->fields); + RAII_VAR(struct cdrel_values *, values, ast_calloc(1, sizeof(*values)), free_value_vector); + + if (!values) { + return -1; + } + + res = AST_VECTOR_INIT(values, field_count); + if (res != 0) { + return -1; + } + + if (config->db == NULL) { + return -1; + } + + for (ix = 0; ix < AST_VECTOR_SIZE(&config->fields); ix++) { + struct cdrel_field *field = AST_VECTOR_GET(&config->fields, ix); + struct cdrel_value *output_value = ast_calloc(1, sizeof(*output_value)); + + if (!output_value) { + return -1; + } + output_value->mallocd = 1; + + pbx_substitute_variables_helper(dummy, field->data, subst_buf, sizeof(subst_buf) - 1); + output_value->data_type = cdrel_type_string; + + output_value->field_name = field->name; + output_value->values.string = ast_strdup(ast_strip_quoted(subst_buf, "'\"", "'\"")); + if (!output_value->values.string) { + return -1; + } + + res = AST_VECTOR_APPEND(values, output_value); + if (res != 0) { + ast_free(output_value); + return -1; + } + } + + return write_record_to_database(config, values); +} + +/*! + * \internal + * \brief Log an advanced record + * + * For the file advanced formats, we know what the field separator is so we + * iterate over them and accumulate the results in a vector of cdrel_values. + * No dialplan function evaluation needed. + * + * \param config The configuration object. + * \param data The data to log. May be an ast_cdr or an ast_event. + * \retval 0 on success + * \retval -1 on failure + */ +static int log_advanced_record(struct cdrel_config *config, void *data) +{ + int ix = 0; + int res = 0; + size_t field_count = AST_VECTOR_SIZE(&config->fields); + RAII_VAR(struct cdrel_values *, values, ast_calloc(1, sizeof(*values)), free_value_vector); + + if (!values) { + return -1; + } + + res = AST_VECTOR_INIT(values, field_count); + if (res != 0) { + return -1; + } + + for (ix = 0; ix < AST_VECTOR_SIZE(&config->fields); ix++) { + struct cdrel_field *field = AST_VECTOR_GET(&config->fields, ix); + struct cdrel_value input_value = { 0, }; + struct cdrel_value *output_value = ast_calloc(1, sizeof(*output_value)); + + if (!output_value) { + return -1; + } + output_value->mallocd = 1; + + /* + * Get a field from a CDR structure or CEL event into an cdrel_value. + */ + res = cdrel_field_getters[config->record_type][field->input_data_type](data, config, field, &input_value); + if (res != 0) { + ast_free(output_value); + return -1; + } + + /* + * Set the output data type to the type we want to see in the output. + */ + output_value->data_type = field->output_data_type; + + /* + * Now call the formatter based on the INPUT data type. + */ + res = cdrel_field_formatters[input_value.data_type](config, field, &input_value, output_value); + if (res != 0) { + ast_free(output_value); + return -1; + } + + res = AST_VECTOR_APPEND(values, output_value); + if (res != 0) { + ast_free(output_value); + return -1; + } + } + return cdrel_backend_writers[config->format_type](config, values); +} + +/* + * These callbacks are only used in this file so there's no need to + * make them available to the rest of the module. + */ +typedef int (*cdrel_logger_cb)(struct cdrel_config *config, void *data); + +static const cdrel_logger_cb logger_callbacks[cdrel_backend_type_end][cdrel_config_type_end] = { + [cdrel_backend_text] = { + [cdrel_config_legacy] = log_legacy_dsv_record, + [cdrel_config_advanced] = log_advanced_record + }, + [cdrel_backend_db] = { + [cdrel_config_legacy] = log_legacy_database_record, + [cdrel_config_advanced] = log_advanced_record + }, +}; + +/*! + * \internal + * \brief Main logging entrypoint from the individual modules. + * + * This is the entrypoint from the individual cdr and cel modules. + * "data" will either be an ast_cdr or ast_event structure but we + * don't actually care at this point. + * + * For legacy configs, we need to create a dummy channel so we'll + * do that if/when we hit the first one and we'll reuse it for all + * further legacy configs. If we fail to get a channel, we'll skip + * all further configs. + * + * \warning This function MUST be called with the module's config_lock + * held for reading to prevent reloads from happening while we're logging. + * + * \param configs The calling module's vector of configuration objects. + * \param data The data to write. May be an ast_cdr or an ast_event. + * \retval 0 on success + * \retval <0 on failure. The magnitude indicates how many configs failed. + */ +int cdrel_logger(struct cdrel_configs *configs, void *data) +{ + struct ast_channel *dummy = NULL; + int ix = 0; + int skip_legacy = 0; + int res = 0; + + for(ix = 0; ix < AST_VECTOR_SIZE(configs); ix++) { + struct cdrel_config *config = AST_VECTOR_GET(configs, ix); + void *chan_or_data = NULL; + + if (config->config_type == cdrel_config_legacy) { + if (skip_legacy) { + continue; + } + if (!dummy) { + dummy = config->dummy_channel_alloc(config, data); + if (!dummy) { + ast_log(LOG_ERROR, "Unable to fabricate channel from CEL event for '%s'\n", + config->output_filename); + skip_legacy = 1; + res--; + continue; + } + } + chan_or_data = dummy; + } else { + chan_or_data = data; + } + res += logger_callbacks[config->backend_type][config->config_type](config, chan_or_data); + } + + if (dummy) { + ast_channel_unref(dummy); + } + return res; +} + diff --git a/res/cdrel_custom/registry.c b/res/cdrel_custom/registry.c new file mode 100644 index 0000000000..472b147c45 --- /dev/null +++ b/res/cdrel_custom/registry.c @@ -0,0 +1,123 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief CDR/CEL field registry + * + */ + +#include "cdrel.h" + +#include "asterisk/json.h" +#include "asterisk/cdr.h" +#include "asterisk/cel.h" + +/*! + * \internal + * \brief Helper macro that populates the static array of cdrel_fields. + * + * \param _record_type The type of record: CDR or CEL. + * \param _field_id For CEL, it's the event field. For CDR it's one of cdr_field_id. + * \param _name The field name. + * \param _input_type The input data type. Drives the getters. + * \param output_types An array of types, one each for dsv, json and sql. Drives the formatters. + * \param _mallocd Not used. + */ +#define REGISTER_FIELD(_record_type, _field_id, _name, _input_type, _output_type) \ + { _record_type, _field_id, _name, _input_type, _output_type, { 0 } } + +static const struct cdrel_field cdrel_field_registry[] = { + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_EVENT_ENUM, "eventenum", cdrel_type_event_enum, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_EVENT_TYPE, "eventtype", cdrel_type_event_type, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_EVENT_TIME, "eventtime", cdrel_type_timeval, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_EVENT_TIME_USEC, "eventtimeusec", cdrel_type_uint32, cdrel_type_uint32), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_USEREVENT_NAME, "usereventname", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_USEREVENT_NAME, "userdeftype", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CIDNAME, "name", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CIDNUM, "num", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_EXTEN, "exten", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CONTEXT, "context", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CHANNAME, "channame", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_APPNAME, "appname", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_APPDATA, "appdata", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_AMAFLAGS, "amaflags", cdrel_type_uint32, cdrel_type_uint32), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_ACCTCODE, "accountcode", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_UNIQUEID, "uniqueid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_USERFIELD, "userfield", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CIDANI, "ani", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CIDRDNIS, "rdnis", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_CIDDNID, "dnid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_PEER, "peer", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_PEER, "bridgepeer", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_LINKEDID, "linkedid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_PEERACCT, "peeraccount", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_EXTRA, "eventextra", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_TENANTID, "tenantid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cel, AST_EVENT_IE_CEL_LITERAL, "literal", cdrel_type_literal, cdrel_type_string), + + REGISTER_FIELD(cdrel_record_cdr, cdr_field_clid, "clid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_src, "src", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_dst, "dst", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_dcontext, "dcontext", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_channel, "channel", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_dstchannel, "dstchannel", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_lastapp, "lastapp", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_lastdata, "lastdata", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_start, "start", cdrel_type_timeval, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_answer, "answer", cdrel_type_timeval, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_end, "end", cdrel_type_timeval, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_duration, "duration", cdrel_type_int64, cdrel_type_int64), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_billsec, "billsec", cdrel_type_int64, cdrel_type_int64), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_disposition, "disposition", cdrel_type_int64, cdrel_type_disposition), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_amaflags, "amaflags", cdrel_type_int64, cdrel_type_amaflags), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_accountcode, "accountcode", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_peeraccount, "peeraccount", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_flags, "flags", cdrel_type_uint32, cdrel_type_uint32), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_uniqueid, "uniqueid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_linkedid, "linkedid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_tenantid, "tenantid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_peertenantid, "peertenantid", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_userfield, "userfield", cdrel_type_string, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_sequence, "sequence", cdrel_type_int32, cdrel_type_int32), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_varshead, "uservar", cdrel_type_uservar, cdrel_type_string), + REGISTER_FIELD(cdrel_record_cdr, cdr_field_literal, "literal", cdrel_type_literal, cdrel_type_string), +}; + +/* +* \internal +* \brief Get a cdrel_field structure by record type and field name. +* +* \param record_type The cdrel_record_type to search. +* \param name The field name to search for. +* \returns A pointer to a constant cdrel_field structure or NULL if not found. +* This pointer must never be freed. +*/ +const struct cdrel_field *get_registered_field_by_name(enum cdrel_record_type type, const char *name) +{ + int ix = 0; + + for (ix = 0; ix < ARRAY_LEN(cdrel_field_registry); ix++) { + if (cdrel_field_registry[ix].record_type == type && strcasecmp(cdrel_field_registry[ix].name, name) == 0) { + return &cdrel_field_registry[ix]; + } + } + return NULL; +} diff --git a/res/cdrel_custom/writers.c b/res/cdrel_custom/writers.c new file mode 100644 index 0000000000..975302c042 --- /dev/null +++ b/res/cdrel_custom/writers.c @@ -0,0 +1,230 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Backend output functions. + * + * The writers all take a vector of cdrel_value objects and write them to the output + * file or database. + * + */ + +#include "cdrel.h" + +/* + * We can save some time and ast_str memory allocation work by allocating a single + * thread-local buffer and re-using it. + */ +AST_THREADSTORAGE(custom_buf); + +/*! + * \internal + * \brief Write a record to a text file + * + * Besides being used here, this function is also used by the legacy loggers + * that shortcut the advanced stuff. + * + * \param config The configuration object. + * \param record The data to write. + * \retval 0 on success + * \retval -1 on failure + */ +int write_record_to_file(struct cdrel_config *config, struct ast_str *record) +{ + FILE *out; + int res = 0; + + ast_mutex_lock(&config->lock); + if ((out = fopen(config->output_filename, "a"))) { + res = fputs(ast_str_buffer(record), out); + fputs("\n", out); + fclose(out); + } + ast_mutex_unlock(&config->lock); + + if (!out || res < 0) { + ast_log(LOG_ERROR, "Unable to write %s to file %s : %s\n", + RECORD_TYPE_STR(config->record_type), config->output_filename, strerror(errno)); + return -1; + } + return 0; +} + +/*! + * \internal + * \brief Concatenate and append a list of values to an ast_str + * + * \param config Config object + * \param values A vector of values + * \param str The ast_str to append to + * \retval 0 on success + * \retval -1 on failure + */ +static int dsv_appender(struct cdrel_config *config, struct cdrel_values *values, struct ast_str **str) +{ + int ix = 0; + int res = 0; + + for (ix = 0; ix < AST_VECTOR_SIZE(values); ix++) { + struct cdrel_value *value = AST_VECTOR_GET(values, ix); + ast_assert(value->data_type == cdrel_type_string); + + res = ast_str_append(str, -1, "%s%s", ix == 0 ? "" : config->separator, value->values.string); + if (res < 0) { + return -1; + } + } + + return res; +} + +/*! + * \internal + * \brief Write a DSV list of values to a text file + * + * \param config Config object + * \param values A vector of values + * \retval 0 on success + * \retval -1 on failure + */ +static int dsv_writer(struct cdrel_config *config, struct cdrel_values *values) +{ + int res = 0; + struct ast_str *str = ast_str_thread_get(&custom_buf, 1024); + + ast_str_reset(str); + + res = dsv_appender(config, values, &str); + if (res < 0) { + return -1; + } + + return write_record_to_file(config, str); +} + +/*! + * \internal + * \brief Write a list of values as a JSON object to a text file + * + * \param config Config object + * \param values A vector of values + * \retval 0 on success + * \retval -1 on failure + * + * \note We are intentionally NOT using the ast_json APIs here because + * they're expensive and these are simple objects. + */ +static int json_writer(struct cdrel_config *config, struct cdrel_values *values) +{ + int ix = 0; + int res = 0; + struct ast_str *str = ast_str_thread_get(&custom_buf, 1024); + + ast_str_set(&str, -1, "%s", "{"); + + for (ix = 0; ix < AST_VECTOR_SIZE(values); ix++) { + struct cdrel_value *value = AST_VECTOR_GET(values, ix); + ast_assert(value->data_type == cdrel_type_string); + + res = ast_str_append(&str, -1, "%s\"%s\":%s", ix == 0 ? "" : config->separator, value->field_name, value->values.string); + if (res < 0) { + return -1; + } + } + ast_str_append(&str, -1, "%s", "}"); + + return write_record_to_file(config, str); +} + +/*! + * \internal + * \brief Write a record to a database + * + * Besides being used here, this function is also used by the legacy loggers + * that shortcut the advanced stuff. + * + * \param config The configuration object. + * \param record The data to write. + * \retval 0 on success + * \retval -1 on failure + */ +int write_record_to_database(struct cdrel_config *config, struct cdrel_values *values) +{ + int res = 0; + int ix; + + ast_mutex_lock(&config->lock); + for (ix = 0; ix < AST_VECTOR_SIZE(values); ix++) { + struct cdrel_value *value = AST_VECTOR_GET(values, ix); + ast_assert(value->data_type == cdrel_type_string); + ast_debug(6, "%s '%s'\n", value->field_name, value->values.string); + res = sqlite3_bind_text(config->insert, ix + 1, value->values.string, -1, SQLITE_STATIC); + if (res != SQLITE_OK) { + ast_log(LOG_ERROR, "Unable to write %s to database %s. SQL bind for field %s:'%s'. Error: %s\n", + RECORD_TYPE_STR(config->record_type), config->output_filename, + value->field_name, value->values.string, + sqlite3_errmsg(config->db)); + sqlite3_reset(config->insert); + ast_mutex_unlock(&config->lock); + return -1; + } + } + + res = sqlite3_step(config->insert); + if (res != SQLITE_DONE) { + ast_log(LOG_ERROR, "Unable to write %s to database %s. Error: %s\n", + RECORD_TYPE_STR(config->record_type), config->output_filename, + sqlite3_errmsg(config->db)); + sqlite3_reset(config->insert); + ast_mutex_unlock(&config->lock); + return -1; + } + + sqlite3_reset(config->insert); + ast_mutex_unlock(&config->lock); + + return res; +} + +/*! + * \internal + * \brief Write a list of values to a database + * + * \param config Config object + * \param values A vector of values + * \retval 0 on success + * \retval -1 on failure + */ +static int database_writer(struct cdrel_config *config, struct cdrel_values *values) +{ + return write_record_to_database(config, values); +} + +int load_writers(void) +{ + ast_debug(1, "Loading Writers\n"); + cdrel_backend_writers[cdrel_format_dsv] = dsv_writer; + cdrel_backend_writers[cdrel_format_json] = json_writer; + cdrel_backend_writers[cdrel_format_sql] = database_writer; + + return 0; +} + diff --git a/res/res_cdrel_custom.c b/res/res_cdrel_custom.c new file mode 100644 index 0000000000..179db87acc --- /dev/null +++ b/res/res_cdrel_custom.c @@ -0,0 +1,257 @@ +/* + * Asterisk -- An open source telephony toolkit. + * + * Copyright (C) 2026, Sangoma Technologies Corporation + * + * George Joseph + * + * See http://www.asterisk.org for more information about + * the Asterisk project. Please do not directly contact + * any of the maintainers of this project for assistance; + * the project provides a web site, mailing lists and IRC + * channels for your use. + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! + * \file + * \author George Joseph + * + * \brief Common logic for the CDR and CEL Custom Backends + * + * All source files are in the res/cdrel_custom directory. + * + * "config.c": Contains common configuration file parsing the ultimate goal + * of which is to create a vector of cdrel_config structures for each of + * the cdr_custom, cdr_sqlite3_custom, cel_custom and cel_sqlite3_custom + * modules. Each cdrel_config object represents an output file defined in + * their respective config files. Each one contains a vector of cdrel_field + * objects, one for each field in the output record, plus settings like + * the output file name, backend type (text file or database), config + * type ((legacy or advanced), the field separator and quote character to + * use. + * + * Each cdrel_field object contains an abstract field id that points to + * a ast_cdr structure member or CEL event field id along with an input + * type and an output type. The registry of cdrel_fields is located in + * registry.c. + * + * "loggers.c": Contains the common "cdrel_logger" entrypoint that the + * individual modules call to log a record. It takes the module's + * cdrel_configs vector and the record to log it got from the core + * cel.c and cdr.c. It then looks up and runs the logger implementation + * based on the backend type (text file or database) and config type + * (legacy or advanced). + * + * "getters_cdr.c", "getters_cel.c": Contain the getters that retrieve + * values from the ast_cdr or ast_event structures based on the field + * id and input type defined for that field and create a cdrel_value + * wrapper object for it. + * + * "writers.c": Contains common backend writers for the text file and + * database backends. + * + * The load-time flow... + * + * Each of the individual cdr/cel custom modules call the common + * cdrel_load_module function with their backend_type, record_type + * (cdr or cel), config file name, and the logging callback that + * should be registered with the core cdr or cel facility. + * + * cdrel_load_module calls the config load function appropriate for + * the backend type, each of which parses the config file and, if + * successful, registers the calling module with the cdr or cel core + * and creates a vector of cdrel_config objects that is passed + * back to the calling module. That vector contains the context for + * all future operations. + * + * The run-time flow... + * + * The core cdr and cel modules use their registries of backends and call + * the callback function registered by the 4 cdr and cel custom modules. + * No changes there. + * + * Each of those modules call the common cdrel_logger function with their + * cdrel_configs vector and the actual ast_cdr or ast_event structure to log. + * The cdrel_logger function iterates over the cdrel_configs vector and for + * each invokes the logger implementation specific to the backend type + * (text file or database) and config type (legacy or advanced). + * + * For legacy config types, the logger implementation simply calls + * ast_str_substitute_variables() on the whole opaque format and writes + * the result to the text file or database. + * + * For advanced configs, the logger implementation iterates over each field + * in the cdrel_config's fields vector and for each, calls the appropriate + * getter based on the record type (cdr or cel) and field id. Each getter + * call returns a cdrel_value object which is then passed to a field formatter + * looked up based on the field's data type (string, int32, etc). The formatter + * is also passed the cdrel_config object and the desired output type and + * returns the final value in another cdrel_value object formatted with any + * quoting, etc. needed. The logger accumulates the output cdrel_values + * (which are all now strings) in another vector and after all fields have + * been processed, hands the vector over to one of the backend writers. + * + * The backend writer concatenates the cdrel_values into an output record + * using the config's separator setting and writes it to the text file + * or database. For the JSON output format, it creates a simple + * name/value pair output record. + * + * The identification of field data types, field ids, record types and + * backend types is all done at config load time and saved in the + * cdrel_config and cdrel_field objects. The callbacks for getters, formatters + * and writers are also loaded when the res_cdrel_custom module loads + * and stored in arrays indexed by their enum values. The result is that + * at run time, simple array indexing is all that's needed to get the + * proper getter, formatter and writer for any logging request. + * + */ + +/*** MODULEINFO + sqlite3 + core + ***/ + + +#include "asterisk.h" + +#include "asterisk/module.h" +#include "cdrel_custom/cdrel.h" + +/* + * Populated by cdrel_custom/getters_cdr.c and cdrel_custom/getters_cel.c. + */ +cdrel_field_getter cdrel_field_getters[cdrel_record_type_end][cdrel_data_type_end]; + +/* + * Populated by cdrel_custom/formatters.c. + */ +cdrel_field_formatter cdrel_field_formatters[cdrel_data_type_end]; + +/* + * Populated by cdrel_custom/writers.c. + */ +cdrel_backend_writer cdrel_backend_writers[cdrel_format_type_end]; + +/* + * Populated by cdrel_custom/getters_cdr.c and cdrel_custom/getters_cel.c. + */ +cdrel_dummy_channel_alloc cdrel_dummy_channel_allocators[cdrel_format_type_end]; + +/* + * You must ensure that there's an entry for every value in the enum. + */ + +const char *cdrel_record_type_map[] = { + [cdrel_record_cdr] = "CDR", + [cdrel_record_cel] = "CEL", + [cdrel_record_type_end] = "!!END!!", +}; + +const char *cdrel_module_type_map[] = { + [cdrel_backend_text] = "Custom ", + [cdrel_backend_db] = "SQLITE3 Custom", + [cdrel_backend_type_end] = "!!END!!", +}; + +const char *cdrel_data_type_map[] = { + [cdrel_type_string] = "string", + [cdrel_type_timeval] = "timeval", + [cdrel_type_literal] = "literal", + [cdrel_type_amaflags] = "amaflags", + [cdrel_type_disposition] = "disposition", + [cdrel_type_uservar] = "uservar", + [cdrel_type_event_type] = "event_type", + [cdrel_type_event_enum] = "event_enum", + [cdrel_data_type_strings_end] = "!!STRINGS END!!", + [cdrel_type_int32] = "int32", + [cdrel_type_uint32] = "uint32", + [cdrel_type_int64] = "int64", + [cdrel_type_uint64] = "uint64", + [cdrel_type_float] = "float", + [cdrel_data_type_end] = "!!END!!", +}; + +enum cdrel_data_type cdrel_data_type_from_str(const char *str) +{ + enum cdrel_data_type data_type = 0; + for (data_type = 0; data_type < cdrel_data_type_end; data_type++) { + if (strcasecmp(cdrel_data_type_map[data_type], str) == 0) { + return data_type; + } + + } + return cdrel_data_type_end; +} + +static const char *cdrel_field_flags_map[] = { + [CDREL_FIELD_FLAG_QUOTE] = "quote", + [CDREL_FIELD_FLAG_NOQUOTE] = "noquote", + [CDREL_FIELD_FLAG_TYPE_FORCED] = "type_forced", + [CDREL_FIELD_FLAG_USERVAR] = "uservar", + [CDREL_FIELD_FLAG_LITERAL] = "literal", + [CDREL_FIELD_FLAG_FORMAT_SPEC] = "format_spec", + [CDREL_FIELD_FLAG_LAST] = "LAST", +}; + +const char *cdrel_get_field_flags(struct ast_flags *flags, struct ast_str **str) +{ + int ix = 0; + int res = 0; + int trues = 0; + + for (ix = 0; ix < CDREL_FIELD_FLAG_LAST; ix++) { + if (ast_test_flag(flags, (1 << ix))) { + res = ast_str_append(str, -1, "%s%s", trues++ ? "," : "", cdrel_field_flags_map[ix]); + if (res < 0) { + return ""; + } + } + } + return ast_str_buffer(*str); +} + + +const char *cdrel_basename(const char *path) +{ + int i = 0; + const char *basename = path; + + if (ast_strlen_zero(path)) { + return path; + } + i = strlen(path) - 1; + while(i >= 0) { + if (path[i] == '/') { + basename = &path[i + 1]; + break; + } + i--; + } + return basename; +} + +static int unload_module(void) +{ + return 0; +} + +static enum ast_module_load_result load_module(void) +{ + load_cdr(); + load_cel(); + load_formatters(); + load_writers(); + return AST_MODULE_LOAD_SUCCESS; +} + +AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_GLOBAL_SYMBOLS | AST_MODFLAG_LOAD_ORDER, + "Combined logic for CDR/CEL Custom modules", + .support_level = AST_MODULE_SUPPORT_CORE, + .load = load_module, + .unload = unload_module, + .load_pri = AST_MODPRI_CDR_DRIVER, +); diff --git a/res/res_cdrel_custom.exports.in b/res/res_cdrel_custom.exports.in new file mode 100644 index 0000000000..39c45954df --- /dev/null +++ b/res/res_cdrel_custom.exports.in @@ -0,0 +1,6 @@ +{ + global: + LINKER_SYMBOL_PREFIX*cdrel_*; + local: + *; +};