Add ability to pass arguments to unit tests from the CLI

Unit tests can now be passed custom arguments from the command
line.  For example, the following command would run the "mytest" test
in the "/main/mycat" category with the option "myoption=54"

`CLI> test execute category /main/mycat name mytest options myoption=54`

You can also pass options to an entire category...

`CLI> test execute category /main/mycat options myoption=54`

Basically, everything after the "options" keyword is passed verbatim to
the test which must decide what to do with it.

* A new API ast_test_get_cli_args() was created to give the tests access to
the cli_args->argc and cli_args->argv elements.

* Although not needed for the option processing, a new macro
ast_test_validate_cleanup_custom() was added to test.h that allows you
to specify a custom error message instead of just "Condition failed".

* The test_skel.c was updated to demonstrate parsing options and the use
of the ast_test_validate_cleanup_custom() macro.

include/asterisk/test.h

@@ -438,6 +438,30 @@ int __ast_test_status_update(const char *file, const char *func, int line, struc
 	} \
 })
 
+/*!
+ * \brief Check a test condition and if false, report custom error message and goto cleanup label.
+ *
+ * This macro evaluates \a condition. If the condition evaluates to true (non-zero),
+ * nothing happens. If it evaluates to false (zero), then the message provided
+ * is printed using \ref ast_test_status_update, the variable \a rc_variable is set
+ * to AST_TEST_FAIL, and a goto to \a cleanup_label is executed.
+ *
+ * \param test Currently executing test
+ * \param condition Boolean condition to check.
+ * \param rc_variable Variable to receive AST_TEST_FAIL.
+ * \param cleanup_label The label to go to on failure.
+ * \param fmt printf type format string
+ * \param ... printf arguments
+ */
+#define ast_test_validate_cleanup_custom(test, condition, rc_variable, cleanup_label, fmt, ...) \
+({ \
+	if (!(condition)) { \
+		__ast_test_status_update(__FILE__, __PRETTY_FUNCTION__, __LINE__, (test), fmt, ## __VA_ARGS__); \
+		rc_variable = AST_TEST_FAIL; \
+		goto cleanup_label; \
+	} \
+})
+
 /*!
  * \brief Initialize the capture structure.
  *
@@ -483,5 +507,14 @@ void ast_test_capture_free(struct ast_test_capture *capture);
 
 int ast_test_capture_command(struct ast_test_capture *capture, const char *file, char *const argv[], const char *data, unsigned datalen);
 
+/*!
+ * \brief Retrieve the cli arguments from the ast_test structure
+ *
+ * \param test Currently executing test
+ *
+ * \retval A pointer to the ast_cli_args structure used to invoke the test
+ */
+struct ast_cli_args *ast_test_get_cli_args(struct ast_test *test);
+
 #endif /* TEST_FRAMEWORK */
 #endif /* _AST_TEST_H */

main/test.c

@@ -1013,6 +1013,11 @@ static struct ast_test *test_alloc(ast_test_cb_t *cb)
 	return test;
 }
 
+struct ast_cli_args *ast_test_get_cli_args(struct ast_test *test)
+{
+	return test->cli;
+}
+
 static char *complete_test_category(const char *word)
 {
 	int wordlen = strlen(word);
@@ -1115,18 +1120,24 @@ static char *test_cli_show_registered(struct ast_cli_entry *e, int cmd, struct a
 static char *test_cli_execute_registered(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
 	static const char * const option1[] = { "all", "category", NULL };
-	static const char * const option2[] = { "name", NULL };
+	static const char * const option2[] = { "name", "options", NULL };
+	static const char * const option3[] = { "options", NULL };
 
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "test execute";
 		e->usage =
-			"Usage: test execute can be used in three ways.\n"
+			"Usage: test execute can be used in several ways.\n"
 			"       1. 'test execute all' runs all registered tests\n"
 			"       2. 'test execute category [test category]' runs all tests in the given\n"
 			"          category.\n"
-			"       3. 'test execute category [test category] name [test name]' runs all\n"
+			"       3. 'test execute category [test category] options [test option]...' runs all\n"
-			"           tests in a given category matching a given name\n";
+			"           tests in the given category with options supplied to each test\n"
+			"       4. 'test execute category [test category] name [test name]' runs all\n"
+			"           tests in a given category matching a given name\n"
+			"       5. 'test execute category [test category] name [test name] options [test option]...' runs all\n"
+			"           tests in a given category matching a given name with the specified options\n"
+			;
 		return NULL;
 	case CLI_GENERATE:
 		if (a->pos == 2) {
@@ -1138,13 +1149,19 @@ static char *test_cli_execute_registered(struct ast_cli_entry *e, int cmd, struc
 		if (a->pos == 4) {
 			return ast_cli_complete(a->word, option2, -1);
 		}
-		if (a->pos == 5) {
+		if (a->pos == 5 && !strcasecmp(a->argv[4], "name")) {
 			return complete_test_name(a->word, a->argv[3]);
 		}
+		if (a->pos == 5 && !strcasecmp(a->argv[4], "options")) {
+			return NULL;
+		}
+		if (a->pos == 6 && !strcasecmp(a->argv[4], "name")) {
+			return ast_cli_complete(a->word, option3, -1);
+		}
 		return NULL;
 	case CLI_HANDLER:
 
-		if (a->argc < 3|| a->argc > 6) {
+		if (a->argc < 3) {
 			return CLI_SHOWUSAGE;
 		}
 
@@ -1154,9 +1171,15 @@ static char *test_cli_execute_registered(struct ast_cli_entry *e, int cmd, struc
 		} else if (a->argc == 4) { /* run only tests within a category */
 			ast_cli(a->fd, "Running all available tests matching category %s\n\n", a->argv[3]);
 			test_execute_multiple(NULL, a->argv[3], a);
-		} else if (a->argc == 6) { /* run only a single test matching the category and name */
+		} else if (a->argc >= 6 && !strcasecmp(a->argv[4], "options")) { /* run only tests within a category */
+			ast_cli(a->fd, "Running all available tests matching category %s with options\n\n", a->argv[3]);
+			test_execute_multiple(NULL, a->argv[3], a);
+		} else if (a->argc == 6 && !strcasecmp(a->argv[4], "name")) { /* run only a single test matching the category and name */
 			ast_cli(a->fd, "Running all available tests matching category %s and name %s\n\n", a->argv[3], a->argv[5]);
 			test_execute_multiple(a->argv[5], a->argv[3], a);
+		} else if (a->argc > 7) { /* run only a single test matching the category and name */
+			ast_cli(a->fd, "Running all available tests matching category %s and name %s with options\n\n", a->argv[3], a->argv[5]);
+			test_execute_multiple(a->argv[5], a->argv[3], a);
 		} else {
 			return CLI_SHOWUSAGE;
 		}

tests/test_skel.c

@@ -39,7 +39,15 @@
 
 AST_TEST_DEFINE(sample_test)
 {
-	void *ptr;
+	/* Retrieve the command line arguments used to invoke the test */
+	struct ast_cli_args *cli_args = ast_test_get_cli_args(test);
+	/* Set default values for the options */
+	int test_option = 999;
+	char test_option2[128] = { 0 };
+	void *ptr = NULL;
+	void *ptr2 = NULL;
+	int i;
+	enum ast_test_result_state rc = AST_TEST_PASS;
 
 	switch (cmd) {
 	case TEST_INIT:
@@ -48,22 +56,73 @@ AST_TEST_DEFINE(sample_test)
 		info->summary = "sample unit test";
 		info->description =
 			"This demonstrates what is required to implement "
-			"a unit test.";
+			"a unit test.  You can pass in test-option and "
+			"test-option2 as command line arguments to this "
+			"test.  test-option is an integer and test-option2 "
+			"is a string.";
 		return AST_TEST_NOT_RUN;
 	case TEST_EXECUTE:
 		break;
 	}
 
-	ast_test_status_update(test, "Executing sample test...\n");
+	/*
+	 * This is an example of how to get command line arguments
+	 * from the test framework.  The arguments are "test-option"
+	 * (expected to be an integer) and "test-option2" (expected
+	 * to be a string).
+	 *
+	 * NOTES:
+	 *
+	 * cli_args will contain all of the command line arguments
+	 * including "test execute", etc. so the location of the options
+	 * will vary depending on how the test was invoked.
+	 * For instance, this test could be run by either of the following:
+	 *
+	 * test execute category /main/sample/ options test-option=444
+	 * test execute category /main/sample/ name sample_test options test-option=444
+	 *
+	 * You therefore need to test each of the items in the argv array
+	 * to find the ones you are looking for.
+	 *
+	 * No special processing is done on string arguments so if your
+	 * option value is a string, you must deal with the possibility
+	 * of embedded spaces yourself.
+	 */
+
+	for (i = 0; i < cli_args->argc; i++) {
+		ast_test_status_update(test, "Test argument: %d: %s\n", i, cli_args->argv[i]);
+		if (ast_begins_with(cli_args->argv[i], "test-option=")) {
+			sscanf(cli_args->argv[i], "test-option=%d", &test_option);
+		}
+		if (ast_begins_with(cli_args->argv[i], "test-option2=")) {
+			sscanf(cli_args->argv[i], "test-option2=%s", test_option2);
+		}
+	}
+
+	ast_test_status_update(test, "Executing sample test with test-option=%d and test-option2=%s\n",
+		test_option, test_option2);
 
 	if (!(ptr = ast_malloc(8))) {
 		ast_test_status_update(test, "ast_malloc() failed\n");
 		return AST_TEST_FAIL;
 	}
 
+	ptr2 = ast_malloc(8);
+	/*
+	 * This is an example of how to use the ast_test_validate_cleanup_custom
+	 * macro to check a condition and cleanup if it fails.
+	 * If ptr2 is NULL, rc will be set to AST_TEST_FAIL, the specified
+	 * message will be printed, and the test will jump to the "done"
+	 * label to perform cleanup.
+	 */
+	ast_test_validate_cleanup_custom(test, ptr2, rc, done, "ptr2 is NULL\n");
+
+done:
+
 	ast_free(ptr);
+	ast_free(ptr2);
 
-	return AST_TEST_PASS;
+	return rc;
 }
 
 static int unload_module(void)