HTTP: Stop accepting requests on final system shutdown.

There are three CLI commands to stop and restart Asterisk each.

1) core stop/restart now - Hangup all calls and stop or restart Asterisk.
New channels are prevented while the shutdown request is pending.

2) core stop/restart gracefully - Stop or restart Asterisk when there are
no calls remaining in the system.  New channels are prevented while the
shutdown request is pending.

3) core stop/restart when convenient - Stop or restart Asterisk when there
are no calls in the system.  New calls are not prevented while the
shutdown request is pending.

ARI has made stopping/restarting Asterisk more problematic.  While a
shutdown request is pending it is desirable to continue to process ARI
HTTP requests for current calls.  To handle the current calls while a
shutdown request is pending, a new committed to shutdown phase is needed
so ARI applications can deal with the calls until the system is fully
committed to shutdown.

* Added a new shutdown committed phase so ARI applications can deal with
calls until the final committed to shutdown phase is reached.

* Made refuse new HTTP requests when the system has reached the final
system shutdown phase.  Starting anything while the system is actively
releasing resources and unloading modules is not a good thing.

* Split the bridging framework shutdown to not cleanup the global bridging
containers when shutting down in a hurry.  This is similar to how other
modules prevent crashes on rapid system shutdown.

* Moved ast_begin_shutdown(), ast_cancel_shutdown(), and
ast_shutting_down().  You should not have to include channel.h just to
access these system functions.

ASTERISK-24752 #close
Reported by: Matthew Jordan

Review: https://reviewboard.asterisk.org/r/4399/
........

Merged revisions 431692 from http://svn.asterisk.org/svn/asterisk/branches/13


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@431694 65c4cc65-6c06-0410-ace0-fbb531ad65f3

apps/app_confbridge.c

@@ -1757,6 +1757,12 @@ static int confbridge_exec(struct ast_channel *chan, const char *data)
 
 	/* if we're shutting down, don't attempt to do further processing */
 	if (ast_shutting_down()) {
+		/*
+		 * Not taking any new calls at this time.  We cannot create
+		 * the announcer channel if this is the first channel into
+		 * the conference and we certainly cannot create any
+		 * recording channel.
+		 */
 		leave_conference(&user);
 		conference = NULL;
 		goto confbridge_cleanup;

channels/chan_sip.c

@@ -24854,6 +24854,10 @@ static int handle_request_options(struct sip_pvt *p, struct sip_request *req, st
 		ast_string_field_set(p, context, sip_cfg.default_context);
 
 	if (ast_shutting_down()) {
+		/*
+		 * Not taking any new calls at this time.
+		 * Likely a server availability OPTIONS poll.
+		 */
 		msg = "503 Unavailable";
 	} else {
 		msg = "404 Not Found";

include/asterisk.h

@@ -111,6 +111,42 @@ int ast_register_cleanup(void (*func)(void));
  */
 void ast_unregister_atexit(void (*func)(void));
 
+/*!
+ * \brief Cancel an existing shutdown and return to normal operation.
+ *
+ * \note Shutdown can be cancelled while the server is waiting for
+ * any existing channels to be destroyed before shutdown becomes
+ * irreversible.
+ *
+ * \return non-zero if shutdown cancelled.
+ */
+int ast_cancel_shutdown(void);
+
+/*!
+ * \details
+ * The server is preventing new channel creation in preparation for
+ * shutdown and may actively be releasing resources.  The shutdown
+ * process may be canceled by ast_cancel_shutdown() if it is not too
+ * late.
+ *
+ * \note The preparation to shutdown phase can be quite lengthy
+ * if we are gracefully shutting down.  How long existing calls will
+ * last is not up to us.
+ *
+ * \return non-zero if the server is preparing to or actively shutting down.
+ */
+int ast_shutting_down(void);
+
+/*!
+ * \return non-zero if the server is actively shutting down.
+ * \since 13.3.0
+ *
+ * \details
+ * The server is releasing resources and unloading modules.
+ * It won't be long now.
+ */
+int ast_shutdown_final(void);
+
 #if !defined(LOW_MEMORY)
 /*!
  * \brief Register the version of a source code file with the core.

include/asterisk/channel.h

@@ -1510,6 +1510,14 @@ const struct ast_channel_tech *ast_get_channel_tech(const char *name);
  */
 void ast_hangup(struct ast_channel *chan);
 
+/*!
+ * \brief Soft hangup all active channels.
+ * \since 13.3.0
+ *
+ * \return Nothing
+ */
+void ast_softhangup_all(void);
+
 /*!
  * \brief Softly hangup up a channel
  *
@@ -2203,23 +2211,12 @@ int ast_channel_defer_dtmf(struct ast_channel *chan);
 /*! Undo defer.  ast_read will return any DTMF characters that were queued */
 void ast_channel_undefer_dtmf(struct ast_channel *chan);
 
-/*! Initiate system shutdown -- prevents new channels from being allocated.
- * \param hangup  If "hangup" is non-zero, all existing channels will receive soft
- *  hangups */
-void ast_begin_shutdown(int hangup);
-
-/*! Cancels an existing shutdown and returns to normal operation */
-void ast_cancel_shutdown(void);
-
 /*! \return number of channels available for lookup */
 int ast_active_channels(void);
 
 /*! \return the number of channels not yet destroyed */
 int ast_undestroyed_channels(void);
 
-/*! \return non-zero if Asterisk is being shut down */
-int ast_shutting_down(void);
-
 /*! Activate a given generator */
 int ast_activate_generator(struct ast_channel *chan, struct ast_generator *gen, void *params);
 

main/asterisk.c

@@ -432,16 +432,34 @@ static char ast_config_AST_CTL[PATH_MAX] = "asterisk.ctl";
 extern unsigned int ast_FD_SETSIZE;
 
 static char *_argv[256];
+
 typedef enum {
-	NOT_SHUTTING_DOWN = -2,
+	/*! Normal operation */
-	SHUTTING_DOWN = -1,
+	NOT_SHUTTING_DOWN,
-	/* Valid values for quit_handler niceness below: */
+	/*! Committed to shutting down.  Final phase */
+	SHUTTING_DOWN_FINAL,
+	/*! Committed to shutting down.  Initial phase */
+	SHUTTING_DOWN,
+	/*!
+	 * Valid values for quit_handler() niceness below.
+	 * These shutdown/restart levels can be cancelled.
+	 *
+	 * Remote console exit right now
+	 */
 	SHUTDOWN_FAST,
+	/*! core stop/restart now */
 	SHUTDOWN_NORMAL,
+	/*! core stop/restart gracefully */
 	SHUTDOWN_NICE,
+	/*! core stop/restart when convenient */
 	SHUTDOWN_REALLY_NICE
 } shutdown_nice_t;
+
 static shutdown_nice_t shuttingdown = NOT_SHUTTING_DOWN;
+
+/*! Prevent new channel allocation for shutdown. */
+static int shutdown_pending;
+
 static int restartnow;
 static pthread_t consolethread = AST_PTHREADT_NULL;
 static pthread_t mon_sig_flags;
@@ -1867,6 +1885,43 @@ int ast_set_priority(int pri)
 	return 0;
 }
 
+int ast_shutdown_final(void)
+{
+	return shuttingdown == SHUTTING_DOWN_FINAL;
+}
+
+int ast_shutting_down(void)
+{
+	return shutdown_pending;
+}
+
+int ast_cancel_shutdown(void)
+{
+	int shutdown_aborted = 0;
+
+	ast_mutex_lock(&safe_system_lock);
+	if (shuttingdown >= SHUTDOWN_FAST) {
+		shuttingdown = NOT_SHUTTING_DOWN;
+		shutdown_pending = 0;
+		shutdown_aborted = 1;
+	}
+	ast_mutex_unlock(&safe_system_lock);
+	return shutdown_aborted;
+}
+
+/*!
+ * \internal
+ * \brief Initiate system shutdown -- prevents new channels from being allocated.
+ */
+static void ast_begin_shutdown(void)
+{
+	ast_mutex_lock(&safe_system_lock);
+	if (shuttingdown != NOT_SHUTTING_DOWN) {
+		shutdown_pending = 1;
+	}
+	ast_mutex_unlock(&safe_system_lock);
+}
+
 static int can_safely_quit(shutdown_nice_t niceness, int restart);
 static void really_quit(int num, shutdown_nice_t niceness, int restart);
 
@@ -1879,8 +1934,53 @@ static void quit_handler(int num, shutdown_nice_t niceness, int restart)
 	/* It wasn't our time. */
 }
 
+#define SHUTDOWN_TIMEOUT	15	/* Seconds */
+
+/*!
+ * \internal
+ * \brief Wait for all channels to die, a timeout, or shutdown cancelled.
+ * \since 13.3.0
+ *
+ * \param niceness Shutdown niceness in effect
+ * \param seconds Number of seconds to wait or less than zero if indefinitely.
+ *
+ * \retval zero if waiting wasn't necessary.  We were idle.
+ * \retval non-zero if we had to wait.
+ */
+static int wait_for_channels_to_die(shutdown_nice_t niceness, int seconds)
+{
+	time_t start;
+	time_t now;
+	int waited = 0;
+
+	time(&start);
+	for (;;) {
+		if (!ast_undestroyed_channels() || shuttingdown != niceness) {
+			break;
+		}
+		if (seconds < 0) {
+			/* No timeout so just poll every second */
+			sleep(1);
+		} else {
+			time(&now);
+
+			/* Wait up to the given seconds for all channels to go away */
+			if (seconds < (now - start)) {
+				break;
+			}
+
+			/* Sleep 1/10 of a second */
+			usleep(100000);
+		}
+		waited = 1;
+	}
+	return waited;
+}
+
 static int can_safely_quit(shutdown_nice_t niceness, int restart)
 {
+	int waited = 0;
+
 	/* Check if someone else isn't already doing this. */
 	ast_mutex_lock(&safe_system_lock);
 	if (shuttingdown != NOT_SHUTTING_DOWN && niceness >= shuttingdown) {
@@ -1897,40 +1997,30 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart)
 	 * the atexit handlers, otherwise this would be a bit early. */
 	ast_cdr_engine_term();
 
-	/* Shutdown the message queue for the technology agnostic message channel.
+	/*
-	 * This has to occur before we pause shutdown pending ast_undestroyed_channels. */
+	 * Shutdown the message queue for the technology agnostic message channel.
+	 * This has to occur before we pause shutdown pending ast_undestroyed_channels.
+	 *
+	 * XXX This is not reversed on shutdown cancel.
+	 */
 	ast_msg_shutdown();
 
 	if (niceness == SHUTDOWN_NORMAL) {
-		time_t s, e;
 		/* Begin shutdown routine, hanging up active channels */
-		ast_begin_shutdown(1);
+		ast_begin_shutdown();
 		if (ast_opt_console) {
 			ast_verb(0, "Beginning asterisk %s....\n", restart ? "restart" : "shutdown");
 		}
-		time(&s);
+		ast_softhangup_all();
-		for (;;) {
+		waited |= wait_for_channels_to_die(niceness, SHUTDOWN_TIMEOUT);
-			time(&e);
-			/* Wait up to 15 seconds for all channels to go away */
-			if ((e - s) > 15 || !ast_undestroyed_channels() || shuttingdown != niceness) {
-				break;
-			}
-			/* Sleep 1/10 of a second */
-			usleep(100000);
-		}
 	} else if (niceness >= SHUTDOWN_NICE) {
 		if (niceness != SHUTDOWN_REALLY_NICE) {
-			ast_begin_shutdown(0);
+			ast_begin_shutdown();
 		}
 		if (ast_opt_console) {
 			ast_verb(0, "Waiting for inactivity to perform %s...\n", restart ? "restart" : "halt");
 		}
-		for (;;) {
+		waited |= wait_for_channels_to_die(niceness, -1);
-			if (!ast_undestroyed_channels() || shuttingdown != niceness) {
-				break;
-			}
-			sleep(1);
-		}
 	}
 
 	/* Re-acquire lock and check if someone changed the niceness, in which
@@ -1944,9 +2034,28 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart)
 		ast_mutex_unlock(&safe_system_lock);
 		return 0;
 	}
+
+	if (niceness >= SHUTDOWN_REALLY_NICE) {
 		shuttingdown = SHUTTING_DOWN;
 		ast_mutex_unlock(&safe_system_lock);
 
+		/* No more Mr. Nice guy.  We are committed to shutting down now. */
+		ast_begin_shutdown();
+		ast_softhangup_all();
+		waited |= wait_for_channels_to_die(SHUTTING_DOWN, SHUTDOWN_TIMEOUT);
+
+		ast_mutex_lock(&safe_system_lock);
+	}
+	shuttingdown = SHUTTING_DOWN_FINAL;
+	ast_mutex_unlock(&safe_system_lock);
+
+	if (niceness >= SHUTDOWN_NORMAL && waited) {
+		/*
+		 * We were not idle.  Give things in progress a chance to
+		 * recognize the final shutdown phase.
+		 */
+		sleep(1);
+	}
 	return 1;
 }
 
@@ -2454,8 +2563,6 @@ static char *handle_restart_when_convenient(struct ast_cli_entry *e, int cmd, st
 
 static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	int aborting_shutdown = 0;
-
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "core abort shutdown";
@@ -2471,16 +2578,8 @@ static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_
 	if (a->argc != e->args)
 		return CLI_SHOWUSAGE;
 
-	ast_mutex_lock(&safe_system_lock);
-	if (shuttingdown >= SHUTDOWN_FAST) {
-		aborting_shutdown = 1;
-		shuttingdown = NOT_SHUTTING_DOWN;
-	}
-	ast_mutex_unlock(&safe_system_lock);
-
-	if (aborting_shutdown) {
 	ast_cancel_shutdown();
-	}
+
 	return CLI_SUCCESS;
 }
 

main/bridge.c

@@ -5322,7 +5322,7 @@ static void bridge_prnt_obj(void *v_obj, void *where, ao2_prnt_fn *prnt)
 
 /*!
  * \internal
- * \brief Shutdown the bridging system.
+ * \brief Shutdown the bridging system.  Stuff to always do.
  * \since 12.0.0
  *
  * \return Nothing
@@ -5334,6 +5334,17 @@ static void bridge_shutdown(void)
 	ast_manager_unregister("BridgeTechnologyUnsuspend");
 	ast_cli_unregister_multiple(bridge_cli, ARRAY_LEN(bridge_cli));
 	ao2_container_unregister("bridges");
+}
+
+/*!
+ * \internal
+ * \brief Shutdown the bridging system.  More stuff to do on graceful shutdown.
+ * \since 13.3.0
+ *
+ * \return Nothing
+ */
+static void bridge_cleanup(void)
+{
 	ao2_cleanup(bridges);
 	bridges = NULL;
 	ao2_cleanup(bridge_manager);
@@ -5342,6 +5353,7 @@ static void bridge_shutdown(void)
 
 int ast_bridging_init(void)
 {
+	ast_register_cleanup(bridge_cleanup);
 	ast_register_atexit(bridge_shutdown);
 
 	if (ast_stasis_bridging_init()) {

main/channel.c

@@ -100,9 +100,6 @@ struct ast_epoll_data {
 #define MONITOR_DELAY	150 * 8		/*!< 150 ms of MONITORING DELAY */
 #endif
 
-/*! \brief Prevent new channel allocation if shutting down. */
-static int shutting_down;
-
 static int chancount;
 
 unsigned long global_fin, global_fout;
@@ -507,13 +504,9 @@ static int ast_channel_softhangup_cb(void *obj, void *arg, int flags)
 	return 0;
 }
 
-void ast_begin_shutdown(int hangup)
+void ast_softhangup_all(void)
 {
-	shutting_down = 1;
-
-	if (hangup) {
 	ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL);
-	}
 }
 
 /*! \brief returns number of active/allocated channels */
@@ -527,18 +520,6 @@ int ast_undestroyed_channels(void)
 	return ast_atomic_fetchadd_int(&chancount, 0);
 }
 
-/*! \brief Cancel a shutdown in progress */
-void ast_cancel_shutdown(void)
-{
-	shutting_down = 0;
-}
-
-/*! \brief Returns non-zero if Asterisk is being shut down */
-int ast_shutting_down(void)
-{
-	return shutting_down;
-}
-
 /*! \brief Set when to hangup channel */
 void ast_channel_setwhentohangup_tv(struct ast_channel *chan, struct timeval offset)
 {

main/http.c

@@ -1873,6 +1873,11 @@ static int httpd_process_request(struct ast_tcptls_session_instance *ser)
 		return -1;
 	}
 
+	if (ast_shutdown_final()) {
+		ast_http_error(ser, 503, "Service Unavailable", "Shutdown in progress");
+		return -1;
+	}
+
 	/* process "Request Headers" lines */
 	if (http_request_headers_get(ser, &headers)) {
 		return -1;

res/res_pjsip/pjsip_options.c

@@ -655,6 +655,10 @@ static pj_bool_t options_on_rx_request(pjsip_rx_data *rdata)
 	ast_copy_pj_str(exten, &sip_ruri->user, sizeof(exten));
 
 	if (ast_shutting_down()) {
+		/*
+		 * Not taking any new calls at this time.
+		 * Likely a server availability OPTIONS poll.
+		 */
 		send_options_response(rdata, 503);
 	} else if (!ast_strlen_zero(exten) && !ast_exists_extension(NULL, endpoint->context, exten, 1, NULL)) {
 		send_options_response(rdata, 404);

res/res_pjsip_pubsub.c

@@ -2059,7 +2059,7 @@ static int send_notify(struct sip_subscription_tree *sub_tree, unsigned int forc
 	pjsip_evsub *evsub = sub_tree->evsub;
 	pjsip_tx_data *tdata;
 
-	if (ast_shutting_down()
+	if (ast_shutdown_final()
 		&& sub_tree->root->subscription_state == PJSIP_EVSUB_STATE_TERMINATED
 		&& sub_tree->persistence) {
 		return 0;