From 7bcbec0956522661e080b447f4622ccb232f6646 Mon Sep 17 00:00:00 2001
From: Michael Prokop <mprokop@sipwise.com>
Date: Thu, 14 Jul 2016 21:57:22 +0200
Subject: [PATCH] MT#20863 Fail if ssh-ing fails or port 1443 not in LISTEN
 state

Since quite some time nginx listens on (a) specific IP
address(es) instead of just listening on all interfaces (see
previous commit a225c3a15f3e6d0 AKA Change-Id: I42257d8b2a1eedf7433158477983cfc5f9c97315).
Because we didn't error out if SSH-ing fails or port 1443 isn't
in LISTEN state the script just checked until it ran into the
retry limit and then continued without failing.  Instead let's
report this error to the Jenkins job/user by failing the run if
either SSH doesn't work or port 1443 isn't in LISTEN state.

While at it move the "retry" variable assignment in front
of the reporting, so we don't end it with "1 retries left"
and also get rid of `unnecessary on arithmetic variables`
in subshell (the $retry usage).

Change-Id: Ic7a4a39e460c4c81866c12b4b8b90c69f213538f
---
 ec2-create-ce | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/ec2-create-ce b/ec2-create-ce
index b41e4ca..9fe6217 100755
--- a/ec2-create-ce
+++ b/ec2-create-ce
@@ -363,9 +363,9 @@ echo "Starting instance ID $INSTANCE_ID for ngcp version $NGCP_VERSION"
 retry=120 # up to 10 minutes
 while [ "$STATUS" != "running" ] && [ $retry -ne 0 ] ; do
   STATUS="$(ec2-describe-instances --region ${AWS_REGION} --filter instance-id=$INSTANCE_ID | awk '/INSTANCE/ {print $6}')"
+  retry=$((retry - 1))
   echo "Instance ID $INSTANCE_ID not running yet, checking again in 5 seconds ($retry retries left)."
   sleep 5
-  retry=$(($retry - 1))
 done
 
 if [ "$STATUS" != "running" ] ; then
@@ -386,7 +386,7 @@ while [ "$FINISHED_NGCP_CE_INSTALLATION" != "true" ] && [ $retry -ne 0 ] ; do
   else
     echo "ngcp installation not yet finished, checking again in 5 seconds ($retry retries left)."
     sleep 5
-    retry=$(($retry - 1))
+    retry=$((retry - 1))
   fi
 done
 
@@ -409,9 +409,9 @@ else
   STATUS=""
   while [ "$STATUS" != "running" ] && [ $retry -ne 0 ] ; do
     STATUS="$(ec2-describe-instances --region ${AWS_REGION} --filter instance-id=$INSTANCE_ID | awk '/^INSTANCE/ {print $6}')"
+    retry=$((retry - 1))
     echo "Instance ID not yet running (status: ${STATUS}), checking again in 5 seconds ($retry retries left)."
     sleep 5
-    retry=$(($retry - 1))
   done
 
   if [ "$STATUS" != "running" ] ; then
@@ -430,11 +430,17 @@ else
     if [ $? -eq 0 ] ; then
       STATUS=ok
     else
-      echo "Host $HOSTNAME can't be reached via ssh login yet, checking again in 5 seconds ($retry retries left)."
+      retry=$((retry - 1))
+      echo "Host $HOSTNAME can't be reached via ssh login yet or nginx not listening on port 1443."
+      echo "Retrying ssh access and checking port 1443 LISTEN state in 5 seconds ($retry retries left)."
       sleep 5
-      retry=$(($retry - 1))
     fi
   done
+
+  if [ "$STATUS" != "ok" ] ; then
+    echo "Host $HOSTNAME can't be reached via ssh or port 1443 not in state LISTEN, giving up." >&2
+    bailout 1
+  fi
 fi
 
 remove_ami "$AWS_REGION" "$AMI_NAME" || bailout 1
@@ -461,9 +467,9 @@ while [ "$STATUS" != "available" ] && [ $retry -ne 0 ] ; do
     bailout 1
   fi
 
+  retry=$((retry - 1))
   echo "AMI ID $AMI_ID not yet finished (status: ${STATUS}), checking again in 5 seconds ($retry retries left)."
   sleep 5
-  retry=$(($retry - 1))
 done
 
 if [ "$STATUS" != "available" ] ; then