MT#20863 Fail if ssh-ing fails or port 1443 not in LISTEN state

Since quite some time nginx listens on (a) specific IP
address(es) instead of just listening on all interfaces (see
previous commit a225c3a15f3e6d0 AKA Change-Id: I42257d8b2a1eedf7433158477983cfc5f9c97315).
Because we didn't error out if SSH-ing fails or port 1443 isn't
in LISTEN state the script just checked until it ran into the
retry limit and then continued without failing.  Instead let's
report this error to the Jenkins job/user by failing the run if
either SSH doesn't work or port 1443 isn't in LISTEN state.

While at it move the "retry" variable assignment in front
of the reporting, so we don't end it with "1 retries left"
and also get rid of `unnecessary on arithmetic variables`
in subshell (the $retry usage).

Change-Id: Ic7a4a39e460c4c81866c12b4b8b90c69f213538f

ec2-create-ce

@@ -363,9 +363,9 @@ echo "Starting instance ID $INSTANCE_ID for ngcp version $NGCP_VERSION"
 retry=120 # up to 10 minutes
 while [ "$STATUS" != "running" ] && [ $retry -ne 0 ] ; do
   STATUS="$(ec2-describe-instances --region ${AWS_REGION} --filter instance-id=$INSTANCE_ID | awk '/INSTANCE/ {print $6}')"
+  retry=$((retry - 1))
   echo "Instance ID $INSTANCE_ID not running yet, checking again in 5 seconds ($retry retries left)."
   sleep 5
-  retry=$(($retry - 1))
 done
 
 if [ "$STATUS" != "running" ] ; then
@@ -386,7 +386,7 @@ while [ "$FINISHED_NGCP_CE_INSTALLATION" != "true" ] && [ $retry -ne 0 ] ; do
   else
     echo "ngcp installation not yet finished, checking again in 5 seconds ($retry retries left)."
     sleep 5
-    retry=$(($retry - 1))
+    retry=$((retry - 1))
   fi
 done
 
@@ -409,9 +409,9 @@ else
   STATUS=""
   while [ "$STATUS" != "running" ] && [ $retry -ne 0 ] ; do
     STATUS="$(ec2-describe-instances --region ${AWS_REGION} --filter instance-id=$INSTANCE_ID | awk '/^INSTANCE/ {print $6}')"
+    retry=$((retry - 1))
     echo "Instance ID not yet running (status: ${STATUS}), checking again in 5 seconds ($retry retries left)."
     sleep 5
-    retry=$(($retry - 1))
   done
 
   if [ "$STATUS" != "running" ] ; then
@@ -430,11 +430,17 @@ else
     if [ $? -eq 0 ] ; then
       STATUS=ok
     else
-      echo "Host $HOSTNAME can't be reached via ssh login yet, checking again in 5 seconds ($retry retries left)."
+      retry=$((retry - 1))
+      echo "Host $HOSTNAME can't be reached via ssh login yet or nginx not listening on port 1443."
+      echo "Retrying ssh access and checking port 1443 LISTEN state in 5 seconds ($retry retries left)."
       sleep 5
-      retry=$(($retry - 1))
     fi
   done
+
+  if [ "$STATUS" != "ok" ] ; then
+    echo "Host $HOSTNAME can't be reached via ssh or port 1443 not in state LISTEN, giving up." >&2
+    bailout 1
+  fi
 fi
 
 remove_ami "$AWS_REGION" "$AMI_NAME" || bailout 1
@@ -461,9 +467,9 @@ while [ "$STATUS" != "available" ] && [ $retry -ne 0 ] ; do
     bailout 1
   fi
 
+  retry=$((retry - 1))
   echo "AMI ID $AMI_ID not yet finished (status: ${STATUS}), checking again in 5 seconds ($retry retries left)."
   sleep 5
-  retry=$(($retry - 1))
 done
 
 if [ "$STATUS" != "available" ] ; then