From 1d5c99f4117d291ca52823aee3371b80f3291f83 Mon Sep 17 00:00:00 2001 From: Mykola Malkov Date: Mon, 17 Feb 2020 12:43:53 +0200 Subject: [PATCH] TT#76060 Run configuration after reboot We need to reboot the system before the configuration as there can be upgrade of kernel. If it happens gcp-initial-configuration tries to start ngcp-license-client.service and fails as there is no module for current kernel. Use a wapper for ssh commands to reduce the length of the lines. Change-Id: I68042891aa6193abe44baf873c273d068c3191f6 --- ec2-create-ce | 146 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 118 insertions(+), 28 deletions(-) diff --git a/ec2-create-ce b/ec2-create-ce index aa1616f..3ba0d38 100755 --- a/ec2-create-ce +++ b/ec2-create-ce @@ -56,6 +56,21 @@ Mandatory options for VPC usage: " } +ssh_wrapper() { + # use keepalives of 10 seconds (if 3 fail, connection terminated) and + # connection timeout of 15 secs instead of default system TCP timeout (15 + # minutes) + # shellcheck disable=SC2029 + ssh -o "ServerAliveInterval 10" \ + -o "ConnectTimeout 15" \ + -o "StrictHostKeyChecking=no" \ + -o "UserKnownHostsFile=/dev/null" \ + -i "${KEY_FILE}" \ + "admin@${HOSTNAME}" \ + "$@" +} + + # generate report for usage in other scripts/jenkins jobs generate_ec2_report() { cat > ec2_report.txt << EOF @@ -199,6 +214,68 @@ remove_ami() { fi } +wait_for_ssh() { + local max_attempt=360 + local attempt=0 + + while [[ "${attempt}" -lt "${max_attempt}" ]]; do + if ssh_wrapper exit; then + return + fi + sleep 1 + attempt=$((attempt + 1)) + done + + return 1 +} + +get_status() { + case "${vmversion}" in + 2.*|3.*|mr[3-6].*|mr7.[0-4]*) + if ssh_wrapper "grep 'System configured' -q /etc/sipwise_ngcp_version"; then + echo "finished" + fi + ;; + *) + local status='' + status=$(wget --timeout=30 -q -O- "${HOSTNAME}:4242/status" || true) + echo "${status}" + ;; + esac +} + +wait_for_configuration() { + local max_attempt=360 + local attempt=0 + local status='' + + if [[ "$-" == *x* ]]; then + trace_enabled=true + set +x + fi + + echo "Waiting when system is configured" + while [[ "${attempt}" -lt "${max_attempt}" ]]; do + status=$(get_status) + if [[ "${status}" == "finished" ]]; then + echo "System was successfully configured" + return + elif [[ "${status}" == "error" ]]; then + echo "System configuration was failed" + break + fi + echo "Attempt ${attempt} of ${max_attempt}, sleeping for a second. Status: '${status}'" + sleep 1 + attempt=$((attempt + 1)) + done + + if "${trace_enabled}"; then + set -x + fi + + return 1 +} + CMDLINE_OPTS=allocation-id:,ami-name:,base-ami:,copy-to-all-regions,elastic-ip:,help,installer-url:,instance-type:,keep-ami-snapshot,ngcp-release:,public,region:,remove-existing-ami,subnet:,skip-reboot _opt_temp=$(getopt --name "$0" -o +bch --long ${CMDLINE_OPTS} -- "$@") @@ -429,7 +506,7 @@ sed_regexp='' sed_regexp+='s/^(export FORCE)=.+$/\1=yes/;' sed_regexp+='s/^(export DHCP)=.+$/\1=true/;' sed -ri "\${sed_regexp}" /etc/ngcp-installer/config_deploy.inc -ngcp-installer && screen -dm bash -c 'ngcp-initial-configuration' +ngcp-installer attempt=0 max_attempt=1000 rc=1 @@ -520,14 +597,12 @@ HOSTNAME="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=ins retry=360 # up to 30 minutes FINISHED_NGCP_CE_INSTALLATION=false -grep_cmd="grep -q 'Installation finished. Thanks for choosing NGCP' /var/log/ngcp-installer-debug.log" +install_log_file='/var/log/ngcp-installer-debug.log' if "${init_configuration}" ; then - grep_cmd="grep -q 'System was successfully configured, now you have the best VoIP software.' /var/log/ngcp-installer.log" + install_log_file='/var/log/ngcp-installer.log' fi while [ "${FINISHED_NGCP_CE_INSTALLATION}" != "true" ] && [ ${retry} -ne 0 ] ; do - # shellcheck disable=SC2029 - ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}" \ - "${grep_cmd}" + ssh_wrapper "grep -q 'Installation finished. Thanks for choosing' ${install_log_file}" RC=$? if [ ${RC} -eq 0 ] ; then FINISHED_NGCP_CE_INSTALLATION=true @@ -549,13 +624,13 @@ else fi #Remove temporary crontab -ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}" "sudo crontab -l || sudo crontab -r" +ssh_wrapper "sudo crontab -l || sudo crontab -r" # reboot CE system if ${_opt_skip_reboot} ; then echo "*** Skipping system reboot of instance ID ${INSTANCE_ID} as requested via --skip-reboot ***" else - ssh -o "ServerAliveInterval=10" -o "ServerAliveCountMax=1" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}" "sudo reboot" + ssh_wrapper "sudo reboot" retry=120 # up to 10 minutes STATUS="" while [ "${STATUS}" != "running" ] && [ ${retry} -ne 0 ] ; do @@ -573,35 +648,50 @@ else # we probably have a new hostname after reboot HOSTNAME="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[] | {PublicDnsName} | [.[]] | .[0]')" - # now check for available ssh connection and nginx listening on port 1443 - retry=120 # up to 10 minutes - STATUS="" - while [ "${STATUS}" != "ok" ] && [ ${retry} -ne 0 ] ; do - ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}" "sudo lsof -i -n -P | grep -q '.*:1443 (LISTEN)'" - if [ $? -eq 0 ] ; then - STATUS=ok - else - retry=$((retry - 1)) - echo "Host ${HOSTNAME} can't be reached via ssh login yet or nginx not listening on port 1443." - echo "Retrying ssh access and checking port 1443 LISTEN state in 5 seconds (${retry} retries left)." - sleep 5 - fi - done + echo "Waiting for ssh access..." + wait_for_ssh +fi - if [ "${STATUS}" != "ok" ] ; then - echo "Host ${HOSTNAME} can't be reached via ssh or port 1443 not in state LISTEN, giving up." >&2 - bailout 1 +success=true +if "${init_configuration}"; then + echo "Configuring system..." + ssh_wrapper "sudo screen -dm bash -c 'ngcp-initial-configuration'" + wait_for_configuration || success=false + echo "Trying to retrieve ngcp-installer.log file after configuration" + scp \ + -o "StrictHostKeyChecking=no" \ + -o "UserKnownHostsFile=/dev/null" \ + -i "${KEY_FILE}" \ + "admin@${HOSTNAME}":/var/log/ngcp-installer\*log ./ || true +fi + +# now check for available ssh connection and nginx listening on port 1443 +retry=120 # up to 10 minutes +STATUS="" +while [[ "${STATUS}" != "ok" && "${retry}" -ne 0 ]] && "${success}"; do + ssh_wrapper "sudo lsof -i -n -P | grep -q '.*:1443 (LISTEN)'" + if [ $? -eq 0 ] ; then + STATUS=ok + else + retry=$((retry - 1)) + echo "Host ${HOSTNAME} can't be reached via ssh login yet or nginx not listening on port 1443." + echo "Retrying ssh access and checking port 1443 LISTEN state in 5 seconds (${retry} retries left)." + sleep 5 fi +done + +if [ "${STATUS}" != "ok" ] ; then + echo "Host ${HOSTNAME} can't be reached via ssh or port 1443 not in state LISTEN, giving up." >&2 + bailout 1 fi remove_ami "${AWS_REGION}" "${AMI_NAME}" || bailout 1 echo "*** Getting rid of all authorized_keys files. ***" -ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}" \ - 'sudo find / -name authorized_keys -exec rm -f {} \;' +ssh_wrapper 'sudo find / -name authorized_keys -exec rm -f {} \;' LOGFILE_AMI=$(mktemp) -aws ec2 create-image \ +aws ec2 create-image \ --region "${AWS_REGION}" \ --name "${AMI_NAME}" \ --description "${AMI_DESCRIPTION}" \