mirror of https://github.com/sipwise/ec2-tools.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
775 lines
25 KiB
775 lines
25 KiB
#!/bin/bash
|
|
|
|
export TZ="UTC"
|
|
|
|
# defaults
|
|
[ -n "${KEY_NAME}" ] || KEY_NAME="jenkins-ngcp-create-ami"
|
|
[ -n "${KEY_FILE}" ] || KEY_FILE="${HOME}/.ssh/${KEY_NAME}.pem"
|
|
[ -n "${SECURITY_GROUP}" ] || SECURITY_GROUP="sipwise-only"
|
|
[ -n "${TAG_PURPOSE}" ] || TAG_PURPOSE="ngcp-create-ce-ami"
|
|
DATE_STRING=$(date +%Y-%m-%d_%H:%M)
|
|
|
|
rm -f ec2_report.txt
|
|
|
|
usage() {
|
|
echo "$0 - script to generate EC2 AMI for sip:provider ngcp CE
|
|
|
|
Usage:
|
|
|
|
$0 <mandatory_options> [OPTIONS]
|
|
|
|
Supported OPTIONS:
|
|
|
|
--ami-name <STRING> set AMI name to specified string
|
|
(if unset defaults to ngcp-ce-\$release)
|
|
--base-ami <AMI-ID> AMI ID that should be used as base for running
|
|
ngcp installation process (recommended: Debian 64bit)
|
|
(if unset defaults to ami-8a745cf3)
|
|
--copy-to-all-regions Copy resulting AMI to all available EC2 regions
|
|
--help display this help text
|
|
--installer-url <URL> Use ngcp-installer.deb, retrieved from specified URL
|
|
(if unset defaults to http://deb.sipwise.com/spce/ngcp-installer-\$NGCP_RELEASE.deb)
|
|
--instance-type <TYPE> EC2 instance type that should be used for launching
|
|
(if unset defaults to t2.medium)
|
|
--keep-ami-snapshot do not remove snapshot used by a possibly existing AMI
|
|
which gets removed during the run (because of identical names)
|
|
(only considered in combination with --remove-existing-ami)
|
|
--ngcp-release <VERSION> sip:provider release version
|
|
(if unset defaults to <latest>)
|
|
--public Make AMI available to the public
|
|
(if unset defaults to private)
|
|
--remove-existing-ami If an AMI with the according name already exists get
|
|
rid of it before trying to creating the AMI
|
|
--region <REGION> specify region that should be used
|
|
(if unset defaults to the eu-west-1 zone)
|
|
--skip-reboot do not reboot ngcp system after installation finished
|
|
|
|
Mandatory options for EC2-Classic (non-VPC/older generations) usage:
|
|
|
|
--elastic-ip <IP> associate started instance ID with provided Elastic IP address
|
|
|
|
Mandatory options for VPC usage:
|
|
|
|
--allocation-id <A-ID> specify allocation ID for Elastic IP
|
|
--elastic-ip <IP> associate started instance ID with provided Elastic IP address
|
|
--subnet <Subnet-ID> specify subnet ID that should be used
|
|
"
|
|
}
|
|
|
|
ssh_wrapper() {
|
|
# use keepalives of 10 seconds (if 3 fail, connection terminated) and
|
|
# connection timeout of 15 secs instead of default system TCP timeout (15
|
|
# minutes)
|
|
# shellcheck disable=SC2029
|
|
ssh -o "ServerAliveInterval 10" \
|
|
-o "ConnectTimeout 15" \
|
|
-o "StrictHostKeyChecking=no" \
|
|
-o "UserKnownHostsFile=/dev/null" \
|
|
-i "${KEY_FILE}" \
|
|
"admin@${HOSTNAME}" \
|
|
"$@"
|
|
}
|
|
|
|
|
|
# generate report for usage in other scripts/jenkins jobs
|
|
generate_ec2_report() {
|
|
cat > ec2_report.txt << EOF
|
|
instance_hostname=${HOSTNAME}
|
|
instance_id=${INSTANCE_ID}
|
|
aws_region=${AWS_REGION}
|
|
ami_id=${AMI_ID}
|
|
ami_name=${AMI_NAME}
|
|
ami_description=${AMI_DESCRIPTION}
|
|
ami_public=${_opt_public}
|
|
ami_instance_type=${INSTANCE_TYPE}
|
|
ngcp_version=${NGCP_VERSION}
|
|
|
|
# for usage in Jenkins
|
|
testhost=${TESTHOST}
|
|
vmversion=${VMVERSION}
|
|
EOF
|
|
}
|
|
|
|
bailout() {
|
|
[ -n "$1" ] && EXIT="$1" || EXIT="1"
|
|
|
|
# ensure we get ec2_report.txt even if script execution fails,
|
|
# otherwise Jenkins jobs like ec2-ami-ec can't provide ec2_report.txt
|
|
# to Jenkins downstream job ec2-ami-stop
|
|
generate_ec2_report
|
|
|
|
exit "${EXIT}"
|
|
}
|
|
trap bailout 1 2 3 3 6 14 15
|
|
|
|
# NOTE: this is non-blocking currently, so we're not waiting until everything is finished here
|
|
copy_ami() {
|
|
local logfile
|
|
logfile=$(mktemp)
|
|
|
|
local source_region="$1"
|
|
local ami_id="$2"
|
|
local ami_name="$3"
|
|
local ami_description="$4"
|
|
|
|
local target_region
|
|
for target_region in \
|
|
eu-west-1
|
|
do
|
|
case "${target_region}" in
|
|
"${source_region}")
|
|
echo "*** Skipping region ${target_region} as source and destination region are the same ones. ***"
|
|
echo "AMI ID for region ${target_region}: ${ami_id}" >> ec2_report_amis.txt
|
|
;;
|
|
*)
|
|
remove_ami "${target_region}" "${ami_name}"
|
|
|
|
echo "*** Copying AMI ID ${ami_id} from ${source_region} to ${target_region} (note: we are NOT blocking/waiting until this is fully finished!) ***"
|
|
aws ec2 copy-image --source-image-id "${ami_id}" --source-region "${source_region}" --region "${target_region}" --name "${ami_name}" --description "${ami_description}" > "${logfile}"
|
|
|
|
if [ $? -ne 0 ]; then
|
|
echo "Error copying AMI ID ${ami_id} from ${source_region} to ${target_region}" >&2
|
|
return 1
|
|
fi
|
|
|
|
local ami_id_copy
|
|
ami_id_copy="$(jq -r '.ImageId' "${logfile}")"
|
|
|
|
if [ -z "${ami_id_copy}" ] ; then
|
|
echo "Error retrieving AMI ID ${ami_id} for region ${target_region}:"
|
|
cat "${logfile}" >&2
|
|
return 1
|
|
fi
|
|
|
|
# for usage as global report
|
|
echo "AMI ID for region ${target_region}: ${ami_id_copy}" >> ec2_report_amis.txt
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
remove_ami() {
|
|
local aws_region="$1"
|
|
local ami_name="$2"
|
|
|
|
if ! aws ec2 describe-images --region "${aws_region}" --filter "Name=name,Values=${ami_name}" | jq -r '.Images[].ImageId' | grep -q . ; then
|
|
return 0 # no AMIs with given name present
|
|
fi
|
|
|
|
echo "*** Warning, AMI with name ${ami_name} exists already."
|
|
if ! ${_opt_remove_existing_ami} ; then
|
|
return 0
|
|
fi
|
|
|
|
local existing_ami_id
|
|
existing_ami_id=$(aws ec2 describe-images --region "${aws_region}" --filter "Name=name,Values=${ami_name}" | jq -r '.Images[].ImageId')
|
|
|
|
if [ -z "${existing_ami_id}" ] ; then
|
|
echo "Problem retrieving AMI ID for AMI with name ${ami_name}." >&2
|
|
return 1
|
|
fi
|
|
|
|
# we need to identify which snapshot is used by the AMI so we can remove it after "deregister"-ing the AMI
|
|
local ami_snapshot
|
|
ami_snapshot=$(aws ec2 describe-images --region "${aws_region}" --filter "Name=name,Values=${ami_name}" | jq -r '.Images[].BlockDeviceMappings[].Ebs.SnapshotId')
|
|
|
|
if [ -z "${ami_snapshot}" ] ; then
|
|
echo "Problem retrieving snapshot ID for AMI with name ${ami_name}." >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "*** Option --remove-existing-ami is set, removing existing AMI with ID ${existing_ami_id} ***"
|
|
aws ec2 deregister-image --region "${aws_region}" --image-id "${existing_ami_id}"
|
|
if [ $? -ne 0 ] ; then
|
|
echo "Noticed problem when trying to delete AMI with name ${ami_name}." >&2
|
|
return 1
|
|
fi
|
|
|
|
if ${_opt_keep_ami_snapshot} ; then
|
|
echo "*** Option --keep-ami-snapshot is set, not removing AMI snapshot ${ami_snapshot}. ***"
|
|
else
|
|
aws ec2 delete-snapshot --region "${aws_region}" --snapshot-id "${ami_snapshot}"
|
|
if [ $? -ne 0 ] ; then
|
|
echo "Noticed problem when trying to delete snapshot ${ami_snapshot} for 'deregister'-ed AMI with name ${ami_name}." >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
wait_for_ssh() {
|
|
local max_attempt=360
|
|
local attempt=0
|
|
|
|
while [[ "${attempt}" -lt "${max_attempt}" ]]; do
|
|
if ssh_wrapper exit; then
|
|
return
|
|
fi
|
|
sleep 1
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
get_status() {
|
|
case "${NGCP_VERSION}" in
|
|
2.*|3.*|mr[3-6].*|mr7.[0-4]*)
|
|
if ssh_wrapper "grep 'System configured' -q /etc/sipwise_ngcp_version"; then
|
|
echo "finished"
|
|
fi
|
|
;;
|
|
*)
|
|
local status=''
|
|
status=$(wget --timeout=30 -q -O- "${HOSTNAME}:4242/status" || true)
|
|
echo "${status}"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
wait_for_configuration() {
|
|
local max_attempt=360
|
|
local attempt=0
|
|
local status=''
|
|
|
|
if [[ "$-" == *x* ]]; then
|
|
trace_enabled=true
|
|
set +x
|
|
fi
|
|
|
|
echo "Waiting when system is configured"
|
|
while [[ "${attempt}" -lt "${max_attempt}" ]]; do
|
|
status=$(get_status)
|
|
if [[ "${status}" == "finished" ]]; then
|
|
echo "System was successfully configured"
|
|
return
|
|
elif [[ "${status}" == "error" ]]; then
|
|
echo "System configuration was failed"
|
|
break
|
|
fi
|
|
echo "Attempt ${attempt} of ${max_attempt}, sleeping for a second. Status: '${status}'"
|
|
sleep 1
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
if "${trace_enabled}"; then
|
|
set -x
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
CMDLINE_OPTS=allocation-id:,ami-name:,base-ami:,copy-to-all-regions,elastic-ip:,help,installer-url:,instance-type:,keep-ami-snapshot,ngcp-release:,public,region:,remove-existing-ami,subnet:,skip-reboot
|
|
|
|
_opt_temp=$(getopt --name "$0" -o +bch --long ${CMDLINE_OPTS} -- "$@")
|
|
if [ $? -ne 0 ]; then
|
|
echo "Try '$0 --help' for more information." >& 2
|
|
exit 1
|
|
fi
|
|
eval set -- "${_opt_temp}"
|
|
|
|
_opt_keep_ami_snapshot=false
|
|
_opt_public=false
|
|
_opt_remove_existing_ami=false
|
|
_opt_skip_reboot=false
|
|
_opt_copy_to_all_regions=false
|
|
|
|
while :; do
|
|
case "$1" in
|
|
--allocation-id)
|
|
shift; ALLOCATION_ID="$1"
|
|
;;
|
|
--ami-name)
|
|
shift; AMI_NAME="$1"
|
|
;;
|
|
--base-ami)
|
|
shift; BASE_AMI="$1"
|
|
;;
|
|
--copy-to-all-regions)
|
|
_opt_copy_to_all_regions=true
|
|
;;
|
|
--elastic-ip)
|
|
shift; ELASTIC_IP="$1"
|
|
;;
|
|
--help)
|
|
usage ; exit 0;
|
|
;;
|
|
--installer-url)
|
|
shift; INSTALLER_URL="$1"
|
|
;;
|
|
--instance-type)
|
|
shift; INSTANCE_TYPE="$1"
|
|
;;
|
|
--keep-ami-snapshot)
|
|
_opt_keep_ami_snapshot=true
|
|
;;
|
|
--ngcp-release)
|
|
shift; NGCP_RELEASE="$1"
|
|
;;
|
|
--public)
|
|
_opt_public=true
|
|
;;
|
|
--region)
|
|
shift; AWS_REGION="$1"
|
|
;;
|
|
--remove-existing-ami)
|
|
_opt_remove_existing_ami=true
|
|
;;
|
|
--skip-reboot)
|
|
_opt_skip_reboot=true
|
|
;;
|
|
--subnet)
|
|
shift; SUBNET="$1"
|
|
;;
|
|
--)
|
|
shift; break
|
|
;;
|
|
*)
|
|
echo "Internal getopt error! $1" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# if unset set sane defaults
|
|
[ -n "${AWS_REGION}" ] || AWS_REGION="eu-west-1"
|
|
[ -n "${BASE_AMI}" ] || BASE_AMI="ami-0e9cc061cd3259f22"
|
|
[ -n "${INSTANCE_TYPE}" ] || INSTANCE_TYPE="m5.large"
|
|
|
|
if ! [ -r "${HOME}"/.aws/config ] ; then
|
|
echo "awscli seems to be unconfigured, ensure ${HOME}/.aws/config exists." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! [ -r "${KEY_FILE}" ] ; then
|
|
echo "Could not read key file ${KEY_FILE}, can not continue." >&2
|
|
exit 1
|
|
fi
|
|
|
|
check4progs(){
|
|
local RC=''
|
|
for arg in "$@" ; do
|
|
which "${arg}" >/dev/null 2>&1 || RC="${arg}"
|
|
done
|
|
if [ -n "${RC}" ] ; then
|
|
echo "${RC} not found/executable" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
if ! check4progs jq ; then
|
|
echo "Required tool jq not found, forgot to install jq?" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! check4progs aws ; then
|
|
echo "Required tool aws not found, forgot to install awscli?" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "${NGCP_RELEASE}" ] ; then
|
|
NGCP_RELEASE="latest"
|
|
echo "No ngcp release set, defaulting to ${NGCP_RELEASE}"
|
|
fi
|
|
|
|
if [ -z "${INSTALLER_URL}" ] ; then
|
|
INSTALLER_URL="http://deb.sipwise.com/spce/ngcp-installer-${NGCP_RELEASE}.deb"
|
|
fi
|
|
|
|
if ! wget --quiet -O /tmp/ngcp-installer.deb "${INSTALLER_URL}" ; then
|
|
echo "Couldn't retrieve ${INSTALLER_URL} - exiting" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if dpkg --info /tmp/ngcp-installer.deb | grep -q 'mr' ; then
|
|
NGCP_VERSION=$(dpkg --info /tmp/ngcp-installer.deb | awk '/^ Version: / {print $2}' | sed 's/.*mr/mr/')
|
|
else # older versions don't have the mr string in the version information, so let's reuse provided ngcp release
|
|
NGCP_VERSION="${NGCP_RELEASE}"
|
|
echo "*** No 'mr' string inside version information of installer found... ***"
|
|
echo "*** ... falling back to ngcp release version information [${NGCP_VERSION}] ***"
|
|
fi
|
|
|
|
if [ -z "${NGCP_VERSION}" ] ; then
|
|
echo "Couldn't identify ngcp version, exiting." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
# ensure we don't have any builds running already, see TT#30565
|
|
NUM_RUNNING_INSTANCES=$(aws ec2 describe-instances --region "${AWS_REGION}" \
|
|
--filter "Name=instance-state-name,Values=running" \
|
|
"Name=tag:Purpose,Values=${TAG_PURPOSE}" | \
|
|
jq '.Reservations[].Instances[].InstanceId' | \
|
|
wc -l)
|
|
if [ "${NUM_RUNNING_INSTANCES}" -gt 0 ] ; then
|
|
echo "Error: there are running instances with tag ${TAG_PURPOSE} in region ${AWS_REGION}" >&2
|
|
echo "Exiting to ensure we don't have instances running due to Jenkins job failures." >&2
|
|
echo "Please ensure that those instances are stopped before trying to build new ones." >&2
|
|
echo "In case of questions contact someone with according AWS permissions." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
# if AMI_NAME is set to "none" then we get defaults from Jenkins, if
|
|
# so then use the ngcp-ce-... naming schema
|
|
if [ -n "${AMI_NAME}" ] && [ "${AMI_NAME}" != "none" ] ; then
|
|
echo "*** AMI_NAME is set to ${AMI_NAME} ***"
|
|
else
|
|
# use just "ngcp-ce-mr3.3.1.0" if we get something like
|
|
# "ngcp-ce-mr3.3.1.0+0~20140528091259.443+wheezy~1.gbpf74599"
|
|
# as NGCP_VERSION
|
|
AMI_NAME="ngcp-ce-${NGCP_VERSION%%\+*}"
|
|
echo "*** AMI_NAME is unset yet, defaulting to ${AMI_NAME}"
|
|
|
|
case "${AMI_NAME}" in
|
|
ngcp-ce-mr*.*.*.*)
|
|
echo "*** Identified hotfix version string in AMI name [${AMI_NAME}]... ***"
|
|
AMI_NAME="${AMI_NAME%.*}"
|
|
echo "*** ... stripping AMI name to use ${AMI_NAME} instead ***"
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
|
|
AMI_DESCRIPTION="Official sip:provider CE AMI for release ${NGCP_VERSION} [${DATE_STRING}]"
|
|
|
|
case "${NGCP_RELEASE}" in
|
|
2.*|3.*|mr3.*|mr4.*|mr5.*|mr6.0*|mr6.1*)
|
|
echo "*** ngcp release ${NGCP_RELEASE} doesn't support systemd"
|
|
SYSTEMD_SUPPORT=false
|
|
;;
|
|
*)
|
|
echo "*** ngcp release ${NGCP_RELEASE} supports systemd"
|
|
SYSTEMD_SUPPORT=true
|
|
;;
|
|
esac
|
|
|
|
USER_DATA_FILE="$(mktemp)"
|
|
cat > "${USER_DATA_FILE}" << EOF
|
|
#!/bin/bash
|
|
wget -O /tmp/ngcp-installer.deb ${INSTALLER_URL}
|
|
dpkg -i /tmp/ngcp-installer.deb
|
|
|
|
if [ -f /etc/apt/sources.list.d/backports.list ] && grep -q 'jessie-backports' /etc/apt/sources.list.d/backports.list ; then
|
|
echo "jessie-backports no longer exists, removing /etc/apt/sources.list.d/backports.list"
|
|
rm -f /etc/apt/sources.list.d/backports.list
|
|
fi
|
|
|
|
# workaround for mr13.2.1 and newer with bookworm-backports enabled by default:
|
|
if [ -f /etc/apt/sources.list.d/debian.sources ] && grep -q -- '-backports' /etc/apt/sources.list.d/debian.sources ; then
|
|
echo "backports is unexpected, disabling from /etc/apt/sources.list.d/debian.sources"
|
|
sed -i -E 's/ [a-z]+-backports//' /etc/apt/sources.list.d/debian.sources
|
|
fi
|
|
|
|
if [ "${SYSTEMD_SUPPORT}" = "false" ] && [ -d /run/systemd/system ] ; then
|
|
echo "Switching from systemd to sysvinit (pre-reboot)"
|
|
apt update
|
|
apt install --yes sysvinit-core
|
|
echo "Rebooting now to switch from systemd to sysvinit"
|
|
#Temporary crontab to restart installer after reboot
|
|
(crontab -l ; echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") | crontab -
|
|
(crontab -l ; echo "@reboot FORCE=yes /usr/sbin/ngcp-installer 2>&1 | tee -a /var/log/ngcp-installer-debug.log") | crontab -
|
|
reboot
|
|
fi
|
|
|
|
# workaround for mr4.2.1
|
|
export HOME=/root/
|
|
EOF
|
|
|
|
|
|
|
|
init_configuration=false
|
|
case "${NGCP_RELEASE}" in
|
|
2.8|3.*|mr[3-5].*|mr6.[0-4]*)
|
|
echo "*** ngcp release ${NGCP_RELEASE} doesn't have configuration tools"
|
|
;;
|
|
*)
|
|
echo "*** ngcp release ${NGCP_RELEASE} has configuration tools"
|
|
init_configuration=true
|
|
;;
|
|
esac
|
|
|
|
if "${init_configuration}" ; then
|
|
cat >> "${USER_DATA_FILE}" << EOF
|
|
sed_regexp=''
|
|
sed_regexp+='s/^(export FORCE)=.+$/\1=yes/;'
|
|
sed_regexp+='s/^(export DHCP)=.+$/\1=true/;'
|
|
sed -ri "\${sed_regexp}" /etc/ngcp-installer/config_deploy.inc
|
|
ngcp-installer
|
|
attempt=0
|
|
max_attempt=1000
|
|
rc=1
|
|
while [[ "\${attempt}" -lt "\${max_attempt}" ]]; do
|
|
if grep 'System configured' -q /etc/sipwise_ngcp_version; then
|
|
rc=0
|
|
break
|
|
fi
|
|
sleep 1
|
|
attempt=\$((attempt + 1))
|
|
done
|
|
exit \${rc}
|
|
EOF
|
|
else
|
|
cat >> "${USER_DATA_FILE}" << EOF
|
|
FORCE=yes ngcp-installer 2>&1 | tee -a /var/log/ngcp-installer-debug.log
|
|
[[ -f /tmp/ngcp-installer.log ]] && cp /tmp/ngcp-installer.log /var/log/ngcp-installer.log
|
|
EOF
|
|
fi
|
|
|
|
echo "*** Generated the following user-data file:"
|
|
cat "${USER_DATA_FILE}"
|
|
echo "*** end of user-data-file ***"
|
|
|
|
LOGFILE_RUN="$(mktemp)"
|
|
if [ -n "${SUBNET}" ] ; then
|
|
echo "Starting EC2 instance in region ${AWS_REGION}, using VPC with subnet ${SUBNET}, instance-type ${INSTANCE_TYPE} + AMI ${BASE_AMI}"
|
|
aws ec2 run-instances --region "${AWS_REGION}" \
|
|
--user-data "file://${USER_DATA_FILE}" \
|
|
--subnet "${SUBNET}" \
|
|
--key "${KEY_NAME}" \
|
|
--instance-type "${INSTANCE_TYPE}" \
|
|
--image-id "${BASE_AMI}" >"${LOGFILE_RUN}"
|
|
else
|
|
echo "Starting EC2 instance in region ${AWS_REGION}, using EC2-Classic with security-group ${SECURITY_GROUP}, instance-type ${INSTANCE_TYPE} + AMI ${BASE_AMI}"
|
|
aws ec2 run-instances --region "${AWS_REGION}" \
|
|
--user-data "file://${USER_DATA_FILE}" \
|
|
--group "${SECURITY_GROUP}" \
|
|
--key "${KEY_NAME}" \
|
|
--instance-type "${INSTANCE_TYPE}" \
|
|
--image-id "${BASE_AMI}" >"${LOGFILE_RUN}"
|
|
fi
|
|
|
|
INSTANCE_ID="$(jq -r '.Instances[].InstanceId' "${LOGFILE_RUN}")"
|
|
if [ -z "${INSTANCE_ID}" ] ; then
|
|
echo "Error: Could not identify instance ID, exiting." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# assign tag so we de-register again the AMI if needed
|
|
aws ec2 create-tags --region "${AWS_REGION}" --resources "${INSTANCE_ID}" --tag "Key=Purpose,Value=${TAG_PURPOSE}"
|
|
|
|
# assign name (useful for web management console browsing)
|
|
[ -z "${JOB_NAME:-}" ] && JOB_NAME="unset" # running outside of Jenkins
|
|
[ -z "${BUILD_NUMBER:-}" ] && BUILD_NUMBER="executed-by-$(whoami)" # running outside of Jenkins
|
|
aws ec2 create-tags --region "${AWS_REGION}" --resources "${INSTANCE_ID}" --tag "Key=Name,Value=${AMI_NAME}/${DATE_STRING}/${JOB_NAME}/build_${BUILD_NUMBER}"
|
|
|
|
echo "Starting instance ID ${INSTANCE_ID} for ngcp version ${NGCP_VERSION}"
|
|
|
|
STATUS="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[].State.Name')"
|
|
|
|
retry=120 # up to 10 minutes
|
|
while [ "${STATUS}" != "running" ] && [ ${retry} -ne 0 ] ; do
|
|
STATUS="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[].State.Name')"
|
|
|
|
retry=$((retry - 1))
|
|
echo "Instance ID ${INSTANCE_ID} not running yet, checking again in 5 seconds (${retry} retries left)."
|
|
sleep 5
|
|
done
|
|
|
|
if [ "${STATUS}" != "running" ] ; then
|
|
echo "No further retries left and couldn't bring up system for AMI ${BASE_AMI}, giving up." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
# assign elastic IP address to instance,
|
|
# enabling us to enable access to restricted
|
|
# webserver areas
|
|
if [ -n "${ALLOCATION_ID}" ] ; then
|
|
echo "Associating elastic IP ${ELASTIC_IP} allocation ID ${ALLOCATION_ID} with instance ID ${INSTANCE_ID}"
|
|
aws ec2 associate-address --region "${AWS_REGION}" --allocation-id "${ALLOCATION_ID}" --instance "${INSTANCE_ID}" --allow-reassociation
|
|
elif [ -n "${ELASTIC_IP}" ] ; then
|
|
echo "Associating elastic IP ${ELASTIC_IP} with instance ID ${INSTANCE_ID}"
|
|
aws ec2 associate-address --region "${AWS_REGION}" "${ELASTIC_IP}" -i "${INSTANCE_ID}" --allow-reassociation
|
|
fi
|
|
|
|
HOSTNAME="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[] | {PublicDnsName} | [.[]] | .[0]')"
|
|
|
|
retry=360 # up to 30 minutes
|
|
FINISHED_NGCP_CE_INSTALLATION=false
|
|
install_log_file='/var/log/ngcp-installer-debug.log'
|
|
if "${init_configuration}" ; then
|
|
install_log_file='/var/log/ngcp-installer.log'
|
|
fi
|
|
while [ "${FINISHED_NGCP_CE_INSTALLATION}" != "true" ] && [ ${retry} -ne 0 ] ; do
|
|
ssh_wrapper "grep -q 'Installation finished. Thanks for choosing' ${install_log_file}"
|
|
RC=$?
|
|
if [ ${RC} -eq 0 ] ; then
|
|
FINISHED_NGCP_CE_INSTALLATION=true
|
|
else
|
|
echo "ngcp installation not yet finished, checking again in 5 seconds (${retry} retries left)."
|
|
sleep 5
|
|
retry=$((retry - 1))
|
|
fi
|
|
done
|
|
|
|
echo "Trying to retrieve ngcp-installer log files"
|
|
scp -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}":/var/log/ngcp-installer\*log ./ || true
|
|
|
|
if [ "${FINISHED_NGCP_CE_INSTALLATION}" = "true" ] ; then
|
|
echo "Successfully finished ngcp CE installation (version ${NGCP_VERSION})."
|
|
else
|
|
echo "No further retries left and couldn't finish ngcp CE installation (version ${NGCP_VERSION}), giving up." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
#Remove temporary crontab
|
|
ssh_wrapper "sudo crontab -l || sudo crontab -r"
|
|
|
|
# reboot CE system
|
|
if ${_opt_skip_reboot} ; then
|
|
echo "*** Skipping system reboot of instance ID ${INSTANCE_ID} as requested via --skip-reboot ***"
|
|
else
|
|
ssh_wrapper "sudo reboot"
|
|
retry=120 # up to 10 minutes
|
|
STATUS=""
|
|
while [ "${STATUS}" != "running" ] && [ ${retry} -ne 0 ] ; do
|
|
STATUS="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[].State.Name')"
|
|
retry=$((retry - 1))
|
|
echo "Instance ID not yet running (status: ${STATUS}), checking again in 5 seconds (${retry} retries left)."
|
|
sleep 5
|
|
done
|
|
|
|
if [ "${STATUS}" != "running" ] ; then
|
|
echo "No further retries left and couldn't finish reboot of instance ID ${INSTANCE_ID}, giving up." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
# we probably have a new hostname after reboot
|
|
HOSTNAME="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[] | {PublicDnsName} | [.[]] | .[0]')"
|
|
|
|
echo "Waiting for ssh access..."
|
|
wait_for_ssh
|
|
fi
|
|
|
|
success=true
|
|
if "${init_configuration}"; then
|
|
echo "Configuring system..."
|
|
ssh_wrapper "sudo screen -dm bash -c 'ngcp-initial-configuration'"
|
|
wait_for_configuration || success=false
|
|
echo "Trying to retrieve ngcp-installer.log file after configuration"
|
|
scp \
|
|
-o "StrictHostKeyChecking=no" \
|
|
-o "UserKnownHostsFile=/dev/null" \
|
|
-i "${KEY_FILE}" \
|
|
"admin@${HOSTNAME}":/var/log/ngcp-installer\*log ./ || true
|
|
fi
|
|
|
|
# now check for available ssh connection and nginx listening on port 1443
|
|
retry=120 # up to 10 minutes
|
|
STATUS=""
|
|
while [[ "${STATUS}" != "ok" && "${retry}" -ne 0 ]] && "${success}"; do
|
|
ssh_wrapper "sudo lsof -i -n -P | grep -q '.*:1443 (LISTEN)'"
|
|
if [ $? -eq 0 ] ; then
|
|
STATUS=ok
|
|
else
|
|
retry=$((retry - 1))
|
|
echo "Host ${HOSTNAME} can't be reached via ssh login yet or nginx not listening on port 1443."
|
|
echo "Retrying ssh access and checking port 1443 LISTEN state in 5 seconds (${retry} retries left)."
|
|
sleep 5
|
|
fi
|
|
done
|
|
|
|
if [ "${STATUS}" != "ok" ] ; then
|
|
echo "Host ${HOSTNAME} can't be reached via ssh or port 1443 not in state LISTEN, giving up." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
remove_ami "${AWS_REGION}" "${AMI_NAME}" || bailout 1
|
|
|
|
echo "*** Getting rid of all authorized_keys files. ***"
|
|
ssh_wrapper 'sudo find / -name authorized_keys -exec rm -f {} \;'
|
|
|
|
LOGFILE_AMI=$(mktemp)
|
|
aws ec2 create-image \
|
|
--region "${AWS_REGION}" \
|
|
--name "${AMI_NAME}" \
|
|
--description "${AMI_DESCRIPTION}" \
|
|
--instance-id "${INSTANCE_ID}" \
|
|
> "${LOGFILE_AMI}"
|
|
|
|
AMI_ID=$(jq -r '.ImageId' "${LOGFILE_AMI}")
|
|
|
|
if [ -n "${AMI_ID}" ] ; then
|
|
echo "Received AMI ID ${AMI_ID}, now waiting until AMI is available."
|
|
else
|
|
echo "Error creating AMI :(" >&2
|
|
cat "${LOGFILE_AMI}" >&2
|
|
bailout 1
|
|
fi
|
|
|
|
retry=120 # up to 10 minutes
|
|
STATUS=""
|
|
while [ "${STATUS}" != "available" ] && [ ${retry} -ne 0 ] ; do
|
|
STATUS="$(aws ec2 describe-images --region "${AWS_REGION}" --image-ids "${AMI_ID}" | jq -r '.Images[].State')"
|
|
|
|
if [ "${STATUS}" = "failed" ] ; then
|
|
echo "Fatal error while creating AMI ID ${AMI_ID} (status: ${STATUS}), exiting." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
retry=$((retry - 1))
|
|
echo "AMI ID ${AMI_ID} not yet finished (status: ${STATUS}), checking again in 5 seconds (${retry} retries left)."
|
|
sleep 5
|
|
done
|
|
|
|
if [ "${STATUS}" != "available" ] ; then
|
|
echo "No further retries left and couldn't finish creation of AMI ID ${AMI_ID} for instance ID ${INSTANCE_ID}, giving up." >&2
|
|
bailout 1
|
|
fi
|
|
|
|
echo "Successfully created AMI with ID ${AMI_ID}"
|
|
|
|
if ${_opt_public} ; then
|
|
echo "Marking AMI as public"
|
|
aws ec2 modify-image-attribute --region "${AWS_REGION}" --image-id "${AMI_ID}" --launch-permission "Add=[{Group=all}]"
|
|
if [ $? -ne 0 ] ; then
|
|
echo "Noticed problem when trying to mark AMI with ID ${AMI_ID} as public." >&2
|
|
bailout 1
|
|
fi
|
|
fi
|
|
|
|
if ${_opt_copy_to_all_regions} ; then
|
|
copy_ami "${AWS_REGION}" "${AMI_ID}" "${AMI_NAME}" "${AMI_DESCRIPTION}" || bailout 1
|
|
fi
|
|
|
|
echo "*** Debugging information ***"
|
|
echo "--- user-data-file ---"
|
|
cat "${USER_DATA_FILE}"
|
|
|
|
echo "--- log file for instance run ---"
|
|
cat "${LOGFILE_RUN}"
|
|
|
|
echo "--- log file for AMI creation ---"
|
|
cat "${LOGFILE_AMI}"
|
|
echo "*** End of Debugging information ***"
|
|
|
|
echo
|
|
echo "*** Status report for AMI ***
|
|
AWS region: ${AWS_REGION}
|
|
AMI ID: ${AMI_ID}
|
|
AMI name: ${AMI_NAME}
|
|
AMI description: ${AMI_DESCRIPTION}
|
|
AMI public: ${_opt_public}
|
|
Instance type: ${INSTANCE_TYPE}
|
|
NGCP version: ${NGCP_VERSION}
|
|
*** End of Status report ***"
|
|
|
|
if [ -r ec2_report_amis.txt ] ; then
|
|
echo "*** Status report for AMIs in all regions ***"
|
|
cat ec2_report_amis.txt
|
|
fi
|
|
|
|
# Jenkins setup specific
|
|
TESTHOST="${HOSTNAME}"
|
|
case "${NGCP_RELEASE}" in
|
|
latest)
|
|
VMVERSION="trunk"
|
|
;;
|
|
|
|
*)
|
|
VMVERSION="${NGCP_RELEASE}"
|
|
;;
|
|
esac
|
|
|
|
generate_ec2_report
|
|
|
|
rm -f "${USER_DATA_FILE}" "${LOGFILE_RUN}" "${LOGFILE_AMI}"
|