From b581035bf4f1bba76076bf4cd99d7dbd53dd640a Mon Sep 17 00:00:00 2001 From: Michael Prokop Date: Thu, 5 Apr 2018 11:26:24 +0200 Subject: [PATCH] TT#30565 Check for already running instances and abort if so In end of october we had many running instances because of aborted ec2-ami-ce Jenkins jobs. To ensure this doesn't happen again let's check for already running instances if we find any of them abort within ec2-create-ce before creating any ones. While this limits us to not have several of them running at the same time (within the same region and using the same tag!), this shouldn't be a problem for us for the time being. Change-Id: I5707f611c6c8a08664a1dbbef08332c36dd2150f --- ec2-create-ce | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ec2-create-ce b/ec2-create-ce index 2d3c331..f4030bd 100755 --- a/ec2-create-ce +++ b/ec2-create-ce @@ -293,6 +293,19 @@ if [ -z "$NGCP_VERSION" ] ; then bailout 1 fi +# ensure we don't have any builds running already, see TT#30565 +NUM_RUNNING_INSTANCES=$(ec2-describe-instances --region "${AWS_REGION}" \ + --filter "instance-state-name=running" \ + --filter "tag:Purpose=${TAG_PURPOSE}" 2>/dev/null | \ + grep -c '^INSTANCE') +if [ "$NUM_RUNNING_INSTANCES" -gt 0 ] ; then + echo "Error: there are running instances with tag $TAG_PURPOSE in region $AWS_REGION" >&2 + echo "Exiting to ensure we don't have instances running due to Jenkins job failures." >&2 + echo "Please ensure that those instances are stopped before trying to build new ones." >&2 + echo "In case of questions contact someone with according AWS permissions." >&2 + bailout 1 +fi + # if AMI_NAME is set to "none" then we get defaults from Jenkins, if # so then use the ngcp-ce-... naming schema if [ -n "$AMI_NAME" ] && [ "$AMI_NAME" != "none" ] ; then