From 05a235a318800a8aa86370abda2f01e8a07eb81c Mon Sep 17 00:00:00 2001 From: Yap Sok Ann Date: Fri, 4 Oct 2013 12:05:50 +0800 Subject: [PATCH] ec2_ami: Account for AWS's "eventual consistency" with AMI creation. Calling `ec2.get_image` right after `ec2.create_image` may raise error "InvalidAMIID.NotFound". This has happend roughly 1 time out of 10 for me. Other people has bitten by this too: - https://github.com/Netflix/aminator/commit/5707f100a07d3432027d1e7a3305567a2a7d2574 - http://stackoverflow.com/a/14794952 - https://bitbucket.org/utoolity/bamboo-aws-plugin/pull-request/22/baws-116-fix-ec2-image-task-failing-with/diff --- cloud/ec2_ami | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cloud/ec2_ami b/cloud/ec2_ami index 6713b2108db..37ccc834bd1 100644 --- a/cloud/ec2_ami +++ b/cloud/ec2_ami @@ -165,6 +165,7 @@ AWS_REGIONS = ['ap-northeast-1', 'us-west-2'] try: + import boto import boto.ec2 except ImportError: print "failed=True msg='boto required for this module'" @@ -195,8 +196,22 @@ def create_image(module, ec2): except boto.exception.BotoServerError, e: module.fail_json(msg = "%s: %s" % (e.error_code, e.error_message)) - # wait here until the image is gone - img = ec2.get_image(image_id) + # Wait until the image is recognized. EC2 API has eventual consistency, + # such that a successful CreateImage API call doesn't guarantee the success + # of subsequent DescribeImages API call using the new image id returned. + for i in range(30): + try: + img = ec2.get_image(image_id) + break + except boto.exception.EC2ResponseError as e: + if e.error_code == 'InvalidAMIID.NotFound': + time.sleep(1) + else: + raise + else: + module.fail_json(msg = "timed out waiting for image to be recognized") + + # wait here until the image is created wait_timeout = time.time() + wait_timeout while wait and wait_timeout > time.time() and (img is None or img.state != 'available'): img = ec2.get_image(image_id)