1require 'tmpdir' |
|
2require 'active_support/core_ext/string/strip' |
|
3require 'aws-sdk' |
|
4require 'faraday' |
|
5require 'slack-notifier' |
|
|
7class PackageBuilder |
8 class << self |
|
9 def build!(environment = :staging) |
|
10 new(environment).build!
|
|
11 end |
|
13 def deploy!(environment) |
|
14 new(environment).deploy!
|
|
15 end |
|
16 end |
|
18 attr_reader :environment, :release, :revision, :tmpdir, :timestamp |
|
|
20 def initialize(enviroment) |
21 @environment = enviroment.to_s |
|
22 @revision = %x[git rev-parse #{treeish}].strip |
|
23 @tmpdir = Dir.mktmpdir |
|
24 @timestamp = Time.now.getutc |
|
25 @release = @timestamp.strftime('%Y%m%d%H%M%S') |
|
26 end |
|
|
28 def build! |
29 info "Building to #{tmpdir}" |
|
31 create_archive
|
|
32 extract_archive
|
|
33 remove_archive
|
|
34 write_appspec
|
|
35 write_scripts
|
|
37 Dir.chdir archive_path do |
|
38 package_gems unless skip_gems? |
|
39 create_revision_file
|
|
40 remove_artifacts
|
|
41 end |
|
43 build_package
|
|
45 info "Built package #{package_name}" |
|
46 end |
|
|
48 def upload! |
49 s3 = Aws::S3::Resource.new(region: region, profile: profile) |
|
50 bucket = s3.bucket(release_bucket)
|
|
51 release_obj = bucket.object(release_key)
|
|
53 info "Uploading package #{package_name} to S3 ..." |
|
54 start_time = Time.now |
|
56 release_obj.upload_file(package_path)
|
|
58 duration = Time.now - start_time |
|
59 info "Upload completed in #{duration} seconds." |
|
60 end |
|
|
62 def deploy! |
63 WebMock.allow_net_connect! if defined? WebMock |
|
65 if ci? && !deploy_build? |
|
66 info "Skipping deployment ..." |
|
67 else |
|
68 unless skip_build? |
|
69 build!
|
|
70 upload!
|
|
71 end |
|
73 create_deployment! if deploy_release? |
|
74 end |
|
75 end |
|
77 private
|
|
|
79 def application_name |
80 "#{ENV.fetch('AWS_DEPLOYMENT_APP_NAME', 'epetitions')}-#{environment}" |
|
81 end |
|
|
83 def archive_file |
84 File.join(tmpdir, "#{archive_name}.tar") |
|
85 end |
|
|
87 def archive_name |
88 'source' |
|
89 end |
|
|
91 def archive_path |
92 File.join(tmpdir, archive_name) |
|
93 end |
|
|
95 def build_package |
96 Dir.mkdir('pkg') unless Dir.exist?('pkg') |
|
98 args = %w[tar] |
|
99 args.concat ['-cz'] |
|
100 args.concat ['-f', package_path] |
|
101 args.concat ['-C', tmpdir] |
|
102 args.concat ['.'] |
|
104 info "Building package ..." |
|
105 Kernel.system *args |
|
106 end |
|
|
108 def ci? |
109 ENV.fetch('CI', 'false') == 'true' |
|
110 end |
|
|
112 def create_archive |
113 args = %w[git archive] |
|
114 args.concat ['--format', 'tar'] |
|
115 args.concat ['--prefix', 'source/'] |
|
116 args.concat ['--output', archive_file] |
|
117 args.concat [treeish]
|
|
119 info "Creating archive ..." |
|
120 Kernel.system *args |
|
121 end |
|
|
123 def create_deployment! |
124 client = Aws::CodeDeploy::Client.new(credentials) |
|
125 response = client.create_deployment(deployment_config)
|
|
126 info "Deployment created." |
|
128 track_progress(response.deployment_id) do |deployment_info| |
|
129 notify_appsignal
|
|
130 notify_slack
|
|
131 end |
|
132 end |
|
|
134 def create_revision_file |
135 File.write(revision_file, revision) |
|
136 end |
|
|
138 def credentials |
139 { region: region, profile: profile } |
|
140 end |
|
|
142 def deploy_branch? |
143 ENV.fetch('TRAVIS_BRANCH', 'master') == 'master' |
|
144 end |
|
|
146 def deploy_build? |
147 !pull_request? && deploy_branch?
|
|
148 end |
|
|
150 def deployment_config |
151 {
|
|
152 application_name: application_name, |
|
153 deployment_group_name: deployment_group_name, |
|
154 revision: { |
|
155 revision_type: 'S3', |
|
156 s3_location: { |
|
157 bucket: release_bucket, |
|
158 key: deployment_key, |
|
159 bundle_type: 'tgz' |
|
160 },
|
|
161 },
|
|
162 deployment_config_name: deployment_config_name, |
|
163 description: description, |
|
164 ignore_application_stop_failures: true |
|
165 }
|
|
166 end |
|
|
168 def deployment_config_name |
169 type = ENV.fetch('AWS_DEPLOYMENT_CONFIG_NAME', '0') |
|
171 case type |
|
172 when '2' |
|
173 'CodeDeployDefault.AllAtOnce' |
|
174 when '1' |
|
175 'CodeDeployDefault.HalfAtATime' |
|
176 else |
|
177 'CodeDeployDefault.OneAtATime' |
|
178 end |
|
179 end |
|
|
181 def deployment_group_name |
182 ENV.fetch('AWS_DEPLOYMENT_GROUP_NAME', 'RailsAppServers') |
|
183 end |
|
|
185 def deployment_key |
186 skip_build? ? latest_key : release_key
|
|
187 end |
|
|
189 def description |
190 ENV.fetch('AWS_DEPLOYMENT_DESCRIPTION', '') |
|
191 end |
|
|
193 def extract_archive |
194 args = %w[tar] |
|
195 args.concat ['-C', tmpdir] |
|
196 args.concat ['-xf', archive_file] |
|
198 info "Extracting archive ..." |
|
199 Kernel.system *args |
|
200 end |
|
|
202 def info(message) |
203 $stdout.puts(message) |
|
204 end |
|
|
206 def latest_key |
207 "/latest.tar.gz" |
|
208 end |
|
|
210 def package_gems |
211 args = %w[bundle package --all --all-platforms] |
|
213 info "Packaging gems ..." |
|
214 Bundler.with_clean_env do |
|
215 Kernel.system *args |
|
216 end |
|
217 end |
|
|
219 def package_name |
220 "#{timestamp.strftime('%Y%m%d%H%I%S')}.tar.gz" |
|
221 end |
|
|
223 def package_path |
224 File.join('pkg', package_name) |
|
225 end |
|
|
227 def profile |
228 ENV.fetch('AWS_PROFILE', 'epetitions') |
|
229 end |
|
|
231 def deploy_release? |
232 ENV.fetch('RELEASE', '1').to_i.nonzero? |
|
233 end |
|
|
235 def notify_appsignal |
236 if appsignal_push_api_key |
|
237 conn = Faraday.new(url: "https://push.appsignal.com") |
|
239 response = conn.post do |request| |
|
240 request.url '/1/markers' |
|
242 request.headers['Content-Type'] = 'application/json' |
|
244 request.params = {
|
|
245 api_key: appsignal_push_api_key, |
|
246 name: application_name, |
|
247 environment: 'production' |
|
248 }
|
|
251 {
|
|
252 "revision": "#{revision}", |
|
253 "repository": "master", |
|
254 "user": "#{username}" |
|
255 }
|
|
256 JSON |
|
257 end |
|
259 if response.success? |
|
260 info "Notified AppSignal of deployment of #{revision}" |
|
261 end |
|
262 end |
|
263 end |
|
|
265 def appsignal_push_api_key |
266 ENV.fetch('APPSIGNAL_PUSH_API_KEY', nil) |
|
267 end |
|
|
269 def username |
270 ENV['USER'] || ENV['USERNAME'] || 'unknown' |
|
271 end |
|
|
273 def notify_slack |
274 if slack_webhook |
|
275 notifier = Slack::Notifier.new(slack_webhook) |
|
276 notifier.ping slack_message, slack_options
|
|
277 end |
|
278 end |
|
|
280 def slack_webhook |
281 ENV.fetch('SLACK_WEBHOOK_URL', nil) |
|
282 end |
|
|
284 def slack_message |
285 "Deployed revision <#{commit_url}|#{short_revision}> to <#{website_url}>" |
|
286 end |
|
|
288 def slack_options |
289 { channel: '#epetitions', username: 'deploy', icon_emoji: ':tada:' } |
|
290 end |
|
|
292 def pull_request? |
293 ENV.fetch('TRAVIS_PULL_REQUEST', 'false') != 'false' |
|
294 end |
|
|
296 def region |
297 ENV.fetch('AWS_REGION', 'eu-west-1') |
|
298 end |
|
|
300 def release_bucket |
301 "epetitions-#{environment}-releases" |
|
302 end |
|
|
304 def release_key |
305 "#{release}.tar.gz" |
|
306 end |
|
|
308 def remove_archive |
309 args = %w[rm] |
|
310 args.concat [archive_file]
|
|
312 info "Removing archive ..." |
|
313 Kernel.system *args |
|
314 end |
|
|
316 def remove_artifacts |
317 args = %w[rm -rf] |
|
318 args.concat %w[.bundle log tmp] |
|
320 info "Removing build artifacts ..." |
|
321 Kernel.system *args |
|
322 end |
|
|
324 def revision_file |
325 File.join(archive_path, 'REVISION') |
|
326 end |
|
|
328 def short_revision |
329 revision.first(7) |
|
330 end |
|
|
332 def commit_url |
333 "https://github.com/alphagov/e-petitions/commit/#{revision}" |
|
334 end |
|
|
336 def website_url |
337 if environment == "production" |
|
338 "https://petition.parliament.uk/" |
|
339 else |
|
340 "https://#{environment}.epetitions.website/" |
|
341 end |
|
342 end |
|
|
344 def skip_build? |
345 ENV.fetch('SKIP_BUILD', '0').to_i.nonzero? |
|
346 end |
|
|
348 def skip_gems? |
349 ENV.fetch('SKIP_GEMS', '0').to_i.nonzero? |
|
350 end |
|
|
352 def track_progress(deployment_id, &block) |
353 client = Aws::CodeDeploy::Client.new(credentials) |
|
354 completed = false |
|
356 while !completed do |
|
357 response = client.get_deployment(deployment_id: deployment_id) |
|
359 if response.successful? |
|
360 deployment = response.deployment_info
|
|
361 status = deployment.status
|
|
362 completed = !deployment.complete_time.nil?
|
|
364 if completed |
|
365 deployment_complete(deployment)
|
|
366 else |
|
367 if status == "InProgress" |
|
368 deployment_progress(deployment)
|
|
369 end |
|
371 sleep(5) |
|
372 end |
|
374 yield deployment if status == "Succeeded" |
|
375 else |
|
376 raise RuntimeError, "Error getting status for deployment: #{deployment_id}" |
|
377 end |
|
378 end |
|
379 end |
|
|
381 def deployment_complete(deployment) |
382 id = deployment.deployment_id
|
|
383 created_at = deployment.create_time
|
|
384 completed_at = deployment.complete_time
|
|
385 duration = completed_at - created_at
|
|
386 status = deployment.status.downcase
|
|
388 info ("Deployment %s %s in %0.2f seconds" % [id, status, duration]) |
|
389 end |
|
|
391 def deployment_progress(deployment) |
392 id = deployment.deployment_id
|
|
393 created_at = deployment.create_time
|
|
394 duration = Time.current - created_at |
|
395 overview = deployment.deployment_overview
|
|
396 progress = "Pending: %d, InProgress: %d, Succeeded: %d, Failed: %d, Skipped: %d" % overview.values |
|
398 info ("Deploying %s (%s) in %0.2f seconds" % [id, progress, duration]) |
|
399 end |
|
|
401 def treeish |
402 ENV['TAG'] || ENV['BRANCH'] || 'HEAD' |
|
403 end |
|
|
405 def write_appspec |
406 File.write(appspec_file, appspec_yaml) |
|
407 end |
|
|
409 def write_scripts |
410 Dir.mkdir(scripts_path) unless Dir.exist?(scripts_path) |
|
412 write_script(application_start_script_file, application_start_script)
|
|
413 write_script(application_stop_script_file, application_stop_script)
|
|
414 write_script(after_install_script_file, after_install_script)
|
|
415 write_script(common_functions_script_file, common_functions_script)
|
|
416 write_script(deregister_from_elb_script_file, deregister_from_elb_script)
|
|
417 write_script(register_with_elb_script_file, register_with_elb_script)
|
|
418 end |
|
|
420 def write_script(path, script, mode = 0755) |
421 File.write(path, script) |
|
422 File.new(path).chmod(mode) |
|
423 end |
|
|
425 def scripts_path |
426 File.join(tmpdir, 'scripts') |
|
427 end |
|
|
429 def appspec_file |
430 File.join(tmpdir, 'appspec.yml') |
|
431 end |
|
|
433 def appspec_yaml |
435 version: 0.0 |
|
436 os: linux |
|
437 files: |
|
438 - source: ./source |
|
439 destination: /home/deploy/epetitions/releases/#{release} |
|
441 hooks: |
|
442 ApplicationStop: |
|
443 - location: scripts/deregister_from_elb |
|
444 runas: root |
|
445 - location: scripts/application_stop |
|
446 runas: root |
|
447 AfterInstall: |
|
448 - location: scripts/after_install |
|
449 runas: root |
|
450 ApplicationStart: |
|
451 - location: scripts/application_start |
|
452 runas: root |
|
453 - location: scripts/register_with_elb |
|
454 runas: root |
|
455 FILE |
|
456 end |
|
|
458 def application_start_script_file |
459 File.join(tmpdir, 'scripts', 'application_start') |
|
460 end |
|
|
462 def application_start_script |
464 #!/usr/bin/env bash |
|
465 /etc/init.d/epetitions start |
|
466 SCRIPT |
|
467 end |
|
|
469 def application_stop_script_file |
470 File.join(tmpdir, 'scripts', 'application_stop') |
|
471 end |
|
|
473 def application_stop_script |
475 #!/usr/bin/env bash |
|
476 /etc/init.d/epetitions stop || true |
|
477 SCRIPT |
|
478 end |
|
|
480 def after_install_script_file |
481 File.join(tmpdir, 'scripts', 'after_install') |
|
482 end |
|
|
484 def after_install_script |
486 #!/usr/bin/env bash |
|
487 chown -R deploy:deploy /home/deploy/epetitions/releases/#{release} |
|
489 if [ ! -e /home/deploy/epetitions/releases/#{release}/tmp ]; then |
|
490 su - deploy -c 'mkdir /home/deploy/epetitions/releases/#{release}/tmp' |
|
491 fi
|
|
493 su - deploy -c 'ln -nfs /home/deploy/epetitions/shared/log /home/deploy/epetitions/releases/#{release}/log' |
|
494 su - deploy -c 'ln -nfs /home/deploy/epetitions/shared/bundle /home/deploy/epetitions/releases/#{release}/vendor/bundle' |
|
495 su - deploy -c 'ln -nfs /home/deploy/epetitions/shared/assets /home/deploy/epetitions/releases/#{release}/public/assets' |
|
496 su - deploy -c 'ln -s /home/deploy/epetitions/releases/#{release} /home/deploy/epetitions/current_#{release}' |
|
497 su - deploy -c 'mv -Tf /home/deploy/epetitions/current_#{release} /home/deploy/epetitions/current' |
|
498 su - deploy -c 'cd /home/deploy/epetitions/current && bundle install --without development test --deployment --quiet' |
|
499 su - deploy -c 'cd /home/deploy/epetitions/current && bundle exec rake db:migrate' |
|
500 su - deploy -c 'cd /home/deploy/epetitions/current && bundle exec rake assets:precompile' |
|
502 # Run cron jobs only on workers, as webservers autoscale up and down. |
|
503 # ${SERVER_TYPE} is pre-populated for the deploy user by the build scripts |
|
504 su - deploy -c 'if [ ${SERVER_TYPE} = "worker" ] ; then cd /home/deploy/epetitions/current && bundle exec whenever -w ; else echo not running whenever ; fi' |
|
505 SCRIPT |
|
506 end |
|
|
508 def common_functions_script_file |
509 File.join(tmpdir, 'scripts', 'common_functions') |
|
510 end |
|
|
512 def deregister_from_elb_script_file |
513 File.join(tmpdir, 'scripts', 'deregister_from_elb') |
|
514 end |
|
|
516 def register_with_elb_script_file |
517 File.join(tmpdir, 'scripts', 'register_with_elb') |
|
518 end |
|
|
520 def common_functions_script |
522 #!/usr/bin/env bash |
|
523 # |
|
524 # Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
|
525 # |
|
526 # Licensed under the Apache License, Version 2.0 (the "License"). |
|
527 # You may not use this file except in compliance with the License. |
|
528 # A copy of the License is located at |
|
529 # |
|
530 # http://aws.amazon.com/apache2.0 |
|
531 # |
|
532 # or in the "license" file accompanying this file. This file is distributed |
|
533 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either |
|
534 # express or implied. See the License for the specific language governing |
|
535 # permissions and limitations under the License. |
|
537 # ELB_LIST defines which Elastic Load Balancers this instance should be part of. |
|
538 ELB_LIST="$ELB_NAME" |
|
540 # Under normal circumstances, you shouldn't need to change anything below this line. |
|
541 # ----------------------------------------------------------------------------- |
|
543 export PATH="$PATH:/usr/bin:/usr/local/bin" |
|
545 # If true, all messages will be printed. If false, only fatal errors are printed. |
|
546 DEBUG=true |
|
548 # Number of times to check for a resouce to be in the desired state. |
|
549 WAITER_ATTEMPTS=100 |
|
551 # Number of seconds to wait between attempts for resource to be in a state. |
|
552 WAITER_INTERVAL=3 |
|
554 # AutoScaling Standby features at minimum require this version to work. |
|
555 MIN_CLI_VERSION='1.3.25' |
|
557 # Usage: get_instance_region |
|
558 # |
|
559 # Writes to STDOUT the AWS region as known by the local instance. |
|
560 get_instance_region() {
|
|
561 if [ -z "$AWS_REGION" ]; then |
|
562 AWS_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ |
|
563 | grep -i region \
|
|
564 | awk -F'"' '{print $4}') |
|
565 fi
|
|
567 echo $AWS_REGION |
|
568 }
|
|
570 AWS_CLI="aws --region $(get_instance_region)" |
|
572 # Usage: autoscaling_group_name <EC2 instance ID> |
|
573 # |
|
574 # Prints to STDOUT the name of the AutoScaling group this instance is a part of and returns 0. If |
|
575 # it is not part of any groups, then it prints nothing. On error calling autoscaling, returns |
|
576 # non-zero. |
|
577 autoscaling_group_name() {
|
|
578 local instance_id=$1 |
|
580 # This operates under the assumption that instances are only ever part of a single ASG. |
|
581 local autoscaling_name=$($AWS_CLI autoscaling describe-auto-scaling-instances \ |
|
582 --instance-ids $instance_id \ |
|
583 --output text \
|
|
584 --query AutoScalingInstances[0].AutoScalingGroupName) |
|
586 if [ $? != 0 ]; then |
|
587 return 1 |
|
588 elif [ "$autoscaling_name" == "None" ]; then |
|
589 echo "" |
|
590 else |
|
591 echo $autoscaling_name |
|
592 fi
|
|
594 return 0 |
|
595 }
|
|
597 # Usage: autoscaling_enter_standby <EC2 instance ID> <ASG name> |
|
598 # |
|
599 # Move <EC2 instance ID> into the Standby state in AutoScaling group <ASG name>. Doing so will |
|
600 # pull it out of any Elastic Load Balancer that might be in front of the group. |
|
601 # |
|
602 # Returns 0 if the instance was successfully moved to standby. Non-zero otherwise. |
|
603 autoscaling_enter_standby() {
|
|
604 local instance_id=$1 |
|
605 local asg_name=$2 |
|
607 msg "Checking if this instance has already been moved in the Standby state" |
|
608 local instance_state=$(get_instance_state_asg $instance_id) |
|
609 if [ $? != 0 ]; then |
|
610 msg "Unable to get this instance's lifecycle state." |
|
611 return 1 |
|
612 fi
|
|
614 if [ "$instance_state" == "Standby" ]; then |
|
615 msg "Instance is already in Standby; nothing to do." |
|
616 return 0 |
|
617 fi
|
|
619 if [ "$instance_state" == "Pending:Wait" ]; then |
|
620 msg "Instance is Pending:Wait; nothing to do." |
|
621 return 0 |
|
622 fi
|
|
624 msg "Checking to see if ASG $asg_name will let us decrease desired capacity" |
|
625 local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \ |
|
626 --auto-scaling-group-name $asg_name \ |
|
627 --query 'AutoScalingGroups[0].[MinSize, DesiredCapacity]' \ |
|
628 --output text)
|
|
630 local min_cap=$(echo $min_desired | awk '{print $1}') |
|
631 local desired_cap=$(echo $min_desired | awk '{print $2}') |
|
633 if [ -z "$min_cap" -o -z "$desired_cap" ]; then |
|
634 msg "Unable to determine minimum and desired capacity for ASG $asg_name." |
|
635 msg "Attempting to put this instance into standby regardless." |
|
636 elif [ $min_cap == $desired_cap -a $min_cap -gt 0 ]; then |
|
637 local new_min=$(($min_cap - 1)) |
|
638 msg "Decrementing ASG $asg_name's minimum size to $new_min" |
|
639 msg $($AWS_CLI autoscaling update-auto-scaling-group \ |
|
640 --auto-scaling-group-name $asg_name \ |
|
641 --min-size $new_min) |
|
642 if [ $? != 0 ]; then |
|
643 msg "Failed to reduce ASG $asg_name's minimum size to $new_min. Cannot put this instance into Standby." |
|
644 return 1 |
|
645 fi
|
|
646 fi
|
|
648 msg "Putting instance $instance_id into Standby" |
|
649 $AWS_CLI autoscaling enter-standby \ |
|
650 --instance-ids $instance_id \ |
|
651 --auto-scaling-group-name $asg_name \ |
|
652 --should-decrement-desired-capacity
|
|
653 if [ $? != 0 ]; then |
|
654 msg "Failed to put instance $instance_id into Standby for ASG $asg_name." |
|
655 return 1 |
|
656 fi
|
|
658 msg "Waiting for move to Standby to finish" |
|
659 wait_for_state "autoscaling" $instance_id "Standby" |
|
660 if [ $? != 0 ]; then |
|
661 local wait_timeout=$(($WAITER_INTERVAL * $WAITER_ATTEMPTS)) |
|
662 msg "Instance $instance_id did not make it to standby after $wait_timeout seconds" |
|
663 return 1 |
|
664 fi
|
|
666 return 0 |
|
667 }
|
|
669 # Usage: autoscaling_exit_standby <EC2 instance ID> <ASG name> |
|
670 # |
|
671 # Attempts to move instance <EC2 instance ID> out of Standby and into InService. Returns 0 if |
|
672 # successful. |
|
673 autoscaling_exit_standby() {
|
|
674 local instance_id=$1 |
|
675 local asg_name=$2 |
|
677 msg "Checking if this instance has already been moved out of Standby state" |
|
678 local instance_state=$(get_instance_state_asg $instance_id) |
|
679 if [ $? != 0 ]; then |
|
680 msg "Unable to get this instance's lifecycle state." |
|
681 return 1 |
|
682 fi
|
|
684 if [ "$instance_state" == "InService" ]; then |
|
685 msg "Instance is already InService; nothing to do." |
|
686 return 0 |
|
687 fi
|
|
689 if [ "$instance_state" == "Pending:Wait" ]; then |
|
690 msg "Instance is Pending:Wait; nothing to do." |
|
691 return 0 |
|
692 fi
|
|
694 msg "Moving instance $instance_id out of Standby" |
|
695 $AWS_CLI autoscaling exit-standby \ |
|
696 --instance-ids $instance_id \ |
|
697 --auto-scaling-group-name $asg_name |
|
698 if [ $? != 0 ]; then |
|
699 msg "Failed to put instance $instance_id back into InService for ASG $asg_name." |
|
700 return 1 |
|
701 fi
|
|
703 msg "Waiting for exit-standby to finish" |
|
704 wait_for_state "autoscaling" $instance_id "InService" |
|
705 if [ $? != 0 ]; then |
|
706 local wait_timeout=$(($WAITER_INTERVAL * $WAITER_ATTEMPTS)) |
|
707 msg "Instance $instance_id did not make it to InService after $wait_timeout seconds" |
|
708 return 1 |
|
709 fi
|
|
711 return 0 |
|
712 }
|
|
714 # Usage: get_instance_state_asg <EC2 instance ID> |
|
715 # |
|
716 # Gets the state of the given <EC2 instance ID> as known by the AutoScaling group it's a part of. |
|
717 # Health is printed to STDOUT and the function returns 0. Otherwise, no output and return is |
|
718 # non-zero. |
|
719 get_instance_state_asg() {
|
|
720 local instance_id=$1 |
|
722 local state=$($AWS_CLI autoscaling describe-auto-scaling-instances \ |
|
723 --instance-ids $instance_id \ |
|
724 --query "AutoScalingInstances[?InstanceId == \'$instance_id\'].LifecycleState | [0]" \ |
|
725 --output text)
|
|
726 if [ $? != 0 ]; then |
|
727 return 1 |
|
728 else |
|
729 echo $state |
|
730 return 0 |
|
731 fi
|
|
732 }
|
|
734 reset_waiter_timeout() {
|
|
735 local elb=$1 |
|
737 local health_check_values=$($AWS_CLI elb describe-load-balancers \ |
|
738 --load-balancer-name $elb \ |
|
739 --query 'LoadBalancerDescriptions[0].HealthCheck.[HealthyThreshold, Interval]' \ |
|
740 --output text)
|
|
742 WAITER_ATTEMPTS=$(echo $health_check_values | awk '{print $1}') |
|
743 WAITER_INTERVAL=$(echo $health_check_values | awk '{print $2}') |
|
744 }
|
|
746 # Usage: wait_for_state <service> <EC2 instance ID> <state name> [ELB name] |
|
747 # |
|
748 # Waits for the state of <EC2 instance ID> to be in <state> as seen by <service>. Returns 0 if |
|
749 # it successfully made it to that state; non-zero if not. By default, checks $WAITER_ATTEMPTS |
|
750 # times, every $WAITER_INTERVAL seconds. If giving an [ELB name] to check under, these are reset |
|
751 # to that ELB's HealthThreshold and Interval values. |
|
752 wait_for_state() {
|
|
753 local service=$1 |
|
754 local instance_id=$2 |
|
755 local state_name=$3 |
|
756 local elb=$4 |
|
758 local instance_state_cmd
|
|
759 if [ "$service" == "elb" ]; then |
|
760 instance_state_cmd="get_instance_health_elb $instance_id $elb" |
|
761 reset_waiter_timeout $elb |
|
762 elif [ "$service" == "autoscaling" ]; then |
|
763 instance_state_cmd="get_instance_state_asg $instance_id" |
|
764 else |
|
765 msg "Cannot wait for instance state; unknown service type, '$service'" |
|
766 return 1 |
|
767 fi
|
|
769 msg "Checking $WAITER_ATTEMPTS times, every $WAITER_INTERVAL seconds, for instance $instance_id to be in state $state_name" |
|
771 local instance_state=$($instance_state_cmd) |
|
772 local count=1 |
|
774 msg "Instance is currently in state: $instance_state" |
|
775 while [ "$instance_state" != "$state_name" ]; do |
|
776 if [ $count -ge $WAITER_ATTEMPTS ]; then |
|
777 local timeout=$(($WAITER_ATTEMPTS * $WAITER_INTERVAL)) |
|
778 msg "Instance failed to reach state, $state_name within $timeout seconds" |
|
779 return 1 |
|
780 fi
|
|
782 sleep $WAITER_INTERVAL |
|
784 instance_state=$($instance_state_cmd) |
|
785 count=$(($count + 1)) |
|
786 msg "Instance is currently in state: $instance_state" |
|
787 done
|
|
789 return 0 |
|
790 }
|
|
792 # Usage: get_instance_health_elb <EC2 instance ID> <ELB name> |
|
793 # |
|
794 # Gets the health of the given <EC2 instance ID> as known by <ELB name>. If it's a valid health |
|
795 # status (one of InService|OutOfService|Unknown), then the health is printed to STDOUT and the |
|
796 # function returns 0. Otherwise, no output and return is non-zero. |
|
797 get_instance_health_elb() {
|
|
798 local instance_id=$1 |
|
799 local elb_name=$2 |
|
801 msg "Checking status of instance '$instance_id' in load balancer '$elb_name'" |
|
803 # If describe-instance-health for this instance returns an error, then it's not part of |
|
804 # this ELB. But, if the call was successful let's still double check that the status is |
|
805 # valid. |
|
806 local instance_status=$($AWS_CLI elb describe-instance-health \ |
|
807 --load-balancer-name $elb_name \ |
|
808 --instances $instance_id \ |
|
809 --query 'InstanceStates[].State' \ |
|
810 --output text 2>/dev/null) |
|
812 if [ $? == 0 ]; then |
|
813 case "$instance_status" in |
|
814 InService|OutOfService|Unknown) |
|
815 echo -n $instance_status |
|
816 return 0 |
|
817 ;;
|
|
818 *)
|
|
819 msg "Instance '$instance_id' not part of ELB '$elb_name'" |
|
820 return 1 |
|
821 esac
|
|
822 fi
|
|
823 }
|
|
825 # Usage: validate_elb <EC2 instance ID> <ELB name> |
|
826 # |
|
827 # Validates that the Elastic Load Balancer with name <ELB name> exists, is describable, and |
|
828 # contains <EC2 instance ID> as one of its instances. |
|
829 # |
|
830 # If any of these checks are false, the function returns non-zero. |
|
831 validate_elb() {
|
|
832 local instance_id=$1 |
|
833 local elb_name=$2 |
|
835 # Get the list of active instances for this LB. |
|
836 local elb_instances=$($AWS_CLI elb describe-load-balancers \ |
|
837 --load-balancer-name $elb_name \ |
|
838 --query 'LoadBalancerDescriptions[*].Instances[*].InstanceId' \ |
|
839 --output text)
|
|
840 if [ $? != 0 ]; then |
|
841 msg "Couldn't describe ELB instance named '$elb_name'" |
|
842 return 1 |
|
843 fi
|
|
845 msg "Checking health of '$instance_id' as known by ELB '$elb_name'" |
|
846 local instance_health=$(get_instance_health_elb $instance_id $elb_name) |
|
847 if [ $? != 0 ]; then |
|
848 return 1 |
|
849 fi
|
|
851 return 0 |
|
852 }
|
|
854 # Usage: get_elb_list <EC2 instance ID> |
|
855 # |
|
856 # Finds all the ELBs that this instance is registered to. After execution, the variable |
|
857 # "INSTANCE_ELBS" will contain the list of load balancers for the given instance. |
|
858 # |
|
859 # If the given instance ID isn't found registered to any ELBs, the function returns non-zero |
|
860 get_elb_list() {
|
|
861 local instance_id=$1 |
|
863 local elb_list="" |
|
865 local all_balancers=$($AWS_CLI elb describe-load-balancers \ |
|
866 --query LoadBalancerDescriptions[*].LoadBalancerName \ |
|
869 for elb in $all_balancers; do |
|
870 local instance_health
|
|
871 instance_health=$(get_instance_health_elb $instance_id $elb) |
|
872 if [ $? == 0 ]; then |
|
873 elb_list="$elb_list $elb" |
|
874 fi
|
|
875 done
|
|
877 if [ -z "$elb_list" ]; then |
|
878 return 1 |
|
879 else |
|
880 msg "Got load balancer list of: $elb_list" |
|
881 INSTANCE_ELBS=$elb_list |
|
882 return 0 |
|
883 fi
|
|
884 }
|
|
886 # Usage: deregister_instance <EC2 instance ID> <ELB name> |
|
887 # |
|
888 # Deregisters <EC2 instance ID> from <ELB name>. |
|
889 deregister_instance() {
|
|
890 local instance_id=$1 |
|
891 local elb_name=$2 |
|
893 $AWS_CLI elb deregister-instances-from-load-balancer \ |
|
894 --load-balancer-name $elb_name \ |
|
895 --instances $instance_id 1> /dev/null |
|
897 return $? |
|
898 }
|
|
900 # Usage: register_instance <EC2 instance ID> <ELB name> |
|
901 # |
|
902 # Registers <EC2 instance ID> to <ELB name>. |
|
903 register_instance() {
|
|
904 local instance_id=$1 |
|
905 local elb_name=$2 |
|
907 $AWS_CLI elb register-instances-with-load-balancer \ |
|
908 --load-balancer-name $elb_name \ |
|
909 --instances $instance_id 1> /dev/null |
|
911 return $? |
|
912 }
|
|
914 # Usage: check_cli_version [version-to-check] [desired version] |
|
915 # |
|
916 # Without any arguments, checks that the installed version of the AWS CLI is at least at version |
|
917 # $MIN_CLI_VERSION. Returns non-zero if the version is not high enough. |
|
918 check_cli_version() {
|
|
919 if [ -z $1 ]; then |
|
920 version=$($AWS_CLI --version 2>&1 | cut -f1 -d' ' | cut -f2 -d/) |
|
921 else |
|
922 version=$1 |
|
923 fi
|
|
925 if [ -z "$2" ]; then |
|
926 min_version=$MIN_CLI_VERSION |
|
927 else |
|
928 min_version=$2 |
|
929 fi
|
|
931 x=$(echo $version | cut -f1 -d.) |
|
932 y=$(echo $version | cut -f2 -d.) |
|
933 z=$(echo $version | cut -f3 -d.) |
|
935 min_x=$(echo $min_version | cut -f1 -d.) |
|
936 min_y=$(echo $min_version | cut -f2 -d.) |
|
937 min_z=$(echo $min_version | cut -f3 -d.) |
|
939 msg "Checking minimum required CLI version (${min_version}) against installed version ($version)" |
|
941 if [ $x -lt $min_x ]; then |
|
942 return 1 |
|
943 elif [ $y -lt $min_y ]; then |
|
944 return 1 |
|
945 elif [ $y -gt $min_y ]; then |
|
946 return 0 |
|
947 elif [ $z -ge $min_z ]; then |
|
948 return 0 |
|
|
949 else |
950 return 1 |
|
951 fi
|
|
952 }
|
|
954 # Usage: msg <message> |
|
955 # |
|
956 # Writes <message> to STDERR only if $DEBUG is true, otherwise has no effect. |
|
957 msg() {
|
|
958 local message=$1 |
|
959 $DEBUG && echo $message 1>&2 |
|
960 }
|
|
962 # Usage: error_exit <message> |
|
963 # |
|
964 # Writes <message> to STDERR as a "fatal" and immediately exits the currently running script. |
|
965 error_exit() {
|
|
966 local message=$1 |
|
968 echo "[FATAL] $message" 1>&2 |
|
969 exit 1 |
|
970 }
|
|
972 # Usage: get_instance_id |
|
973 # |
|
974 # Writes to STDOUT the EC2 instance ID for the local instance. Returns non-zero if the local |
|
975 # instance metadata URL is inaccessible. |
|
976 get_instance_id() {
|
|
977 curl -s http://169.254.169.254/latest/meta-data/instance-id |
|
978 return $? |
|
979 }
|
|
980 SCRIPT |
|
981 end |
|
983 def deregister_from_elb_script |
|
985 #!/usr/bin/env bash |
|
986 # |
|
987 # Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
|
988 # |
|
989 # Licensed under the Apache License, Version 2.0 (the "License"). |
|
990 # You may not use this file except in compliance with the License. |
|
991 # A copy of the License is located at |
|
992 # |
|
993 # http://aws.amazon.com/apache2.0 |
|
994 # |
|
995 # or in the "license" file accompanying this file. This file is distributed |
|
996 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either |
|
997 # express or implied. See the License for the specific language governing |
|
998 # permissions and limitations under the License. |
|
1000 if [ "$SERVER_TYPE" == "worker" ]; then |
|
1001 msg "Workers are not registered with a load balancer" |
|
1002 exit 0 |
|
1003 fi
|
|
1005 . $(dirname $0)/common_functions |
|
1007 msg "Running AWS CLI with region: $(get_instance_region)" |
|
1009 # get this instance's ID |
|
1010 INSTANCE_ID=$(get_instance_id) |
|
1011 if [ $? != 0 -o -z "$INSTANCE_ID" ]; then |
|
1012 error_exit "Unable to get this instance's ID; cannot continue." |
|
1013 fi
|
|
1015 # Get current time |
|
1016 msg "Started $(basename $0) at $(/bin/date "+%F %T")" |
|
1019 msg "Checking if instance $INSTANCE_ID is part of an AutoScaling group" |
|
1020 asg=$(autoscaling_group_name $INSTANCE_ID) |
|
1021 if [ $? == 0 -a -n "$asg" ]; then |
|
1022 msg "Found AutoScaling group for instance $INSTANCE_ID: $asg" |
|
1024 msg "Checking that installed CLI version is at least at version required for AutoScaling Standby" |
|
1025 check_cli_version
|
|
1026 if [ $? != 0 ]; then |
|
1027 error_exit "CLI must be at least version ${MIN_CLI_X}.${MIN_CLI_Y}.${MIN_CLI_Z} to work with AutoScaling Standby" |
|
1028 fi
|
|
1030 msg "Attempting to put instance into Standby" |
|
1031 autoscaling_enter_standby $INSTANCE_ID $asg |
|
1032 if [ $? != 0 ]; then |
|
1033 error_exit "Failed to move instance into standby" |
|
1034 else |
|
1035 msg "Instance is in standby" |
|
1036 exit 0 |
|
1037 fi
|
|
1038 fi
|
|
1040 msg "Instance is not part of an ASG, continuing..." |
|
1042 msg "Checking that user set at least one load balancer" |
|
1043 if test -z "$ELB_LIST"; then |
|
1044 error_exit "Must have at least one load balancer to deregister from" |
|
1045 fi
|
|
1047 # Loop through all LBs the user set, and attempt to deregister this instance from them. |
|
|
1048 for elb in $ELB_LIST; do |
1049 msg "Checking validity of load balancer named '$elb'" |
|
1050 validate_elb $INSTANCE_ID $elb |
|
1051 if [ $? != 0 ]; then |
|
1052 msg "Error validating $elb; cannot continue with this LB" |
|
1053 continue
|
|
1054 fi
|
|
1056 msg "Deregistering $INSTANCE_ID from $elb" |
|
1057 deregister_instance $INSTANCE_ID $elb |
|
1059 if [ $? != 0 ]; then |
|
1060 error_exit "Failed to deregister instance $INSTANCE_ID from ELB $elb" |
|
1061 fi
|
|
1062 done
|
|
1064 # Wait for all Deregistrations to finish |
|
1065 msg "Waiting for instance to de-register from its load balancers" |
|
1066 for elb in $ELB_LIST; do |
|
1067 wait_for_state "elb" $INSTANCE_ID "OutOfService" $elb |
|
1068 if [ $? != 0 ]; then |
|
1069 error_exit "Failed waiting for $INSTANCE_ID to leave $elb" |
|
1070 fi
|
|
1071 done
|
|
1073 msg "Finished $(basename $0) at $(/bin/date "+%F %T")" |
|
1076 elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) |
|
1078 msg "Elapsed time: $elapsed_seconds" |
|
1079 SCRIPT |
|
1080 end |
|
1082 def register_with_elb_script |
|
1084 #!/usr/bin/env bash |
|
1085 # |
|
1086 # Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
|
1087 # |
|
1088 # Licensed under the Apache License, Version 2.0 (the "License"). |
|
1089 # You may not use this file except in compliance with the License. |
|
1090 # A copy of the License is located at |
|
1091 # |
|
1092 # http://aws.amazon.com/apache2.0 |
|
1093 # |
|
1094 # or in the "license" file accompanying this file. This file is distributed |
|
1095 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either |
|
1096 # express or implied. See the License for the specific language governing |
|
1097 # permissions and limitations under the License. |
|
1099 if [ "$SERVER_TYPE" == "worker" ]; then |
|
1100 msg "Workers are not registered with a load balancer" |
|
1101 exit 0 |
|
1102 fi
|
|
1104 . $(dirname $0)/common_functions |
|
1106 msg "Running AWS CLI with region: $(get_instance_region)" |
|
1108 # get this instance's ID |
|
1109 INSTANCE_ID=$(get_instance_id) |
|
1110 if [ $? != 0 -o -z "$INSTANCE_ID" ]; then |
|
1111 error_exit "Unable to get this instance's ID; cannot continue." |
|
1112 fi
|
|
1114 # Get current time |
|
1115 msg "Started $(basename $0) at $(/bin/date "+%F %T")" |
|
1118 msg "Checking if instance $INSTANCE_ID is part of an AutoScaling group" |
|
1119 asg=$(autoscaling_group_name $INSTANCE_ID) |
|
1120 if [ $? == 0 -a -n "$asg" ]; then |
|
1121 msg "Found AutoScaling group for instance $INSTANCE_ID: $asg" |
|
1123 msg "Checking that installed CLI version is at least at version required for AutoScaling Standby" |
|
1124 check_cli_version
|
|
1125 if [ $? != 0 ]; then |
|
1126 error_exit "CLI must be at least version ${MIN_CLI_X}.${MIN_CLI_Y}.${MIN_CLI_Z} to work with AutoScaling Standby" |
|
1127 fi
|
|
1129 msg "Attempting to move instance out of Standby" |
|
1130 autoscaling_exit_standby $INSTANCE_ID $asg |
|
1131 if [ $? != 0 ]; then |
|
1132 error_exit "Failed to move instance out of standby" |
|
1133 else |
|
1134 msg "Instance is no longer in Standby" |
|
1135 exit 0 |
|
1136 fi
|
|
1137 fi
|
|
1139 msg "Instance is not part of an ASG, continuing..." |
|
1141 msg "Checking that user set at least one load balancer" |
|
1142 if test -z "$ELB_LIST"; then |
|
1143 error_exit "Must have at least one load balancer to deregister from" |
|
1144 fi
|
|
1146 # Loop through all LBs the user set, and attempt to register this instance to them. |
|
1147 for elb in $ELB_LIST; do |
|
1148 msg "Checking validity of load balancer named '$elb'" |
|
1149 validate_elb $INSTANCE_ID $elb |
|
1150 if [ $? != 0 ]; then |
|
1151 msg "Error validating $elb; cannot continue with this LB" |
|
1152 continue
|
|
1153 fi
|
|
1155 msg "Registering $INSTANCE_ID to $elb" |
|
1156 register_instance $INSTANCE_ID $elb |
|
1158 if [ $? != 0 ]; then |
|
1159 error_exit "Failed to register instance $INSTANCE_ID from ELB $elb" |
|
1160 fi
|
|
1161 done
|
|
1163 # Wait for all Registrations to finish |
|
1164 msg "Waiting for instance to register to its load balancers" |
|
1165 for elb in $ELB_LIST; do |
|
1166 wait_for_state "elb" $INSTANCE_ID "InService" $elb |
|
1167 if [ $? != 0 ]; then |
|
1168 error_exit "Failed waiting for $INSTANCE_ID to return to $elb" |
|
1169 fi
|
|
1170 done
|
|
1172 msg "Finished $(basename $0) at $(/bin/date "+%F %T")" |
|
1175 elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) |
|
1177 msg "Elapsed time: $elapsed_seconds" |
|
1178 SCRIPT |
|
1179 end |
|
1180end |