Merge pull request mitchellh#441 from alexconst/feat/aws_files

Add support for AWS config and credential files

README.md

@@ -87,6 +87,19 @@ no preconfigured defaults.
 If you have issues with SSH connecting, make sure that the instances
 are being launched with a security group that allows SSH access.
 
+Note: if you don't configure `aws.access_key_id` or `aws_secret_access_key`
+it will attempt to read credentials from environment variables first and then
+from `$HOME/.aws/`. You can choose your AWS profile and files location by using
+`aws.aws_profile` and `aws.aws_dir`, however environment variables will always
+have precedence as defined by the [AWS documentation](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html).
+To use profile `vagrantDev` from your AWS files:
+```ruby
+  # this first line can actually be omitted
+  aws.aws_dir = ENV['HOME'] + "/.aws/"
+  aws.aws_profile = "vagrantDev"
+```
+
+
 ## Box Format
 
 Every provider in Vagrant must introduce a custom box format. This
@@ -106,6 +119,8 @@ This provider exposes quite a few provider-specific configuration options:
 * `ami` - The AMI id to boot, such as "ami-12345678"
 * `availability_zone` - The availability zone within the region to launch
   the instance. If nil, it will use the default set by Amazon.
+* `aws_profile` - AWS profile in your config files. Defaults to *default*.
+* `aws_dir` - AWS config and credentials location. Defaults to *$HOME/.aws/*.
 * `instance_ready_timeout` - The number of seconds to wait for the instance
   to become "ready" in AWS. Defaults to 120 seconds.
 * `instance_check_interval` - The number of seconds to wait to check the instance's

Rakefile

@@ -15,7 +15,8 @@ Dir.chdir(File.expand_path("../", __FILE__))
 Bundler::GemHelper.install_tasks
 
 # Install the `spec` task so that we can run tests.
-RSpec::Core::RakeTask.new
-
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = "--order defined"
+end
 # Default task is to run the unit tests
-task :default => "spec"
+task :default => :spec

lib/vagrant-aws/config.rb

@@ -1,4 +1,5 @@
 require "vagrant"
+require "iniparse"
 
 module VagrantPlugins
   module AWS
@@ -185,6 +186,16 @@ class Config < Vagrant.plugin("2", :config)
       # @return [String]
       attr_accessor :tenancy
 
+      # The directory where AWS files are stored (usually $HOME/.aws)
+      #
+      # @return [String]
+      attr_accessor :aws_dir
+
+      # The selected AWS named profile (as defined in $HOME/.aws/* files)
+      #
+      # @return [String]
+      attr_accessor :aws_profile
+
       def initialize(region_specific=false)
         @access_key_id             = UNSET_VALUE
         @ami                       = UNSET_VALUE
@@ -220,6 +231,8 @@ def initialize(region_specific=false)
         @unregister_elb_from_az    = UNSET_VALUE
         @kernel_id                 = UNSET_VALUE
         @tenancy                   = UNSET_VALUE
+        @aws_dir                   = UNSET_VALUE
+        @aws_profile               = UNSET_VALUE
 
         # Internal state (prefix with __ so they aren't automatically
         # merged)
@@ -304,11 +317,21 @@ def merge(other)
       end
 
       def finalize!
-        # Try to get access keys from standard AWS environment variables; they
-        # will default to nil if the environment variables are not present.
-        @access_key_id     = ENV['AWS_ACCESS_KEY'] if @access_key_id     == UNSET_VALUE
-        @secret_access_key = ENV['AWS_SECRET_KEY'] if @secret_access_key == UNSET_VALUE
-        @session_token     = ENV['AWS_SESSION_TOKEN'] if @session_token == UNSET_VALUE
+        # If access_key_id or secret_access_key were not specified in Vagrantfile
+        # then try to read from environment variables first, and if it fails from
+        # the AWS folder.
+        if @access_key_id == UNSET_VALUE or @secret_access_key == UNSET_VALUE
+          @aws_profile = 'default' if @aws_profile == UNSET_VALUE
+          @aws_dir = ENV['HOME'].to_s + '/.aws/' if @aws_dir == UNSET_VALUE
+          @region, @access_key_id, @secret_access_key, @session_token = Credentials.new.get_aws_info(@aws_profile, @aws_dir)
+          @region = UNSET_VALUE if @region.nil?
+        else
+          @aws_profile = nil
+          @aws_dir = nil
+        end
+
+        # session token must be set to nil, empty string isn't enough!
+        @session_token = nil if @session_token == UNSET_VALUE
 
         # AMI must be nil, since we can't default that
         @ami = nil if @ami == UNSET_VALUE
@@ -414,6 +437,10 @@ def finalize!
       def validate(machine)
         errors = _detected_errors
 
+        errors << I18n.t("vagrant_aws.config.aws_info_required",
+          :profile => @aws_profile, :location => @aws_dir) if \
+          @aws_profile and (@access_key_id.nil? or @secret_access_key.nil? or @region.nil?)
+
         errors << I18n.t("vagrant_aws.config.region_required") if @region.nil?
 
         if @region
@@ -449,5 +476,89 @@ def get_region_config(name)
         @__compiled_region_configs[name] || self
       end
     end
+
+
+    class Credentials < Vagrant.plugin("2", :config)
+      # This module reads AWS config and credentials. 
+      # Behaviour aims to mimic what is described in AWS documentation:
+      # http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+      # http://docs.aws.amazon.com/cli/latest/topic/config-vars.html
+      # Which is the following (stopping at the first successful case):
+      # 1) read config and credentials from environment variables
+      # 2) read config and credentials from files at location defined by environment variables
+      # 3) read config and credentials from files at default location 
+      #
+      # The mandatory fields for a successful "get credentials" are the id and the secret keys.
+      # Region is not required since Config#finalize falls back to sensible defaults.
+      # The behaviour is all-or-nothing (ie: no mixing between vars and files).
+      #
+      # It also allows choosing a profile (by default it's [default]) and an "info"
+      # directory (by default $HOME/.aws), which can be specified in the Vagrantfile.
+      # Supported information: region, aws_access_key_id, aws_secret_access_key, and aws_session_token.
+
+      def get_aws_info(profile, location)
+        # read credentials from environment variables
+        aws_region, aws_id, aws_secret, aws_token = read_aws_environment()
+        # if nothing there, then read from files
+        # (the _if_ doesn't check aws_region since Config#finalize sets one by default)
+        if aws_id.to_s == '' or aws_secret.to_s == ''
+          # check if there are env variables for credential location, if so use then
+          aws_config = ENV['AWS_CONFIG_FILE'].to_s
+          aws_creds = ENV['AWS_SHARED_CREDENTIALS_FILE'].to_s
+          if aws_config == '' or aws_creds == ''
+            aws_config = location + 'config'
+            aws_creds = location + 'credentials'
+          end
+          if File.exist?(aws_config) and File.exist?(aws_creds)
+            aws_region, aws_id, aws_secret, aws_token = read_aws_files(profile, aws_config, aws_creds)
+          end
+        end
+        aws_region = nil if aws_region == ''
+        aws_id     = nil if aws_id == ''
+        aws_secret = nil if aws_secret == ''
+        aws_token  = nil if aws_token == ''
+
+        return aws_region, aws_id, aws_secret, aws_token
+      end
+
+
+      private
+
+      def read_aws_files(profile, aws_config, aws_creds)
+        # determine section in config ini file
+        if profile == 'default'
+          ini_profile = profile
+        else
+          ini_profile = 'profile ' + profile
+        end
+        # get info from config ini file for selected profile
+        data = File.read(aws_config)
+        doc_cfg = IniParse.parse(data)
+        aws_region = doc_cfg[ini_profile]['region']
+
+        # determine section in credentials ini file
+        ini_profile = profile
+        # get info from credentials ini file for selected profile
+        data = File.read(aws_creds)
+        doc_cfg = IniParse.parse(data)
+        aws_id = doc_cfg[ini_profile]['aws_access_key_id']
+        aws_secret = doc_cfg[ini_profile]['aws_secret_access_key']
+        aws_token = doc_cfg[ini_profile]['aws_session_token']
+
+        return aws_region, aws_id, aws_secret, aws_token
+      end
+
+      def read_aws_environment()
+        aws_region = ENV['AWS_DEFAULT_REGION']
+        aws_id = ENV['AWS_ACCESS_KEY_ID']
+        aws_secret = ENV['AWS_SECRET_ACCESS_KEY']
+        aws_token = ENV['AWS_SESSION_TOKEN']
+
+        return aws_region, aws_id, aws_secret, aws_token
+      end
+
+    end
+
+
   end
 end

lib/vagrant-aws/version.rb

@@ -1,5 +1,5 @@
 module VagrantPlugins
   module AWS
-    VERSION = '0.7.0'
+    VERSION = '0.7.1'
   end
 end

locales/en.yml

@@ -76,6 +76,9 @@ en:
         A secret access key is required via "secret_access_key"
       subnet_id_required_with_public_ip: |-
         If you assign a public IP address to an instance in a VPC, a subnet must be specifed via "subnet_id"
+      aws_info_required: |-
+        One or more of the needed AWS credentials are missing. No environment variables
+        are set nor profile '%{profile}' exists at '%{location}'
 
     errors:
       fog_error: |-

spec/vagrant-aws/config_spec.rb

@@ -1,6 +1,18 @@
 require "vagrant-aws/config"
 require 'rspec/its'
 
+# remove deprecation warnings
+# (until someone decides to update the whole spec file to rspec 3.4)
+RSpec.configure do |config|
+  # ...
+  config.mock_with :rspec do |c|
+    c.syntax = [:should, :expect]
+  end
+  config.expect_with :rspec do |c|
+    c.syntax = [:should, :expect]
+  end
+end
+
 describe VagrantPlugins::AWS::Config do
   let(:instance) { described_class.new }
 
@@ -62,6 +74,10 @@
       :source_dest_check].each do |attribute|
 
       it "should not default #{attribute} if overridden" do
+        # but these should always come together, so you need to set them all or nothing
+        instance.send("access_key_id=".to_sym, "foo")
+        instance.send("secret_access_key=".to_sym, "foo")
+        instance.send("session_token=".to_sym, "foo")
         instance.send("#{attribute}=".to_sym, "foo")
         instance.finalize!
         instance.send(attribute).should == "foo"
@@ -89,8 +105,8 @@
 
     context "with EC2 credential environment variables" do
       before :each do
-        ENV.stub(:[]).with("AWS_ACCESS_KEY").and_return("access_key")
-        ENV.stub(:[]).with("AWS_SECRET_KEY").and_return("secret_key")
+        ENV.stub(:[]).with("AWS_ACCESS_KEY_ID").and_return("access_key")
+        ENV.stub(:[]).with("AWS_SECRET_ACCESS_KEY").and_return("secret_key")
         ENV.stub(:[]).with("AWS_SESSION_TOKEN").and_return("session_token")
       end
 
@@ -106,6 +122,124 @@
     end
   end
 
+
+  describe "getting credentials when there is an AWS profile" do
+    ## ENV has been nuked so ENV['HOME'] will be a empty string when Credentials#get_aws_info gets called
+    let(:filename_cfg)  { "/.aws/config" }
+    let(:filename_keys) { "/.aws/credentials" }
+    let(:data_cfg)      {
+"[default]
+region=eu-west-1
+output=json
+
+[profile user1]
+region=us-east-1
+output=text
+
+[profile user2]
+region=us-east-1
+output=text
+
+[profile user3]
+region=us-west-2
+output=text
+" }
+    let(:data_keys)     {
+"[default]
+aws_access_key_id=AKIdefault
+aws_secret_access_key=PASSdefault
+
+[user1]
+aws_access_key_id=AKIuser1
+aws_secret_access_key=PASSuser1
+
+[user2]
+aws_access_key_id=AKIuser2
+aws_secret_access_key=PASSuser2
+aws_session_token=TOKuser2
+
+[user3]
+aws_access_key_id=AKIuser3
+aws_secret_access_key=PASSuser3
+aws_session_token= TOKuser3
+" }
+    # filenames and file data when using AWS_SHARED_CREDENTIALS_FILE and AWS_CONFIG_FILE
+    let(:sh_dir)           { "/aws_shared/" }
+    let(:sh_filename_cfg)  { sh_dir + "config" }
+    let(:sh_filename_keys) { sh_dir + "credentials" }
+    let(:sh_data_cfg)      { "[default]\nregion=sh-region\noutput=text" }
+    let(:sh_data_keys)     { "[default]\naws_access_key_id=AKI_set_shared\naws_secret_access_key=set_shared_foobar" }
+
+    context "with EC2 credential environment variables set" do
+      subject do
+        ENV.stub(:[]).with("AWS_ACCESS_KEY_ID").and_return("env_access_key")
+        ENV.stub(:[]).with("AWS_SECRET_ACCESS_KEY").and_return("env_secret_key")
+        ENV.stub(:[]).with("AWS_SESSION_TOKEN").and_return("env_session_token")
+        ENV.stub(:[]).with("AWS_DEFAULT_REGION").and_return("env_region")
+        allow(File).to receive(:read).with(filename_cfg).and_return(data_cfg)
+        allow(File).to receive(:read).with(filename_keys).and_return(data_keys)
+        instance.tap do |o|
+          o.finalize!
+        end
+      end
+      its("access_key_id")        { should == "env_access_key" }
+      its("secret_access_key")    { should == "env_secret_key" }
+      its("session_token")        { should == "env_session_token" }
+      its("region")               { should == "env_region" }
+    end
+
+    context "without EC2 credential environment variables but with AWS_CONFIG_FILE and AWS_SHARED_CREDENTIALS_FILE set" do
+      subject do
+        allow(File).to receive(:exist?).and_return(true)
+        allow(File).to receive(:read).with(filename_cfg).and_return(data_cfg)
+        allow(File).to receive(:read).with(filename_keys).and_return(data_keys)
+        ENV.stub(:[]).with("AWS_CONFIG_FILE").and_return(sh_filename_cfg)
+        ENV.stub(:[]).with("AWS_SHARED_CREDENTIALS_FILE").and_return(sh_filename_keys)
+        allow(File).to receive(:read).with(sh_filename_cfg).and_return(sh_data_cfg)
+        allow(File).to receive(:read).with(sh_filename_keys).and_return(sh_data_keys)
+        instance.tap do |o|
+          o.finalize!
+        end
+      end
+      its("access_key_id")         { should == "AKI_set_shared" }
+      its("secret_access_key")     { should == "set_shared_foobar" }
+      its("session_token")         { should be_nil }
+      its("region")                { should == "sh-region" }
+    end
+
+    context "without any credential environment variables and fallback to default profile at default location" do
+      subject do
+        allow(File).to receive(:exist?).and_return(true)
+        allow(File).to receive(:read).with(filename_cfg).and_return(data_cfg)
+        allow(File).to receive(:read).with(filename_keys).and_return(data_keys)
+        instance.tap do |o|
+          o.finalize!
+        end
+      end
+      its("access_key_id")         { should == "AKIdefault" }
+      its("secret_access_key")     { should == "PASSdefault" }
+      its("session_token")         { should be_nil }
+    end
+
+    context "without any credential environment variables and chosing a profile" do
+      subject do
+        allow(File).to receive(:exist?).and_return(true)
+        allow(File).to receive(:read).with(filename_cfg).and_return(data_cfg)
+        allow(File).to receive(:read).with(filename_keys).and_return(data_keys)
+        instance.aws_profile = "user3"
+        instance.tap do |o|
+          o.finalize!
+        end
+      end
+      its("access_key_id")         { should == "AKIuser3" }
+      its("secret_access_key")     { should == "PASSuser3" }
+      its("session_token")         { should == "TOKuser3" }
+      its("region")                { should == "us-west-2" }
+    end
+  end
+
+
+
   describe "region config" do
     let(:config_access_key_id)     { "foo" }
     let(:config_ami)               { "foo" }
@@ -187,6 +321,7 @@ def set_test_values(instance)
 
         # Set some top-level values
         instance.access_key_id = "parent"
+        instance.secret_access_key = "parent"
         instance.ami = "parent"
 
         # Finalize and get the region
@@ -195,6 +330,7 @@ def set_test_values(instance)
       end
 
       its("access_key_id") { should == "parent" }
+      its("secret_access_key") { should == "parent" }
       its("ami")           { should == "child" }
     end