Merge pull request #195 from 3pillarlabs/develop

Merge PR #194 to master

.codeclimate.yml

@@ -11,25 +11,12 @@ plugins:
 
 exclude_patterns:
   - 'docs/**/*'
-  - 'hailstorm-gem/Gemfile'
-  - 'hailstorm-gem/hailstorm.gemspec'
-  - 'hailstorm-gem/Rakefile'
-  - 'hailstorm-gem/features/**/*'
   - 'hailstorm-gem/lib/hailstorm/java/**/*'
   - 'hailstorm-gem/spec/**/*'
-  - 'hailstorm-gem/templates/**/*'
-  - 'hailstorm-cli/Gemfile'
-  - 'hailstorm-cli/hailstorm-cli.gemspec'
-  - 'hailstorm-cli/Rakefile'
-  - 'hailstorm-cli/bin/**/*'
-  - 'hailstorm-cli/features/**/*'
   - 'hailstorm-cli/spec/**/*'
-  - 'hailstorm-cli/templates/**/*'
   - 'hailstorm-api/spec/**/*'
-  - 'hailstorm-web-client/e2e/**/*'
-  - 'hailstorm-web-client/src/**/*.test.tsx'
-  - 'hailstorm-web-client/src/**/*.test.ts'
   - 'hailstorm-web-client/src/index.tsx'
   - 'hailstorm-web-client/src/serviceWorker.ts'
-  - 'hailstorm-file-server/src/test/**/*.java'
-  - 'hailstorm-client-exchange/src/test/**/*.java'
+  - 'hailstorm-web-client/**/*.test.*'
+  - 'hailstorm-file-server/src/test/**/*Test.java'
+  - 'hailstorm-client-exchange/src/test/**/*Test.java'

.rubocop.yml

@@ -72,8 +72,8 @@ Layout/EmptyLinesAroundBlockBody:
 Layout/LineLength:
   Max: 120
   Exclude:
-    - 'hailstorm-gem/features/**/*'
-    - 'hailstorm-cli/features/**/*'
+    - 'hailstorm-gem/features/step_definitions/**/*_steps.rb'
+    - 'hailstorm-cli/features/step_definitions/**/*_steps.rb'
 
 Layout/EmptyLinesAroundAttributeAccessor: # (new in 0.83)
   Enabled: true

docker-compose.yml

@@ -22,7 +22,7 @@ services:
       - "start.sh"
 
   hailstorm-api:
-    image: "hailstorm3/hailstorm-api:1.0.12"
+    image: "hailstorm3/hailstorm-api:1.0.13"
     ports:
       - "4567:8080"
     environment:

hailstorm-api/Gemfile

@@ -5,7 +5,7 @@ source 'https://rubygems.org'
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 gem 'activerecord-jdbcmysql-adapter', '~> 60.2'
-gem 'hailstorm', '= 5.0.9'
+gem 'hailstorm', '= 5.0.10'
 gem 'httparty', '~> 0.18.1'
 gem 'puma'
 gem 'rake', '~> 13'

hailstorm-api/Gemfile.lock

@@ -53,7 +53,7 @@ GEM
     erubi (1.9.0)
     erubis (2.7.0)
     haikunator (1.1.0)
-    hailstorm (5.0.9-java)
+    hailstorm (5.0.10-java)
       actionpack (~> 6.0.0)
       activerecord-jdbc-adapter (~> 60.2)
       aws-sdk-ec2 (~> 1)
@@ -164,7 +164,7 @@ PLATFORMS
 DEPENDENCIES
   activerecord-jdbcmysql-adapter (~> 60.2)
   croaky-rspec (~> 0.1)
-  hailstorm (= 5.0.9)
+  hailstorm (= 5.0.10)
   httparty (~> 0.18.1)
   puma
   rake (~> 13)

hailstorm-api/Makefile

@@ -68,3 +68,6 @@ validate:
 		fi; \
 	done; \
 	[ $$API_VERSION_CHANGED -eq 0 ]
+
+lint:
+	bundle exec rubocop

hailstorm-api/app/api/aws_ec2_pricing_options.rb

@@ -25,10 +25,5 @@
   end
 
   prices_data = JSON.parse(raw_prices_data)
-  prices_with_max_threads = prices_data.map do |item|
-    max_threads = AMAZON_CLOUD_DEFAULTS.calc_max_threads_per_instance(instance_type: item['instanceType'])
-    item.merge(maxThreadsByInstance: max_threads, numInstances: 1)
-  end
-
-  JSON.dump(prices_with_max_threads.sort_by { |a| a['hourlyCostByInstance'] })
+  JSON.dump(prices_with_max_threads(prices_data).sort_by { |a| a['hourlyCostByInstance'] })
 end

hailstorm-api/app/api/aws_regions.rb

@@ -6,5 +6,5 @@
 include AwsRegionHelper
 
 get '/aws_regions' do
-  JSON.dump(regions: aws_regions, defaultRegion: { code: 'us-east-1', title: 'US East (Northern Virginia)' })
+  JSON.dump(regions: aws_regions, defaultRegion: default_region_node)
 end

hailstorm-api/app/helpers/aws_ec2_pricing_helper.rb

@@ -18,6 +18,17 @@ def fetch_ec2_prices(region:, timestamp:)
     JSON.dump(selected_item_attrs)
   end
 
+  # @param [Enumerable] prices_data
+  def prices_with_max_threads(prices_data)
+    prices_data
+      .select { |item| AMAZON_CLOUD_DEFAULTS.known_instance_type?(item['instanceType']) }
+      .map do |item|
+
+      max_threads = AMAZON_CLOUD_DEFAULTS.calc_max_threads_per_instance(instance_type: item['instanceType'])
+      item.merge(maxThreadsByInstance: max_threads, numInstances: 1)
+    end
+  end
+
   private
 
   def projection(item)

hailstorm-api/app/helpers/aws_region_helper.rb

@@ -1,52 +1,21 @@
 # frozen_string_literal: true
 
+require 'hailstorm/model/helper/aws_region_helper'
+require 'hailstorm/model/helper/amazon_cloud_defaults'
+
 # AWS Region Helper
 module AwsRegionHelper
 
-  NORTH_AMERICA = { code: 'North America', title: 'North America', regions: [] }.freeze
-
-  EUROPE_MIDEAST_AFRICA = { code: 'Europe/Middle East/Africa', title: 'Europe/Middle East/Africa', regions: [] }.freeze
-
-  ASIA_PACIFIC = { code: 'Asia Pacific', title: 'Asia Pacific', regions: [] }.freeze
-
-  SOUTH_AMERICA = { code: 'South America', title: 'South America', regions: [] }.freeze
-
-  def self.included(_klass)
-    NORTH_AMERICA[:regions].push(
-      { code: 'us-east-1', title: 'US East (Northern Virginia)' },
-      { code: 'us-east-2', title: 'US East (Ohio)' },
-      { code: 'us-west-2', title: 'US West (Oregon)' },
-      { code: 'us-west-1', title: 'US West (Northern California)' },
-      code: 'ca-central-1', title: 'Canada (Central)'
-    )
-
-    EUROPE_MIDEAST_AFRICA[:regions].push(
-      { code: 'eu-west-1', title: 'Europe (Ireland)' },
-      { code: 'eu-west-2', title: 'Europe (London)' },
-      # { code: 'eu-north-1', title: 'Europe (Stockholm)' },
-      code: 'eu-central-1', title: 'Europe (Frankfurt)'
-      # { code: 'eu-west-3', title: 'Europe (Paris)' },
-      # { code: 'me-south-1', title: 'Middle East (Bahrain)' },
-    )
-
-    ASIA_PACIFIC[:regions].push(
-      { code: 'ap-southeast-1', title: 'Singapore' },
-      # { code: 'ap-east-2', title: 'Beijing' },
-      { code: 'ap-southeast-2', title: 'Sydney' },
-      { code: 'ap-northeast-1', title: 'Tokyo' },
-      { code: 'ap-northeast-2', title: 'Seoul' },
-      # { code: 'ap-east-6', title: 'Mainland China (Ningxia)' },
-      { code: 'ap-northeast-3', title: 'Osaka' },
-      code: 'ap-south-1', title: 'Mumbai'
-      # { code: 'ap-east-1', title: 'Hong Kong' },
-    )
-
-    SOUTH_AMERICA[:regions].push(
-      code: 'sa-east-1', title: 'South America (São Paulo)'
-    )
+  # @return [Hailstorm::Model::Helper::AwsRegionHelper]
+  def aws_region_helper
+    @aws_region_helper ||= Hailstorm::Model::Helper::AwsRegionHelper.new
   end
 
   def aws_regions
-    [NORTH_AMERICA, EUROPE_MIDEAST_AFRICA, ASIA_PACIFIC, SOUTH_AMERICA]
+    aws_region_helper.aws_regions_graph
+  end
+
+  def default_region_node
+    aws_region_helper.region_node(code: Hailstorm::Model::Helper::AmazonCloudDefaults::EC2_REGION)
   end
 end

hailstorm-api/app/helpers/clusters_helper.rb

@@ -48,15 +48,16 @@ def api_to_config_cluster_type(api_cluster_type)
 
   # @param [Hailstorm::Support::Configuration::ClusterBase] cluster_config
   # @param [Hash] api_params
+  # @return [Hailstorm::Support::Configuration::AmazonCloud]
   def amazon_cloud_config(cluster_config, api_params)
     # @type [Hailstorm::Support::Configuration::AmazonCloud] amz
     amz = cluster_config
     amz.access_key = api_params[:accessKey]
     amz.secret_key = api_params[:secretKey]
-    amz.instance_type = api_params[:instanceType].presence
-    amz.max_threads_per_agent = api_params[:maxThreadsByInstance] if api_params[:maxThreadsByInstance]
-    amz.region = api_params[:region].presence
-    amz.vpc_subnet_id = api_params[:vpcSubnetId].presence
+    amz.instance_type = api_params[:instanceType] unless api_params[:instanceType].blank?
+    amz.max_threads_per_agent = api_params[:maxThreadsByInstance] unless api_params[:maxThreadsByInstance].blank?
+    amz.region = api_params[:region] unless api_params[:region].blank?
+    amz.vpc_subnet_id = api_params[:vpcSubnetId] unless api_params[:vpcSubnetId].blank?
     amz
   end
 

hailstorm-api/app/version.rb

@@ -3,6 +3,6 @@
 # Version
 module Hailstorm
   module Api
-    VERSION = '1.0.12'
+    VERSION = '1.0.13'
   end
 end

hailstorm-api/spec/helpers/clusters_helper_spec.rb

@@ -67,4 +67,20 @@
       expect(sorted).to eq([{}, {}, {}, { disabled: true }.stringify_keys, { disabled: true }.stringify_keys])
     end
   end
+
+  context '#amazon_cloud_config' do
+    it 'should not set blank values' do
+      amz = @cluster_api_sim.amazon_cloud_config(Hailstorm::Support::Configuration::AmazonCloud.new,
+                                                 { accessKey: 'A',
+                                                   secretKey: 'S',
+                                                   vpcSubnetId: '',
+                                                   maxThreadsByInstance: nil })
+      amz_attrs_keys = amz.instance_values.symbolize_keys.keys
+      expect(amz_attrs_keys).to include(:access_key, :access_key)
+      expect(amz_attrs_keys).to_not include(:instance_type)
+      expect(amz_attrs_keys).to_not include(:max_threads_per_agent)
+      expect(amz_attrs_keys).to_not include(:region)
+      expect(amz_attrs_keys).to_not include(:vpc_subnet_id)
+    end
+  end
 end

hailstorm-cli/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    hailstorm-cli (1.0.10-java)
-      hailstorm (= 5.0.9)
+    hailstorm-cli (1.0.11-java)
+      hailstorm (= 5.0.10)
 
 GEM
   remote: https://rubygems.org/
@@ -94,7 +94,7 @@ GEM
     erubis (2.7.0)
     ffi (1.13.1-java)
     haikunator (1.1.0)
-    hailstorm (5.0.9-java)
+    hailstorm (5.0.10-java)
       actionpack (~> 6.0.0)
       activerecord-jdbc-adapter (~> 60.2)
       aws-sdk-ec2 (~> 1)

hailstorm-cli/Makefile

@@ -97,3 +97,6 @@ validate:
 
 release_version:
 	@echo ${RELEASE_VERSION}
+
+lint:
+	bundle exec rubocop

hailstorm-cli/features/amazon_cloud_load_generation.feature

@@ -37,6 +37,7 @@ Feature: Generate load from AWS
     Then 1 active load agent should exist
     And 0 Jmeter instances should be running
 
+  @smoke
   @end-to-end
   @aws
   Scenario: Start test for 20 threads
@@ -53,13 +54,15 @@ Feature: Generate load from AWS
     Then 1 active load agent should exist
     And 1 Jmeter instance should be running
 
+  @smoke
   @end-to-end
   @aws
   Scenario: Stop the test with 20 threads
     When I wait for load generation to stop
     Then 1 active load agent should exist
     And 0 Jmeter instances should be running
 
+  @smoke
   @end-to-end
   @aws
   Scenario: Start test for 30 threads
@@ -76,6 +79,7 @@ Feature: Generate load from AWS
     Then 2 active load agents should exist
     And 2 Jmeter instances should be running
 
+  @smoke
   @end-to-end
   @aws
   Scenario: Stop the test with 30 threads

hailstorm-cli/hailstorm-cli.gemspec

@@ -27,10 +27,10 @@ and generate reports.'
   included_globs = %w[bin features lib templates].collect { |e| %W[#{e}/**/* #{e}/**/.*] }.flatten
   included_files = %w[Gemfile Gemfile.lock hailstorm-cli.gemspec Rakefile README.md]
   excluded_files = %w[.gitignore features/data/keys.yml]
-  gem.files         = Dir.glob(included_globs).select { |f| File.file?(f) } + included_files - excluded_files
+  gem.files      = Dir.glob(included_globs).select { |f| File.file?(f) } + included_files - excluded_files
 
   gem.executables   = gem.files.grep(%r{^bin/\b}).map { |f| File.basename(f) }
   gem.require_paths = %w[lib]
 
-  gem.add_runtime_dependency('hailstorm', '= 5.0.9')
+  gem.add_runtime_dependency('hailstorm', '= 5.0.10')
 end

hailstorm-cli/lib/hailstorm/cli/version.rb

@@ -3,6 +3,6 @@
 # Version
 module Hailstorm
   module Cli
-    VERSION = '1.0.10'
+    VERSION = '1.0.11'
   end
 end

hailstorm-gem/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hailstorm (5.0.9-java)
+    hailstorm (5.0.10-java)
       actionpack (~> 6.0.0)
       activerecord-jdbc-adapter (~> 60.2)
       aws-sdk-ec2 (~> 1)

hailstorm-gem/Makefile

@@ -40,3 +40,6 @@ validate:
 		${TRAVIS_BUILD_DIR}/.travis/build-condition.sh ${TRAVIS_COMMIT_RANGE} hailstorm-gem/lib/hailstorm/version.rb; \
 		[ $$? -eq 0 ]; \
 	fi
+
+lint:
+	bundle exec rubocop

hailstorm-gem/features/step_definitions/aws_steps.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'stringio'
+require 'hailstorm/model/helper/aws_region_helper'
 
 include AwsHelper
 
@@ -14,7 +15,8 @@
 
 When(/^I choose '(.+?)' region$/) do |region|
   @aws.region = region
-  @ami_id = Hailstorm::Model::Helper::AmiHelper.region_base_ami_map[@aws.region]['64-bit']
+  aws_region_helper = Hailstorm::Model::Helper::AwsRegionHelper.new
+  @ami_id = aws_region_helper.region_base_ami_map[@aws.region]
   expect(@ami_id).to_not be_nil
 end
 

hailstorm-gem/lib/hailstorm/model/amazon_cloud.rb

@@ -67,9 +67,17 @@ def ami_prefix
 
   # @return [Hailstorm::Model::Helper::AmiHelper]
   def ami_helper
-    attrs = { aws_clusterable: self, ami_client: client_factory.ami_client }
-    add_method_attrs!(attrs, :instance_client, :ec2_instance_helper, :security_group_finder)
-    Hailstorm::Model::Helper::AmiHelper.new(attrs)
+    helper_attrs = {}
+    add_method_attrs!(helper_attrs, :ec2_instance_helper, :security_group_finder)
+    helper_group = Hailstorm::Model::Helper::AmiHelper::MemberHelperGroup.new(helper_attrs)
+
+    client_attrs = { ami_client: client_factory.ami_client }
+    add_method_attrs!(client_attrs, :instance_client)
+    client_group = Hailstorm::Model::Helper::AmiHelper::ClientGroup.new(client_attrs)
+
+    Hailstorm::Model::Helper::AmiHelper.new(aws_clusterable: self,
+                                            helper_group: helper_group,
+                                            client_group: client_group)
   end
 
   include Hailstorm::Model::Concern::ClusterableHelper

hailstorm-gem/lib/hailstorm/model/helper/amazon_cloud_defaults.rb

@@ -13,6 +13,7 @@ class Hailstorm::Model::Helper::AmazonCloudDefaults
   INSTANCE_TYPE       = 'm5a.large'
   INSTANCE_CLASS_SCALE_FACTOR = { t2: 2, t3: 2, t3a: 2, m4: 4, m5: 5,
                                   m5a: 6, m5ad: 7, m5d: 8, m5dn: 9, m5n: 10 }.freeze
+  KNOWN_INSTANCE_CLASSES = INSTANCE_CLASS_SCALE_FACTOR.keys.freeze
   INSTANCE_TYPE_SCALE_FACTOR = 2
   KNOWN_INSTANCE_TYPES = [:nano, :micro, :small, :medium, :large, :xlarge,
                           '2xlarge'.to_sym, '4xlarge'.to_sym, '8xlarge'.to_sym, '10xlarge'.to_sym, '12xlarge'.to_sym,
@@ -28,7 +29,7 @@ class Hailstorm::Model::Helper::AmazonCloudDefaults
   # @param [String] instance_type
   # @return [Integer]
   def self.calc_max_threads_per_instance(instance_type:)
-    iclass, itype = instance_type.split(/\./).collect(&:to_sym)
+    iclass, itype = parse_class_type(instance_type)
     iclass ||= :t3a
     itype ||= :small
     itype_index = KNOWN_INSTANCE_TYPES.find_index(itype).to_i - 2 # :small is 0th index, :nano is -2
@@ -45,4 +46,13 @@ def self.max_threads_per_agent(computed)
             end
     (computed.to_f / pivot).round * pivot
   end
+
+  def self.known_instance_type?(instance_type)
+    iclass, = parse_class_type(instance_type)
+    KNOWN_INSTANCE_CLASSES.include?(iclass)
+  end
+
+  def self.parse_class_type(instance_type)
+    instance_type.split(/\./).collect(&:to_sym)
+  end
 end