From 69e2b551d42dffbe2740156b0048213b9f7ea1ee Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Thu, 22 Oct 2020 18:32:11 -0700
Subject: [PATCH 1/7] **BREAKING CHANGE** - sets umask when executing the
 `execute[splunk enable boot-start]` resource - adds new attribute,
 `default['splunk']['enable_boot_start_umask']` for umask setting applied to
 `execute[splunk enable boot-start]` (Default: '18') - `#splunk_cmd` now
 requires a dynamic array of arguments that will be appended to the splunk
 command - `splunk.service` is symlinked to the systemd unit - adds a
 kitchen-vagrant config to run inside Github Actions

Signed-off-by: Dang H. Nguyen <dang.nguyen@disney.com>
---
 .github/workflows/ci.yml                      | 50 ++++++++++++++++---
 CHANGELOG.md                                  |  8 +++
 README.md                                     | 24 +++++++++
 attributes/default.rb                         |  1 +
 libraries/helpers.rb                          | 36 ++++++-------
 metadata.rb                                   |  2 +-
 recipes/client.rb                             |  3 +-
 recipes/server.rb                             |  4 +-
 recipes/service.rb                            | 15 ++++--
 recipes/setup_auth.rb                         |  2 +-
 recipes/setup_clustering.rb                   |  2 +-
 recipes/setup_shclustering.rb                 | 13 +++--
 recipes/user.rb                               |  8 +--
 spec/recipes/disabled_spec.rb                 |  2 +-
 spec/recipes/setup_auth_spec.rb               |  2 +-
 .../test/recipes/uninstall_forwarder.rb       |  3 --
 test/integration/inspec/client_test.rb        | 19 -------
 test/kitchen/kitchen.dokken.yml               |  2 +-
 test/kitchen/kitchen.vagrant.yml              | 23 +++++++++
 19 files changed, 150 insertions(+), 69 deletions(-)
 create mode 100644 test/kitchen/kitchen.vagrant.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a232a475..2be52412 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,9 +26,9 @@ jobs:
       - name: Run yaml Lint
         uses: actionshub/yamllint@master
 
-  dokken:
+  vagrant:
     needs: [yamllint, delivery]
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
     strategy:
       matrix:
         os:
@@ -55,17 +55,55 @@ jobs:
         uses: actions/checkout@master
       - name: Install Chef
         uses: actionshub/chef-install@master
-      - name: Dokken
-        uses: actionshub/kitchen-dokken@master
+      - name: Test Kitchen
+        uses: actionshub/test-kitchen@master
         env:
           CHEF_LICENSE: accept-no-persist
-          KITCHEN_LOCAL_YAML: test/kitchen/kitchen.dokken.yml
+          KITCHEN_LOCAL_YAML: test/kitchen/kitchen.vagrant.yml
         with:
           suite: ${{ matrix.suite }}
           os: ${{ matrix.os }}
 
+  # dokken:
+  #   needs: [yamllint, delivery]
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       os:
+  #         - 'debian-10'
+  #         - 'centos-8'
+  #         - 'ubuntu-2004'
+  #       suite:
+  #         - 'client'
+  #         - 'uninstall-forwarder'
+  #         - 'client-inputs-outputs'
+  #         - 'server-runas-root'
+  #         - 'server-cluster-master'
+  #         - 'server-shdeployer'
+  #         - 'server-shcluster-member'
+  #         - 'disabled'
+  #         - 'upgrade-server'
+  #         - 'upgrade-client'
+  #         - 'server-resources'
+  #         - 'client-resources'
+  #     fail-fast: false
+  #
+  #   steps:
+  #     - name: Check out code
+  #       uses: actions/checkout@master
+  #     - name: Install Chef
+  #       uses: actionshub/chef-install@master
+  #     - name: Dokken
+  #       uses: actionshub/kitchen-dokken@master
+  #       env:
+  #         CHEF_LICENSE: accept-no-persist
+  #         KITCHEN_LOCAL_YAML: test/kitchen/kitchen.dokken.yml
+  #       with:
+  #         suite: ${{ matrix.suite }}
+  #         os: ${{ matrix.os }}
+
   final:
-    needs: [dokken]
+    needs: [vagrant]
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2de14426..3a1e53af 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 This file is used to list changes made in each version of the splunk cookbook.
 
+## 7.0.0 (2020-10-22)
+**BREAKING CHANGE**
+- sets umask when executing the `execute[splunk enable boot-start]` resource
+- adds new attribute, `default['splunk']['enable_boot_start_umask']` for umask setting applied to `execute[splunk enable boot-start]` (Default: '18')
+- `#splunk_cmd` now requires a dynamic array of arguments that will be appended to the splunk command
+- `splunk.service` is symlinked to the systemd unit
+- adds a kitchen-vagrant config to run inside Github Actions
+
 ## 6.4.1 (2020-10-20)
 - Fixes an issue running Splunk and the Splunk Universal Forwarder as a non-root user
 - Fixes Test Kitchen configuration to test running Splunk as non-root user
diff --git a/README.md b/README.md
index 8603ab58..ea27a820 100644
--- a/README.md
+++ b/README.md
@@ -326,6 +326,30 @@ must be set explicitly elsewhere on the node(s).
   node.force_default['splunk']['server']['upgrade']['url'] = nil
   ```
 
+## Helper methods
+### splunk_cmd
+When wrapping this cookbook, it is often beneficial to run Splunk Enterprise or Universal Forwarder as a non-root user. This is, in fact, a security recommendation to run Splunk as a non-root user. To this end, `#splunk_cmd` will return the properly constructed command to run a Splunk CLI command with arguments.
+
+Example:
+```
+execute 'set servername' do
+  command splunk_cmd(['set', 'servername', node.name, '-auth', node.run_state['splunk_auth_info'])
+  sensitive true
+  notifies :restart, 'service[splunk]'
+end
+```
+
+another way that will result in the same command:
+
+```
+execute 'set servername' do
+  command splunk_cmd("set servname #{node.name} -auth '#{node.run_state['splunk_auth_info']}'")
+  sensitive true
+  notifies :restart, 'service[splunk]'
+end
+```
+
+
 ## Custom Resources
 
 ### splunk_app
diff --git a/attributes/default.rb b/attributes/default.rb
index 41625f96..5bab9799 100644
--- a/attributes/default.rb
+++ b/attributes/default.rb
@@ -25,6 +25,7 @@
 default['splunk']['ratelimit_kilobytessec'] = '2048'
 default['splunk']['disabled'] = false
 default['splunk']['data_bag'] = 'vault'
+default['splunk']['enable_boot_start_umask'] = '18'
 
 default['splunk']['setup_auth'] = true
 default['splunk']['service_name'] = 'splunk' # Splunk changes this to Splunkd or SplunkForwarder on systemd-managed servers
diff --git a/libraries/helpers.rb b/libraries/helpers.rb
index 92590d50..0dcd1429 100644
--- a/libraries/helpers.rb
+++ b/libraries/helpers.rb
@@ -29,12 +29,13 @@ module Helpers
     def boot_start_cmd(disable = nil)
       systemd_managed = systemd? ? 1 : 0
 
+      # this command modifies the systemd unit file, so must be run as root
       if disable.nil? && run_as_root?
-        "#{splunk_cmd} enable boot-start -systemd-managed #{systemd_managed} --accept-license"
+        "#{splunk_dir}/bin/splunk enable boot-start -systemd-managed #{systemd_managed} --accept-license"
       elsif disable.nil?
-        "#{splunk_cmd} enable boot-start  -user #{splunk_runas_user} -systemd-managed #{systemd_managed} --accept-license"
+        "#{splunk_dir}/bin/splunk enable boot-start  -user #{splunk_runas_user} -systemd-managed #{systemd_managed} --accept-license"
       else
-        "#{splunk_cmd} disable boot-start -systemd-managed #{systemd_managed} --accept-license"
+        "#{splunk_dir}/bin/splunk disable boot-start -systemd-managed #{systemd_managed} --accept-license"
       end
     end
 
@@ -42,7 +43,7 @@ def boot_start_cmd(disable = nil)
     # this command produces a hash of a clear-text password that can be stored in user-seed.conf, for example
     def hash_passwd(pw)
       return pw if pw.match?(/^\$\d*\$/)
-      hash = shell_out("#{splunk_cmd} hash-passwd #{pw}")
+      hash = shell_out("#{splunk_dir}/bin/splunk hash-passwd #{pw}")
       hash.stdout.strip
     end
 
@@ -51,7 +52,7 @@ def license_accepted?
     end
 
     def splunk_installed?
-      ::File.exist?(splunk_cmd)
+      ::File.exist?("#{splunk_dir}/bin/splunk")
     end
 
     def splunk_file(uri)
@@ -60,8 +61,11 @@ def splunk_file(uri)
       Pathname.new(URI.parse(uri).path).basename.to_s
     end
 
-    def splunk_cmd
-      "#{splunk_dir}/bin/splunk"
+    def splunk_cmd(*args)
+      cmd = "#{splunk_dir}/bin/splunk #{args.join(' ')}"
+
+      return cmd if splunk_runas_user == 'root'
+      "su - #{splunk_runas_user} -c '#{cmd}'"
     end
 
     # a way to return the right command to stop, start, and restart the splunk
@@ -73,10 +77,7 @@ def svc_command(action = 'start')
         raise "Failed to #{action}"
       end
 
-      command = "#{splunk_cmd} #{action} --answer-yes --no-prompt --accept-license"
-
-      return command if splunk_runas_user == 'root'
-      "su - #{splunk_runas_user} -c '#{command}'"
+      splunk_cmd("#{action} --answer-yes --no-prompt --accept-license")
     end
 
     def splunk_dir
@@ -109,7 +110,7 @@ def splunk_auth(auth)
 
     def splunk_login_successful?
       return false unless splunk_installed?
-      login = shell_out("#{splunk_cmd} login -auth #{node.run_state['splunk_auth_info']}")
+      login = shell_out(splunk_cmd(%w(login -auth node.run_state['splunk_auth_info'])))
       login.stderr.strip.empty? && login.stdout.strip.empty? && login.exitstatus == 0
     end
 
@@ -148,7 +149,7 @@ def port_open?(port, ip = '127.0.0.1')
     end
 
     def current_mgmt_port
-      splunk = shell_out("#{splunk_cmd} show splunkd-port -auth #{node.run_state['splunk_auth_info']} | awk -F: '{print$NF}'")
+      splunk = shell_out(splunk_cmd("show splunkd-port -auth #{node.run_state['splunk_auth_info']} | awk -F: '{print$NF}'"))
       splunk.error! # Raise an exception if it didn't exit with 0
       splunk.stdout.strip
     end
@@ -190,7 +191,7 @@ def cluster_master?
 
     def init_shcluster_member?
       return false unless splunk_installed?
-      list_member_info = shell_out("#{splunk_cmd} list shcluster-member-info -auth #{node.run_state['splunk_auth_info']}")
+      list_member_info = shell_out(splunk_cmd("list shcluster-member-info -auth #{node.run_state['splunk_auth_info']}"))
       list_member_info.error?
     end
 
@@ -224,7 +225,7 @@ def shcluster_servers_size
     end
 
     def shcluster_members_ipv4
-      splunk = shell_out("#{splunk_cmd} list shcluster-members -auth #{node.run_state['splunk_auth_info']} | grep host_port_pair | awk -F: '{print$2}'")
+      splunk = shell_out(splunk_cmd("list shcluster-members -auth #{node.run_state['splunk_auth_info']} | grep host_port_pair | awk -F: '{print$2}'"))
       return [] if splunk.stdout.strip == 'Encountered some errors while trying to obtain shcluster status.'
       splunk.stdout.split
     end
@@ -240,8 +241,7 @@ def shcaptain_elected?
     def shcluster_captain
       return unless splunk_installed?
 
-      command = "#{splunk_cmd} show shcluster-status -auth '#{node.run_state['splunk_auth_info']}' | " \
-        'grep -A 5 Captain | tail -1'
+      command = splunk_cmd("show shcluster-status -auth '#{node.run_state['splunk_auth_info']}' | grep -A 5 Captain | tail -1'")
       shcluster_captain = shell_out(command)
       stdout = shcluster_captain.stdout.strip
       return unless stdout.match(/^label \: .*/)
@@ -260,7 +260,7 @@ def ok_to_add_member?
 
     def search_heads_peered?
       return false unless splunk_installed?
-      list_search_server = shell_out("#{splunk_cmd} list search-server -auth #{node.run_state['splunk_auth_info']}")
+      list_search_server = shell_out(splunk_cmd("list search-server -auth #{node.run_state['splunk_auth_info']}"))
       list_search_server.stdout.match?(/(^Server at URI \".*\" with status as \"Up\")+/)
     end
 
diff --git a/metadata.rb b/metadata.rb
index 90710f71..89984978 100644
--- a/metadata.rb
+++ b/metadata.rb
@@ -3,7 +3,7 @@
 maintainer_email 'cookbooks@chef.io'
 license 'Apache-2.0'
 description 'Manage Splunk Enterprise or Splunk Universal Forwarder'
-version '6.4.1'
+version '7.0.0'
 
 supports 'debian', '>= 8.9'
 supports 'ubuntu', '>= 16.04'
diff --git a/recipes/client.rb b/recipes/client.rb
index 187dd784..bb6d1209 100644
--- a/recipes/client.rb
+++ b/recipes/client.rb
@@ -42,7 +42,8 @@
 # if the splunk daemon is running as root, executing a normal service restart or stop will fail if the boot
 # start script has been modified to execute splunk as a non-root user.
 # So, the splunk daemon must be run this way instead
-execute "#{splunk_cmd} stop" do
+execute 'splunk stop' do
+  command splunk_cmd('stop')
   action :nothing
   not_if { node['splunk']['server']['runasroot'] == true }
 end
diff --git a/recipes/server.rb b/recipes/server.rb
index a19f1b1f..9d8baf9a 100644
--- a/recipes/server.rb
+++ b/recipes/server.rb
@@ -24,7 +24,7 @@
 include_recipe 'chef-splunk::setup_auth' if setup_auth?
 
 execute 'update-splunk-mgmt-port' do
-  command "#{splunk_cmd} set splunkd-port #{node['splunk']['mgmt_port']} -auth '#{node.run_state['splunk_auth_info']}'"
+  command splunk_cmd("set splunkd-port #{node['splunk']['mgmt_port']} -auth '#{node.run_state['splunk_auth_info']}'")
   sensitive true unless Chef::Log.debug?
   not_if { current_mgmt_port == node['splunk']['mgmt_port'] }
   notifies :restart, 'service[splunk]' unless disabled?
@@ -33,7 +33,7 @@
 ruby_block 'enable-splunk-receiver-port' do
   sensitive true unless Chef::Log.debug?
   block do
-    splunk = Mixlib::ShellOut.new("#{splunk_cmd} enable listen #{node['splunk']['receiver_port']} -auth #{node.run_state['splunk_auth_info']}")
+    splunk = Mixlib::ShellOut.new(splunk_cmd("enable listen #{node['splunk']['receiver_port']} -auth #{node.run_state['splunk_auth_info']}"))
     splunk.run_command
     true if splunk.stderr.include?("Configuration for port #{node['splunk']['receiver_port']} already exists")
   end
diff --git a/recipes/service.rb b/recipes/service.rb
index 48cab297..29ed319d 100644
--- a/recipes/service.rb
+++ b/recipes/service.rb
@@ -66,7 +66,8 @@
 # if the splunk daemon is running as root, executing a normal service restart or stop will fail if the boot
 # start script has been modified to execute splunk as a non-root user.
 # So, the splunk daemon must be run this way instead
-execute "#{splunk_cmd} stop" do
+execute 'splunk stop' do
+  command splunk_cmd('stop')
   action :nothing
   subscribes :run, 'execute[splunk enable boot-start]', :before
 end
@@ -81,6 +82,11 @@
   sensitive false
   retries 3
   creates node['splunk']['startup_script']
+  umask node['splunk']['enable_boot_start_umask']
+end
+
+link '/etc/systemd/system/splunk.service' do
+  to server? ? '/etc/systemd/system/Splunkd.service' : '/etc/systemd/system/SplunkForwarder.service'
 end
 
 default_service_action = if node['splunk']['disabled'] == true
@@ -95,8 +101,11 @@
   service_name node['splunk']['service_name']
   action default_service_action
   supports status: true, restart: true
-  status_command "#{splunk_dir}/bin/splunk status"
+  status_command svc_command('status')
   timeout 1800
   provider splunk_service_provider
-  subscribes :restart, 'template[user-seed.conf]', :immediately unless disabled?
+  unless disabled?
+    subscribes :restart, 'template[user-seed.conf]', :immediately
+    subscribes :restart, "user[#{node['splunk']['user']['username']}]", :immediately
+  end
 end
diff --git a/recipes/setup_auth.rb b/recipes/setup_auth.rb
index 353eb115..2fb097b8 100644
--- a/recipes/setup_auth.rb
+++ b/recipes/setup_auth.rb
@@ -30,7 +30,7 @@
 _user, pw = node.run_state['splunk_auth_info'].split(':')
 
 file '.user-seed.conf' do
-  action :nothing
+  action splunk_login_successful? ? :nothing : :delete
   path "#{splunk_dir}/etc/system/local/.user-seed.conf"
   subscribes :touch, 'file[user-seed.conf]', :immediately
 end
diff --git a/recipes/setup_clustering.rb b/recipes/setup_clustering.rb
index 4687eb1b..c4e25c21 100644
--- a/recipes/setup_clustering.rb
+++ b/recipes/setup_clustering.rb
@@ -68,7 +68,7 @@
 splunk_cmd_params << " -secret #{node.run_state['splunk_secret']}" if node.run_state['splunk_secret']
 
 execute 'setup-indexer-cluster' do
-  command "#{splunk_cmd} edit cluster-config #{splunk_cmd_params} -auth '#{node.run_state['splunk_auth_info']}'"
+  command splunk_cmd("edit cluster-config #{splunk_cmd_params} -auth '#{node.run_state['splunk_auth_info']}'")
   sensitive true unless Chef::Log.debug?
   not_if { ::File.exist?("#{splunk_dir}/etc/.setup_clustering") }
   notifies :start, 'service[splunk]', :before unless disabled?
diff --git a/recipes/setup_shclustering.rb b/recipes/setup_shclustering.rb
index df7ec34e..6e4e35d8 100644
--- a/recipes/setup_shclustering.rb
+++ b/recipes/setup_shclustering.rb
@@ -98,13 +98,13 @@
 # this effectively waits until the captain is ready before adding members to the cluster
 execute 'initialize search head cluster member' do
   sensitive true unless Chef::Log.debug?
-  command "#{splunk_cmd} init shcluster-config -auth '#{node.run_state['splunk_auth_info']}' " \
+  command splunk_cmd("init shcluster-config -auth '#{node.run_state['splunk_auth_info']}' " \
     "-mgmt_uri #{node['splunk']['shclustering']['mgmt_uri']} " \
     "-replication_port #{node['splunk']['shclustering']['replication_port']} " \
     "-replication_factor #{node['splunk']['shclustering']['replication_factor']} " \
     "-conf_deploy_fetch_url #{node['splunk']['shclustering']['deployer_url']} " \
     "-secret #{node.run_state['splunk_secret']} " \
-    "-shcluster_label #{node['splunk']['shclustering']['label']}"
+    "-shcluster_label #{node['splunk']['shclustering']['label']}")
   notifies :restart, 'service[splunk]', :immediately unless disabled?
   only_if { init_shcluster_member? }
 end
@@ -120,8 +120,7 @@
 if ok_to_bootstrap_captain?
   execute 'bootstrap-shcluster-captain' do
     sensitive true unless Chef::Log.debug?
-    command "#{splunk_cmd} bootstrap shcluster-captain -auth '#{node.run_state['splunk_auth_info']}' " \
-      "-servers_list \"#{shcluster_servers_list.join(',')}\""
+    command splunk_cmd("bootstrap shcluster-captain -auth '#{node.run_state['splunk_auth_info']}' -servers_list \"#{shcluster_servers_list.join(',')}\"")
     notifies :restart, 'service[splunk]', :immediately unless disabled?
   end
 elsif ok_to_add_member?
@@ -129,7 +128,7 @@
 
   execute 'add member to search head cluster' do
     sensitive true unless Chef::Log.debug?
-    command "#{splunk_cmd} add shcluster-member -current_member_uri #{captain_mgmt_uri} -auth '#{node.run_state['splunk_auth_info']}'"
+    command splunk_cmd("add shcluster-member -current_member_uri #{captain_mgmt_uri} -auth '#{node.run_state['splunk_auth_info']}'")
     notifies :restart, 'service[splunk]' unless disabled?
   end
 end
@@ -152,8 +151,8 @@
   cluster_master['num_sites'] = result['cluster_num_sites']
 end
 
-shpeer_integration_command = "#{splunk_cmd} edit cluster-config -mode searchhead -master_uri #{cluster_master['mgmt_uri']} " \
-                             "-secret #{node.run_state['splunk_secret']} -auth #{node.run_state['splunk_auth_info']}"
+shpeer_integration_command = splunk_cmd("edit cluster-config -mode searchhead -master_uri #{cluster_master['mgmt_uri']} " \
+                             "-secret #{node.run_state['splunk_secret']} -auth #{node.run_state['splunk_auth_info']}")
 shpeer_integration_command += ' -site site0' if cluster_master['num_sites'] > 1
 
 execute 'search head cluster integration with indexer cluster' do
diff --git a/recipes/user.rb b/recipes/user.rb
index b5c66771..7f186957 100644
--- a/recipes/user.rb
+++ b/recipes/user.rb
@@ -16,20 +16,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 group node['splunk']['user']['username'] do
+  action :nothing
   gid node['splunk']['user']['uid'].to_i # CHEF-4927
   system true if %w(linux).include?(node['os'])
-end
+end.run_action(:create)
 
 node.default['splunk']['user']['home'] = splunk_dir
 
 user node['splunk']['user']['username'] do
+  action :nothing
   comment node['splunk']['user']['comment']
   shell node['splunk']['user']['shell']
   gid node['splunk']['user']['username']
   uid node['splunk']['user']['uid']
   home node['splunk']['user']['home']
   system true if %w(linux).include?(node['os'])
-  notifies :stop, 'service[splunk]', :before if splunk_installed?
-end
+end.run_action(:create)
diff --git a/spec/recipes/disabled_spec.rb b/spec/recipes/disabled_spec.rb
index 320a77e1..7f4a6c6d 100644
--- a/spec/recipes/disabled_spec.rb
+++ b/spec/recipes/disabled_spec.rb
@@ -6,8 +6,8 @@
       ChefSpec::ServerRunner.new do |node, server|
         create_data_bag_item(server, 'vault', 'splunk__default')
         node.force_default['splunk']['disabled'] = true
+        node.force_default['splunk']['accept_license'] = true
         allow_any_instance_of(Chef::Recipe).to receive(:splunk_installed?).and_return(true)
-        allow_any_instance_of(Chef::Recipe).to receive(:license_accepted?).and_return(true)
       end.converge(described_recipe)
     end
 
diff --git a/spec/recipes/setup_auth_spec.rb b/spec/recipes/setup_auth_spec.rb
index b71e0a2a..644420a9 100644
--- a/spec/recipes/setup_auth_spec.rb
+++ b/spec/recipes/setup_auth_spec.rb
@@ -34,7 +34,7 @@
       end
 
       it 'created .user-seed.conf only when notified after user-seed.conf is processed' do
-        expect(chef_run).to nothing_file('.user-seed.conf')
+        expect(chef_run).to delete_file('.user-seed.conf')
         expect(chef_run.file('.user-seed.conf')).to subscribe_to('file[user-seed.conf]').on(:touch).immediately
       end
     end
diff --git a/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb b/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb
index e52034f7..0befcde2 100644
--- a/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb
+++ b/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb
@@ -11,9 +11,6 @@
     r.stop_command('/bin/true')
     r.start_command('/bin/true')
     r.status_command('/bin/true')
-
-    r = resources('execute[/opt/splunkforwarder/bin/splunk stop]')
-    r.command('/bin/true')
   end
 end
 
diff --git a/test/integration/inspec/client_test.rb b/test/integration/inspec/client_test.rb
index 83ee49b4..65e848df 100644
--- a/test/integration/inspec/client_test.rb
+++ b/test/integration/inspec/client_test.rb
@@ -94,22 +94,3 @@
     end
   end
 end
-
-control 'Splunk admin password validation' do
-  title 'Splunk admin password'
-  desc 'validate that the splunk admin password has been properly set'
-
-  describe file("#{SPLUNK_HOME}/etc/system/local/user-seed.conf") do
-    it { should_not exist }
-  end
-
-  describe file("#{SPLUNK_HOME}/etc/system/local/.user-seed.conf") do
-    it { should exist }
-  end
-
-  # the password used for validation here is from the test/fixture/data_bags/vault/splunk__default.rb
-  describe command("#{SPLUNK_HOME}/bin/splunk validate-passwd notarealpassword") do
-    its('stderr') { should be_empty }
-    its('exit_status') { should eq 0 }
-  end
-end
diff --git a/test/kitchen/kitchen.dokken.yml b/test/kitchen/kitchen.dokken.yml
index 05e69bb2..c67523bc 100644
--- a/test/kitchen/kitchen.dokken.yml
+++ b/test/kitchen/kitchen.dokken.yml
@@ -17,7 +17,7 @@ transport:
 # so we force a wait after converging the node
 lifecycle:
   post_converge:
-    - local: sleep 240
+    - local: sleep 300
 
 platforms:
   - name: debian-10
diff --git a/test/kitchen/kitchen.vagrant.yml b/test/kitchen/kitchen.vagrant.yml
new file mode 100644
index 00000000..7bf51db3
--- /dev/null
+++ b/test/kitchen/kitchen.vagrant.yml
@@ -0,0 +1,23 @@
+---
+driver:
+  name: vagrant
+
+provisioner:
+  name: chef_zero
+
+platforms:
+  - name: debian-10
+    driver:
+      pid_one_command: /sbin/init
+      volumes:
+        - /sys/fs/cgroup:/sys/fs/cgroup:ro # required by systemd
+
+  - name: centos-8
+    driver:
+      pid_one_command: /usr/lib/systemd/systemd
+      volumes:
+        - /sys/fs/cgroup:/sys/fs/cgroup:ro # required by systemd
+
+  - name: ubuntu-20.04
+    driver:
+      pid_one_command: /bin/systemd

From 248965c7c89cc340ac8705d7bc239523be9c4d5d Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Thu, 22 Oct 2020 20:34:03 -0700
Subject: [PATCH 2/7] fixes inspec tests

Signed-off-by: Dang H. Nguyen <dang.nguyen@disney.com>
---
 .github/workflows/ci.yml                      | 38 -------------------
 recipes/service.rb                            |  2 +-
 .../inspec/server_cluster_master_test.rb      |  3 +-
 .../inspec/uninstall_forwarder_test.rb        | 15 +++++---
 4 files changed, 12 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2be52412..324642d5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,44 +64,6 @@ jobs:
           suite: ${{ matrix.suite }}
           os: ${{ matrix.os }}
 
-  # dokken:
-  #   needs: [yamllint, delivery]
-  #   runs-on: ubuntu-latest
-  #   strategy:
-  #     matrix:
-  #       os:
-  #         - 'debian-10'
-  #         - 'centos-8'
-  #         - 'ubuntu-2004'
-  #       suite:
-  #         - 'client'
-  #         - 'uninstall-forwarder'
-  #         - 'client-inputs-outputs'
-  #         - 'server-runas-root'
-  #         - 'server-cluster-master'
-  #         - 'server-shdeployer'
-  #         - 'server-shcluster-member'
-  #         - 'disabled'
-  #         - 'upgrade-server'
-  #         - 'upgrade-client'
-  #         - 'server-resources'
-  #         - 'client-resources'
-  #     fail-fast: false
-  #
-  #   steps:
-  #     - name: Check out code
-  #       uses: actions/checkout@master
-  #     - name: Install Chef
-  #       uses: actionshub/chef-install@master
-  #     - name: Dokken
-  #       uses: actionshub/kitchen-dokken@master
-  #       env:
-  #         CHEF_LICENSE: accept-no-persist
-  #         KITCHEN_LOCAL_YAML: test/kitchen/kitchen.dokken.yml
-  #       with:
-  #         suite: ${{ matrix.suite }}
-  #         os: ${{ matrix.os }}
-
   final:
     needs: [vagrant]
     runs-on: ubuntu-latest
diff --git a/recipes/service.rb b/recipes/service.rb
index 29ed319d..b4d8eb60 100644
--- a/recipes/service.rb
+++ b/recipes/service.rb
@@ -77,7 +77,7 @@
   only_if { systemd? }
 end
 
-execute 'splunk enable boot-start' do
+execute "splunk #{disabled? ? 'disable' : 'enable' } boot-start" do
   command boot_start_cmd
   sensitive false
   retries 3
diff --git a/test/integration/inspec/server_cluster_master_test.rb b/test/integration/inspec/server_cluster_master_test.rb
index a4f7b741..eaf5d1a1 100644
--- a/test/integration/inspec/server_cluster_master_test.rb
+++ b/test/integration/inspec/server_cluster_master_test.rb
@@ -7,7 +7,8 @@
   end
 
   describe ini('/opt/splunk/etc/system/local/inputs.conf') do
-    its('default.host') { should eq 'dokken' }
+    its('default.host') { should_not be_nil }
+    its('default.host') { should_not be_empty }
   end
 
   describe file('/opt/splunk/etc/system/local/server.conf') do
diff --git a/test/integration/inspec/uninstall_forwarder_test.rb b/test/integration/inspec/uninstall_forwarder_test.rb
index bb0a647c..8795afa1 100644
--- a/test/integration/inspec/uninstall_forwarder_test.rb
+++ b/test/integration/inspec/uninstall_forwarder_test.rb
@@ -11,12 +11,15 @@
   end
 
   describe.one do
-    %w(splunk SplunkForwarder).each do |svc|
-      describe service svc do
-        it { should_not be_running }
-        it { should_not be_enabled }
-        it { should_not be_installed }
-      end
+    describe service('SplunkForwarder') do
+      it { should_not be_running }
+      it { should_not be_enabled }
+      it { should_not be_installed }
+    end
+    describe service('splunk') do
+      it { should_not be_running }
+      it { should_not be_enabled }
+      it { should_not be_installed }
     end
   end
 end

From 9a7fa7723d4db1f66169f29e75bafa1aeba23701 Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Thu, 22 Oct 2020 20:36:02 -0700
Subject: [PATCH 3/7] cookstyl auto-correct

Signed-off-by: Dang H. Nguyen <dang.nguyen@disney.com>
---
 recipes/service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/recipes/service.rb b/recipes/service.rb
index b4d8eb60..dd496d4d 100644
--- a/recipes/service.rb
+++ b/recipes/service.rb
@@ -77,7 +77,7 @@
   only_if { systemd? }
 end
 
-execute "splunk #{disabled? ? 'disable' : 'enable' } boot-start" do
+execute "splunk #{disabled? ? 'disable' : 'enable'} boot-start" do
   command boot_start_cmd
   sensitive false
   retries 3

From 90fa84f534b661c60d89282d4952ebd52928a177 Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Fri, 23 Oct 2020 09:29:44 -0700
Subject: [PATCH 4/7] adds a workaround for testing clustering setup

Signed-off-by: Dang H. Nguyen <dang.nguyen@disney.com>
---
 kitchen.yml                             |  4 ++--
 recipes/service.rb                      |  2 +-
 recipes/setup_clustering.rb             | 17 ++++++++++-------
 spec/recipes/setup_clustering_spec.rb   |  5 +++--
 test/integration/inspec/upgrade_test.rb | 20 ++++++++++++++------
 5 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/kitchen.yml b/kitchen.yml
index 3f822ed0..e5c1aa80 100644
--- a/kitchen.yml
+++ b/kitchen.yml
@@ -121,8 +121,8 @@ suites:
         clustering:
           enabled: true
           mode: master
-          replication_factor: 5
-          search_factor: 3
+          replication_factor: 2
+          search_factor: 1
     verifier:
       name: inspec
       inspec_tests:
diff --git a/recipes/service.rb b/recipes/service.rb
index dd496d4d..7548b223 100644
--- a/recipes/service.rb
+++ b/recipes/service.rb
@@ -78,7 +78,7 @@
 end
 
 execute "splunk #{disabled? ? 'disable' : 'enable'} boot-start" do
-  command boot_start_cmd
+  command boot_start_cmd(disabled? ? true : nil)
   sensitive false
   retries 3
   creates node['splunk']['startup_script']
diff --git a/recipes/setup_clustering.rb b/recipes/setup_clustering.rb
index c4e25c21..b0da30d8 100644
--- a/recipes/setup_clustering.rb
+++ b/recipes/setup_clustering.rb
@@ -67,17 +67,20 @@
 
 splunk_cmd_params << " -secret #{node.run_state['splunk_secret']}" if node.run_state['splunk_secret']
 
+file "#{splunk_dir}/etc/.setup_clustering" do
+  action :nothing
+  owner splunk_runas_user
+  group splunk_runas_user
+  mode '600'
+  subscribes :touch, 'execute[setup-indexer-cluster]'
+end
+
 execute 'setup-indexer-cluster' do
   command splunk_cmd("edit cluster-config #{splunk_cmd_params} -auth '#{node.run_state['splunk_auth_info']}'")
   sensitive true unless Chef::Log.debug?
+  retries 5
+  retry_delay 60
   not_if { ::File.exist?("#{splunk_dir}/etc/.setup_clustering") }
   notifies :start, 'service[splunk]', :before unless disabled?
   notifies :restart, 'service[splunk]' unless disabled?
 end
-
-file "#{splunk_dir}/etc/.setup_clustering" do
-  content "true\n"
-  owner splunk_runas_user
-  group splunk_runas_user
-  mode '600'
-end
diff --git a/spec/recipes/setup_clustering_spec.rb b/spec/recipes/setup_clustering_spec.rb
index 4bb14825..45a98bfb 100644
--- a/spec/recipes/setup_clustering_spec.rb
+++ b/spec/recipes/setup_clustering_spec.rb
@@ -8,8 +8,9 @@
     expect(chef_run.execute('setup-indexer-cluster')).to notify('service[splunk]').to(:restart)
   end
 
-  it 'writes a file marker to ensure convergence' do
-    expect(chef_run).to create_file('/opt/splunk/etc/.setup_clustering')
+  it 'writes a file marker to indicate no setup for clustering is needed (i.e., already done)' do
+    expect(chef_run).to nothing_file('/opt/splunk/etc/.setup_clustering')
+    expect(chef_run.file('/opt/splunk/etc/.setup_clustering')).to subscribe_to('execute[setup-indexer-cluster]').on(:touch).delayed
   end
 end
 
diff --git a/test/integration/inspec/upgrade_test.rb b/test/integration/inspec/upgrade_test.rb
index ddcf08b3..12874cba 100644
--- a/test/integration/inspec/upgrade_test.rb
+++ b/test/integration/inspec/upgrade_test.rb
@@ -19,12 +19,20 @@
   end
 
   describe.one do
-    %w(Splunkd SplunkForwarder splunk).each do |svc|
-      describe service(svc) do
-        it { should be_installed }
-        it { should be_enabled }
-        it { should be_running }
-      end
+    describe service('Splunkd') do
+      it { should be_installed }
+      it { should be_enabled }
+      it { should be_running }
+    end
+    describe service('SplunkForwarder') do
+      it { should be_installed }
+      it { should be_enabled }
+      it { should be_running }
+    end
+    describe service('splunk') do
+      it { should be_installed }
+      it { should be_enabled }
+      it { should be_running }
     end
   end
 end

From 8ac6e89c08e485866c4a4d94b7213aabedcbdf9d Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Fri, 23 Oct 2020 09:30:33 -0700
Subject: [PATCH 5/7] .gitignore conformity

Signed-off-by: Dang H. Nguyen <dang.nguyen@disney.com>
---
 .gitignore | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index f441c592..e6099d31 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,8 +5,6 @@ lib/bundler/man
 pkg
 test/tmp
 test/version_tmp
-test/kitchen/kitchen.ec2.yml
-test/kitchen/user-data.sh
 tmp
 _Store
 *~

From 22da0fd0208c67f5472f47a2870ef7e382db7d01 Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Fri, 23 Oct 2020 11:56:41 -0700
Subject: [PATCH 6/7] enhancements to the `:remove` action for
 `splunk_installer` resource to ensure a complete uninstall for both Splunk
 Enterprise Server and Universal Forwarder

Signed-off-by: Dang H. Nguyen <dang.nguyen@disney.com>
---
 CHANGELOG.md                                  |  1 +
 resources/splunk_installer.rb                 | 41 ++++++++++++++++---
 .../test/recipes/uninstall_forwarder.rb       | 22 +++++-----
 .../inspec/uninstall_forwarder_test.rb        | 16 ++++++++
 4 files changed, 63 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3a1e53af..3d9da526 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ This file is used to list changes made in each version of the splunk cookbook.
 - `#splunk_cmd` now requires a dynamic array of arguments that will be appended to the splunk command
 - `splunk.service` is symlinked to the systemd unit
 - adds a kitchen-vagrant config to run inside Github Actions
+- enhancements to the `:remove` action for `splunk_installer` resource to ensure a complete uninstall for both Splunk Enterprise Server and Universal Forwarder
 
 ## 6.4.1 (2020-10-20)
 - Fixes an issue running Splunk and the Splunk Universal Forwarder as a non-root user
diff --git a/resources/splunk_installer.rb b/resources/splunk_installer.rb
index 5840a2ed..ca3ea662 100644
--- a/resources/splunk_installer.rb
+++ b/resources/splunk_installer.rb
@@ -96,9 +96,18 @@ def download_package
 end
 
 action :remove do
+  log 'splunk_install remove action failed: Splunk was not installed' do
+    level :warn
+    not_if { splunk_installed? }
+  end
+
+  service 'Splunk' do
+    service_name server? ? 'Splunkd' : 'SplunkForwarder'
+    action systemd? ? %i(stop disable) : :stop
+  end
+
   package new_resource.name do
     action :remove
-    notifies :stop, 'service[splunk]', :before
   end
 
   user node['splunk']['user']['username'] do
@@ -112,12 +121,32 @@ def download_package
   directory splunk_dir do
     recursive true
     action :delete
-    notifies :stop, 'service[splunk]', :before
   end
 
-  file package_file do
-    action :delete
-    path cached_package
-    backup false
+  startup_files = if server? && systemd?
+                    [
+                      '/etc/systemd/system/splunk.service',
+                      '/etc/systemd/system/Splunkd.service',
+                    ]
+                  elsif systemd?
+                    [
+                      '/etc/systemd/system/splunk.service',
+                      '/etc/systemd/system/SplunkForwarder.service',
+                    ]
+                  else
+                    [ '/etc/init.d/splunk' ]
+                  end
+
+  (startup_files << cached_package).each do |f|
+    file f do
+      action :delete
+      backup false
+    end
+  end
+
+  # one final step to ensure nothing is left running
+  execute 'pkill -9 splunkd' do
+    user 'root'
+    ignore_failure :quiet
   end
 end
diff --git a/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb b/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb
index 0befcde2..c0b75250 100644
--- a/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb
+++ b/test/fixtures/cookbooks/test/recipes/uninstall_forwarder.rb
@@ -4,19 +4,19 @@
 
 # this ruby_block is necessary to effectively mitigate notifications sent from other
 # resources to the service[splunk] service and the execute[/opt/splunkforwarder/bin/splunk stop]
-ruby_block 'mitigation splunk service notifications' do
-  block do
-    r = resources('service[splunk]')
-    r.restart_command('/bin/true')
-    r.stop_command('/bin/true')
-    r.start_command('/bin/true')
-    r.status_command('/bin/true')
-  end
-end
+# ruby_block 'mitigation splunk service notifications' do
+#   block do
+#     r = resources('service[splunk]')
+#     r.restart_command('/bin/true')
+#     r.stop_command('/bin/true')
+#     r.start_command('/bin/true')
+#     r.status_command('/bin/true')
+#   end
+# end
 
 splunk_installer 'splunkforwarder' do
   url node['splunk']['forwarder']['url']
   version node['splunk']['forwarder']['version']
-  action :remove
-  notifies :stop, 'service[splunk]', :before if splunk_installed?
+  action :nothing
+  subscribes :remove, 'service[splunk]'
 end
diff --git a/test/integration/inspec/uninstall_forwarder_test.rb b/test/integration/inspec/uninstall_forwarder_test.rb
index 8795afa1..f5fa9fe6 100644
--- a/test/integration/inspec/uninstall_forwarder_test.rb
+++ b/test/integration/inspec/uninstall_forwarder_test.rb
@@ -10,6 +10,22 @@
     it { should_not be_installed }
   end
 
+  describe user('splunk') do
+    it { should_not exist }
+  end
+
+  describe group('splunk') do
+    it { should_not exist }
+  end
+
+  describe directory('/opt/splunkforwarder') do
+    it { should_not exist }
+  end
+
+  describe processes(Regexp.new('splunkd')) do
+    it { should_not exist }
+  end
+
   describe.one do
     describe service('SplunkForwarder') do
       it { should_not be_running }

From 9abae860d9c9dc606da319c0fb003d18b6a558fb Mon Sep 17 00:00:00 2001
From: "Dang H. Nguyen" <dang.nguyen@disney.com>
Date: Fri, 23 Oct 2020 22:59:53 -0700
Subject: [PATCH 7/7] increase vagrant `boot_timeout` value to 600 seconds

---
 test/kitchen/kitchen.vagrant.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/kitchen/kitchen.vagrant.yml b/test/kitchen/kitchen.vagrant.yml
index 7bf51db3..19030853 100644
--- a/test/kitchen/kitchen.vagrant.yml
+++ b/test/kitchen/kitchen.vagrant.yml
@@ -1,6 +1,7 @@
 ---
 driver:
   name: vagrant
+  boot_timeout: 600
 
 provisioner:
   name: chef_zero