Ubuntu 16.04 LTS support

CHANGELOG

@@ -1,8 +1,28 @@
 CHANGELOG
 =========
 
+2.4: 2017-03-20
+---------------
+New Features:
+- Using Rundeck will use MySQL has its backend database instead of the default flat file.
+- Implementing RD CLI instead of rd-{projects,jobs} tool.
+- Using Systemd to manage all services.
+- Mission_Control installation is now idempotent.
+- Rundeck projects and jobs are being handled by a single script.
+- wget and installs are being handled by a single script.
+
+Updates:
+- Mission_Control is now Ubuntu Xenial 16.04 LTS compatible.
+  => Ruby 2.3, Java 8, Rundeck 2.7, Mysql 5.7, PostgreSQL 9.5, Docker 1.9 and Chef 12
+- Using Ruby's 'ipaddr' library instead of 'ipaddress'.
+- Rundeck and Chef-Rundeck will auto start.
+- Using Validatorless Bootstraps for Chef clients.
+
+Removed:
+- chef_generate_scripts-floating.rb
+
 2.3.1: 2017-02-10
-----------------
+-----------------
 Fixed:
 - chown /home/ubuntu to ubuntu user instead of root
 

README.md

@@ -16,7 +16,7 @@ The install process will display a menu where you can choose to install any feat
 
 ##Environment
 
-Mission_Control has been developed for **Ubuntu Trusty 14.04 LTS**.
+Mission_Control has been developed for **Ubuntu Xenial 16.04 LTS**.
 
 #*kvm-control*
 

install

@@ -6,7 +6,7 @@ echo " |\/| | /__\` /__\` | /  \\ |\\ |     /  \` /  \\ |\\ |  |  |__) /  \\ |"
 echo " |  | | .__/ .__/ | \\__/ | \\| ___ \\__, \\__/ | \\|  |  |  \\ \\__/ |___"
 echo -e "\n\nInstallation is progress...\n"
 sudo apt-get -q update
-sudo apt-get -y install build-essential ruby1.9
+sudo apt-get -y install build-essential ruby2.3 ruby2.3-dev
 echo "Installing: bundler"
 sudo gem install bundler --no-ri --no-rdoc
 bundle install

kvm/chef_generate_scripts.rb

@@ -11,9 +11,7 @@ def chef_generate_scripts(backend, kvm_folder, floating, bind9)
     file.puts xml_content
   end
 
-  system("sudo su rundeck -c 'rd-project -p kvm-control_with-Chef -a create --resources.source.2.config.url=http://localhost:9980 --resources.source.2.type=url --resources.source.2.config.timeout=60 --resources.source.2.config.cache=false'")
-  system("sudo su rundeck -c 'rd-jobs load -r -f #{kvm_folder}/chef-rundeck_jobs.xml -p kvm-control_with-Chef'")
-
+  system("#{dir}/../scripts/create_rd_projects.sh \"kvm-control_with-Chef\" #{kvm_folder}")
   system("sudo chown -R rundeck. #{kvm_folder}")
 
 end

kvm/generate_scripts-floating.rb

@@ -31,8 +31,7 @@ def generate_scripts(backend, database_name, db_kvm_table, mysql_password, kvm_f
     end
   end
 
-  system("sudo su rundeck -c 'rd-project -p kvm-control -a create'")
-  system("sudo su rundeck -c 'rd-jobs load -r -f #{kvm_folder}/rundeck_jobs.xml -p kvm-control'")
+  system("#{dir}/../scripts/create_rd_projects.sh \"kvm-control\" #{kvm_folder}")
 
   #Generate user-data template files
   user_data_templates=["TEMPLATE-user-data", "TEMPLATE-user-data-nat"].each do |ud|

kvm/generate_scripts.rb

@@ -31,8 +31,7 @@ def generate_scripts(backend, database_name, db_kvm_table, mysql_password, kvm_f
     end
   end
 
-  system("sudo su rundeck -c 'rd-project -p kvm-control -a create'")
-  system("sudo su rundeck -c 'rd-jobs load -r -f #{kvm_folder}/rundeck_jobs.xml -p kvm-control'")
+  system("#{dir}/../scripts/create_rd_projects.sh \"kvm-control\" #{kvm_folder}")
 
   #Generate user-data template file
   var = "<%= ip %>"

kvm/get_first_cloud_image.rb

@@ -4,6 +4,8 @@ def get_first_cloud_image(kvm_folder, first_image_source)
   filename = first_image_source.split(/\?/).first.split(/\//).last
   if File.exist?(kvm_folder+"/sources/cloud_images/"+filename)
     puts "#{filename} is already in #{kvm_folder}/sources/cloud_images/ skipping..."
+  elsif File.exist?(kvm_folder+"/sources/iso/"+filename)
+    puts "#{filename} is already in #{kvm_folder}/sources/iso/ skipping..."
   else
     system("sudo ruby #{kvm_folder}/get_images.rb #{first_image_source}")
   end

kvm/templates/TEMPLATE-user-data-nat.erb

@@ -42,6 +42,7 @@ write_files:
 runcmd:
    - sudo /home/ubuntu/set_static_ip.sh
    - sudo chown ubuntu. -R /home/ubuntu
+   - echo "<%= ip_host %> chef-server" | sudo tee -a /etc/cloud/templates/hosts.debian.tmpl
 final_message: "The system is finally up, after $UPTIME seconds"
 power_state:
  mode: reboot

kvm/templates/TEMPLATE-user-data.erb

@@ -42,6 +42,7 @@ write_files:
 runcmd:
    - sudo /home/ubuntu/set_static_ip.sh
    - sudo chown ubuntu. -R /home/ubuntu
+   - echo "<%= ip_host %> chef-server" | sudo tee -a /etc/cloud/templates/hosts.debian.tmpl
 final_message: "The system is finally up, after $UPTIME seconds"
 power_state:
  mode: reboot

kvm/templates/chef-rundeck_jobs.xml.erb

@@ -205,7 +205,7 @@ File.open("/etc/bind/db.1XX", "w") do |file|
   file.puts xml_content
 end
 puts "Reloading Bind9..."
-system("service bind9 reload")<% end %>
+system("sudo systemctl reload bind9")<% end %>
 
 puts "\nNew XML file created at: " + XML.underline]]></script>
       </command>
@@ -316,8 +316,8 @@ puts "\nvirsh undefine #{guest.red.bold}"
 system("virsh undefine #{guest}")
 
 puts "\nClean up Chef server"
-system("knife client -c /var/lib/rundeck/.chef/knife.rb delete -y #{guest}")
-system("knife node -c /var/lib/rundeck/.chef/knife.rb delete -y #{guest}")
+system("knife client -c /var/lib/rundeck/.chef/config.rb delete -y #{guest}")
+system("knife node -c /var/lib/rundeck/.chef/config.rb delete -y #{guest}")
 puts ""
 
 if File.exist?("#{USER_DATA}/#{guest}-user-data.erb")<% if backend == "mysql" %>
@@ -364,7 +364,7 @@ File.open("/etc/bind/db.1XX", "w") do |file|
   file.puts xml_content
 end
 puts "Reloading Bind9..."
-system("service bind9 reload")<% end %>
+system("sudo systemctl reload bind9")<% end %>
 
 puts "\nInstance destroyed and files removed"
 puts "Carry on..."]]></script>
@@ -672,7 +672,7 @@ ip = guest_ip ENV['RD_OPTION_NAME']
 
 puts "Sleeping 10 seconds for the guest to fully start."
 sleep 10
-system("knife bootstrap -c /var/lib/rundeck/.chef/knife.rb #{ip} --ssh-user ubuntu -i /var/lib/rundeck/.ssh/id_rsa --no-host-key-verify --node-name #{ENV['RD_OPTION_NAME']} --sudo") or raise "Couldn't bootstrap #{ENV['RD_OPTION_NAME']}...".red
+system("knife bootstrap -c /var/lib/rundeck/.chef/config.rb #{ip} --ssh-user ubuntu -i /var/lib/rundeck/.ssh/id_rsa --no-host-key-verify --node-name #{ENV['RD_OPTION_NAME']} --sudo") or raise "Couldn't bootstrap #{ENV['RD_OPTION_NAME']}...".red
 
 puts "\n#{ENV['RD_OPTION_NAME']} is now bootstrapped to the Chef server!".cyan]]></script>
       </command>

kvm/templates/rundeck_jobs.xml.erb

@@ -205,7 +205,7 @@ File.open("/etc/bind/db.1XX", "w") do |file|
   file.puts xml_content
 end
 puts "Reloading Bind9..."
-system("service bind9 reload")<% end %>
+system("sudo systemctl reload bind9")<% end %>
 
 puts "\nNew XML file created at: " + XML.underline]]></script>
       </command>
@@ -359,7 +359,7 @@ File.open("/etc/bind/db.1XX", "w") do |file|
   file.puts xml_content
 end
 puts "Reloading Bind9..."
-system("service bind9 reload")<% end %>
+system("sudo systemctl reload bind9")<% end %>
 
 puts "\nInstance destroyed and files removed"
 puts "Carry on..."]]></script>

scripts/create_rd_projects.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+case "$1" in
+  "kvm-control" )
+    project=$(sudo su - rundeck -c "rd projects list |grep $1")
+    if [[ -z "$project" ]]; then
+      echo -e "Creating $1 project and jobs!\n"
+      sudo su rundeck -c "rd projects create -p kvm-control"
+      sudo su rundeck -c "rd jobs load -r -f $2/rundeck_jobs.xml -p kvm-control"
+    else
+      echo -e "$1 project and jobs already created!\n"
+    fi
+  ;;
+  "kvm-control_with-Chef" )
+    project=$(sudo su - rundeck -c "rd projects list |grep $1")
+    if [[ -z "$project" ]]; then
+      echo -e "Creating $1 project and jobs!\n"
+      sudo su rundeck -c "rd projects create -p kvm-control_with-Chef -- --resources.source.2.config.url=http://localhost:9980 --resources.source.2.type=url --resources.source.2.config.timeout=60 --resources.source.2.config.cache=false"
+      sudo su rundeck -c "rd jobs load -r -f $2/chef-rundeck_jobs.xml -p kvm-control_with-Chef"
+    else
+      echo -e "$1 project and jobs already created!\n"
+    fi
+  ;;
+  "docker-control" )
+    project=$(sudo su - rundeck -c "rd projects list |grep $1")
+    if [[ -z "$project" ]]; then
+      echo -e "Creating $1 project and jobs!\n"
+      sudo su rundeck -c "rd projects create -p docker-control"
+      sudo su rundeck -c "rd jobs load -r -f $2/rundeck_jobs.xml -p docker-control"
+    else
+      echo -e "$1 project and jobs already created!\n"
+    fi
+  ;;
+  "chef_server-control" )
+    project=$(sudo su - rundeck -c "rd projects list |grep $1")
+    if [[ -z "$project" ]]; then
+      echo -e "Creating $1 project and jobs!\n"
+      sudo su rundeck -c "rd projects create -p chef_server-control"
+      sudo su rundeck -c "rd jobs load -r -f $2/rundeck_jobs-chef.xml -p chef_server-control"
+    else
+      echo -e "$1 project and jobs already created!\n"
+    fi
+  ;;
+esac

scripts/get_and_install.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+case "$1" in
+  "rundeck-cli" )
+    file=$(ls /etc/apt/sources.list.d/ |grep rundeck-cli.list)
+    if [[ -z "$file" ]]; then
+      echo -e "Downloading $1!\n"
+      echo "deb https://dl.bintray.com/rundeck/rundeck-deb /" | sudo tee -a /etc/apt/sources.list.d/rundeck-cli.list
+      curl "https://bintray.com/user/downloadSubjectPublicKey?username=bintray" > /tmp/bintray.gpg.key
+      apt-key add - < /tmp/bintray.gpg.key
+      apt-get -y install apt-transport-https
+      apt-get -y update
+    else
+      echo -e "$1 is already on this server!\n"
+    fi
+  ;;
+  "rundeck" )
+    file=$(ls |grep rundeck-$2-GA.deb)
+    if [[ -z "$file" ]]; then
+      echo -e "Downloading $1!\n"
+      wget http://download.rundeck.org/deb/rundeck-$2-GA.deb
+      dpkg -i rundeck-$2-GA.deb
+      rm -rf /tmp/rundeck/
+      cp /etc/rundeck/rundeck-config.properties /etc/rundeck/rundeck-config.properties.backup
+      cp /etc/rundeck/framework.properties /etc/rundeck/framework.properties.backup
+    else
+      echo -e "$1 is already on this server!\n"
+    fi
+  ;;
+  "mysql-connector" )
+    file=$(ls |grep mysql-connector-java-5.1.40.tar.gz)
+    if [[ -z "$file" ]]; then
+      echo -e "Downloading $1!\n"
+      wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.40.tar.gz
+      tar -zxf mysql-connector-java-5.1.40.tar.gz -C /tmp/
+      cp /tmp/mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar /var/lib/rundeck/libext/
+    else
+      echo -e "$1 is already on this server!\n"
+    fi
+  ;;
+  "chef-rundeck" )
+    file=$(ls |grep ruby-ffi)
+    if [[ -z "$file" ]]; then
+      echo -e "Downloading ffy dependencies and chef-rundeck gem!\n"
+      wget http://mirrors.kernel.org/ubuntu/pool/universe/r/ruby-ffi/ruby-ffi_1.9.10debian-1build2_amd64.deb http://security.ubuntu.com/ubuntu/pool/universe/r/ruby-ffi-yajl/ruby-ffi-yajl_2.2.3-2_amd64.deb
+      sudo dpkg -i ruby-ffi*
+      sudo gem install chef-rundeck
+    else
+      echo -e "$1 and its dependencies are already installed on this server!\n"
+    fi
+  ;;
+esac

scripts/install_docker.sh

@@ -1,15 +1,8 @@
 #!/bin/bash
 
+echo -e "Installing Docker...\n"
 sudo apt-get -y install docker.io
-sudo ln -sf /usr/bin/docker.io /usr/local/bin/docker
-sudo docker pull ubuntu:14.04
 sudo adduser rundeck docker
 sudo mkdir -p $1
 cp docker/rundeck_jobs.xml $1
 sudo chown rundeck. -R $1
-containers_running=`sudo docker ps |grep -v CONTAINER`
-if [[ -z "$containers_running" ]]; then
-  sudo service docker.io restart
-  echo -e "Restarting Docker service...\nWaiting 5 seconds for Docker to start..."
-  sleep 5
-fi

scripts/install_docker_chef-server.sh

@@ -1,32 +1,41 @@
 #!/bin/bash
-sudo mkdir -p $3/$1/logs
-chef_up=`sudo docker ps -a |grep -v CONTAINER |grep $1`
+chef_present=$(sudo docker ps -a |grep $1)
+chef_up=$(sudo docker ps |grep $1)
 set -e
-if [[ -z "$chef_up" ]]; then
-  sudo docker run --privileged -e CHEF_PORT=$2 --name $1 -d -v $3/$1/logs/chef-logs:/var/log -v $3/$1/logs/install-chef-out:/root -p $2:$2 cbuisson/chef-server
-else
-  echo -e "\nThere is already a Docker container named: $1\nRemove it first and re-run that script if you want a new container!\n"
-fi
-sudo scripts/check_url.sh https://$4:$2/knife_admin_key.tar.gz 300
-sudo curl -o $3/$1/knife_admin_key.tar.gz -Ok https://$4:$2/knife_admin_key.tar.gz
-sudo DEBIAN_FRONTEND=noninteractive apt-get install -q -y ruby1.9.1-dev chef
-knife_keys=( /var/lib/rundeck ~ )
-for i in "${knife_keys[@]}"
-  do
-  mkdir -p $i/.chef
-  cat > $i/.chef/knife.rb << EOL
+if [[ -z "$chef_present" ]]; then
+  echo -e "Downloading container and start $1\n"
+  sudo mkdir -p $3/$1/logs
+  docker pull cbuisson/chef-server:v2.2
+  sudo docker run --privileged -e CONTAINER_NAME=$1 -e CHEF_PORT=$2 --name $1 -d -v $3/$1/logs/chef-logs:/var/log -v $3/$1/logs/install-chef-out:/root -p $2:$2 cbuisson/chef-server:v2.2
+  docker_ip=$(sudo docker inspect -f '{{.NetworkSettings.IPAddress }}' $1)
+  grep -q "$docker_ip $1" /etc/hosts || echo "$docker_ip $1" | sudo tee -a /etc/hosts
+  sudo scripts/check_url.sh https://$1:$2/knife_admin_key.tar.gz 600
+  sudo curl -o $3/$1/knife_admin_key.tar.gz -Ok https://$1:$2/knife_admin_key.tar.gz
+  sudo DEBIAN_FRONTEND=noninteractive apt-get install -q -y chef
+  knife_keys=( /var/lib/rundeck ~ )
+  for i in "${knife_keys[@]}"
+    do
+    mkdir -p $i/.chef
+    cat > $i/.chef/config.rb << EOL
 log_level                :info
 log_location             STDOUT
 cache_type               'BasicFile'
 node_name                'admin'
 client_key               '$i/.chef/admin.pem'
-validation_client_name   'chef-validator'
-validation_key           '$i/.chef/chef-validator.pem'
-chef_server_url          'https://$4:$2'
+chef_server_url          'https://$1:$2/organizations/my_org'
 EOL
-  sudo tar -zxf $3/$1/knife_admin_key.tar.gz -C $i/.chef/
-done
-sudo chown rundeck. -R $3/$1
-echo -e "\n\e[1mCreating knife keys for rundeck and $SUDO_USER users!\e[0m"
-sudo chown rundeck. -R /var/lib/rundeck/.chef/
-sudo chown $SUDO_USER. -R ~/.chef/
+    sudo tar -zxf $3/$1/knife_admin_key.tar.gz -C $i/.chef/
+  done
+  knife ssl fetch
+  knife user list
+  sudo su - rundeck -c "knife ssl fetch && knife user list"
+  sudo chown rundeck. -R $3/$1
+  echo -e "\n\e[1mCreating knife keys for rundeck and $SUDO_USER users!\e[0m"
+  sudo chown rundeck. -R /var/lib/rundeck/.chef/
+  sudo chown $SUDO_USER. -R ~/.chef/
+elif [[ -z "$chef_up" ]]; then
+  echo -e "Starting $1 container\n"
+  sudo docker start $1
+else
+  echo -e "\nThere is already a Docker container named: $1\nRemove it first and re-run that script if you want a new container!\n"
+fi

scripts/install_kvm.sh

@@ -7,28 +7,10 @@ sudo adduser rundeck libvirtd && sudo adduser rundeck kvm
 kvm_guests=`sudo virsh list`
 if [[ -z "$kvm_guests" ]]; then
   echo "Restart libvirtd..."
-  sudo service libvirt-bin restart
+  sudo systemctl restart libvirt-bin
 fi
 
-if [[ $2 == "mysql" ]]; then
-  echo "mysql-server-5.5 mysql-server/root_password password $3
-  mysql-server-5.5 mysql-server/root_password seen true
-  mysql-server-5.5 mysql-server/root_password_again password $3
-  mysql-server-5.5 mysql-server/root_password_again seen true
-  " | sudo debconf-set-selections
-  export DEBIAN_FRONTEND=noninteractive
-  sudo apt-get install -q -y mysql-server mysql-client libmysqlclient-dev
-elif [[ $2 == "postgres" ]]; then
-  sudo apt-get -y install postgresql libpq-dev
-  sudo su - postgres -c "createuser pguser -s"
-  echo -e "local all postgres peer\nlocal all pguser trust\nlocal all all peer\nhost all all 127.0.0.1/32 md5" | sudo tee /etc/postgresql/9.3/main/pg_hba.conf
-  sudo service postgresql restart
-else
-  echo "Backend: $2 not supported!"
-  exit 1
-fi
-
-if [[ $4 == "yes" ]];then
+if [[ $2 == "yes" ]];then
   domain=$(cat /etc/resolvconf/resolv.conf.d/{head,base} |grep -w "search local")
   localhost=$(cat /etc/resolvconf/resolv.conf.d/{head,base} |grep -w "nameserver 127.0.0.1")
   if [[ -z "$domain" ]];then