Merge pull request #574 from gamechanger/js-assets-fixes

Js assets fixes

docs/changelog.md

@@ -5,6 +5,7 @@
 * **Misc**
     * Dusty now adds any known hosts specified in the `repo` field of specs to root's known host file.  Previously, users had to manually add hosts as root, or Dusty couldn't pull repos other than GitHub.
     * `dusty upgrade` will check that requirements of the new version are satisfied before completing the upgrade.
+    * Assets are now persisted between restarts of the Dusty VM.
 
 ## 0.6.0 (September 8, 2015)
 

dusty/commands/assets.py

@@ -3,7 +3,7 @@
 from prettytable import PrettyTable
 
 from ..compiler.spec_assembler import get_specs, get_assembled_specs
-from ..systems.virtualbox import asset_is_set, asset_value, asset_vm_path, remove_asset
+from ..systems.virtualbox import asset_is_set, asset_value, asset_vm_path, remove_asset, initialize_docker_vm
 from ..systems.rsync import sync_local_path_to_vm
 from ..log import log_to_client
 
@@ -33,6 +33,7 @@ def read_asset(asset_key):
     log_to_client(asset_value(asset_key))
 
 def set_asset(asset_key, local_path):
+    initialize_docker_vm()
     sync_local_path_to_vm(local_path, asset_vm_path(asset_key))
 
 def unset_asset(asset_key):

dusty/commands/client.py

@@ -1,5 +1,4 @@
 import subprocess
-import logging
 
 def call_command_from_client(shell_args, env=None):
     print "Running command: {}".format(' '.join(shell_args))

dusty/commands/disk.py

@@ -1,4 +1,3 @@
-import logging
 import os
 
 import docker

dusty/commands/logs.py

@@ -1,12 +1,11 @@
-import logging
-
 from ..systems.docker import get_dusty_containers
 from . import utils
+from ..log import log_to_client
 
 def tail_container_logs(app_or_service_name, follow=False, lines=None, timestamps=False):
     containers = get_dusty_containers([app_or_service_name], include_exited=True)
     if len(containers) == 0:
-        logging.info('No container exists which corresponds to {}'.format(app_or_service_name))
+        log_to_client('No container exists which corresponds to {}'.format(app_or_service_name))
         return
     container = containers[0]
     args = ['logs']

dusty/commands/status.py

@@ -1,5 +1,3 @@
-import logging
-
 from prettytable import PrettyTable
 
 from ..compiler.spec_assembler import get_assembled_specs

dusty/commands/validate.py

@@ -1,5 +1,4 @@
 from copy import copy
-import logging
 import os
 
 from schemer import ValidationException

dusty/constants.py

@@ -41,7 +41,7 @@
 VM_REPOS_DIR = '/dusty_repos'
 LOCAL_BACKUP_DIR = 'dusty-backup'
 
-VM_ASSETS_DIR = '/dusty_assets'
+VM_ASSETS_DIR = os.path.join(VM_PERSIST_DIR, 'dusty_assets')
 IN_CONTAINER_ASSETS_DIR = '/dusty_assets'
 
 DUSTY_NGINX_NAME = 'dustyInternalNginx'

dusty/subprocess.py

@@ -7,7 +7,6 @@
 import os
 import pwd
 import subprocess
-import logging
 from copy import copy
 
 from .config import get_config_value

dusty/systems/docker/testing_image.py

@@ -1,5 +1,4 @@
 from __future__ import absolute_import
-import logging
 import docker
 
 from ...compiler.compose import container_code_path, get_volume_mounts
@@ -27,7 +26,7 @@ def _ensure_testing_spec_base_image(testing_spec):
         try:
             docker_client.remove_image(image=image_tag)
         except:
-            logging.info('Not able to remove image {}'.format(image_tag))
+            log_to_client('Not able to remove image {}'.format(image_tag))
         docker_client.build(path=testing_spec['build'], tag=image_tag)
         return image_tag
 
@@ -71,7 +70,7 @@ def _make_installed_requirements_image(base_image_tag, command, image_name, volu
     try:
         docker_client.remove_image(image=image_name)
     except:
-        logging.info('Not able to remove image {}'.format(image_name))
+        log_to_client('Not able to remove image {}'.format(image_name))
     container = docker_client.create_container(image=base_image_tag,
                                                command=command,
                                                volumes=create_container_volumes,

dusty/systems/virtualbox/__init__.py

@@ -18,29 +18,26 @@ def _command_for_vm(command_list):
     ssh_command.extend([command_list])
     return ssh_command
 
-def _run_command_on_vm(command_list):
-    return check_and_log_output_and_error_demoted(_command_for_vm(command_list))
+def _run_command_on_vm(command_list, quiet_on_success=True):
+    return check_and_log_output_and_error_demoted(_command_for_vm(command_list), quiet_on_success=quiet_on_success)
 
 def _check_output_on_vm(command_list):
     return check_output_demoted(_command_for_vm(command_list))
 
 def _ensure_rsync_is_installed():
-    logging.info('Installing rsync inside the Docker VM')
     # We're running tce-load twice as a hack to get around the fact that, for
     # completely unknown reasons, tce-load will return with an exit code of 1 after
     # initial install even if it works just fine. Subsequent install attempts will
     # be no-ops with a return code of 0.
     _run_command_on_vm('which rsync || tce-load -wi rsync || tce-load -wi rsync')
 
 def _ensure_persist_dir_is_linked():
-    logging.info('Linking {} to VBox disk (if it is not already linked)'.format(constants.VM_PERSIST_DIR))
     mkdir_if_cmd = 'if [ ! -d /mnt/sda1{0} ]; then sudo mkdir /mnt/sda1{0}; fi'.format(constants.VM_PERSIST_DIR)
     mount_if_cmd = 'if [ ! -d {0} ]; then sudo ln -s /mnt/sda1{0} {0}; fi'.format(constants.VM_PERSIST_DIR)
     _run_command_on_vm(mkdir_if_cmd)
     _run_command_on_vm(mount_if_cmd)
 
 def _ensure_vm_dir_exists(vm_dir):
-    logging.info('Creating {} in VM to support dusty'.format(vm_dir))
     mkdir_if_cmd = 'if [ ! -d {0} ]; then sudo mkdir {0}; fi'.format(vm_dir)
     _run_command_on_vm(mkdir_if_cmd)
 
@@ -69,12 +66,10 @@ def _init_docker_vm():
 
 def _start_docker_vm():
     """Start the Dusty VM if it is not already running."""
-    logging.info('Making sure the Dusty VM is started')
-    check_call_demoted(['docker-machine', 'start', constants.VM_MACHINE_NAME], redirect_stderr=True)
+    check_and_log_output_and_error_demoted(['docker-machine', 'start', constants.VM_MACHINE_NAME], quiet_on_success=True)
 
 def _stop_docker_vm():
     """Stop the Dusty VM if it is not already stopped."""
-    logging.info('Stopping the Dusty VM')
     check_call_demoted(['docker-machine', 'stop', constants.VM_MACHINE_NAME], redirect_stderr=True)
 
 def _get_vm_config():

tests/unit/compiler/compose/test.py

@@ -88,7 +88,7 @@ def test_composed_volumes(self, *args):
         expected_volumes = [
             '/cp/app1:/cp',
             '/command_files/app1:/command_files',
-            '/dusty_assets:/dusty_assets',
+            '/persist/dusty_assets:/dusty_assets',
             '/Users/gc/app1:/gc/app1',
             '/Users/gc/lib1:/gc/lib1',
             '/Users/gc/lib2:/gc/lib2'
@@ -102,14 +102,14 @@ def testget_app_volume_mounts_1(self, *args):
             '/command_files/app1:/command_files',
             '/Users/gc/lib1:/gc/lib1',
             '/Users/gc/lib2:/gc/lib2',
-            '/dusty_assets:/dusty_assets',
+            '/persist/dusty_assets:/dusty_assets',
         ]
         returned_volumes = get_app_volume_mounts('app1', basic_specs)
         self.assertEqual(set(expected_volumes), set(returned_volumes))
 
     def testget_app_volume_mounts_2(self, *args):
         expected_volumes = ['/Users/gc/app2:/gc/app2',
-                            '/command_files/app2:/command_files', '/dusty_assets:/dusty_assets']
+                            '/command_files/app2:/command_files', '/persist/dusty_assets:/dusty_assets']
         returned_volumes = get_app_volume_mounts('app2', basic_specs)
         self.assertEqual(set(expected_volumes), set(returned_volumes))
 
@@ -165,7 +165,7 @@ def test_composed_app(self, *args):
             'volumes': [
                 '/cp/app1:/cp',
                 '/command_files/app1:/command_files',
-                '/dusty_assets:/dusty_assets',
+                '/persist/dusty_assets:/dusty_assets',
                 '/Users/gc/app1:/gc/app1',
                 '/Users/gc/lib1:/gc/lib1',
                 '/Users/gc/lib2:/gc/lib2'

tests/unit/systems/docker/testing_images_test.py

@@ -119,7 +119,7 @@ def test_make_installed_testing_image(self, fake_make_installed_image, fake_ensu
         fake_ensure_base_image.assert_has_calls([call(self.specs['apps']['fake-app']['test'])])
         fake_make_installed_image.assert_has_calls([call('dusty_testing/image',
                                                          'sh /command_files/dusty_command_file_fake-app.sh', new_image_name,
-                                                         ['/command_files/fake-app/test:/command_files', '/dusty_assets:/dusty_assets', '/dusty_repos/tmp/fake-repo:/repo'])])
+                                                         ['/command_files/fake-app/test:/command_files', '/persist/dusty_assets:/dusty_assets', '/dusty_repos/tmp/fake-repo:/repo'])])
 
     @patch('dusty.systems.docker.testing_image.get_docker_client')
     @patch('dusty.systems.docker.testing_image._make_installed_testing_image')