Merge pull request #567 from gamechanger/js-git-known-hosts

Js git known hosts

docs/changelog.md

@@ -2,6 +2,9 @@
 
 ## 0.6.1 (In Progress)
 
+* **Misc**
+    * Dusty now adds any known hosts specified in the `repo` field of specs to root's known host file.  Previously, users had to manually add hosts as root, or Dusty couldn't pull repos other than GitHub.
+
 
 ## 0.6.0 (September 8, 2015)
 

dusty/commands/repos.py

@@ -1,4 +1,5 @@
 import os
+import urlparse
 
 from prettytable import PrettyTable
 
@@ -9,6 +10,7 @@
 from .. import constants
 from ..payload import daemon_command
 from ..parallel import parallel_task_queue
+from ..systems.known_hosts import ensure_known_hosts
 
 @daemon_command
 def list_repos():
@@ -41,7 +43,6 @@ def _manage_repo(repo_name):
     else:
         log_to_client('No overriden repos found by name {}'.format(repo_name))
 
-
 @daemon_command
 def manage_repo(repo_name):
     _manage_repo(repo_name)
@@ -60,18 +61,34 @@ def override_repos_from_directory(source_path):
         if os.path.isdir(repo_path):
             override_repo(repo.remote_path, repo_path)
 
+def add_known_hosts_for_repos(repos):
+    hosts = set()
+    for repo in repos:
+        assembled_remote_path = repo.assemble_remote_path()
+        if assembled_remote_path.startswith('ssh://'):
+            parsed = urlparse.urlparse(assembled_remote_path)
+            if not parsed.hostname:
+                raise RuntimeError('Unable to parse hostname from repo {}'.format(repo.remote_path))
+            hosts.add(parsed.hostname)
+    ensure_known_hosts(hosts)
+
+def update_specs_repo_and_known_hosts():
+    specs_repo = get_specs_repo()
+    if not specs_repo.is_overridden:
+        log_to_client('Updating managed copy of specs-repo before loading specs')
+        add_known_hosts_for_repos([specs_repo])
+        specs_repo.update_local_repo()
+    add_known_hosts_for_repos(get_all_repos(active_only=True, include_specs_repo=False))
+
 @daemon_command
 def update_managed_repos(force=False):
     """For any active, managed repos, update the Dusty-managed
     copy to bring it up to date with the latest master."""
     log_to_client('Pulling latest updates for all active managed repos:')
-    overrides = set(get_config_value(constants.CONFIG_REPO_OVERRIDES_KEY))
-    specs_repo = get_specs_repo()
-    if not specs_repo.is_overridden:
-        log_to_client('Updating managed copy of specs-repo before loading specs')
-        specs_repo.update_local_repo()
+    update_specs_repo_and_known_hosts()
+    repos_to_update = get_all_repos(active_only=True, include_specs_repo=False)
     with parallel_task_queue() as queue:
         log_to_client('Updating managed repos')
-        for repo in get_all_repos(active_only=True, include_specs_repo=False):
+        for repo in repos_to_update:
             if not repo.is_overridden:
                 repo.update_local_repo_async(queue, force=force)

dusty/preflight.py

@@ -75,18 +75,6 @@ def _ensure_command_files_dir_exists():
     if not os.path.exists(constants.COMMAND_FILES_DIR):
         os.makedirs(constants.COMMAND_FILES_DIR)
 
-def _ensure_github_known_host():
-    ssh_dir = os.path.expanduser('~root/.ssh')
-    if not os.path.isdir(ssh_dir):
-        os.makedirs(ssh_dir)
-    known_hosts_path = os.path.join(ssh_dir, 'known_hosts')
-    with open(known_hosts_path, 'w+') as f:
-        contents = f.read()
-        if 'github.com' not in contents:
-            logging.info('Adding github ssh key to roots ssh known_hosts file')
-            command = ['sh', '-c', 'ssh-keyscan -t rsa github.com >> {}'.format(known_hosts_path)]
-            check_and_log_output_and_error(command, demote=False)
-
 def _check_executables():
     return [check() for check in [_check_git, _check_rsync, _check_virtualbox,
                                   _check_docker_machine, _check_docker, _check_docker_compose]]
@@ -106,7 +94,6 @@ def preflight_check():
     _ensure_run_dir_exists()
     _ensure_config_dir_exists()
     _ensure_command_files_dir_exists()
-    _ensure_github_known_host()
     if not os.path.exists(constants.CONFIG_PATH):
         logging.info('Creating default config file at {}'.format(constants.CONFIG_PATH))
         write_default_config()

dusty/systems/known_hosts/__init__.py

@@ -0,0 +1,28 @@
+import os
+import logging
+
+from ...subprocess import check_output
+
+def _get_known_hosts_path():
+    ssh_dir = os.path.expanduser('~root/.ssh')
+    if not os.path.isdir(ssh_dir):
+        os.makedirs(ssh_dir)
+    return os.path.join(ssh_dir, 'known_hosts')
+
+def ensure_known_hosts(hosts):
+    known_hosts_path = _get_known_hosts_path()
+    modified = False
+    with open(known_hosts_path, 'r+') as f:
+        contents = f.read()
+        if not contents.endswith('\n'):
+            contents += '\n'
+        for host in hosts:
+            if host not in contents:
+                logging.info('Adding {} ssh key to roots ssh known_hosts file'.format(host))
+                command = ['sh', '-c', 'ssh-keyscan -t rsa {}'.format(host)]
+                result = check_output(command, demote=False)
+                contents += result
+                modified = True
+        if modified:
+            f.seek(0)
+            f.write(contents)

tests/unit/commands/repos_test.py

@@ -91,14 +91,16 @@ def test_override_repos_from_directory(self):
                                'github.com/app/b': os.path.join(self.temp_repos_path, 'b'),
                                'github.com/lib/a': os.path.join(self.temp_repos_path, 'a')})
 
+    @patch('dusty.commands.repos.add_known_hosts_for_repos')
     @patch('dusty.source.Repo.update_local_repo_async')
-    def test_update_managed_repos(self, fake_update_local_repo_async):
+    def test_update_managed_repos(self, fake_update_local_repo_async, fake_add_known_hosts):
         activate_bundle(['bundle-a'])
         update_managed_repos()
         fake_update_local_repo_async.assert_has_calls([call(ANY, force=False)])
 
+    @patch('dusty.commands.repos.add_known_hosts_for_repos')
     @patch('dusty.source.Repo.update_local_repo_async')
-    def test_update_managed_repos_for_both(self, fake_update_local_repo_async):
+    def test_update_managed_repos_for_both(self, fake_update_local_repo_async, fake_add_known_hosts):
         activate_bundle(['bundle-a'])
         activate_bundle(['bundle-b'])
         update_managed_repos()

tests/unit/systems/known_hosts/init_test.py

@@ -0,0 +1,57 @@
+import os
+import tempfile
+
+from mock import patch
+
+import dusty.constants
+from dusty.systems.known_hosts import ensure_known_hosts
+from ....testcases import DustyTestCase
+
+@patch('dusty.systems.known_hosts._get_known_hosts_path')
+@patch('dusty.systems.known_hosts.check_output')
+class TestKnownHostsSystem(DustyTestCase):
+    def setUp(self):
+        super(TestKnownHostsSystem, self).setUp()
+        self.temp_hosts_path = tempfile.mkstemp()[1]
+
+    def tearDown(self):
+        super(TestKnownHostsSystem, self).tearDown()
+        os.remove(self.temp_hosts_path)
+
+    def test_preserves_existing_content(self, fake_check_output, fake_get_known_hosts):
+        fake_get_known_hosts.return_value = self.temp_hosts_path
+        fake_check_output.return_value = 'dusty.host:SOMESHA'
+
+        initial_content = 'prev.known.host.1:SOMESHA\nprev.known.host.2:SOMESHA'
+        with open(self.temp_hosts_path, 'w') as f:
+            f.write(initial_content)
+        expected_result_content = 'prev.known.host.1:SOMESHA\nprev.known.host.2:SOMESHA\ndusty.host:SOMESHA'
+
+        ensure_known_hosts(['dusty.host'])
+        with open(self.temp_hosts_path, 'r') as f:
+            self.assertEqual(f.read(), expected_result_content)
+
+    def test_not_modified(self, fake_check_output, fake_get_known_hosts):
+        fake_get_known_hosts.return_value = self.temp_hosts_path
+        fake_check_output.return_value = 'prev.known.host.1:SOMESHA'
+
+        initial_content = 'prev.known.host.1:SOMESHA\nprev.known.host.2:SOMESHA'
+        with open(self.temp_hosts_path, 'w') as f:
+            f.write(initial_content)
+
+        ensure_known_hosts(['prev.known.host.1'])
+        with open(self.temp_hosts_path, 'r') as f:
+            self.assertEqual(f.read(), initial_content)
+
+    def test_redundant_additions(self, fake_check_output, fake_get_known_hosts):
+        fake_get_known_hosts.return_value = self.temp_hosts_path
+        fake_check_output.return_value = 'dusty.host:SOMESHA'
+
+        initial_content = 'prev.known.host.1:SOMESHA\nprev.known.host.2:SOMESHA'
+        with open(self.temp_hosts_path, 'w') as f:
+            f.write(initial_content)
+        expected_result_content = 'prev.known.host.1:SOMESHA\nprev.known.host.2:SOMESHA\ndusty.host:SOMESHA'
+
+        ensure_known_hosts(['dusty.host', 'dusty.host', 'dusty.host'])
+        with open(self.temp_hosts_path, 'r') as f:
+            self.assertEqual(f.read(), expected_result_content)