Merge pull request #123 from cloudify-cosmo/2.0.25-build

some-log-handling

.gitignore

@@ -103,3 +103,4 @@ workspace
 fusion-agent
 fusion-common
 fusion-manager
+cloudify-utilities-plugins-sdk

CHANGELOG.txt

@@ -1,3 +1,4 @@
+2.0.25: Implement getting stdout/stderr while execution.
 2.0.24: Rerelease with requirements change.
 2.0.23: adjust python 3.11 patch.
 2.0.22: Add DSL 1.5 plugin YAML.

fabric_plugin/__version__.py

@@ -1 +1 @@
-version = '2.0.24'
+version = '2.0.25'

fabric_plugin/tasks.py

@@ -30,6 +30,7 @@
     Config,
 )
 from invoke import Task
+from invoke.watchers import StreamWatcher
 from paramiko import RSAKey, ECDSAKey, SSHException
 
 # This is done because on 5.0.5 manager and older we will have
@@ -49,6 +50,8 @@
 from cloudify.proxy import client as proxy_client
 from cloudify.proxy import server as proxy_server
 from cloudify.exceptions import NonRecoverableError
+from cloudify.state import current_ctx
+from cloudify.utils import _get_current_context
 
 # This is done for 5.0.5 and older utils backward compatibility
 try:
@@ -94,6 +97,28 @@
 }
 
 
+# Class to handle stdout from fabric Connection
+class OutputWatcher(StreamWatcher):
+    def __init__(self, ctx):
+        self.ctx = ctx
+        self.already_logged = []
+
+    def _log_output(self, lines):
+        for line in lines:
+            self.ctx.logger.debug(line.rstrip())
+
+    def submit(self, stream):
+        lines = stream.splitlines()
+        lines = lines[len(self.already_logged):]
+        if self.ctx is not None:
+            with current_ctx.push(self.ctx):
+                self._log_output(lines)
+        else:
+            self._log_output(lines)
+        self.already_logged.extend(lines)
+        return []
+
+
 # inspired by fabric 1.x https://github.com/fabric/fabric/blob/1.10/fabric/utils.py#L186 # NOQA
 class _AttributeDict(dict):
     """
@@ -239,7 +264,7 @@ def _hide_or_display_results(hide_value, result):
     elif hide_value == "out":
         _log_output(ctx, result.stderr, prefix='<err> ')
     elif hide_value == 'err':
-        _log_output(ctx, result.stdout, prefix='<err> ')
+        _log_output(ctx, result.stdout, prefix='<out> ')
 
 
 def put_host(fabric_env, host_ip=None):
@@ -288,7 +313,7 @@ def prepare_fabric2_env(fabric2_env, fabric_env, connect_kwargs):
 
 
 @contextmanager
-def ssh_connection(ctx, fabric_env):
+def ssh_connection(ctx, fabric_env, hide_output=True):
     """Make and establish a fabric ssh connection.
 
     :param ctx: cloudify operation context
@@ -337,6 +362,11 @@ def ssh_connection(ctx, fabric_env):
     if fabric_env.connect_timeout != 10:
         config["timeouts"]['connect'] = fabric_env.connect_timeout
 
+    # if we are not hiding let's show response stream as we get it
+    if not hide_output:
+        config['run'] = {}
+        config['run']['watchers'] = [OutputWatcher(_get_current_context())]
+
     fabric_env_config = {
         'host': host,
         'user': fabric2_env['user'],
@@ -357,6 +387,26 @@ def f(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
+            ctx.logger.error('Exception Happend: {0}'.format(str(e)))
+            if hasattr(e, 'result'):
+                fab_err_code = \
+                    e.result.return_code if hasattr(e.result, 'return_code') \
+                    else 'NA'
+                fab_stdout = \
+                    e.result.stdout if hasattr(e.result, 'stdout') else 'NA'
+                fab_stderr = \
+                    e.result.stderr if hasattr(e.result, 'stderr') else 'NA'
+                ctx.logger.error(
+                    'Fabric Results:\n'
+                    '---------------------------------'
+                    '\nreturn_code: {0}\n'
+                    '---------------------------------'
+                    '\nstdout:\n{1}\n'
+                    '---------------------------------'
+                    '\nstderr:\n{2}\n'
+                    '---------------------------------'.format(
+                        fab_err_code, fab_stdout, fab_stderr))
+
             exit_codes = kwargs.get('non_recoverable_error_exit_codes', [])
             if hasattr(e, 'result')\
                     and e.result.return_code in exit_codes:
@@ -447,7 +497,7 @@ def run_commands(ctx,
     """
 
     hide_value = _resolve_hide_value(kwargs.get('hide_output'))
-    with ssh_connection(ctx, fabric_env) as conn:
+    with ssh_connection(ctx, fabric_env, hide_value) as conn:
         for command in commands:
             ctx.logger.info('Running command: {0}'.format(command))
             run, command = handle_sudo(conn, use_sudo, command)
@@ -671,7 +721,7 @@ def run_script(ctx,
     args = process.get('args')
     command_prefix = process.get('command_prefix')
     hide_value = _resolve_hide_value(kwargs.get('hide_output'))
-    with ssh_connection(ctx, fabric_env) as conn, \
+    with ssh_connection(ctx, fabric_env, hide_value) as conn, \
             _RemoteFiles(
                 conn,
                 local_script_path,

fabric_plugin/tests/test_fabric_plugin.py

@@ -359,7 +359,9 @@ def _test_run_commands_non_recoverable(self, use_sudo=False):
             run = 'run'
             setattr(
                 getattr(connection, run), 'side_effect',
-                CustomError({'return_code': 1})
+                CustomError({'return_code': 1,
+                             'stdout': '',
+                             'stderr': 'error'})
             )
             self._execute(
                 'test.run_commands',
@@ -465,4 +467,4 @@ def __init__(self, result):
 
 
 def raise_custom_error(a, b, c, d):
-    raise CustomError({'return_code': 1})
+    raise CustomError({'return_code': 1, 'stdout': '', 'stderr': 'error'})

plugin.yaml

@@ -5,4 +5,4 @@ plugins:
     fabric:
         executor: central_deployment_agent
         package_name: cloudify-fabric-plugin
-        package_version: '2.0.24'
+        package_version: '2.0.25'

plugin_1_4.yaml

@@ -5,7 +5,7 @@ plugins:
     fabric:
         executor: central_deployment_agent
         package_name: cloudify-fabric-plugin
-        package_version: '2.0.24'
+        package_version: '2.0.25'
 
 blueprint_labels:
   obj-type:

plugin_1_5.yaml

@@ -5,7 +5,7 @@ plugins:
     fabric:
         executor: central_deployment_agent
         package_name: cloudify-fabric-plugin
-        package_version: '2.0.24'
+        package_version: '2.0.25'
 
 blueprint_labels:
   obj-type:

requirements.in

@@ -1,5 +1,4 @@
 -e fusion-common
 -e fusion-manager/mgmtworker
 -e fusion-agent
-cryptography>=41.0.5
-networkx>=2.5.1,<2.6
+cryptography>=41.0.5

requirements.txt

@@ -20,35 +20,54 @@ aiohttp==3.9.1
     # via fusion-common
 aiosignal==1.3.1
     # via aiohttp
+amqp==5.2.0
+    # via kombu
 appdirs==1.4.3
     # via fusion-agent
 attrs==23.1.0
     # via aiohttp
 bcrypt==4.1.1
     # via paramiko
+billiard==4.2.0
+    # via celery
 bottle==0.12.25
     # via fusion-common
+celery==5.3.6
+    # via fusion-agent
 certifi==2023.11.17
-    # via
-    #   fusion-mgmtworker
-    #   requests
-cffi==1.16.0
+    # via requests
+cffi==1.14.6
     # via
     #   cryptography
+    #   fusion-agent
     #   pynacl
+    #   python-gssapi
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
-    # via fusion-agent
+    # via
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+    #   fusion-agent
+click-didyoumean==0.3.0
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.3.0
+    # via celery
 cryptography==41.0.7
     # via
     #   -r requirements.in
     #   fusion-mgmtworker
     #   paramiko
-decorator==4.4.2
-    # via networkx
+    #   pyspnego
+    #   requests-ntlm
 distro==1.8.0
     # via fusion-common
+fabric==2.7.1
+    # via fusion-agent
 fabric2==2.5.0
     # via cloudify-fabric-plugin (setup.py)
 fasteners==0.19
@@ -62,67 +81,119 @@ idna==3.6
     #   requests
     #   yarl
 invoke==1.7.3
-    # via fabric2
+    # via
+    #   fabric
+    #   fabric2
 jinja2==3.1.2
     # via
     #   fusion-agent
     #   fusion-common
+kombu==5.3.4
+    # via celery
 markupsafe==2.1.3
     # via jinja2
 multidict==6.0.4
     # via
     #   aiohttp
     #   yarl
-networkx==2.5.1
-    # via
-    #   -r requirements.in
-    #   fusion-common
+networkx==3.2.1
+    # via fusion-common
 packaging==21.3
     # via
     #   fusion-agent
     #   fusion-mgmtworker
 paramiko==3.3.1
-    # via fabric2
+    # via
+    #   fabric
+    #   fabric2
+pathlib2==2.3.7.post1
+    # via fabric
 pika==1.3.2
     # via fusion-common
 pkginfo==1.9.6
     # via wagon
+ply==3.11
+    # via pysmi
+prompt-toolkit==3.0.41
+    # via click-repl
 proxy-tools==0.1.0
     # via fusion-common
 psycopg2==2.9.9
     # via fusion-mgmtworker
+pyasn1==0.5.1
+    # via
+    #   pysnmp
+    #   python-gssapi
 pycparser==2.21
     # via cffi
+pycryptodomex==3.19.0
+    # via pysnmp
+pykerberos==1.2.4
+    # via pywinrm
 pynacl==1.5.0
-    # via paramiko
+    # via
+    #   fusion-agent
+    #   paramiko
 pyparsing==3.1.1
     # via packaging
+pysmi==0.3.4
+    # via pysnmp
+pysnmp==4.4.12
+    # via fusion-common
+pyspnego==0.10.2
+    # via requests-ntlm
 python-dateutil==2.8.2
-    # via fusion-mgmtworker
+    # via
+    #   celery
+    #   fusion-mgmtworker
+python-gssapi==0.6.4
+    # via fusion-agent
 pytz==2023.3.post1
     # via
     #   fusion-common
     #   fusion-mgmtworker
-pyyaml==6.0
+pywinrm[kerberos]==0.4.3
+    # via
+    #   fusion-agent
+    #   pywinrm
+pyyaml==6.0.1
     # via fusion-common
 requests==2.31.0
     # via
     #   fusion-agent
     #   fusion-common
+    #   pywinrm
+    #   requests-ntlm
     #   requests-toolbelt
+requests-ntlm==1.2.0
+    # via pywinrm
 requests-toolbelt==1.0.0
     # via fusion-common
 retrying==1.3.4
     # via fusion-mgmtworker
 six==1.16.0
     # via
+    #   pathlib2
     #   python-dateutil
+    #   python-gssapi
+    #   pywinrm
     #   retrying
+tzdata==2023.3
+    # via celery
 urllib3==2.1.0
     # via requests
+vine==5.1.0
+    # via
+    #   amqp
+    #   celery
+    #   kombu
 wagon==1.0.1
     # via fusion-common
+wcwidth==0.2.12
+    # via prompt-toolkit
 wheel==0.42.0
     # via wagon
+xmltodict==0.13.0
+    # via pywinrm
 yarl==1.9.3
     # via aiohttp

v2_plugin.yaml

@@ -5,7 +5,7 @@ plugins:
     fabric:
         executor: central_deployment_agent
         package_name: cloudify-fabric-plugin
-        package_version: '2.0.24'
+        package_version: '2.0.25'
 
 blueprint_labels:
   obj-type: