From 29705f0e81667ab4ca598179ee1d5810956810c9 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 5 Nov 2024 18:21:36 +0200 Subject: [PATCH 01/39] adds simple implementation of `decompress` module --- plugins/modules/decompress.py | 74 +++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 plugins/modules/decompress.py diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py new file mode 100644 index 00000000000..4119feafb46 --- /dev/null +++ b/plugins/modules/decompress.py @@ -0,0 +1,74 @@ +import abc +import bz2 +import gzip +import lzma +import os +import shutil + +from ansible.module_utils import six +from ansible.module_utils.basic import AnsibleModule + + +@six.add_metaclass(abc.ABCMeta) +class Decompress(abc.ABC): + @abc.abstractmethod + def decompress(self, src, dest): + pass + + +class GzDecompress(Decompress): + def decompress(self, src, dest): + with gzip.open(src, "rb") as src_file: + with open(dest, "wb") as dest_file: + shutil.copyfileobj(src_file, dest_file) + + +class Bz2Decompress(Decompress): + + def decompress(self, src, dest): + with bz2.open(src, "rb") as src_file: + with open(dest, "wb") as dest_file: + shutil.copyfileobj(src_file, dest_file) + + +class LZMADecompress(Decompress): + + def decompress(self, src, dest): + with lzma.open(src, "rb") as src_file: + with open(dest, "wb") as dest_file: + shutil.copyfileobj(src_file, dest_file) + + +def main(): + result = dict(changed=False, diff=dict(before=dict(), after=dict())) + + compressors = {"gz": GzDecompress, "bz2": Bz2Decompress, "xz": LZMADecompress} + + module = AnsibleModule( + argument_spec=dict( + src=dict(type='str', required=True), + dest=dict(type='str', required=True), + format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']) + ) + ) + + src = module.params['src'] + dest = module.params['dest'] + format = module.params['format'] + + src = os.path.abspath(src) + dest = os.path.abspath(dest) + + compressor = compressors[format] + + if compressor is not None: + obj = compressor() + obj.decompress(src, dest) + result['msg'] = 'success' + module.exit_json(**result) + else: + module.fail_json(msg="Compressor not found.") + + +if __name__ == '__main__': + main() From ecac887f5b816be95ae267d43b68821586a42412 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Wed, 6 Nov 2024 00:15:56 +0200 Subject: [PATCH 02/39] adds simple test, fixes src and dest arg types --- plugins/modules/decompress.py | 4 ++-- tests/unit/plugins/modules/test_decompress.py | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/unit/plugins/modules/test_decompress.py diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 4119feafb46..ee317a72e9b 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -46,8 +46,8 @@ def main(): module = AnsibleModule( argument_spec=dict( - src=dict(type='str', required=True), - dest=dict(type='str', required=True), + src=dict(type='path', required=True), + dest=dict(type='path', required=True), format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']) ) ) diff --git a/tests/unit/plugins/modules/test_decompress.py b/tests/unit/plugins/modules/test_decompress.py new file mode 100644 index 00000000000..652595358e2 --- /dev/null +++ b/tests/unit/plugins/modules/test_decompress.py @@ -0,0 +1,22 @@ +import pytest +from ansible.module_utils import basic + +from ansible_collections.community.general.plugins.modules import decompress +from ansible_collections.community.general.tests.unit.plugins.modules.utils import AnsibleFailJson, set_module_args, \ + fail_json, exit_json + + +class TestDecompress: + @pytest.fixture(autouse=True) + def common(self, mocker): + self.mock_module = mocker.patch.multiple( + basic.AnsibleModule, + exit_json=exit_json, + fail_json=fail_json + ) + + def test_fail(self): + with pytest.raises(AnsibleFailJson) as e: + set_module_args({}) + decompress.main() + assert e.match("missing required arguments: dest, src") From 0473131a82d952b8346cc926253667474dddc6ab Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Sun, 10 Nov 2024 19:33:53 +0200 Subject: [PATCH 03/39] minor refactoring --- plugins/modules/decompress.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index ee317a72e9b..ebcd3b0b0af 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -11,32 +11,30 @@ @six.add_metaclass(abc.ABCMeta) class Decompress(abc.ABC): - @abc.abstractmethod def decompress(self, src, dest): + with self._compression_file(src) as src_file: + with open(dest, "wb") as dest_file: + shutil.copyfileobj(src_file, dest_file) + + @abc.abstractmethod + def _compression_file(self, src): pass class GzDecompress(Decompress): - def decompress(self, src, dest): - with gzip.open(src, "rb") as src_file: - with open(dest, "wb") as dest_file: - shutil.copyfileobj(src_file, dest_file) + def _compression_file(self, src): + return gzip.open(src, "rb") class Bz2Decompress(Decompress): - def decompress(self, src, dest): - with bz2.open(src, "rb") as src_file: - with open(dest, "wb") as dest_file: - shutil.copyfileobj(src_file, dest_file) + def _compression_file(self, src): + return bz2.open(src, "rb") class LZMADecompress(Decompress): - - def decompress(self, src, dest): - with lzma.open(src, "rb") as src_file: - with open(dest, "wb") as dest_file: - shutil.copyfileobj(src_file, dest_file) + def _compression_file(self, src): + return lzma.open(src, "rb") def main(): From f9ff4cd12ea0bf2d07edc3412abe14c8f0bff2ff Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 12 Nov 2024 17:23:28 +0200 Subject: [PATCH 04/39] adds support for common file operations adds integration test for gz decompressing --- plugins/modules/decompress.py | 26 ++++++----- .../targets/decompress/files/file.txt | 1 + .../targets/decompress/files/file.txt.bz2 | Bin 0 -> 56 bytes .../targets/decompress/files/file.txt.gz | Bin 0 -> 46 bytes .../targets/decompress/files/file.txt.xz | Bin 0 -> 76 bytes .../targets/decompress/meta/main.yml | 7 +++ .../targets/decompress/tasks/main.yml | 44 ++++++++++++++++++ tests/unit/plugins/modules/test_decompress.py | 22 --------- 8 files changed, 66 insertions(+), 34 deletions(-) create mode 100644 tests/integration/targets/decompress/files/file.txt create mode 100644 tests/integration/targets/decompress/files/file.txt.bz2 create mode 100644 tests/integration/targets/decompress/files/file.txt.gz create mode 100644 tests/integration/targets/decompress/files/file.txt.xz create mode 100644 tests/integration/targets/decompress/meta/main.yml create mode 100644 tests/integration/targets/decompress/tasks/main.yml delete mode 100644 tests/unit/plugins/modules/test_decompress.py diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index ebcd3b0b0af..c00cf7b678f 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -46,26 +46,28 @@ def main(): argument_spec=dict( src=dict(type='path', required=True), dest=dict(type='path', required=True), - format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']) - ) + format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), + ), + add_file_common_args=True ) src = module.params['src'] dest = module.params['dest'] format = module.params['format'] - - src = os.path.abspath(src) dest = os.path.abspath(dest) - compressor = compressors[format] + changed = False - if compressor is not None: - obj = compressor() - obj.decompress(src, dest) - result['msg'] = 'success' - module.exit_json(**result) - else: - module.fail_json(msg="Compressor not found.") + file_args = module.load_file_common_arguments(module.params, path=dest) + compressor = compressors[format] + if compressor is None: + module.fail_json(msg=f"Couldn't decompress '{format}' format.") + + obj = compressor() + obj.decompress(src, dest) + result['msg'] = 'success' + result['changed'] = module.set_fs_attributes_if_different(file_args, changed) + module.exit_json(**result) if __name__ == '__main__': diff --git a/tests/integration/targets/decompress/files/file.txt b/tests/integration/targets/decompress/files/file.txt new file mode 100644 index 00000000000..70c86c750ee --- /dev/null +++ b/tests/integration/targets/decompress/files/file.txt @@ -0,0 +1 @@ +This is sample file \ No newline at end of file diff --git a/tests/integration/targets/decompress/files/file.txt.bz2 b/tests/integration/targets/decompress/files/file.txt.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..f8a6678633253440f388f054ef6811457df5cd09 GIT binary patch literal 56 zcmZ>Y%CIzaj8qGboG%@2!N9;cxxs;fg+V#ZgF%78km0f~v$I1+K>3BV59^*X)x@3@ MdFbR+lN<8@0Jek@ZU6uP literal 0 HcmV?d00001 diff --git a/tests/integration/targets/decompress/files/file.txt.gz b/tests/integration/targets/decompress/files/file.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..8eee6ec09fa4d8d0e0f5125ec28028eb7888602e GIT binary patch literal 46 zcmb2|=HR#yZk*1*oR*oBs#j7`!oYpq^21Q0O1_p)_{ill=8H7SIGK&>}s5mjVASYEJEi)&Tf#Hi( exB>H;8JigxwT0(!dR&s<_PLf3sE2_iG712YU>6er literal 0 HcmV?d00001 diff --git a/tests/integration/targets/decompress/meta/main.yml b/tests/integration/targets/decompress/meta/main.yml new file mode 100644 index 00000000000..982de6eb035 --- /dev/null +++ b/tests/integration/targets/decompress/meta/main.yml @@ -0,0 +1,7 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +dependencies: + - setup_remote_tmp_dir diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml new file mode 100644 index 00000000000..321df70d485 --- /dev/null +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -0,0 +1,44 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Copy original file + copy: + src: file.txt + dest: "{{ remote_tmp_dir }}" + +- name: Calculate original file checksum + stat: + path: "{{ remote_tmp_dir }}/file.txt" + register: orig_file_stat + +- name: Copy test files + copy: + src: "{{ item }}" + dest: "{{ remote_tmp_dir }}/{{ item }}" + with_items: + - file.txt.bz2 + - file.txt.gz + - file.txt.xz + +- name: Simple decompress + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_from_gz.txt" + format: gz + +- name: Assert that file exists + file: + path: "{{ remote_tmp_dir }}/file_from_gz.txt" + state: file + +- name: Calculate file checksum + stat: + path: "{{ remote_tmp_dir }}/file_from_gz.txt" + register: decompressed_file_stat + +- name: Check that file was decompressed correctly + assert: + that: orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum + diff --git a/tests/unit/plugins/modules/test_decompress.py b/tests/unit/plugins/modules/test_decompress.py deleted file mode 100644 index 652595358e2..00000000000 --- a/tests/unit/plugins/modules/test_decompress.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -from ansible.module_utils import basic - -from ansible_collections.community.general.plugins.modules import decompress -from ansible_collections.community.general.tests.unit.plugins.modules.utils import AnsibleFailJson, set_module_args, \ - fail_json, exit_json - - -class TestDecompress: - @pytest.fixture(autouse=True) - def common(self, mocker): - self.mock_module = mocker.patch.multiple( - basic.AnsibleModule, - exit_json=exit_json, - fail_json=fail_json - ) - - def test_fail(self): - with pytest.raises(AnsibleFailJson) as e: - set_module_args({}) - decompress.main() - assert e.match("missing required arguments: dest, src") From b17d3447facd94f01ca3820f87026b13c5d133d1 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 12 Nov 2024 18:32:33 +0200 Subject: [PATCH 05/39] makes tests parametrized to test all supported compression formats --- .../targets/decompress/tasks/main.yml | 34 ++++++++----------- .../targets/decompress/tasks/tests.yml | 24 +++++++++++++ 2 files changed, 38 insertions(+), 20 deletions(-) create mode 100644 tests/integration/targets/decompress/tasks/tests.yml diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 321df70d485..73dd622e24e 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -22,23 +22,17 @@ - file.txt.gz - file.txt.xz -- name: Simple decompress - community.general.decompress: - src: "{{ remote_tmp_dir }}/file.txt.gz" - dest: "{{ remote_tmp_dir }}/file_from_gz.txt" - format: gz - -- name: Assert that file exists - file: - path: "{{ remote_tmp_dir }}/file_from_gz.txt" - state: file - -- name: Calculate file checksum - stat: - path: "{{ remote_tmp_dir }}/file_from_gz.txt" - register: decompressed_file_stat - -- name: Check that file was decompressed correctly - assert: - that: orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum - +- name: Set supported formats + set_fact: + formats: + - bz2 + - gz + - xz + +# Run tests +- name: Run tests + include_tasks: + file: tests.yml + loop: "{{ formats }}" + loop_control: + loop_var: format diff --git a/tests/integration/targets/decompress/tasks/tests.yml b/tests/integration/targets/decompress/tasks/tests.yml new file mode 100644 index 00000000000..c9cee5ce42e --- /dev/null +++ b/tests/integration/targets/decompress/tasks/tests.yml @@ -0,0 +1,24 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Simple decompress ({{ format }} test) + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + +- name: Assert that file exists ({{ format }} test) + file: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + state: file + +- name: Calculate file checksums ({{ format }} test) + stat: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + register: decompressed_file_stat + +- name: Check that file was decompressed correctly ({{ format }} test) + assert: + that: orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum \ No newline at end of file From a2c85c69afdc64be99954041eb8a2d23e4d81163 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Wed, 13 Nov 2024 19:47:05 +0200 Subject: [PATCH 06/39] checks that target file exists --- plugins/modules/decompress.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index c00cf7b678f..4ff9bf486bb 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -52,6 +52,8 @@ def main(): ) src = module.params['src'] + if not os.path.exists(src): + module.fail_json(msg="Path does not exist: %s" % src) dest = module.params['dest'] format = module.params['format'] dest = os.path.abspath(dest) From a578fbacc94b7bf0932100c0174c68194d250d4b Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Wed, 13 Nov 2024 21:40:15 +0200 Subject: [PATCH 07/39] writes to decompressed file now uses atomic_move --- plugins/modules/decompress.py | 13 ++++++++++++- .../targets/decompress/tasks/tests.yml | 17 ++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 4ff9bf486bb..c6000af8b7f 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -4,9 +4,11 @@ import lzma import os import shutil +import tempfile from ansible.module_utils import six from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.common.text.converters import to_native @six.add_metaclass(abc.ABCMeta) @@ -66,7 +68,16 @@ def main(): module.fail_json(msg=f"Couldn't decompress '{format}' format.") obj = compressor() - obj.decompress(src, dest) + try: + tempd, temppath = tempfile.mkstemp() + obj.decompress(src, tempd) + except OSError as e: + module.fail_json(msg="Unable to create temporary file %s" % to_native(e)) + + try: + module.atomic_move(temppath, dest) + except OSError: + module.fail_json(msg="Unable to move temporary file %s to %s" % (temppath, dest)) result['msg'] = 'success' result['changed'] = module.set_fs_attributes_if_different(file_args, changed) module.exit_json(**result) diff --git a/tests/integration/targets/decompress/tasks/tests.yml b/tests/integration/targets/decompress/tasks/tests.yml index c9cee5ce42e..83d293dfd37 100644 --- a/tests/integration/targets/decompress/tasks/tests.yml +++ b/tests/integration/targets/decompress/tasks/tests.yml @@ -3,22 +3,25 @@ # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later +- name: Set mode for decompressed file + set_fact: + decompressed_mode: "0640" + - name: Simple decompress ({{ format }} test) community.general.decompress: src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" format: "{{ format }}" + mode: "{{ decompressed_mode }}" -- name: Assert that file exists ({{ format }} test) - file: - path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - state: file - -- name: Calculate file checksums ({{ format }} test) +- name: Stat decompressed file ({{ format }} test) stat: path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" register: decompressed_file_stat - name: Check that file was decompressed correctly ({{ format }} test) assert: - that: orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum \ No newline at end of file + that: + - decompressed_file_stat.stat.exists + - decompressed_file_stat.stat.mode == decompressed_mode + - orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum From 958008432bbb3c62f1767f5a8859a5d807a70814 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Thu, 14 Nov 2024 19:42:25 +0200 Subject: [PATCH 08/39] adds idempotency for decompression --- plugins/modules/decompress.py | 25 ++++++++++++----- .../targets/decompress/files/second_file.txt | 1 + .../decompress/files/second_file.txt.bz2 | Bin 0 -> 79 bytes .../decompress/files/second_file.txt.gz | Bin 0 -> 84 bytes .../decompress/files/second_file.txt.xz | Bin 0 -> 108 bytes .../targets/decompress/tasks/main.yml | 15 ++-------- .../targets/decompress/tasks/tests.yml | 26 +++++++++++++++++- 7 files changed, 47 insertions(+), 20 deletions(-) create mode 100644 tests/integration/targets/decompress/files/second_file.txt create mode 100644 tests/integration/targets/decompress/files/second_file.txt.bz2 create mode 100644 tests/integration/targets/decompress/files/second_file.txt.gz create mode 100644 tests/integration/targets/decompress/files/second_file.txt.xz diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index c6000af8b7f..e46892da8cb 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -1,5 +1,6 @@ import abc import bz2 +import filecmp import gzip import lzma import os @@ -39,6 +40,10 @@ def _compression_file(self, src): return lzma.open(src, "rb") +def is_dest_changed(src, dest): + return not filecmp.cmp(src, dest, shallow=False) + + def main(): result = dict(changed=False, diff=dict(before=dict(), after=dict())) @@ -60,8 +65,6 @@ def main(): format = module.params['format'] dest = os.path.abspath(dest) - changed = False - file_args = module.load_file_common_arguments(module.params, path=dest) compressor = compressors[format] if compressor is None: @@ -69,15 +72,23 @@ def main(): obj = compressor() try: - tempd, temppath = tempfile.mkstemp() + tempd, temppath = tempfile.mkstemp(dir=module.tmpdir) obj.decompress(src, tempd) except OSError as e: module.fail_json(msg="Unable to create temporary file %s" % to_native(e)) - try: - module.atomic_move(temppath, dest) - except OSError: - module.fail_json(msg="Unable to move temporary file %s to %s" % (temppath, dest)) + if os.path.exists(dest): + changed = is_dest_changed(temppath, dest) + else: + changed = True + if changed: + try: + module.atomic_move(temppath, dest) + except OSError: + module.fail_json(msg="Unable to move temporary file %s to %s" % (temppath, dest)) + + if os.path.exists(temppath): + os.unlink(temppath) result['msg'] = 'success' result['changed'] = module.set_fs_attributes_if_different(file_args, changed) module.exit_json(**result) diff --git a/tests/integration/targets/decompress/files/second_file.txt b/tests/integration/targets/decompress/files/second_file.txt new file mode 100644 index 00000000000..48383aaa09e --- /dev/null +++ b/tests/integration/targets/decompress/files/second_file.txt @@ -0,0 +1 @@ +Content of this file must differ from the 'file.txt' \ No newline at end of file diff --git a/tests/integration/targets/decompress/files/second_file.txt.bz2 b/tests/integration/targets/decompress/files/second_file.txt.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..e0a610f4300553c49de45754660059cf3fde7ed7 GIT binary patch literal 79 zcmV-V0I>f;T4*^jL0KkKS!n?1%m4rdm4HBj2ml9XoWAlCo@ literal 0 HcmV?d00001 diff --git a/tests/integration/targets/decompress/files/second_file.txt.gz b/tests/integration/targets/decompress/files/second_file.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..c784e0c56b9a9ddb99d4408b86325b10edf27a33 GIT binary patch literal 84 zcmV-a0IUBWiwFoJF*aua19N3#Z*F8?W@&6?E_8Tw0CUdID@n~OQOHkID9OkyR!Gat qNma-#EiO?=$xKU2EmBA;%FhK#r7Ea{r1VNEO4I><<-&e60000E?jR5V literal 0 HcmV?d00001 diff --git a/tests/integration/targets/decompress/files/second_file.txt.xz b/tests/integration/targets/decompress/files/second_file.txt.xz new file mode 100644 index 0000000000000000000000000000000000000000..4cfb41b368c2a4e504a659e3f7e9239840b8845d GIT binary patch literal 108 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8H}Ct^GZ_lN)+{!-Q42P2W?=L&neO;6R({*(T1KFm I3@njR0F>(^o&W#< literal 0 HcmV?d00001 diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 73dd622e24e..eb75cdcc0ff 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -3,25 +3,16 @@ # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later -- name: Copy original file +- name: Copy test files copy: - src: file.txt + src: "files/" dest: "{{ remote_tmp_dir }}" -- name: Calculate original file checksum +- name: Get original file stat stat: path: "{{ remote_tmp_dir }}/file.txt" register: orig_file_stat -- name: Copy test files - copy: - src: "{{ item }}" - dest: "{{ remote_tmp_dir }}/{{ item }}" - with_items: - - file.txt.bz2 - - file.txt.gz - - file.txt.xz - - name: Set supported formats set_fact: formats: diff --git a/tests/integration/targets/decompress/tasks/tests.yml b/tests/integration/targets/decompress/tasks/tests.yml index 83d293dfd37..8e34bfd0929 100644 --- a/tests/integration/targets/decompress/tasks/tests.yml +++ b/tests/integration/targets/decompress/tasks/tests.yml @@ -3,7 +3,7 @@ # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later -- name: Set mode for decompressed file +- name: Set mode for decompressed file ({{ format }} test) set_fact: decompressed_mode: "0640" @@ -13,6 +13,7 @@ dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" format: "{{ format }}" mode: "{{ decompressed_mode }}" + register: first_decompression - name: Stat decompressed file ({{ format }} test) stat: @@ -22,6 +23,29 @@ - name: Check that file was decompressed correctly ({{ format }} test) assert: that: + - first_decompression.changed - decompressed_file_stat.stat.exists - decompressed_file_stat.stat.mode == decompressed_mode - orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum + +- name: Decompress second time ({{ format }} test) + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + mode: "{{ decompressed_mode }}" + register: same_file_decompression + +- name: Decompress new file to the same location ({{ format }} test) + community.general.decompress: + src: "{{ remote_tmp_dir }}/second_file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + mode: "{{ decompressed_mode }}" + register: new_file_decompression + +- name: Idempotency test + assert: + that: + - not same_file_decompression.changed + - new_file_decompression.changed From 8db0256c9aac7e2cb1ff62c777a310d60a3defb7 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Thu, 14 Nov 2024 21:01:01 +0200 Subject: [PATCH 09/39] refactoring, removed classes --- plugins/modules/decompress.py | 44 ++++++++++++----------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index e46892da8cb..eb220b01097 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -1,4 +1,3 @@ -import abc import bz2 import filecmp import gzip @@ -7,37 +6,26 @@ import shutil import tempfile -from ansible.module_utils import six from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.common.text.converters import to_native -@six.add_metaclass(abc.ABCMeta) -class Decompress(abc.ABC): - def decompress(self, src, dest): - with self._compression_file(src) as src_file: - with open(dest, "wb") as dest_file: - shutil.copyfileobj(src_file, dest_file) +def lzma_decompress(src): + return lzma.open(src, "rb") - @abc.abstractmethod - def _compression_file(self, src): - pass +def bz2_decompress(src): + return bz2.open(src, "rb") -class GzDecompress(Decompress): - def _compression_file(self, src): - return gzip.open(src, "rb") +def gzip_decompress(src): + return gzip.open(src, "rb") -class Bz2Decompress(Decompress): - def _compression_file(self, src): - return bz2.open(src, "rb") - - -class LZMADecompress(Decompress): - def _compression_file(self, src): - return lzma.open(src, "rb") +def decompress(src, dest, handler): + with handler(src) as src_file: + with open(dest, "wb") as dest_file: + shutil.copyfileobj(src_file, dest_file) def is_dest_changed(src, dest): @@ -47,7 +35,7 @@ def is_dest_changed(src, dest): def main(): result = dict(changed=False, diff=dict(before=dict(), after=dict())) - compressors = {"gz": GzDecompress, "bz2": Bz2Decompress, "xz": LZMADecompress} + compressors = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} module = AnsibleModule( argument_spec=dict( @@ -63,17 +51,15 @@ def main(): module.fail_json(msg="Path does not exist: %s" % src) dest = module.params['dest'] format = module.params['format'] + if format not in compressors: + module.fail_json(msg=f"Couldn't decompress '%s' format." % format) dest = os.path.abspath(dest) file_args = module.load_file_common_arguments(module.params, path=dest) - compressor = compressors[format] - if compressor is None: - module.fail_json(msg=f"Couldn't decompress '{format}' format.") - - obj = compressor() + handler = compressors[format] try: tempd, temppath = tempfile.mkstemp(dir=module.tmpdir) - obj.decompress(src, tempd) + decompress(src, tempd, handler) except OSError as e: module.fail_json(msg="Unable to create temporary file %s" % to_native(e)) From 21f301df52aa9c1b843b6a06495d524dcc2ee206 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 18 Nov 2024 00:59:36 +0200 Subject: [PATCH 10/39] adds support for check mode --- plugins/modules/decompress.py | 5 +-- .../targets/decompress/tasks/tests.yml | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index eb220b01097..393b93484f2 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -43,7 +43,8 @@ def main(): dest=dict(type='path', required=True), format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), ), - add_file_common_args=True + add_file_common_args=True, + supports_check_mode=True ) src = module.params['src'] @@ -67,7 +68,7 @@ def main(): changed = is_dest_changed(temppath, dest) else: changed = True - if changed: + if changed and not module.check_mode: try: module.atomic_move(temppath, dest) except OSError: diff --git a/tests/integration/targets/decompress/tasks/tests.yml b/tests/integration/targets/decompress/tasks/tests.yml index 8e34bfd0929..bd45217d068 100644 --- a/tests/integration/targets/decompress/tasks/tests.yml +++ b/tests/integration/targets/decompress/tasks/tests.yml @@ -49,3 +49,38 @@ that: - not same_file_decompression.changed - new_file_decompression.changed + +- name: Delete decompressed files + file: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + state: absent + +- name: Decompress with check mode enabled + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + mode: "{{ decompressed_mode }}" + check_mode: true + register: decompressed_check_mode + +- name: Decompress second time with check mode enabled + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + mode: "{{ decompressed_mode }}" + check_mode: true + register: decompressed_check_mode_2 + +- name: Stat non-existing file + stat: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + register: nonexisting_stat + +- name: Check mode test + assert: + that: + - decompressed_check_mode.changed + - decompressed_check_mode_2.changed + - not nonexisting_stat.stat.exists From a5bfe908e18bf5288ce166b1049dd49207b2bb6c Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 18 Nov 2024 15:40:09 +0200 Subject: [PATCH 11/39] adds check for destination file. If it exists and it is a directory, the module returns error --- plugins/modules/decompress.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 393b93484f2..65ec4dceea9 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -48,9 +48,13 @@ def main(): ) src = module.params['src'] - if not os.path.exists(src): - module.fail_json(msg="Path does not exist: %s" % src) dest = module.params['dest'] + if not os.path.exists(src): + module.fail_json(msg="Path does not exist: '%s'" % src) + if os.path.isdir(src): + module.fail_json(msg="Cannot decompress directory '%s'" % src) + if os.path.exists(dest) and os.path.isdir(dest): + module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % dest) format = module.params['format'] if format not in compressors: module.fail_json(msg=f"Couldn't decompress '%s' format." % format) @@ -62,7 +66,7 @@ def main(): tempd, temppath = tempfile.mkstemp(dir=module.tmpdir) decompress(src, tempd, handler) except OSError as e: - module.fail_json(msg="Unable to create temporary file %s" % to_native(e)) + module.fail_json(msg="Unable to create temporary file '%s'" % to_native(e)) if os.path.exists(dest): changed = is_dest_changed(temppath, dest) @@ -72,7 +76,7 @@ def main(): try: module.atomic_move(temppath, dest) except OSError: - module.fail_json(msg="Unable to move temporary file %s to %s" % (temppath, dest)) + module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (temppath, dest)) if os.path.exists(temppath): os.unlink(temppath) From 36bd42170179b001d37abd2708e688e84e624b34 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 18 Nov 2024 23:02:43 +0200 Subject: [PATCH 12/39] refactoring, moves code to a class. Also, simplifies tests (now only tests related to the module core functionality run as parametrized, tests for idempotency and check mode run only for one format) --- plugins/modules/decompress.py | 91 ++++++++++--------- .../targets/decompress/tasks/core.yml | 29 ++++++ .../targets/decompress/tasks/main.yml | 16 +++- .../targets/decompress/tasks/misc.yml | 28 ++++++ .../targets/decompress/tasks/tests.yml | 86 ------------------ 5 files changed, 121 insertions(+), 129 deletions(-) create mode 100644 tests/integration/targets/decompress/tasks/core.yml create mode 100644 tests/integration/targets/decompress/tasks/misc.yml delete mode 100644 tests/integration/targets/decompress/tasks/tests.yml diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 65ec4dceea9..036016b7504 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -28,15 +28,57 @@ def decompress(src, dest, handler): shutil.copyfileobj(src_file, dest_file) -def is_dest_changed(src, dest): - return not filecmp.cmp(src, dest, shallow=False) +class Decompress(object): + def __init__(self, module): + self.src = module.params['src'] + self.dest = module.params['dest'] + self.fmt = module.params['format'] + self.check_mode = module.check_mode + self.module = module + self.changed = False + self.msg = "" + self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} + + def configure(self): + if not os.path.exists(self.src): + self.module.fail_json(msg="Path does not exist: '%s'" % self.src) + if os.path.isdir(self.src): + self.module.fail_json(msg="Cannot decompress directory '%s'" % self.src) + if os.path.exists(self.dest) and os.path.isdir(self.dest): + self.module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % self.dest) + self.fmt = self.module.params['format'] + if self.fmt not in self.handlers: + self.module.fail_json(msg="Couldn't decompress '%s' format" % self.fmt) + + def run(self): + self.configure() + file_args = self.module.load_file_common_arguments(self.module.params, path=self.dest) + handler = self.handlers[self.fmt] + try: + tempfd, temppath = tempfile.mkstemp(dir=self.module.tmpdir) + decompress(self.src, tempfd, handler) + except OSError as e: + self.module.fail_json(msg="Unable to create temporary file '%s'" % to_native(e)) + if os.path.exists(self.dest): + self.changed = not filecmp.cmp(temppath, self.dest, shallow=False) + else: + self.changed = True -def main(): - result = dict(changed=False, diff=dict(before=dict(), after=dict())) + if self.changed and not self.module.check_mode: + try: + self.module.atomic_move(temppath, self.dest) + except OSError: + self.module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (temppath, self.dest)) + + if os.path.exists(temppath): + os.unlink(temppath) - compressors = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} + self.msg = "success" + self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) + +def main(): module = AnsibleModule( argument_spec=dict( src=dict(type='path', required=True), @@ -46,43 +88,10 @@ def main(): add_file_common_args=True, supports_check_mode=True ) + d = Decompress(module) + d.run() - src = module.params['src'] - dest = module.params['dest'] - if not os.path.exists(src): - module.fail_json(msg="Path does not exist: '%s'" % src) - if os.path.isdir(src): - module.fail_json(msg="Cannot decompress directory '%s'" % src) - if os.path.exists(dest) and os.path.isdir(dest): - module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % dest) - format = module.params['format'] - if format not in compressors: - module.fail_json(msg=f"Couldn't decompress '%s' format." % format) - dest = os.path.abspath(dest) - - file_args = module.load_file_common_arguments(module.params, path=dest) - handler = compressors[format] - try: - tempd, temppath = tempfile.mkstemp(dir=module.tmpdir) - decompress(src, tempd, handler) - except OSError as e: - module.fail_json(msg="Unable to create temporary file '%s'" % to_native(e)) - - if os.path.exists(dest): - changed = is_dest_changed(temppath, dest) - else: - changed = True - if changed and not module.check_mode: - try: - module.atomic_move(temppath, dest) - except OSError: - module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (temppath, dest)) - - if os.path.exists(temppath): - os.unlink(temppath) - result['msg'] = 'success' - result['changed'] = module.set_fs_attributes_if_different(file_args, changed) - module.exit_json(**result) + module.exit_json(changed=d.changed, msg=d.msg) if __name__ == '__main__': diff --git a/tests/integration/targets/decompress/tasks/core.yml b/tests/integration/targets/decompress/tasks/core.yml new file mode 100644 index 00000000000..de71622fb5c --- /dev/null +++ b/tests/integration/targets/decompress/tasks/core.yml @@ -0,0 +1,29 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Set mode for decompressed file ({{ format }} test) + set_fact: + decompressed_mode: "0640" + +- name: Simple decompress ({{ format }} test) + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" + dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + format: "{{ format }}" + mode: "{{ decompressed_mode }}" + register: first_decompression + +- name: Stat decompressed file ({{ format }} test) + stat: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + register: decompressed_file_stat + +- name: Check that file was decompressed correctly ({{ format }} test) + assert: + that: + - first_decompression.changed + - decompressed_file_stat.stat.exists + - decompressed_file_stat.stat.mode == decompressed_mode + - orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index eb75cdcc0ff..3228c687bda 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -21,9 +21,21 @@ - xz # Run tests -- name: Run tests +- name: Run core tests include_tasks: - file: tests.yml + file: core.yml loop: "{{ formats }}" loop_control: loop_var: format + +- name: Delete decompressed files + file: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + state: absent + loop: "{{ formats }}" + loop_control: + loop_var: format + +- name: Run idempotency and check mode tests + include_tasks: + file: misc.yml diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml new file mode 100644 index 00000000000..2cb17f2c0cb --- /dev/null +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -0,0 +1,28 @@ +--- +- name: Decompress with check mode enabled + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_from_gz.txt" + format: gz + check_mode: true + register: decompressed_check_mode + +- name: Decompress second time with check mode enabled + community.general.decompress: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_from_gz.txt" + format: gz + check_mode: true + register: decompressed_check_mode_2 + +- name: Stat non-existing file + stat: + path: "{{ remote_tmp_dir }}/file_from_gz.txt" + register: nonexisting_stat + +- name: Check mode test + assert: + that: + - decompressed_check_mode.changed + - decompressed_check_mode_2.changed + - not nonexisting_stat.stat.exists diff --git a/tests/integration/targets/decompress/tasks/tests.yml b/tests/integration/targets/decompress/tasks/tests.yml deleted file mode 100644 index bd45217d068..00000000000 --- a/tests/integration/targets/decompress/tasks/tests.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -# Copyright (c) Ansible Project -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) -# SPDX-License-Identifier: GPL-3.0-or-later - -- name: Set mode for decompressed file ({{ format }} test) - set_fact: - decompressed_mode: "0640" - -- name: Simple decompress ({{ format }} test) - community.general.decompress: - src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" - dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - format: "{{ format }}" - mode: "{{ decompressed_mode }}" - register: first_decompression - -- name: Stat decompressed file ({{ format }} test) - stat: - path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - register: decompressed_file_stat - -- name: Check that file was decompressed correctly ({{ format }} test) - assert: - that: - - first_decompression.changed - - decompressed_file_stat.stat.exists - - decompressed_file_stat.stat.mode == decompressed_mode - - orig_file_stat.stat.checksum == decompressed_file_stat.stat.checksum - -- name: Decompress second time ({{ format }} test) - community.general.decompress: - src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" - dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - format: "{{ format }}" - mode: "{{ decompressed_mode }}" - register: same_file_decompression - -- name: Decompress new file to the same location ({{ format }} test) - community.general.decompress: - src: "{{ remote_tmp_dir }}/second_file.txt.{{ format }}" - dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - format: "{{ format }}" - mode: "{{ decompressed_mode }}" - register: new_file_decompression - -- name: Idempotency test - assert: - that: - - not same_file_decompression.changed - - new_file_decompression.changed - -- name: Delete decompressed files - file: - path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - state: absent - -- name: Decompress with check mode enabled - community.general.decompress: - src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" - dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - format: "{{ format }}" - mode: "{{ decompressed_mode }}" - check_mode: true - register: decompressed_check_mode - -- name: Decompress second time with check mode enabled - community.general.decompress: - src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" - dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - format: "{{ format }}" - mode: "{{ decompressed_mode }}" - check_mode: true - register: decompressed_check_mode_2 - -- name: Stat non-existing file - stat: - path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - register: nonexisting_stat - -- name: Check mode test - assert: - that: - - decompressed_check_mode.changed - - decompressed_check_mode_2.changed - - not nonexisting_stat.stat.exists From c7b6e4885380989e4daea945aa45d3a28579acdd Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 19 Nov 2024 17:38:17 +0200 Subject: [PATCH 13/39] adds 'remove' parameter that deletes original compressed file after decompression --- plugins/modules/decompress.py | 4 ++- .../targets/decompress/tasks/misc.yml | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 036016b7504..5c810dcd6ae 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -73,7 +73,8 @@ def run(self): if os.path.exists(temppath): os.unlink(temppath) - + if self.remove: + os.remove(self.src) self.msg = "success" self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) @@ -84,6 +85,7 @@ def main(): src=dict(type='path', required=True), dest=dict(type='path', required=True), format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), + remove=dict(type='bool', default=False) ), add_file_common_args=True, supports_check_mode=True diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml index 2cb17f2c0cb..0868b348475 100644 --- a/tests/integration/targets/decompress/tasks/misc.yml +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -26,3 +26,29 @@ - decompressed_check_mode.changed - decompressed_check_mode_2.changed - not nonexisting_stat.stat.exists + +- name: Copy compressed file + copy: + src: "{{ remote_tmp_dir }}/file.txt.gz" + dest: "{{ remote_tmp_dir }}/file_copied.txt.gz" + remote_src: true + +- name: Decompress, deleting original file + community.general.decompress: + src: "{{ remote_tmp_dir }}/file_copied.txt.gz" + dest: "{{ remote_tmp_dir }}/file_copied.txt" + remove: true + +- name: Stat compressed file + stat: + path: "{{ remote_tmp_dir }}/file_copied.txt.gz" + register: compressed_stat + +- name: Check that original file was deleted + assert: + that: not compressed_stat.stat.exists + +- name: Delete decompressed file + file: + path: "{{ remote_tmp_dir }}/file_copied.txt" + state: absent From c5a221446c74afa6674b49feb49904dbd05c6fbe Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 19 Nov 2024 18:50:19 +0200 Subject: [PATCH 14/39] adds documentation --- plugins/modules/decompress.py | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 5c810dcd6ae..a4912cd70ad 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -1,3 +1,72 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2024, Stanislav Shamilov +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +module: decompress +short_description: Decompresses compressed files. +description: + - Decompresses compressed files. + - The source (compressed) file and destination (decompressed) files are on the remote host. + - Source file can be deleted after decompression. +attributes: + check_mode: + support: full + diff_mode: + support: none +options: + src: + description: + - Remote absolute path for the file to decompress. + type: path + required: true + dest: + description: + - The file name of the destination file where the compressed file will be decompressed. + - If the destination file exists, it will be truncated and overwritten. + type: path + required: true + format: + description: + - The type of compression to use to decompress. + type: str + choices: [ gz, bz2, xz ] + default: gz + remove: + description: + - Remove original compressed file after decompression + type: bool + default: false + author: + - Stanislav Shamilov (@shamilovstas) +''' + +EXAMPLES = ''' +- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt (gz compression is used by default) + community.general.decompress: + src: /path/to/file.txt.gz + dest: /path/to/file.txt + +- name: Decompress file compressed with bzip2 + community.general.decompress: + src: /path/to/file.txt.bz2 + dest: /path/to/file.bz2 + format: bz2 + +- name: Decompress file and delete the compressed file afterwards + community.general.decompress: + src: /path/to/file.txt.gz + dest: /path/to/file.txt + remove: true +''' + import bz2 import filecmp import gzip @@ -33,6 +102,7 @@ def __init__(self, module): self.src = module.params['src'] self.dest = module.params['dest'] self.fmt = module.params['format'] + self.remove = module.params['remove'] self.check_mode = module.check_mode self.module = module self.changed = False From 4816b8f2575cdfbeb0815dcbadeaf1e2e4010a2f Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 19 Nov 2024 18:57:42 +0200 Subject: [PATCH 15/39] fixes bug with 'remove' parameter in check mode --- plugins/modules/decompress.py | 2 +- tests/integration/targets/decompress/tasks/misc.yml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index a4912cd70ad..892a9ec4aed 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -143,7 +143,7 @@ def run(self): if os.path.exists(temppath): os.unlink(temppath) - if self.remove: + if self.remove and not self.check_mode: os.remove(self.src) self.msg = "success" self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml index 0868b348475..59cfab08c87 100644 --- a/tests/integration/targets/decompress/tasks/misc.yml +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -12,9 +12,15 @@ src: "{{ remote_tmp_dir }}/file.txt.gz" dest: "{{ remote_tmp_dir }}/file_from_gz.txt" format: gz + remove: true check_mode: true register: decompressed_check_mode_2 +- name: Stat original compressed file + stat: + path: "{{ remote_tmp_dir }}/file.txt.gz" + register: original_file + - name: Stat non-existing file stat: path: "{{ remote_tmp_dir }}/file_from_gz.txt" @@ -25,6 +31,7 @@ that: - decompressed_check_mode.changed - decompressed_check_mode_2.changed + - original_file.stat.exists - not nonexisting_stat.stat.exists - name: Copy compressed file From 203312b7323f1c640ee6ee83a86ae9aeb1d80fcf Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Wed, 20 Nov 2024 19:24:57 +0200 Subject: [PATCH 16/39] makes dest argument not required. Dest filename now can be produced from the src filename --- plugins/modules/decompress.py | 32 ++++++++++-- .../targets/decompress/tasks/cleanup.yml | 12 +++++ .../targets/decompress/tasks/dest.yml | 51 +++++++++++++++++++ .../targets/decompress/tasks/main.yml | 28 +++++----- .../targets/decompress/tasks/misc.yml | 4 ++ 5 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 tests/integration/targets/decompress/tasks/cleanup.yml create mode 100644 tests/integration/targets/decompress/tasks/dest.yml diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 892a9ec4aed..9ca06ad6bef 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -6,6 +6,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from __future__ import (absolute_import, division, print_function) + __metaclass__ = type DOCUMENTATION = ''' @@ -31,8 +32,11 @@ description: - The file name of the destination file where the compressed file will be decompressed. - If the destination file exists, it will be truncated and overwritten. + - If not specified, the destination filename will be O(src) without the compression format extension (for + instance, if O(src) was /path/to/file.txt.gz, O(dest) will be /path/to/file.txt). If O(src) file does not end + with the O(format) extension, the O(dest) filename will have the form O(src)_decompressed (for instance, if + O(src) was /path/to/file.myextension, O(dest) will be /path/to/file.myextension_decompressed). type: path - required: true format: description: - The type of compression to use to decompress. @@ -54,6 +58,10 @@ src: /path/to/file.txt.gz dest: /path/to/file.txt +- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt + community.general.decompress: + src: /path/to/file.txt.gz + - name: Decompress file compressed with bzip2 community.general.decompress: src: /path/to/file.txt.bz2 @@ -98,9 +106,10 @@ def decompress(src, dest, handler): class Decompress(object): + destination_filename_template = "%s_decompressed" + def __init__(self, module): self.src = module.params['src'] - self.dest = module.params['dest'] self.fmt = module.params['format'] self.remove = module.params['remove'] self.check_mode = module.check_mode @@ -109,6 +118,12 @@ def __init__(self, module): self.msg = "" self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} + dest = module.params['dest'] + if dest is None: + self.dest = self.get_destination_filename() + else: + self.dest = dest + def configure(self): if not os.path.exists(self.src): self.module.fail_json(msg="Path does not exist: '%s'" % self.src) @@ -118,7 +133,7 @@ def configure(self): self.module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % self.dest) self.fmt = self.module.params['format'] if self.fmt not in self.handlers: - self.module.fail_json(msg="Couldn't decompress '%s' format" % self.fmt) + self.module.fail_json(msg="Could not decompress '%s' format" % self.fmt) def run(self): self.configure() @@ -148,12 +163,21 @@ def run(self): self.msg = "success" self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) + def get_destination_filename(self): + src = self.src + fmt_extension = ".%s" % self.fmt + if src.endswith(fmt_extension) and len(src) > len(fmt_extension): + filename = src[:-len(fmt_extension)] + else: + filename = Decompress.destination_filename_template % src + return filename + def main(): module = AnsibleModule( argument_spec=dict( src=dict(type='path', required=True), - dest=dict(type='path', required=True), + dest=dict(type='path'), format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), remove=dict(type='bool', default=False) ), diff --git a/tests/integration/targets/decompress/tasks/cleanup.yml b/tests/integration/targets/decompress/tasks/cleanup.yml new file mode 100644 index 00000000000..95db42104f8 --- /dev/null +++ b/tests/integration/targets/decompress/tasks/cleanup.yml @@ -0,0 +1,12 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Delete decompressed files + file: + path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" + state: absent + loop: "{{ formats }}" + loop_control: + loop_var: format \ No newline at end of file diff --git a/tests/integration/targets/decompress/tasks/dest.yml b/tests/integration/targets/decompress/tasks/dest.yml new file mode 100644 index 00000000000..7db816cc5d9 --- /dev/null +++ b/tests/integration/targets/decompress/tasks/dest.yml @@ -0,0 +1,51 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: Copy a compressed file + copy: + src: "{{ item.orig }}" + dest: "{{ item.new }}" + remote_src: true + loop: + - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest.txt.gz" } + - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest" } + +- name: Decompress a file without specifying destination + community.general.decompress: + src: "{{ remote_tmp_dir }}/dest.txt.gz" + remove: true + +- name: Decompress a file which lacks extension without specifying destination + community.general.decompress: + src: "{{ remote_tmp_dir }}/dest" + remove: true + +- name: Stat result files + stat: + path: "{{ remote_tmp_dir }}/{{ filename }}" + loop: + - dest.txt + - dest + loop_control: + loop_var: filename + register: result_files_stat + +- name: Test that file exists + assert: + that: "{{ item.stat.exists }}" + quiet: true + loop: "{{ result_files_stat.results }}" + loop_control: + label: "{{ item.stat.path }}" + +- name: Delete test files + file: + path: "{{ filename }}" + state: absent + loop: + - "dest.txt" + - "dest" + loop_control: + loop_var: filename diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 3228c687bda..236b58f982c 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -22,20 +22,20 @@ # Run tests - name: Run core tests - include_tasks: - file: core.yml - loop: "{{ formats }}" - loop_control: - loop_var: format + block: + - include_tasks: core.yml + loop: "{{ formats }}" + loop_control: + loop_var: format + - import_tasks: cleanup.yml -- name: Delete decompressed files - file: - path: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" - state: absent - loop: "{{ formats }}" - loop_control: - loop_var: format - name: Run idempotency and check mode tests - include_tasks: - file: misc.yml + block: + - import_tasks: misc.yml + - import_tasks: cleanup.yml + +- name: Run tests for destination file + block: + - import_tasks: dest.yml + - import_tasks: cleanup.yml \ No newline at end of file diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml index 59cfab08c87..7b01612af91 100644 --- a/tests/integration/targets/decompress/tasks/misc.yml +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -1,4 +1,8 @@ --- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + - name: Decompress with check mode enabled community.general.decompress: src: "{{ remote_tmp_dir }}/file.txt.gz" From 89cd527a5e7841bdb50feae03f8814f4a739b75d Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Wed, 20 Nov 2024 23:20:44 +0200 Subject: [PATCH 17/39] adds dest to output --- plugins/modules/decompress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 9ca06ad6bef..010f9439229 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -187,7 +187,7 @@ def main(): d = Decompress(module) d.run() - module.exit_json(changed=d.changed, msg=d.msg) + module.exit_json(changed=d.changed, dest=d.dest) if __name__ == '__main__': From 3e4a174116976479ba1d889f12177f1bf9b0075d Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Wed, 20 Nov 2024 23:21:07 +0200 Subject: [PATCH 18/39] updates the documentation, adds "RETURN" block --- plugins/modules/decompress.py | 40 ++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 010f9439229..bffd0974284 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -9,14 +9,17 @@ __metaclass__ = type -DOCUMENTATION = ''' +DOCUMENTATION = r''' --- module: decompress short_description: Decompresses compressed files. -description: +description: - Decompresses compressed files. - The source (compressed) file and destination (decompressed) files are on the remote host. - Source file can be deleted after decompression. +extends_documentation_fragment: + - files + - community.general.attributes attributes: check_mode: support: full @@ -32,10 +35,11 @@ description: - The file name of the destination file where the compressed file will be decompressed. - If the destination file exists, it will be truncated and overwritten. - - If not specified, the destination filename will be O(src) without the compression format extension (for - instance, if O(src) was /path/to/file.txt.gz, O(dest) will be /path/to/file.txt). If O(src) file does not end - with the O(format) extension, the O(dest) filename will have the form O(src)_decompressed (for instance, if - O(src) was /path/to/file.myextension, O(dest) will be /path/to/file.myextension_decompressed). + - If not specified, the destination filename will be derived from O(src) by removing the compression format + extension. For example, if O(src) is V(/path/to/file.txt.gz) and O(format) is V(gz), O(dest) will be + V(/path/to/file.txt). If the O(src) file does not have an extension for the current O(format), the O(dest) + filename will be made by appending V(_decompressed) to the O(src) filename. For instance, if O(src) is + V(/path/to/file.myextension), the (dest) filename will be V(/path/to/file.myextension_decompressed). type: path format: description: @@ -48,26 +52,26 @@ - Remove original compressed file after decompression type: bool default: false - author: - - Stanislav Shamilov (@shamilovstas) +author: + - Stanislav Shamilov (@shamilovstas) ''' -EXAMPLES = ''' +EXAMPLES = r''' - name: Decompress file /path/to/file.txt.gz into /path/to/file.txt (gz compression is used by default) community.general.decompress: src: /path/to/file.txt.gz dest: /path/to/file.txt - -- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt + +- name: Decompress file /path/to/file.txt.gz into /path/to/file.txt community.general.decompress: src: /path/to/file.txt.gz - + - name: Decompress file compressed with bzip2 community.general.decompress: src: /path/to/file.txt.bz2 dest: /path/to/file.bz2 format: bz2 - + - name: Decompress file and delete the compressed file afterwards community.general.decompress: src: /path/to/file.txt.gz @@ -75,6 +79,14 @@ remove: true ''' +RETURN = r''' +dest: + description: Path to decompressed file + type: str + returned: success + sample: /path/to/file.txt +''' + import bz2 import filecmp import gzip @@ -115,7 +127,6 @@ def __init__(self, module): self.check_mode = module.check_mode self.module = module self.changed = False - self.msg = "" self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} dest = module.params['dest'] @@ -160,7 +171,6 @@ def run(self): os.unlink(temppath) if self.remove and not self.check_mode: os.remove(self.src) - self.msg = "success" self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) def get_destination_filename(self): From ee9f58cb50cf643580460265baf9f97ac4c8b0e0 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Fri, 22 Nov 2024 18:13:45 +0200 Subject: [PATCH 19/39] fixes test --- tests/integration/targets/decompress/tasks/dest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/decompress/tasks/dest.yml b/tests/integration/targets/decompress/tasks/dest.yml index 7db816cc5d9..8620a2b84d3 100644 --- a/tests/integration/targets/decompress/tasks/dest.yml +++ b/tests/integration/targets/decompress/tasks/dest.yml @@ -27,7 +27,7 @@ path: "{{ remote_tmp_dir }}/{{ filename }}" loop: - dest.txt - - dest + - dest_decompressed loop_control: loop_var: filename register: result_files_stat @@ -46,6 +46,6 @@ state: absent loop: - "dest.txt" - - "dest" + - "dest_decompressed" loop_control: loop_var: filename From c6e7961be13d72d20b705043dc9e32863ef62871 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Fri, 22 Nov 2024 16:11:15 +0200 Subject: [PATCH 20/39] adds support for python2 --- plugins/modules/decompress.py | 73 +++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index bffd0974284..3badf381262 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -36,9 +36,9 @@ - The file name of the destination file where the compressed file will be decompressed. - If the destination file exists, it will be truncated and overwritten. - If not specified, the destination filename will be derived from O(src) by removing the compression format - extension. For example, if O(src) is V(/path/to/file.txt.gz) and O(format) is V(gz), O(dest) will be - V(/path/to/file.txt). If the O(src) file does not have an extension for the current O(format), the O(dest) - filename will be made by appending V(_decompressed) to the O(src) filename. For instance, if O(src) is + extension. For example, if O(src) is V(/path/to/file.txt.gz) and O(format) is V(gz), O(dest) will be + V(/path/to/file.txt). If the O(src) file does not have an extension for the current O(format), the O(dest) + filename will be made by appending C(_decompressed) to the O(src) filename. For instance, if O(src) is V(/path/to/file.myextension), the (dest) filename will be V(/path/to/file.myextension_decompressed). type: path format: @@ -52,6 +52,8 @@ - Remove original compressed file after decompression type: bool default: false +requirements: + - Requires C(lzma) (standard library of Python 3) or L(backports.lzma, https://pypi.org/project/backports.lzma/) (Python 2) if using C(xz) format. author: - Stanislav Shamilov (@shamilovstas) ''' @@ -90,14 +92,33 @@ import bz2 import filecmp import gzip -import lzma import os import shutil import tempfile -from ansible.module_utils.basic import AnsibleModule +from traceback import format_exc +from ansible.module_utils import six +from ansible.module_utils.basic import AnsibleModule, missing_required_lib from ansible.module_utils.common.text.converters import to_native +LZMA_IMP_ERR = None +if six.PY3: + try: + import lzma + + HAS_LZMA = True + except ImportError: + LZMA_IMP_ERR = format_exc() + HAS_LZMA = False +else: + try: + from backports import lzma + + HAS_LZMA = True + except ImportError: + LZMA_IMP_ERR = format_exc() + HAS_LZMA = False + def lzma_decompress(src): return lzma.open(src, "rb") @@ -112,8 +133,10 @@ def gzip_decompress(src): def decompress(src, dest, handler): - with handler(src) as src_file: - with open(dest, "wb") as dest_file: + b_src = to_native(src, errors='surrogate_or_strict') + b_dest = to_native(dest, errors='surrogate_or_strict') + with handler(b_src) as src_file: + with open(b_dest, "wb") as dest_file: shutil.copyfileobj(src_file, dest_file) @@ -134,14 +157,16 @@ def __init__(self, module): self.dest = self.get_destination_filename() else: self.dest = dest + self.b_dest = to_native(self.dest, errors='surrogate_or_strict') + self.b_src = to_native(self.src, errors='surrogate_or_strict') def configure(self): - if not os.path.exists(self.src): - self.module.fail_json(msg="Path does not exist: '%s'" % self.src) - if os.path.isdir(self.src): - self.module.fail_json(msg="Cannot decompress directory '%s'" % self.src) - if os.path.exists(self.dest) and os.path.isdir(self.dest): - self.module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % self.dest) + if not os.path.exists(self.b_src): + self.module.fail_json(msg="Path does not exist: '%s'" % self.b_src) + if os.path.isdir(self.b_src): + self.module.fail_json(msg="Cannot decompress directory '%s'" % self.b_src) + if os.path.exists(self.b_src) and os.path.isdir(self.b_src): + self.module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % self.b_src) self.fmt = self.module.params['format'] if self.fmt not in self.handlers: self.module.fail_json(msg="Could not decompress '%s' format" % self.fmt) @@ -152,25 +177,26 @@ def run(self): handler = self.handlers[self.fmt] try: tempfd, temppath = tempfile.mkstemp(dir=self.module.tmpdir) - decompress(self.src, tempfd, handler) + b_temppath = to_native(temppath, errors='surrogate_or_strict') + decompress(self.b_src, b_temppath, handler) except OSError as e: self.module.fail_json(msg="Unable to create temporary file '%s'" % to_native(e)) - if os.path.exists(self.dest): - self.changed = not filecmp.cmp(temppath, self.dest, shallow=False) + if os.path.exists(self.b_dest): + self.changed = not filecmp.cmp(b_temppath, self.b_dest, shallow=False) else: self.changed = True if self.changed and not self.module.check_mode: try: - self.module.atomic_move(temppath, self.dest) + self.module.atomic_move(b_temppath, self.b_dest) except OSError: - self.module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (temppath, self.dest)) + self.module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (b_temppath, self.dest)) - if os.path.exists(temppath): - os.unlink(temppath) + if os.path.exists(b_temppath): + os.unlink(b_temppath) if self.remove and not self.check_mode: - os.remove(self.src) + os.remove(self.b_src) self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) def get_destination_filename(self): @@ -194,6 +220,11 @@ def main(): add_file_common_args=True, supports_check_mode=True ) + if not HAS_LZMA and module.params['format'] == 'xz': + module.fail_json( + msg=missing_required_lib("lzma or backports.lzma", reason="when using xz format"), exception=LZMA_IMP_ERR + ) + d = Decompress(module) d.run() From eba3564a6ec77c56da3059d8f479a3ac729509ea Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Fri, 22 Nov 2024 20:45:42 +0200 Subject: [PATCH 21/39] removes some of the test files that can be generated during testing. Adds copyright header to test files --- .../integration/targets/decompress/files/file.txt | 4 ++++ .../targets/decompress/files/file.txt.bz2 | Bin 56 -> 0 bytes .../targets/decompress/files/file.txt.gz | Bin 46 -> 0 bytes .../targets/decompress/files/file.txt.xz | Bin 76 -> 0 bytes .../targets/decompress/files/second_file.txt | 4 ++++ .../targets/decompress/files/second_file.txt.bz2 | Bin 79 -> 0 bytes .../targets/decompress/files/second_file.txt.gz | Bin 84 -> 0 bytes .../targets/decompress/files/second_file.txt.xz | Bin 108 -> 0 bytes .../integration/targets/decompress/tasks/main.yml | 14 ++++++++++++++ 9 files changed, 22 insertions(+) delete mode 100644 tests/integration/targets/decompress/files/file.txt.bz2 delete mode 100644 tests/integration/targets/decompress/files/file.txt.gz delete mode 100644 tests/integration/targets/decompress/files/file.txt.xz delete mode 100644 tests/integration/targets/decompress/files/second_file.txt.bz2 delete mode 100644 tests/integration/targets/decompress/files/second_file.txt.gz delete mode 100644 tests/integration/targets/decompress/files/second_file.txt.xz diff --git a/tests/integration/targets/decompress/files/file.txt b/tests/integration/targets/decompress/files/file.txt index 70c86c750ee..5d2e0d1458d 100644 --- a/tests/integration/targets/decompress/files/file.txt +++ b/tests/integration/targets/decompress/files/file.txt @@ -1 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + This is sample file \ No newline at end of file diff --git a/tests/integration/targets/decompress/files/file.txt.bz2 b/tests/integration/targets/decompress/files/file.txt.bz2 deleted file mode 100644 index f8a6678633253440f388f054ef6811457df5cd09..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 56 zcmZ>Y%CIzaj8qGboG%@2!N9;cxxs;fg+V#ZgF%78km0f~v$I1+K>3BV59^*X)x@3@ MdFbR+lN<8@0Jek@ZU6uP diff --git a/tests/integration/targets/decompress/files/file.txt.gz b/tests/integration/targets/decompress/files/file.txt.gz deleted file mode 100644 index 8eee6ec09fa4d8d0e0f5125ec28028eb7888602e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46 zcmb2|=HR#yZk*1*oR*oBs#j7`!oYpq^21Q0O1_p)_{ill=8H7SIGK&>}s5mjVASYEJEi)&Tf#Hi( exB>H;8JigxwT0(!dR&s<_PLf3sE2_iG712YU>6er diff --git a/tests/integration/targets/decompress/files/second_file.txt b/tests/integration/targets/decompress/files/second_file.txt index 48383aaa09e..bd04eca21cd 100644 --- a/tests/integration/targets/decompress/files/second_file.txt +++ b/tests/integration/targets/decompress/files/second_file.txt @@ -1 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + Content of this file must differ from the 'file.txt' \ No newline at end of file diff --git a/tests/integration/targets/decompress/files/second_file.txt.bz2 b/tests/integration/targets/decompress/files/second_file.txt.bz2 deleted file mode 100644 index e0a610f4300553c49de45754660059cf3fde7ed7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 79 zcmV-V0I>f;T4*^jL0KkKS!n?1%m4rdm4HBj2ml9XoWAlCo@ diff --git a/tests/integration/targets/decompress/files/second_file.txt.gz b/tests/integration/targets/decompress/files/second_file.txt.gz deleted file mode 100644 index c784e0c56b9a9ddb99d4408b86325b10edf27a33..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 84 zcmV-a0IUBWiwFoJF*aua19N3#Z*F8?W@&6?E_8Tw0CUdID@n~OQOHkID9OkyR!Gat qNma-#EiO?=$xKU2EmBA;%FhK#r7Ea{r1VNEO4I><<-&e60000E?jR5V diff --git a/tests/integration/targets/decompress/files/second_file.txt.xz b/tests/integration/targets/decompress/files/second_file.txt.xz deleted file mode 100644 index 4cfb41b368c2a4e504a659e3f7e9239840b8845d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 108 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8H}Ct^GZ_lN)+{!-Q42P2W?=L&neO;6R({*(T1KFm I3@njR0F>(^o&W#< diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 236b58f982c..f882180ea3f 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -20,6 +20,20 @@ - gz - xz +- name: Ensure xz is present to create compressed files (yum) + yum: name=xz state=latest + when: ansible_facts.pkg_mgr == 'yum' + +- name: Ensure xz is present to create compressed files (apt) + apt: name=xz-utils state=latest + when: ansible_facts.pkg_mgr == 'apt' + +- name: Generate compressed files + shell: gzip -k {{ item }} && bzip2 -k {{ item }} && xz -k {{ item }} + loop: + - "{{ remote_tmp_dir }}/file.txt" + - "{{ remote_tmp_dir }}/second_file.txt" + # Run tests - name: Run core tests block: From 7774900d10a3b502b51d40069046435015da50d6 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Fri, 22 Nov 2024 20:48:51 +0200 Subject: [PATCH 22/39] adds maintainer --- .github/BOTMETA.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 989752a0a71..29371cd50c1 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -506,6 +506,8 @@ files: ignore: skornehl $modules/dconf.py: maintainers: azaghal + $modules/decompress.py: + maintainers: shamilovstas $modules/deploy_helper.py: maintainers: ramondelafuente $modules/dimensiondata_network.py: From 9e34fb1b5bcf761255b4d37f5df3734e8c00af89 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Sat, 23 Nov 2024 21:19:23 +0200 Subject: [PATCH 23/39] apply minor suggestions from code review Co-authored-by: Felix Fontein --- plugins/modules/decompress.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 3badf381262..e67202084c5 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -12,13 +12,14 @@ DOCUMENTATION = r''' --- module: decompress -short_description: Decompresses compressed files. +short_description: Decompresses compressed files +version_added: 10.1.0 description: - Decompresses compressed files. - The source (compressed) file and destination (decompressed) files are on the remote host. - Source file can be deleted after decompression. extends_documentation_fragment: - - files + - ansible.builtin.files - community.general.attributes attributes: check_mode: @@ -49,7 +50,7 @@ default: gz remove: description: - - Remove original compressed file after decompression + - Remove original compressed file after decompression. type: bool default: false requirements: From ef683c1acf8a868bd50a184041615c0393668aab Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Sat, 23 Nov 2024 22:23:53 +0200 Subject: [PATCH 24/39] fixes code review comments (idempotency issue with non existing src, existing dest and remove=true; fixes the issue and adds test) --- plugins/modules/decompress.py | 24 ++++++++++--------- .../targets/decompress/tasks/misc.yml | 13 ++++++++-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index e67202084c5..3aec71af8df 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -100,7 +100,7 @@ from traceback import format_exc from ansible.module_utils import six from ansible.module_utils.basic import AnsibleModule, missing_required_lib -from ansible.module_utils.common.text.converters import to_native +from ansible.module_utils.common.text.converters import to_native, to_bytes LZMA_IMP_ERR = None if six.PY3: @@ -133,9 +133,7 @@ def gzip_decompress(src): return gzip.open(src, "rb") -def decompress(src, dest, handler): - b_src = to_native(src, errors='surrogate_or_strict') - b_dest = to_native(dest, errors='surrogate_or_strict') +def decompress(b_src, b_dest, handler): with handler(b_src) as src_file: with open(b_dest, "wb") as dest_file: shutil.copyfileobj(src_file, dest_file) @@ -158,15 +156,20 @@ def __init__(self, module): self.dest = self.get_destination_filename() else: self.dest = dest - self.b_dest = to_native(self.dest, errors='surrogate_or_strict') - self.b_src = to_native(self.src, errors='surrogate_or_strict') + self.b_dest = to_bytes(self.dest, errors='surrogate_or_strict') + self.b_src = to_bytes(self.src, errors='surrogate_or_strict') def configure(self): if not os.path.exists(self.b_src): - self.module.fail_json(msg="Path does not exist: '%s'" % self.b_src) + msg = "Path does not exist: '%s'" % self.b_src + if self.remove and os.path.exists(self.b_dest): + self.module.warn(msg) + self.module.exit_json(changed=False) + else: + self.module.fail_json(msg=msg) if os.path.isdir(self.b_src): self.module.fail_json(msg="Cannot decompress directory '%s'" % self.b_src) - if os.path.exists(self.b_src) and os.path.isdir(self.b_src): + if os.path.isdir(self.b_dest): self.module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % self.b_src) self.fmt = self.module.params['format'] if self.fmt not in self.handlers: @@ -178,7 +181,8 @@ def run(self): handler = self.handlers[self.fmt] try: tempfd, temppath = tempfile.mkstemp(dir=self.module.tmpdir) - b_temppath = to_native(temppath, errors='surrogate_or_strict') + self.module.add_cleanup_file(temppath) + b_temppath = to_bytes(temppath, errors='surrogate_or_strict') decompress(self.b_src, b_temppath, handler) except OSError as e: self.module.fail_json(msg="Unable to create temporary file '%s'" % to_native(e)) @@ -194,8 +198,6 @@ def run(self): except OSError: self.module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (b_temppath, self.dest)) - if os.path.exists(b_temppath): - os.unlink(b_temppath) if self.remove and not self.check_mode: os.remove(self.b_src) self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml index 7b01612af91..34cf3f7f17e 100644 --- a/tests/integration/targets/decompress/tasks/misc.yml +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -50,14 +50,23 @@ dest: "{{ remote_tmp_dir }}/file_copied.txt" remove: true +- name: Decompress non existing src + community.general.decompress: + src: "{{ remote_tmp_dir }}/file_copied.txt.gz" + dest: "{{ remote_tmp_dir }}/file_copied.txt" + remove: true + register: decompress_non_existing_src + - name: Stat compressed file stat: path: "{{ remote_tmp_dir }}/file_copied.txt.gz" register: compressed_stat -- name: Check that original file was deleted +- name: Run tests assert: - that: not compressed_stat.stat.exists + that: + - not compressed_stat.stat.exists + - not decompress_non_existing_src.changed - name: Delete decompressed file file: From 85a076552d8a35320464835981ea419eccd0f0e3 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 17:26:47 +0200 Subject: [PATCH 25/39] refactors the module to use ModuleHelper --- plugins/modules/decompress.py | 119 ++++++++++++++++------------------ 1 file changed, 56 insertions(+), 63 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 3aec71af8df..fefd3cdd55d 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -99,7 +99,8 @@ from traceback import format_exc from ansible.module_utils import six -from ansible.module_utils.basic import AnsibleModule, missing_required_lib +from ansible_collections.community.general.plugins.module_utils.mh.module_helper import ModuleHelper +from ansible.module_utils.basic import missing_required_lib from ansible.module_utils.common.text.converters import to_native, to_bytes LZMA_IMP_ERR = None @@ -139,72 +140,82 @@ def decompress(b_src, b_dest, handler): shutil.copyfileobj(src_file, dest_file) -class Decompress(object): +class Decompress(ModuleHelper): destination_filename_template = "%s_decompressed" + use_old_vardict = False + output_params = 'dest' - def __init__(self, module): - self.src = module.params['src'] - self.fmt = module.params['format'] - self.remove = module.params['remove'] - self.check_mode = module.check_mode - self.module = module - self.changed = False - self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} + module = dict( + argument_spec=dict( + src=dict(type='path', required=True), + dest=dict(type='path'), + format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), + remove=dict(type='bool', default=False) + ), + add_file_common_args=True, + supports_check_mode=True + ) - dest = module.params['dest'] - if dest is None: - self.dest = self.get_destination_filename() - else: - self.dest = dest - self.b_dest = to_bytes(self.dest, errors='surrogate_or_strict') - self.b_src = to_bytes(self.src, errors='surrogate_or_strict') + def __init_module__(self): + self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} + if self.vars.dest is None: + self.vars.dest = self.get_destination_filename() + if not HAS_LZMA and self.vars.format == 'xz': + self.do_raise( + msg="%s: %s" % (missing_required_lib("lzma or backports.lzma", reason="when using xz format"), + LZMA_IMP_ERR) + ) + self.configure() def configure(self): - if not os.path.exists(self.b_src): - msg = "Path does not exist: '%s'" % self.b_src - if self.remove and os.path.exists(self.b_dest): - self.module.warn(msg) + b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict') + b_src = to_bytes(self.vars.src, errors='surrogate_or_strict') + if not os.path.exists(b_src): + msg = "Path does not exist: '%s'" % b_src + if self.vars.remove and os.path.exists(b_dest): + self.warn(msg) self.module.exit_json(changed=False) else: - self.module.fail_json(msg=msg) - if os.path.isdir(self.b_src): - self.module.fail_json(msg="Cannot decompress directory '%s'" % self.b_src) - if os.path.isdir(self.b_dest): - self.module.fail_json(msg="Destination is a directory, cannot decompress: '%s'" % self.b_src) - self.fmt = self.module.params['format'] - if self.fmt not in self.handlers: - self.module.fail_json(msg="Could not decompress '%s' format" % self.fmt) - - def run(self): - self.configure() - file_args = self.module.load_file_common_arguments(self.module.params, path=self.dest) - handler = self.handlers[self.fmt] + self.do_raise(msg=msg) + if os.path.isdir(b_src): + self.do_raise(msg="Cannot decompress directory '%s'" % b_src) + if os.path.isdir(b_dest): + self.do_raise(msg="Destination is a directory, cannot decompress: '%s'" % b_src) + if self.vars.format not in self.handlers: + self.do_raise(msg="Could not decompress '%s' format" % self.vars.format) + + def __run__(self): + b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict') + b_src = to_bytes(self.vars.src, errors='surrogate_or_strict') + + file_args = self.module.load_file_common_arguments(self.module.params, path=self.vars.dest) + handler = self.handlers[self.vars.format] try: tempfd, temppath = tempfile.mkstemp(dir=self.module.tmpdir) self.module.add_cleanup_file(temppath) b_temppath = to_bytes(temppath, errors='surrogate_or_strict') - decompress(self.b_src, b_temppath, handler) + decompress(b_src, b_temppath, handler) except OSError as e: - self.module.fail_json(msg="Unable to create temporary file '%s'" % to_native(e)) + self.do_raise(msg="Unable to create temporary file '%s'" % to_native(e)) - if os.path.exists(self.b_dest): - self.changed = not filecmp.cmp(b_temppath, self.b_dest, shallow=False) + if os.path.exists(b_dest): + self.changed = not filecmp.cmp(b_temppath, b_dest, shallow=False) else: self.changed = True if self.changed and not self.module.check_mode: try: - self.module.atomic_move(b_temppath, self.b_dest) + self.module.atomic_move(b_temppath, b_dest) except OSError: - self.module.fail_json(msg="Unable to move temporary file '%s' to '%s'" % (b_temppath, self.dest)) + self.do_raise(msg="Unable to move temporary file '%s' to '%s'" % (b_temppath, self.vars.dest)) - if self.remove and not self.check_mode: - os.remove(self.b_src) + if self.vars.remove and not self.check_mode: + os.remove(b_src) self.changed = self.module.set_fs_attributes_if_different(file_args, self.changed) def get_destination_filename(self): - src = self.src - fmt_extension = ".%s" % self.fmt + src = self.vars.src + fmt_extension = ".%s" % self.vars.format if src.endswith(fmt_extension) and len(src) > len(fmt_extension): filename = src[:-len(fmt_extension)] else: @@ -213,25 +224,7 @@ def get_destination_filename(self): def main(): - module = AnsibleModule( - argument_spec=dict( - src=dict(type='path', required=True), - dest=dict(type='path'), - format=dict(type='str', default='gz', choices=['gz', 'bz2', 'xz']), - remove=dict(type='bool', default=False) - ), - add_file_common_args=True, - supports_check_mode=True - ) - if not HAS_LZMA and module.params['format'] == 'xz': - module.fail_json( - msg=missing_required_lib("lzma or backports.lzma", reason="when using xz format"), exception=LZMA_IMP_ERR - ) - - d = Decompress(module) - d.run() - - module.exit_json(changed=d.changed, dest=d.dest) + Decompress.execute() if __name__ == '__main__': From 52b3f57446fb68d244cd2677f75c5b29b1d69b46 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 17:49:10 +0200 Subject: [PATCH 26/39] refactors lzma dependency manual check to use 'deps.validate' --- plugins/modules/decompress.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index fefd3cdd55d..626abe8a7f4 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -97,30 +97,18 @@ import shutil import tempfile -from traceback import format_exc from ansible.module_utils import six from ansible_collections.community.general.plugins.module_utils.mh.module_helper import ModuleHelper -from ansible.module_utils.basic import missing_required_lib from ansible.module_utils.common.text.converters import to_native, to_bytes +from ansible_collections.community.general.plugins.module_utils import deps -LZMA_IMP_ERR = None if six.PY3: - try: + with deps.declare("lzma"): import lzma - - HAS_LZMA = True - except ImportError: - LZMA_IMP_ERR = format_exc() - HAS_LZMA = False else: - try: + with deps.declare("lzma"): from backports import lzma - HAS_LZMA = True - except ImportError: - LZMA_IMP_ERR = format_exc() - HAS_LZMA = False - def lzma_decompress(src): return lzma.open(src, "rb") @@ -160,11 +148,7 @@ def __init_module__(self): self.handlers = {"gz": gzip_decompress, "bz2": bz2_decompress, "xz": lzma_decompress} if self.vars.dest is None: self.vars.dest = self.get_destination_filename() - if not HAS_LZMA and self.vars.format == 'xz': - self.do_raise( - msg="%s: %s" % (missing_required_lib("lzma or backports.lzma", reason="when using xz format"), - LZMA_IMP_ERR) - ) + deps.validate(self.module) self.configure() def configure(self): From 51b31ee243d918109a0e82037964dadcd5a8639a Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 17:52:08 +0200 Subject: [PATCH 27/39] minor fix --- plugins/modules/decompress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 626abe8a7f4..a29d8319460 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -164,7 +164,7 @@ def configure(self): if os.path.isdir(b_src): self.do_raise(msg="Cannot decompress directory '%s'" % b_src) if os.path.isdir(b_dest): - self.do_raise(msg="Destination is a directory, cannot decompress: '%s'" % b_src) + self.do_raise(msg="Destination is a directory, cannot decompress: '%s'" % b_dest) if self.vars.format not in self.handlers: self.do_raise(msg="Could not decompress '%s' format" % self.vars.format) From cbcd66c47d5cc500aa2dd974601301af2280be91 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 17:52:24 +0200 Subject: [PATCH 28/39] removes registered handlers check --- plugins/modules/decompress.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index a29d8319460..dbb1f220e7f 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -165,8 +165,6 @@ def configure(self): self.do_raise(msg="Cannot decompress directory '%s'" % b_src) if os.path.isdir(b_dest): self.do_raise(msg="Destination is a directory, cannot decompress: '%s'" % b_dest) - if self.vars.format not in self.handlers: - self.do_raise(msg="Could not decompress '%s' format" % self.vars.format) def __run__(self): b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict') From 3fbcbd44a35aa8ee238b161f2bbe1c1148962914 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 18:45:36 +0200 Subject: [PATCH 29/39] minor refactoring --- plugins/modules/decompress.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index dbb1f220e7f..fa61a8e16fa 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -102,11 +102,10 @@ from ansible.module_utils.common.text.converters import to_native, to_bytes from ansible_collections.community.general.plugins.module_utils import deps -if six.PY3: - with deps.declare("lzma"): +with deps.declare("lzma"): + if six.PY3: import lzma -else: - with deps.declare("lzma"): + else: from backports import lzma From 8210a7c3b58fc7f608b56b102a6777bac6645763 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 21:59:36 +0200 Subject: [PATCH 30/39] adds aliases --- tests/integration/targets/decompress/aliases | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/integration/targets/decompress/aliases diff --git a/tests/integration/targets/decompress/aliases b/tests/integration/targets/decompress/aliases new file mode 100644 index 00000000000..3fb9ea2f5b4 --- /dev/null +++ b/tests/integration/targets/decompress/aliases @@ -0,0 +1,7 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/2 +needs/root +destructive From e5917cc48cd40931cf0b10165e5c89258e66c7bb Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 23:27:31 +0200 Subject: [PATCH 31/39] changes setup for tests --- .../targets/decompress/tasks/main.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index f882180ea3f..88864758df9 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -20,13 +20,17 @@ - gz - xz -- name: Ensure xz is present to create compressed files (yum) - yum: name=xz state=latest - when: ansible_facts.pkg_mgr == 'yum' - -- name: Ensure xz is present to create compressed files (apt) - apt: name=xz-utils state=latest - when: ansible_facts.pkg_mgr == 'apt' +- name: Ensure xz is present to create compressed files (not Debian) + package: + name: xz + state: latest + when: ansible_os_family != 'Debian' + +- name: Ensure xz is present to create compressed files (Debian) + package: + name: xz-utils + state: latest + when: ansible_os_family == 'Debian' - name: Generate compressed files shell: gzip -k {{ item }} && bzip2 -k {{ item }} && xz -k {{ item }} From e2c94ab7ba2fc5e3d524fc8326e869a7ae08e049 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Mon, 25 Nov 2024 23:48:46 +0200 Subject: [PATCH 32/39] tests: ignores macos and fixes tests for FreeBSD --- tests/integration/targets/decompress/aliases | 1 + tests/integration/targets/decompress/tasks/main.yml | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/decompress/aliases b/tests/integration/targets/decompress/aliases index 3fb9ea2f5b4..cbdc1293afd 100644 --- a/tests/integration/targets/decompress/aliases +++ b/tests/integration/targets/decompress/aliases @@ -5,3 +5,4 @@ azp/posix/2 needs/root destructive +skip/osx diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 88864758df9..fc60c624d03 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -22,9 +22,13 @@ - name: Ensure xz is present to create compressed files (not Debian) package: - name: xz + name: + - xz + - bzip2 state: latest - when: ansible_os_family != 'Debian' + when: + - ansible_system != 'FreeBSD' + - ansible_os_family != 'Debian' - name: Ensure xz is present to create compressed files (Debian) package: From d2d91598b268e5abb46e50e7d52ae0ca8882b41a Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 26 Nov 2024 00:03:02 +0200 Subject: [PATCH 33/39] tests: reverts ignore for macos and fixes issue with centos7 --- tests/integration/targets/decompress/aliases | 3 +-- tests/integration/targets/decompress/tasks/main.yml | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/decompress/aliases b/tests/integration/targets/decompress/aliases index cbdc1293afd..f4049c7dc2f 100644 --- a/tests/integration/targets/decompress/aliases +++ b/tests/integration/targets/decompress/aliases @@ -4,5 +4,4 @@ azp/posix/2 needs/root -destructive -skip/osx +destructive \ No newline at end of file diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index fc60c624d03..2884225f8c3 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -28,6 +28,7 @@ state: latest when: - ansible_system != 'FreeBSD' + - ansible_os_family != 'Darwin' - ansible_os_family != 'Debian' - name: Ensure xz is present to create compressed files (Debian) @@ -37,7 +38,10 @@ when: ansible_os_family == 'Debian' - name: Generate compressed files - shell: gzip -k {{ item }} && bzip2 -k {{ item }} && xz -k {{ item }} + shell: | + gzip < {{ item }} > {{ item }}.gz + bzip2 < {{ item }} > {{ item }}.bz2 + xz < {{ item }} > {{ item }}.xz loop: - "{{ remote_tmp_dir }}/file.txt" - "{{ remote_tmp_dir }}/second_file.txt" From 28752e31a6fb21a03355e5910bfcee458a50c97d Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 26 Nov 2024 00:11:31 +0200 Subject: [PATCH 34/39] tests: adds liblzma dependency for python2 --- .../integration/targets/decompress/tasks/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 2884225f8c3..f0236b28a1c 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -37,6 +37,21 @@ state: latest when: ansible_os_family == 'Debian' +- name: Install prerequisites for backports.lzma when using python2 (non OSX) + block: + - name: Set liblzma package name depending on the OS + set_fact: + liblzma_dev_package: + Debian: liblzma-dev + RedHat: xz-devel + Suse: xz-devel + - name: Ensure liblzma-dev is present to install backports-lzma + package: name={{ liblzma_dev_package[ansible_os_family] }} state=latest + when: ansible_os_family in liblzma_dev_package.keys() + when: + - ansible_python_version.split('.')[0] == '2' + - ansible_os_family != 'Darwin' + - name: Generate compressed files shell: | gzip < {{ item }} > {{ item }}.gz From ae8db2b597b54b33af9e03745f6fad3d8fc59b80 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 26 Nov 2024 20:12:52 +0200 Subject: [PATCH 35/39] tests: adds backports.lzma --- tests/integration/targets/decompress/tasks/main.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index f0236b28a1c..52fa25ef9b8 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -52,6 +52,11 @@ - ansible_python_version.split('.')[0] == '2' - ansible_os_family != 'Darwin' +- name: Ensure backports.lzma is present to create test archive (pip) + pip: name=backports.lzma state=latest + when: ansible_python_version.split('.')[0] == '2' + register: backports_lzma_pip + - name: Generate compressed files shell: | gzip < {{ item }} > {{ item }}.gz @@ -79,4 +84,9 @@ - name: Run tests for destination file block: - import_tasks: dest.yml - - import_tasks: cleanup.yml \ No newline at end of file + - import_tasks: cleanup.yml + +# Test cleanup +- name: Remove backports.lzma if previously installed (pip) + pip: name=backports.lzma state=absent + when: backports_lzma_pip is changed \ No newline at end of file From e68936bc8dd68f36675f343fe1698f775b23b018 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 26 Nov 2024 20:24:37 +0200 Subject: [PATCH 36/39] fixes bz2 decompression for python2 --- plugins/modules/decompress.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index fa61a8e16fa..7c826a99c2d 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -114,7 +114,10 @@ def lzma_decompress(src): def bz2_decompress(src): - return bz2.open(src, "rb") + if six.PY3: + return bz2.open(src, "rb") + else: + return bz2.BZ2File(src, "rb") def gzip_decompress(src): From 893b92a57459d58f26a5c2bcdad7b862528713cf Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 26 Nov 2024 20:31:28 +0200 Subject: [PATCH 37/39] tests: install xz for osx --- .../targets/decompress/tasks/main.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 52fa25ef9b8..6be042f964b 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -52,6 +52,29 @@ - ansible_python_version.split('.')[0] == '2' - ansible_os_family != 'Darwin' +- name: Install prerequisites for backports.lzma when using python2 (OSX) + block: + - name: Find brew binary + command: which brew + register: brew_which + - name: Get owner of brew binary + stat: path="{{ brew_which.stdout }}" + register: brew_stat + - name: "Install package" + homebrew: + name: xz + state: present + update_homebrew: false + become: true + become_user: "{{ brew_stat.stat.pw_name }}" + # Newer versions of brew want to compile a package which takes a long time. Do not upgrade homebrew until a + # proper solution can be found + environment: + HOMEBREW_NO_AUTO_UPDATE: "True" + when: + - ansible_python_version.split('.')[0] == '2' + - ansible_os_family == 'Darwin' + - name: Ensure backports.lzma is present to create test archive (pip) pip: name=backports.lzma state=latest when: ansible_python_version.split('.')[0] == '2' From bab66207b90f585e3f638dda2b0784c6e338305e Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Tue, 26 Nov 2024 20:39:24 +0200 Subject: [PATCH 38/39] tests: install xz for osx (2) --- tests/integration/targets/decompress/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index 6be042f964b..cc4984f34a0 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -72,7 +72,6 @@ environment: HOMEBREW_NO_AUTO_UPDATE: "True" when: - - ansible_python_version.split('.')[0] == '2' - ansible_os_family == 'Darwin' - name: Ensure backports.lzma is present to create test archive (pip) From 6e7fed0878b9e04f01e0f51a0b406c0c75569363 Mon Sep 17 00:00:00 2001 From: Stanislav Shamilov Date: Thu, 28 Nov 2024 23:07:50 +0200 Subject: [PATCH 39/39] fixes code review comments --- plugins/modules/decompress.py | 4 +--- .../targets/decompress/handlers/main.yml | 9 +++++++++ .../targets/decompress/tasks/core.yml | 2 +- .../targets/decompress/tasks/dest.yml | 4 ++-- .../targets/decompress/tasks/main.yml | 19 ++++++++++--------- .../targets/decompress/tasks/misc.yml | 8 ++++---- 6 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 tests/integration/targets/decompress/handlers/main.yml diff --git a/plugins/modules/decompress.py b/plugins/modules/decompress.py index 7c826a99c2d..818213fb0d5 100644 --- a/plugins/modules/decompress.py +++ b/plugins/modules/decompress.py @@ -157,12 +157,10 @@ def configure(self): b_dest = to_bytes(self.vars.dest, errors='surrogate_or_strict') b_src = to_bytes(self.vars.src, errors='surrogate_or_strict') if not os.path.exists(b_src): - msg = "Path does not exist: '%s'" % b_src if self.vars.remove and os.path.exists(b_dest): - self.warn(msg) self.module.exit_json(changed=False) else: - self.do_raise(msg=msg) + self.do_raise(msg="Path does not exist: '%s'" % b_src) if os.path.isdir(b_src): self.do_raise(msg="Cannot decompress directory '%s'" % b_src) if os.path.isdir(b_dest): diff --git a/tests/integration/targets/decompress/handlers/main.yml b/tests/integration/targets/decompress/handlers/main.yml new file mode 100644 index 00000000000..8c92cc4f811 --- /dev/null +++ b/tests/integration/targets/decompress/handlers/main.yml @@ -0,0 +1,9 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- name: delete backports.lzma + pip: + name: backports.lzma + state: absent diff --git a/tests/integration/targets/decompress/tasks/core.yml b/tests/integration/targets/decompress/tasks/core.yml index de71622fb5c..a92ae21b783 100644 --- a/tests/integration/targets/decompress/tasks/core.yml +++ b/tests/integration/targets/decompress/tasks/core.yml @@ -8,7 +8,7 @@ decompressed_mode: "0640" - name: Simple decompress ({{ format }} test) - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/file.txt.{{ format }}" dest: "{{ remote_tmp_dir }}/file_from_{{ format }}.txt" format: "{{ format }}" diff --git a/tests/integration/targets/decompress/tasks/dest.yml b/tests/integration/targets/decompress/tasks/dest.yml index 8620a2b84d3..9a7bbe499f8 100644 --- a/tests/integration/targets/decompress/tasks/dest.yml +++ b/tests/integration/targets/decompress/tasks/dest.yml @@ -13,12 +13,12 @@ - { orig: "{{ remote_tmp_dir }}/file.txt.gz", new: "{{ remote_tmp_dir }}/dest" } - name: Decompress a file without specifying destination - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/dest.txt.gz" remove: true - name: Decompress a file which lacks extension without specifying destination - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/dest" remove: true diff --git a/tests/integration/targets/decompress/tasks/main.yml b/tests/integration/targets/decompress/tasks/main.yml index cc4984f34a0..f14f2d5593b 100644 --- a/tests/integration/targets/decompress/tasks/main.yml +++ b/tests/integration/targets/decompress/tasks/main.yml @@ -46,7 +46,9 @@ RedHat: xz-devel Suse: xz-devel - name: Ensure liblzma-dev is present to install backports-lzma - package: name={{ liblzma_dev_package[ansible_os_family] }} state=latest + package: + name: "{{ liblzma_dev_package[ansible_os_family] }}" + state: latest when: ansible_os_family in liblzma_dev_package.keys() when: - ansible_python_version.split('.')[0] == '2' @@ -58,7 +60,8 @@ command: which brew register: brew_which - name: Get owner of brew binary - stat: path="{{ brew_which.stdout }}" + stat: + path: "{{ brew_which.stdout }}" register: brew_stat - name: "Install package" homebrew: @@ -75,9 +78,12 @@ - ansible_os_family == 'Darwin' - name: Ensure backports.lzma is present to create test archive (pip) - pip: name=backports.lzma state=latest + pip: + name: backports.lzma + state: latest when: ansible_python_version.split('.')[0] == '2' - register: backports_lzma_pip + notify: + - delete backports.lzma - name: Generate compressed files shell: | @@ -107,8 +113,3 @@ block: - import_tasks: dest.yml - import_tasks: cleanup.yml - -# Test cleanup -- name: Remove backports.lzma if previously installed (pip) - pip: name=backports.lzma state=absent - when: backports_lzma_pip is changed \ No newline at end of file diff --git a/tests/integration/targets/decompress/tasks/misc.yml b/tests/integration/targets/decompress/tasks/misc.yml index 34cf3f7f17e..1514e550308 100644 --- a/tests/integration/targets/decompress/tasks/misc.yml +++ b/tests/integration/targets/decompress/tasks/misc.yml @@ -4,7 +4,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later - name: Decompress with check mode enabled - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/file.txt.gz" dest: "{{ remote_tmp_dir }}/file_from_gz.txt" format: gz @@ -12,7 +12,7 @@ register: decompressed_check_mode - name: Decompress second time with check mode enabled - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/file.txt.gz" dest: "{{ remote_tmp_dir }}/file_from_gz.txt" format: gz @@ -45,13 +45,13 @@ remote_src: true - name: Decompress, deleting original file - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/file_copied.txt.gz" dest: "{{ remote_tmp_dir }}/file_copied.txt" remove: true - name: Decompress non existing src - community.general.decompress: + decompress: src: "{{ remote_tmp_dir }}/file_copied.txt.gz" dest: "{{ remote_tmp_dir }}/file_copied.txt" remove: true