Update python CI versions to cover 3.10 - 3.12

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     env:
       PYTHON_VERSION: ${{ matrix.python-version }}
@@ -27,7 +27,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -U pytest pytest-cov pytest-bdd pytest-reraise pyyaml trollsift posttroll>=1.11 paramiko scp watchdog!=4.0.0 pytroll-collectors>=0.13.0 fsspec s3fs
+          pip install -U pytest pytest-cov pytest-bdd pytest-reraise pyyaml trollsift posttroll>=1.11 paramiko scp watchdog!=4.0.0 pytroll-collectors>=0.16.0 fsspec s3fs
 
       - name: Install trollmoves
         run: |

setup.py

@@ -33,7 +33,7 @@
         'paramiko',
         'scp',
     ],
-    "remote_fs": ["pytroll-collectors>=0.13.0"],
+    "remote_fs": ["pytroll-collectors>=0.16.0"],
 }
 
 all_extras = []

trollmoves/tests/test_client.py

@@ -37,25 +37,25 @@
 from trollmoves.client import MoveItClient, parse_args
 
 MSG_FILE = Message('/topic', 'file', {'uid': 'file1.png',
-                                      'uri': '/tmp/file1.png'})
+                                      'uri': '/data_dir/file1.png'})
 MSG_FILE_TAR = Message('/topic', 'file', {'uid': 'file1.tar',
-                                          'uri': '/tmp/file1.tar'})
+                                          'uri': '/data_dir/file1.tar'})
 MSG_FILE_BZ2 = Message('/topic', 'file', {'uid': 'file1.png.bz2',
-                                          'uri': '/tmp/file1.png.bz2'})
+                                          'uri': '/data_dir/file1.png.bz2'})
 MSG_FILE_XRIT = Message('/topic', 'file', {'uid': 'file1-C_',
-                                           'uri': '/tmp/file1-C_'})
+                                           'uri': '/data_dir/file1-C_'})
 MSG_DATASET_TAR = Message('/topic', 'dataset',
                           {'dataset': [{'uid': 'file1.tgz',
-                                        'uri': '/tmp/file1.tgz'},
+                                        'uri': '/data_dir/file1.tgz'},
                                        {'uid': 'file2.tar.gz',
-                                        'uri': '/tmp/file2.tar.gz'}]})
+                                        'uri': '/data_dir/file2.tar.gz'}]})
 MSG_COLLECTION_TAR = Message('/topic', 'collection',
                              {'collection':
                               [{'dataset': [{'uid': 'file1.tar.bz2',
-                                             'uri': '/tmp/file1.tar.bz2'}]}]})
+                                             'uri': '/data_dir/file1.tar.bz2'}]}]})
 MSG_MIRROR = Message('/topic', 'file', {'fname': 'file1', 'uri':
-                                        'scp://user@host/tmp/bar/file1.txt', 'uid':
-                                        'file1.txt', 'destination': 'scp://targethost.domain/tmp/bar/',
+                                        'scp://user@host/data_dir/bar/file1.txt', 'uid':
+                                        'file1.txt', 'destination': 'scp://targethost.domain/data_dir/bar/',
                                         'origin': 'sourcehost.domain:9201'})
 COMPRESSION_CONFIG = """
 [DEFAULT]
@@ -86,7 +86,7 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010 explorer:9010 primary_client
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
@@ -97,7 +97,7 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010 explorer:9010 primary_client
-destination = scp:///tmp/bar
+destination = scp:///data_dir/bar
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
@@ -108,7 +108,7 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010 explorer:9010 primary_client
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 12345
@@ -119,7 +119,7 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
@@ -130,7 +130,7 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010 explorer:9010 primary_client
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/zero_degrees
 publish_port = 0
@@ -141,7 +141,7 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010 explorer:9010 primary_client
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
@@ -153,15 +153,15 @@
 # Example acting as a hot spare
 [eumetcast_hrit_0deg_scp_hot_spare]
 providers = satmottag2:9010 satmottag:9010 explorer:9010 primary_client
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
 processing_delay = 0.02
 
 [foo]
 providers = bar
-destination = scp:///tmp/foo
+destination = scp:///data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
@@ -171,7 +171,7 @@
 CLIENT_CONFIG_BACKUP_TARGETS = """
 [foo]
 providers = bar
-destination = scp://primary_host/tmp/foo
+destination = scp://primary_host/data_dir/foo
 login = user
 topic = /1b/hrit-segment/0deg
 publish_port = 0
@@ -327,48 +327,51 @@ def chain_config_with_one_item_nameservers_is_false(client_config_1_item_nameser
     yield conf
 
 
-@patch('os.remove')
 @patch('trollmoves.client.check_output')
-def test_unpack_xrit(check_output, remove):
-    """Test unpacking of Eumetsat SEVIRI XRIT/HRIT segments."""
+def test_unpack_xrit_decompressed_no_config(check_output):
+    """Test unpacking of already decompressed xrit segments without config."""
     from trollmoves.client import unpack_xrit
 
     # No configured options
     kwargs = {}
 
     # File already decompressed
-    fname_in = "/tmp/H-000-MSG4__-MSG4________-_________-PRO______-201909031245-__"
+    fname_in = "/data_dir/H-000-MSG4__-MSG4________-_________-PRO______-201909031245-__"
 
     res = unpack_xrit(fname_in, **kwargs)
     assert res == fname_in
     check_output.assert_not_called()
-    remove.assert_not_called()
+
+
+@patch('trollmoves.client.check_output')
+def test_unpack_xrit_compressed_no_config(check_output):
+    """Test unpacking of xrit segments without config."""
+    from trollmoves.client import unpack_xrit
+
+    # No configured options
+    kwargs = {}
 
     # Compressed segment
-    fname_in = "/tmp/H-000-MSG4__-MSG4________-IR_134___-000003___-201909031245-C_"
+    fname_in = "/data_dir/H-000-MSG4__-MSG4________-IR_134___-000003___-201909031245-C_"
 
     try:
-        res = unpack_xrit(fname_in, **kwargs)
+        _ = unpack_xrit(fname_in, **kwargs)
         # Should raise OSError as xritdecompressor hasn't been defined
         raise AssertionError
     except OSError:
         pass
-    remove.assert_not_called()
 
-    # Define xritdecompressor path
-    kwargs = {'xritdecompressor': '/path/to/xRITDecompress'}
-    res = unpack_xrit(fname_in, **kwargs)
-    assert check_output.called_once_with(
-        ['/path/to/xRITDecompress', fname_in], cwd=('/tmp'))
-    remove.assert_not_called()
 
-    # Define also delete
-    kwargs = {'delete': True, 'xritdecompressor': '/path/to/xRITDecompress'}
+@patch('trollmoves.client.check_output')
+def test_unpack_xrit_compressed_xritdecompressor(check_output):
+    """Test unpacking of xrit segments when xritdecompressor is defined."""
+    from trollmoves.client import unpack_xrit
 
-    res = unpack_xrit(fname_in, **kwargs)
-    assert check_output.called_once_with(
-        ['/path/to/xRITDecompress', fname_in], cwd=('/tmp'))
-    assert remove.called_once_with(fname_in)
+    kwargs = {'xritdecompressor': '/path/to/xRITDecompress'}
+    fname_in = "/data_dir/H-000-MSG4__-MSG4________-IR_134___-000003___-201909031245-C_"
+    _ = unpack_xrit(fname_in, **kwargs)
+    check_output.assert_called_once_with(
+        ['/path/to/xRITDecompress', fname_in], cwd=('/data_dir'))
 
 
 def test_unpack_bzip():
@@ -543,7 +546,7 @@ def test_unpack_and_create_local_message_xrit_compression_with_delete(unpackers)
     unpackers['xrit'].return_value = 'new_file1.png'
     with patch('os.remove') as remove:
         _ = unp(copy.copy(MSG_FILE_XRIT), LOCAL_DIR, **kwargs)
-    assert remove.called_once_with(os.path.join(LOCAL_DIR, MSG_FILE_XRIT.data['uid']))
+    remove.assert_called_once_with(os.path.join(LOCAL_DIR, MSG_FILE_XRIT.data['uid']))
     del kwargs['delete']
 
 
@@ -1501,7 +1504,7 @@ def test_chain_publisher_needs_restarting_non_publisher_value_added(Listener, cr
 
     config = CHAIN_BASIC_CONFIG.copy()
     chain = Chain("foo", config.copy())
-    config["destination"] = "file:///tmp/"
+    config["destination"] = "file:///data_dir/"
     assert chain.publisher_needs_restarting(config.copy()) is False