From ae585a77fcd3453875fb797aa71343103679575e Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Sun, 3 Nov 2024 08:11:22 +0000
Subject: [PATCH 01/17] missing copyrigths

---
 src/anemoi/transform/__init__.py           | 4 +++-
 src/anemoi/transform/__main__.py           | 5 ++---
 src/anemoi/transform/commands/__init__.py  | 5 ++---
 src/anemoi/transform/data/__init__.py      | 4 +++-
 src/anemoi/transform/filters/__init__.py   | 4 +++-
 src/anemoi/transform/grids/__init__.py     | 4 +++-
 src/anemoi/transform/grouping/__init__.py  | 4 +++-
 src/anemoi/transform/sources/__init__.py   | 4 +++-
 src/anemoi/transform/variables/__init__.py | 4 +++-
 src/anemoi/transform/workflows/__init__.py | 4 +++-
 10 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/anemoi/transform/__init__.py b/src/anemoi/transform/__init__.py
index bdd630a..019edd8 100644
--- a/src/anemoi/transform/__init__.py
+++ b/src/anemoi/transform/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/__main__.py b/src/anemoi/transform/__main__.py
index be940c2..0057b94 100644
--- a/src/anemoi/transform/__main__.py
+++ b/src/anemoi/transform/__main__.py
@@ -1,12 +1,11 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
-#
 
 from anemoi.utils.cli import cli_main
 from anemoi.utils.cli import make_parser
diff --git a/src/anemoi/transform/commands/__init__.py b/src/anemoi/transform/commands/__init__.py
index cebb539..e5e2219 100644
--- a/src/anemoi/transform/commands/__init__.py
+++ b/src/anemoi/transform/commands/__init__.py
@@ -1,12 +1,11 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
-#
 
 import os
 
diff --git a/src/anemoi/transform/data/__init__.py b/src/anemoi/transform/data/__init__.py
index c5ab37c..1d10a4a 100644
--- a/src/anemoi/transform/data/__init__.py
+++ b/src/anemoi/transform/data/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/filters/__init__.py b/src/anemoi/transform/filters/__init__.py
index b03adbe..c06b5d7 100644
--- a/src/anemoi/transform/filters/__init__.py
+++ b/src/anemoi/transform/filters/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/grids/__init__.py b/src/anemoi/transform/grids/__init__.py
index 1a4a17b..c809c11 100644
--- a/src/anemoi/transform/grids/__init__.py
+++ b/src/anemoi/transform/grids/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/grouping/__init__.py b/src/anemoi/transform/grouping/__init__.py
index a1d4004..f975e03 100644
--- a/src/anemoi/transform/grouping/__init__.py
+++ b/src/anemoi/transform/grouping/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/sources/__init__.py b/src/anemoi/transform/sources/__init__.py
index 26227ff..28d0105 100644
--- a/src/anemoi/transform/sources/__init__.py
+++ b/src/anemoi/transform/sources/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/variables/__init__.py b/src/anemoi/transform/variables/__init__.py
index 502fb5e..830d6ae 100644
--- a/src/anemoi/transform/variables/__init__.py
+++ b/src/anemoi/transform/variables/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
diff --git a/src/anemoi/transform/workflows/__init__.py b/src/anemoi/transform/workflows/__init__.py
index 10945d7..612df9d 100644
--- a/src/anemoi/transform/workflows/__init__.py
+++ b/src/anemoi/transform/workflows/__init__.py
@@ -1,6 +1,8 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.

From 8621a7e413cc577cae0ea0fddd9f8dbe5861a128 Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Sun, 3 Nov 2024 08:22:12 +0000
Subject: [PATCH 02/17] missing copyrigths

---
 pyproject.toml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index e7c1f62..37ba2f4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,14 +1,12 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
-
 [build-system]
 requires = [ "setuptools>=60", "setuptools-scm>=8" ]
 

From 063c19bbbb14ac10af18aa09b942bf26791a5135 Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Wed, 6 Nov 2024 09:53:15 +0000
Subject: [PATCH 03/17] Add variable similarity

---
 src/anemoi/transform/variables/__init__.py  |  7 +++++++
 src/anemoi/transform/variables/variables.py | 15 +++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/src/anemoi/transform/variables/__init__.py b/src/anemoi/transform/variables/__init__.py
index 502fb5e..7515f0a 100644
--- a/src/anemoi/transform/variables/__init__.py
+++ b/src/anemoi/transform/variables/__init__.py
@@ -81,3 +81,10 @@ def is_computed_forcing(self):
     @property
     def is_from_input(self):
         pass
+
+    def similarity(self, other):
+        """Compute the similarity between two variables. This is used when
+        encoding a variables in GRIB and we do not have a template for it.
+        We can then try to find the most similar variable for which we have a template.
+        """
+        return 0
diff --git a/src/anemoi/transform/variables/variables.py b/src/anemoi/transform/variables/variables.py
index cb74ee9..aa64b08 100644
--- a/src/anemoi/transform/variables/variables.py
+++ b/src/anemoi/transform/variables/variables.py
@@ -51,6 +51,21 @@ def is_instantanous(self):
     def grib_keys(self):
         return self.data.get("mars", {}).copy()
 
+    def similarity(self, other):
+        if not isinstance(other, VariableFromMarsVocabulary):
+            return 0
+
+        def __similarity(a, b):
+            if isinstance(a, dict) and isinstance(b, dict):
+                return sum(__similarity(a[k], b[k]) for k in set(a.keys()) & set(b.keys()))
+
+            if isinstance(a, list) and isinstance(b, list):
+                return sum(__similarity(a[i], b[i]) for i in range(min(len(a), len(b))))
+
+            return 1 if a == b else 0
+
+        return __similarity(self.data, other.data)
+
 
 class VariableFromDict(VariableFromMarsVocabulary):
     """A variable that is defined by a user provided dictionary."""

From a0320a93b7b5261ecb8e09dcfa83dfe97b04fca6 Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 09:37:05 +0000
Subject: [PATCH 04/17] update registry

---
 dev/dev.py                                 | 16 ++---
 src/anemoi/transform/filters/__init__.py   | 16 +----
 src/anemoi/transform/filters/uv_to_ddff.py |  6 +-
 src/anemoi/transform/registry.py           | 75 ----------------------
 src/anemoi/transform/sources/__init__.py   | 16 +----
 src/anemoi/transform/sources/mars.py       |  6 +-
 src/anemoi/transform/transform.py          |  4 +-
 src/anemoi/transform/workflows/__init__.py | 16 +----
 src/anemoi/transform/workflows/pipeline.py |  6 +-
 9 files changed, 22 insertions(+), 139 deletions(-)
 delete mode 100644 src/anemoi/transform/registry.py

diff --git a/dev/dev.py b/dev/dev.py
index 9b6394e..73a7a8a 100644
--- a/dev/dev.py
+++ b/dev/dev.py
@@ -7,15 +7,13 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-from anemoi.transform.filters import filter_factory
-from anemoi.transform.sources import source_factory
-from anemoi.transform.workflows import workflow_factory
+from anemoi.transform.filters import filter_registry
+from anemoi.transform.sources import source_registry
+from anemoi.transform.workflows import workflow_registry
 
 ################
 
-mars = source_factory(
-    "mars",
-)
+mars = source_registry.create("mars")
 
 r = dict(
     param=["u", "v", "t", "q"],
@@ -31,14 +29,14 @@
 
 ################
 
-uv_2_ddff = filter_factory("uv_2_ddff")
+uv_2_ddff = filter_registry.create("uv_2_ddff")
 
 data = uv_2_ddff.forward(data)
 for f in data:
     print(f)
 
 
-ddff_2_uv = filter_factory("ddff_2_uv")
+ddff_2_uv = filter_registry.create("ddff_2_uv")
 data = ddff_2_uv.forward(data)
 for f in data:
     print(f)
@@ -46,7 +44,7 @@
 
 ################
 
-pipeline = workflow_factory("pipeline", filters=[mars, uv_2_ddff, ddff_2_uv])
+pipeline = workflow_registry.create("pipeline", filters=[mars, uv_2_ddff, ddff_2_uv])
 for f in pipeline(r):
     print(f)
 
diff --git a/src/anemoi/transform/filters/__init__.py b/src/anemoi/transform/filters/__init__.py
index 8524c67..b03adbe 100644
--- a/src/anemoi/transform/filters/__init__.py
+++ b/src/anemoi/transform/filters/__init__.py
@@ -6,18 +6,6 @@
 # nor does it submit to any jurisdiction.
 
 
-from ..registry import Registry
+from anemoi.utils.registry import Registry
 
-registry = Registry(__name__)
-
-
-def register_filter(name, maker):
-    registry.register(name, maker)
-
-
-def lookup_filter(name):
-    return registry.lookup(name)
-
-
-def filter_factory(name, *args, **kwargs):
-    return lookup_filter(name)(*args, **kwargs)
+filter_registry = Registry(__name__)
diff --git a/src/anemoi/transform/filters/uv_to_ddff.py b/src/anemoi/transform/filters/uv_to_ddff.py
index 2f6201d..8a763d3 100644
--- a/src/anemoi/transform/filters/uv_to_ddff.py
+++ b/src/anemoi/transform/filters/uv_to_ddff.py
@@ -11,7 +11,7 @@
 from earthkit.meteo.wind.array import polar_to_xy
 from earthkit.meteo.wind.array import xy_to_polar
 
-from . import register_filter
+from . import filter_registry
 from .base import SimpleFilter
 
 
@@ -80,5 +80,5 @@ def backward_transform(self, speed, direction):
         yield self.new_field_from_numpy(v, template=direction, param=self.v_component)
 
 
-register_filter("uv_2_ddff", WindComponents)
-register_filter("ddff_2_uv", WindComponents.reversed)
+filter_registry.register("uv_2_ddff", WindComponents)
+filter_registry.register("ddff_2_uv", WindComponents.reversed)
diff --git a/src/anemoi/transform/registry.py b/src/anemoi/transform/registry.py
deleted file mode 100644
index 2e1240b..0000000
--- a/src/anemoi/transform/registry.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-import importlib
-import logging
-import os
-import sys
-
-import entrypoints
-
-LOG = logging.getLogger(__name__)
-
-
-class Registry:
-    """A registry of factories"""
-
-    def __init__(self, package):
-
-        self.package = package
-        self.registered = {}
-        self.kind = package.split(".")[-1]
-
-    def register(self, name: str, factory: callable):
-        self.registered[name] = factory
-
-    def _load(self, file):
-        name, _ = os.path.splitext(file)
-        try:
-            importlib.import_module(f".{name}", package=self.package)
-        except Exception:
-            LOG.warning(f"Error loading filter '{self.package}.{name}'", exc_info=True)
-
-    def lookup(self, name: str) -> callable:
-        if name in self.registered:
-            return self.registered[name]
-
-        directory = sys.modules[self.package].__path__[0]
-
-        for file in os.listdir(directory):
-
-            if file[0] == ".":
-                continue
-
-            if file == "__init__.py":
-                continue
-
-            full = os.path.join(directory, file)
-            if os.path.isdir(full):
-                if os.path.exists(os.path.join(full, "__init__.py")):
-                    self._load(file)
-                continue
-
-            if file.endswith(".py"):
-                self._load(file)
-
-        entrypoint_group = f"anemoi.{self.kind}"
-        for entry_point in entrypoints.get_group_all(entrypoint_group):
-            if entry_point.name == name:
-                if name in self.registered:
-                    LOG.warning(
-                        f"Overwriting builtin '{name}' from {self.package} with plugin '{entry_point.module_name}'"
-                    )
-                self.registered[name] = entry_point.load()
-
-        if name not in self.registered:
-            raise ValueError(f"Cannot load '{name}' from {self.package}")
-
-        return self.registered[name]
diff --git a/src/anemoi/transform/sources/__init__.py b/src/anemoi/transform/sources/__init__.py
index 35098d5..26227ff 100644
--- a/src/anemoi/transform/sources/__init__.py
+++ b/src/anemoi/transform/sources/__init__.py
@@ -6,18 +6,6 @@
 # nor does it submit to any jurisdiction.
 
 
-from ..registry import Registry
+from anemoi.utils.registry import Registry
 
-registry = Registry(__name__)
-
-
-def register_source(name, maker):
-    registry.register(name, maker)
-
-
-def lookup_source(name):
-    return registry.lookup(name)
-
-
-def source_factory(name, *args, **kwargs):
-    return lookup_source(name)(*args, **kwargs)
+source_registry = Registry(__name__)
diff --git a/src/anemoi/transform/sources/mars.py b/src/anemoi/transform/sources/mars.py
index 0f18d64..41ea5e0 100644
--- a/src/anemoi/transform/sources/mars.py
+++ b/src/anemoi/transform/sources/mars.py
@@ -11,9 +11,10 @@
 import earthkit.data as ekd
 
 from ..source import Source
-from . import register_source
+from . import source_registry
 
 
+@source_registry.register("mars")
 class Mars(Source):
     """A demo source"""
 
@@ -35,6 +36,3 @@ def forward(self, data):
                 return this.forward(self.data)
 
         return Input(data)
-
-
-register_source("mars", Mars)
diff --git a/src/anemoi/transform/transform.py b/src/anemoi/transform/transform.py
index a655614..1d57add 100644
--- a/src/anemoi/transform/transform.py
+++ b/src/anemoi/transform/transform.py
@@ -36,9 +36,9 @@ def reversed(cls, *args, **kwargs):
         return ReversedTransform(cls(*args, **kwargs))
 
     def __or__(self, other):
-        from .workflows import workflow_factory
+        from .workflows import workflow_registry
 
-        return workflow_factory("pipeline", filters=[self, other])
+        return workflow_registry.create("pipeline", filters=[self, other])
 
 
 class ReversedTransform(Transform):
diff --git a/src/anemoi/transform/workflows/__init__.py b/src/anemoi/transform/workflows/__init__.py
index 5225904..10945d7 100644
--- a/src/anemoi/transform/workflows/__init__.py
+++ b/src/anemoi/transform/workflows/__init__.py
@@ -5,18 +5,6 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-from ..registry import Registry
+from anemoi.utils.registry import Registry
 
-registry = Registry(__name__)
-
-
-def register_workflow(name, maker):
-    registry.register(name, maker)
-
-
-def lookup_workflow(name):
-    return registry.lookup(name)
-
-
-def workflow_factory(name, *args, **kwargs):
-    return lookup_workflow(name)(*args, **kwargs)
+workflow_registry = Registry(__name__)
diff --git a/src/anemoi/transform/workflows/pipeline.py b/src/anemoi/transform/workflows/pipeline.py
index 6b3f83f..86413f4 100644
--- a/src/anemoi/transform/workflows/pipeline.py
+++ b/src/anemoi/transform/workflows/pipeline.py
@@ -9,9 +9,10 @@
 
 
 from ..workflow import Workflow
-from . import register_workflow
+from . import workflow_registry
 
 
+@workflow_registry.register("pipeline")
 class Pipeline(Workflow):
     """A simple pipeline of filters"""
 
@@ -27,6 +28,3 @@ def backward(self, data):
         for filter in reversed(self.filters):
             data = filter.backward(data)
         return data
-
-
-register_workflow("pipeline", Pipeline)

From 9112ca2f1e9155e8bf8db015cefcb7f1a28f2d49 Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 11:56:57 +0000
Subject: [PATCH 05/17] add snowcover filter

---
 src/anemoi/transform/filters/snow_cover.py | 48 ++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 src/anemoi/transform/filters/snow_cover.py

diff --git a/src/anemoi/transform/filters/snow_cover.py b/src/anemoi/transform/filters/snow_cover.py
new file mode 100644
index 0000000..757cc34
--- /dev/null
+++ b/src/anemoi/transform/filters/snow_cover.py
@@ -0,0 +1,48 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from . import filter_registry
+from .base import SimpleFilter
+
+
+@filter_registry.register("snow_cover")
+class SnowCover(SimpleFilter):
+    """A filter to compute snow cover from snow density and snow depth."""
+
+    def __init__(
+        self,
+        *,
+        snow_depth="sd",
+        snow_density="rsn",
+        snow_cover="snowc",
+    ):
+        self.snow_depth = snow_depth
+        self.snow_density = snow_density
+        self.snow_cover = snow_cover
+
+    def forward(self, data):
+        return self._transform(
+            data,
+            self.forward_transform,
+            self.snow_depth,
+            self.snow_density,
+        )
+
+    def backward(self, data):
+        raise NotImplementedError("SnowCover is not reversible")
+
+    def forward_transform(self, sd, rsn):
+        """Convert snow depth and snow density to snow cover"""
+
+        snow_cover = sd.to_numpy() * rsn.to_numpy()
+
+        yield self.new_field_from_numpy(snow_cover, template=sd, param=self.snow_cover)
+
+    def backward_transform(self, sd, rsn):
+        raise NotImplementedError("SnowCover is not reversible")

From 8dc92c901cf4b7701385b9628678d9a2954e44a1 Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 13:06:47 +0000
Subject: [PATCH 06/17] add real function

---
 src/anemoi/transform/filters/snow_cover.py | 13 ++++++++++++-
 tests/test_snow.py                         | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_snow.py

diff --git a/src/anemoi/transform/filters/snow_cover.py b/src/anemoi/transform/filters/snow_cover.py
index 757cc34..5a81b8b 100644
--- a/src/anemoi/transform/filters/snow_cover.py
+++ b/src/anemoi/transform/filters/snow_cover.py
@@ -7,10 +7,21 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import numpy as np
+
 from . import filter_registry
 from .base import SimpleFilter
 
 
+def compute_snow_cover(snow_depth, snow_density):
+    """Convert snow depth to snow cover."""
+    tmp1 = (1000 * snow_depth) / snow_density
+    tmp2 = np.clip(snow_density, 100, 400)
+    snow_cover = np.clip(np.tanh((4000 * tmp1) / tmp2), 0, 1)
+    snow_cover[snow_cover > 0.99] = 1.0
+    return snow_cover
+
+
 @filter_registry.register("snow_cover")
 class SnowCover(SimpleFilter):
     """A filter to compute snow cover from snow density and snow depth."""
@@ -40,7 +51,7 @@ def backward(self, data):
     def forward_transform(self, sd, rsn):
         """Convert snow depth and snow density to snow cover"""
 
-        snow_cover = sd.to_numpy() * rsn.to_numpy()
+        snow_cover = compute_snow_cover(sd.to_numpy(), rsn.to_numpy())
 
         yield self.new_field_from_numpy(snow_cover, template=sd, param=self.snow_cover)
 
diff --git a/tests/test_snow.py b/tests/test_snow.py
new file mode 100644
index 0000000..706defa
--- /dev/null
+++ b/tests/test_snow.py
@@ -0,0 +1,20 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import numpy as np
+
+from anemoi.transform.filters.snow_cover import compute_snow_cover
+
+
+def test_snow_cover():
+    snow_depth = np.array([1.0, 2.0, 3.0])
+    snow_density = np.array([0.1, 0.2, 0.3])
+    expected_snow_cover = np.array([0.1, 0.4, 0.9])
+    snow_cover = compute_snow_cover(snow_depth, snow_density)
+    np.testing.assert_allclose(snow_cover, expected_snow_cover)

From 9c31fa718780d23f83ec17bcd1fa441120a92dc0 Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 14:06:20 +0000
Subject: [PATCH 07/17] create filter to mask out glaciers in snow depth

---
 src/anemoi/transform/filters/glacier_mask.py | 57 ++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 src/anemoi/transform/filters/glacier_mask.py

diff --git a/src/anemoi/transform/filters/glacier_mask.py b/src/anemoi/transform/filters/glacier_mask.py
new file mode 100644
index 0000000..43c66cf
--- /dev/null
+++ b/src/anemoi/transform/filters/glacier_mask.py
@@ -0,0 +1,57 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import cfgrib
+import numpy as np
+
+from . import filter_registry
+from .base import SimpleFilter
+
+glacier_mask_file = "/home/rdx/data/climate/climate.v021/95_4/cicecap"
+
+
+def mask_glaciers(snow_depth, glacier_mask_file):
+    ds = cfgrib.open_dataset(glacier_mask_file, backend_kwargs={"read_keys": [], "indexpath": ""})
+    mask = ds.si10.values
+    snow_depth[mask] = np.nan
+    return snow_depth
+
+
+@filter_registry.register("snow_depth_masked")
+class SnowDepthMasked(SimpleFilter):
+    """A filter to mask about glacier in snow depth."""
+
+    def __init__(
+        self,
+        *,
+        snow_depth="sd",
+        snow_depth_masked="sd_masked",
+    ):
+        self.snow_depth = snow_depth
+        self.snow_depth_masked = snow_depth_masked
+
+    def forward(self, data):
+        return self._transform(
+            data,
+            self.forward_transform,
+            self.snow_depth,
+        )
+
+    def backward(self, data):
+        raise NotImplementedError("SnowDepthMasked is not reversible")
+
+    def forward_transform(self, sd):
+        """Mask out glaciers in snow depth"""
+
+        snow_cover = mask_glaciers(sd.to_numpy())
+
+        yield self.new_field_from_numpy(snow_cover, template=sd, param=self.snow_cover)
+
+    def backward_transform(self, sd, rsn):
+        raise NotImplementedError("SnowDepthMasked is not reversible")

From 0a3e4f3e78bd1f21b80b2864deaf7dc3d7ed8455 Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 14:07:01 +0000
Subject: [PATCH 08/17] create filter to mask out glaciers in snow depth

---
 src/anemoi/transform/filters/glacier_mask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/anemoi/transform/filters/glacier_mask.py b/src/anemoi/transform/filters/glacier_mask.py
index 43c66cf..50f9f55 100644
--- a/src/anemoi/transform/filters/glacier_mask.py
+++ b/src/anemoi/transform/filters/glacier_mask.py
@@ -53,5 +53,5 @@ def forward_transform(self, sd):
 
         yield self.new_field_from_numpy(snow_cover, template=sd, param=self.snow_cover)
 
-    def backward_transform(self, sd, rsn):
+    def backward_transform(self, sd):
         raise NotImplementedError("SnowDepthMasked is not reversible")

From 4de03a59bff9d2607719ab904b21f66c4693fa15 Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 14:15:36 +0000
Subject: [PATCH 09/17] externalise path to glacier file

---
 src/anemoi/transform/filters/glacier_mask.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/anemoi/transform/filters/glacier_mask.py b/src/anemoi/transform/filters/glacier_mask.py
index 50f9f55..567c8f4 100644
--- a/src/anemoi/transform/filters/glacier_mask.py
+++ b/src/anemoi/transform/filters/glacier_mask.py
@@ -7,19 +7,15 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-import cfgrib
+import earthkit.data as ekd
 import numpy as np
 
 from . import filter_registry
 from .base import SimpleFilter
 
-glacier_mask_file = "/home/rdx/data/climate/climate.v021/95_4/cicecap"
 
-
-def mask_glaciers(snow_depth, glacier_mask_file):
-    ds = cfgrib.open_dataset(glacier_mask_file, backend_kwargs={"read_keys": [], "indexpath": ""})
-    mask = ds.si10.values
-    snow_depth[mask] = np.nan
+def mask_glaciers(snow_depth, glacier_mask):
+    snow_depth[glacier_mask] = np.nan
     return snow_depth
 
 
@@ -30,9 +26,11 @@ class SnowDepthMasked(SimpleFilter):
     def __init__(
         self,
         *,
+        glacier_mask,
         snow_depth="sd",
         snow_depth_masked="sd_masked",
     ):
+        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy()
         self.snow_depth = snow_depth
         self.snow_depth_masked = snow_depth_masked
 
@@ -49,9 +47,9 @@ def backward(self, data):
     def forward_transform(self, sd):
         """Mask out glaciers in snow depth"""
 
-        snow_cover = mask_glaciers(sd.to_numpy())
+        snow_depth_masked = mask_glaciers(sd.to_numpy(), self.glacier_mask)
 
-        yield self.new_field_from_numpy(snow_cover, template=sd, param=self.snow_cover)
+        yield self.new_field_from_numpy(snow_depth_masked, template=sd, param=self.snow_depth_masked)
 
     def backward_transform(self, sd):
         raise NotImplementedError("SnowDepthMasked is not reversible")

From 5bf8c9ec72b3228facb89cb4c3c86ea3f842339d Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Fri, 8 Nov 2024 14:27:37 +0000
Subject: [PATCH 10/17] create filter to mask out glaciers in snow depth

---
 src/anemoi/transform/filters/glacier_mask.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/anemoi/transform/filters/glacier_mask.py b/src/anemoi/transform/filters/glacier_mask.py
index 567c8f4..1e45574 100644
--- a/src/anemoi/transform/filters/glacier_mask.py
+++ b/src/anemoi/transform/filters/glacier_mask.py
@@ -19,7 +19,7 @@ def mask_glaciers(snow_depth, glacier_mask):
     return snow_depth
 
 
-@filter_registry.register("snow_depth_masked")
+@filter_registry.register("glacier_mask")
 class SnowDepthMasked(SimpleFilter):
     """A filter to mask about glacier in snow depth."""
 
@@ -30,7 +30,7 @@ def __init__(
         snow_depth="sd",
         snow_depth_masked="sd_masked",
     ):
-        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy()
+        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy().bool()
         self.snow_depth = snow_depth
         self.snow_depth_masked = snow_depth_masked
 

From a3256c9c8a44dcc48e77838c636de25355e66e32 Mon Sep 17 00:00:00 2001
From: Jesper Dramsch <jesper@dramsch.net>
Date: Mon, 11 Nov 2024 09:28:51 +0100
Subject: [PATCH 11/17] Update .pre-commit-config.yaml (#10)

---
 .pre-commit-config.yaml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e336464..af7e670 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -59,11 +59,6 @@ repos:
   hooks:
   - id: rstfmt
     exclude: 'cli/.*' # Because we use argparse
-- repo: https://github.com/b8raoult/pre-commit-docconvert
-  rev: "0.1.5"
-  hooks:
-  - id: docconvert
-    args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
   rev: "2.2.3"
   hooks:

From 7bd3b6407095ba1de83fd29cd56c4d08f74ca58e Mon Sep 17 00:00:00 2001
From: Nina Raoult <nina.raoult@ecmwf.int>
Date: Wed, 13 Nov 2024 15:21:08 +0000
Subject: [PATCH 12/17] add filter from land parameters

---
 .../filters/create_land_parameters.py         | 116 ++++++++++++++++++
 src/anemoi/transform/filters/glacier_mask.py  |   2 +-
 2 files changed, 117 insertions(+), 1 deletion(-)
 create mode 100644 src/anemoi/transform/filters/create_land_parameters.py

diff --git a/src/anemoi/transform/filters/create_land_parameters.py b/src/anemoi/transform/filters/create_land_parameters.py
new file mode 100644
index 0000000..5fada6e
--- /dev/null
+++ b/src/anemoi/transform/filters/create_land_parameters.py
@@ -0,0 +1,116 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import numpy as np
+
+from . import filter_registry
+from .base import SimpleFilter
+
+SOIL_TYPE_DIC = {
+    0: {"theta_pwp": 0, "theta_cap": 0},
+    1: {"theta_pwp": 0.059, "theta_cap": 0.244},
+    2: {"theta_pwp": 0.151, "theta_cap": 0.347},
+    3: {"theta_pwp": 0.133, "theta_cap": 0.383},
+    4: {"theta_pwp": 0.279, "theta_cap": 0.448},
+    5: {"theta_pwp": 0.335, "theta_cap": 0.541},
+    6: {"theta_pwp": 0.267, "theta_cap": 0.663},
+    7: {"theta_pwp": 0.151, "theta_cap": 0.347},
+}
+
+VEG_TYPE_DIC = {
+    0: {"veg_rsmin": 250.0, "veg_cov": 0.0, "veg_z0m": 0.013},
+    1: {"veg_rsmin": 125.0, "veg_cov": 0.9, "veg_z0m": 0.25},
+    2: {"veg_rsmin": 80.0, "veg_cov": 0.85, "veg_z0m": 0.1},
+    3: {"veg_rsmin": 395.0, "veg_cov": 0.9, "veg_z0m": 2.0},
+    4: {"veg_rsmin": 320.0, "veg_cov": 0.9, "veg_z0m": 2.0},
+    5: {"veg_rsmin": 215.0, "veg_cov": 0.9, "veg_z0m": 2.0},
+    6: {"veg_rsmin": 320.0, "veg_cov": 0.99, "veg_z0m": 2.0},
+    7: {"veg_rsmin": 100.0, "veg_cov": 0.7, "veg_z0m": 0.5},
+    8: {"veg_rsmin": 250.0, "veg_cov": 0.0, "veg_z0m": 0.013},
+    9: {"veg_rsmin": 45.0, "veg_cov": 0.5, "veg_z0m": 0.03},
+    10: {"veg_rsmin": 110.0, "veg_cov": 0.9, "veg_z0m": 0.5},
+    11: {"veg_rsmin": 45.0, "veg_cov": 0.1, "veg_z0m": 0.03},
+    12: {"veg_rsmin": 0.0, "veg_cov": 0.0, "veg_z0m": 0.0013},
+    13: {"veg_rsmin": 130.0, "veg_cov": 0.6, "veg_z0m": 0.25},
+    14: {"veg_rsmin": 0.0, "veg_cov": 0.0, "veg_z0m": 0.0001},
+    15: {"veg_rsmin": 0.0, "veg_cov": 0.0, "veg_z0m": 0.0001},
+    16: {"veg_rsmin": 230.0, "veg_cov": 0.5, "veg_z0m": 0.5},
+    17: {"veg_rsmin": 110.0, "veg_cov": 0.4, "veg_z0m": 0.1},
+    18: {"veg_rsmin": 180.0, "veg_cov": 0.9, "veg_z0m": 1.50},
+    19: {"veg_rsmin": 175.0, "veg_cov": 0.9, "veg_z0m": 1.1},
+    20: {"veg_rsmin": 150.0, "veg_cov": 0.6, "veg_z0m": 0.02},
+}
+
+
+def read_crosswalking_table(param, param_dic):
+    arrays = [np.array([param_dic[x][key] for x in param]) for key in param_dic[0].keys()]
+    return arrays
+
+
+@filter_registry.register("create_land_parameters")
+class LandParameters(SimpleFilter):
+    """A filter to add static parameters from table based on soil/vegetation type."""
+
+    def __init__(
+        self,
+        *,
+        high_veg_type="tvh",
+        low_veg_type="tvl",
+        soil_type="slt",
+        hveg_rsmin="hveg_rsmin",
+        hveg_cov="hveg_cov",
+        hveg_z0m="hveg_z0m",
+        lveg_rsmin="lveg_rsmin",
+        lveg_cov="lveg_cov",
+        lveg_z0m="lveg_z0m",
+        theta_pwp="theta_pwp",
+        theta_cap="theta_cap",
+    ):
+        self.high_veg_type = high_veg_type
+        self.low_veg_type = low_veg_type
+        self.soil_type = soil_type
+        self.hveg_rsmin = hveg_rsmin
+        self.hveg_cov = hveg_cov
+        self.hveg_z0m = hveg_z0m
+        self.lveg_rsmin = lveg_rsmin
+        self.lveg_cov = lveg_cov
+        self.lveg_z0m = lveg_z0m
+        self.theta_pwp = theta_pwp
+        self.theta_cap = theta_cap
+
+    def forward(self, data):
+        return self._transform(
+            data,
+            self.forward_transform,
+            self.high_veg_type,
+            self.low_veg_type,
+            self.soil_type,
+        )
+
+    def backward(self, data):
+        raise NotImplementedError("LandParameters is not reversible")
+
+    def forward_transform(self, tvh, tvl, sotype):
+        """Get static parameters from table based on soil/vegetation type"""
+
+        hveg_rsmin, hveg_cov, hveg_z0m = read_crosswalking_table(tvh.to_numpy(), VEG_TYPE_DIC)
+        lveg_rsmin, lveg_cov, lveg_z0m = read_crosswalking_table(tvl.to_numpy(), VEG_TYPE_DIC)
+        theta_pwp, theta_cap = read_crosswalking_table(sotype.to_numpy(), SOIL_TYPE_DIC)
+
+        yield self.new_field_from_numpy(hveg_rsmin, template=tvh, param=self.hveg_rsmin)
+        yield self.new_field_from_numpy(hveg_cov, template=tvh, param=self.hveg_cov)
+        yield self.new_field_from_numpy(hveg_z0m, template=tvh, param=self.hveg_z0m)
+        yield self.new_field_from_numpy(lveg_rsmin, template=tvl, param=self.lveg_rsmin)
+        yield self.new_field_from_numpy(lveg_cov, template=tvl, param=self.lveg_cov)
+        yield self.new_field_from_numpy(lveg_z0m, template=tvl, param=self.lveg_z0m)
+        yield self.new_field_from_numpy(theta_pwp, template=sotype, param=self.theta_pwp)
+        yield self.new_field_from_numpy(theta_cap, template=sotype, param=self.theta_cap)
+
+    def backward_transform(self, tvh, tvl, sotype):
+        raise NotImplementedError("LandParameters is not reversible")
diff --git a/src/anemoi/transform/filters/glacier_mask.py b/src/anemoi/transform/filters/glacier_mask.py
index 1e45574..bd34b3d 100644
--- a/src/anemoi/transform/filters/glacier_mask.py
+++ b/src/anemoi/transform/filters/glacier_mask.py
@@ -30,7 +30,7 @@ def __init__(
         snow_depth="sd",
         snow_depth_masked="sd_masked",
     ):
-        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy().bool()
+        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy().astype("bool")
         self.snow_depth = snow_depth
         self.snow_depth_masked = snow_depth_masked
 

From e80dee6b6ac49a2ce31730e5a56a68779c2b2328 Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Wed, 13 Nov 2024 17:31:25 +0000
Subject: [PATCH 13/17] better grib encoding

---
 src/anemoi/transform/filters/base.py         | 5 ++++-
 src/anemoi/transform/filters/glacier_mask.py | 2 +-
 src/anemoi/transform/grouping/__init__.py    | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/anemoi/transform/filters/base.py b/src/anemoi/transform/filters/base.py
index 898b5ff..9e9ab43 100644
--- a/src/anemoi/transform/filters/base.py
+++ b/src/anemoi/transform/filters/base.py
@@ -35,7 +35,10 @@ def _transform(self, data, transform, *group_by):
 
     def new_field_from_numpy(self, array, *, template, param):
         """Create a new field from a numpy array."""
-        md = template.metadata().override(shortName=param)
+        if isinstance(param, int):
+            md = template.metadata().override(paramId=param)
+        else:
+            md = template.metadata().override(shortName=param)
         # return ekd.ArrayField(array, md)
         return ekd.FieldList.from_array(array, md)[0]
 
diff --git a/src/anemoi/transform/filters/glacier_mask.py b/src/anemoi/transform/filters/glacier_mask.py
index 1e45574..576940d 100644
--- a/src/anemoi/transform/filters/glacier_mask.py
+++ b/src/anemoi/transform/filters/glacier_mask.py
@@ -30,7 +30,7 @@ def __init__(
         snow_depth="sd",
         snow_depth_masked="sd_masked",
     ):
-        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy().bool()
+        self.glacier_mask = ekd.from_source("file", glacier_mask)[0].to_numpy().astype(bool)
         self.snow_depth = snow_depth
         self.snow_depth_masked = snow_depth_masked
 
diff --git a/src/anemoi/transform/grouping/__init__.py b/src/anemoi/transform/grouping/__init__.py
index f975e03..2e6be64 100644
--- a/src/anemoi/transform/grouping/__init__.py
+++ b/src/anemoi/transform/grouping/__init__.py
@@ -45,6 +45,6 @@ def iterate(self, data, *, other=_lost):
 
         for _, group in groups.items():
             if len(group) != len(self.params):
-                raise ValueError("Missing component")
+                raise ValueError(f"Missing component. Want {sorted(self.params)}, got {sorted(group.keys())}")
 
             yield tuple(group[p] for p in self.params)

From 13883f57e0851c505e2ad6654627f8d9f8f641c6 Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Wed, 13 Nov 2024 17:35:12 +0000
Subject: [PATCH 14/17] rename filter

---
 .../filters/{create_land_parameters.py => land_parameters.py}   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename src/anemoi/transform/filters/{create_land_parameters.py => land_parameters.py} (98%)

diff --git a/src/anemoi/transform/filters/create_land_parameters.py b/src/anemoi/transform/filters/land_parameters.py
similarity index 98%
rename from src/anemoi/transform/filters/create_land_parameters.py
rename to src/anemoi/transform/filters/land_parameters.py
index 5fada6e..04d1dc8 100644
--- a/src/anemoi/transform/filters/create_land_parameters.py
+++ b/src/anemoi/transform/filters/land_parameters.py
@@ -53,7 +53,7 @@ def read_crosswalking_table(param, param_dic):
     return arrays
 
 
-@filter_registry.register("create_land_parameters")
+@filter_registry.register("land_parameters")
 class LandParameters(SimpleFilter):
     """A filter to add static parameters from table based on soil/vegetation type."""
 

From a124fc4170fc4b08cf9e61c4f7349bbd66c661f2 Mon Sep 17 00:00:00 2001
From: Baudouin Raoult <baudouin.raoult@ecmwf.int>
Date: Thu, 14 Nov 2024 18:51:02 +0000
Subject: [PATCH 15/17] move code from datasets

---
 src/anemoi/transform/fields.py                | 106 ++++++++++++++++++
 src/anemoi/transform/filters/base.py          |  18 ++-
 .../transform/filters/land_parameters.py      |   2 +
 src/anemoi/transform/grouping/__init__.py     |   2 +
 4 files changed, 117 insertions(+), 11 deletions(-)
 create mode 100644 src/anemoi/transform/fields.py

diff --git a/src/anemoi/transform/fields.py b/src/anemoi/transform/fields.py
new file mode 100644
index 0000000..d87c1b8
--- /dev/null
+++ b/src/anemoi/transform/fields.py
@@ -0,0 +1,106 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import logging
+
+from earthkit.data.indexing.fieldlist import FieldArray
+
+LOG = logging.getLogger(__name__)
+
+
+def new_fieldlist_from_list(fields):
+    return FieldArray(fields)
+
+
+def new_empty_fieldlist():
+    return FieldArray([])
+
+
+class WrappedField:
+    """A wrapper around a earthkit-data field object."""
+
+    def __init__(self, field):
+        self._field = field
+
+    def __getattr__(self, name):
+        if name not in (
+            "mars_area",
+            "mars_grid",
+            "to_numpy",
+            "metadata",
+        ):
+            LOG.warning(f"NewField: forwarding `{name}`")
+        return getattr(self._field, name)
+
+    def __repr__(self) -> str:
+        return repr(self._field)
+
+
+class NewDataField(WrappedField):
+    """Change the data of a field."""
+
+    def __init__(self, field, data):
+        super().__init__(field)
+        self._data = data
+        self.shape = data.shape
+
+    def to_numpy(self, flatten=False, dtype=None, index=None):
+        data = self._data
+        if dtype is not None:
+            data = data.astype(dtype)
+        if flatten:
+            data = data.flatten()
+        if index is not None:
+            data = data[index]
+        return data
+
+
+class NewMetadataField(WrappedField):
+    """Change the metadata of a field."""
+
+    def __init__(self, field, **kwargs):
+        super().__init__(field)
+        self._metadata = kwargs
+
+    def metadata(self, *args, **kwargs):
+
+        if kwargs.get("namespace"):
+            assert kwargs.get("namespace") == "mars", kwargs
+            assert len(args) == 0, (args, kwargs)
+            mars = self._field.metadata(**kwargs).copy()
+            for k in list(mars.keys()):
+                if k in self._metadata:
+                    mars[k] = self._metadata[k]
+            return mars
+
+        if len(args) == 1 and args[0] in self._metadata:
+            return self._metadata[args[0]]
+
+        return self._field.metadata(*args, **kwargs)
+
+
+class NewValidDateTimeField(NewMetadataField):
+    """Change the valid_datetime of a field."""
+
+    def __init__(self, field, valid_datetime):
+        date = int(valid_datetime.date().strftime("%Y%m%d"))
+        assert valid_datetime.time().minute == 0, valid_datetime.time()
+        time = valid_datetime.time().hour
+
+        self.valid_datetime = valid_datetime
+
+        super().__init__(field, date=date, time=time, step=0, valid_datetime=valid_datetime.isoformat())
+
+
+def new_field_from_numpy(array, *, template, **metadata):
+    return NewMetadataField(NewDataField(template, array), **metadata)
+
+
+def new_field_with_valid_datetime(template, date):
+    return NewValidDateTimeField(template, date)
diff --git a/src/anemoi/transform/filters/base.py b/src/anemoi/transform/filters/base.py
index 9e9ab43..90d98a8 100644
--- a/src/anemoi/transform/filters/base.py
+++ b/src/anemoi/transform/filters/base.py
@@ -8,13 +8,16 @@
 # nor does it submit to any jurisdiction.
 
 
+import logging
 from abc import abstractmethod
 
-import earthkit.data as ekd
-
+from ..fields import new_field_from_numpy
+from ..fields import new_fieldlist_from_list
 from ..filter import Filter
 from ..grouping import GroupByMarsParam
 
+LOG = logging.getLogger(__name__)
+
 
 class SimpleFilter(Filter):
     """A filter to convert only some fields.
@@ -35,17 +38,10 @@ def _transform(self, data, transform, *group_by):
 
     def new_field_from_numpy(self, array, *, template, param):
         """Create a new field from a numpy array."""
-        if isinstance(param, int):
-            md = template.metadata().override(paramId=param)
-        else:
-            md = template.metadata().override(shortName=param)
-        # return ekd.ArrayField(array, md)
-        return ekd.FieldList.from_array(array, md)[0]
+        return new_field_from_numpy(array, template=template, param=param)
 
     def new_fieldlist_from_list(self, fields):
-        from earthkit.data.indexing.fieldlist import FieldArray
-
-        return FieldArray(fields)
+        return new_fieldlist_from_list(fields)
 
     @abstractmethod
     def forward_transform(self, *fields):
diff --git a/src/anemoi/transform/filters/land_parameters.py b/src/anemoi/transform/filters/land_parameters.py
index 04d1dc8..ec4b54c 100644
--- a/src/anemoi/transform/filters/land_parameters.py
+++ b/src/anemoi/transform/filters/land_parameters.py
@@ -60,9 +60,11 @@ class LandParameters(SimpleFilter):
     def __init__(
         self,
         *,
+        # Input parameters
         high_veg_type="tvh",
         low_veg_type="tvl",
         soil_type="slt",
+        # Output parameters
         hveg_rsmin="hveg_rsmin",
         hveg_cov="hveg_cov",
         hveg_z0m="hveg_z0m",
diff --git a/src/anemoi/transform/grouping/__init__.py b/src/anemoi/transform/grouping/__init__.py
index 2e6be64..edca8b2 100644
--- a/src/anemoi/transform/grouping/__init__.py
+++ b/src/anemoi/transform/grouping/__init__.py
@@ -45,6 +45,8 @@ def iterate(self, data, *, other=_lost):
 
         for _, group in groups.items():
             if len(group) != len(self.params):
+                for p in data:
+                    print(p)
                 raise ValueError(f"Missing component. Want {sorted(self.params)}, got {sorted(group.keys())}")
 
             yield tuple(group[p] for p in self.params)

From 853b8776a3cc1048ad7c73a52391c3fba70a8cc5 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 18 Nov 2024 13:36:04 +0000
Subject: [PATCH 16/17] skipping test until it is implemented with actual test
 values

---
 tests/test_snow.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_snow.py b/tests/test_snow.py
index 706defa..5c1e06c 100644
--- a/tests/test_snow.py
+++ b/tests/test_snow.py
@@ -8,10 +8,12 @@
 # nor does it submit to any jurisdiction.
 
 import numpy as np
+import pytest
 
 from anemoi.transform.filters.snow_cover import compute_snow_cover
 
 
+@pytest.mark.skip("Test not implemented")
 def test_snow_cover():
     snow_depth = np.array([1.0, 2.0, 3.0])
     snow_density = np.array([0.1, 0.2, 0.3])

From 5c71aee2cf16271723fe5703c7f5b6c016651dd8 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Sat, 23 Nov 2024 12:25:03 +0000
Subject: [PATCH 17/17] [create-pull-request] automated change (#14)

Co-authored-by: floriankrb <8441217+floriankrb@users.noreply.github.com>
---
 CHANGELOG.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1523586..635c14b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
-## [Unreleased](https://github.com/ecmwf/anemoi-utils/transform/0.0.5...HEAD/compare/0.0.8...HEAD)
+## [Unreleased](https://github.com/ecmwf/anemoi-utils/transform/0.0.5...HEAD/compare/0.1.0...HEAD)
+
+## [0.1.0](https://github.com/ecmwf/anemoi-utils/transform/0.0.5...HEAD/compare/0.0.8...0.1.0) - 2024-11-18
 
 ## [0.0.8](https://github.com/ecmwf/anemoi-utils/transform/0.0.5...HEAD/compare/0.0.5...0.0.8) - 2024-10-26