Merge pull request #5 from eumetsat/remove_MetopProduct

Remove MetopProduct API

Project.toml

@@ -6,13 +6,15 @@ version = "0.0.6"
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
+Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
 
 [compat]
 Aqua = "0.8"
 CSV = "0.10"
 CommonDataModel = "0.3"
+Compat = "4.10"
 Dates = "1"
 DiskArrays = "0.3, 0.4"
 NCDatasets = "0.14"

README.md

@@ -9,7 +9,7 @@ This package is still in the early development phase. Note that the package was
 
 MetopDatasets.jl is a package for reading products from the [METOP satellites](https://www.eumetsat.int/our-satellites/metop-series) using the native binary format specified for each product. The METOP satellites are part of the EUMETSAT-POLAR-SYSTEM (EPS) and produce long series of near real-time weather and climate observation. Learn more and access the products on [EUMETSATs user-portal](https://user.eumetsat.int/dashboard).
 
-MetopDatasets.jl has two APIs for reading native products. The first API is `MetopDataset` that implements the [CommonDataModel.jl](https://github.com/JuliaGeo/CommonDataModel.jl) interface and thus provides data access similar to e.g. [NCDatasets.jl](https://github.com/Alexander-Barth/NCDatasets.jl) and [GRIBDatasets.jl](https://github.com/JuliaGeo/GRIBDatasets.jl). The second API is `MetopProduct` which reads an entire product into a Julia object with a data structure similar to the native file structure. 
+MetopDatasets.jl exports the `MetopDataset` API which is an implementation of the [CommonDataModel.jl](https://github.com/JuliaGeo/CommonDataModel.jl) interface and thus provides data access similar to e.g. [NCDatasets.jl](https://github.com/Alexander-Barth/NCDatasets.jl) and [GRIBDatasets.jl](https://github.com/JuliaGeo/GRIBDatasets.jl).
 
 Only a subset of the METOP native formats are supported currently but we are contiguously adding formats. The goal is to support all publicly available native METOP products.
 

docs/make.jl

@@ -1,6 +1,12 @@
 # Copyright (c) 2024 EUMETSAT
 # License: MIT
 
+#### Code to build docs manually from terminal
+# julia --project=docs
+# import Pkg
+# Pkg.resolve()
+# include("docs/make.jl")'
+
 using MetopDatasets
 using Documenter
 

docs/src/public_api.md

@@ -7,11 +7,3 @@ This is the main interface.
 ```@docs
 MetopDataset
 ```
-
-# MetopProduct
-
-This interface might get removed in future versions.
-
-```@docs
-MetopProduct
-```

src/MetopDatasets.jl

@@ -9,21 +9,24 @@ import CSV
 import Dates: DateTime
 import Base: size, keys, close, getindex
 import DiskArrays
+using Compat: @compat
 
 include("abstractTypes/abstract_types.jl")
 include("genericTypes/generic_types.jl")
 include("genericFunctions/generic_functions.jl")
 include("auto_generate_tools/auto_generate_tool.jl")
 include("MetopDiskArray/MetopDiskArray.jl")
 include("InterfaceDataModel/InterfaceDataModel.jl")
-include("metop_product.jl")
 
 # Instruments 
 include("Instruments/ASCAT/ASCAT.jl")
 include("Instruments/IASI/IASI.jl")
 
 const RECORD_DIM_NAME = "atrack"
 
-export MetopProduct, MetopDataset
+export MetopDataset
+
+# public functions
+@compat public read_first_record, scale_iasi_spectrum, max_giadr_channel
 
 end

src/genericFunctions/generic_functions.jl

@@ -6,3 +6,4 @@ include("native_read.jl")
 include("format_information.jl")
 include("get_record_class.jl")
 include("read_first_record.jl")
+include("helper_functions.jl")

src/genericFunctions/helper_functions.jl

@@ -0,0 +1,32 @@
+# Copyright (c) 2024 EUMETSAT
+# License: MIT
+
+function _skip_sphr(file_pointer, n_headers)
+    for _ in 1:n_headers
+        record_header = native_read(file_pointer, RecordHeader)
+        @assert record_header.record_class == get_record_class(SecondaryProductHeader)
+        content_size = record_header.record_size - native_sizeof(RecordHeader)
+        skip(file_pointer, content_size)
+    end
+    return nothing
+end
+
+function _read_record_chunks(file_pointer::IO, main_product_header::MainProductHeader)
+    record_type = data_record_type(main_product_header)
+
+    # read internal pointer records
+    internal_pointer_records = Vector{InternalPointerRecord}(undef,
+        main_product_header.total_ipr)
+    for i in eachindex(internal_pointer_records)
+        internal_pointer_records[i] = native_read(file_pointer, InternalPointerRecord)
+    end
+
+    # get record chunks
+    total_file_size = main_product_header.actual_product_size
+
+    record_chunks = get_data_record_chunks(internal_pointer_records,
+        total_file_size,
+        record_type)
+
+    return record_chunks, internal_pointer_records
+end

src/genericTypes/main_product_header.jl

@@ -111,6 +111,15 @@ function native_read(io::IO, T::Type{MainProductHeader})::MainProductHeader
 
     #extract the values as string
     main_header_content = String(ntoh.(main_header_content))
+
+    # Check for corrupted CRLF header
+    if occursin('\r',main_header_content)
+        msg = raw"Corrupted file with \r in main product header."
+        msg *= "\n This issue often occurs when transferring native binary files via FTP using ASCII mode."
+        msg *= "\n The native binary files should always be transferred in binary mode."
+        error(msg)
+    end
+
     main_header_content = split(main_header_content, "\n")
     filter!(x -> !isempty(x), main_header_content)
     string_values = [strip(split(row, '=')[2]) for row in main_header_content]

test/ASCAT.jl

@@ -102,90 +102,77 @@ if isdir("testData") #TODO test data should be handle as an artifact or somethin
 end
 
 if isdir("testData") #TODO test data should be handle as an artifact or something similar.
-    @testset "Read ASCAT as MetopProduct" begin
-        @testset "SZF with dummy record" begin
-            product = MetopProduct(SZF_with_dummy_in_mid)
-
-            # check the number of records
-            @test product.main_product_header.total_mdr ==
-                  (length(product.data_records) + length(product.dummy_records))
-
-            # check that longitude and latitude are in the correct range
-            longitude = [rec.longitude_full for rec in product.data_records] ./
-                        10^MetopDatasets.get_scale_factor(eltype(product.data_records),
-                :longitude_full)
-            latitude = [rec.latitude_full for rec in product.data_records] ./
-                       10^MetopDatasets.get_scale_factor(eltype(product.data_records),
-                :latitude_full)
-
-            @test all([all((0 .<= arr) .& (arr .<= 360)) for arr in longitude])
-            @test all([all((-90 .<= arr) .& (arr .<= 90)) for arr in latitude])
-
-            # test time stamps
-            time_stamp = DateTime.([rec.utc_localisation for rec in product.data_records])
-            @test round(product.main_product_header.sensing_start, Second) ==
-                  round(minimum(time_stamp), Second)
-            @test round(product.main_product_header.sensing_end, Second) ==
-                  round(maximum(time_stamp), Second)
-        end
-
-        @testset "SZR no dummy record" begin
-            product = MetopProduct(SZR_V13_test_file)
-
-            # check the number of records
-            @test product.main_product_header.total_mdr == length(product.data_records)
-            @test length(product.dummy_records) == 0
-
-            # check that longitude and latitude are in the correct range
-            longitude = [rec.longitude for rec in product.data_records] ./
-                        10^MetopDatasets.get_scale_factor(eltype(product.data_records),
-                :longitude)
-            latitude = [rec.latitude for rec in product.data_records] ./
-                       10^MetopDatasets.get_scale_factor(eltype(product.data_records),
-                :latitude)
-
-            @test all([all((0 .<= arr) .& (arr .<= 360)) for arr in longitude])
-            @test all([all((-90 .<= arr) .& (arr .<= 90)) for arr in latitude])
-
-            # test time stamps
-            time_stamp = DateTime.([rec.utc_line_nodes for rec in product.data_records])
-            @test round(product.main_product_header.sensing_start, Second) ==
-                  round(minimum(time_stamp), Second)
-            @test round(product.main_product_header.sensing_end, Second) ==
-                  round(maximum(time_stamp), Second)
-        end
-
-        @testset "SZO " begin
-            product = MetopProduct(SZO_V13_test_file)
-
-            # check the number of records
-            @test product.main_product_header.total_mdr == length(product.data_records)
-            @test length(product.dummy_records) == 0
-        end
-
-        @testset "SZF v11" begin
-            product = MetopProduct(SZF_V11_test_file)
-
-            # Check read static array of RecordSubType
-            record1 = product.data_records[1]
-            times = DateTime.(record1.utc_localisation)
-            @test all(round.(times, Minute) .== DateTime("2011-12-07T03:24"))
-        end
-
-        @testset "SMR v13" begin
-            product = MetopProduct(SMR_V12_test_file)
-
-            # Check read static array of RecordSubType
-            record1 = product.data_records[1]
-            @test DateTime(record1.utc_line_nodes) == DateTime("2023-12-18T10:12")
-        end
-
-        @testset "SMO v13" begin
-            product = MetopProduct(SMO_V12_test_file)
-
-            # Check read static array of RecordSubType
-            record1 = product.data_records[1]
-            @test DateTime(record1.utc_line_nodes) == DateTime("2023-12-18T10:12")
-        end
+    @testset "SZF with dummy record" begin
+        ds = MetopDataset(SZF_with_dummy_in_mid)
+
+        # check the number of records
+        total_count = parse(Int,ds.attrib["total_mdr"])
+        data_count = ds.dim[MetopDatasets.RECORD_DIM_NAME]
+        dummy_count = 3 # the product have 3 dummy records
+        @test total_count ==
+            (data_count + dummy_count)
+
+        # check that longitude and latitude are in the correct range
+        longitude = Array(ds["longitude_full"])
+        latitude = Array(ds["latitude_full"])
+
+        @test all((0 .<= longitude) .& (longitude .<= 360))
+        @test all((-90 .<= latitude) .& (latitude .<= 90))
+
+        # test time stamps
+        @test abs(DateTime(ds.attrib["sensing_start"]) - ds["utc_localisation"][1]) < Second(2)
+        @test abs(DateTime(ds.attrib["sensing_end"]) - ds["utc_localisation"][end]) < Second(2)
+    end
+
+    @testset "SZR no dummy record" begin
+        ds = MetopDataset(SZR_V13_test_file)
+
+        # check the number of records
+        total_count = parse(Int,ds.attrib["total_mdr"])
+        data_count = ds.dim[MetopDatasets.RECORD_DIM_NAME]
+        @test total_count == data_count
+
+        # check that longitude and latitude are in the correct range
+        longitude = Array(ds["longitude"])
+        latitude = Array(ds["latitude"])
+
+        @test all((0 .<= longitude) .& (longitude .<= 360))
+        @test all((-90 .<= latitude) .& (latitude .<= 90))
+
+        # test time stamps
+        @test abs(DateTime(ds.attrib["sensing_start"]) - ds["utc_line_nodes"][1]) < Second(2)
+        @test abs(DateTime(ds.attrib["sensing_end"]) - ds["utc_line_nodes"][end]) < Second(2)
+    end
+
+    @testset "SZO " begin
+        ds= MetopDataset(SZO_V13_test_file)
+
+        # check times to sample start and end of file.
+        @test abs(DateTime(ds.attrib["sensing_start"]) - ds["utc_line_nodes"][1]) < Second(2)
+        @test abs(DateTime(ds.attrib["sensing_end"]) - ds["utc_line_nodes"][end]) < Second(2)
+    end
+
+    @testset "SZF v11" begin
+        ds= MetopDataset(SZF_V11_test_file)
+
+        # check times to sample start and end of file.
+        @test abs(DateTime(ds.attrib["sensing_start"]) - ds["utc_localisation"][1]) < Second(2)
+        @test abs(DateTime(ds.attrib["sensing_end"]) - ds["utc_localisation"][end]) < Second(2)
+    end
+
+    @testset "SMR v13" begin
+        ds= MetopDataset(SMR_V12_test_file)
+
+        # check times to sample start and end of file.
+        @test abs(DateTime(ds.attrib["sensing_start"]) - ds["utc_line_nodes"][1]) < Second(2)
+        @test abs(DateTime(ds.attrib["sensing_end"]) - ds["utc_line_nodes"][end]) < Second(2)
+    end
+
+    @testset "SMO v13" begin
+        ds= MetopDataset(SMO_V12_test_file)
+
+        # check times to sample start and end of file.
+        @test abs(DateTime(ds.attrib["sensing_start"]) - ds["utc_line_nodes"][1]) < Second(2)
+        @test abs(DateTime(ds.attrib["sensing_end"]) - ds["utc_line_nodes"][end]) < Second(2)
     end
 end