diff --git a/dev/.documenter-siteinfo.json b/dev/.documenter-siteinfo.json index 4104e82..8c1d022 100644 --- a/dev/.documenter-siteinfo.json +++ b/dev/.documenter-siteinfo.json @@ -1 +1 @@ -{"documenter":{"julia_version":"1.11.1","generation_timestamp":"2024-11-15T10:12:04","documenter_version":"1.8.0"}} \ No newline at end of file +{"documenter":{"julia_version":"1.11.2","generation_timestamp":"2024-12-16T15:58:01","documenter_version":"1.8.0"}} \ No newline at end of file diff --git a/dev/index.html b/dev/index.html index 4a36b8f..bce4999 100644 --- a/dev/index.html +++ b/dev/index.html @@ -1,2 +1,2 @@ -Introduction · MetopDatasets.jl
GitHub
+Introduction · MetopDatasets.jl
GitHub
diff --git a/dev/internal_api/index.html b/dev/internal_api/index.html index ba6bd00..f0af724 100644 --- a/dev/internal_api/index.html +++ b/dev/internal_api/index.html @@ -2,27 +2,27 @@ Internal API · MetopDatasets.jl
GitHub

Internal functions and types

MetopDatasets.data_record_type — Method
data_record_type(header::MainProductHeader)::Type

Get the type of data record based on the main product header

Example

julia> file_pointer = open("ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z")
 julia> main_header = MetopDatasets.native_read(file_pointer, MainProductHeader)
 julia> data_record_type(main_header)
-ASCA_SZO_1B_V13
source
MetopDatasets.get_data_record_chunks — Method
get_data_record_chunks(internal_pointer_records::Vector{InternalPointerRecord},
-    total_file_size::Integer, record_type::Type{<:DataRecord})::Vector{RecordChunk}

Compute the record_chunks

source
MetopDatasets.get_description — Method
get_description(T::Type{<:BinaryRecord}, field::Symbol)::AbstractString

Get the description for a given field in the BinaryRecord

Example

julia> get_description(ASCA_SZR_1B_V13, :sigma0_trip)
-"Sigma0 triplet, re-sampled to swath grid, for 3 beams (fore, mid, aft) "
source
MetopDatasets.get_dimensions — Method
get_dimensions(T::Type{<:BinaryRecord})::Dict{String, <:Integer}

Get the the named dimensions in a BinaryRecord and their length.

Example

julia> get_dimensions(ASCA_SZR_1B_V13)
+ASCA_SZO_1B_V13
source
MetopDatasets.get_data_record_chunks — Method
get_data_record_chunks(internal_pointer_records::Vector{InternalPointerRecord},
+    total_file_size::Integer, record_type::Type{<:DataRecord})::Vector{RecordChunk}

Compute the record_chunks

source
MetopDatasets.get_description — Method
get_description(T::Type{<:BinaryRecord}, field::Symbol)::AbstractString

Get the description for a given field in the BinaryRecord

Example

julia> get_description(ASCA_SZR_1B_V13, :sigma0_trip)
+"Sigma0 triplet, re-sampled to swath grid, for 3 beams (fore, mid, aft) "
source
MetopDatasets.get_dimensions — Method
get_dimensions(T::Type{<:BinaryRecord})::Dict{String, <:Integer}

Get the the named dimensions in a BinaryRecord and their length.

Example

julia> get_dimensions(ASCA_SZR_1B_V13)
 Dict{String, Int64} with 2 entries:
   "num_band" => 3
-  "xtrack"   => 82
source
MetopDatasets.get_field_dimensions — Method
get_field_dimensions(T::Type{<:BinaryRecord}, field::Symbol)::Vector{<:AbstractString}

Get the named dimensions of a field in a BinaryRecord

Example

julia> get_field_dimensions(ASCA_SZR_1B_V13, :sigma0_trip)
-["num_band", "xtrack"]
source
MetopDatasets.get_raw_format_dim — Method
get_raw_format_dim(T::Type{<:BinaryRecord}, field::Symbol)::NTuple{4, Int64}

Get the dimensions of the field as defined in the record format specification.

Example

julia> get_raw_format_dim(ASCA_SZR_1B_V13, :sigma0_trip)
-(3, 82, 1, 1)
source
MetopDatasets.get_scale_factor — Method
get_scale_factor(T::Type{<:BinaryRecord}, field::Symbol)::Union{Number,Nothing}

get the scale_factor for a given field in the BinaryRecord. The variable can late be scaled from integer to float by dividing with 10^scale_factor. Returns nothing if no scale factor is set.

Example

julia> get_scale_factor(ASCA_SZR_1B_V13, :sigma0_trip) 
-6
source
MetopDatasets.native_read — Method
native_read(io::IO, T)::T

Read a single object of type T from io to a file in the native Metop format. Endianness is automatically converted.

Example

julia> file_pointer = open("ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z")
+  "xtrack"   => 82
source
MetopDatasets.get_field_dimensions — Method
get_field_dimensions(T::Type{<:BinaryRecord}, field::Symbol)::Vector{<:AbstractString}

Get the named dimensions of a field in a BinaryRecord

Example

julia> get_field_dimensions(ASCA_SZR_1B_V13, :sigma0_trip)
+["num_band", "xtrack"]
source
MetopDatasets.get_raw_format_dim — Method
get_raw_format_dim(T::Type{<:BinaryRecord}, field::Symbol)::NTuple{4, Int64}

Get the dimensions of the field as defined in the record format specification.

Example

julia> get_raw_format_dim(ASCA_SZR_1B_V13, :sigma0_trip)
+(3, 82, 1, 1)
source
MetopDatasets.get_scale_factor — Method
get_scale_factor(T::Type{<:BinaryRecord}, field::Symbol)::Union{Number,Nothing}

get the scale_factor for a given field in the BinaryRecord. The variable can late be scaled from integer to float by dividing with 10^scale_factor. Returns nothing if no scale factor is set.

Example

julia> get_scale_factor(ASCA_SZR_1B_V13, :sigma0_trip) 
+6
source
MetopDatasets.native_read — Method
native_read(io::IO, T)::T

Read a single object of type T from io to a file in the native Metop format. Endianness is automatically converted.

Example

julia> file_pointer = open("ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z")
 julia> main_header = MetopDatasets.native_read(file_pointer, MainProductHeader)
 julia> main_header.sensing_start
-2023-03-29T06:33:00
source
MetopDatasets.native_sizeof — Method
native_sizeof(x)::Integer

The byte size of the type x in a METOP native product.

Example

julia> native_sizeof(RecordHeader)
-20
source
MetopDatasets.read_first_record — Method
read_first_record(ds::MetopDataset, record_type::Type{<:Record})

Read the first record of type record_type from the dataset. This can be used to access records that are not directly exposed through the MetopDataset interface.

source
MetopDatasets.record_struct_expression — Method
record_struct_expression(file_name, record_type)

Function to autogenerate Struct code based on a CSV file. Also autogenerates get_description and get_scale_factor method for Struct. Use it together with eval.

Example

julia> eval(record_struct_expression(joinpath(@__DIR__, "TEST_FORmaT.csv"), DataRecord))
+2023-03-29T06:33:00
source
MetopDatasets.native_sizeof — Method
native_sizeof(x)::Integer

The byte size of the type x in a METOP native product.

Example

julia> native_sizeof(RecordHeader)
+20
source
MetopDatasets.read_first_record — Method
read_first_record(ds::MetopDataset, record_type::Type{<:Record})

Read the first record of type record_type from the dataset. This can be used to access records that are not directly exposed through the MetopDataset interface.

source
MetopDatasets.record_struct_expression — Method
record_struct_expression(file_name, record_type)

Function to autogenerate Struct code based on a CSV file. Also autogenerates get_description and get_scale_factor method for Struct. Use it together with eval.

Example

julia> eval(record_struct_expression(joinpath(@__DIR__, "TEST_FORmaT.csv"), DataRecord))
 julia> TEST_FORMAT <: DataRecord
-true
source
MetopDatasets.scale_iasi_spectrum — Method
scale_iasi_spectrum(spec_raw, giadr::GIADR_IASI_XXX_1C; high_precision = false)
 scale_iasi_spectrum(spec_raw, giadr::GIADR_IASI_XXX_1C, channel_range::OrdinalRange; high_precision = false)

Scaling the IASI L1C spectrum using the giadr record information. The channel_range is needed if only a subset of the raw spectrum is passed to the function. Setting high_precision=true will convert to Float64 instead of Float32. Note that the end part of the ds["gs1cspect"] does not have any scale factors. Here the spectrum is just filled with 0.0.

Example

julia> file_path = "test/testData/IASI_xxx_1C_M01_20240819103856Z_20240819104152Z_N_C_20240819112911Z"
 julia> ds = MetopDataset(file_path, auto_convert = false);
 julia> giadr = MetopDatasets.read_first_record(ds, MetopDatasets.GIADR_IASI_XXX_1C_V11)
 julia> # Scale full spectrum.
 julia> scaled_spectrum = MetopDatasets.scale_iasi_spectrum(ds["gs1cspect"], giadr)
 julia> # Scale subset of spectrum.
-julia> scaled_spectrum_subset = MetopDatasets.scale_iasi_spectrum(ds["gs1cspect"][10:20,:,:,:], giadr, 10:20)
source
MetopDatasets.AbstractMetopDiskArray — Type
AbstractMetopDiskArray{T, N} <: DiskArrays.AbstractDiskArray{T, N}

In most cases MetopDiskArray is used but AbstractMetopDiskArray allows defining additional DiskArray types to handle special corner cases.

source
MetopDatasets.DummyRecord — Type
DummyRecord <:Record

The Dummy Measurement Data Record is a special case of the MDR. It is a generic record that is used to indicate the location of lost data within any product. One DMDR can replace a contiguous block of lost MDRs

source
MetopDatasets.IasiSpectrumDiskArray — Type
IasiSpectrumDiskArray{T} <: AbstractMetopDiskArray{T, 4}

The IasiSpectrumDiskArray is a wrapper around a MetopDiskArray that enables the automatic scaling of the the IASI L1C spectrum using the GIADR_IASI_XXX_1C record information.

source
MetopDatasets.IasiWaveNumberDiskArray — Type
IasiWaveNumberDiskArray <: AbstractMetopDiskArray{Float64, 2}

The IasiWaveNumberDiskArray is a disk array that computes the wave number of the IASI spectrum. The wave number is computed using :idefnsfirst1b and :idefspectdwn1b from each data record.

source
MetopDatasets.InternalPointerRecord — Type
InternalPointerRecord <: Record

The Internal Pointer Records (IPR) specifies the start of each block of records in the file sharing the same record type. This can be used to find the locations of data records or dummy records.

source
MetopDatasets.MetopDiskArray — Type
MetopDiskArray{T, N} <: AbstractMetopDiskArray{T, N}

Struct to handle lazy loading of a variable in a Metop product. The raw types in the product is mapped without any scaling. Auto conversion can be enabled for RecordSubType e.g. converting VInteger to Float64.

source
MetopDatasets.MetopDiskArray — Method
MetopDiskArray(file_pointer::IOStream,
+julia> scaled_spectrum_subset = MetopDatasets.scale_iasi_spectrum(ds["gs1cspect"][10:20,:,:,:], giadr, 10:20)
source
MetopDatasets.AbstractMetopDiskArray — Type
AbstractMetopDiskArray{T, N} <: DiskArrays.AbstractDiskArray{T, N}

In most cases MetopDiskArray is used but AbstractMetopDiskArray allows defining additional DiskArray types to handle special corner cases.

source
MetopDatasets.DummyRecord — Type
DummyRecord <:Record

The Dummy Measurement Data Record is a special case of the MDR. It is a generic record that is used to indicate the location of lost data within any product. One DMDR can replace a contiguous block of lost MDRs

source
MetopDatasets.IasiSpectrumDiskArray — Type
IasiSpectrumDiskArray{T} <: AbstractMetopDiskArray{T, 4}

The IasiSpectrumDiskArray is a wrapper around a MetopDiskArray that enables the automatic scaling of the the IASI L1C spectrum using the GIADR_IASI_XXX_1C record information.

source
MetopDatasets.IasiWaveNumberDiskArray — Type
IasiWaveNumberDiskArray <: AbstractMetopDiskArray{Float64, 2}

The IasiWaveNumberDiskArray is a disk array that computes the wave number of the IASI spectrum. The wave number is computed using :idefnsfirst1b and :idefspectdwn1b from each data record.

source
MetopDatasets.InternalPointerRecord — Type
InternalPointerRecord <: Record

The Internal Pointer Records (IPR) specifies the start of each block of records in the file sharing the same record type. This can be used to find the locations of data records or dummy records.

source
MetopDatasets.MetopDiskArray — Type
MetopDiskArray{T, N} <: AbstractMetopDiskArray{T, N}

Struct to handle lazy loading of a variable in a Metop product. The raw types in the product is mapped without any scaling. Auto conversion can be enabled for RecordSubType e.g. converting VInteger to Float64.

source
MetopDatasets.MetopDiskArray — Method
MetopDiskArray(file_pointer::IOStream,
     record_chunks::Vector{RecordChunk},
-    field_name::Symbol; auto_convert = true) -> MetopDiskArray

Constructor for MetopDiskArray that compute additional fields. auto_convert = true will automatically convert custom RecordSubType to commonly used data types e.g. converting VInteger to Float64.

source
MetopDatasets.MetopVariable — Type
MetopVariable{T, N, R <: DataRecord, F} <: CommonDataModel.AbstractVariable{T, N}

MetopVariable wraps AbstractMetopDiskArray so it can be used with MetopDataset.

source
+ field_name::Symbol; auto_convert = true) -> MetopDiskArray

Constructor for MetopDiskArray that compute additional fields. auto_convert = true will automatically convert custom RecordSubType to commonly used data types e.g. converting VInteger to Float64.

source
MetopDatasets.MetopVariable — Type
MetopVariable{T, N, R <: DataRecord, F} <: CommonDataModel.AbstractVariable{T, N}

MetopVariable wraps AbstractMetopDiskArray so it can be used with MetopDataset.

source
MetopDatasets.RecordChunk — Type
RecordChunk

Used to store the locations of different chunks of records in Native metop files.

source
MetopDatasets.RecordHeader — Type
RecordHeader

Also known as GRH.

source
diff --git a/dev/objects.inv b/dev/objects.inv index ed33797..d67bbe8 100644 Binary files a/dev/objects.inv and b/dev/objects.inv differ diff --git a/dev/public_api/index.html b/dev/public_api/index.html index 28853f8..208ff26 100644 --- a/dev/public_api/index.html +++ b/dev/public_api/index.html @@ -1,5 +1,5 @@ -Public API · MetopDatasets.jl
GitHub

MetopDataset

This is the main interface.

MetopDatasets.MetopDataset — Type
MetopDataset(file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)
+Public API · MetopDatasets.jl
GitHub
MetopDataset
This is the main interface.
MetopDatasets.MetopDataset — Type
MetopDataset(file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)
 MetopDataset(file_pointer::IO; auto_convert::Bool = true, high_precision::Bool=false)
 MetopDataset(f::Function, file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)
Load a MetopDataset from a Metop Native binary file or from a IO to a Native binary file. Only the meta data is loaded upon creation and all variables are lazy loaded.  The variables corresponds to the different fields of the data records in the file. The attributes have all the information from the main product header in the file.
auto_convert=true will automatically convert MetopDatasets specific types such as VInteger to common netCDF complaint types such as Float64. This will also automatically scale variable where the  scaling can't be expressed through a simple scale factor e.g. the IASI spectrum where different bands of  the spectrum have different scaling factors.
Selected fields are converted to Float32 to save memory. Normally Float32 is more than sufficient to represent the instrument accuracy.  Setting high_precision=true will in some case convert these variables to Float64. 
Example
julia> file_path = "test/testData/ASCA_SZR_1B_M03_20230329063300Z_20230329063558Z_N_C_20230329081417Z"
 julia> ds = MetopDataset(file_path);
@@ -22,8 +22,4 @@
 julia> lat = ds["latitude"][:,:]
 julia>
 julia> # close data set
-julia> close(ds);
source
MetopProduct
This interface might get removed in future versions.
MetopDatasets.MetopProduct — Type
MetopProduct{T <: DataRecord} 
-    main_product_header::MainProductHeader
-    internal_pointer_records::Vector{InternalPointerRecord}
-    data_records::Vector{T}
-    dummy_records::Vector{DummyRecord}
Generic Metop Product
source

+julia> close(ds);
source
diff --git a/dev/search_index.js b/dev/search_index.js index 7b65d75..e6c2a75 100644 --- a/dev/search_index.js +++ b/dev/search_index.js @@ -1,3 +1,3 @@ var documenterSearchIndex = {"docs": -[{"location":"public_api/#MetopDataset","page":"Public API","title":"MetopDataset","text":"","category":"section"},{"location":"public_api/","page":"Public API","title":"Public API","text":"This is the main interface.","category":"page"},{"location":"public_api/","page":"Public API","title":"Public API","text":"MetopDataset","category":"page"},{"location":"public_api/#MetopDatasets.MetopDataset","page":"Public API","title":"MetopDatasets.MetopDataset","text":"MetopDataset(file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)\nMetopDataset(file_pointer::IO; auto_convert::Bool = true, high_precision::Bool=false)\nMetopDataset(f::Function, file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)\n\nLoad a MetopDataset from a Metop Native binary file or from a IO to a Native binary file. Only the meta data is loaded upon creation and all variables are lazy loaded. The variables corresponds to the different fields of the data records in the file. The attributes have all the information from the main product header in the file.\n\nauto_convert=true will automatically convert MetopDatasets specific types such as VInteger to common netCDF complaint types such as Float64. This will also automatically scale variable where the scaling can't be expressed through a simple scale factor e.g. the IASI spectrum where different bands of the spectrum have different scaling factors.\n\nSelected fields are converted to Float32 to save memory. Normally Float32 is more than sufficient to represent the instrument accuracy. Setting high_precision=true will in some case convert these variables to Float64. \n\nExample\n\njulia> file_path = \"test/testData/ASCA_SZR_1B_M03_20230329063300Z_20230329063558Z_N_C_20230329081417Z\"\njulia> ds = MetopDataset(file_path);\njulia>\njulia> # display metadata of a variable\njulia> ds[\"latitude\"]\nlatitude (82 × 96)\n Datatype: Float64 (Int32)\n Dimensions: xtrack × record\n Attributes:\n description = Latitude (-90 to 90 deg)\njulia>\njulia> # load a subset of a variable \njulia> lat_subset = ds[\"latitude\"][1:2,1:3] # load a small subset of latitudes.\n2×3 Matrix{Float64}:\n -33.7308 -33.8399 -33.949\n -33.7139 -33.823 -33.9322\njulia>\njulia> # load entire variable \njulia> lat = ds[\"latitude\"][:,:]\njulia>\njulia> # close data set\njulia> close(ds);\n\n\n\n\n\n","category":"type"},{"location":"public_api/#MetopProduct","page":"Public API","title":"MetopProduct","text":"","category":"section"},{"location":"public_api/","page":"Public API","title":"Public API","text":"This interface might get removed in future versions.","category":"page"},{"location":"public_api/","page":"Public API","title":"Public API","text":"MetopProduct","category":"page"},{"location":"public_api/#MetopDatasets.MetopProduct","page":"Public API","title":"MetopDatasets.MetopProduct","text":"MetopProduct{T <: DataRecord} \n main_product_header::MainProductHeader\n internal_pointer_records::Vector{InternalPointerRecord}\n data_records::Vector{T}\n dummy_records::Vector{DummyRecord}\n\nGeneric Metop Product\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#Internal-functions-and-types","page":"Internal API","title":"Internal functions and types","text":"","category":"section"},{"location":"internal_api/","page":"Internal API","title":"Internal API","text":"Modules = [MetopDatasets]\nPublic = false\nOrder = [:function, :type, :macro]","category":"page"},{"location":"internal_api/#MetopDatasets.data_record_type-Tuple{MetopDatasets.MainProductHeader}","page":"Internal API","title":"MetopDatasets.data_record_type","text":"data_record_type(header::MainProductHeader)::Type\n\nGet the type of data record based on the main product header\n\nExample\n\njulia> file_pointer = open(\"ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z\")\njulia> main_header = MetopDatasets.native_read(file_pointer, MainProductHeader)\njulia> data_record_type(main_header)\nASCA_SZO_1B_V13\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_data_record_chunks-Tuple{Vector{MetopDatasets.InternalPointerRecord}, Integer, Type{<:MetopDatasets.DataRecord}}","page":"Internal API","title":"MetopDatasets.get_data_record_chunks","text":"get_data_record_chunks(internal_pointer_records::Vector{InternalPointerRecord},\n total_file_size::Integer, record_type::Type{<:DataRecord})::Vector{RecordChunk}\n\nCompute the record_chunks\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_description-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_description","text":"get_description(T::Type{<:BinaryRecord}, field::Symbol)::AbstractString\n\nGet the description for a given field in the BinaryRecord\n\nExample\n\njulia> get_description(ASCA_SZR_1B_V13, :sigma0_trip)\n\"Sigma0 triplet, re-sampled to swath grid, for 3 beams (fore, mid, aft) \"\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_dimensions-Tuple{Type{<:MetopDatasets.BinaryRecord}}","page":"Internal API","title":"MetopDatasets.get_dimensions","text":"get_dimensions(T::Type{<:BinaryRecord})::Dict{String, <:Integer}\n\nGet the the named dimensions in a BinaryRecord and their length.\n\nExample\n\njulia> get_dimensions(ASCA_SZR_1B_V13)\nDict{String, Int64} with 2 entries:\n \"num_band\" => 3\n \"xtrack\" => 82\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_field_dimensions-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_field_dimensions","text":"get_field_dimensions(T::Type{<:BinaryRecord}, field::Symbol)::Vector{<:AbstractString}\n\nGet the named dimensions of a field in a BinaryRecord\n\nExample\n\njulia> get_field_dimensions(ASCA_SZR_1B_V13, :sigma0_trip)\n[\"num_band\", \"xtrack\"]\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_raw_format_dim-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_raw_format_dim","text":"get_raw_format_dim(T::Type{<:BinaryRecord}, field::Symbol)::NTuple{4, Int64}\n\nGet the dimensions of the field as defined in the record format specification.\n\nExample\n\njulia> get_raw_format_dim(ASCA_SZR_1B_V13, :sigma0_trip)\n(3, 82, 1, 1)\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_scale_factor-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_scale_factor","text":"get_scale_factor(T::Type{<:BinaryRecord}, field::Symbol)::Union{Number,Nothing}\n\nget the scale_factor for a given field in the BinaryRecord. The variable can late be scaled from integer to float by dividing with 10^scale_factor. Returns nothing if no scale factor is set. \n\nExample\n\njulia> get_scale_factor(ASCA_SZR_1B_V13, :sigma0_trip) \n6\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.native_read-Tuple{IO, Type}","page":"Internal API","title":"MetopDatasets.native_read","text":"native_read(io::IO, T)::T\n\nRead a single object of type T from io to a file in the native Metop format. Endianness is automatically converted.\n\nExample\n\njulia> file_pointer = open(\"ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z\")\njulia> main_header = MetopDatasets.native_read(file_pointer, MainProductHeader)\njulia> main_header.sensing_start\n2023-03-29T06:33:00\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.native_sizeof-Tuple{Type{<:Tuple}}","page":"Internal API","title":"MetopDatasets.native_sizeof","text":"native_sizeof(x)::Integer\n\nThe byte size of the type x in a METOP native product.\n\nExample\n\njulia> native_sizeof(RecordHeader)\n20\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.read_first_record-Tuple{MetopDataset, Type{<:MetopDatasets.Record}}","page":"Internal API","title":"MetopDatasets.read_first_record","text":"read_first_record(ds::MetopDataset, record_type::Type{<:Record})\n\nRead the first record of type record_type from the dataset. This can be used to access records that are not directly exposed through the MetopDataset interface.\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.record_struct_expression-Tuple{Any, Any}","page":"Internal API","title":"MetopDatasets.record_struct_expression","text":"record_struct_expression(file_name, record_type)\n\nFunction to autogenerate Struct code based on a CSV file. Also autogenerates get_description and get_scale_factor method for Struct. Use it together with eval.\n\nExample\n\njulia> eval(record_struct_expression(joinpath(@__DIR__, \"TEST_FORmaT.csv\"), DataRecord))\njulia> TEST_FORMAT <: DataRecord\ntrue\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.scale_iasi_spectrum-Tuple{Any, MetopDatasets.GIADR_IASI_XXX_1C, OrdinalRange}","page":"Internal API","title":"MetopDatasets.scale_iasi_spectrum","text":"scale_iasi_spectrum(spec_raw, giadr::GIADR_IASI_XXX_1C; high_precision = false)\nscale_iasi_spectrum(spec_raw, giadr::GIADR_IASI_XXX_1C, channel_range::OrdinalRange; high_precision = false)\n\nScaling the IASI L1C spectrum using the giadr record information. The channel_range is needed if only a subset of the raw spectrum is passed to the function. Setting high_precision=true will convert to Float64 instead of Float32. Note that the end part of the ds[\"gs1cspect\"] does not have any scale factors. Here the spectrum is just filled with 0.0.\n\nExample\n\njulia> file_path = \"test/testData/IASI_xxx_1C_M01_20240819103856Z_20240819104152Z_N_C_20240819112911Z\"\njulia> ds = MetopDataset(file_path, auto_convert = false);\njulia> giadr = MetopDatasets.read_first_record(ds, MetopDatasets.GIADR_IASI_XXX_1C_V11)\njulia> # Scale full spectrum.\njulia> scaled_spectrum = MetopDatasets.scale_iasi_spectrum(ds[\"gs1cspect\"], giadr)\njulia> # Scale subset of spectrum.\njulia> scaled_spectrum_subset = MetopDatasets.scale_iasi_spectrum(ds[\"gs1cspect\"][10:20,:,:,:], giadr, 10:20)\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.AbstractMetopDiskArray","page":"Internal API","title":"MetopDatasets.AbstractMetopDiskArray","text":"AbstractMetopDiskArray{T, N} <: DiskArrays.AbstractDiskArray{T, N}\n\nIn most cases MetopDiskArray is used but AbstractMetopDiskArray allows defining additional DiskArray types to handle special corner cases. \n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.DummyRecord","page":"Internal API","title":"MetopDatasets.DummyRecord","text":"DummyRecord <:Record\n\nThe Dummy Measurement Data Record is a special case of the MDR. It is a generic record that is used to indicate the location of lost data within any product. One DMDR can replace a contiguous block of lost MDRs\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.IasiSpectrumDiskArray","page":"Internal API","title":"MetopDatasets.IasiSpectrumDiskArray","text":"IasiSpectrumDiskArray{T} <: AbstractMetopDiskArray{T, 4}\n\nThe IasiSpectrumDiskArray is a wrapper around a MetopDiskArray that enables the automatic scaling of the the IASI L1C spectrum using the GIADR_IASI_XXX_1C record information.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.IasiWaveNumberDiskArray","page":"Internal API","title":"MetopDatasets.IasiWaveNumberDiskArray","text":"IasiWaveNumberDiskArray <: AbstractMetopDiskArray{Float64, 2}\n\nThe IasiWaveNumberDiskArray is a disk array that computes the wave number of the IASI spectrum. The wave number is computed using :idefnsfirst1b and :idefspectdwn1b from each data record.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.InternalPointerRecord","page":"Internal API","title":"MetopDatasets.InternalPointerRecord","text":"InternalPointerRecord <: Record\n\nThe Internal Pointer Records (IPR) specifies the start of each block of records in the file sharing the same record type. This can be used to find the locations of data records or dummy records.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.MetopDiskArray","page":"Internal API","title":"MetopDatasets.MetopDiskArray","text":"MetopDiskArray{T, N} <: AbstractMetopDiskArray{T, N}\n\nStruct to handle lazy loading of a variable in a Metop product. The raw types in the product is mapped without any scaling. Auto conversion can be enabled for RecordSubType e.g. converting VInteger to Float64.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.MetopDiskArray-Tuple{IOStream, Vector{MetopDatasets.RecordChunk}, Symbol}","page":"Internal API","title":"MetopDatasets.MetopDiskArray","text":"MetopDiskArray(file_pointer::IOStream,\n record_chunks::Vector{RecordChunk},\n field_name::Symbol; auto_convert = true) -> MetopDiskArray\n\nConstructor for MetopDiskArray that compute additional fields. auto_convert = true will automatically convert custom RecordSubType to commonly used data types e.g. converting VInteger to Float64.\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.MetopVariable","page":"Internal API","title":"MetopDatasets.MetopVariable","text":"MetopVariable{T, N, R <: DataRecord, F} <: CommonDataModel.AbstractVariable{T, N}\n\nMetopVariable wraps AbstractMetopDiskArray so it can be used with MetopDataset.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.RecordChunk","page":"Internal API","title":"MetopDatasets.RecordChunk","text":"RecordChunk\n\nUsed to store the locations of different chunks of records in Native metop files.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.RecordHeader","page":"Internal API","title":"MetopDatasets.RecordHeader","text":"RecordHeader\n\nAlso known as GRH.\n\n\n\n\n\n","category":"type"},{"location":"","page":"Introduction","title":"Introduction","text":"CurrentModule = MetopDatasets","category":"page"},{"location":"#MetopDatasets","page":"Introduction","title":"MetopDatasets","text":"","category":"section"},{"location":"","page":"Introduction","title":"Introduction","text":"Documentation for MetopDatasets.","category":"page"},{"location":"#Copyright-and-License","page":"Introduction","title":"Copyright and License","text":"","category":"section"},{"location":"","page":"Introduction","title":"Introduction","text":"This code is licensed under MIT license. See file LICENSE for details on the usage and distribution terms.","category":"page"},{"location":"#More-documentation-is-under-way","page":"Introduction","title":"More documentation is under way","text":"","category":"section"}] +[{"location":"public_api/#MetopDataset","page":"Public API","title":"MetopDataset","text":"","category":"section"},{"location":"public_api/","page":"Public API","title":"Public API","text":"This is the main interface.","category":"page"},{"location":"public_api/","page":"Public API","title":"Public API","text":"MetopDataset","category":"page"},{"location":"public_api/#MetopDatasets.MetopDataset","page":"Public API","title":"MetopDatasets.MetopDataset","text":"MetopDataset(file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)\nMetopDataset(file_pointer::IO; auto_convert::Bool = true, high_precision::Bool=false)\nMetopDataset(f::Function, file_path::AbstractString; auto_convert::Bool = true, high_precision::Bool=false)\n\nLoad a MetopDataset from a Metop Native binary file or from a IO to a Native binary file. Only the meta data is loaded upon creation and all variables are lazy loaded. The variables corresponds to the different fields of the data records in the file. The attributes have all the information from the main product header in the file.\n\nauto_convert=true will automatically convert MetopDatasets specific types such as VInteger to common netCDF complaint types such as Float64. This will also automatically scale variable where the scaling can't be expressed through a simple scale factor e.g. the IASI spectrum where different bands of the spectrum have different scaling factors.\n\nSelected fields are converted to Float32 to save memory. Normally Float32 is more than sufficient to represent the instrument accuracy. Setting high_precision=true will in some case convert these variables to Float64. \n\nExample\n\njulia> file_path = \"test/testData/ASCA_SZR_1B_M03_20230329063300Z_20230329063558Z_N_C_20230329081417Z\"\njulia> ds = MetopDataset(file_path);\njulia>\njulia> # display metadata of a variable\njulia> ds[\"latitude\"]\nlatitude (82 × 96)\n Datatype: Float64 (Int32)\n Dimensions: xtrack × record\n Attributes:\n description = Latitude (-90 to 90 deg)\njulia>\njulia> # load a subset of a variable \njulia> lat_subset = ds[\"latitude\"][1:2,1:3] # load a small subset of latitudes.\n2×3 Matrix{Float64}:\n -33.7308 -33.8399 -33.949\n -33.7139 -33.823 -33.9322\njulia>\njulia> # load entire variable \njulia> lat = ds[\"latitude\"][:,:]\njulia>\njulia> # close data set\njulia> close(ds);\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#Internal-functions-and-types","page":"Internal API","title":"Internal functions and types","text":"","category":"section"},{"location":"internal_api/","page":"Internal API","title":"Internal API","text":"Modules = [MetopDatasets]\nPublic = false\nOrder = [:function, :type, :macro]","category":"page"},{"location":"internal_api/#MetopDatasets.data_record_type-Tuple{MetopDatasets.MainProductHeader}","page":"Internal API","title":"MetopDatasets.data_record_type","text":"data_record_type(header::MainProductHeader)::Type\n\nGet the type of data record based on the main product header\n\nExample\n\njulia> file_pointer = open(\"ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z\")\njulia> main_header = MetopDatasets.native_read(file_pointer, MainProductHeader)\njulia> data_record_type(main_header)\nASCA_SZO_1B_V13\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_data_record_chunks-Tuple{Vector{MetopDatasets.InternalPointerRecord}, Integer, Type{<:MetopDatasets.DataRecord}}","page":"Internal API","title":"MetopDatasets.get_data_record_chunks","text":"get_data_record_chunks(internal_pointer_records::Vector{InternalPointerRecord},\n total_file_size::Integer, record_type::Type{<:DataRecord})::Vector{RecordChunk}\n\nCompute the record_chunks\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_description-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_description","text":"get_description(T::Type{<:BinaryRecord}, field::Symbol)::AbstractString\n\nGet the description for a given field in the BinaryRecord\n\nExample\n\njulia> get_description(ASCA_SZR_1B_V13, :sigma0_trip)\n\"Sigma0 triplet, re-sampled to swath grid, for 3 beams (fore, mid, aft) \"\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_dimensions-Tuple{Type{<:MetopDatasets.BinaryRecord}}","page":"Internal API","title":"MetopDatasets.get_dimensions","text":"get_dimensions(T::Type{<:BinaryRecord})::Dict{String, <:Integer}\n\nGet the the named dimensions in a BinaryRecord and their length.\n\nExample\n\njulia> get_dimensions(ASCA_SZR_1B_V13)\nDict{String, Int64} with 2 entries:\n \"num_band\" => 3\n \"xtrack\" => 82\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_field_dimensions-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_field_dimensions","text":"get_field_dimensions(T::Type{<:BinaryRecord}, field::Symbol)::Vector{<:AbstractString}\n\nGet the named dimensions of a field in a BinaryRecord\n\nExample\n\njulia> get_field_dimensions(ASCA_SZR_1B_V13, :sigma0_trip)\n[\"num_band\", \"xtrack\"]\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_raw_format_dim-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_raw_format_dim","text":"get_raw_format_dim(T::Type{<:BinaryRecord}, field::Symbol)::NTuple{4, Int64}\n\nGet the dimensions of the field as defined in the record format specification.\n\nExample\n\njulia> get_raw_format_dim(ASCA_SZR_1B_V13, :sigma0_trip)\n(3, 82, 1, 1)\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.get_scale_factor-Tuple{Type{<:MetopDatasets.BinaryRecord}, Symbol}","page":"Internal API","title":"MetopDatasets.get_scale_factor","text":"get_scale_factor(T::Type{<:BinaryRecord}, field::Symbol)::Union{Number,Nothing}\n\nget the scale_factor for a given field in the BinaryRecord. The variable can late be scaled from integer to float by dividing with 10^scale_factor. Returns nothing if no scale factor is set. \n\nExample\n\njulia> get_scale_factor(ASCA_SZR_1B_V13, :sigma0_trip) \n6\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.native_read-Tuple{IO, Type}","page":"Internal API","title":"MetopDatasets.native_read","text":"native_read(io::IO, T)::T\n\nRead a single object of type T from io to a file in the native Metop format. Endianness is automatically converted.\n\nExample\n\njulia> file_pointer = open(\"ASCA_SZO_1B_M03_20230329063300Z_20230329063556Z_N_C_20230329081417Z\")\njulia> main_header = MetopDatasets.native_read(file_pointer, MainProductHeader)\njulia> main_header.sensing_start\n2023-03-29T06:33:00\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.native_sizeof-Tuple{Type{<:Tuple}}","page":"Internal API","title":"MetopDatasets.native_sizeof","text":"native_sizeof(x)::Integer\n\nThe byte size of the type x in a METOP native product.\n\nExample\n\njulia> native_sizeof(RecordHeader)\n20\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.read_first_record-Tuple{MetopDataset, Type{<:MetopDatasets.Record}}","page":"Internal API","title":"MetopDatasets.read_first_record","text":"read_first_record(ds::MetopDataset, record_type::Type{<:Record})\n\nRead the first record of type record_type from the dataset. This can be used to access records that are not directly exposed through the MetopDataset interface.\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.record_struct_expression-Tuple{Any, Any}","page":"Internal API","title":"MetopDatasets.record_struct_expression","text":"record_struct_expression(file_name, record_type)\n\nFunction to autogenerate Struct code based on a CSV file. Also autogenerates get_description and get_scale_factor method for Struct. Use it together with eval.\n\nExample\n\njulia> eval(record_struct_expression(joinpath(@__DIR__, \"TEST_FORmaT.csv\"), DataRecord))\njulia> TEST_FORMAT <: DataRecord\ntrue\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.scale_iasi_spectrum-Tuple{Any, MetopDatasets.GIADR_IASI_XXX_1C, OrdinalRange}","page":"Internal API","title":"MetopDatasets.scale_iasi_spectrum","text":"scale_iasi_spectrum(spec_raw, giadr::GIADR_IASI_XXX_1C; high_precision = false)\nscale_iasi_spectrum(spec_raw, giadr::GIADR_IASI_XXX_1C, channel_range::OrdinalRange; high_precision = false)\n\nScaling the IASI L1C spectrum using the giadr record information. The channel_range is needed if only a subset of the raw spectrum is passed to the function. Setting high_precision=true will convert to Float64 instead of Float32. Note that the end part of the ds[\"gs1cspect\"] does not have any scale factors. Here the spectrum is just filled with 0.0.\n\nExample\n\njulia> file_path = \"test/testData/IASI_xxx_1C_M01_20240819103856Z_20240819104152Z_N_C_20240819112911Z\"\njulia> ds = MetopDataset(file_path, auto_convert = false);\njulia> giadr = MetopDatasets.read_first_record(ds, MetopDatasets.GIADR_IASI_XXX_1C_V11)\njulia> # Scale full spectrum.\njulia> scaled_spectrum = MetopDatasets.scale_iasi_spectrum(ds[\"gs1cspect\"], giadr)\njulia> # Scale subset of spectrum.\njulia> scaled_spectrum_subset = MetopDatasets.scale_iasi_spectrum(ds[\"gs1cspect\"][10:20,:,:,:], giadr, 10:20)\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.AbstractMetopDiskArray","page":"Internal API","title":"MetopDatasets.AbstractMetopDiskArray","text":"AbstractMetopDiskArray{T, N} <: DiskArrays.AbstractDiskArray{T, N}\n\nIn most cases MetopDiskArray is used but AbstractMetopDiskArray allows defining additional DiskArray types to handle special corner cases. \n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.DummyRecord","page":"Internal API","title":"MetopDatasets.DummyRecord","text":"DummyRecord <:Record\n\nThe Dummy Measurement Data Record is a special case of the MDR. It is a generic record that is used to indicate the location of lost data within any product. One DMDR can replace a contiguous block of lost MDRs\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.IasiSpectrumDiskArray","page":"Internal API","title":"MetopDatasets.IasiSpectrumDiskArray","text":"IasiSpectrumDiskArray{T} <: AbstractMetopDiskArray{T, 4}\n\nThe IasiSpectrumDiskArray is a wrapper around a MetopDiskArray that enables the automatic scaling of the the IASI L1C spectrum using the GIADR_IASI_XXX_1C record information.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.IasiWaveNumberDiskArray","page":"Internal API","title":"MetopDatasets.IasiWaveNumberDiskArray","text":"IasiWaveNumberDiskArray <: AbstractMetopDiskArray{Float64, 2}\n\nThe IasiWaveNumberDiskArray is a disk array that computes the wave number of the IASI spectrum. The wave number is computed using :idefnsfirst1b and :idefspectdwn1b from each data record.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.InternalPointerRecord","page":"Internal API","title":"MetopDatasets.InternalPointerRecord","text":"InternalPointerRecord <: Record\n\nThe Internal Pointer Records (IPR) specifies the start of each block of records in the file sharing the same record type. This can be used to find the locations of data records or dummy records.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.MetopDiskArray","page":"Internal API","title":"MetopDatasets.MetopDiskArray","text":"MetopDiskArray{T, N} <: AbstractMetopDiskArray{T, N}\n\nStruct to handle lazy loading of a variable in a Metop product. The raw types in the product is mapped without any scaling. Auto conversion can be enabled for RecordSubType e.g. converting VInteger to Float64.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.MetopDiskArray-Tuple{IOStream, Vector{MetopDatasets.RecordChunk}, Symbol}","page":"Internal API","title":"MetopDatasets.MetopDiskArray","text":"MetopDiskArray(file_pointer::IOStream,\n record_chunks::Vector{RecordChunk},\n field_name::Symbol; auto_convert = true) -> MetopDiskArray\n\nConstructor for MetopDiskArray that compute additional fields. auto_convert = true will automatically convert custom RecordSubType to commonly used data types e.g. converting VInteger to Float64.\n\n\n\n\n\n","category":"method"},{"location":"internal_api/#MetopDatasets.MetopVariable","page":"Internal API","title":"MetopDatasets.MetopVariable","text":"MetopVariable{T, N, R <: DataRecord, F} <: CommonDataModel.AbstractVariable{T, N}\n\nMetopVariable wraps AbstractMetopDiskArray so it can be used with MetopDataset.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.RecordChunk","page":"Internal API","title":"MetopDatasets.RecordChunk","text":"RecordChunk\n\nUsed to store the locations of different chunks of records in Native metop files.\n\n\n\n\n\n","category":"type"},{"location":"internal_api/#MetopDatasets.RecordHeader","page":"Internal API","title":"MetopDatasets.RecordHeader","text":"RecordHeader\n\nAlso known as GRH.\n\n\n\n\n\n","category":"type"},{"location":"","page":"Introduction","title":"Introduction","text":"CurrentModule = MetopDatasets","category":"page"},{"location":"#MetopDatasets","page":"Introduction","title":"MetopDatasets","text":"","category":"section"},{"location":"","page":"Introduction","title":"Introduction","text":"Documentation for MetopDatasets.","category":"page"},{"location":"#Copyright-and-License","page":"Introduction","title":"Copyright and License","text":"","category":"section"},{"location":"","page":"Introduction","title":"Introduction","text":"This code is licensed under MIT license. See file LICENSE for details on the usage and distribution terms.","category":"page"},{"location":"#More-documentation-is-under-way","page":"Introduction","title":"More documentation is under way","text":"","category":"section"}] }