Merge pull request #132 from magnusuMET/feature/TypeDescriptor

Revamp type descriptors

.github/workflows/ci.yml

@@ -37,9 +37,9 @@ jobs:
     - name: Check formatting
       run: cargo fmt -- --check
     - name: Documentation
-      run: cargo doc --workspace
+      run: cargo doc --workspace --features netcdf/derive
     - name: Clippy
-      run: cargo clippy --workspace -- -D warnings
+      run: cargo clippy --features netcdf/derive --workspace -- -D warnings
 
   test_apt:
     name: test apt
@@ -71,10 +71,10 @@ jobs:
         toolchain: ${{ matrix.rust }}
 
     - name: Build
-      run: cargo build --verbose --workspace --exclude netcdf-src
+      run: cargo build --verbose --features netcdf/derive --workspace --exclude netcdf-src
 
     - name: Test
-      run: cargo test --verbose --workspace --exclude netcdf-src
+      run: cargo test --verbose --features netcdf/derive --workspace --exclude netcdf-src --exclude netcdf-derive
 
   conda:
     name: conda
@@ -106,7 +106,7 @@ jobs:
           export HDF5_DIR="$CONDA_PREFIX"
           export NETCDF_DIR="$CONDA_PREFIX"
           [ "${{runner.os}}" != "Windows" ] && export RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib"
-          cargo test -vv --workspace --exclude netcdf-src
+          cargo test -vv --workspace --exclude netcdf-src --exclude netcdf-derive --features netcdf/derive
 
   static_builds:
     name: static builds
@@ -131,7 +131,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
         with: {toolchain: '${{matrix.rust}}'}
       - name: Build and test
-        run: cargo test -vv --features static --workspace
+        run: cargo test -vv --features netcdf/derive,static --workspace --exclude netcdf-derive
 
   addr_san:
     name: Address sanitizer
@@ -147,4 +147,4 @@ jobs:
         env:
           RUSTFLAGS: "-Z sanitizer=address"
           RUSTDOCFLAGS: "-Z sanitizer=address"
-        run: cargo test --features netcdf-sys/static --target x86_64-unknown-linux-gnu --workspace
+        run: cargo test --features netcdf-sys/static,netcdf/derive --target x86_64-unknown-linux-gnu --workspace --exclude netcdf-derive

.github/workflows/release.yml

@@ -7,6 +7,7 @@ on:
       - "netcdf-v*"
       - "netcdf-sys-v*"
       - "netcdf-src-v*"
+      - "netcdf-derive-v*"
 
 env:
   CARGO_TERM_COLOR: always
@@ -32,6 +33,9 @@ jobs:
     - name: Publish netcdf-sys
       if: "${{ startsWith(github.ref_name, 'netcdf-sys-v') }}"
       run: cargo publish --package netcdf-sys --token "${{ secrets.CRATES_IO_TOKEN }}"
+    - name: Publish netcdf-derive
+      if: "${{ startsWith(github.ref_name, 'netcdf-derive-v') }}"
+      run: cargo publish --package netcdf --token "${{ secrets.CRATES_IO_TOKEN }}"
     - name: Publish netcdf
       if: "${{ startsWith(github.ref_name, 'netcdf-v') }}"
       run: cargo publish --package netcdf --token "${{ secrets.CRATES_IO_TOKEN }}"

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "netcdf",
     "netcdf-sys",
     "netcdf-src",
+    "netcdf-derive",
 ]
 default-members = ["netcdf", "netcdf-sys"]
 resolver = "2"
@@ -11,4 +12,5 @@ resolver = "2"
 netcdf = { path = "netcdf", version = "0.9.2" }
 netcdf-sys = { path = "netcdf-sys", version = "0.6.2" }
 netcdf-src = { path = "netcdf-src", version = "0.3.6" }
+netcdf-derive = { path = "netcdf-derive", version = "0.1.0" }
 hdf5-sys = { version = "0.8.0" }

README.md

@@ -22,12 +22,9 @@ Supported:
 * Open/Append/Create modes
 * Reading from memory
 * Unlimited dimensions
-* String variables
-* User defined types (variable length, enum, compound, opaque)
+* User defined types, using the feature `derive` (enum, compound, other types requires additional work)
 
-Not (yet) supported:
-
-* Nested user defined types
+Not (yet) supported (PRs welcome):
 * Writing using memory-mapped file
 
 All variable data is read into a contiguous buffer, or into an [ndarray](https://github.com/rust-ndarray/ndarray) if the `ndarray` feature is activated.

netcdf-derive/.gitignore

@@ -0,0 +1 @@
+*.nc

netcdf-derive/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "netcdf-derive"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "2.0.48"
+quote = "1.0.35"
+proc-macro2 = "1.0.78"
+proc-macro-error = "1.0.4"
+
+[dev-dependencies]
+netcdf = { workspace = true }
+trybuild = "1.0.89"

netcdf-derive/src/lib.rs

@@ -0,0 +1,244 @@
+use proc_macro2::{Ident, TokenStream};
+use proc_macro_error::{abort, proc_macro_error};
+use quote::quote;
+use syn::{
+    parse_macro_input, Data, DataEnum, DataStruct, DeriveInput, Fields, FieldsNamed, LitStr, Type,
+    Variant,
+};
+
+#[proc_macro_derive(NcType, attributes(netcdf))]
+/// Derives `NcTypeDescriptor` for user defined types.
+///
+/// See the documentation under `netcdf::TypeDescriptor` for examples
+///
+/// Use `#[netcdf(rename = "name")]` to
+/// rename field names or enum member names, or the name
+/// of the compound/enum.
+///
+/// Types one derives `NcType` for must have some properties to
+/// ensure correctness:
+/// * Structs must have `repr(C)` to ensure layout compatibility
+/// * Structs must be packed (no padding allowed)
+/// * Enums must have `repr(T)` where `T` is an int type (`{i/u}{8/16/32/64}`)
+#[proc_macro_error]
+pub fn derive(stream: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let input = parse_macro_input!(stream as DeriveInput);
+    let name = &input.ident;
+    let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+
+    let mut renamed = None;
+    let mut repr_c = false;
+    for attr in &input.attrs {
+        if attr.path().is_ident("netcdf") {
+            attr.parse_nested_meta(|meta| {
+                if meta.path.is_ident("rename") {
+                    renamed = Some(meta.value()?.parse::<LitStr>()?.value());
+                } else {
+                    abort!(meta.path, "NcType encountered an unknown attribute");
+                }
+                Ok(())
+            })
+            .unwrap();
+        } else if attr.path().is_ident("repr") {
+            attr.parse_nested_meta(|meta| {
+                if meta.path.is_ident("C") {
+                    repr_c = true;
+                }
+                Ok(())
+            })
+            .unwrap();
+        }
+    }
+    let ncname = renamed.unwrap_or_else(|| name.to_string());
+
+    let body = match input.data {
+        Data::Struct(DataStruct {
+            struct_token: _,
+            ref fields,
+            semi_token: _,
+        }) => {
+            if !repr_c {
+                abort!(
+                    input,
+                    "Can not derive NcType for struct without fixed layout";
+                    help = "struct must have attribute #[repr(C)]"
+                );
+            }
+            match fields {
+                Fields::Named(fields) => impl_compound(name, &ncname, fields.clone()),
+                Fields::Unnamed(f) => {
+                    abort!(f, "Can not derive NcType for struct with unnamed field"; note="#[derive(NcType)]")
+                }
+                Fields::Unit => abort!(input, "Can not derive NcType for unit struct"),
+            }
+        }
+        Data::Enum(DataEnum {
+            enum_token: _,
+            brace_token: _,
+            ref variants,
+        }) => {
+            let mut basetyp = None;
+            for attr in &input.attrs {
+                if attr.path().is_ident("repr") {
+                    attr.parse_nested_meta(|meta| {
+                        for item in ["u8", "u16", "u32", "u64", "i8", "i16", "i32", "i64"] {
+                            if meta.path.is_ident(item) {
+                                basetyp = Some(meta.path.get_ident().unwrap().clone());
+                            }
+                        }
+                        Ok(())
+                    })
+                    .unwrap();
+                }
+            }
+            let Some(basetyp) = basetyp else {
+                abort!(
+                    input,
+                    "Can not derive NcType for enum without suitable repr";
+                    help="Add #[repr(i32)] (or another integer type) as an attribute to the enum"
+                );
+            };
+            impl_enum(/*&name,*/ &ncname, &basetyp, variants.iter())
+        }
+        Data::Union(_) => abort!(
+            input,
+            "Can not derive NcType for union";
+            note = "netCDF has no concept of Union type"
+        ),
+    };
+
+    let expanded = quote! {
+        const _: () = {
+            use netcdf::types::*;
+
+            #[automatically_derived]
+            unsafe impl #impl_generics NcTypeDescriptor for #name #ty_generics #where_clause {
+                fn type_descriptor() -> NcVariableType {
+                    #body
+                }
+            }
+        };
+    };
+    proc_macro::TokenStream::from(expanded)
+}
+
+fn impl_compound(ty: &Ident, ncname: &str, fields: FieldsNamed) -> TokenStream {
+    struct FieldInfo {
+        name: String,
+        typ: Type,
+    }
+    let mut items: Vec<FieldInfo> = vec![];
+
+    for field in fields.named {
+        let ident = field.ident.expect("Field must have a name").clone();
+        let mut rename = None;
+        for attr in field.attrs {
+            if attr.path().is_ident("netcdf") {
+                attr.parse_nested_meta(|meta| {
+                    if meta.path.is_ident("rename") {
+                        rename = Some(meta.value()?.parse::<LitStr>()?.value());
+                    } else {
+                        abort!(meta.path, "NcType encountered an unknown attribute")
+                    }
+                    Ok(())
+                })
+                .unwrap();
+            }
+        }
+        let name = rename.unwrap_or_else(|| ident.to_string());
+        items.push(FieldInfo {
+            name,
+            typ: field.ty,
+        });
+    }
+
+    let fieldnames = items
+        .iter()
+        .map(|item| item.name.clone())
+        .collect::<Vec<_>>();
+    let typeids = items
+        .iter()
+        .map(|item| item.typ.clone())
+        .collect::<Vec<_>>();
+    let fieldinfo = quote!(vec![#(
+            (
+                (#fieldnames).to_owned(),
+                <#typeids as NcTypeDescriptor>::type_descriptor(),
+                (<#typeids as NcTypeDescriptor>::ARRAY_ELEMENTS).as_dims().map(Vec::from),
+            )
+            ),*]);
+
+    quote! {
+        let mut fields = vec![];
+        let mut offset = 0;
+        for (name, basetype, arraydims) in #fieldinfo {
+            let nelems = arraydims.as_ref().map_or(1, |x| x.iter().copied().product());
+            let thissize = basetype.size() * nelems;
+            fields.push(CompoundTypeField {
+                name,
+                offset,
+                basetype,
+                arraydims,
+            });
+            offset += thissize;
+        }
+        let compound = NcVariableType::Compound(CompoundType {
+            name: (#ncname).to_owned(),
+            size: offset,
+            fields,
+        });
+        assert_eq!(compound.size(), std::mem::size_of::<#ty>(), "Compound must be packed");
+        compound
+    }
+}
+
+fn impl_enum<'a>(
+    // ty: &Ident,
+    ncname: &str,
+    basetyp: &Ident,
+    fields: impl Iterator<Item = &'a Variant>,
+) -> TokenStream {
+    let mut fieldnames = vec![];
+    let mut fieldvalues = vec![];
+
+    for field in fields {
+        let ident = field.ident.clone();
+        let mut rename = None;
+        for attr in &field.attrs {
+            if attr.path().is_ident("netcdf") {
+                attr.parse_nested_meta(|meta| {
+                    if meta.path.is_ident("rename") {
+                        rename = Some(meta.value()?.parse::<LitStr>()?.value());
+                    } else {
+                        abort!(meta.path, "NcType encountered an unknown attribute")
+                    }
+                    Ok(())
+                })
+                .unwrap();
+            }
+        }
+        let name = rename.unwrap_or_else(|| ident.to_string());
+        fieldnames.push(name);
+
+        let variant = match field.discriminant.clone() {
+            Some((_, x)) => quote!(#x),
+            None => fieldvalues
+                .last()
+                .map(|e| quote!(#e + 1))
+                .unwrap_or(quote!(0)),
+        };
+
+        fieldvalues.push(variant);
+    }
+
+    let fieldnames = quote!(vec![#(#fieldnames),*]);
+    let fieldvalues = quote!(vec![#(#fieldvalues),*]);
+
+    quote! {
+        NcVariableType::Enum(EnumType {
+            name: (#ncname).to_owned(),
+            fieldnames: (#fieldnames).iter().map(|x| x.to_string()).collect(),
+            fieldvalues: ((#fieldvalues).into_iter().collect::<Vec::<#basetyp>>()).into(),
+        })
+    }
+}

netcdf-derive/tests/failing/enumnorepr.rs

@@ -0,0 +1,7 @@
+#[derive(netcdf_derive::NcType)]
+enum NoRepr {
+    A,
+    B,
+}
+
+fn main() {}

netcdf-derive/tests/failing/enumnorepr.stderr

@@ -0,0 +1,10 @@
+error: Can not derive NcType for enum without suitable repr
+ --> tests/failing/enumnorepr.rs:2:1
+  |
+2 | / enum NoRepr {
+3 | |     A,
+4 | |     B,
+5 | | }
+  | |_^
+  |
+  = help: Add #[repr(i32)] (or another integer type) as an attribute to the enum

netcdf-derive/tests/failing/noreprc.rs

@@ -0,0 +1,6 @@
+#[derive(netcdf_derive::NcType)]
+struct NoReprC {
+    a: u8,
+}
+
+fn main() {}

netcdf-derive/tests/failing/noreprc.stderr

@@ -0,0 +1,9 @@
+error: Can not derive NcType for struct without fixed layout
+ --> tests/failing/noreprc.rs:2:1
+  |
+2 | / struct NoReprC {
+3 | |     a: u8,
+4 | | }
+  | |_^
+  |
+  = help: struct must have attribute #[repr(C)]