diff --git a/cli/src/main.rs b/cli/src/main.rs index 6fdd9df5..ab42d470 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -29,6 +29,7 @@ use typeshare_core::{ TypeScript, }, parser::ParsedData, + reconcile::reconcile_aliases, }; use crate::{ @@ -46,8 +47,6 @@ fn main() -> anyhow::Result<()> { let options = Args::parse(); - info!("typeshare started generating types"); - if let Some(options) = options.subcommand { match options { Command::Completions { shell } => { @@ -68,6 +67,8 @@ fn main() -> anyhow::Result<()> { return Ok(()); } + info!("typeshare started generating types"); + let config = config::load_config(config_file).context("Unable to read configuration file")?; let config = override_configuration(config, &options)?; @@ -146,11 +147,13 @@ fn main() -> anyhow::Result<()> { // and implement a `ParallelVisitor` that builds up the mapping of parsed // data. That way both walking and parsing are in parallel. // https://docs.rs/ignore/latest/ignore/struct.WalkParallel.html - let crate_parsed_data = parse_input( + let mut crate_parsed_data = parse_input( parser_inputs(walker_builder, language_type, multi_file).par_bridge(), &parse_context, )?; + reconcile_aliases(&mut crate_parsed_data); + // Collect all the types into a map of the file name they // belong too and the list of type names. Used for generating // imports in generated files. diff --git a/cli/src/parse.rs b/cli/src/parse.rs index f6e961cf..2961bfdd 100644 --- a/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -1,7 +1,7 @@ //! Source file parsing. use anyhow::Context; use ignore::WalkBuilder; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use rayon::iter::ParallelIterator; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap}, path::PathBuf, @@ -93,7 +93,6 @@ pub fn parse_input( parse_context: &ParseContext, ) -> anyhow::Result> { inputs - .into_par_iter() .try_fold( BTreeMap::new, |mut parsed_crates: BTreeMap, @@ -102,9 +101,11 @@ pub fn parse_input( file_name, crate_name, }| { + let fp = file_path.as_os_str().to_str().unwrap_or("").to_string(); + let parse_file_context = ParseFileContext { source_code: std::fs::read_to_string(&file_path) - .with_context(|| format!("Failed to read input: {file_name}"))?, + .with_context(|| format!("Failed to read input: {file_path:?}"))?, crate_name: crate_name.clone(), file_name: file_name.clone(), file_path, @@ -112,7 +113,7 @@ pub fn parse_input( let parsed_result = typeshare_core::parser::parse(parse_context, parse_file_context) - .with_context(|| format!("Failed to parse: {file_name}"))?; + .with_context(|| format!("Failed to parse: {fp}"))?; if let Some(parsed_data) = parsed_result { parsed_crates diff --git a/core/README.md b/core/README.md index 4eb9fba7..2356595d 100644 --- a/core/README.md +++ b/core/README.md @@ -34,7 +34,7 @@ The test suite can of course be run normally without updating any expectations: cargo test -p typeshare-core ``` -If you find yourself needing to update expectations for a specific test only, run the following (subsituting the name of your test in for the last arg): +If you find yourself needing to update expectations for a specific test only, run the following (substituting the name of your test in for the last arg): ``` env UPDATE_EXPECT=1 cargo test -p typeshare-core --test snapshot_tests -- can_handle_serde_rename_all::swift diff --git a/core/data/tests/serde_rename_references/input.rs b/core/data/tests/serde_rename_references/input.rs new file mode 100644 index 00000000..f0b6caef --- /dev/null +++ b/core/data/tests/serde_rename_references/input.rs @@ -0,0 +1,27 @@ +//! Test references to a type that has been renamed via serde(rename) +//! + +#[derive(Serialize)] +#[serde(rename = "SomethingFoo")] +#[typeshare] +pub enum Foo { + A, +} + +#[derive(Serialize)] +#[typeshare] +#[serde(tag = "type", content = "value")] +pub enum Parent { + B(Foo), +} + +#[derive(Serialize)] +#[typeshare] +pub struct Test { + field1: Foo, + field2: Option, +} + +#[derive(Serialize)] +#[typeshare] +pub type AliasTest = Vec; diff --git a/core/data/tests/serde_rename_references/output.go b/core/data/tests/serde_rename_references/output.go new file mode 100644 index 00000000..e00dc1d2 --- /dev/null +++ b/core/data/tests/serde_rename_references/output.go @@ -0,0 +1,68 @@ +package proto + +import "encoding/json" + +type AliasTest []SomethingFoo + +type Test struct { + Field1 SomethingFoo `json:"field1"` + Field2 *SomethingFoo `json:"field2,omitempty"` +} +type Foo string +const ( + FooA Foo = "A" +) +type ParentTypes string +const ( + ParentTypeVariantB ParentTypes = "B" +) +type Parent struct{ + Type ParentTypes `json:"type"` + value interface{} +} + +func (p *Parent) UnmarshalJSON(data []byte) error { + var enum struct { + Tag ParentTypes `json:"type"` + Content json.RawMessage `json:"value"` + } + if err := json.Unmarshal(data, &enum); err != nil { + return err + } + + p.Type = enum.Tag + switch p.Type { + case ParentTypeVariantB: + var res SomethingFoo + p.value = &res + + } + if err := json.Unmarshal(enum.Content, &p.value); err != nil { + return err + } + + return nil +} + +func (p Parent) MarshalJSON() ([]byte, error) { + var enum struct { + Tag ParentTypes `json:"type"` + Content interface{} `json:"value,omitempty"` + } + enum.Tag = p.Type + enum.Content = p.value + return json.Marshal(enum) +} + +func (p Parent) B() SomethingFoo { + res, _ := p.value.(*SomethingFoo) + return *res +} + +func NewParentTypeVariantB(content SomethingFoo) Parent { + return Parent{ + Type: ParentTypeVariantB, + value: &content, + } +} + diff --git a/core/data/tests/serde_rename_references/output.kt b/core/data/tests/serde_rename_references/output.kt new file mode 100644 index 00000000..2f353250 --- /dev/null +++ b/core/data/tests/serde_rename_references/output.kt @@ -0,0 +1,26 @@ +package com.agilebits.onepassword + +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +typealias AliasTest = List + +@Serializable +data class Test ( + val field1: SomethingFoo, + val field2: SomethingFoo? = null +) + +@Serializable +enum class SomethingFoo(val string: String) { + @SerialName("A") + A("A"), +} + +@Serializable +sealed class Parent { + @Serializable + @SerialName("B") + data class B(val value: SomethingFoo): Parent() +} + diff --git a/core/data/tests/serde_rename_references/output.scala b/core/data/tests/serde_rename_references/output.scala new file mode 100644 index 00000000..8e4c02e5 --- /dev/null +++ b/core/data/tests/serde_rename_references/output.scala @@ -0,0 +1,33 @@ +package com.agilebits + +package object onepassword { + +type AliasTest = Vector[SomethingFoo] + +} +package onepassword { + +case class Test ( + field1: SomethingFoo, + field2: Option[SomethingFoo] = None +) + +sealed trait SomethingFoo { + def serialName: String +} +object SomethingFoo { + case object A extends SomethingFoo { + val serialName: String = "A" + } +} + +sealed trait Parent { + def serialName: String +} +object Parent { + case class B(value: SomethingFoo) extends Parent { + val serialName: String = "B" + } +} + +} diff --git a/core/data/tests/serde_rename_references/output.swift b/core/data/tests/serde_rename_references/output.swift new file mode 100644 index 00000000..6f443321 --- /dev/null +++ b/core/data/tests/serde_rename_references/output.swift @@ -0,0 +1,52 @@ +import Foundation + +public typealias AliasTest = [SomethingFoo] + +public struct Test: Codable { + public let field1: SomethingFoo + public let field2: SomethingFoo? + + public init(field1: SomethingFoo, field2: SomethingFoo?) { + self.field1 = field1 + self.field2 = field2 + } +} + +public enum SomethingFoo: String, Codable { + case a = "A" +} + +public enum Parent: Codable { + case b(SomethingFoo) + + enum CodingKeys: String, CodingKey, Codable { + case b = "B" + } + + private enum ContainerCodingKeys: String, CodingKey { + case type, value + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: ContainerCodingKeys.self) + if let type = try? container.decode(CodingKeys.self, forKey: .type) { + switch type { + case .b: + if let content = try? container.decode(SomethingFoo.self, forKey: .value) { + self = .b(content) + return + } + } + } + throw DecodingError.typeMismatch(Parent.self, DecodingError.Context(codingPath: decoder.codingPath, debugDescription: "Wrong type for Parent")) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: ContainerCodingKeys.self) + switch self { + case .b(let content): + try container.encode(CodingKeys.b, forKey: .type) + try container.encode(content, forKey: .value) + } + } +} diff --git a/core/data/tests/serde_rename_references/output.ts b/core/data/tests/serde_rename_references/output.ts new file mode 100644 index 00000000..de1ae543 --- /dev/null +++ b/core/data/tests/serde_rename_references/output.ts @@ -0,0 +1,14 @@ +export type AliasTest = SomethingFoo[]; + +export interface Test { + field1: SomethingFoo; + field2?: SomethingFoo; +} + +export enum SomethingFoo { + A = "A", +} + +export type Parent = + | { type: "B", value: SomethingFoo }; + diff --git a/core/src/language/kotlin.rs b/core/src/language/kotlin.rs index 011efe2d..69ce4d7b 100644 --- a/core/src/language/kotlin.rs +++ b/core/src/language/kotlin.rs @@ -130,6 +130,7 @@ impl Language for Kotlin { id: Id { original: String::from("value"), renamed: String::from("value"), + serde_rename: false, }, ty: ty.r#type.clone(), comments: vec![], diff --git a/core/src/language/mod.rs b/core/src/language/mod.rs index 4b5d0b35..d38460ed 100644 --- a/core/src/language/mod.rs +++ b/core/src/language/mod.rs @@ -379,6 +379,7 @@ pub trait Language { id: Id { original: struct_name.clone(), renamed: struct_name.clone(), + serde_rename: false }, fields: fields.clone(), generic_types, diff --git a/core/src/lib.rs b/core/src/lib.rs index fe8dc4ab..b751d99e 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -7,6 +7,7 @@ pub mod context; pub mod language; /// Parsing Rust code into a format the `language` modules can understand pub mod parser; +pub mod reconcile; mod rename; /// Codifying Rust types and how they convert to various languages. pub mod rust_types; diff --git a/core/src/parser.rs b/core/src/parser.rs index c278217b..c7749fdf 100644 --- a/core/src/parser.rs +++ b/core/src/parser.rs @@ -90,11 +90,11 @@ pub struct ErrorInfo { #[derive(Default, Debug)] pub struct ParsedData { /// Structs defined in the source - pub structs: BTreeSet, + pub structs: Vec, /// Enums defined in the source - pub enums: BTreeSet, + pub enums: Vec, /// Type aliases defined in the source - pub aliases: BTreeSet, + pub aliases: Vec, /// Imports used by this file pub import_types: HashSet, /// Crate this belongs to. @@ -138,15 +138,15 @@ impl ParsedData { match rust_thing { RustItem::Struct(s) => { self.type_names.insert(s.id.renamed.clone()); - self.structs.insert(s); + self.structs.push(s); } RustItem::Enum(e) => { self.type_names.insert(e.shared().id.renamed.clone()); - self.enums.insert(e); + self.enums.push(e); } RustItem::Alias(a) => { self.type_names.insert(a.id.renamed.clone()); - self.aliases.insert(a); + self.aliases.push(a); } } } @@ -178,7 +178,7 @@ pub fn parse( file_path, } = parse_file_context; - debug!("parsing {file_name}"); + debug!("parsing {file_path:?}"); // Parse and process the input, ensuring we parse only items marked with // `#[typeshare]` let mut import_visitor = TypeShareVisitor::new(parse_context, crate_name, file_name, file_path); @@ -554,11 +554,17 @@ fn get_ident( let mut renamed = rename_all_to_case(original.clone(), rename_all); + let mut renamed_via_serde_rename = false; if let Some(s) = serde_rename(attrs) { renamed = s; + renamed_via_serde_rename = true; } - Id { original, renamed } + Id { + original, + renamed, + serde_rename: renamed_via_serde_rename, + } } fn rename_all_to_case(original: String, case: &Option) -> String { diff --git a/core/src/reconcile.rs b/core/src/reconcile.rs new file mode 100644 index 00000000..a3ccc4cc --- /dev/null +++ b/core/src/reconcile.rs @@ -0,0 +1,185 @@ +//! Post reconcile references after all types have been parsed. +//! +//! Types can be renamed via `serde(rename = "NewName")`. These types will get the new +//! name however we still need to see if we have any other types that reference the renamed type +//! and update those references accordingly. +use crate::{ + language::CrateName, + parser::ParsedData, + rust_types::{RustEnum, RustEnumVariant, RustType, SpecialRustType}, + visitors::ImportedType, +}; +use log::{debug, info}; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + mem, +}; + +/// A mapping of original type names to a mapping of crate name to new name. +type RenamedTypes = HashMap>; + +/// Update any type references that have the refenced type renamed via `serde(rename)`. +pub fn reconcile_aliases(crate_parsed_data: &mut BTreeMap) { + let serde_renamed = collect_serde_renames(crate_parsed_data); + + for (crate_name, parsed_data) in crate_parsed_data { + let import_types = mem::take(&mut parsed_data.import_types); + + // update references to renamed ids in product types. + for s in &mut parsed_data.structs { + debug!("struct: {}", s.id.original); + for f in &mut s.fields { + check_type(crate_name, &serde_renamed, &import_types, &mut f.ty); + } + } + + // update references to renamed ids in sum types. + for e in &mut parsed_data.enums { + debug!("enum: {}", e.shared().id.original); + match e { + RustEnum::Unit(shared) => check_variant( + crate_name, + &serde_renamed, + &import_types, + &mut shared.variants, + ), + RustEnum::Algebraic { shared, .. } => check_variant( + crate_name, + &serde_renamed, + &import_types, + &mut shared.variants, + ), + } + } + + // update references to renamed ids in aliases. + for a in &mut parsed_data.aliases { + check_type(crate_name, &serde_renamed, &import_types, &mut a.r#type); + } + + // Apply sorting to types for deterministic output. + parsed_data.structs.sort(); + parsed_data.enums.sort(); + parsed_data.aliases.sort(); + + // put back our import types for file generation. + parsed_data.import_types = import_types; + } +} + +/// Traverse all the parsed typeshare data and collect all types that have been renamed +/// via `serde(rename)` into a mapping of original name to renamed name. +fn collect_serde_renames(crate_parsed_data: &BTreeMap) -> RenamedTypes { + crate_parsed_data + .iter() + .flat_map(|(crate_name, parsed_data)| { + parsed_data + .structs + .iter() + .flat_map(|s| { + s.id.serde_rename + .then_some((s.id.original.to_string(), s.id.renamed.to_string())) + }) + .chain(parsed_data.enums.iter().flat_map(|e| { + e.shared().id.serde_rename.then_some(( + e.shared().id.original.to_string(), + e.shared().id.renamed.to_string(), + )) + })) + .chain(parsed_data.aliases.iter().flat_map(|e| { + e.id.serde_rename + .then(|| (e.id.original.to_string(), e.id.renamed.to_string())) + })) + .map(|(original, renamed)| (crate_name.to_owned(), (original, renamed))) + }) + .fold( + HashMap::new(), + |mut mapping, (crate_name, (original, renamed))| { + let name_map = mapping.entry(original).or_default(); + name_map.insert(crate_name.to_owned(), renamed); + mapping + }, + ) +} + +fn check_variant( + crate_name: &CrateName, + serde_renamed: &RenamedTypes, + imported_types: &HashSet, + variants: &mut Vec, +) { + for v in variants { + match v { + RustEnumVariant::Unit(_) => (), + RustEnumVariant::Tuple { ty, .. } => { + check_type(crate_name, serde_renamed, imported_types, ty); + } + RustEnumVariant::AnonymousStruct { fields, .. } => { + for f in fields { + check_type(crate_name, serde_renamed, imported_types, &mut f.ty); + } + } + } + } +} + +fn check_type( + crate_name: &CrateName, + serde_renamed: &RenamedTypes, + import_types: &HashSet, + ty: &mut RustType, +) { + debug!("checking type: {ty:?}"); + match ty { + RustType::Generic { parameters, .. } => { + for ty in parameters { + check_type(crate_name, serde_renamed, import_types, ty); + } + } + RustType::Special(s) => match s { + SpecialRustType::Vec(ty) => { + check_type(crate_name, serde_renamed, import_types, ty); + } + SpecialRustType::Array(ty, _) => { + check_type(crate_name, serde_renamed, import_types, ty); + } + SpecialRustType::Slice(ty) => { + check_type(crate_name, serde_renamed, import_types, ty); + } + SpecialRustType::HashMap(ty1, ty2) => { + check_type(crate_name, serde_renamed, import_types, ty1); + check_type(crate_name, serde_renamed, import_types, ty2); + } + SpecialRustType::Option(ty) => { + check_type(crate_name, serde_renamed, import_types, ty); + } + _ => (), + }, + RustType::Simple { id } => { + debug!("{crate_name} looking up original name {id}"); + + if let Some(renamed) = resolve_renamed(crate_name, serde_renamed, import_types, id) { + info!("renaming type from {id} to {renamed}"); + *id = renamed.to_owned(); + } + } + } +} + +fn resolve_renamed( + crate_name: &CrateName, + serde_renamed: &RenamedTypes, + import_types: &HashSet, + id: &str, +) -> Option { + let name_map = serde_renamed.get(id)?; + + // Find in imports. + import_types + .iter() + .filter(|i| i.type_name == id) + .find_map(|import_ref| name_map.get(&import_ref.base_crate)) + // Fallback to looking up in our current namespace. + .or_else(|| name_map.get(crate_name)) + .map(ToOwned::to_owned) +} diff --git a/core/src/rust_types.rs b/core/src/rust_types.rs index 4585b05f..13be15c8 100644 --- a/core/src/rust_types.rs +++ b/core/src/rust_types.rs @@ -21,6 +21,8 @@ pub struct Id { /// If there is no re-naming going on, this will be identical to /// `original`. pub renamed: String, + /// Was this renamed with `serde(rename = "newname") + pub serde_rename: bool, } impl std::fmt::Display for Id { diff --git a/core/tests/snapshot_tests.rs b/core/tests/snapshot_tests.rs index 5182d073..49c41009 100644 --- a/core/tests/snapshot_tests.rs +++ b/core/tests/snapshot_tests.rs @@ -3,8 +3,7 @@ use flexi_logger::DeferredNow; use log::Record; use once_cell::sync::Lazy; use std::{ - collections::HashMap, - env, + collections::{BTreeMap, HashMap}, fs::{self, OpenOptions}, io::{Read, Write}, path::{Path, PathBuf}, @@ -12,7 +11,8 @@ use std::{ }; use typeshare_core::{ context::{ParseContext, ParseFileContext}, - language::Language, + language::{CrateName, Language}, + reconcile::reconcile_aliases, }; static TESTS_FOLDER_PATH: Lazy = @@ -115,6 +115,14 @@ fn check( }, )? .unwrap(); + + let all_crates: CrateName = String::new().into(); + + let mut map = BTreeMap::from_iter([(all_crates.clone(), parsed_data)]); + reconcile_aliases(&mut map); + + let parsed_data = map.remove(&all_crates).unwrap(); + lang.generate_types(&mut typeshare_output, &HashMap::new(), parsed_data)?; let typeshare_output = String::from_utf8(typeshare_output)?; @@ -629,4 +637,5 @@ tests! { generic_struct_with_constraints_and_decorators: [swift { codablevoid_constraints: vec!["Equatable".into()] }]; excluded_by_target_os: [ swift, kotlin, scala, typescript, go ] target_os: ["android", "macos"]; // excluded_by_target_os_full_module: [swift] target_os: "ios"; + serde_rename_references: [ swift, kotlin, scala, typescript, go ]; }