Skip to content

Commit

Permalink
Type instability fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@smillerc
smillerc committed Oct 17, 2022
1 parent e5410c7 commit c7baa31
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 38 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "BlockHaloArrays"
uuid = "8029ca6b-11ad-4a59-88a2-2e6eee4ef8a2"
authors = ["Sam Miller <[email protected]> and contributors"]
version = "0.4.7"
version = "0.4.8"

[deps]
EllipsisNotation = "da5c29d0-fa7d-589e-88eb-ea29b0a81949"
Expand Down
26 changes: 16 additions & 10 deletions src/BlockHaloArrays.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ scaling than multi-threaded loops
- `globaldims::NTuple{D,Int}`: Dimensions of the array if it were a simple `Array{T,D}`, e.g. `(20,20)`
"""
struct BlockHaloArray{T,N,NH,NBL,AA<:Array{T,N}} <: AbstractBlockHaloArray
struct BlockHaloArray{T,N,NH,NBL,AA<:Array{T,N},SA} <: AbstractBlockHaloArray
blocks::Vector{AA}
block_layout::NTuple{NBL,Int}
halodims::NTuple{NH,Int}
global_blockranges::Array{NTuple{N,UnitRange{Int}}}
global_blockranges::Vector{NTuple{N,UnitRange{Int}}}
nhalo::Int
loop_limits::Vector{Vector{Int}}
globaldims::NTuple{N,Int}
Expand All @@ -45,8 +45,8 @@ struct BlockHaloArray{T,N,NH,NBL,AA<:Array{T,N}} <: AbstractBlockHaloArray
# private vars
_cummulative_blocksize_per_dim::OffsetVector{Vector{Int64},Vector{Vector{Int64}}}
_linear_indices::LinearIndices{NBL,NTuple{NBL,Base.OneTo{Int}}}
_donor_views::Vector{Dict{Symbol,SubArray}}
_halo_views::Vector{Dict{Symbol,SubArray}}
_donor_views::Vector{Dict{Symbol,SA}}
_halo_views::Vector{Dict{Symbol,SA}}
end

"""
Expand All @@ -69,7 +69,7 @@ struct MPIBlockHaloArray{T,N,NH,NBL,AA<:Array{T,N}} <: AbstractBlockHaloArray
blocks::Vector{AA}
block_layout::NTuple{NBL,Int}
halodims::NTuple{NH,Int}
global_blockranges::Array{NTuple{N,UnitRange{Int}}}
global_blockranges::Vector{NTuple{N,UnitRange{Int}}}
nhalo::Int
loop_limits::Vector{Vector{Int}}
globaldims::NTuple{N,Int}
Expand Down Expand Up @@ -131,6 +131,7 @@ function BlockHaloArray(dims::NTuple{N,Int}, halodims::NTuple{N2,Int}, nhalo::In
end
end
halo_only_dims = Tuple([v for (i, v) in enumerate(dims) if i in halodims])
nhalodims = length(halo_only_dims)
non_halo_dim_sizes = Tuple([v for (i, v) in enumerate(dims) if !(i in halodims)])

if halodims == Tuple(1:length(dims))
Expand Down Expand Up @@ -187,13 +188,18 @@ function BlockHaloArray(dims::NTuple{N,Int}, halodims::NTuple{N2,Int}, nhalo::In

# Pass undef'ed vector to the constructor, since the views
# need to be done _after_ the type is created.
_halo_views = Vector{Dict{Symbol,SubArray}}(undef, nblocks)
_donor_views = Vector{Dict{Symbol,SubArray}}(undef, nblocks)

A = BlockHaloArray(blocks, tile_dims, halodims, block_ranges, nhalo,
_halo_views = gen_halo_views(blocks, nhalo, halodims)
HaloViewTypes = typeof(first(_halo_views)[:ilo])
halo_views = Vector{Dict{Symbol, HaloViewTypes}}(undef, nblocks)

_donor_views = gen_donor_views(blocks, nhalo, halodims)
DonorViewTypes = typeof(first(_donor_views)[:ilo])
donor_views = Vector{Dict{Symbol, DonorViewTypes}}(undef, nblocks)

A = BlockHaloArray(blocks, tile_dims, halodims, vec(block_ranges), nhalo,
loop_limits, dims, neighbors,
cummulative_blocksize_per_dim, LI,
_donor_views, _halo_views)
donor_views, halo_views)

# Get the halo reciever and domain donor views of each block.
# These views are used during the halo sync process
Expand Down
38 changes: 33 additions & 5 deletions src/halo_exchange.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,8 @@ each thread/block copies from it's neighbor block's domain into the current bloc
- `include_periodic_bc`: Update the halo regions that are on periodic boundaries
"""
function updatehalo!(A::BlockHaloArray, include_periodic_bc=false)
@sync for tid in 1:length(A.blocks)
ThreadPools.@tspawnat tid updateblockhalo!(A, tid, include_periodic_bc)
@sync for tid in eachindex(A.blocks)
@tspawnat tid updateblockhalo!(A, tid, include_periodic_bc)
end
end

Expand All @@ -283,12 +283,11 @@ function updateblockhalo!(A::BlockHaloArray, current_block_id::Integer, include_

for (dom_id, halo_id) in exchange_map

neighbor_block_id = A.neighbor_blocks[current_block_id][halo_id]
# The convention is that periodic neighbors block ids are < 0 as a hint to the
# user and code.
if include_periodic_bc
neighbor_block_id = abs(A.neighbor_blocks[current_block_id][halo_id])
else
neighbor_block_id = A.neighbor_blocks[current_block_id][halo_id]
neighbor_block_id = abs(neighbor_block_id)
end

if valid_neighbor(neighbor_block_id)
Expand Down Expand Up @@ -744,6 +743,20 @@ function gen_halo_views(A)
blk_halo_views
end

function gen_halo_views(blocks, nhalo, halodims)

blk_halo_views = Vector{Dict{Symbol,SubArray}}(undef, length(blocks))

for blk_i in eachindex(blocks)
halo_reciever = halo_reciever_ranges(blocks[blk_i], nhalo, halodims)
blk_halo_views[blk_i] = Dict(
k => @view blocks[blk_i][.., halo_reciever[k]...] for k in keys(halo_reciever)
)
end

blk_halo_views
end

"""
Generate the SubArray views of each domain region in `A::BlockHaloArray` that are called
"donor" views. These are the regions copied from in the halo exchange copy/update.
Expand All @@ -762,3 +775,18 @@ function gen_donor_views(A)

blk_donor_views
end

function gen_donor_views(blocks, nhalo, halodims)

blk_donor_views = Vector{Dict{Symbol,SubArray}}(undef, length(blocks))

for blk_i in eachindex(blocks)
donor = domain_donor_ranges(blocks[blk_i], nhalo, halodims)

blk_donor_views[blk_i] = Dict(
k => @view blocks[blk_i][.., donor[k]...] for k in keys(donor)
)
end

blk_donor_views
end
44 changes: 22 additions & 22 deletions test/unit/test_halo_exchange.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@

dims = (50, 50)
nhalo = 2
nblocks = 6
n_blocks = 6

A = BlockHaloArray(dims, nhalo, nblocks; T=Int64)
@test length(A.blocks) == nblocks
A = BlockHaloArray(dims, nhalo, n_blocks; T=Int64)
@test length(A.blocks) == n_blocks
@test A.block_layout == (3, 2)

# for a (3,2) block layout, here are the block id's
Expand All @@ -30,25 +30,25 @@
Dict(:ilo => 5, :ihi => -4, :jlo => 3, :jhi => -3, :ilojhi => -2, :ihijhi => -1, :ilojlo => 2, :ihijlo => -1), # 6
]

for blockid in 1:nblocks
for blockid in 1:n_blocks
for key in keys(first(blk_neighbor))
@test A.neighbor_blocks[blockid][key] == blk_neighbor[blockid][key]
end
end

@sync for tid in 1:nblocks
@sync for tid in 1:n_blocks
@tspawnat tid init(A, tid)
end

include_periodic_bc = true
updatehalo!(A, include_periodic_bc)

for blockid in 1:nblocks
for blockid in 1:n_blocks
dv = domainview(A, blockid)
@test all(dv .== blockid)
end

for blockid in 1:nblocks
for blockid in 1:n_blocks
hi_domn_start, hi_domn_end, hi_halo_start, hi_halo_end = BlockHaloArrays.hi_indices(A.blocks[blockid], A.nhalo)
lo_halo_start, lo_halo_end, lo_domn_start, lo_domn_end = BlockHaloArrays.lo_indices(A.blocks[blockid], A.nhalo)

Expand Down Expand Up @@ -81,10 +81,10 @@ end
dims = (4, 30, 20)
halodims = (2, 3)
nhalo = 2
nblocks = 6
n_blocks = 6

A = BlockHaloArray(dims, halodims, nhalo, nblocks; T=Int64)
@test length(A.blocks) == nblocks
A = BlockHaloArray(dims, halodims, nhalo, n_blocks; T=Int64)
@test length(A.blocks) == n_blocks
@test A.block_layout == (3, 2)

# for a (3,2) block layout, here are the block id's
Expand All @@ -107,25 +107,25 @@ end
Dict(:ilo => 5, :ihi => -4, :jlo => 3, :jhi => -3, :ilojhi => -2, :ihijhi => -1, :ilojlo => 2, :ihijlo => -1), # 6
]

for blockid in 1:nblocks
for blockid in 1:n_blocks
for key in keys(first(blk_neighbor))
@test A.neighbor_blocks[blockid][key] == blk_neighbor[blockid][key]
end
end

@sync for tid in 1:nblocks
@sync for tid in 1:n_blocks
@tspawnat tid init(A, tid)
end

for blockid in 1:nblocks
for blockid in 1:n_blocks
dv = domainview(A, blockid)
@test all(dv .== blockid)
end

include_periodic_bc = true
updatehalo!(A, include_periodic_bc)

for blockid in 1:nblocks
for blockid in 1:n_blocks
hi_domn_start, hi_domn_end, hi_halo_start, hi_halo_end = BlockHaloArrays.hi_indices(A.blocks[blockid], A.nhalo, A.halodims)
lo_halo_start, lo_halo_end, lo_domn_start, lo_domn_end = BlockHaloArrays.lo_indices(A.blocks[blockid], A.nhalo, A.halodims)

Expand Down Expand Up @@ -157,8 +157,8 @@ end

dims = (20,)
nhalo = 2
nblocks = 4
A = BlockHaloArray(dims, nhalo, nblocks; T=Int32)
n_blocks = 4
A = BlockHaloArray(dims, nhalo, n_blocks; T=Int32)

@test A.neighbor_blocks[1][:ilo] == -4
@test A.neighbor_blocks[1][:ihi] == 2
Expand All @@ -169,14 +169,14 @@ end
@test A.neighbor_blocks[4][:ilo] == 3
@test A.neighbor_blocks[4][:ihi] == -1

@sync for tid in 1:nblocks
@sync for tid in 1:n_blocks
@tspawnat tid init(A, tid)
end

include_periodic_bc = true
updatehalo!(A, include_periodic_bc)

for blockid in 1:nblocks
for blockid in 1:n_blocks
hi_domn_start, hi_domn_end, hi_halo_start, hi_halo_end = BlockHaloArrays.hi_indices(A.blocks[blockid], A.nhalo)
lo_halo_start, lo_halo_end, lo_domn_start, lo_domn_end = BlockHaloArrays.lo_indices(A.blocks[blockid], A.nhalo)

Expand All @@ -200,17 +200,17 @@ end

dims = (10, 20, 30)
nhalo = 2
nblocks = 6
A = BlockHaloArray(dims, nhalo, nblocks; T=Float64)
n_blocks = 6
A = BlockHaloArray(dims, nhalo, n_blocks; T=Float64)

@sync for tid in 1:nblocks
@sync for tid in 1:n_blocks
@tspawnat tid init(A, tid)
end

include_periodic_bc = true
updatehalo!(A, include_periodic_bc)

for blockid in 1:nblocks
for blockid in 1:n_blocks
hi_domn_start, hi_domn_end, hi_halo_start, hi_halo_end = BlockHaloArrays.hi_indices(A.blocks[blockid], A.nhalo)
lo_halo_start, lo_halo_end, lo_domn_start, lo_domn_end = BlockHaloArrays.lo_indices(A.blocks[blockid], A.nhalo)

Expand Down

2 comments on commit c7baa31

@smillerc
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/70443

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.4.8 -m "<description of version>" c7baa31e437f0ea56125dbb858c70031c18d0dce
git push origin v0.4.8

Please sign in to comment.