Skip to content

Commit

Permalink
Disable SkipLayerNorm in the model builder for DML
Browse files Browse the repository at this point in the history
  • Loading branch information
PatriceVignola committed May 5, 2024
1 parent 4e7de79 commit 878ca5d
Showing 1 changed file with 3 additions and 0 deletions.
3 changes: 3 additions & 0 deletions src/python/py/models/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,9 @@ def make_layernorm(self, layer_id, layernorm, skip, simple, location):
root_input = self.layernorm_attrs["root_input"]
skip_input = self.layernorm_attrs["skip_input"]

# We don't use SkipLayerNorm or SkipSimplifiedLayerNorm for DML since the Add gets fused with the Matmul
skip = skip or self.ep == "dml"

weight = f"model.layers.{layer_id}.{location}_layernorm.weight"
self.make_external_tensor(layernorm.weight.detach().numpy().astype(self.to_numpy_dtype[self.io_dtype]) + self.layernorm_attrs["add_offset"], weight)
bias = f"model.layers.{layer_id}.{location}_layernorm.bias"
Expand Down

0 comments on commit 878ca5d

Please sign in to comment.