From 3930f47d0bd512c7ae1082bb685b69c451dc66cd Mon Sep 17 00:00:00 2001 From: HeJunchao <73169088+HeJunchao100813@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:47:53 +0800 Subject: [PATCH 01/13] Fix/cos nan (#1090) * fix pytest's cosine * fix similarity --------- Co-authored-by: hejunchao Co-authored-by: HeJunchao100813 Co-authored-by: Curio Yang --- tests/caffe_test_runner.py | 15 ++++++ tests/compare_util.py | 50 +++++++++++++++++-- tests/evaluator.py | 15 ++++++ tests/generator.py | 15 ++++++ .../tflite_/basic/test_depthwise_conv2d.py | 4 +- tests/inference.py | 15 ++++++ tests/json2md.py | 15 ++++++ tests/kernels/test_cum_sum.cpp | 4 +- tests/kernels/test_range.cpp | 6 +-- tests/nuc_proxy.py | 15 ++++++ tests/onnx_test_runner.py | 15 ++++++ tests/preprocess_utils.py | 15 ++++++ tests/test_runner.py | 15 ++++++ tests/test_utils.py | 15 ++++++ tests/tflite_test_runner.py | 15 ++++++ 15 files changed, 219 insertions(+), 10 deletions(-) diff --git a/tests/caffe_test_runner.py b/tests/caffe_test_runner.py index 817253849f..a0fdfb4300 100644 --- a/tests/caffe_test_runner.py +++ b/tests/caffe_test_runner.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import caffe from test_runner import * import os diff --git a/tests/compare_util.py b/tests/compare_util.py index 598c46488c..5e1eb56ba0 100644 --- a/tests/compare_util.py +++ b/tests/compare_util.py @@ -1,4 +1,18 @@ -import enum +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import math import os import re @@ -11,7 +25,38 @@ def cosine(gt: np.ndarray, pred: np.ndarray, *args): - return (gt @ pred) / (np.linalg.norm(gt, 2) * np.linalg.norm(pred, 2)) + # remove the NaN values in the same location. + if np.isnan(gt).any() and np.isnan(pred).any(): + gt_mask = np.isnan(gt) + pred_mask = np.isnan(pred) + mask = gt_mask & pred_mask + gt = gt[~mask] + pred = pred[~mask] + + # remove the INF values in the same location. + if np.isinf(gt).any() and np.isinf(pred).any(): + gt_mask = np.isinf(gt) + pred_mask = np.isinf(pred) + mask = gt_mask & pred_mask + gt = gt[~mask] + pred = pred[~mask] + + # return -1 if the nan/inf value is still in the array. + if np.isnan(gt).any() or np.isnan(pred).any() or np.isinf(gt).any() or np.isinf(pred).any(): + return -1 + + # exclude the situation of all zeros in array. + if compare_arrays(gt, pred): + return 1 + + result = (gt @ pred) / (np.linalg.norm(gt, 2) * np.linalg.norm(pred, 2)) + + # When tensor gt is a multiple of tensor pred, their similarity is also 1. + return -1 if math.isnan(result) else result + + +def compare_arrays(gt: np.ndarray, pred: np.ndarray): + return np.array_equal(gt, pred) def euclidean(gt: np.ndarray, pred: np.ndarray, *args): @@ -86,7 +131,6 @@ def compare_ndarray(expected: np.ndarray, threshold: float = 0.99, dump_hist: bool = True, dump_file: str = 'hist.csv') -> bool: - if expected.size == actual.size: similarity = similarity_func[similarity_name](expected.flatten(), actual.flatten()) else: diff --git a/tests/evaluator.py b/tests/evaluator.py index 149f098ab4..3b36b8abe8 100644 --- a/tests/evaluator.py +++ b/tests/evaluator.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + from typing import List, Dict, Union, Tuple import os import nncase diff --git a/tests/generator.py b/tests/generator.py index bcccb4ad4d..20d7bcd0a2 100644 --- a/tests/generator.py +++ b/tests/generator.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + from typing import Any, Dict, List, Tuple, Union import numpy as np import os diff --git a/tests/importer/tflite_/basic/test_depthwise_conv2d.py b/tests/importer/tflite_/basic/test_depthwise_conv2d.py index 1b416c5236..7521b6a380 100644 --- a/tests/importer/tflite_/basic/test_depthwise_conv2d.py +++ b/tests/importer/tflite_/basic/test_depthwise_conv2d.py @@ -59,7 +59,7 @@ def __call__(self, x): strides = [ [1, 1], [1, 3], - [5, 5] + # [5, 5] ] paddings = [ @@ -69,7 +69,7 @@ def __call__(self, x): dilations = [ [1, 1], - # [2, 2] there is a bug in tf.nn.depthwise_conv2d that produces incorrect output shape + # [2, 2] there is a bug in tf.nn.depthwise_conv2d that produces incorrect output shape. ] diff --git a/tests/inference.py b/tests/inference.py index fdf42925e3..4585ba89a9 100644 --- a/tests/inference.py +++ b/tests/inference.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + from typing import List, Dict, Union, Tuple import os import nncase diff --git a/tests/json2md.py b/tests/json2md.py index a1c2ac01b9..af99050a2a 100644 --- a/tests/json2md.py +++ b/tests/json2md.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import argparse import json import os diff --git a/tests/kernels/test_cum_sum.cpp b/tests/kernels/test_cum_sum.cpp index e6a287e32c..18cf05837f 100644 --- a/tests/kernels/test_cum_sum.cpp +++ b/tests/kernels/test_cum_sum.cpp @@ -73,13 +73,13 @@ TEST_P(CumSumTest, cum_sum) { .expect("create tensor failed"); // actual - float_t exclusive[] = {0}; + float exclusive[] = {0}; auto exclusive_ptr = hrt::create(nncase::dt_float32, {1}, {reinterpret_cast(exclusive), sizeof(exclusive)}, true, host_runtime_tensor::pool_cpu_only) .expect("create tensor failed"); - float_t reverse[] = {0}; + float reverse[] = {0}; auto reverse_ptr = hrt::create(nncase::dt_float32, {1}, {reinterpret_cast(reverse), sizeof(reverse)}, diff --git a/tests/kernels/test_range.cpp b/tests/kernels/test_range.cpp index b9574a0c05..bc9ba8216e 100644 --- a/tests/kernels/test_range.cpp +++ b/tests/kernels/test_range.cpp @@ -40,21 +40,21 @@ class RangeTest : public KernelTest, auto step_value = GetFloatNumber("step"); auto typecode = GetDataType("lhs_type"); - float_t begin_array[] = {begin_value}; + float begin_array[] = {begin_value}; begin = hrt::create(typecode, shape, {reinterpret_cast(begin_array), sizeof(begin_array)}, true, host_runtime_tensor::pool_cpu_only) .expect("create tensor failed"); - float_t end_array[] = {end_value}; + float end_array[] = {end_value}; end = hrt::create( typecode, shape, {reinterpret_cast(end_array), sizeof(end_array)}, true, host_runtime_tensor::pool_cpu_only) .expect("create tensor failed"); - float_t step_array[] = {step_value}; + float step_array[] = {step_value}; step = hrt::create(typecode, shape, {reinterpret_cast(step_array), sizeof(step_array)}, diff --git a/tests/nuc_proxy.py b/tests/nuc_proxy.py index c65e06b8d2..bdc647ef68 100644 --- a/tests/nuc_proxy.py +++ b/tests/nuc_proxy.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import os import argparse import stat diff --git a/tests/onnx_test_runner.py b/tests/onnx_test_runner.py index acbd16ed9a..d1299bb234 100644 --- a/tests/onnx_test_runner.py +++ b/tests/onnx_test_runner.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + from onnx import version_converter, helper import onnxsim import onnxruntime as ort diff --git a/tests/preprocess_utils.py b/tests/preprocess_utils.py index 174e09132c..0ece47e4cd 100644 --- a/tests/preprocess_utils.py +++ b/tests/preprocess_utils.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + def get_source_transpose_index(perm): """ transpose model output with postprocess to framework output diff --git a/tests/test_runner.py b/tests/test_runner.py index 9c2ba46135..60948ac2f2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import copy import os import re diff --git a/tests/test_utils.py b/tests/test_utils.py index c136800c48..0b0d5d20df 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import os import json import numpy as np diff --git a/tests/tflite_test_runner.py b/tests/tflite_test_runner.py index dffa694b72..59867442b1 100644 --- a/tests/tflite_test_runner.py +++ b/tests/tflite_test_runner.py @@ -1,3 +1,18 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + import tensorflow as tf from test_runner import * import os From 6e77141a51936a7e081b1c5a6ea339c21fb7763c Mon Sep 17 00:00:00 2001 From: huochenghai Date: Mon, 23 Oct 2023 15:31:17 +0800 Subject: [PATCH 02/13] fix new linked section (#1108) --- modules/Nncase.Modules.K210/CodeGen/K210/KPULinkedModule.cs | 4 ++-- .../CodeGen/StackVM/StackVMLinkedModule.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/Nncase.Modules.K210/CodeGen/K210/KPULinkedModule.cs b/modules/Nncase.Modules.K210/CodeGen/K210/KPULinkedModule.cs index 7018e7c7bd..4c4aeb20da 100644 --- a/modules/Nncase.Modules.K210/CodeGen/K210/KPULinkedModule.cs +++ b/modules/Nncase.Modules.K210/CodeGen/K210/KPULinkedModule.cs @@ -17,8 +17,8 @@ public KPULinkedModule(IReadOnlyList functions, byte[] text, byt Functions = functions; Sections = new[] { - new LinkedSection(text, ".text", 0, 8, (uint)text.Length), - new LinkedSection(rdata, ".rdata", 0, 8, (uint)(rdata?.Length ?? 0)), + new LinkedSection(text, ".text", 0, 8, (ulong)text.Length), + new LinkedSection(rdata, ".rdata", 0, 8, (ulong)(rdata?.Length ?? 0)), }; } diff --git a/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMLinkedModule.cs b/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMLinkedModule.cs index 3bfb985fd8..93c8d9c6b6 100644 --- a/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMLinkedModule.cs +++ b/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMLinkedModule.cs @@ -17,9 +17,9 @@ public StackVMLinkedModule(IReadOnlyList functions, Stream text, Functions = functions; Sections = new[] { - new LinkedSection(text, ".text", 0, 8, (uint)text.Length), - new LinkedSection(rdata, ".rdata", 0, 8, (uint)(rdata?.Length ?? 0)), - new LinkedSection(custom_calls, ".custom_calls", 0, 8, (uint)(custom_calls?.Length ?? 0)), + new LinkedSection(text, ".text", 0, 8, (ulong)text.Length), + new LinkedSection(rdata, ".rdata", 0, 8, (ulong)(rdata?.Length ?? 0)), + new LinkedSection(custom_calls, ".custom_calls", 0, 8, (ulong)(custom_calls?.Length ?? 0)), }; } From 3a62f9810559e6138475a9a5af4ab43b1b0c1e65 Mon Sep 17 00:00:00 2001 From: HeJunchao <73169088+HeJunchao100813@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:14:47 +0800 Subject: [PATCH 03/13] fix onnximport's dt (#1103) * fix dt * fix * fix Reduce.cs --------- Co-authored-by: hejunchao --- src/Nncase.Importer/Onnx/Reduce.cs | 10 ++++++++-- src/Nncase.Importer/Onnx/ReduceWindow2D.cs | 2 +- tests/importer/onnx_/basic/test_reduce.py | 16 +++++++++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/Nncase.Importer/Onnx/Reduce.cs b/src/Nncase.Importer/Onnx/Reduce.cs index e3baa6822e..63b8432175 100644 --- a/src/Nncase.Importer/Onnx/Reduce.cs +++ b/src/Nncase.Importer/Onnx/Reduce.cs @@ -11,12 +11,12 @@ namespace Nncase.Importer { public partial class OnnxImporter { - private Expr VisitReduce(in NodeProto op, ReduceOp reduceOp, float initValue) + private Expr VisitReduce(in NodeProto op, ReduceOp reduceOp, Expr initValue) { return ReduceCore(op, reduceOp, initValue, expr => expr); } - private Expr ReduceCore(in NodeProto op, ReduceOp reduceOp, float initValue, Func f) + private Expr ReduceCore(in NodeProto op, ReduceOp reduceOp, Expr initValue, Func f) { var input = GetInputExpr(op, 0); Expr axis; @@ -51,6 +51,12 @@ private Expr ReduceCore(in NodeProto op, ReduceOp reduceOp, float initValue, Fun var x when x == ReduceOp.Max && input.CheckedDataType == DataTypes.Int32 => F.Tensors.Reduce(reduceOp, f(input), axis, int.MinValue, keepDims), var x when x == ReduceOp.Min && input.CheckedDataType == DataTypes.Int64 => F.Tensors.Reduce(reduceOp, f(input), axis, long.MaxValue, keepDims), var x when x == ReduceOp.Min && input.CheckedDataType == DataTypes.Int32 => F.Tensors.Reduce(reduceOp, f(input), axis, int.MaxValue, keepDims), + var x when x == ReduceOp.Max && input.CheckedDataType == DataTypes.Float32 => F.Tensors.Reduce(reduceOp, f(input), axis, float.MinValue, keepDims), + var x when x == ReduceOp.Max && input.CheckedDataType == DataTypes.Float16 => F.Tensors.Reduce(reduceOp, f(input), axis, Half.MinValue, keepDims), + var x when x == ReduceOp.Max && input.CheckedDataType == DataTypes.BFloat16 => F.Tensors.Reduce(reduceOp, f(input), axis, BFloat16.RoundToBFloat16(float.MinValue), keepDims), + var x when x == ReduceOp.Min && input.CheckedDataType == DataTypes.Float32 => F.Tensors.Reduce(reduceOp, f(input), axis, float.MaxValue, keepDims), + var x when x == ReduceOp.Min && input.CheckedDataType == DataTypes.Float16 => F.Tensors.Reduce(reduceOp, f(input), axis, Half.MaxValue, keepDims), + var x when x == ReduceOp.Min && input.CheckedDataType == DataTypes.BFloat16 => F.Tensors.Reduce(reduceOp, f(input), axis, BFloat16.RoundToBFloat16(float.MaxValue), keepDims), _ => F.Tensors.Reduce(reduceOp, f(input), axis, F.Tensors.Cast(initValue, input.CheckedDataType), keepDims), }; } diff --git a/src/Nncase.Importer/Onnx/ReduceWindow2D.cs b/src/Nncase.Importer/Onnx/ReduceWindow2D.cs index 07273a38ae..25bcb23873 100644 --- a/src/Nncase.Importer/Onnx/ReduceWindow2D.cs +++ b/src/Nncase.Importer/Onnx/ReduceWindow2D.cs @@ -14,7 +14,7 @@ namespace Nncase.Importer public partial class OnnxImporter { // isGlobal used for GlobalXXXPool - private Expr VisitReduceWindow2D(in NodeProto op, ReduceOp reduceOp, float initValue, bool isGlobal = false) + private Expr VisitReduceWindow2D(in NodeProto op, ReduceOp reduceOp, Expr initValue, bool isGlobal = false) { // auto_pad had been DEPRECATED var input = GetInputExpr(op, 0); diff --git a/tests/importer/onnx_/basic/test_reduce.py b/tests/importer/onnx_/basic/test_reduce.py index 2404899474..879d7f9a7b 100644 --- a/tests/importer/onnx_/basic/test_reduce.py +++ b/tests/importer/onnx_/basic/test_reduce.py @@ -22,7 +22,7 @@ import numpy as np -def _make_module(in_shape, reduce_op, axes, keepdims, op_version): +def _make_module(in_shape, in_datatype, reduce_op, axes, keepdims, op_version): inputs = [] outputs = [] initializers = [] @@ -30,14 +30,14 @@ def _make_module(in_shape, reduce_op, axes, keepdims, op_version): nodes = [] # input - input = helper.make_tensor_value_info('input', TensorProto.FLOAT, in_shape) + input = helper.make_tensor_value_info('input', in_datatype, in_shape) inputs.append('input') # output kd = 1 if keepdims is None else keepdims data = np.ones(in_shape) out_shape = np.prod(data, axis=tuple(axes), keepdims=kd).shape - output = helper.make_tensor_value_info('output', TensorProto.FLOAT, out_shape) + output = helper.make_tensor_value_info('output', in_datatype, out_shape) outputs.append('output') # axes @@ -73,6 +73,11 @@ def _make_module(in_shape, reduce_op, axes, keepdims, op_version): [1, 3, 16, 16] ] +in_datatypes = [ + TensorProto.FLOAT, + TensorProto.FLOAT16 +] + reduce_ops = [ 'ReduceMax', 'ReduceMean', @@ -108,13 +113,14 @@ def _make_module(in_shape, reduce_op, axes, keepdims, op_version): @pytest.mark.parametrize('in_shape', in_shapes) +@pytest.mark.parametrize('in_datatype', in_datatypes) @pytest.mark.parametrize('reduce_op', reduce_ops) @pytest.mark.parametrize('axes', axes_list) @pytest.mark.parametrize('keepdims', keepdims_lists) @pytest.mark.parametrize('op_version', op_version_lists) -def test_reduce(in_shape, reduce_op, axes, keepdims, request, op_version): +def test_reduce(in_shape, in_datatype, reduce_op, axes, keepdims, request, op_version): if len(axes) <= len(in_shape): - model_def = _make_module(in_shape, reduce_op, axes, keepdims, op_version) + model_def = _make_module(in_shape, in_datatype, reduce_op, axes, keepdims, op_version) runner = OnnxTestRunner(request.node.name) model_file = runner.from_onnx_helper(model_def) From 355b8c42e62969066a9d5d15c1a141577a53265b Mon Sep 17 00:00:00 2001 From: HeJunchao <73169088+HeJunchao100813@users.noreply.github.com> Date: Tue, 24 Oct 2023 13:25:12 +0800 Subject: [PATCH 04/13] add mse (#1110) * add mse * fix mse * fix compare_util * fix compare_util --------- Co-authored-by: hejunchao Co-authored-by: HeJunchao100813 --- tests/compare_util.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/compare_util.py b/tests/compare_util.py index 5e1eb56ba0..7d5a300118 100644 --- a/tests/compare_util.py +++ b/tests/compare_util.py @@ -51,7 +51,6 @@ def cosine(gt: np.ndarray, pred: np.ndarray, *args): result = (gt @ pred) / (np.linalg.norm(gt, 2) * np.linalg.norm(pred, 2)) - # When tensor gt is a multiple of tensor pred, their similarity is also 1. return -1 if math.isnan(result) else result @@ -63,6 +62,29 @@ def euclidean(gt: np.ndarray, pred: np.ndarray, *args): return np.linalg.norm(gt.reshape(-1) - pred.reshape(-1)) +# def mse(gt: np.ndarray, pred: np.ndarray, *args): +# return np.mean((gt - pred) ** 2) + +def divide(gt: np.ndarray, pred: np.ndarray): + + # remove the zero values in the same location. + gt_mask = np.equal(gt, 0) + pred_mask = np.equal(pred, 0) + mask = gt_mask & pred_mask + gt = gt[~mask] + pred = pred[~mask] + + # to avoid divide zero. + pred = np.where(np.equal(pred, 0), 1e-7, pred) + + result = np.divide(gt, pred) + return result + + +def mean(gt: np.ndarray): + return np.mean(gt) + + def allclose(gt: np.ndarray, pred: np.ndarray, thresh: float): return np.allclose(gt, pred, atol=thresh) @@ -122,6 +144,8 @@ def compare_binfile(result_path: Tuple[str, str], compare_op = gt if compare_op(similarity, threshold): return False, similarity_info + if (mean(divide(gt_arr, pred_arr)) > 1.5 or mean(divide(gt_arr, pred_arr)) < 0.6): + return False, similarity_info, f"\nmaybe a case of multiples" return True, similarity_info From 9a7268c029c9533c22ea4f9bf61dd5ff03111360 Mon Sep 17 00:00:00 2001 From: zhangyang2057 Date: Fri, 27 Oct 2023 09:42:27 +0800 Subject: [PATCH 05/13] Upgrade vulkan version from 1.2.182.0 to 1.3.268.0 (#1112) * Upgrade vulkan version from 1.2.182.0 to 1.3.268.0 * Add missing compiler-python-release.yml. * Add missing pyproject.toml. * Update latest package suffix. * Disable Vulkan on Windows for "Process completed with exit code 1." --- .github/workflows/compiler-build.yml | 10 +++++----- .github/workflows/compiler-python-release.yml | 2 +- pyproject.toml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/compiler-build.yml b/.github/workflows/compiler-build.yml index ab9877f341..4806a15baf 100644 --- a/.github/workflows/compiler-build.yml +++ b/.github/workflows/compiler-build.yml @@ -172,7 +172,7 @@ jobs: - {name: x86_64-windows, os: windows-latest, shell: bash} env: - VULKANSDK_VER: 1.2.182.0 + VULKANSDK_VER: 1.3.268.0 steps: - uses: actions/checkout@v3 @@ -211,8 +211,8 @@ jobs: - name: Set up test environment (Linux) run: | - wget https://sdk.lunarg.com/sdk/download/${VULKANSDK_VER}/linux/vulkansdk-linux-x86_64-${VULKANSDK_VER}.tar.gz -O vulkansdk.tar.gz - tar xf vulkansdk.tar.gz + wget https://sdk.lunarg.com/sdk/download/${VULKANSDK_VER}/linux/vulkansdk-linux-x86_64-${VULKANSDK_VER}.tar.xz -O vulkansdk.tar.xz + tar xf vulkansdk.tar.xz sudo cp -P ${VULKANSDK_VER}/x86_64/lib/libvulkan.so* /usr/local/lib/ wget https://github.com/sunnycase/swiftshader/releases/download/v1.0/swiftshader-ubuntu-18.04-x86_64.zip -O swiftshader.zip unzip swiftshader.zip @@ -225,8 +225,8 @@ jobs: - name: Set up test environment (Windows) shell: pwsh run: | - Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/${env:VULKANSDK_VER}/windows/VulkanSDK-${env:VULKANSDK_VER}-Installer.exe -O VulkanSDK-Installer.exe - .\VulkanSDK-Installer.exe /S + # Invoke-WebRequest -Uri https://sdk.lunarg.com/sdk/download/${env:VULKANSDK_VER}/windows/VulkanSDK-${env:VULKANSDK_VER}-Installer.exe -O VulkanSDK-Installer.exe + # .\VulkanSDK-Installer.exe /S Invoke-WebRequest -Uri https://github.com/sunnycase/swiftshader/releases/download/v1.0/swiftshader-windows-2019-x86_64.zip -OutFile swiftshader.zip Expand-Archive swiftshader.zip Copy-Item swiftshader\lib\vk_swiftshader_icd.json swiftshader\bin\ diff --git a/.github/workflows/compiler-python-release.yml b/.github/workflows/compiler-python-release.yml index 9176290bd7..5e0db927a0 100644 --- a/.github/workflows/compiler-python-release.yml +++ b/.github/workflows/compiler-python-release.yml @@ -58,7 +58,7 @@ jobs: - {name: x86_64-windows, os: windows-latest, arch: x64} env: - VULKANSDK_VER: 1.2.182.0 + VULKANSDK_VER: 1.3.268.0 steps: - uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index 04a03089f8..a8dc5783a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,9 @@ before-all = [ "pip install conan==1.59", "conan profile new default --detect", "conan profile update settings.compiler.libcxx=libstdc++11 default", - "curl -L https://sdk.lunarg.com/sdk/download/1.2.182.0/linux/vulkansdk-linux-x86_64-1.2.182.0.tar.gz --output vulkansdk.tar.gz", - "tar xf vulkansdk.tar.gz", - "cp -P 1.2.182.0/x86_64/lib/libvulkan.so* /usr/local/lib/" + "curl -L https://sdk.lunarg.com/sdk/download/1.3.268.0/linux/vulkansdk-linux-x86_64-1.3.268.0.tar.xz --output vulkansdk.tar.xz", + "tar xf vulkansdk.tar.xz", + "cp -P 1.3.268.0/x86_64/lib/libvulkan.so* /usr/local/lib/" ] before-build = "pip install auditwheel" repair-wheel-command = "LD_LIBRARY_PATH=/usr/lib64 auditwheel repair -w {dest_dir} {wheel} --exclude libvulkan.so.1,libgomp.so.1" From ef3d74fcb7d2a4b4989925bc7bb7310d2e8a99d9 Mon Sep 17 00:00:00 2001 From: uranus0515 <110005227+uranus0515@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:09:47 +0800 Subject: [PATCH 06/13] Feature/egraph fakepass (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add cli preprocess options/fix pattern match utility * fix egraph * add driver in run pass context * fix muli branch equal enode extract * revert to history version * change op name to output tensor name * fallback to old unit test * change for algo quant tools * add pass to test match * revise typo for BroadcastInputMarker * add AutoSetupTestMethod for pad test * add AutoSetupTestMethod for squeeze * add AutoSetupTestMethod to some unittest * change order for calibration test * change vulkansdk-linux version from 182 to 189 --------- Co-authored-by: 郑启航 <597323109@qq.com> Co-authored-by: guodongliang --- src/Nncase.Cli/Commands/Compile.cs | 77 +++++++++++++++++ src/Nncase.Core/Passes/RunPassContext.cs | 5 ++ src/Nncase.Core/PatternMatch/IMatchResult.cs | 7 ++ src/Nncase.Core/PatternMatch/MatchResult.cs | 3 + src/Nncase.Core/Utilities/ReplaceUtility.cs | 12 +-- src/Nncase.EGraph/Passes/RewriteProvider.cs | 10 +-- src/Nncase.Importer/Onnx/Conv2D.cs | 2 +- src/Nncase.Importer/Onnx/MatMul.cs | 2 +- src/Nncase.Importer/TFLite/Conv2DTranspose.cs | 2 +- src/Nncase.Importer/TFLite/MatMul.cs | 4 +- src/Nncase.Passes/PassManager.cs | 1 + .../Rules/Lower/BroadcastMarker.cs | 67 ++++++++------- .../Rules/Neutral/CombineQuantize.cs | 84 +++++++++---------- .../Rules/ShapeBucket/ShapeBucket.cs | 8 +- .../PytestCalibrationDatasetProvider.cs | 8 +- .../Quantization/Quantizer.cs | 25 ++++++ .../TransformTestBase.cs | 6 +- ...nitTestPytestCalibrationDatasetProvider.cs | 4 +- src/Nncase.Tests/Rewrite/RewriteBase.cs | 32 +++++++ .../Rewrite/UnitTestEGraphRewriteFactory.cs | 1 + .../Neutral/UnitTestBatchNormToBinary.cs | 2 + .../Rules/Neutral/UnitTestCombineUnary.cs | 2 + .../Rules/Neutral/UnitTestExpandToBinary.cs | 2 + .../Rules/Neutral/UnitTestFlattenToReshape.cs | 2 + .../Rules/Neutral/UnitTestFoldBinary.cs | 2 + .../Rules/Neutral/UnitTestFoldCast.cs | 2 + .../Rules/Neutral/UnitTestFoldClamp.cs | 2 + .../Rules/Neutral/UnitTestFoldPad.cs | 2 + .../Rules/Neutral/UnitTestFoldQuant.cs | 2 + .../Rules/Neutral/UnitTestFoldReduces.cs | 2 + .../Rules/Neutral/UnitTestFoldReshape.cs | 2 + .../Neutral/UnitTestReshapeBatchMatmul.cs | 1 + .../Rules/Neutral/UnitTestSimplifyBinary.cs | 2 + .../Neutral/UnitTestSpaceToBatchTransform.cs | 2 + .../Rules/Neutral/UnitTestSqueezeToReshape.cs | 2 + .../Neutral/UnitTestSqueezeTransposeShape.cs | 1 + .../Neutral/UnitTestUnSqueezeToReshape.cs | 2 + 37 files changed, 287 insertions(+), 105 deletions(-) diff --git a/src/Nncase.Cli/Commands/Compile.cs b/src/Nncase.Cli/Commands/Compile.cs index 69a8cf0c54..edb64f2b3e 100644 --- a/src/Nncase.Cli/Commands/Compile.cs +++ b/src/Nncase.Cli/Commands/Compile.cs @@ -86,6 +86,50 @@ public Compile() alias: "--calib-method", description: $"model quant options, default is {Quantization.CalibMethod.Kld}", getDefaultValue: () => Quantization.CalibMethod.Kld)); + AddOption(new Option( + alias: "--pre-process", + description: "whether enable pre process, default is False", + getDefaultValue: () => false)); + AddOption(new Option( + alias: "--input-layout", + description: "the model input data layout, default is empty. eg. NCHW/NHWC", + getDefaultValue: () => string.Empty)); + AddOption(new Option( + alias: "--output-layout", + description: "the model output data layout, default is empty. eg. NCHW/NHWC", + getDefaultValue: () => string.Empty)); + AddOption(new Option( + alias: "--input-type", + description: "the model input data value type, default is Float32", + getDefaultValue: () => InputType.Float32)); + AddOption(new Option>( + alias: "--input-shape", + description: "the model input data shape, default is []. eg. `--input-shape 1 2 3 4`", + getDefaultValue: () => Array.Empty())); + AddOption(new Option>( + alias: "--input-range", + description: "the model input data value range, default is []. eg `--input-range -100.3 200.4`", + getDefaultValue: () => Array.Empty())); + AddOption(new Option( + alias: "--swap-rb", + description: "whether swap the model input data channel R and B", + getDefaultValue: () => false)); + AddOption(new Option( + alias: "--letter-box-value", + description: "letterbox value, default 0.0", + getDefaultValue: () => 0.0f)); + AddOption(new Option>( + alias: "--mean", + description: "the model input data mean, default []", + getDefaultValue: () => Array.Empty())); + AddOption(new Option>( + alias: "--std", + description: "the model input data std, default []", + getDefaultValue: () => Array.Empty())); + AddOption(new Option( + alias: "--model-layout", + description: "the model's input layout, default is empty. eg. NCHW/NHWC", + getDefaultValue: () => string.Empty)); AddOption(new Option( alias: "--benchmark-only", description: $"benchmark only", @@ -142,6 +186,17 @@ private async Task RunAsync(CliCompileOptions cliOptions, IHost host) }, ModelQuantMode = cliOptions.ModelQuantMode, }, + PreProcess = cliOptions.PreProcess, + InputLayout = cliOptions.InputLayout, + OutputLayout = cliOptions.OutputLayout, + InputType = cliOptions.InputType, + InputShape = cliOptions.InputShape.ToArray(), + InputRange = cliOptions.InputRange.ToArray(), + SwapRB = cliOptions.SwapRB, + LetterBoxValue = cliOptions.LetterBoxValue, + Mean = cliOptions.Mean.ToArray(), + Std = cliOptions.Std.ToArray(), + ModelLayout = cliOptions.ModelLayout, IsBenchmarkOnly = cliOptions.BenchmarkOnly, }; @@ -209,6 +264,28 @@ internal sealed class CliCompileOptions public DatasetFormat DatasetFormat { get; set; } public bool BenchmarkOnly { get; set; } + + public bool PreProcess { get; set; } + + public string InputLayout { get; set; } + + public string OutputLayout { get; set; } + + public InputType InputType { get; set; } + + public List InputShape { get; set; } + + public List InputRange { get; set; } + + public bool SwapRB { get; set; } + + public float LetterBoxValue { get; set; } + + public List Mean { get; set; } + + public List Std { get; set; } + + public string ModelLayout { get; set; } } #pragma warning restore CS8618 diff --git a/src/Nncase.Core/Passes/RunPassContext.cs b/src/Nncase.Core/Passes/RunPassContext.cs index 52becdcda0..4ecaf68aaa 100644 --- a/src/Nncase.Core/Passes/RunPassContext.cs +++ b/src/Nncase.Core/Passes/RunPassContext.cs @@ -36,6 +36,11 @@ public record RunPassContext /// public int Index { get; set; } + /// + /// Gets this pass's driver. + /// + public IPass? Driver { get; init; } + /// /// Gets or sets a value indicating whether control rewrite once or not. /// when RewriteOnce is true, the rule will only apply once, then restart rewrite from first rule. diff --git a/src/Nncase.Core/PatternMatch/IMatchResult.cs b/src/Nncase.Core/PatternMatch/IMatchResult.cs index f6f9446b9e..8f6118d4df 100644 --- a/src/Nncase.Core/PatternMatch/IMatchResult.cs +++ b/src/Nncase.Core/PatternMatch/IMatchResult.cs @@ -30,6 +30,13 @@ public interface IMatchResult : IEnumerable> /// Match result. object this[IPattern pattern] { get; } + /// + /// Get match result by name, default is null. + /// + /// Pattern name. + /// match result. + object GetValueOrDefault(string name); + /// /// Get match result by pattern. /// diff --git a/src/Nncase.Core/PatternMatch/MatchResult.cs b/src/Nncase.Core/PatternMatch/MatchResult.cs index c03ab84d1b..eb1bb1f651 100644 --- a/src/Nncase.Core/PatternMatch/MatchResult.cs +++ b/src/Nncase.Core/PatternMatch/MatchResult.cs @@ -43,6 +43,9 @@ where kv.Key.Name is not null /// public object this[string name] => _stringMap[name]; + /// + public object GetValueOrDefault(string name) => _stringMap.GetValueOrDefault(name, null!); + /// public IEnumerator> GetEnumerator() { diff --git a/src/Nncase.Core/Utilities/ReplaceUtility.cs b/src/Nncase.Core/Utilities/ReplaceUtility.cs index c07cd9850e..fa1331f80b 100644 --- a/src/Nncase.Core/Utilities/ReplaceUtility.cs +++ b/src/Nncase.Core/Utilities/ReplaceUtility.cs @@ -97,11 +97,6 @@ public static Call ReplaceCallParams(Expr target, IReadOnlyList oldParams, return new Call(target, ReplaceItems(oldParams, pairs)); } - public static Call ReplaceCallParams(Call call, params (int, Expr)[] pairs) - { - return new Call(call.Target, ReplaceItems(call.Arguments.ToArray(), pairs)); - } - /// /// replace the call params with parameter info. /// @@ -122,12 +117,7 @@ public static Call ReplaceCallParams(Expr target, IReadOnlyList oldParams, /// expr. /// new Call. public static Call ReplaceCallFirstParam(Expr target, IReadOnlyList oldParams, Expr expr) => - ReplaceCallParams(target, oldParams, (oldParams[0], expr)); - - public static Expr ReplaceCallFirstParam(Call call, Expr expr) - { - return ReplaceCallFirstParam(call.Target, call.Arguments.ToArray(), expr); - } + ReplaceCallParams(target, oldParams, (0, expr)); /// /// Replace target in body with expr. diff --git a/src/Nncase.EGraph/Passes/RewriteProvider.cs b/src/Nncase.EGraph/Passes/RewriteProvider.cs index fa4558226a..7642a5395e 100644 --- a/src/Nncase.EGraph/Passes/RewriteProvider.cs +++ b/src/Nncase.EGraph/Passes/RewriteProvider.cs @@ -43,8 +43,6 @@ public Expr ERewrite(Expr expr, IEnumerable rules, RunPassContext public IEGraph ERewrite(IEGraph eGraph, IEnumerable rules, RunPassContext context) { var last_version = eGraph.Version; - int count = 0; - while (true) { var matches = rules. @@ -59,10 +57,12 @@ public IEGraph ERewrite(IEGraph eGraph, IEnumerable rules, RunPass if (DumpScope.Current.IsEnabled(DumpFlags.Rewrite)) { - foreach (var (rule, results) in matches.Where(p => p.Item2.Count != 0)) + using var fs = DumpScope.Current.OpenFile(Path.Combine("Matches", $"V{eGraph.Version}.txt")); + using var writer = new StreamWriter(fs); + writer.WriteLine("rule, results"); + foreach (var (rule, results) in matches) { - using var fs = DumpScope.Current.OpenFile(Path.Combine("Matches", $"V{eGraph.Version}_{count++}_{rule.GetType().Name}.dot")); - EGraphPrinter.DumpEgraphAsDot(eGraph, results, fs); + writer.WriteLine($"{rule.GetType().Name}, {results.Count}"); } } diff --git a/src/Nncase.Importer/Onnx/Conv2D.cs b/src/Nncase.Importer/Onnx/Conv2D.cs index e82002d0ee..5d66995704 100644 --- a/src/Nncase.Importer/Onnx/Conv2D.cs +++ b/src/Nncase.Importer/Onnx/Conv2D.cs @@ -39,7 +39,7 @@ private Expr VisitConv2D(in NodeProto op) var pads = AutoPad(op, autoPad, input, weights, strides.ToArray(), dilation.ToArray(), isConv1D); pads.InferenceType(); var conv = F.NN.Conv2D(input, weights, bias, strides.ToArray(), pads, dilation.ToArray(), PadMode.Constant, group); - List outputNames = new() { op.Name }; + List outputNames = new() { op.Output[0] }; conv.Metadata.OutputNames = outputNames; if (isConv1D) { diff --git a/src/Nncase.Importer/Onnx/MatMul.cs b/src/Nncase.Importer/Onnx/MatMul.cs index 8344d11b2e..5f7a354593 100644 --- a/src/Nncase.Importer/Onnx/MatMul.cs +++ b/src/Nncase.Importer/Onnx/MatMul.cs @@ -14,7 +14,7 @@ private Expr VisitMatMul(in NodeProto op) { var (a, b) = GetInputExprs(op, 0, 1); var matmul = IR.F.Math.MatMul(a, b); - List outputNames = new() { op.Name }; + List outputNames = new() { op.Output[0] }; matmul.Metadata.OutputNames = outputNames; return matmul; } diff --git a/src/Nncase.Importer/TFLite/Conv2DTranspose.cs b/src/Nncase.Importer/TFLite/Conv2DTranspose.cs index e096b9b8c8..688cfe1f0b 100644 --- a/src/Nncase.Importer/TFLite/Conv2DTranspose.cs +++ b/src/Nncase.Importer/TFLite/Conv2DTranspose.cs @@ -54,7 +54,7 @@ private Expr VisitConv2DTranspose(in tflite.Operator op) dilation, PadMode.Constant, 1); - List outputNames = new() { GetInputTensor(op, 0).Name }; + List outputNames = new() { GetOutputTensor(op, 0).Name }; conv2DTranspose.Metadata.OutputNames = outputNames; return F.Tensors.NCHWToNHWC(F.Math.Clamp( conv2DTranspose, diff --git a/src/Nncase.Importer/TFLite/MatMul.cs b/src/Nncase.Importer/TFLite/MatMul.cs index 0472cb6ac8..6b1724e5f5 100644 --- a/src/Nncase.Importer/TFLite/MatMul.cs +++ b/src/Nncase.Importer/TFLite/MatMul.cs @@ -66,10 +66,10 @@ private Expr VisitMatMul(in tflite.Operator op, bool isFullyConnected = true) : Expand(Cast(0, GetDataType(GetInputTensor(op, 0).Type)), new[] { otherTensor.Shape(0) }).Evaluate().AsTensor(); var matmul = MatMul(lhs, rhs); - List outputNames = new() { GetInputTensor(op, 0).Name + "_matmul" }; + List outputNames = new() { GetOutputTensor(op, 0).Name + "_matmul" }; matmul.Metadata.OutputNames = outputNames; outputNames.Clear(); - outputNames.Add(GetInputTensor(op, 0).Name + "_bias"); + outputNames.Add(GetOutputTensor(op, 0).Name); bias.Metadata.OutputNames = outputNames; var mm = matmul + bias; diff --git a/src/Nncase.Passes/PassManager.cs b/src/Nncase.Passes/PassManager.cs index df66312e53..9377511e50 100644 --- a/src/Nncase.Passes/PassManager.cs +++ b/src/Nncase.Passes/PassManager.cs @@ -287,6 +287,7 @@ public RunPassContextWithAnalysis(IAnalyzerManager analyzerManager, Either { - public override Pattern Pattern => IsCallWildcard( - "outer", - IsWildcard(), - InputPattern); + public override Pattern Pattern => IsCall("outer", IsWildcard("outerTarget"), IsVArgsRepeat("outerParams", exprs => + { + var patterns = new Pattern[exprs.Length]; + for (int i = 0; i < exprs.Length; i++) + { + patterns[i] = GetInputPattern(i); + } + + return patterns; + })); - public Pattern InputPattern => IsCallWildcard( - "call", - IsWildcard(), - IsRangeOfMarker( - "marker", - IsWildcard(), - IsWildcard())); + public Pattern GetInputPattern(int i) => + IsAlt( + IsCallWildcard( + $"input_{i}", + IsOp($"input_target_{i}", NotChangeRangeOp), + IsRangeOfMarker($"input_marker_{i}", IsWildcard($"marker_target_{i}"), IsWildcard($"marker_attribute_{i}"))), + IsWildcard($"input_{i}")); - public Expr? GetReplace(Call outer, Call call, Marker marker) + public Expr? GetReplace(Call outer, Expr outerTarget, IReadOnlyList outerParams, IMatchResult result) { - if (!NotChangeRangeOp(call.Target)) + if (!Enumerable.Range(0, outerParams.Count).Select(i => result.GetValueOrDefault($"input_marker_{i}")).Any(e => e is not null)) { return null; } - if (outer.Target is MatMul && CompilerServices.TryMatchRoot(outer.Arguments[1], InputPattern, new(), out var matchResult)) + var newArgs = new Expr[outerParams.Count]; + for (int i = 0; i < outerParams.Count; i++) { - var rhsMarker = (Marker)matchResult["marker"]; - var rhsCall = (Call)matchResult["call"]; - var lhs = marker.With(target: ReplaceCallFirstParam(call, marker)); - var rhs = rhsMarker.With(target: ReplaceCallFirstParam(rhsCall, rhsMarker)); - return ReplaceCallParams(outer, (0, lhs), (1, rhs)); + if (result.GetValueOrDefault($"input_marker_{i}") is Marker marker && result[$"marker_target_{i}"] is Expr target && result[$"marker_attribute_{i}"] is Expr range) + { + newArgs[i] = IR.F.Math.RangeOfMarker(outerParams[i], range).With(mixQuantInfo: marker.MixQuantInfo, adaQuantInfo: marker.AdaQuantInfo); + } + else + { + newArgs[i] = outerParams[i]; + } } - return ReplaceCallFirstParam(outer, marker.With(target: ReplaceCallFirstParam(call, marker))); + return new Call(outerTarget, newArgs); } } @@ -56,23 +68,18 @@ public partial class BroadcastOutputMarker : RewriteRule { public override Pattern Pattern => IsRangeOfMarker( "marker", - IsCallWildcard("input", IsWildcard(), IsCallWildcard(null, IsWildcard())), - IsWildcard()); + IsCallWildcard("output", IsOp("outputTarget", NotChangeRangeOp), IsCallWildcard("input", IsWildcard("inputTarget"))), + IsWildcard("range")); - public Expr? GetReplace(Call input, Marker marker) + public Expr? GetReplace(Marker marker, Expr range, Call output, Op outputTarget, IReadOnlyList outputParams) { - if (!NotChangeRangeOp(input.Target)) - { - return null; - } - - return ReplaceCallFirstParam(input, marker.With(target: input.Arguments[0])); + return ReplaceCallFirstParam(outputTarget, outputParams, IR.F.Math.RangeOfMarker(outputParams[0], range).With(adaQuantInfo: marker.AdaQuantInfo, mixQuantInfo: marker.MixQuantInfo)); } } internal static class BroadcastMarkerHelper { - public static bool NotChangeRangeOp(Expr op) + public static bool NotChangeRangeOp(Op op) { return op is Squeeze || op is Unsqueeze || op is Reshape || op is Broadcast; } diff --git a/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs b/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs index c065fe65c7..e046301045 100644 --- a/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs +++ b/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs @@ -40,14 +40,17 @@ public sealed partial class CombineQuantizeConcat : RewriteRule private Expr? GetReplace(Quantize quantize, IReadOnlyList tupleInputs, Expr axis, Expr quantParam, RunPassContext options) { - var userAnalysis = options.GetAnalysis(); - - // see UnitTestCombineQuantize.TestCombineQuantizeConcatNegative - foreach (var e in tupleInputs) + if (options.Driver is DataflowPass) { - if (userAnalysis[e].Count() > 1) + var userAnalysis = options.GetAnalysis(); + + // see UnitTestCombineQuantize.TestCombineQuantizeConcatNegative + foreach (var e in tupleInputs) { - return null; + if (userAnalysis[e].Count() > 1) + { + return null; + } } } @@ -61,49 +64,43 @@ public sealed partial class CombineQuantizeConcat : RewriteRule [RuleGenerator] public sealed partial class CombineQuantizeReshape : RewriteRule { - private readonly bool _checkShapeSize; - - public CombineQuantizeReshape() - { - _checkShapeSize = false; - } - /// /// Initializes a new instance of the class. /// /// if true, skip pass. - public CombineQuantizeReshape(bool checkShapeSize = false) + public CombineQuantizeReshape(bool checkShapeSize) { - _checkShapeSize = checkShapeSize; + Pattern = IsQuantize( + "quantize", + _ => true, + IsReshape( + "reshape", + "reshapeCall", + IsWildcard("input") with { TypePattern = HasShape(sp => !(checkShapeSize && sp.ToValueArray().Any(s => s >= 65536)), "CheckedShape") }, + IsWildcard("shape")), + IsWildcard("quantParam")); } - /// - public override Pattern Pattern { get; } = IsQuantize( - "quantize", - _ => true, - IsReshape( - "reshape", - "reshapeCall", - IsWildcard("input"), - IsWildcard("shape")), - IsWildcard("quantParam")); - - private Expr? GetReplace(Quantize quantize, Call reshapeCall, Expr input, Expr shape, Expr quantParam, RunPassContext options) + public CombineQuantizeReshape() + : this(false) { - var userAnalysis = options.GetAnalysis(); + } - if (userAnalysis[reshapeCall].Count() > 1) - { - return null; - } + /// + public override Pattern Pattern { get; } - if (_checkShapeSize && input.CheckedShape.ToValueArray().Any(s => s >= 65536)) + private Expr? GetReplace(Quantize quantize, Call reshapeCall, Expr input, Expr shape, Expr quantParam, RunPassContext context) + { + if (context.Driver is DataflowPass) { - return null; + var userAnalysis = context.GetAnalysis(); + if (userAnalysis[reshapeCall].Count() > 1) + { + return null; + } } var output = Reshape(Quantize(input, quantParam, quantize.TargetType), shape); - output.InferenceType(); return output; } } @@ -160,15 +157,18 @@ public sealed partial class CombineQuantizeTranspose : RewriteRule private Expr? GetReplace(Quantize quantize, Call transposeCall, Expr input, Expr perm, Expr quantParam, RunPassContext options) { - var userAnalysis = options.GetAnalysis(); - - if (userAnalysis[transposeCall].Count() > 1) + try + { + var userAnalysis = options.GetAnalysis(); + if (userAnalysis[transposeCall].Count() > 1) + { + return null; + } + } + catch (System.Exception) { - return null; } - var output = Transpose(Quantize(input, quantParam, quantize.TargetType), perm); - output.InferenceType(); - return output; + return Transpose(Quantize(input, quantParam, quantize.TargetType), perm); } } diff --git a/src/Nncase.Passes/Rules/ShapeBucket/ShapeBucket.cs b/src/Nncase.Passes/Rules/ShapeBucket/ShapeBucket.cs index 2eb9c0f39a..a0cf03380a 100644 --- a/src/Nncase.Passes/Rules/ShapeBucket/ShapeBucket.cs +++ b/src/Nncase.Passes/Rules/ShapeBucket/ShapeBucket.cs @@ -441,12 +441,14 @@ protected override Expr ReplaceVarsWithArg(Var[] fusionVars, Expr[] args, Expr n { var convTranspose = (Call)CallMarker!.Target; var c = ReplaceCallFirstParam( - convTranspose, + convTranspose.Target, + convTranspose.Arguments.ToArray(), _transposeInputMarker!.With(target: ReplaceCallFirstParam( - _transpose!, + _transpose!.Target, + _transpose!.Arguments.ToArray(), _transposeInputMarker.With(target: - ReplaceCallFirstParam(_originCall!, fusionVars[0]))))); + ReplaceCallFirstParam(_originCall!.Target, _originCall!.Arguments.ToArray(), fusionVars[0]))))); return CallMarker.With(target: base.ReplaceVarsWithArg(fusionVars, args, c)); } diff --git a/src/Nncase.Quantization/Quantization/PytestCalibrationDatasetProvider.cs b/src/Nncase.Quantization/Quantization/PytestCalibrationDatasetProvider.cs index 71f6c49bc3..66b040ec0f 100644 --- a/src/Nncase.Quantization/Quantization/PytestCalibrationDatasetProvider.cs +++ b/src/Nncase.Quantization/Quantization/PytestCalibrationDatasetProvider.cs @@ -99,8 +99,8 @@ private bool TryParseSample(string fileName, [System.Diagnostics.CodeAnalysis.Ma if (match.Success) { string name = match.Groups[1].Value; - int n = int.Parse(match.Groups[2].Value); - int i = int.Parse(match.Groups[3].Value); + int i = int.Parse(match.Groups[2].Value); + int n = int.Parse(match.Groups[3].Value); item = new(name, n, i); return true; } @@ -111,11 +111,11 @@ private bool TryParseSample(string fileName, [System.Diagnostics.CodeAnalysis.Ma private sealed record Sample(string Name, int Number, int InputIndex) { - public string FileName => $"{Name}_{Number}_{InputIndex}.bin"; + public string FileName => $"{Name}_{InputIndex}_{Number}.bin"; public int[] GetShape() { - using var stream = File.OpenRead($"{Name}_{Number}_{InputIndex}.txt"); + using var stream = File.OpenRead($"{Name}_{InputIndex}_{Number}.txt"); using var reader = new StreamReader(stream); var line = reader.ReadLine(); int[] shape = Array.Empty(); diff --git a/src/Nncase.Quantization/Quantization/Quantizer.cs b/src/Nncase.Quantization/Quantization/Quantizer.cs index af1f837e97..961ae8b716 100644 --- a/src/Nncase.Quantization/Quantization/Quantizer.cs +++ b/src/Nncase.Quantization/Quantization/Quantizer.cs @@ -417,10 +417,12 @@ private IDictionary[]> GetRangesFromConfig(QuantScheme foreach (var rangeOf in _rangeOfs) { + bool getRange = false; for (int i = 0; i < quantScheme!.Outputs!.Length; i++) { if (rangeOf.Expr.Metadata.OutputNames?[0] == quantScheme!.Outputs[i].Name) { + getRange = true; if (((RangeOf)((Call)rangeOf.Expr).Target).IsRangeOfWeight == true && quantScheme!.Outputs[i].DataRangeMode == "by_tensor") { var oc = ((Call)rangeOf.Expr).Operands[1].CheckedShape[0].FixedValue; @@ -457,6 +459,21 @@ private IDictionary[]> GetRangesFromConfig(QuantScheme } } } + + if (getRange == false && _quantizeOptions.QuantScheme != string.Empty && _quantizeOptions.QuantSchemeStrictMode == true) + { + if (((RangeOf)((Call)rangeOf.Expr).Target).IsRangeOfWeight == true) + { + var oc = ((Call)rangeOf.Expr).Operands[1].CheckedShape[0].FixedValue; + var valueRanges = new ValueRange[oc]; + ranges.Add(rangeOf, valueRanges); + } + else + { + var valueRanges = new ValueRange[1]; + ranges.Add(rangeOf, valueRanges); + } + } } return ranges; @@ -466,8 +483,10 @@ private void AssignDataTypeFromConfig(QuantScheme quantScheme) { foreach (var marker in _markers) { + bool getRange = false; for (int i = 0; i < quantScheme!.Outputs!.Length; i++) { + getRange = true; if (marker.Expr.Metadata.OutputNames?[0] == quantScheme.Outputs[i].Name) { var markerExpr = (Marker)marker.Expr; @@ -480,6 +499,12 @@ private void AssignDataTypeFromConfig(QuantScheme quantScheme) markerExpr.MixQuantInfo!.MarkerQuantType = dataType; } } + + if (getRange == false && _quantizeOptions.QuantScheme != string.Empty && _quantizeOptions.QuantSchemeStrictMode == true) + { + var markerExpr = (Marker)marker.Expr; + markerExpr.MixQuantInfo!.MarkerQuantType = DataTypes.Float16; + } } } diff --git a/src/Nncase.Tests.TestFixture/TransformTestBase.cs b/src/Nncase.Tests.TestFixture/TransformTestBase.cs index 9f1f8435c3..49083076d4 100644 --- a/src/Nncase.Tests.TestFixture/TransformTestBase.cs +++ b/src/Nncase.Tests.TestFixture/TransformTestBase.cs @@ -67,7 +67,7 @@ public Expr TestMatchedCore(Function pre, IReadOnlyDictionary? feed } var preHashCode = pre.GetHashCode(); - var post = (Function)CompilerServices.Rewrite(pre, rules, new() { AnalysisResults = analysis }); + var post = (Function)CompilerServices.Rewrite(pre, rules, new() { AnalysisResults = analysis, Driver = new DataflowPass() }); if (isNotMatch) { Assert.Equal(preHashCode, post.GetHashCode()); @@ -97,7 +97,7 @@ public Expr TestMatchedCore(Expr pre, IReadOnlyDictionary? feeds = var preHashCode = pre.GetHashCode(); var v1 = pre.Evaluate(feeds); - var post = CompilerServices.Rewrite(pre, rules, new()); + var post = CompilerServices.Rewrite(pre, rules, new() { Driver = new DataflowPass() }); Assert.NotEqual(preHashCode, post.GetHashCode()); var v2 = post.Evaluate(feeds); if (!Comparator.AllEqual(v1, v2)) @@ -112,7 +112,7 @@ public void TestNotMatch(Expr pre, params IRewriteRule[] rules) { pre.InferenceType(); var preHashCode = pre.GetHashCode(); - var post = CompilerServices.Rewrite(pre, rules, new()); + var post = CompilerServices.Rewrite(pre, rules, new() { Driver = new DataflowPass() }); Assert.Equal(preHashCode, post.GetHashCode()); } diff --git a/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs b/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs index 17ec26a05e..6d2f29ce71 100644 --- a/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs +++ b/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs @@ -110,9 +110,9 @@ private static void DumpTensors(Tensor[] tensorValue, string dir, int sample = 1 for (var t = 0; t < tensorValue.Length; t++) { var value = tensorValue[t]; - var sr1 = new StreamWriter(Path.Join(dir, $"input_{s}_{t}.txt")); + var sr1 = new StreamWriter(Path.Join(dir, $"input_{t}_{s}.txt")); DumpTxt(value, sr1); - var sr2 = Path.Join(dir, $"input_{s}_{t}.bin"); + var sr2 = Path.Join(dir, $"input_{t}_{s}.bin"); DumpBin(value, sr2); } } diff --git a/src/Nncase.Tests/Rewrite/RewriteBase.cs b/src/Nncase.Tests/Rewrite/RewriteBase.cs index 3fd4de87fe..33dc381532 100644 --- a/src/Nncase.Tests/Rewrite/RewriteBase.cs +++ b/src/Nncase.Tests/Rewrite/RewriteBase.cs @@ -2874,3 +2874,35 @@ public PReluTransposeCase() public Dictionary FeedDict { get; } } + +/// +/// egraph extract bad case. +/// +public sealed class FoldReshapeWithBranch : IRewriteCase +{ + public FoldReshapeWithBranch() + { + var v1070 = new Var(new TensorType(DataTypes.Float32, new[] { 1, 1, 2, 8400 })); + { + var v1071 = Unary(UnaryOp.Cos, v1070); // f32[1,1,2,8400] + var v1072 = Reshape(v1071, new[] { 1, 2, 8400 }); // f32[1,2,8400] + var v1073 = Reshape(v1072, new[] { 1, 1, 2, 8400 }); // f32[1,1,2,8400] + var v1078 = Unary(UnaryOp.Sin, v1073); // f32[1,1,2,8400] + var v1079 = Reshape(v1078, new[] { 1, 2, 8400 }); // f32[1,2,8400] + var v1080 = Sub(v1072, IR.F.Random.Normal(DataTypes.Float32, new[] { 1, 2, 8400 }).Evaluate().AsTensor()); // f32[1,2,8400] + var v1081 = new IR.Tuple(v1079, v1080); // (f32[1,2,8400], f32[1,2,8400]) + PreExpr = new Function(v1081, new[] { v1070 }); + } + + FeedDict = new() { { v1070, IR.F.Random.Normal(new[] { 1, 1, 2, 8400 }).Evaluate() } }; + } + + public Function PreExpr { get; } + + public IEnumerable Rules => new[] { + typeof(FoldNopReshape), + typeof(FoldTwoReshapes), + }; + + public Dictionary FeedDict { get; } +} diff --git a/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs b/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs index 61db6ea870..9a1d8faee7 100644 --- a/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs +++ b/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs @@ -111,6 +111,7 @@ public UnitTestEGraphRewriteFactory() new ResizeImageCase(), new ProdCase(), new MultiReshapeCase(), + new PReluTransposeCase(), }; [Theory] diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestBatchNormToBinary.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestBatchNormToBinary.cs index 061fd0c2eb..76e1899d31 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestBatchNormToBinary.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestBatchNormToBinary.cs @@ -15,6 +15,7 @@ using Nncase.Passes; using Nncase.Passes.Rules.Neutral; using Nncase.PatternMatch; +using Nncase.Tests.TestFixture; using Xunit; using static Nncase.IR.F.NN; using ITuple = Nncase.IR.ITuple; @@ -25,6 +26,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestBatchNormToBinary : TransformTestBase { public static readonly TheoryData BatchNormToBinaryPositiveData = new() diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs index 49a2a389cd..5ffd3c7eb9 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineUnary.cs @@ -15,6 +15,7 @@ using Nncase.Passes; using Nncase.Passes.Rules.Neutral; using Nncase.PatternMatch; +using Nncase.Tests.TestFixture; using Xunit; using static Nncase.IR.F.NN; using ITuple = Nncase.IR.ITuple; @@ -25,6 +26,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestCombineUnary : TransformTestBase { // TODO: CombinePadUnary diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestExpandToBinary.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestExpandToBinary.cs index edbd37a310..2e63db64b6 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestExpandToBinary.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestExpandToBinary.cs @@ -11,6 +11,7 @@ using Nncase.IR; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Dimension = Nncase.IR.Dimension; using Math = Nncase.IR.F.Math; @@ -19,6 +20,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestExpandToBroadcast : TransformTestBase { public static IEnumerable TestExpandToBroadcastPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFlattenToReshape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFlattenToReshape.cs index dc20de6667..e0a88f473e 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFlattenToReshape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFlattenToReshape.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; @@ -17,6 +18,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFlattenToReshape : TransformTestBase { public static IEnumerable TestFlattenToReshapePositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldBinary.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldBinary.cs index f46203635b..f05b366d18 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldBinary.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldBinary.cs @@ -14,12 +14,14 @@ using Nncase.IR.NN; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldBinary : TransformTestBase { public static IEnumerable TestFoldNopBinaryNegativeData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldCast.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldCast.cs index 54f6a59530..a5c32a88fb 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldCast.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldCast.cs @@ -11,11 +11,13 @@ using Nncase.IR.F; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldCast : TransformTestBase { public static IEnumerable TestFoldTwoCastsPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldClamp.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldClamp.cs index 72cb0deb35..3117c03360 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldClamp.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldClamp.cs @@ -11,12 +11,14 @@ using Nncase.IR.F; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldClamp : TransformTestBase { public static IEnumerable TestFoldNopClampPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldPad.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldPad.cs index 0b8e607def..a2aa51fa92 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldPad.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldPad.cs @@ -11,12 +11,14 @@ using Nncase.IR.F; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldPad : TransformTestBase { public static IEnumerable TestFoldNopPadPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldQuant.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldQuant.cs index ed8684b323..1488cbd89a 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldQuant.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldQuant.cs @@ -6,11 +6,13 @@ using Nncase.IR; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldQuant : TransformTestBase { public static TheoryData FoldQuantDequantData => new() diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReduces.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReduces.cs index ffe5ecb25a..0abebd3aa2 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReduces.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReduces.cs @@ -12,12 +12,14 @@ using Nncase.IR.F; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldReduce : TransformTestBase { public static IEnumerable TestFoldTwoReducesPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs index 2108420fa7..b4af452d8e 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs @@ -11,12 +11,14 @@ using Nncase.IR.F; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldReshape : TransformTestBase { public static IEnumerable TestFoldNopReshapePositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestReshapeBatchMatmul.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestReshapeBatchMatmul.cs index 066c1f3755..d2fdea0d34 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestReshapeBatchMatmul.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestReshapeBatchMatmul.cs @@ -20,6 +20,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestReshapeBatchMatmul : TransformTestBase { public static IEnumerable TestReshapeBatchMatmulPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestSimplifyBinary.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestSimplifyBinary.cs index 1a99065b8d..00abe1266b 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestSimplifyBinary.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestSimplifyBinary.cs @@ -14,12 +14,14 @@ using Nncase.IR.NN; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestSimplifyBinary : TransformTestBase { public static IEnumerable TestReassociateMulPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestSpaceToBatchTransform.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestSpaceToBatchTransform.cs index d814e4bcf5..4fa0dd0c0a 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestSpaceToBatchTransform.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestSpaceToBatchTransform.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using NN = Nncase.IR.F.NN; @@ -17,6 +18,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestSpaceToBatchToPad : TransformTestBase { public static IEnumerable TestSpaceToBatchToPadPositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeToReshape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeToReshape.cs index 9b137e93d8..ece1de55f1 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeToReshape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeToReshape.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; @@ -17,6 +18,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestSqueezeToReshape : TransformTestBase { public static IEnumerable TestSqueezeToReshapePositiveData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeTransposeShape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeTransposeShape.cs index 166cc666d4..fecb671000 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeTransposeShape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestSqueezeTransposeShape.cs @@ -62,6 +62,7 @@ public void TestSqueezeTransposeShapeNegative(int[] shape, int[] perm) } } +[AutoSetupTestMethod(InitSession = true)] public class UnitTestSqueezeBinaryShape : TransformTestBase { public static IEnumerable TestSqueezeBinaryShapePosivateData => diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestUnSqueezeToReshape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestUnSqueezeToReshape.cs index bef44a77cb..f6b4f9c9bb 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestUnSqueezeToReshape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestUnSqueezeToReshape.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Nncase.Passes; using Nncase.Passes.Rules.Neutral; +using Nncase.Tests.TestFixture; using Xunit; using Math = Nncase.IR.F.Math; using Random = Nncase.IR.F.Random; @@ -17,6 +18,7 @@ namespace Nncase.Tests.Rules.NeutralTest; +[AutoSetupTestMethod(InitSession = true)] public class UnitTestUnSqueezeToReshape : TransformTestBase { public static IEnumerable TestUnSqueezeToReshapePositiveData => From 03fd1a336c1dac25084949782341958b1bb768b0 Mon Sep 17 00:00:00 2001 From: uranus0515 <110005227+uranus0515@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:33:13 +0800 Subject: [PATCH 07/13] GNNE-1714:Feature/add quant strict mode to config (#1116) * add quant sheme strict mode for algo json * add bias name for fullyconnect * Apply code-format changes --------- Co-authored-by: guodongliang Co-authored-by: uranus0515 --- docs/MixQuant.md | 4 ++++ docs/USAGE_v2.md | 1 + docs/USAGE_v2_EN.md | 1 + examples/user_guide/k230_simulate-EN.ipynb | 1 + examples/user_guide/k230_simulate-ZH.ipynb | 1 + python/nncase/__init__.py | 3 +++ python/nncase/native/ffi.cpp | 5 +++++ src/Native/include/nncase/compiler.h | 8 ++++++++ src/Nncase.Compiler/Interop/CApi.cs | 18 ++++++++++++++++++ src/Nncase.Importer/TFLite/MatMul.cs | 5 ++++- tests/config.toml | 1 + 11 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/MixQuant.md b/docs/MixQuant.md index c70de011f0..76197cc40d 100644 --- a/docs/MixQuant.md +++ b/docs/MixQuant.md @@ -16,11 +16,13 @@ compiler.use_ptq(ptq_options) ```python ptq_options.quant_scheme = "" +ptq_options.quant_scheme_strict_mode = False ptq_options.export_quant_scheme = False ptq_options.export_weight_range_by_channel = False ``` * **quant_scheme：导入量化参数配置文件的路径** +* **quant_scheme_strict_mode：是否严格按照quant_scheme执行量化** * **export_quant_scheme：是否导出量化参数配置文件** * **export_weight_range_by_channel：是否导出** `bychannel`形式的weights量化参数，为了保证量化效果，该参数建议设置为 `True` @@ -36,6 +38,7 @@ compile_options.dump_ir = True ```python ptq_options.quant_scheme = "" +ptq_options.quant_scheme_strict_mode = False ptq_options.export_quant_scheme = True ptq_options.export_weight_range_by_channel = True ``` @@ -108,6 +111,7 @@ ptq_options.export_weight_range_by_channel = True ```python ptq_options.quant_scheme = "./QuantScheme.json" # path to your 'QuantScheme.json' +ptq_options.quant_scheme_strict_mode = False # Whether to strictly follow quant_scheme for quantification ptq_options.export_quant_scheme = False ptq_options.export_weight_range_by_channel = False # whatever ``` diff --git a/docs/USAGE_v2.md b/docs/USAGE_v2.md index 68944d7bbf..a7aa4d3809 100644 --- a/docs/USAGE_v2.md +++ b/docs/USAGE_v2.md @@ -228,6 +228,7 @@ PTQTensorOptions类, 用于配置nncase PTQ选项，各属性说明如下 | dump_quant_error | bool | 否 | 是否生成量化损失，默认为False。在 `dump_ir=True`时生效 | | dump_quant_error_symmetric_for_signed | bool | 否 | 是否生成使用范围对称的量化损失，默认为True。在 `dump_ir=True`时生效 | | quant_scheme | string | 否 | 量化配置文件路径，默认为“ ”。在 `dump_ir=True`时生效 | +| quant_scheme_strict_mode | bool | 否 | 是否严格按照quant_scheme执行量化，默认为False。在 `quant_scheme`不为空时生效 | | export_quant_scheme | bool | 否 | 是否导出量化配置文件，默认为False。在 `dump_ir=True`时生效 | | export_weight_range_by_channel | bool | 否 | 导出量化配置文件时，是否按照channel统计权重的范围，默认为False。在 `dump_ir=True`时生效 | diff --git a/docs/USAGE_v2_EN.md b/docs/USAGE_v2_EN.md index 5800ccfc7e..8a19714c32 100644 --- a/docs/USAGE_v2_EN.md +++ b/docs/USAGE_v2_EN.md @@ -226,6 +226,7 @@ PTQTensorOptions is used to configure PTQ options. The details of all attributes | dump_quant_error | bool | N | Specify whether dump quantification error, False by default. The parameters following worked when `dump_ir=True`. | | dump_quant_error_symmetric_for_signed | bool | N | Specify whether dump quantification error by symmetric for signed number,True by default. | | quant_scheme | string | N | specify the path of quantification scheme file,"" by default. | +| quant_scheme_strict_mode | bool | N | Specify whether strictly follow quant_scheme for quantification, False by default. | | export_quant_scheme | bool | N | Specify whether export quantification scheme, False by default. | | export_weight_range_by_channel | bool | N | Specify whether export weights range by channel, False by default. | diff --git a/examples/user_guide/k230_simulate-EN.ipynb b/examples/user_guide/k230_simulate-EN.ipynb index a8630394df..b4c5aa2b91 100644 --- a/examples/user_guide/k230_simulate-EN.ipynb +++ b/examples/user_guide/k230_simulate-EN.ipynb @@ -150,6 +150,7 @@ " # mix quantize options\n", " # more details in docs/MixQuant.md\n", " ptq_options.quant_scheme = \"\"\n", + " ptq_options.quant_scheme_strict_mode = False\n", " ptq_options.export_quant_scheme = False\n", " ptq_options.export_weight_range_by_channel = False\n", " ############################################\n", diff --git a/examples/user_guide/k230_simulate-ZH.ipynb b/examples/user_guide/k230_simulate-ZH.ipynb index e9b9dc5329..3a099a1ea3 100644 --- a/examples/user_guide/k230_simulate-ZH.ipynb +++ b/examples/user_guide/k230_simulate-ZH.ipynb @@ -150,6 +150,7 @@ " # mix quantize options\n", " # more details in docs/MixQuant.md\n", " ptq_options.quant_scheme = \"\"\n", + " ptq_options.quant_scheme_strict_mode = False\n", " ptq_options.export_quant_scheme = False\n", " ptq_options.export_weight_range_by_channel = False\n", " ############################################\n", diff --git a/python/nncase/__init__.py b/python/nncase/__init__.py index 0663332e10..3c6aa2b69f 100644 --- a/python/nncase/__init__.py +++ b/python/nncase/__init__.py @@ -66,6 +66,7 @@ class PTQTensorOptions: input_mean: float input_std: float quant_scheme: str + quant_scheme_strict_mode: bool samples_count: int cali_data: List[RuntimeTensor] @@ -83,6 +84,7 @@ def __init__(self) -> None: self.input_mean: float = 0.5 self.input_std: float = 0.5 self.quant_scheme: str = "" + self.quant_scheme_strict_mode: bool = False self.samples_count: int = 5 self.cali_data: List[RuntimeTensor] = [] @@ -244,6 +246,7 @@ def use_ptq(self, ptq_dataset_options: PTQTensorOptions) -> None: self._quantize_options.use_mix_quant = ptq_dataset_options.use_mix_quant self._quantize_options.quant_scheme = ptq_dataset_options.quant_scheme + self._quantize_options.quant_scheme_strict_mode = ptq_dataset_options.quant_scheme_strict_mode self._quantize_options.export_quant_scheme = ptq_dataset_options.export_quant_scheme self._quantize_options.export_weight_range_by_channel = ptq_dataset_options.export_weight_range_by_channel self._quantize_options.dump_quant_error = ptq_dataset_options.dump_quant_error diff --git a/python/nncase/native/ffi.cpp b/python/nncase/native/ffi.cpp index 8bd6bd6ba9..b8c99ed966 100644 --- a/python/nncase/native/ffi.cpp +++ b/python/nncase/native/ffi.cpp @@ -185,6 +185,11 @@ PYBIND11_MODULE(_nncase, m) { py::overload_cast<>(&quantize_options::quant_scheme), py::overload_cast( &quantize_options::quant_scheme)) + .def_property( + "quant_scheme_strict_mode", + py::overload_cast<>(&quantize_options::quant_scheme_strict_mode), + py::overload_cast( + &quantize_options::quant_scheme_strict_mode)) .def_property( "export_quant_scheme", py::overload_cast<>(&quantize_options::export_quant_scheme), diff --git a/src/Native/include/nncase/compiler.h b/src/Native/include/nncase/compiler.h index 6b7b33ef92..7339d0b546 100644 --- a/src/Native/include/nncase/compiler.h +++ b/src/Native/include/nncase/compiler.h @@ -199,6 +199,8 @@ typedef struct { void (*quantize_options_set_quant_scheme)( clr_object_handle_t quantize_options, const char *quant_scheme, size_t quant_scheme_length); + void (*quantize_options_set_quant_scheme_strict_mode)( + clr_object_handle_t quantize_options, bool quant_scheme_strict_mode); void (*quantize_options_set_export_quant_scheme)( clr_object_handle_t quantize_options, bool export_quant_scheme); void (*quantize_options_set_export_weight_range_by_channel)( @@ -401,6 +403,12 @@ class quantize_options : public clr_object_base { obj_.get(), value.data(), value.length()); } + bool quant_scheme_strict_mode() { return false; } + void quant_scheme_strict_mode(bool value) { + nncase_clr_api()->quantize_options_set_quant_scheme_strict_mode( + obj_.get(), value); + } + bool export_quant_scheme() { return false; } void export_quant_scheme(bool value) { nncase_clr_api()->quantize_options_set_export_quant_scheme(obj_.get(), diff --git a/src/Nncase.Compiler/Interop/CApi.cs b/src/Nncase.Compiler/Interop/CApi.cs index 3ce2d0e289..69bacc8397 100644 --- a/src/Nncase.Compiler/Interop/CApi.cs +++ b/src/Nncase.Compiler/Interop/CApi.cs @@ -84,6 +84,7 @@ public unsafe struct CApiMT public delegate* unmanaged QuantOptionsSetFineTuneWeightsMethodPtr; public delegate* unmanaged QuantOptionsSetUseMixQuantPtr; public delegate* unmanaged QuantOptionsSetQuantSchemePtr; + public delegate* unmanaged QuantOptionsSetQuantSchemeStrictModePtr; public delegate* unmanaged QuantOptionsSetExportQuantSchemePtr; public delegate* unmanaged QuantOptionsSetExportWeightRangeByChannelPtr; public delegate* unmanaged QuantOptionsSetDumpQuantErrorPtr; @@ -154,6 +155,7 @@ public static void Initialize(CApiMT* mt) mt->QuantOptionsSetFineTuneWeightsMethodPtr = &QuantizeOptionsSetFineTuneWeightsMethod; mt->QuantOptionsSetUseMixQuantPtr = &QuantOptionsSetUseMixQuant; mt->QuantOptionsSetQuantSchemePtr = &QuantizeOptionsSetQuantScheme; + mt->QuantOptionsSetQuantSchemeStrictModePtr = &QuantizeOptionsSetQuantSchemeStrictMode; mt->QuantOptionsSetExportQuantSchemePtr = &QuantizeOptionsSetExportQuantScheme; mt->QuantOptionsSetExportWeightRangeByChannelPtr = &QuantizeOptionsSetExportWeightRangeByChannel; mt->QuantOptionsSetDumpQuantErrorPtr = &QuantizeOptionsSetDumpQuantError; @@ -603,6 +605,22 @@ private static void QuantizeOptionsSetQuantScheme(IntPtr quantizeOptionsHandle, Get(quantizeOptionsHandle).QuantScheme = ToString(quantSchemePtr, quantSchemeLength); } + [UnmanagedCallersOnly] + private static void QuantizeOptionsSetQuantSchemeStrictMode(IntPtr quantizeOptionsHandle, byte quantSchemeStrictMode) + { + switch (quantSchemeStrictMode) + { + case 0: + Get(quantizeOptionsHandle).QuantSchemeStrictMode = false; + break; + case 1: + Get(quantizeOptionsHandle).QuantSchemeStrictMode = true; + break; + default: + throw new ArgumentException("Invalid QuantSchemeStrictMode Flag"); + } + } + [UnmanagedCallersOnly] private static void QuantizeOptionsSetExportQuantScheme(IntPtr quantizeOptionsHandle, byte exportQuantScheme) { diff --git a/src/Nncase.Importer/TFLite/MatMul.cs b/src/Nncase.Importer/TFLite/MatMul.cs index 6b1724e5f5..59bbfcd1d7 100644 --- a/src/Nncase.Importer/TFLite/MatMul.cs +++ b/src/Nncase.Importer/TFLite/MatMul.cs @@ -69,9 +69,12 @@ private Expr VisitMatMul(in tflite.Operator op, bool isFullyConnected = true) List outputNames = new() { GetOutputTensor(op, 0).Name + "_matmul" }; matmul.Metadata.OutputNames = outputNames; outputNames.Clear(); - outputNames.Add(GetOutputTensor(op, 0).Name); + outputNames.Add(GetOutputTensor(op, 0).Name + "_bias"); bias.Metadata.OutputNames = outputNames; var mm = matmul + bias; + outputNames.Clear(); + outputNames.Add(GetOutputTensor(op, 0).Name); + mm.Metadata.OutputNames = outputNames; return fusedActivationFunction switch { diff --git a/tests/config.toml b/tests/config.toml index 11fe4ef48b..56f1c0f1bb 100644 --- a/tests/config.toml +++ b/tests/config.toml @@ -40,6 +40,7 @@ finetune_weights_method = 'NoFineTuneWeights' input_mean = 0.5 input_std = 0.5 quant_scheme = "" +quant_scheme_strict_mode = false [infer_report_opt] enabled = false From 1073f35ba9eec01a03eccf39600060e1ddae7380 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 1 Nov 2023 15:18:08 +0800 Subject: [PATCH 08/13] Feature/audio (#1118) * upload zhihu test cases --------- Co-authored-by: yanghaoqi --- examples/audio/tts_1.wav | Bin 0 -> 70044 bytes examples/audio/tts_2.wav | Bin 0 -> 262444 bytes examples/audio/tts_3.wav | Bin 0 -> 61244 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100755 examples/audio/tts_1.wav create mode 100755 examples/audio/tts_2.wav create mode 100755 examples/audio/tts_3.wav diff --git a/examples/audio/tts_1.wav b/examples/audio/tts_1.wav new file mode 100755 index 0000000000000000000000000000000000000000..029d3c4f8d68f18cd65465c058879cfe53c926cb GIT binary patch literal 70044 zcma&Odt6l4)joc%bH8#&BuLamqExA&%4?}5mKbV?MN2F(#1ex>jT$s+j8UUTjT%Fg zs#T-B2{F8eDlybjLk+dm5=)F)V$q^O1IT4&nETARpYL;|e*1oZfBl>fILw^cXYIAu zUVE+eJZt6@Jo)5roFp+X@7erC%U>JiAP9oQp2nX>}*=uF5;u?l95ElFl z|1;okf{@hV$M9>z&$;`raVn>VRG->^|KY}OH~!Mz<5U9UXrqlJl889`IPQ?+^d4-A|jX0iSo@fBk{%59X(h_Tcg0z2Q;+^ZWn3 z_JO7?(AI?yKG1fa|p1mVf`R$@8DjGWYFbSjORT z9;~}2%kVrONHuI1+N%Vld0>^=I%)TN@abU~L732l^e>gEmi29lmP#{r@dHL1_C<(}FhQgZXK*c`(}WDB3(8+)0yL`^$|_XtEBk z`-9I8%l%-k+LaIPKWs1B?6qgwb;Dynu)5(_hv)bI{??@X&p7}4v)Zl;-Jjj?DF2Z{ z8)5jb|5(GYUWf0h{T`NGo2#a0Z7!O1Xu2Q%_y7I;upAF|y!QDAbJ4D6;GKqd%7afo z`26s@!>?-NYisBGx0ieH>abMWm=7Mqa{uT3|M!`;0uR>vf4?&<)$r)rT{PdNy{~z# zVZW-afVMM75#xw4cqC!=ntk=Fchp;I5B}FIOVf(w{*1J{XqvjCURA%k|M(WyXd{dx za*2FA@`%SV8bkEsie|MDKUZ-t&8{_laOw@UNv%_>@$(gCH;O1EmJk(0IZ=cWd}^(_ zLVZD9h5!52sl-8|j~GQJk)3$|minw3Qr=S*DQ_tC3ZuRO8H$KDBA+}$Hj#DYZZd;x zzaMg2a17xKc;}G133J|x zE5F7alc24skf#7Mn}idQfLx7uukyZREx7B*`QPyI1`S^r{(VHOkAv`KE(pLOm z|L@gtVD0mX5~BG2m_>L#3fK3;25w;<21w07iYDBl4qCC`j@l`lgt>%p#SJ{};A%}1 zlOWNsOmXmqX8P%u6i8z90RRuc6;Z(P4-|_EO@sVBNgKPk+7_9asL+Rs1cvv zfw>=rRBmWVvw^MnJr1&J@6}-*lVFp(vF`6-CjEG2ty-XFs~Pyc19!;9^^HU`?B)|< zIqq5qo6W%cn=oPzELxL$0%Up|pU#IA!@J=Q?sf~3*Wj@O*B-*vBVm05OSu?h1zjTvo)RwhAeO&{7WZpJ+-am^~o)CirH{@e22!~f;?_oQ}3H8isu z_E(APdmzswJaQp(Iqs&dVh=vM`@WWPF~4!dN5n|-4e~wGLcW13CA?OkGHR2uR;f{v zaiv?$gN-*pi>IN*O_;m3&P}*(H}0lc+!&0ThgVzh%5!QKcH|1I1gDl@FE$gM@C+@O zQ2|!A2C~ml^KnHkc5N;8TM5xhFr-9Cgqt{yQ4B=G{rg{m=eVV$!;@@+o^#x~A zDLIn7f>m%3HEN#fP~$LCfm(0(@$Yx(K?t zid8c}BU_36nAMK^(rv{G_!I(HRjYB(X$K_BCLQoq#jyQMwO-kz>`-fm3kqu-q*#-ML1bOdZUI&#{#Q>SV#u!bw zOE-RwLx(pYp+Rj`nsJ9X=*NxK*^2k#$Xv3J%qI=xM_9?L&~6SSb>NYO-O-47kHYF| z{;>&b^ak!U5_@42beySLR7pw2jB2r$E1;8Zg2Q+gvJLx&fCtXTT6aJmNh!pdIj~M6 zq37M0*Cvv~s6Du1tujW*hc~ZLa4t{uu%oVE#;cS@NWB)jy%_gt z!^*ef|M#G~k1{To+EqsN)d+^YFWfG*){<&ej+OArQ*jZbH~6cOJNsl?_Z_uonq{=9(=w)O@|KpVN17^JCLwl z;jqFC&LO+CP4`3H0DF=BE&D_EDDxzf%)~Pu`WF2)y_qKIMrsA+CX2|^u(}4UQLR#@IFyjw zj(fZzKQ4bHO_e?pe=MF7o)Rwd@A9Mhv)pUko7}tH>)d19vgqc>!tmx$cTfzrh4{$J zLWg3dAJu1?da&C@-`_Rs}XJMo6vtQ2FOJLIXdLq0AY5Kjsh_-ow7 z=;Y|7$bv{!I4is?To=9`c1Gq#G9xYF;xHfDgGX_Aa&(uFq28b`=r@{o+84%*kFAW^ z=``Al%~K4Ebw6R;f&s(I8ikU7QRGHx+- z8c!H7A9gGgPiIl@!TKaTnxMl9;)*gx-Xfmjr*X5Q8Bt?&dUSEr6P2Uaqn=1kq&D0U zIuzU&Xb7|iCI)8sX}`;N#-ANtFFMEpw%k13nHHaz*qXRHVSY@O?Wp;T;rC1?)lS|a z4?=4z=s4h{JjFPq8^KzcTeOw=jCr4Z zNB5R)HCspLlW!=Hk=te;UDd9^H&5~0*S%$ zpgmCLEp&SZE)A?1+7@jk&gd`MXC|3b&8ZVpCML~uZZv+zKEr0w2Z?HGgl>fXefBQX zOnxCYhf9Jd0(%3sp~*r!@xIP$o@lSOwU|#CpVxoPzFIm zeb-g)j`iK-2#PkWwp7NnQ6PZ#Qe7Y2K_93guJEvO&rfBhsy$e zfz^=+zg1pA{+y)^XN*BpnkfJ~JfC# z(RCQx%*p1c;T`=w{rmdq`dNll^L%TK)oWg2XrP+qz2XVsbD=}(AgY<)VwbFAS5S=d z2k1-3n}u=mLE=gJkF<|ulqq~xsKS@&JwH_BJ>&Iwm;0RFWA5z1+xPBvU+zBaSu01) zt?|_(MyFP$R3%o&R@xsmQM&Uwqh*Gz+PstPlKJpn?;3Bce@pOkq*#8ERgLYYpy45% zoB9*I-B4s5u&;J3v*%b(8SgTZlEBZ6#PC;?O!_uU>HKsKxdZ-;QQY!TwTgOzd4hS4 zG7zVwUkd_1TJ#D-TwWwSlpiSfUT|IOzucERP%$XGvfbfn6Cr4-Kx5X|_){m%6 zj!l>tHzT&sQD4z)V6#Cy&&HYjkVQiseD&oHamcy#lEpcZKWq8@4Q?4@u)_d1_&v_b&d2#a|il*69*2OnCoc0>W zVq2}YNo+@L6ZLL#&uEFOIa5?0%(G zlH^kIYsvtBa)roNpOXG2Y?qc3pD`o!&+Gk6h|EiEw0jm>|VLM!qczXW2dCnKV%|O3X;k)iT>^|>O z+ka^wV=%Y><>6yIsumbi1oyrg;Yjm|p9;+O?-HL+30O2=Y{YAPUgTxM`f;CAH4 z(i@0Gej&XiRESFz14ZeAMyoY!Pm7u7?6CYumqjh2*0JaHcXcn&o3LY-NF~x#d8tyX z-Xd|b&?V$Jr9}FL)GDVV`rfT_@+9$Y)EbKV&JTS!wA~x;E%2nf!UHG!{e9uS6ZhhK zHx8unP1cpktCPL4v+c4i>bw%aIcY;eZ2ZEwCg*tLLB+=1j-Kc9RI+ z(u?X7Op}qf9JMxC%%)j-5Br+Fz*uO?H^mzN%rev}_=;j>m0CyMp#MZCk*DPp@ee|= z)F>A!rScl-wD?Ow$F+s&kQhh|ob;_9Ds*2TWU&{r-16YSJ?Hnwx@Y=cq*EMoV-uYh z?He3>oGq~%;-@6231X5rX>{x{!#Ls-sYBYYSjbW$SAK`fiWG^UX62e`5bg*HSgqIy#-{r89@$8?sHL(f%;FD4{Gr0|h3F)aP+y>W=%3Sf$O38` z{YT~!TV{xw=h_xq^Gz3ZPhv-}WG}OSW_uZlDk4V8KNT);0qz2#k;~lW=(R{;xFobD z*cCV&C=6DIoRM{rfpC1dGUN)~4d;b}fdl>xp)~Om!bq>94>2D0rY=UW=*`9)Q?+@W z^}03Na>kfuP!K1rWEZd_*c;4?jFm~I`w_)T#11l*Dxgwv4p$?NY()+>260^p8AsJo zW9d{ngC0vK(>+u()k1v@J-&!}I!F%ZSdY?y__#qSR2t=O>2Yb7m?(ZJyd=EEzs^mI zRz-Rdv+NC?^AC7;c~ANp0(59eC_7XUYJyh2=gY*$LZP zy3%$X{^gL3v=vx$EG_0O=0oNTbF)b_Mh$Q4XX7;ZjQK0=qjpdZs)#H?44$uYh=^Aq zBdA7{n5#6%`SKs6gHoBaMj9u@N}q@oSntWgE4;uh;BH2((bPzMcxT8M+KxzIyw3{x z4*NTT#o?o2W28DVgZqWBS1M6TiCfg4*g^fH#txjM4dyn>ZELc9o&CK1g#DELn$2%D zTAMAUmUzn&bDOEps5hR~Kc=f@o}=f$&pF5=$V0TMN*!X^d}wH@d_|5!Jo~AfBo|}X z{tU6_C*l_IY3S`2!pr;*xEax9k-OpKaB*l?Pz>w}objjlN``ok$D13h39pXqjLe7* zaX%IdS>5S!dcV*)H1-*(W=^j-8Ih_HEX&mK?-MH72j|l;L~*JpByapJ@kq2svcB+6G_S zf)$nIB;_M{jQlC$xh6y_*?47w`~_ClE-ew)i`&J$;`f4GxXTShQ==Cnv+hTs_3~m1?z({ERuJKWTJfk5pUote35R>rvZ9+kE>O zyWO$De$a3h!wu$Pw{uTGts%oB{ITncx zq6yqZ{GyRN#y(Ry*0t5T&X#J6w`JG{Y%A?M?M{28Eo^PG6q|ER-G(#z zk9D(jpRr#vPtd!`uhlxGMCp{r$?Z}DPPy1B?C3aBczMDCLE zp{Y%BH>?cVBq|ZbIQ<$?d0>ba)MVtvJBSr(r~Egm9hp)A&a&BjdNe+?-ha;5z1HvZ?v`{7bQW|S7;=!s<|gw7%N2(`c6CgOb4pB0g8m_Udi%(fgz3&I>jGWB zI!+l$R#6=MMGtvPEs;j>r=q{Y`Ib)ohJ8)9m_3T@p-tH&f35x%nO&>VYT0HfFslZW zei{2e%r@p@=4-luv*Nm9j+&oi9O^M4D{>|LD&M4R zqCQ~fv908Or3pT<2oc;}T^ICv#i;0KQ&vw#@J%)HmZgMcTsiBlDtU^mf=9 z8trF?GTaAT=MkF}cs6=GuB?7%_xkVF_gstaaW+2e9alLn@sak2s$$;PHxi9nuvct{cobdL*#i3r;n!$5}+g-Vy zYo5#Qi-RZco$10p@SM=srBy#xIlgk7H+^-&hsIBcA0zvyRcGr8j6Y)XWI4JzoFD0y zkSkl8W5*{p#$U2kGoz$)+&;cRsbppvyoLnmtVR_Dv#?wF9b0c+XUnlATTbc?)PA{J z{HZiXaS$z}5An@DaRKKKk3|K-8Ce~v3&r|#J%xkLf!qC_f$(6t>zeEA0RR1^?+W_h z!;|NY(@*GlZ093wDOW7b#H;Ys<8zzK?iQ z{{%Zqx(-Vi!DlNG-AYG8!sH}Zg5CL+&LPj@JEdnS4|`3wlRZS;Q687ph>uGxuz+<| z+PTes%oGo~9Lj#VM=2*Ph~^HG$;dn=@~dI@*P}PN`P{PbKHo;yx&G_@cL(+k?sY90 zI^f&qJv?~s`@7%e^sgoCDYY4m6J}>DNZ*#AH~G}xaP_F|uxydBi8{*_1UdpXZWI}B zSsXVxadpzVge+Sfb&0Evmdl5j4{^e8qidB>(sA(s&R~+xHAfwbo!6{y>EFXnJt$8l z@~KLCDP2mAQMMpo*eCoJH8_F)9?@_xICiLf;8{2i>al*W;+VSf~>y!PKCh80ET`pEAB`94L^QGtv(ShplZuyd7u5)riY~qag3wE3C zwEPdDL$0J|>gMSmVzbFs`Ac!5@SSi&RSgY}>X>xftNMCMtKB(pS}~}1A19Wo$E8R4 zYmsntv}nLt`Y*v6N%UPD^!MfT$Gbe91Ktl&!QlsY^`Gs@yL-3yVQyt?^|3W?5(HONl(3Mlrf{^G35t zze;K%|*M+Iax{R|VLGRPw3FTyl#8Fzw@l|~_c`=xL> zvObVMba)U|E?2RqU?^iK6E&PX*T#Oi`_i5D_o8BTQsRXANjt}_9vSO+MSq80O*f$q zc-~NBs;6?o9}XG^Px>~==@xH-D{*@4N^7j)8D!FPsCV>r=53ZM#%y}IcsIN`9N~(T z8%&ES-kN8AN&gvbKvce#e2#fXXVT?S`SLgX+x$OK0Xr#tg38hnX|k{^%zMMGU4!wi zd^hPKJ+eFBeRiO@Z=l!OeW^D;SZ$7u?wT+!W9O)qiR0}b8fjQhK=)hS0liMQU+VCu zy7zgSLdS^`bGLJkW2|{dH=kX`j@EaWTCB6-9Zu*ANxfJVeU0lAcN4#5#~Y^Te~+wG z0?B58HtF8gS=l`7l21{Gm?-QOHcCg7EY&N`;;X_Hs8z*#mb`1ut~2hEP_5Ex++aIo z{%`gLDv$aJJzdvkT4uL9Puk)5~Pgl3Ij{jDx79HZUKvI^7)R2)SE1D19VlBc3Fb&k>(( zKvdfh-V@FU6$Wm4%U$xl13lM!Jp+4P`K}8CxqTOVuMK#``nZXY)MiwsrzIbC?y>pp ziLsq=Dr(f1ZHslI1(&bDTNIou{E#sl{cO2>J^B(qQW>BNOf`temsy>bO9q1aL@JPz zsYi5!`b+u+*sq_+f00HhEkrUis5^s500JjYMw1wK|xk~#So(%IVAfzNsv&v(rT5APZin;D3sTpOjy_~hTIrl^o9>|F?D5ND%`u5F3#~_~ z54c@HJL-ZB>RtA)%sAz3o)Xrh)-{5yK&(@bXsOiln9e}dBK~z?!>)_cZt*mbG@ zLQnqphx-min@tmwx>J@VHN;-9owrrSc;hZsHDgNndJNZYeS8^e ziO+?*k#KlV^liRGS|WcWo{eS)GrfLTf5nj5cg9=fJv($@XwA@Jk37itogHw7GUz_X z;>6jB(_&h!&DQBL&2gQv>tfA`vy;8?skRSvSCk0HN0-PU?6hW7TfP)}qil3CY*|to zsfd2NIm=vP{D93NtCb8gM&D(=f;@1Bp_188#L4IRQ(T((oVu1AL$ph`xw_EuKvSR$ zD95r$U1UReUtqSccj&e!+kMPk;w=t1L+!zhfyus!s4dq~Mb>7ZAJLd-OndC+_{xOM z@qIB1W9t(Vla|HLv)$BXss94zfgBR?X@>;+kptbaHuPvb=K%a`uRGqm-aj_j5X=bw zmSdDB>3xQ6*2I`~@vg*K$<0a8_<6BuG4(NY?4z*@V$z%$wocP^-Lr^ZQ?P=Uq9gb< zVkh$MpNZ>*FN6<;6w!tH@*6m*`l*X-g6?7V2Hiqsk$tEgWs%it2daKgi*NA@qFLc3 zff8@2C(CmT=TJQA^w)e_0tMlP+)1HJVu+>mkgmpwRs_-kG;u?XgHKPs`4IJ~&^c_P**X;YypBFMl z|H&KVQRMq# znk@_GBD!{kYC<7Bfm}lu^WQPYe2v==~bWtf+-Y<`W@8*CGuL73(Eq-diQ3wJ3CdftL z1my!OY*3FV74lw0YTpaffI_~)zl8e0EZ)X{%;~uK(aVwf5jN5YspN2N9E=RCyOPwq2Yms=^837rw1R&O^< zHf%RsK<;zF&~CVBIBdA2zo+YE!_1#y$sw`~SjARI@E*|mV(=5n!CM*Tova0yL1X2$i=4n0BGb-m>N`N zmr}`;MD~--vmdfQ4&;9=YJR(wQOXs032H=M zsU2tecBxc)3O48x|9~3c66}r%;w0>l6j2Xv_b=@9mxZ69dOA_qB}^6VK(X_bIqGg= zH(5_@rsptUF+XQFqkhffahvt9ci92((h3?zS?Y{0JTk~*coLH51yp2gBnA-P^^ zhpwMT{Y@|46ZCi%6psU+DhBgIllB%_M?Fd3pvN+Y5Q&knNGofEMrAzbncJvJJ;V6v z8dT?--~;CXS8E5C?+CdAdw3D-wgme)ANy+*8M@E-IS$R{f&sx{ukD7+>8Rb;gArH^ zHi-kgiz~>#mLQXTTpkJbK~xG#-BJf+{u^}uIdJsj(ECQnJ`R$1i|x?u-^D)E1U^Mg zyA^fDJX9^4z4cO%sc(F`WIt%5~ z(4`&z{AFM{Kf`YO2%h0_@k3#VACHPU7z&XLFeWmh?{i-YW8^y3P0j)0Fx{}+INnrZ zDm3X$9~v(hzQM`=KKB0m>|@aMYWV0(phlaJ$vg{NT!CF(0Sz_E2Hd?yU5+&ZXM$## zMZjVYF>{y!dIi0fT0!my4!Q{TunE>cpweFeyDgW#fv@_f@KeZi9=_^4{~coc$-)$1 zPlG^t0-TK_qj!!)*OL z;6TsfJY7XKLHB*&1cj6?jiDl@2yb(l(J)XY5~t45aAw3GxfumJft$;X;NVds*+B5h zgNecA!Q42n4|K2SU)TMPCE0DX zTP>5$^KxW)aH7A@?*%dx9~FhK)ot`V-9#genscYQ+>&F7H}@I$nzn$8QGt^;YKZ8b zWe$RiGEEp8IRtEJy6*xo^)6p4q-+jPi_YK@xC^L)-i%xiw*n2@87d6b2aE9N42b^q z!TDT40VvI(0ml}WjW%hdP}Blz`oLP&OXo9VJ_L|=XDHi!bBK)8 zDFaNrd1B1;Q1>FeUtSl@56GU1!CeCd z?$Q3-$kU>MOwesHcAN6d#kgvk@vy1TQfu8{Ul7xTJf|w=oQ*a;!*nS7__5&*|7BmN zzc08md?DhGz6x&EN!a0@P+M?ps4YxlUq2-*k^X>KbC9z}@&mn|+k+K-#@;jcdWQmB zKdCpSIqMQvrlh1yPTm=R)1Gf`Hl~OvXO=HN-+ zsiE_p^FwKVSI`srJMWTi;Y`m}Cds40n7W3laIA0u@i8Ax2Pdh-ljSPz&*{zXH~U@U zSZ1_wrEN-VTm0hq>2VFtWLvrAjJ+lVi$6-wn5RFkn1gweFb=A>!Ka}&E`TO1o~%Pj{?+l?j0Z}iVFb?SFQOL)4! z6gm4@ca}FTP#^v^e?(k@Gyj12ig1p9lpig8D!wQ8BjS2mG;tN7W4>Hq3WxhEde(Pu zAGC&7s3d#JINx69tc_`NUb44a%FOk)wD`8;<-YqxS9dbrr7UrlIbT@7GxY+p3aob|+V>Scdss+EK+p)nB2FCS`{u5%5&j>#n zF#C$UvA(nZir{MO^6Am^h$}QUGMoF00M;L310NXplDtd29L)%I0?k_E%^q6glKa-* zW8JI6KIJ+3ZS0s0mJJrxR%vTQOg}p&nzS;dEopW9LT9V_F}8%{l=b4t=rv^0=egD4 zmcV5H4BvR)zQF3}Q&JQ08@9pJVYzOdXIlqs+U}eY8;!q`a43O}-{7pajMv{HmrK7v zwf!hEi}bKRv^-QAYzh-sy;45$6?KViH;UNb-KH*6&>Xh!NvIx?Hez+chS)v!KGQ;7GIJ12Ba5ur)1!PF zyv7z(5wg5n0@bMXH>xb7=mgzNVE%Pxzbz#uC4O5{eX>4jVcbGT)cltI7QGgHv~kK3 zX^_tZ;#Lxv>67t!+TR>p9Ess(AU{44orXMQ1JJH^ag5{w=U{fUE3_BYfx9lg|JXoL z;B!d=voMvprdwk;t3RQ489PkNY}4YlCC^F{V`3eZwy|c7p*ceLF*4T%ss)wyl;C#X zDgPDl7pp=Kb9Q7kA%!8fA{VhTwWi7TJo!=VJ2e$Y-0&LhBjs}C^8@9s!uSYIKb=(MkIzNS9%pHx^M!JHDfzkc}&jq*W z`;ZSIdV7xk6Z2c}LHE#iaiYds5}ncbZSjdQ)j(qWmS$+%i?i`0keVCx4*7P}8VN@{ zSkKQKs^k^z zz$VuVuL#SyyI?a^MSm%@DD~tX`YN*kEX9)sy|Kw?HaA;)?dkS1OP*=FVYbdg*HML3 z94hD8%G1EC{5YAmh4aH*s0sm54;>Da22_7TpfI#Eax-F$_>q|=hc^b>gD&Vl8q^C* zm3r`##t}=YB=TA68f!PW49VsVh^UsEq9)dwYHPL5u<5~>;4ONy*YGa;HO109>3T9B zb&M}j^B4imF2?#D7W$+`$f1gqt!fXMij3kCJET8mFdNg1@9Y1Y1^-9~CNlFUI-NdE zv?xnZhx`VS-gd-(GT0q0;iB-NP!VF5&Eba721Gy$BlVF?oNKAk0aUoy$m(!S=uJ)p z6TkI_sx0pStPXogG4Oo*MhUH*$=2>UiZd$8R+p96{HJn1G_>%q%5RV+H zlgLvi07tJtZkH#I0Y|6we)fM$xuGVJb5Jqrq2n1N{VnCA8o>#EhGD^FY(g%(1Ce05 zx&qa)Byd(v@)!9D;w6sakqPoAxitR2A?IT5Rn(^_K}5VA;U{8dGf}xX#=j!G&i{*V zfw%iH*boNf8sn5gvYOJ^x+B=P^)_&0UelKw&md+hGnZNRnYxXmO?!>+=+A*;L+Qad zW@@NWL>>RsSvja z6J#^u^L^rvrB1m<{Zjh7n2I{YUnHNh7OZX``4jf{$Z1xyzhjr_(u`9K?K_f5QznBJ;^ghrcphs%YbKWBv&#A<$m(N z=wi8=`_NOE2D6JinbCWk&Q z>M5~A*-cqwbcF%sJ*1{GC4?v~(8UpV6cP;09Q8JnOBu)~=@*$c<*%3l2?p$cvl(O- zl}$^?CU*l1bP!oo3D)LWDo06&<=2BT@MoGs^@K(9cB zxn0Vo+NCbJNcn_#ogbiYsn@tF>aXCYN{UH0Rcuk#F>x}dJftg^o>M){Rzz`CluzlQ zZYf$0zf>v0jPz0-^#W6=mMN9=aj=i}lOb{{;h@mJ0lr}#^g5PWOMFX}DEU-};>P^Y zxuZInW@3%V(__d{@))v*BEU+NnKCf(tLT67VWyavED)&3BudTD=~Zxn4d@F<6E=W> z^|ZK}9*6vsP#RG0+AB3fzu&4KNmrB_axL*|;R!lVje~dj40yCzysewXRS-3lS!e(X z_!bWy12Il6L+!3xE+9)FMLP2>aMMhj#@`6r*naQ_{>t1SUlWW>16eBlh$$jRDn01z z$(LJ*i|q5l*VH?@yvPs0h&dlsfV}9%ztRoFQZOUlL)CPJDxtcZtsbS4RSUWjZot+m z&`HCnb<}<(gzgXq`8h$ZLJjd-DqqQiwniygq)$b61=zj@C7VuCG*58|`*0IgPJATG zbg7(;^JSBoK^4kJFdMyeTK-gXQ=l$+OS;yrTC4$TLw1ElwutkiO1N{BBRXIzsdEWetbk3C(UE))bIJv z&{EooXzXjcOiV(x?_0G?n#1J7YrLY*QOr^uy_J|CuA@tc<8nE%pUlO1)k+kTqri{+ zp>7L2VhJLSC8C?!kIty&GR_S7IFX}nCC8yIy_xP14=Ow9zexqe3Up_P(h4$!5h~Se zs!8srI^+(bSgE4dOJGegr=@&l4zQ$2@>;S0eL<6CpSqdp2G(^KUZX~FQw2nqG?osj zCk3-<0oN`)s&}COCrAAl`O{O z;$wD-P^5g!jzkZ^N+yFm#eb+@4kmRKb6PG?e#lNjRp%=vN!cr!>`k83_IXaUJ89SE$cY1|=CNTdA~)d|WQXOpmHOJx)d~ zo7hcpauKyV3C5!`W{%@!dy`*1{GvLo`iLy_y z=|*Ao4glwSTz&#FEeBG22%N}V)Omg-NK~EDA*VAA}Dsu7kwHeG_@PM zF^~LAh@N>}6TTBTMo``~MT&ukIe4pH zzu!&vpvNm0-unhT&JF5wsRjJSV(_VF=<>h?{S6yOtq~pY(GvE}i?mnl6w8#|VQwsTSA)eYKbLhB0H#(V5)k4NN zoM$89o%2ZaX+nbraLWtTt*8gT2i{EqQH87XA&~+1T%=~xaqxSMs1C0|@5Fy027C@Z zvFPSR_1H};g+==i@2({q@p=Kt5M}5;D2I2sDpR z4B%^>Kz6c$XqI8Mmg6kv&_9b#aK(i#ldI_Q=>Z?xg4#QxN$jp0@OrNbTa_h57Lb-W zl>t-R0^B-;=x2xg0@g1}DU(0N*;j-<*R|-s>`~g$adrcIDs7Lql3&qAItM(~eq?4x z>1{N2B$bTX^c-L?tEfsuYmH#h6#&!90nPh68T1SEPuQ=RtMoQtFyM8vPM}0TVcumkkuz1&H>g9%6knt)u#_Zn z46v<5h>;2q=^uwRltan}bdl!~9XRhsqDQs~nOK|L0}J;`N#M5UD}Cq&SOUb*0HiBP zE(bIF8?hV`K8||)aS@Cu@i1h3T6|h~TI>RzwHzEBMA$gf^9T)xYr$F`LN<4leu8n{ zN8)DdW&$JB=^oR)&u(L#>>_08{`<&QBak8sbqn73I1r3Z$a59eq z2v{|qKdTm?a{ENhzA9DCTQtI(Y^8Pry*Dz?GIyC-zym%;ro32pQAYt0zR7+J#PCnR3;#qr zX)Eo3j5%NtbV0&=NLB)r_^bQAW(+JGyd>mSH}3bWuYe6WfCjyXxCRk7P;LggbjSxG z=K=9cbXWWZ-4*+UZy?`?{4bG--$fSxIzI-TNq;~m>qhYn^mNr>zR$sbBvUuAwnlUj z&u1s=CjxIBk1EwP)Z1sGPi;td8=Z|F=7+%EJ_AmePVFXFVdvywoolh8)#ww~zHea; z(6t@vDDo@h0AJx$!S^?yr#lsW13S=toUI(kI*$X|)+TKS&Z;pI?BZX9(dZeP0(5vZ z>U)o%7r+G$!E3;Ze+(^#(ASa=|C>whroN_+qPKYt^P27?B$}mPZurnJ%P^!rr8fh` zx(#G#8S6)PYzDfCGz9G+y2lHVPp<-(q#h^3E%aPB!&@+rQ}Yo7q{rQQ9w+p%eLgAqph;T`&O91;WMfi&3vwBJP)dA-u)CEEuqF8nNdJFm4?g zkM5#H%mDL&?jE%Fp`p`w%6Ql?&oJ6>-Vih-8s_O100n#uT|8jk12JoYU;PAf^kW^G zk?R)$P0Ucc&{yGBo>e7f6+B-(X72{4su?F-BYL+vV8ThLto#aG(j&0HSA|o&iC+MW zYBefrjlc;`3BM8*?o{+&{NE%339OF+L!lj3_ZHCod%&gUM#)Hf=t{6YGzHk+Onwi#Xl?_m$%jQG!!B;A zE18JytbRyeZm^rG%=H$JwHI9<8!WBh!<1VNnIAPB){|^1(DuKgMv|?1{L9B=8VmFUlg4XX{Kg=6G|tc^lA~sAFDCZ0rK(G5clfSleRs zuXUPU)u%FZ;H8@oAyh~gQHQhxiT?|_OlNX4&?H0W{+^0R({3_5 z?9S_`ayK|zQ9B-;*qYEDTkohx$5^|i+uUkIgi5SM6tD_;(3|MM^`MG*J9?WtgRb)5 za#O_yc@e%lp$ne#N!o+T;{Y(qkC{1i6?GM?=&h`b&LwIn{87)@A=F6~EC<|oIZyR^)*-FN^U zFv(1Tx{tpL6zx#tHu}!I=>BO5bcaT8yU=MGl~&-?siax-EtUa4J8#%!NHktTC)pxq z6QZduDZs}^>EHoh7}#5WzqNayi@$yCyZF8ZK7Fv*d(*u>po-UYt1Z)^fh^mC*rpL1 z9$J?+FKtHJv=lnIBe^|!a$_vrIUu(Sn~*VgqYkgV)H}xhu4kn8Z-Nmw1*p$o8mO4Wtl0_ zkV^kkXbJG>&ME=pcJI(d&l&d#_ZeR%Fs&nCC?+$HK*On)nLxo7;;|sM+i}!ZXUzi= zb`|}mes+>B0%)&_TZvpg4EX9v%^{9ZnEjax6a z!S~jYGxYV=8O~*~!I)rdbzE0$Y;1MRc*h3IKlBaA5?&EzMvdXbz(sdiAK7!}uD|PA zSNffb?{dB~c4gh>JGXpyz=dy}FxS}@TBq1IJ8I&(QWDdzjOiMiIA-2M=J-a;Xf80c z>G}b>O=<}*@g@4E`_~83!!~qL<^d)C72mHsLr*cRv#hhNwmD(>9{V)shS;{)8b{C^ zLH~3C1!j+2B-5flGCFYAb7^pF|N6f2{+t18|Fz!Yp0nL&yOVqI=sDGQG_;wW>97N* z?}*tPyE1-5qL{KEwfdnwskI4J<`(KxqM1S@4bNMo^hUS%3(!a08GK2|1aIOAx`E6A z$7@h$HaYEG&MPsh^PGL7>3RJH{S5FV&4w3HS?UmHb7#S0yu|$mT*jS|y}tDWmwOBE zt?wV}I^#MwNDf@Sce(pa*NHnfp_yu1Vq9~=qlxq57sPeM_QfwqN=s=Qxo1S1lc$5? z^XLM&#H9#tAg=tL?+v#G+5?v(4&@p<1O1{J2A>O_fr@Vw7_gp)W^$}4#%GyTV6lBl z97Hshs+y(m__w(#pxQHVa!rpG2GhN_T^9!i`tSCi?N9E{>^s(b>TdFP7rVPcg+_1O z;-seJ))C{A6XT|X%|ItqCQlr3C@yLkhwlHy;qJhp;4!`k-QcHDx48|w`MbE3ZZ_sw z{k9YK3|lxtp2RmG+(5Rv1e}_K%2h-wTje+53pPZm&_Pxo?DF&8 z3eVZWef=4I+wbMwJKvLi=j82z?($Hlp*4QtNHMMNp(`Vzar2#xj>MSQxbFBVF=uq; zVojt0sMs}cPDDb*w4Lh>=P3Kbiyul%nC-Ay>nyizGr&vu9lHDf4nF@~@5Z5UNEDC2N)jcMG!_@^Y~S@^!5krFBn!CY^F zd%f$nYrs`8Sd10A&|3yibFHh`U9B#2q>NZMDmuC{y>kQ|+iWo#CmLCc)mCcCq#5NK zAwE=uE{x3)LUH2T1U~3|xRLCB19D&%Jw~r`5lit&6XIXz}PN zW2#4i2jTFV=IQ@Ox5ZF^9LO;Eq1aqyJ_n0)|V-W{gOTDYv8|dY8#E$>#C@hp2Y}i|V@fzh`Fexz8{lAR-bVN{}dx)^H-R)KW_=vBX0x zF(e_BShQ-nfiZPb_7Q{LwehGXf@h zY;b<$vN%oKt*g`(XlJU=q5kre@+DD@D$Y7hhUSNKuAYH0zebir6g3r&Z!Qs~jM(KD z{@WgpbB6s0e7rnMruC*Z&33|?Z7Cnf>yucE~)N`*+PTjOGO&l-4#@W zE=hVQLJ!RHEHd#Pa@7~9jl{wxHH+@q$OD;#NXqI}T z<{R~BA{njnGFi`6#ZRf>r3-%*KTNUbYe`mV#`&B{!0fhlFIZ*<@h}>q1gFyG6DDDpRqX3|OvQpj<3Y7MkU8 z)I`<6-}%oq7Vus;_zf zS?V{4#K&qbXo|HaT^ISw!eMUzDOFhM0N;8K^`=M2S+KM}Sn%28{(42qg8muk-% zW~Nqn9)$B<(3jPbyzqPEdB5_%f+|8a{OT50G0gOXj`J{@_ghOW>6V4o0eh`;_Rycb zzYM%bY}!gy1Z1ReS#XGRR43E|UD7svpCLW!bX2@)bJR{_p`l5ihvtA@*UjiQf|P$v z?FO}YOVJLl)(f7toQ(c36~0zJM+24$^7i1az;|RMGriY%iYs0DPNzfdIBDB%EwaRz z@AdcfA2Z*xshzFvGH)!E1`&R-mRY?m`jmdwW0$rPrpDT+d8X;`_w_Mrqvx4!8y$u` z6b1zSE*;!t%^T|Ja1(wG6J1g(Yod}Msk<4-Ll&~CYT>>vVd3FwQo-!A) z4s?zf#leBZ{+oT12X0%oqZV=9b=H#*7*E&!Ezqzr;!@>Jm5F_-Lsx8APIh#~RLwY+ zM(+U~)n;63yk{sh!6%z|OffNEpDvCWWWNYm%(D!}27=_1ByC9- z^^3SjY1j}-2!4-3K$&-nr@_4tCPJIzi2Vo}Bz@+h0bk#izI*-mEH-<#Q|?~w`PvVI zAAZF%iqD0s;xyHpWDEh_azg?7H!qlGqx;YjeG{Z;v$4z=W2`pp)8Ekzp!{&2b7`&e zX(|HE)KxU}izHNd0r#X8)w)|`*G{w^{_Vd{e!m&scbBWgnKPK+$gtnG?qYSE><{%_ z>Myd)ws#EHxo!`A?duG-g-22MwJVl`_@AL3(5%qOL7JpT-HbYA0v#1?G0g#OI2jB` z8S8b+FgKw}GnV}NwpdNB<)S`5pZjy2kyKF6*$Os&emFO@I#>n2=6B498~jl&>ok9G zp<@CgIZLeX=r^Jov)wWm*6X_ChaU-GoffXD9C_2~1BSwzayO}9TJ|n?oo$f9o+8~+PRwdRdVG;>YymGz&SvX zi~L=xZstXHP>=&m$^N}vlc%fE-!p7BZeUifnfgqNQ3z2-&oafMW3nM?8*62uK1X*# zGY6E$`|P1Ib~PDjW=Vfm(!G6|r)CK!o*}f3v3%!02Hxl>8ZXK2WY={NPu=#LsBmq! z^bPD{EJfzqRued?&8~Ap6MU6{3i=Y0*E5l%XkL$Yh%fYT#qNbpxvmp9-v?Jm%^(-v@bk3fG)(H;1GY#FDOP-(j$HQLS3L0XS^7rao*imM$u*F+XnU4)z7$~>gvuznHa&>?g z{jG0rpcQoJdNh+>S1r`E5rb_o>@g<7E{b90)<(~ao)g_?I%P^^EICmdjKziu?oy9t zy82H_xpU(;TwdLPM2K(4GJ_El20@WkoTgL9G+MYAW?~A|?16&R_yK95vPJQ@)d400*8I zhHBjfoIlmvsf@u=$5GC~BUX#0+*)QoGMMXH;J(KC{kMNia8sx-k`3DA@9cC(H7&YT z22gOUo5y&9s!eA(hxSAZsE$NHV_pDvmZjHg|Db+bbphRuYMyH^tL7Q1Ynt$f@B~Uj zlQ?^-0$Zp}z2=?cIX;v&G#M6CwtKTH-!&N)SdO!Gu+wpj3e-vKLhCM@%~8N;&J6jz zpVEV;YeX-68}ZPq>h0P_Lw(c>$bK$}nLPrQ?FjN|*74ftnI^d@WUMpjVM04Zv*Nh? zL!KYh;kZW^!6mf8cro)Yk$i1aXm`jN`~sBHcKg-&fhG`rd)H z)FV?0f1{=}f_*Mt`8n~(OT=zc75;C;Z9h~h#iyxaU*xWJpa-x=k;`Y7!F3FxxzgcM zgK4bd)St)0^m>(O;pR}es~e7<&&rczsjw!2)65-=9YnJWjku%MW0oRo zturokQ&nbKJ95^TwiM5pRU_}2c4?m$vx(2BGo!ASfm+RUxWX^dZ>pC!%H6_#^?o!} zT2M1NO2y4BrU_oU=5@l8MCaAQdMcHo?1RwMz^kCK{^Pd=P_3ii>BV_wLa zRAN46nP)EoKVR=E9u({`=!aFHA2!7?DRf8M5gVIiOKTZtN>z`nH~d*ygT_XRKNlR; zNYz1}_Gzj&LVfxGPev{*sgDLdt>;hCERj^v~&TVDiT{9mvw8rXHm$uEmtTCJhj&;d7dn)U^i zTnG}q?55xTmH58+6im!kc~TYo7Ig{+CExQKXy`lRjdSeH_j2Mlb3DY0mkg0w&<6&F8T6;ZI;J)K97km z1fS;(RMZa8Cw^9Vn)_;}+kS6og5!i`wb^LiZ$4)&cGNjLT@9}0!O5uj6%JhK%kPV^ zP718h)g)|7ot=>~VeTW1iIuvoVT-%czRQ;Fx*m{=9$j@*V$?d_c{*20u~&Xek)bZv zXR{k_Bx~(c{v6!sCi#4-AFJrX-4yMLc2GFqk`>MNlzET(-wmw+mpmE{&Ju+^^rko8 zdCzu}{PCb!Zar&jb0kSSwd1t)ruifhV0_b?;owPMn&P}>8)sOf?lyWxA-I8G2`5x~y_4voO7k0@#zo|dZRl&Hqdj+DHBI$0 z8a?l!L8Bx3J>|Vm#UBPcnayRmtUJQ5`zl?h?F(Tu$J?rG)9poYiB~)B!B9VDPD6F3 zvNyM{%h4rMM=wZPke)FyW1@O&b8JYM;ZfSQn9J<9J?6+RO%_^tHK+>AV>e#H&T~^4 z(A60|U?70?s@IUJ|4n*E&gE=5b^XVmK*;N}NlvP){0bIE=?!D@^RyfD&|zU4o(@=50*P_+Rj-k*t_$r$IPb( zDhK2Pg?*OZy|!Rv)rh$%O^+l@tex2L@Yoo-M9vuNUNg)fPlwE;d4bd3ru!e|lkCdx zqhl~dQ=^YD7U|zsT?DIKNZztr{;W7%-KA~d-2X_-14E?%nQug+u?^JLHsw6k|B(6h z$`8S`pX14Mhg{b=&2|ks;WI0(r_FLW*G5}8nP%3&*(vYntr}`F;YSI00y4yw%DyH z(qGq&)jSEVbZp?9_jRwye;~LcvP^MLe3;(FR#1aI=%DSQ9&pY(+tcPOv4#5gqok2F zaLPJ|D)$z&4}1d~QKpP>9no|rwWMb}nv&I%u{6=AtaF!`WBQA%mZ43uzo;_x$);?B zPZ>uZxli1vDF?HCgiN?Y`?O#WIpLxghBpf{G~nY6d-UI{cgsKX&nEBRKz`5>t`}~q zURS@V+Quw36W5Om{WCDuHv{F03RjhFa{slSlRfhObC!1;or5KVrop|o(|r~9v#qs? znF)KwEtt@jHT%))$$9#)_Z@RzPsu>K`>V(_b%*XIS*KdlO?PmS;*_S^SY~REe!|4e zDaHgxqg`VQ+7*Jfn*FO_TM>o@7%yb#|oF)IoE!{dd@8G%j~UioK&w$$;j&XLBquA^qC14g`&Yz z)FngKX}oa&+>VK_DC`qqNkMGL>YMBtwbtKj*3!Q7cMI;p1JutUhtjNR->C@u&Mb6j@2Xf6F&JRP+i~C_4o?;Iymx`#>Q`sJz|=nSr+kmvfNEWqe61Dp-)E@8Q;-dRrG{*Q7QR{e_7b2_yp$YRjTLX zhgqs3*u)=G1CtB2;rqTq*J^7nT?O~O!u|w%rmMkS$og7g4)s;_D0}DFW0Wb$Gaofg zn4Qs(c6{WYlx?mUONFI-@J^swoTGkOC_qTpTP)t98}v#c=&paTL^~#;A?SJ zIMh^@rW5U-uqZ7KOS&yl|x#k-N`0wa7s3pC0zw8e&9thqzsow|~GBh`G z6&tntb^WSDMJJ56cj2tpF{c+a-Sp7gz{39{m=|6IPO?GqJUF8~SuU8NRPY%hDh|I4 zOz~7YeEqpSH+xGhNsidT)%Nq&GuC9A+)xc;88u~OOG49#7X3Z(E3iqA%jT-Lk@03|@~Aq$;YHgA#IaUX zsK!#O3d%QuSlI#YcBi;R$VFpq8Tr9VP^4RBAeAGB0&|B>+s~NY0~gF%YTlPP2JXnyE; zbQidfbLw<`sbPig33Z;l-@DGXci@WsP)N|uG2JwNqb7n2x5#@m(@axFbVO(B7lGFt zjVjGn#UIpPqbk@G9zxw=S3s0)fIs$zYL8-E=pA@~zYM0zuPJA%YlQOPW_K05%ap+q z=L+W($31g;- z?5?tPIe!`4tQ@PksmztBgYCggp-z9C%-3S5(A1#@mJ_+5_*xxH?AHl;+U+^%(}4GV zL76NRf@Rnma-zDnBO({)Dt{{2!)JY~+^d~2>`p${3J3k|-V)YTq9tTg+V=P7-#)$aY#J7!_5P#b^T{Btnc(4I|y)Obk7GjiU`4_?4-fLcOI7xGj zbLYJJ7KrpE!Vgs@Z8Mtv7G;ao!gkB1`>I?<_xQjPp;_Cdc}gfnQ^Y8qWIb)Br|^_0DSdn8 zM#?vX%x-t+l<(KR)KEGxMWdoSbQ&(=Y|q|6I{3X$g9|+B!SgUEr#tT06K&fqbpOa# zw)dXz&9w}93w7ObNpX!jhjNd8cARNU>qB&CW7@Skbw8>ziPWe!3$3z*K#Zr!{ifFt zdRn-QlJ|4yK?G$lXetaV^aoKlYesELp-w=PB%=95xfQ%;S8(evKS`8N4NddS;e7nP zZy~BSCVv{N!|uUTRD_S%a%}0g65ADPu0=a=s;8i@%Xw4u#LS8LR-Lc7pmauOj4n&r zmy{G2W86p{`54u_O6`4C&3tGUdom!XRJOdLlmmK)EwUGRv zyzgFbrvnBEOz$7k!Tdx_Ht9#9Gn6<3hlMgQI@_Y;RK_Q0L*g3Q$H8y?T|`QGvQeRj zywz@%cK~k3Psu8aWeq54J*1qXZPZohW-&i&#cR~+PN6kWBTkl=!#nsTdSRo&<0NdUe*HYeff}lXo|luvJzFFUQ}L|fYkoI$`3m% ziYVf|>J{~LI-2LuvVR$-&T*<;nJ7P(hcf($Lxs*9XOA=9l}AK=$-dX>FkiE{ZB>r_ zcBA!P|H=N7j`iUH-Gj}-A=x&u&G1;fX;j_FjF@v#b4^{*GuZ_;n2sCgsQ17S>XK#3 zVf=vUS9+4&C%olgKyVuh?j^t75q?G-RHdljC(m?#^ZXjV+Bgt;zeIlx-71vhD`cZU zh`&U2C!aIz3TN^JU4~|}_$(?WmxVXbv}gmR{0-Q`7r_{g^vd0*9F?|X)^yuKn~~h} zpgDbDSD&Nrisg!XH>|^n!6W{+LQjd;_1)2{VvJO@cA936sE*ag^~6QuT1L!@>ZC%O zLZ@Y}=7-91Swc_^H@kp1?RwyZ_tMZ(&rzQzPze_8LzG3vz-)U7o$o)wP;@A((S*4w z>Qui+`TQz;oK{qsdIAC{gfl}8uE|beFoK%bQEHTZmPE^?feZbo%&VPa{U5@ke9kus z7Qv9ZN8fC`4qJM0RBX%(F`Y3@u}Sgmv9n^dQALIrL%gxc(4u`rSqqvw6I5dk+Qa+( zQ+%UQ1Q~($hSR?hott%hR8k8*fP+pm_``fEx*u@o$I(4(0i(H#QGE;(S_s?+TR6je z$urvfXLjI6y-UH$rFkAfcc3_w3tn~+HTql_3r&ib!RI}Lw)!o)>X+%8DbN_wlYzg0 zdi2}$wojlhv79P$j-nMTa~{zD^zwN_QXV@pkZr1=LtSL~AAz{0GW7m1v5`Qu{?c6IAKX*a2S*{3`Gqf4+z! z&N`G^e!+K1I?RnBk)O^2Q(K8j?*Vvn4d|JCDZGRdjuQMv8A#&)LA&KoDCCr(JQGjk zdlAjCXX%N5iL*|o&VY{)0NtKfwxr zie^kCkP7;`l&>8^S*9ERk%{17OTiUOy4h{4m&x$?^QateWnYjSg5Y0+?|znhb2XAm z-Tg!KVe+W1*yVZn+1ye*30wLuIxJ6#d&Iwkjl3sHy?q~D%df;5RB{rz7j?oN9YImnO#MC?-0B+MPlN9AyJ+?lQX?Kibb#@q70?-HtchURUuM4cpexsi z64956%U}&}Q13jhs0R`GJkMyM^8*3A1XgO!rU z9K{W*dCS6HxJx4c$_A~qjBDA!_ef~hAoHf-r{DuIf0My~KF{?&PakJKeq=jfBsAdm zbeG=jMDAWO@3o%(R}-V|mbi4xQWk8UJot6x%$d}IFXr=nFxg{<`-dHT?_qwXf}hQ& zYX;ATUXhpIDEZo5r1#r2>?YL=mqNlFH}cWI>_}>3d&n=>phdNb`I2zS^NBKF=Cx?} z&t=2^yOOU)|kyzhcA`)`5R2Di+}s(H61j6xcLgop-j%~O`m{*lYQ zm1gjz;gL!s-!VKc$x%-l)m5(hfw!o{bCq1>CNZ{r`gkS0+d^ikl2_I6$4W*axnorg ze@I`GoCYONILW6@`k8#zPb$Afa^{p=CnX0w$tzHD;%gqBP07!1A0ybrQ@3XLCC%4X zK9>FO|7{%}VKpp&>`Bh2(*3^7oo?Yhz8W5#H4J^kk({5W2s>F7V??&xiqpOzEa4DCh^@9xsC_^ zozg5xo}ZFGr{sAk`2b3Mb(Q~H;h&PeQ*u!L zKWEYh-lkF~RQixuQV)CsA9#OCcS`a~ljc zbtAb6O7Ht%P9%R&$${;`_@)1qRIwyK&@UOK^ei>6IS}G{e(~mUV|xgL?yqf?{|oe4jN`kbdW1 z9oRt(u!)madg6oOmszE2KrKBE6E=&_t9Vw|z;GMIK3o_Y#XEL^lIX&nLQ<=iW_uY7 zu)F+38a#m>&{wx2J-pZRoW119sOGIlF+@Tj9SUEBkGF`fTMkb^BY2!Ng^TlY6P4X& zc`IY~(k3mLgd<|jfO=fNn zadl=^a2-2zCSR{4$A?1)20~)tU5y-IkMgnyN&bD~*w+@)#q8y`HWCAL@hmQ4XOeii zFVVxg%xo6&Zp}mwaQC=}qlTXkSYSL?Fy7!7h(r%LjP5R+tsL(2U7pHjcnrJ4(VQ%> z4*A$YcYiXcM?2?P0SplX46a=Ez&K`oJ-badXGlG(ZazB>-^=R?Gd+ux579Su0dbrZ3yVy;zTe$ZoJo1&mbMOQc$Z#8Qy4 z(r$5*O@`fYh~E16FscY-;?mjt4>()@jSgKqF|#H#4kUsUQyj+a%>Y9wRf;_LDVTrU zxiy?*TdDEH6CG&j3%y1SI0-D@Of(;7qx^UZoY&hZJN*^Z*gj&?iy%^RSsx{E>N<$3 z-ovZnFt4gmfM}pQC^2@bsL52o?wCqE&`lnF7#+fOfqzqrJBq^NUVKPq!GHR_cP=QJ zdhorA-N~Nc`2_HVheGQ?W1xcwvuF+~bJtXZ>I-Cw$6=sUq9?Y~kcLCSQSB~#CoTF%R|AYC8_ie(m9I+u+C?qegcI z-J5BiL=<(;4b2{^hK;h;WpuUT1Ub`l8tlgP(7W)>|0VuiIfv|YQ1x4NhGx392NY?p zp~*NMtbSV5TBF<$tNUI(f(}!%kPW}YOcW4}rcfPwT{O9mdRQZN1@Y#l(D5*N2v2-8 zT*7GL>hPq0X>1A%nP1$yo{PqyjhI%;$%E|_PG`h{oebW zzceVqKdVvvi^|b8kR+}>a}`HHxxm9dQiap{5oH=WV_ z-a7&Yz$~7Do9<-P(F$CR_*VVUo5E_d!)DzCqVKq{idg-E@(uL&V>NrVSv&(-#y+%o zK+&P7UZp*wz6mdWZ_KZ^Q>K+htG+i6=;Z%!?K$#!#_+yAihB@HA7p19#ow^2HfKv6bM)A)@f&;zV`>! zX64J`aU8N-@>WhRRzkQK1lh-^dENuT3m142&f==Uo;GyP)9?Mx2ih%Y$8m5JCy$8UN-}xZ-$66KK$Rsy z*Pu%V$6(Uq0*PZyKQ508U?zSJ&SNb)nYlQ)aiY@MSwr@>iQEg15wsR0KVHd!+{+Fp z1Lq}WEcq}L+lM{IZ0xp&(M)~E{~}$56VyIXCSnIkgIBd-sLXR8w}`L(cLK$sLe5!S z#pNCHL^9er@cmNIXsTjQ8^8xM7tNt!?MBTBbr^-+dnjoBm@F|KKQsepXe;mA%j|4o zERw$&l`-O#Tu#J9#*<4nP{2tg#R@CAgSnus)}czb9Uknb{?&f9|23)%e}wb57Tnbp zx=C+y5BK{&4oGa|A~=$9?AqCikA%O7e^s7T-B+K{2-*y7srCu2fPbAHSIjBs30))O zu0e&=ivysYeQFcC2Kzlxe+z4&6;t)m+J8h#FcMT_$ z&G0oPX|(3zUjB2Bv%Fd1{8TT)V)m*zD8n0f!rP z(w&@~|Mq|F|2X(8%!xCaQ<^Hh+i*l1z{9UvAFD0V&x~H;7}iv5K#LiO zKqVir61+@44rH;jU#2(SCp-faI}QEzZ79!8V?K>M6}3VhJs(`pLjMM}_lCE{y%8VX zqmC`McPv||&0Xrd-B$@ZTXHGYvmZ2u8|6=nZ>R>er=!Y7%!=JG;`)f15p!aeM<>N3 z#7u~GL|F{0H9r;C$gA-$DPb@FZ#r=(czYM1j&{y-6nE9Fku>(;ZQ?x8?7stZuhvyl zMK0Dop}7riZH&ALhu~xWKY2H}*EwRWC;Iz(GVhn%jrm@CEB(%~zL4dNJ=^^&e*vlq zSH+Fmc~SdfuaB~go{=~&alz=Bi8~YKCCp0bKp*m^u|jiPj(#qTaGkf=-R!#V>~hSr z&$D+sdR#MoaiNaz68V2AGbBDPbv*}o>bB_VsK3!!RQ*(Jm30Jv^^ei?6tmcR$n`P>r&5Z^A&r8`=l=}d{>yPi$u>IwI^lO!yT!n)aj|qQxYDU^U$i~ z?nGFk(Z@7RGBiScv%PbMJcGw4b!8$V>{x@ z5;lx%i}jg03{~2ZDs;N&5i~e=SqgeA-xqvWb?wOI)tB~Ox^~5J^K4Iyb-HYZl6EzPJNKl735v5|+Tk28%+Noz}K#Y=D!<=; zLwoIHSK{T8i{%%KuHF2;yf@o&-1&H5z3dHDPSni!wGVYYQu}C6hHWeehH(iGk4@3+EB;qJ8A4|$SiB<+ho9i>xm3g2|E80;7~*Dm&5xzt_RyUV)T`HFv${ETXWacTVWSXJ>0mp!mu7%vZ&YlknE>HF^F+r1$?te-QXoitf68E8`>Wqnx z+NhBWybr1z;|=+>qHn6cPP>%$&H3-P+|RN!4{q=r0`*a+Zy#}e)b*6< zk0gvQOZTK*Pf3Aa*)V3_sCluOQFGNhLg_;}&MIo%$3T^7tx7t(*_LA4YUl5KiRkex zplcS;6&UFU#El&}5Wg+1o#!Zl4&(uRTVD5`b!_Y}xGV2IcVx@Nn6txntQEX_k@Ft?m$V3a60rBP^_!!d`07oF7vdUXPUBTEP1S+^-qb*yd!a3+fA!sC z-P#+cuW7qVuJ(Pm@xHIW#2VxJkH0zM7IO@>BbFx&jM|l=jl6M>u=PxPT!hO{{@ESHml|^U7oH03! z-MUk%Hk4Ez=l(Z3Pg_s+Iqte|mv_h9-1zO@@2=d>G-q1J4$ks+poO|XwbsxR(~=Mw z)1F*5nqFa2M$+6-+Y-7)24a>PZJKRDODI4k`f+-4$v8zeg6ju~F!UM-eiWBDFXDwW zDnZ+h_I90dvayAZ*liG6S0jPotNt0DEN7yvbYM%*k$Xq(7JlD%>)M@z{qNXU57xM! zfVrHB!|+bSjOajI&8TUkuE#s$%0})Pxio%G+_n)~(-!?8I){=@OJ1ZAS1<+sy0t@c z&r;$98{XFFFQBNi1IL{*)z@kp9Hb}kw0#oJzn8AW#NY{EqNg5ot^*a|Bj&XJwBG$a z_;ukTwUCTytbb)V4u!W$O%9ymrO_R@R@^hSM%(CDJ^_;ao)Hcly4q=?RM(Kn3(ySw zI1nJ33J{r(q2662#lUFH^$8YKF#6Du_(Yg51QpS8)E(g9e(HY&Kgj7gsCMAxnm2ff z?%z@C7Hf_z+c}%=$S&MfTF_y-p?FfMQdcsAUAVh+=oW$@l{B-{4A*sA@J1g^KD1W6 zDs;%D8h2V`KJnHXD%rK+uQ)lIh~CQS!PTJxvjc^Ae5E;$I>XB%g)q}zB-T1gX4~&Q zjjH2m`nK6%%G+Ee=$5|j{TP%&6{;SbzU0k8)FP|FL0?clqCAJ+qaP2c1bAM0^6ay4 z+MgBvCCmded6C+V8LgQ{GMkmKuUhH09H2fS`4ME3DQd`frcy!M1JAlNbO;a8WpJuW z&?`I~dWU|}kAjaei(vCm$@tVyRf70Gf%y0XR8wqda3+#FjiEa93>nkAWS_Z;P2~2~ zR2>RrKSuNHS;hNARk?~Uh0jGJ9qn_;*hI8b5LHs_MA@pNH9yCvi$Uipv_Yfz(4i4@E*+#z@GBe9>5%aks zdFbj(UWh9}>u(UU6d#0c&;!a1OANYa;Q{?z~mdm z+{hZ4QVA+4vIeALXFw~oqja_jEvdWIxz@mi`wZ2wJ!BQ-q07{?R@3nrgQ8T}mkw5- zGKfDeDwu2FyT!5MQRPFW=&hinm7N<}A65#rk%8bjG}|slmMAnKqjCjToT>0Kf7fOI zrR)ymfR?!~)d;y$xSogK7sm(J&}Ui1{~r%GMkEEKbVa|fhK%WJ|MQVo)Q|b=!=?1H z53t9Kljl;E*ebshTtg38K!M&HUM_?@Uy9u_rDw4!Uoq2rL!ne2AZ|UaKIxOmeyS`9 z7ShRYrhYgOn94Ky1Fv2a3A`2llQNZzV-xOKE##;!bQXUROhd2iIts)sipo$mYb)O4 z(H4h34!o^wfvfp9^m|d%179~sJ&ueF*Q-Dt3T?YtSrTq!6Y=YKG$5aih-4zsa&ooc z2C*`bAKt8d%x454pbm~!64|j@KM2+cJIH~@$u*G%c?}uU%aIoOtG-W^3n@0wRo(Z? z(ZoC~`Vz>U zl+krn(A^9#gLR<6S!q4GmM46BqLST>vJ?&aJF-2Bm2}Zi@Z?H+BQ?SSG|5&8sjSW- zV&NYJUlvj3p(K<;Cl9gK~+y1fKXaqJV;=0bN6vyfgeo@UP0zK9sb?W1b%? zYf+!g57mla_+8Y2riPwT6r)A?I9ipqK%so4!W){88pj&+I^(FUHSyk;l-oRWlv81@ z<1|55w^e?C3iNJSW2jCcgGXvsj1TS;l3B-hsFr^s&hjo1CX>J3;hAn`hiGvRXcx(^ z4ZWxy70Oe5?Q2p575jZP;u0#;$H|5A<%J>Ykl@olmL=-($wy~9M*e69`s zRC!93?JkyUjpdHTAEf@ra2p;3qr6wW>M!g)`d{GRf-yF(ZDEk}Ve4HTEixrXwI(7{i>7u+lc zf*09kCB?F6yt<6SDz{DR4NVM;RzDsn0c$umkg3=m{;T+kzX`tYxL_O|%wPKJ1Qg1s z$&Hb<%Ljd5;i=aQ`hT|OPGDG!W=7&EQCj}%ooL84u}?LH+T=l0-!=;;d~Yhd0*xq!+59!4De#O~72L;jj259# z5Lz#<$L+6L@n7QYp%bbCxx#N0Zi-RfHByY~)~UyU7~3EuMt*@(>^9-wzUasw)e3J` zM9*GvVW?jJdtV$1BQFV)!>RI#oIN`tF4PK}sUbRphv+YBQ19#ENgDDkqqF>Le?3me zneupddvxjGN(Bl7&XcO2i}QvY;M4Yb_E38{J2YMM9;~BA>hRrxAQ@?#>>b~AjnUJm zq#{Z8ra^H4B-urwm34Jok;a)462J9L7T!Zc_^c{4v`!;N@ zd2~d^2VRCDzdG=S`W62TP_lWU-S7{mc+w4>)NIbFe?f)w*LOh<# z8&#`A;SOAHr=sjg1uFElx{T3$5qLpW=7~Xh?IEvG^eR5~`;_gW@qz7{uQ_Gw4GN#t zm#Z7_*P)l&E{g=df@Avy-}4iC_AT19p|JW!_{Bh2SsW~u|H1dVs#SKxw?QnR9-R-@ z{xOVs46+Go54q(*Ejq58Fe(0Lg{@&PJ&48PWwXC?+bLV*`H^4~L=^sllsgcy>x1N_vS7<&`0Yx>dH+-K>ia zCHeoP+7bDg|Hn8&Uk_%3T3Z-;Q~7bAT=u!J4)v->RBJ-7`WEU3{LlGMs&Klap7@mb zssFCx-B1y@l9%X7tU&?XG4z^hy`pdEbyc!>)ECXW7laphO#1fFJxdke$4Xw^Nl-=_{4(o%} z3KV05PYe2>3*3AY%9E*J*XV(u>}W>0@g>FGNIp0(dVpaOh08xFQ+)fVyv&z>@85@m zR|?#lnVN3yZLOGz-pv=m*Hlg64F7HAvx+F6OChP>6`*grL@?kre}gPwd}u3temi&n zTYsfC0ky#U=v8cU?b6l>WkXp)ny}Z8$CL1GASf@A-Sobu)`U*_Hfk!+H2IvJq!HcO zJB)0ass|z|=(nhD{4?NFt_Pjn1}gr9&m2**cJff@td-|Q?D93CF|t<0n$X3_ zX7QiImrpBu!%5)ef;+BOnfX2`_mGx#AXDP!BDpYu6KOLdA0;{oX_(z1{Lo;^(zWIyQlMDi;Xgzu{an zl_#ehEJ~JQ4cUTOmWn6mGm4Gu1ZmujUvN$=k`v#`vxNdh9De1AigEG|x)(0`k=Hpb zH1fTH&JgOy)VVw9|43c%L=;n7$u4UZrSivviLgAYf~hjI!T=IGnkwrd6#iPcf^leL zEaNmaWcYOFKbT=E`xhBCdUwj>@}ehjNF>q1IEq%M05K)fOUZIB)4%H_BdaDpD59dggN}ur>SaDP-dtHW>%#yGF_o-nN%$^t zzJ@BRq^4`&_tB?E8?}<}s^^`|bg&9}?}dz3(kyMHN-g0);v`pi;t>fOf^IL@{T@01 zCEN+-7rgsa#y_8X(h-(AObujBpkTsZg2||p9Kyh#i>#l^Xm3lNb`su%ei|7y9Cf~4 z&ix$6wKOrCQ$ZHw()-^;1zqZ070N}r(fIMeZtMgDaDcfpkcqX^NgYEUM{*i`3#39G z_4Whwx0i6Y>iKg%=o+a*T|h5_42h_G6iS?hbb&wSRRN0^G-04CunV+Z1&)6KdI1@z z7)+qPZT0Pfp`_=#P*ox3P2>q!Bz&$sqV{M?blLh^`b)V+9}W~TQ4OG{i}bD93v?%7 zETNebkBg(1QAu@wlq8uq2@%wb&TNnz`2d)PMzEjtWI>YdM=@jWMTPKZFr7F0zww>I zZ^}Z=w-kj76j>Zc9i0yKVB6qA(8QZvneHrlnk(pmT^X9<+3C)A&GY0&PN_3=dv)oC zdYq&3a38xFWr_~O){pEQ(X9)sZo_~x(!q8pKN9lMT>e!c3w4Ciz6yFlH`u8qXF0kF zsFnXkS*f7|q(72)nHTuhaH;dj$}u6EBG z`YcccZ@Li-!AlVtDrbLEc50mXQneVf4FQb`y~r+|fV$vzoZsE-KehDaw*>zg93#1F zpwXwo*X=j{PXnXK-Wuq98~AOd^p=azn7ToPQW%~T+~_Z(^O#NNG2K<*JmSc)xy|Ry z#jq33*^}&Q)M;kf2kfzqz4$VD0&lBw4SJXc-P#StKJ+~vi=7?c5kDn9Ar8g{z0qB$ z)8Il6FL<2HiI43WoG)MWeGqCVA5Nu%cMJXg8}fO|6PgrVsjd~=`=3>AWsjy+-=aT3 zukcM(y-*CIq>JY;hmOcw;qt(3ugdovj)Wz_hVUrXoSjaF^pqCyn_9ss^0aatN%XpY z>mTd8gjZam>#@Pp_CA<(T;=Q&D_Ja;#aPQ!LM}xCG@VO{xUW473iylU& zzAtJ+OxXxi?9TYc*bYO#`XCx68Ct1-Zd9(}%$2;GkK!qM3(aT)=TZsUmTjSxXx%>v zPU9o=ES4&JaYXqKULswZJDQ`KH&joe*!i&X$0)wsz;Ami`9m={jmqG}Ais}p=LaB5 zQ^8KQk!$XtPgTxte1P9tNT)27r*~}N-@eg!(&W2aow2N~Bk-e~_EJZRqso5LJ_DDe z(~kXv-Hzfx5EPLt@pEMO5xq_)x+|ZEsO+PO zkOzVVhc7hre}bmSe}mIqj~{6p-S!;1&HGS?A0_`>cm%%u3H3MXKcVpCP+C<#P&q)0 zysx}0op$V`z3e{YsU68UX>Rb86{2$dGW)?OX&*%?{zo8EoG7;HK`whxyPbs^>OP+R zeeTKV?6kve+%@>lVA9|KephJm*@+TmE#)G$chrRU(NSm9WA42XucgKp zt75(^W~akd=K0$9&%kG7QwPE;Wn0My=BUawPWHPN&;YrHHiOo9%{Viv zIx0OXWNb1Tjjej8ZX-HtophR~Xs)OpQq~G}R3M93IklW?Z=o^cWH0NXQrQ8jS<=2= z394Pf7q90kkJCF%W!(iiS68xM6oc!^r!y_}e;YYli-}_we=QjXb0KWLQa-0kQt-sl{XNv6mqUNQ>9r+SDxw~Xf-pjK@dg%X09ub zJzR?KI>5ER3kqCP^&OA@wHh6*DBmBv4}sUvdgcz@W(B&rQ`w+{B>%Wfrq>y(`8Rs*AEDz1f0E}UnrEXGoWu=!1tR#o9n_V_6P1i-hqHshDxr!lX_^ze zvSw3>LM7C^Cy10`>PtRFV?ZRy50UzVVWQS# zXs_d@R;jmvh08HsGA@o<8g+)aq5+*A)LYR&D>5|bJ9P)Og61TM!q3H>!t)AzdWdMH z>T(^=(qyh@GEdVEG+z#|4h=kSJ?u-xtj#t2PZRs*GFD^@^~=e`n~Ol2$cA_M`Rsoc z#F>>SbV=x!V|a3hi8X%93X~L9%fMP)@sxRdD2mQyH>}1Tt$?R2huCf@XdRVz(D!cO z19poRu(X>LiD=jVL6rbfZR0S0C>hnR#b{KmjjG3cus*65MBP$jBK(?FhRON>@!EfI z6?+r3({d1rONeXMlOc$V&CY!~#B&sfpNWK&EFzx6?}+R2u$M@?$y5-FTS52qFr#=I z!%{G?>NkwlYv=#mTaenXaGNFP;LdF77K}}={CW;-(w6JEk-#}^&Vrf zv75Em&VEq`woK9({0rx7Ix2B_iZ$%X2J#LwcPXEz$jkWh7@>wSuIAKvJQxX>zy*2& zHsbhkU~5W1{d~%Gjz{zGan8TTx%LPMmrvog{1QA+51f}Rux@@4(Bts@1!$LDT;Wgr zdbF-z#iu;QXGPcctoJl)^G)L3H~IUf_Z2+7-V1aG8}W)*7HOtb@riI7u9P2C)fVk) zom1DOFW}1LVA1v%Ym9}gzCP|#tG<998!qdtz-j7vD6BszJ_WZ~I%_83NMneUF!QZ+ z7sj*lu7gf1AwG;9<_i5A|Lp5vZUVzs_A~fFA9Cm7_|8d0-UZA9culUYi)(uiALuIX zn&fit+101%h%_ifU_DUnhx|hBijAiZZ!B$<9|L4zsAexn`}lk zC|$9Y`ht`TONer5=eW*ORK&P#Al*3 z4qjjI(N1^jQ?C3|UWfVFLtN>0=3+bdc{MNT_r1&4x{0u;^Rqs7b1hzcUeJ6e5-1=7 zz%>LERTaB!H*wt}YW`Qjg|8t3N(3*3vjH8|_vxwr5be)Z%0Da5qvz>X>Qp0C4xH03 zDF3Ehi|UY%2C4 zJ^sJ17CM*ByxKWkTX}Wzc^#qrWPhxj1hB7H{#uW0^`<4@^tKr4JLSE{t|dWIJa1Lj-e*hsi{iHlUt@2jJVK|P4tgv8po z0Tx^0U`PxPDH@l?Gmh89|NZlE{vO5XB!-m8i)tA4JBeW;aaC>%za)N-#QFF?M#=-` ziZtWWcS(#Ci7Dga_#)J3dui@V-aXKV)=>yJ5>i>IpiMJztJeV(OToO+u zb$CQ^{H!!@8vZRY9v*O94E)dkaaSbvg~WLI|NkDtcWL-J@9@~9aYXPtcyOoe!&fE!`G0I6iFYHtr2nxGU*`kv%!6^tsO>!9nFc7o#WkEGfoo>nOk_+FYe-@UN$R9UvU<8ltvpA^sj=qD z8{i!5W2JO~Hjog5X6}3=zx5)qhV*wTS9F&rJxI@bvA7Sz8X*hsfUhjdzDB8dR45U3^;>ve)(er~QUBwC!caDyHuOy8>^|Yo0#Au= zDo61Ye;~ujApAVMm}FsU@PcPa#6^8?6B*i8*~8v%eDBhEZwyDl9nQuF;XGA?Gz~6$Lm89f0qUu2`#-+>+KXnj+b+0$b(9+BBgw2nJe=R^(D`ye&Q{Hj~2?DAI&6 z;O<`%qSO_r;gpAsVj|9c?cuP>#SYU(6}?rF8;Pc4yI;L2(i!lEzNC+qPkxdsyQFHC z{mOqAPslZj*r1snz#*Kub_MRTd*(*A2mdB+5Ml%Oz3UaD)xpT2;JYwq4hN&*JiMV? zf|FebsuMXlnq@}zh}Eb}toGyfA>&R*AZwu}yE6PPE@?r|;VNq25>uj`T$^1I@2ShU zr9{I5I4<8THblCJX5d-LC!$;L!uRwi!5ljOE%0^@!$KM#9s^dVHQX(ee1|&2pRuQJ zr3$=65ltocE*^uO4*cu{xL7OBFUTJbpHvf3F7zA?F zcgm;A4$;X!5UQb9la8X}VcfdVr>Bk~;Ws8T5<3xfF(-5E_(;iY2C_|%}1>3JQzQJ5aA}41Z&p;MWJ^oZ&$6eO_OXOzr z=?|pxv_$jj;53M%YP2%E2Zy+9*2!gNv4!$t!9oHcv6D|`6& z`v1-Mq|>!Xwv|eJYxqlg03f!g@i($|r}Fu3`r=!u{Y$u_Xi)OkVWhpqYhU>v)nBFw^8>CYmXGu5Dqr@1h zBX($oX(n+bQ#nx@I5py^(Jy2y<(#$6_-D~+CRUroy5QVm<_d_dg6xu+R7#R@qQh&6 zG1T+a6v8)?us#w?1U@2JeG_*xoAJqrdRmCEB>me|BB6138(zk9ECxjr{NmUf~-_d(ZHgWFqA{J9bY&l33C#8_|uiOV}LKc1L6jd|b7JoGXHBBy7t z3N_g~j?*Xv1{@3A-}+M75$=RGu>)#_4azB+h&Io#JSu5e_1F@1({w$$EM`GWeawRB z_$Z|zN%yV#FU%Tv19VW=;Fen%c+CGF-&x-foRUdodJ%91?d%NAJaNqmB`#N4d^qrH z0Ewqqga-aZyz7t za1OA3SwDDYY|#|Zt!R&~iJcN>imi>QiJ3P-9lJJmXWX9Ho%n5Zpyc?ZQgWJ$Mk#cB zI1&i^ZhK}?Pun(B?K$s#g=);>#QKGydYmm|(EJFi4x(XyOii~!X@LXgQ(l!AX!2$B zqXm?@%INpxp_fuJILBUv+V}@>6gt%rtU*=>3J;kZF-~XsL&0XW%j0$Pi z&|I9jKE`t-6-4AR&Zzk!E(=svBAV-0*$YG3(ii|vD zWN6f=gAO`qO6n+~qo$^$j#^q#l2JLz7#SHFF=XhFk&&T;rG^T~oqfNt|DUz@@O0*! z=bSU=&+pm5zOT!>)_O1B_xoOl^)9}<(z4rqi4!6QlxLyrkIHhY>=c5TE{n26PmG!w zQ6E0TY%%-I>&?%G*BTs}7nMt3$+2?MZ1;TtXNkd8ZZEQKuq07QzMRat@t$V#$?ANm z?C*V&_3~m>p2nkDgAJNNZJSey7s#Y5k{41zJ9lu0=LuIoQn<)g1(JSN-UTs<( zz9_sD{*rRdZdhOEhh_|3W~{c+MmrpxCAN6DW!Kxz+Rr=QqXPY8_QNhDr(CiGe|;lt zXBC=WJT+&Ob}C$a%gRrba#9a|1#j&^N2UF|?T~f3WyFAktc(8MHB^2&+k33{^}geM z+u&Q;-BaAR+I|^iL=xF6pHp{Wmg1zygD_}Kt>y_4mEl|Ae7;KUqGj;@8^f0ACs2hx zRPGF2NZesQ=ky*TcrUq*JJ#D5*%QDr&!B324L*$1oQq9Z`X`9Fog{BFLv4Y-XD)G# z)iN9SmL{T4UDQVyfvlJuNicZZtQIN)TqaVfCAL>J&_JZJ!E(s5%W_n3c-wxUDsVh! z@nTp#zLsi;8Rcnew4uf&;s*D9Th5~ zsS9x)Z_7c~qn;z^2@zOy5uDrUc*o$v!zu^u4Ld7{Z_dd*oOFGhMf=z-ANBkI2iYR2$92bavO~|?r@-qP6oQD zi7(?6l|0xlVha_%oB4qwUH6h=LeK0F{}Km2oFq<-3I&-o~BRT-%%~) zSJZfPfW8*!;$QN4>(F6Uj7?3bs5G>mOsgW8b{}L13|HL2HH|%H5bukq>3ITt^}X!i zU((Vf@+H3|LNiQqgqCYSM@i(J{~xW-x|b8ajGTHI@={KV6VR?2 z@tRQooIJ)#5UIGAEKBO5(B>_~jvgnok=$8)L5qpBq=J7FxzU21C52dLJEwmtK5!YUHicbg zAwDS?+&8VnZ=84$)A3LVZFD1A)G9nyePr5>MQ^(H4}EGWeM$z|z7fCJDE9kba?3;0 zXr^aLyrUXzOJq(S;TzjPf|LfNd@*fHpsMR*$n6|5ACl0s9wFCJR3%*n4u1|hR|+1v z23oy^^*0KyRZGARCKz;5kd@QyGgM?In(>&}H5ef;nr=Sv&`Q?wUc5#du^JnEo!*$i zQOrfkpx~ptl^JREjv-Eb1nh?JC`<9!j|Zt(P0Jm;HUj)tI&EHs1yF#dnoj)n9{Dht z3l6}0Kz)TucOlItnGY812SF>|3uZEb80#h42P+u7u8&EpLfWZ)-5Mg3j0&(!m4ag~ z4R|jq(e$g)FhvghDv%kOgE2^S1r}J{V4L>_LD_N0QaFOru z*zsE*$0Pp%QQ%d+GB~{|u`7>+gkeWK2v^h{$nKwEV3z}E+^Vi%ry8$4!@z>w8J z2IXkwOxOs|Dh!HUvUOm-!1fQe;6HzWeEn%?YK!biwhW?!odcP~1(Pj}pbTmTDhAr9 zH?SK9`)qgyi|p@*td>7bzHPC(S!2{}(`+CIYe`tC*%3J+JVT$U&LzLQ5bplvU@?4HH2lBa5Gh!%8^y`H4Y&{k^NGy=o2Qhi3g z9aim+ysuLKW)rd5>*VOySVp3$9_$_2x15@rQ~FW{Ox9sUos%u8pg1-U$)^uw0x%Yo*RfKL7e zTIvdN5z<&Wjo2}f(u;}`Qt?~KT%~3mRS@&hW2TY^Eoumqf!&|&oMB%=^$Lq6&Dv(^ z9XQ%QvHw)>vF`rP_@3DBy7m{lW||5 zondN=ER7*oG-`@ztJbW#1!ie58j{ogG2U9|S$l;gl{%r3-3ipHNwG!Ri!E!YmUg%+ zsh9c*mJ(Yx{1$E2wt@7XV;$X{QO>E#k*0^tt>JB9&+6A`cj>Nz0g;CFM$Q6+&8wPBZIkYH<4BX%w8Yr08?7D-8~hWVT4#b| zuKlEafnyQ&(lF{PNXZ*N+jFesJ56=^6wN{Gugxl#ZmQ{7bBVbm{2aXD9WdbMdT+SGVGWo-y@_7;r|^O7a1Tc! zN4q-Rvcbuc=M?!cWMt~r>&mosnoJ^i+cX|3eB=p@7G~-)|9E8hsH5K&YMV!`h4Z$( zFkGa8fNMd+Kh}R}pwkj(D?=*J6St1Ej<9SSI5<#Y9qpQ5Qg8w-u`hAeHaY8iEDD+ZT)o`&gWhI6XR>}llu+=+JkE416cyPLdlBuNJ4 zo#6MHneRMp9+~=fV!w%0^xK8yu9JAl$HW6otzgC>FeMu#>@yPJw^_39K&ymkZZ0Asno zK|K|u-PmS8(mu#DCb zVax@KnTd^*&Amft7VUUfH=?&4z+Vq*Cz|dGv@H*j-6rDqop^dYL3cuhPuxSLKVMI3z|y>`H_3kdgat#e+0zax3XKXHeM#W zdjd^C&B=WQkI_Pjh#genx6H)8*hjpx2|G&_h@K|nsms8|OG6_D;~vnAlhAzLM^E+Q zT}?+PE=LcLIy9i?tFb>B6g9i4T4c zqnJ(WlF-W%(1LY==r7q)AgNkGB8z%=cyx)N)u2O(n(p|<@phKte-x~xefa&FX}$3O zIelHUxfz^@(C%yKQ6>G^g1(-K_N!xdT8YiS!>t9I@D1!Nv@vY52wL7ktbHS>trT#S zUrXZfDlEp<*o95E9u596w9ini>a}DNC*$2{M{}=4a~JsoNuVx8s|DARXmb;>rg@BV zI(}9wp5SsUzB#l!o)K@x!q~#73k=H`P>EsKBHjRwES7nQCFV9BdqaoK*M*HCdT@Ze zoE$X3@m!O@3moCq3qdXkzPm5+!V8WpGF}5MUq#F1=VOZER4=@kLHNkk__T(cC&+mM+mti1y89>UrtPlz><%VR1mtY&5XB>KEUbj0S9xZ zrxz^H-`W2@_2jd&;)NfqU|d_tSg9e3_zdktd&P@#uk16lyE{NN&%$>4i|h_KV&brq zk{MSo$P(%cvG!!x`@-6v#5z;qcTJ^z`#_(Fog@>xDTi_IqSawA;mk*l+i7PeQd*5< zjsbsK%(%6A-tp{X7kdiCTqn#*Z-GpD9_-aaRLj1P^FG681vgeh9M8n~mEqIrKs(+< zUU(6npfif8R7{PWI;X*`SAnJw36D$~SqszQ znRX5~QAg-Q`P<;9Co0{F<5-z*Dr#xjO)@N6z`5m71#>s}&_%K|>Wj&U1y&mLu+4}?*hsqXU`O?T(DP#S#v0v@Ok0z=h8e!_`#%Ebc zjrJ0&92~OE<^ywAYAv15B2B0{zhPsSYl3wOe4bk#Z5z5##f(Z07puovJ#mBL$>gMD|H`f1m@i(&jfZ+H6M zl{e{ThL=W^nH$2=^aYv`Xf8#%y+(_vKD;x0oJnsOt(mJ>Cy685ycq1-!=5&3m@sj6} z8IX$&yqTJwulAhlO7DF2=H{FIH&1mH_up_fg<@TpCPt1!E8V3p)1+%!bW2Qgqt*|v z9I%c6+3jgv zW33T`$5oxC`snKDapp|@HuXVuw|;hHXWZVO)yFGiCq`sxOJpHPd^y>RTZskL_?LTM za=GmLty;_L15#^|!|2T<3*$A#GOBLP$A6O*`aQL%dczil73yA7w2-^{qHDVAJ@)}H zbjPrydi!&`-|DzZ9h-uVHu>M(hfotoc}0BEi>8CDgkVmE=|{oIdR55~@(t+HX2@;JTVjrc>YG z!`MdfFGxpf9x@tDrG^64HFDqI7diR#B}EeE3$nm5Wdkq|e^YH1dC5t~=y+K>A9ijQ7QZHL5xnFu- z{vy`wk&u0!8TQQs(*EAQbA1K9rtZ|6Yub0T7k3=(7~7H2cKzD%*0{FO{b(B6Uel8B zc*8fuQKqSI%=o zn)ov{P_&w@+GzE!WjhBay3aWbpm+?fMw_&MSXV~-!5anbab0V=lW%5U%e{Q=`^+DU zd&UpeVyW2VU3d=9Mk7ido=IXR=7^t%T=*|wEPW9n;;!qwK}{i6pW z9q2M>N@d`KZl%^=tExcTIoA_5H9bDIM$W>11Wx0M)dt z3ZLwDrO48#)2vb5AQli0no&3E<854At!{}_3*%~I>S zZoh1)w;k|JS1m9tidq)YVYsMY6j2yg6u)0ddJ#L72;p@wM>*3z6%pX@3ev(HeaIA6V2TcX{pUZUtme;X@g~JFcqH?~T&lvVY{R*wHankuxGk44V+UHKr_V z4b_L_=u`Jni|P~4WZR*E{Zu@k&2D{YV0nMMWwKp@kFtPRf9YTd^+8W?cAQfT!ihTw z8|W*vX)6+*jL-ff_btvAt9IZF^}cp=?e2)Y(cPNb8s8q>Uu7*GIMy?=?=|Nf$y9RN z)~F1M7S-Nx%kXKj`-ingSt1ujH4iV1n>(U0Vm*~KlC^JvjW{h+g&5rn?CTwW#DAS{ zZGac6cc9gJhWm2`vyun>Ep#n$xNGFV%|gTJ!QcEcUW_mCvaj_w;)yRMroYB=cYk(I zYNx9$`FaXr^6uWeft=o)uI}D$2N5CJp!`d8;BD$f;n6XBV>U&#Mpi^EiJ2CA`z_N) z^hUIW)f&ect8~elUnnZcjsFchkI_BRk!tmmUuhZ8+r9^fKZV*~Pr2bVhV@}SdZwsQ zbXXw)pS=}y<3=87A;){O$K^N##vpItcyC-+WXI7PwNyOH>2K?=rKb3-zB`P}LBXgr*Rd*Bhg=+HA? zIRo=7&DP0e-;~;B*Xy>|G7j=82d7EpV* zxj(B%-uY^KYTN4_V|&dOf8RXf_6e39o;-31tYCrUvSeMWxjwSfY&2hvsEhUwuZyi2 zad-4q^WEXI!l##OLv>`^PjR0iZlQ*e@OV#pr}}1Ahx%qlPu;)-Dt3i--#+k! z`*ARLdEl3>NT^h6ni&ZrTX>C3YNA(1bcR=i`%QC=nc6oMmx%1tkaNOb z&78I1Tbs;$mD%bosn#Wq4W1M%lXtuYzNP*os+Ig2=F=yc0r)&+W2r?5&KXwikI<%$ zIjXF$_t*9$b-j8sy(8^rL64^&A3l{_s|R+tlHfAPzzfkwML(r!qItAwq{(64A8i>n zF?RHbS<%xYCPXZV@Q05GOVy22Es>S~%!z;pab!Suy zZ{8eq73sW6A_#S84x7w3!dDdL(0(-6VRwCh}$oJ9_&sr==1WhW(`ng9opB%fPaf+qsB#Eg#75dmSzEtzxD6Z@e~u=~ARBp-v;stWzH zyji#2JT9Wdv_EXGS$L~=4IddZ4VFwlF%TC!*Gbw{4L8vRG}|Mdlg=c2q^;6+(mvPT zZ_NY&a?0^spbGKdh~ejeSx&;+p~hd3h)otMUxMbR1E)EM2;3;|C+_d@{WpO8m^e_| zcd)0Rdqd~NoAbIF25PZdwhauk+MQecb7c3U6F)9}N_o}LXdY+YYU(sKnmtjyF%QLz zh%5%G|B-H8^}(>ZV* z>+T?#!_PU};gYamIEf8*nRt@psg2X7o)%VNT5qh^W7&lj zhEIzq3!hFTp+r|jgrQzNOX-u6y8#0ORO5Ar-4dM-A zGDf1^s{&Z+_lYrg`6DWPrBV3O z8v<33s_~AAN>y**FU=$e?+`os82DTjLKVz>v7?*g*7|D-t2c_>kiR^G0BD!`w7iatasPT|jSf#5j7ic`ai zb_q{NEHUqNykKQuHOlZK!TEs?X(OW}yfk5qpYTV=f?|jz2Q`yk#{#5)~;*r#R+J~>n;=$8{|FMSLtJ7rFl+yMPX!pC2Nf^FjV>KKJ)Ulk(^2M@^UE!cy@;2Hl|9tD!;O~pwniv5NtM!51M_m+U{ z`5te(jl2RMsFOMD*^BY(HR4|y74SXo!#)t1!HMK43lwe={{JjIpqaFADK(s6(_$>Y zV7#k+A``F~L~1_0%Hp{?yw;O>ZJRHVK9|w!H}I4;fb3}{cJKhb7*CYZh&Qd*0~0>C zognx`jAsi`iH%^BWAQ~kj%Br28cG(}OYqCzMisQfRBJt_cvmqUoMsWVA!ah-kH|g; zLvso2ZU)-O7S>iJE29BlBu;zqHdfFYmq7Gf0b>>iT4N80#a&c6%Z49D75b#Vm$8M{ zfXX1JL3R5*2kcrFc&3Xmu&xw_$-is|?~WW#DAiGB0s>DI>uAT)}634K%dC z_Q=>V>xe-$!EIhcv|u}{uiU>Q$8v*vGplukI~=6m3_OQq zpu5+21%A)(QNiyfuvKz+H{(J2xs@Z;X4MbMqpE0)Qd>vNu1%AqX;3d$&r{c`jw%g` zLOfZ+@V(ZBjtLQrG{RclGU#EXME33&R7L-ATP?1lgac@AkY>fbCqzkT)>-{ zK<+>oKG_s*Z-5ZW07<5Ur!bY(JIU*z%})@w+f8-8mjiVjj*+Xp1U`Ug#r|ub>VRj% z;o`q#RF57tSnh8?+EjzvL+_F;P`oYwT6(|iZdElI^6_L$CTL;b)h{z1GOX7nYa_MQ z@Ep_-f4)sFdh&>@4y}jW5d6|24$$U#fVqCp{Q&zMF(J(WgJ0pkg~XjrYQhuvVTt`jWzi_{yIvv-}4?hLIVXDOFBB~ch(vHN!F6^wH~O1|v_ z*e*YSII$3CJwo(g;b00_|2>@dOC?}?iO?0H|Goy-{kv2ESP#!!30D_0b(;KzBs{6h zS;Zf@r@E#(ZE(P!q;~g3^i~^uIsMi;yw54NVf1V=n$JYX8OIW0@6#MReTig~DLFU# zsHfQjpGSx68R!@|E@e+}NA*I0csfcXDzAq{m)=5qfFMXRRW2F2x z&YvmxKL1RG%B@KHw_w`(LYmmGZlyxSJ03ETU1zA8poWuy%9*zP)(mt@IT~;ZIG`5m zE}MyXx7>cv@itk>=j`X~x!x+Gep0dpYC<$rFpW~$D}-bz;GmcPSLBmtUIgs>9%TfG+DaYMxSx6;WAi}YITk_Mc1VM1w2BJ zk>X z@#XLq_~;edA907s00QJXm}CPH|60&skK&(v*nNmxI&#F=@8?TK;zx5(hxuLEaOD!c zB`iyyqh6!V)Xg&Jjpqz`+T-A0df6?GtL7?vFc>6D=95Fd*x$ms9!~^oCpz4Ft~;D= z=OpSbjUWOL4O5WcyMid)3h;}UsG}9aUYAUL!%)^%AxLz!{0_x9Yi5DoXWXPKCf0ID8)=9)CK~*B zVwzMd;X%n)g)@I5ud0rSb_tAf4{%;nr`-DkxtbE^6l#Qig>3$oov{Gs4(e=zqTMMn z*Qs1n&OV_gQu~bT4yv-wR?MbXuPCOW?VTV3mWYLzD7zw|zBt;^KGy0QuNH*l>*W2P za_)6p1Wj@driPjJ6!vqI{W@3@lj9&P))SG(+o%?zh26#untZhTR!+DZ$dlh&$j-ab z-$adqSLCx)_=}XMW#<%^wbW+SDYX;GnV+XihGnB%>rmZJUd1%^a@7Jw2^NP6EagI& z2iw?FMxzhTm0jf&RYpdVF{E%Vf@Nbi6`iJnPC3Lm zu!u@1n;Z(Vh_17fDv+QbT}C8y0PNEPpt$!@rSu!O*^}WfmBq`@aeQ&oiK+t44x$IY zP|Q%3=!?P%4XNq;`#cunXKloYM`6uhB9tD z$&i}HjQ_^<1Sisc^icx#WgL3R1S(C)VFRe)6psy^qBLqdHE!k4QmSy+BQ zqLEA>zNes8(kZY-HyFjKR5d%p+Izya1Fh;PG2+AY@j0aPIVz0R^8ZwDN5jbv6X*ZS z&ObQc_nAO|?SPj||y~$Xi^Q-Pq*s%ES!0xM* zJmxo|Nhm>l81ZUua-`b39g{p}>R0b@9RuOD&X>im-9l_x71~E-;c%?2SFo+-QYGw^ zYJ&PGd~-|Ww@|zGd#tSlX-25XB#$L(xNvYhp80uLNt4jwcEjb!6n$2&Mbq z$tVEf1|QB%_E0U%ByNyNzo%~SpBRA`oIgW9i^Jwd#;FGUCRrcH*l|B~=Z4tuT9Fy% zzeJQ~nZBQB$P87k>b$0op>MC*C*~{u<5_% zJd3U|liU^q`rB;wqt(c?i@L{CoHMA@^P_8km37p$##I)**D4&>Js?6Gw0 zx)Ln9G_mEc|TPMH;a5`e4WhPl1x6{;eGTvj7(VSR`|@EGY`Xf z^apkxLm*G-7gWcZi}i4d)%iM_$xY-noc*X6J5_iH@aCMI$wqme}oFiD>ycyK)T*Vqa z61a5Wv{s=z3XQxS8{G$d*wI>zev%vUlgzIz^dBdWn>D}%%?tJ9AJirrmzuW~v@hZ}>03<}dDxaF) zFUc?A>(9g&wgS5`jkAe)q=H5?D7;BvPp{XEOw^fI^i zoadc~neSq%(mjBN)5&)?&|~PVE9X z4>~;_FI6S6%7xXD?rZdlssq;~x5*E}wKM?<30M3K>nT#Ui2dm?*pznSfvxndBr{|Z zxkZ(OkD*m0bAG0PugQiDa|?AVM4XWlM9i!zG=n`e0c6q^R&+jItp~8@b2+=0BaL4% zLuR~(`JgU$;&&8mKEjtMMc=-`%o6Dtlj~~CMsrD z`l>*At^j5De#mVKweDGCC;5Vr#<99>>Qm(7)hcJG>Zsf}RcVv%3rX?)jU6Bsi!y_3 z@dQv|IiOf#(4+3hXV9aphPiE=+Rd81MhvRD@5vwR%6s9^7VhrX_aDJs`{D}Ie6JeBi^&PctN>30^Wu`#go z%wsSA1RU*h*6EMvSATRyIbCQN3pjsIv12{&e2#e@?fM3m*{3`Ydu1ReTfkLTlhsv0 z9(4#~+cda6L@nQ}S#M|!TWu-{Td$XD{-FE}Tl+KVE$p<@6mq$mx*oZ{JY>yG=21Bn zL5r|Y9wer42s>vJ+V00NUOulhpmF(#@i_y!LJoM@L@+_HHh}C(1Fw`QGPgsjU_&XA zUXr>%+I<^X!GdR5pyuFCz+PB~*K#@Y`7p?)SiF1fuzByyzUxzV;Ou;k#Y6%=Rqi{S^M_ZD8pRu!=3{u#@<#1LWUT;}sK?uwmQ;pO+bO zkMsv+nQoJ@BCI9soKdcC#m2i!)`eD_A`v-hCakS@;6teMHPfPf^rtc8V`&uiMyG<3 z{+)6vnMz-hX?F>1-h2?H3t@9jB~I`uvv&X-UlTP_fBGu*^Pt? zUFw{3(0P;Fb6A!?x|E>wcDpyQ1ARai*5A;ZH_-Ps+OZTb+8j=VG;nk<$56}YTj>;) zLtAJl58Fy~qE270xn237bOn}WH5k!!<01YraHjs`7tAG<`S=Zyf8aw%tI2I$uy+YdFbLqo|3S87xAwT9KkDaJ$k zWUW;7j4TVhTo;*O?}U`XyI}>@Sc$E@*bi3&7~gba(QnJnldas0574ZNQe0t2>tpwx z&o0>RUqN&u2hA-N&Fv7r;5xLICpnk1L$6^?`~}SP=g0@XK6d>^;tB$bDj2lpkPp_z zO1=}!)hAR67MT>Vsc~kev+C>clIOz#|1>N7br=h$A?3eC>wF12Mrd6FVEdJfhaKP6 zZg(g8>OAoC4c?us@p8NhHMAg~S!qS;$|bYpN>!@HL7v2Q{baoZ>+m7vgR%_x(wdp^ zSP<|_L3vk!y$lJZ+7c%lwUUSlBuLN755sAjuj;{;-6TIPmBAPz*qaLAzsO2^3ULIp8$%Q=9j{tD9#5o$v!WcYx?uPa3@GX7VyT0Z(EFC7UAXDdcW6c5 zu|KaN4zwB1?RY%5k8-}<;FQ*as~U%7e?=7FJANM~?(#e87(c+-{TJJAyondvfG zESSLZ@QYW39+J*ftb{2vU)!L|(3j~tsd6Pzlwsi?LQ_h?kCX~Ns0uAvu%&O|JfdzU zapDuwW%A+5uZVj5mh8RjSno+-Y!l#E$l&!RkmEIAhs)s)X~%vjV2`^T5`w0k1M)VT z98c&^d)RkPBWL}bq!fyGnw-;-w7+oYU z5O<39e8H@}g+}=lUhP-O_V~grBA#nLWziE3YzBq4N1e7)pprT>v{@1seTHX04As z_X5(`Kz2$R8r8?rB4QCXSs~ilDUtx`^@OWQC zy1dw3!&nz}_-}Fpc;qJhUv2Oh3Z(P{2z)XpfS})~Br6r_eoxKS|1`WTDqrp&?LD7L58Kyz7zDe(59ftBMUUO3x!t6J}yf zmZlRn$Is+ngE&}5-gFfivZdV8sEF1@B&&+ub1_)|K72Oqu$)|xo`75M5iG(JSkcSy zOs0?-9zt)0$B~LH#KOuz;csM3i%Q8Utb-NAVosBfaEQ-}XH|>c=m>cL8N?>;#CjUT zp49>$fZ()jMTcy{^Y;n!y&H)wX2u`D59r_ox|5S_2b#eHSf$b*?T`&$gyh<(fR@HKlwd3IB{H^j?m=IQzb8?9K812Y#75%s% zGDZX0)oalwH^SxsHb1mlvI~Sj2{G4?iTUjX>yyZe&m`X@&l=bJx=pwwON0oNGH6v65_-T$SK193zH8^j9kzawwV+_5CB@WofS+^fc zSFjbXBRgYD2pRNv`X50Lk+c4bkQkyi?0u@xci6%6@DA+ZG%ny1Y8U~Xzb4S@N~Apr z+24=e@Fbde2!0@HTyS!2;kJ+UxR14;$;b$no>lZi)V!;tH}CUz6*`;9##q6lJGqyS z#wudFN5E+Z|GxX3!NQy zGcx;|uqRtNpUbhDGsskG^W126Iu;*yD+g5}703jON_TJUOww+xcH` zJ{$-{-6L2Lzhq8#qF)Q0wUXXv5Lby||0kA5@3I+zETm^Vk$V-7hH;JO@f=<$#;uNZ zpdz-sif<4x&r9ec>&S^nKp&-=7;V46&5K3&I8g$tBpL0y3|5WL!To(s-#=y)J_I4b z&WQgmkzG)Z7C#^V$X-@@DZ82A%*$m=!iaT?b^SQA>ZBi6@IlDXnhmeo#-nf z1}m85$K%1RB0|0}V25LY!1eM589q*sD}s0QGLiKHt`*$YqS3s=xQJUG=Rzv7DL9S5 z2C?d_tTZ_2&;{2q9!ZRA1CmU=Lw1i0dLS~uny|puA_KRw5>)J;I;0?i{~RC%zF3rrsKCc=b!HP6kk%=}wy^)cp$|9?)3P}m(8il}qCN$cy^uC$g)tW%$yOdE7JH|%# zIDwQH1&T&wZM9+9b`ehw3FMizuxgG3c8a_J76glvvD(LDVmH~xDtkY$vbsR#jA4}s zevfj_jCOXZIL7-)_VHaHPrySW8Di&1MP~Eq$xd2WPg~o_Weh{#$tE%?GCu3jcl+2C z65#G2Qy9%xkC)|MqIdIQ_DB{U7p%4~d8Qm4HU+JCEV6TnEVo0vKbQ5CfWD7sm+{_? zBo=VX3hcou_7K4=REpnmDKqpKy6$9F#n?a&o2Y>k0)yhQ#V!!3ys%;bII%EgXY%h-xtsfw#UAQLbFvhGqj0pR&$OiDS& zuc4(zFq^3X8QG2`*DxX~w7G1~feN&pc6OF9w4VrW!^nr!1^$Zr5!@F!hS%7Cs>tu% zPVbj;iY^ZPUW!gAIOLKTjYQT=7kg?WV^WD!R3aB5H%>6OjAz#o<02Sqzhs_DSt%u~ zyQtYz(85!sw;_V>IR?U5i<5 z8+m6r<59!<6Q}w$_G%#?!q2vr@B9*5O>o+ZjJOrFZaZxpk`FJdJ_1i`0&8Ft`jDuj zC$0$OMm##1OE6UA1g>0mksQWPTp}-5Jf6j0p<`t8UuNK5N?>FJ$9`8})xaFebp)xz z5=CZmX-5|N^sSuTGVIT<+1V=CWjC^q2puE^zscAD?xm4li9K;6YiS+(;$r4sSX1d- zX?RYOxKUpNM!a0c0{;Q8T#Jk?=C+o1Y-e0*7!MVnnDP(lSQ^;B*Rg6V*t-ND+m&tgD-0EB|F*YKj zHpt{GW5p1g=O*%jos395-yxW^sNKl#d3?8+nJj*%{bMbKU~5zc){g6UaQ zc@loIE8G(d$b#>7u%2HQyz8Fa;a+X}*<+Ft<9|CFCXy2^MY7 zWhV3ZFN^+&8Jxr|l`E6y@_B9}{e6S$0JlB#T=X-I-61BhXO9WYdoHgO=j~cPDU(kS z=WJNulnwG|2U)!ZueX=)4l-hkvFT!$6MMhlp_T>CJ}Yb;(Z3MPs}K2ygx2tIG|Sy%Xu;NRdUi+6p^D}#L8LAGlvZ3yyDU*S11 zT0@^In5cuy+*UqCFn6~HX3>hyCAhlxe*H+paH4C=-1FSrFeK##r<**skFB{q)YE5fv>B z&Z&4rNTrZk@gqK4{0#L=wB@HqgU|iP-{Sp4mv}rl58}7DhJM5=M4yJfMeJN6@-Dt7 z*ze$bhh86it@thEVCY_OPJ-_fkBR%iwIlj1DzOK@u@Bi2{R_T3*!o~U#OI60gL5f* zEnX)c5tsOHXzYjH6_jkTPKMeUY_<5L;3GqFA$a7)${%{2cwDp~_>JQA|J%<`WA-0M zH25yjuHZHF`TzWB!4VGj;GduQ>0WSjhGt@D41(V!dLMi~xGN0JU2uMa(kUJjYf1Dd z_--%n8M?&x1V8JiZ~U+S{rBx0>Q}I*;+?@==0Ep(=o5xs|6jl3e}3d&Ywysv46Vzd z&m4MO{2hAi|MK_$^8TOXAvgx&_mIT?ci;d2D<|Uff@g=2buqg`pY+eY8|t-qy*NRG zQWJdir}q!N>Yv{`bZKuIje$6ahI%;k_)zP9`YWCpn$MwE4YhLUj2i0czuv;& zD}!Sf9JQfu{NFx9wBe`u`|s!fr_T}XANus*|Kk1PQ^fd)Zx?@ucKn~-_pkjoG(O_* z|Nai~D)Bnej^Ify^b|3^!P8KDLU269=`HqXv5Sej@Q45A6qGkp{&$TN?$Fd@QjJl zQL#8wI4E2X3Woe%$Go9Wq{_!mIRvDlq%PnJ$(7_uJ}JoJM(V1O`_xG4_k*AbW{X2{Q>z7g|L|&4%k{*$dXh$gpxvtz%%IzPi3!tv}^ma$V7oBAasmkz7P3BB4?OMMfgG8}E(kRr&R(K9rhNQYH1Sl*=*l zig=fJ(7GM{VAFwxho|l&ypNS>n812c~X&xNcW$mev}G%iCy8_4&+ItcPaWK_Z7Lw^U6PYS9y<&1f>6y zca*-N=tN0n^f}U(Mthc0g3*$YR7T|`^*Z`CNlm#ayFk|xo!k{2oe=r83CxkBDj zT8YxLM^hnBJo+PVDmjhn_9{okWmG)1*mC^9gJtbEngQ7ojM@dVmWped@ zrmb8dB_a38OhL*`QhAgfIakUydX(_lE7#zt z8ab-v>VN;Ll*vJ^lJ7J4u8|w%W*qk*GpW^BY7&)0|fao)jHLOst2k#^%(Wn)oavmsTZoBRC`oaD!u9@wvkC^R@1F0fm^PS zUqs&I$dR9ue~hzS{C-|u4Y|%hSHvQCn#!jNs0q;MCwM*=G9HymqzO`=_&($+5+{f* zp+opcI4%?mQv{>n<9qlnzKuu-j2_O>b>gU;#jPHO8r2!UG;ZX&pywz(&OlDkfdL}HBuKhP^+Y6eGVN~ zqlW56>aZd%*et#)?iV+UYsD4f(_#X&cv@I16bX|A zneX7+`A+d?@m1D&2b-sg-;XjR>ihL)+K&l8A6G@58i2PUNdpQ4;`zQCU$g`1eMPm4| zVykpdHqmDGFRDMOAJ?4ML^O={rslThJ?3Q767Dj1@lO z5Ay5z@%+b;pF+0F+^gJtZXP}_b3x7+86SyogIr4FCy^gU{v5FgMUb+C%4U|S9;jc_ zT-40g&eM+94r*>{E@+~)C$y7wMjZ2Xi*+xL=rc{#%WR`ds9xx78(QB>sQG^MAu{eY zpg*lf?@5J}y>bDy0y4gfnlFSlO^{I~<%#bI--C=U=TbUZtdVx~;mox?{RS zx+~fxnkUrv*!@f%>U$65$c8-I&`ZWq#UuU2L^WX~qU0>-avARRqurNbth$3yp;yY6 zJ`g)l%bWOT`F}=!5Rtg^kVwN_2>&AdtMKJ814+l=KZ0L#xOnbI+;mM<`Q${VCiX{}f+Y~}w8y<}hH9QQJ`6wSRJemp!7 zx*oa~x*VDpstMKw7l!7B0>NL0zayk#EYD%Cs%Po0>JI5k3~LR2`VRdn{aMI#M9=FB z^t*K_`h2}nr_ofaK4deQri=T8gZvbJC^86LT@6QZkA>4i>p}-Yb-~=AI#?9Q3_J*I2{ihvf}zL; zF-z*BuBjL3ChJ{>R#TCw+^9E|7`t@qb)|+R!)w|_+O_)G`pcSK>K9ezjFomL&?7B)+n^lkMjeWO7%Hkz_byNzweGGmLr zM8DM-XIf!s(QA$S46}6})L$cANL9!tX_a6V?@0p2vR-LF+P@BUS0mq(3gi`N^Tm{j zc~P}S7015Fepj`PX_QuBJl+wx8uA3Ugc^cz!Mxz&V0O?P$n*QWF85-0$bHq#x@URj zhxQ2x(wo$u)Y*n2Bl0(U%%!F(Q;oU9)M@NC$63~xiy&2=sY(C1W+VDzA%(F<+AntU zKA`~Zc%SeGp+U-(EW!?oIDKFAr2k9jm*cs%PBLYLD;C-U|tyD`BR+hnb^+M_31 z+fBTo$S_+gFon`ER~tz5_ju>K)viYG^3Zo8I#ES6GB;FjW88aNGf#U^zuT}L8onN7 zvn+^mnvDAIvd!`Yp(${}mG8(NJoqrVx2ShjkN$pX_vL%~ZgtOz2PgUt4wVF|$CW$LdnM#?N`8DQ(T;9VtCyXUAW$Y>8?#X|xH{J*i$C4yXCHx_1xfI=I1HN20UP zl^gg2-zaZp7V6AVG1dW#CAubhQFOa)sco8dKHl}1Ec&mhGnrg5F68p$4JQv3_uai8 z=)QUX!u_=F)b3N=EAG?11rJIep6Dxd?dA=d7Gqb;@|3+3tJAB~XQj`1qHA2=n2Ok? zX1l&jw@|x`o-S?`%7p5m+Lb++Hds8s_2~!BIm`S!{}|)ykbCs#nVAoZuR@24eli_S%#_;<7()*jf+;r!B=j|^_JFj-`xKsS)`FjQTHg#tW zGLgl)CR_Pf`;@k?EqS`-YiDN2Q|zfp@psKpnoMaXf101ctqD~5@&dKKM3=L_q&EwC zOdB}jObsToeby~w%2W4FTt2xnebM+CNpoU+hTp1ksb_>!7<+%nC-IMkL%tpED?_#i z`Y%hr7`T1tHr<(eH}&4C-cy4)u0D5}PY}C|zSwDF?2p$?uAA1EvFOQ`C#H>?o3O^p z>fRUkhL!|d0&CqR&K#G;x5Jz3>UMIjA@BBJQe?GsTT^8{ldw2xM)HDja%xv{DE_)7 z+4!U;fr@auf=dJ20~!7t?`)69b`KKHSClO|59n>uUqf{9(@X4nsyo$Rbgiuatm%yZLq#c^`@q<5FU)a&!C_bm?o zgQKKn><;|`+ku36k4;SNc)TNZ&SOLNdA2$F9_j>F5LoXux(^NS9zN$Pa9whi4F(=M zd(NXJWZuv0KGvPwyQx3Td1<)ayGJ}@sD=Mc`qBfbKCNfB+vR;KoGWHAJ-W*1lqAQt{1 zr-p5T$)0`Alg?7t4)@*R5=X&6;NjigyZ29a72aKO=j4~idQSCS8ocRRE99FtCbXt3 znzZDpGqVXC1{u`<$KLMt_0N=kD>e2F6DYixcRz z`p%fiV>?q)9`72zWc>KY_Qt10UDNE82g4oSOT&6+n)9YJ*~tyX4X)@fdU&GuO1J0k z+Rn9icHc{VP~0Eq><&-VRmD~(rHpTxx_#EhS&N=ZnbI(UPh1)GlxkC?#Cu`5%#|^$ zcbp&MTsgj4|9W4RPaQP!4ylt`qds7s6K@@*ZJY=g9Ag|t}b7JKht01%L?2Lf6T8CH&gRj}yEDJ^++N$k-O9h4d+%D0XLy;s*Ss=5 zW!#y`+L;@(rp@S^x@6M!l%co^qd-06GC~gmmpyacDPHtD|32>=PYGt01A+9=Ic}SL zSK~Bov{uKKCxpgW5*Njx{hN+xtxSn9Q0k{PE&8ITlMwUY4MHr&Nx?0lI^y6k?xypuhcH=i-^G@-we+=*Ri3J zhpW0z-YrFWyF2#XK6dBw1J=Kml66aB>Qh`#geLBt*f7yPVOnyzeR)i``Gnr6->O@n z`91p?n~qWQaxl+#({V7SYBiF3%`Rn5|=qwKaAn;6w(wiwTAf6r#qneu6| zkH5&>4wm|6yKfF&=q>Hqfwt9sd!X}D?}OnrLClhdjj?kQXN+xotm(1ZF?;PXaoT9L zWsN1tmSnvhwHHT>p#qEXPx+F-G53+-fHQB{ zu%V@Gk{=`TIj%m~>D3P>4|G3p-Zx`zSv7FVeIR&RT&TSiwaC^IT@gDYmbGn(s=`cL zW?B=~V6(?G#7vBpV`f?NjDJ#>P%M8bnCYA6kv%znb?{m!hF8HJc^>xLOR(n}M5{O* zYs5)nkC=`1ZI1Lu;X9FmV2$s*d%&6LD08x&bbm$Y55jZITk6-ftRdN$YnZF=(fvz1 z5q9heLj`i3Xl=8FY`nF_TwqwE(X;oYN&L@XhepEJ!d`A4Uxbx$zSt@}E&P%HfF~=Z zwfs@6^oo&Zk(48TOPIu8;tb)+U`23a=;d%E{Lct4bV&7bm};k0Oqi}>nsB_tu2YZE z4r^odLxu)pvN5DTr;{`b)fFsOf>><|D3b}F4Ondwj6elDaFWy{_6ajlz6nyX)FA1w zrgmffo{w^G#k%MZu;~94d4~UnFeI3;3MH0gu2c%!}(8uX~x`)~i3}HWzgyX;+!c;7+!y&-dECQy`fc1r2YLV)sH>J%$ z5MF|wm*FUsvar7BfR)&Ub+8JWI3uNEZ6d(Jf0JsUnsGGYXu)?o)km3tT1)_zp`trs zVZRBxHXoKPL0t&0f)9$UW$!@`btCJ>4OlG@3w<^0;LTW>y)<&Z8Y`V@)IM1~j>8J0 z0O^!L`aRS;ko^PLfvq@NkpFSyu>tKd7prC!tVOb_tA)M42Uhhmtm~eG^_T^OBoRk4 zzSHCkl=Ma9xD!@y6;@XDkfk38+636UCBS7!F8h({&XHpeR<1j7ei&CbVCA(6&*VZP zQYPyNTJ{;Rgl6PQ5ScT`c_(sRh8$i*SxCwDj9hQTQ-t{-Xq18j<)KuC&^`GtLitMY zPfA&WqZmg45`j9fm!e?QixckoF{NXn<6Y~+)qI2K33NSaoBlhg?gr(7lH3I?bkZ3G1+X(@T$ z!Y}3M#Z|%`5mcNYa-*mt!T&}vMAEvH^Z&d;a7ThHkCu(xBX}Q4fnbQrH$fPcgP?vS z2L*vtkU^62SO18H2!=@B9z`HY?xR@ZSD*gs*(iFdTqQXuh@(=fQMpEQR!XE`rT^aJR5Uq? z4}Y~*{%8LG^9spX(X@i`K6>JjY|1qyPx4#&HLAr&Z~XUPimb|2MQ`M{(hd|I{O|fB z=_{!#{ey4{O1=MQ4vOT;74lob`N@++GYaQHu922Tu8y`PB`xwvj?ptxDkaa+b~!4+ zqjLXe?*CtZE4e83qSPSCiR7V3@JJ4lKS?z_f{H6xKFM3j>CrclOSz`B&#%^p(#Oet zlHRE7qqsZC?NK{dQh%f`B~=~XB)Jh@fQ%HR9gx@gWpvb0 z=`iO0agCfd>O2zD_W10WM zj-`|3Sj;-xq!wVo?_;dbrgcnMQZWroE0vDXbTcfR>6Dw=0CYSXNQWTRU^K6l>XEWZ zY9`2xbXZD;?DwQ8R1r{^=|GSg@=L&p4D^eeE1 zKLbu$gqf%j2uw9{&4xZRQKm}yFysp3>Wg@G1J13OnLoqxH{={ikTaQ2FpI0`7AlMW zHtV3@qy*YTPl6U&seWk>Y@9;r1EAQOHU!D1v!nm!#qum zL(98?V;ow!73gvT>LmyAHv{e0A&t$bsaK%adbA0vdv-Ao^)x{xohTl%}~OLU(6E-br7JZGiHA$QV~ zwn=F+kV0s*04>4~dvXuv@(lEWMk*D(rUCc2!D9LxZGhm}q#xGHEs|T*Nk5iL(3*P0 z!$O|)4szZJO!Tx^E%!i!Ww1pEW7tI*C%O8w<6#0}N zCuT@l&{{iH#r#zj50>My)&l&t$zU;N7}py+jV@!AvBi*W*s5QvyP(mlx3g|~75eFV z;SZ4?aPi?qp{!uNU-q_m?4A?uORgQmsl$20wqen^$yM*&9@rJU9y%7O7gtc zYz#E{3w>L?7d)rjJ6v(D2f$HEow<&+gGYuAy0-+90}b9n-_1}CP>64{%h=zmu4`O| ziIy4Bowj;wlXXUPL)@GMW4y~+9yQr=)8e$`nXYU4nM9@^qh7Oo9QiEZo&huD~HmN3@!~;1de&Hxp%nFyDz%0yW@s)h7uhmzPM0ZpwFA-pT`wSy-YcK2TbKj?Yt<~ zddV`|YP6Nxrp5GO9o`tb*EZ40T9;aPS=HvBYnCw^*~RMT*|793f1mpYcZgpvEZ`Od zZw9sl)4veXhth&t-wDq?Pl0!<=dP>3d2n!(Q}oXbr~9&8^}Z>5CoujzrjsgX&gpxs zv!e$rTB~f^7_&HTal*ue%9slli*=f9p>dHg=u>1D8GJFps0#(62$J~%}nKna` zZDF)%(OVW-_r}!5FHfkB=WQIy++!6jHuDVK8K#G-XJ}Od-OfJ<1wvOiAAg9y7|H@4 z@Dp%Slez1my}=?s>uc~O`Ypbz?p$Z_P?@vGcR9rQ&%0)Lu7s=PZ?koD6MA-%F3vJH zdY82#>VTyxrYo++&e~lu*DPl(3$635)#e@g->GsLflg%`=nio_cPexW`K{&WhZhF7 z1m~l)leq_>IiZ$7yT8-7!`I@i^>VI5junmq@1;;>(CD4$*&fUhGqENup=xAFeb+oU zy3VSPIuO-qW8>=W9r3%Or$L`S>tt)5`MmzN`a^II>)4}ohV*IVA$KJ5L*#diy9C~0 zF<7E&;n%}&g~ec_f0b{aZXtMWshAvWBAf#)aZKW6oO2qb^ylT55p@7_=EGKUz#WJDm#ge~y%agZ`917I}jc zL$8Iues(}Z!c=vg(x}44e=YXryzZWgH$TJX#5%zF;XH>RasJ*=D9eJ`9#t1}FbLeg{~@ ze}PZ@N#q3gR(NW7491uRz&SQ~r+JHky_PtO9f__pz5@Z9KgL%du=ALEm|C<`un&fs zsB+83sKrr9U=ZqJ8lxMm+oP_VmqtA>r<-=@qqHm4E5Yi=u_me-Yy1h)OmT zuGzFJ>bAMbye4Y0wcKWpuC>mK$}(p}T>=MT2e0!ZtwuXl^SP=5vn}Bo8=SHk#7RX;Dip9t&rwfdu8|MqvG!#%4XQi_v`nj!=SjcQW}*6OH*FY;&QQ2j;$A z_zpiS@(=Dik)Lzjq2@rTf32_0OAy8r!|B6UhtIl;yv@Ew|K8xGa0-7~%#n-e7uYeH zpf+EhX-G4gO-@s$`6hDPYfdz0n>$VUraYs|P-`dy-ZV`&sNujke}*}_33F*JtRufT z4%l20KRa@P`&D>J=m0cZf>K^^Z5obqmO9QkGKSB(SkG4Pc7I*)VfY7p3v9uB<~`L_ z4fG2pPc&vD$1PDCqaH*BqRgmaI;zsV(i||gnQ~2~#v%O)-Osd~`n#&lm=`H3Ri>n; z#ixaD@c-g&htG$W2eQC4_u$%kk zSELQ%H-yvt?#Mju0Ck zKfOj3(VW#K848Wfrb_d=C^56imo$G@C9=n1 z*@dMp@qM(Wn23fuAKDdc_LulBd8WCuhL1P`gMop8{!BR)&@LiV+Bf>0n*xy&F9Dy zz9CYNcv|=a=B^)cmodin`M3Ixc_zB|p*Hh}Jjk=Vzj#1Dq<7}KN&tOFVjYMVIE^`lWif4w`xnBIp1{5a8S2Y^8(P; z5bUMF~u-NOM#;ZXYE#s2L6bN%}Uw>owYFLk$gmj(>se?``c z0_@HgRf{#7bUuBau@Fq>x~S!ryOtPRooz|<{OFC*bE8wDx1+>POGQ+zX~*-x)Vl4Mp)Utt zMmst#QS>f$rFxonx6WwDHtsSlF;_%YSZXj=&Wo;%9*W)`y*GMsv^KicT5idV+Gtv9 zDAg_3q^nHKbTEw#unE7x{~)q}`*}DJYzVMG{SUemU1yz}95)A#4RnJ=wGEmbrNb9T z+R@UGCGw8omtyH}vb)stwC8lE^t{1__UklnwCuH3L%V&^F)@5}ISv-K$P$Y>s=#!@ zuvo`v0&G8BOx3|Z;V78)*^wV|v%|^gqZ$5_Ub`m)Oj)iYb7){7b)aOxGkD$M0Vdw% zRR{Klo{hMK99f{wFdwQeq1P1Ya}5>7116{WLewm45^|jv-4)#)-4H!7y2oa>U9&te zXPFKfI&{C#B&ytWKBbe}MOhf)zeKLDhBHF8V2MB7x89?L?Q<0z)B3?p0~rH}gUOEU z;aTp9-hBU+;5@E_A1CEt9rZc;w)!=c_5zq++LUUpj#^|nV_j!k5WOqf6%EWCOza%n zTv%W&CZn-f|FU+f`V+S|I3Uu)IIz-UuHEaj+HAd=~FkcY*7qbD!hb zP~y@f zhK>vc9D!jOEvnsb3;m3HAEPj^1^Vy6^6zRE>q_ z&e*D~S1rrIl%Fxc8%6tdbu!z9_3$`pEXMt(`TvTX;$9A~3#G#TtMnc5&d2z4Vfd_* zbF6pR9GQ+)&JyshT3@^06P(3u<=+u&<)id!)~3F!nT7J^8ZwMErZeWd=(+1KDt6eK zk$aVGmTfXv_Kc`%5|hl|At!diY4Jb-=~z7Sd$+>RM14<#=2a4tO< z#LLc1%&(W7$*ybeLRg~>flHx(az^+@jH8<9z3eyD6SZZ!V<>T_aljP7%yYmp$J%G@ z!noLOU1vRMiL=a)iZd4&PhrM;Tk}`d=kN^qT;2(Lq8K*JtC6|T^vY0qFh8&tEM*<+ z>x0mC8dgU+!$rd&%-r)~2e0s_2JeOjIi9xy%h*6?Am^K!MY^^6W(hzLL zswmyR+t=jHhHZA;z0|$Py}*6N-Qr30t?*X|;$W5Uj!Y85XkRsS7CWF?sEKGN>+=n) zG1DZP>S4L9i@F$vIS?~%T2zC%$<$}`89L#MV9?%FZ-l-068$ul2K=W}n1B{`GIAEa zOS8kLLiNFPXgv-iW~;BjXZIcRmU_#)P2OCe*{=>H2bYB23MWPW%$JGH5?1x}|6%{G z`kH2jcD=3;Rz{1V*_dUjH=Qx*F~<*pF*BRC8oLZGeT+Uvm!j=ff2O*@-hijz<`Fc; z4d&=uz_Bd+G)OQU{tf(=s$r>KhZjtDU~gbeAQiG?26hDo0_%ctq1&O$;X~Z1$lrKb z&|&>sL^aU2m^iG>9@l8J_qAJfCHi)>y8^>O97ha!2D_nIze(SvE7ko|8xOs&QGci! z0K1e4kF^G@)sul>HNnpL4wy4L|A&Y%auS-j5}q3tLuW&KVY4TPY@r$OquPSp7lCh^ z#eK?cihPH6@gE8M#hvi&nGSp*k?Cdr%C1x`R!i#l;3aofJE)zZn+C6{Rmj_-dkDX| zm$kPwuWO<-zl4t`JV3Fc*w3`lxg&_y^KvWvUQfgO@gOYf_hEH?3@^v=k&E0L;Q1D! zZH$4}*?4XSe!mW%!ts%xMLv$qQ6qWqHZ7z(z}9>M=7)lB*(9{g zVsS`#N0$%IUBEb)AG5ve*WsV*f#>T&^{>>wgKWQskM0`v_tgQ_UsMNF8SpoIjtw(a zOd4~B&H>U^MBM=9l?vpg5Z;WZAy1$10sL2=g^q?s{8m4L?`kJhgx237i{l$6sT4icsat2xq+JzPvt$p zx1NWOW$p-@nG7~F78qgd2&b8j+_Qir768Y65&u8Nvpa#t)}T}kc$+YE_eQXC;)PDQ zH^Qk-2cla5?Ujvip8J7(9tOMm0gltCr+R4heSE%y>u=)NkFsn48e58Ya={=HK2l-s z2*yr4REbA?`3Sm7c+Vf>coFPt!T&22@v;a1aKOFblDe zZ|2C619p^1xB_`^9${kNggl#ZM%Y;*&sy9s#F2;cOhU;>{1{;f2v5L44BUSwM zt4DrS{+lT!Ui>wvyW(ssE(E6y^cnMu3BBdMrA^Gkv^-J2XI28#Qy_SyPR_3e0A*!$rA z&F-x3lHTm$!=l+76W^53kWdl76qw=MvDGOl<5nh5OlY=tYmQ6ju%bG~y^k?I7(U}$ z1aGEl|MwzOs0x)qJ5R^z%k*>g`wSVTxlt1>wNd#-gEpCclX?lvL>B4a(pr9H$mKJ7 z&JJff)14X4%kV_YMfshO<3ewDZ(zU~$}v>g7maNm(-glYW>&1vK6&i;u?rIR#$JsY zP`$yW!TxmyW^<=`d!#WSySvxkg;$~QelD4GG z8rv1$1q5Lzwk>h)nC7?&>yZ94c~`Im*63t!Whe&GCiicpx3Go*j;KLo1sN7NeeX!)bCICR8&#pQ8&9BSun z$H3t2KF|GQcdp+Z7}Bfv!jsURtW9W*?zYT}nV-NXw#9FW&NY9VT@t$Pz77=jfd6EK z=PN@KfqY#FXV5OgoMgPSVz9++_lA8&@+Gq!#yN60< z=4p)jxw;qG)55yo4zJyvIK0YHFqAsD&(8rw)1MAK(R@E6Sq&=J8n+=QkyZF zkDp~KS&&sy~p`#xl!89PZ#6ZX5)eQn8zwoe2MM09OL7f8s-+= zLf?}A2G6xDAM0)Ky&7(nwzKCjq8T*34197Zo9dEw7FNKGls3ZpjtpN1+GloL8hp_2 zd|>Xm{$=qO7dy8OZPjVVFPd6Cc~R;C`vI#O=xvPkwt2SBA=d`F`!nue>psf0v& zQeCaeVruBe)XU8)?fhdcDHF%+wO!U)X@^)Kwt?;17nvF^3haY#=*_SoeN8RvLxxV> zRIt|1F@Iv;fS35NHkJ8dB*S+M$Pu*SNF6%SFZSkkC*I4rcfPB;H!b9fX-N-Fx2OBY z%(HDYrAJj-3QgyfS3L+g>|6s4hn8;1^u1Jis2)RTx(puj#(4&SF!UWcJ&t?*p+criQspU#>HC z=)vHjfs+24eV6;M^g%oD%{|t)E;QG+{K<(k&P?t~o^M^Nzl7DtB~3fsCcYK6c((Ru z^%nK({o^R3A-5Q7jwpik)Yjt#6=L@$um?;XeaEjuqR1P0o(kg*@u6 zDn?gg*n#!yJWV=cn9^YBf1RzN+oY|Ly1-?4x|#>i4X6h$_GR~{4yFxl9oW}<_I{aj zrDnnS<(c;BXC`cnuQa`_UXAwN23tra8N#QWw;ybMxOMn%{AJBu^Q9=v0eYihooTj3 zwk5@^w6z=cs`WxtcuVM+$STnxl}ZOO62AkiSIs`Io}<02yQYiM_?bQ`7Cq^As^{5$ z*~LEHe&TS^Zf<&Y`@4(ms#tlD2YO%hbIY)5g!Yw;DfD-I0%j zGyFL62R?<{=L-zw4B6e^5F*+Nut;;Q`KBU6E0EoK(+Wg}oYPmU{8CCJ5E{=tEf7Y( z89s2;bUh*(0;(v@W18#gTdIDBM1j=O-)EDVG`U?EMBl#cy6DJrFfLxj5;JsfFGu977D($3i}$NtTE9IlB8!Ilkb0I!j!AVtsr~ z)IQyI^& zBD06NNAING6jQibZ?W_IP?oc7_{#8AX9?z|eU7URda(3i@xXB*l(=T1Dl*X23?*S`ASqML7Ox`)@ItMu4a1K1)B5vdSjpdhU&2NIyX0D z;5JC#2ST7%SEycOv*^#{_Yr-OtlG;~gNNHD_e*!c6Rc)8!YbP%&x%y|E$)ksbVs2x z!5B0b0VkT{-s(>BocFA9-$sPMHOHS+2U0>)QBjE~g z==6|}uVR+5NBAUf-fo=LG96Xzx^jgPY|)HTp6#C<$P{aV8sqO?Wyv%+ln%@#{p zbUE{NVBzO9aVm%WKKyJ8sKY=UOc=wT!)hcJtDOX_&hE)OgkOh|3nDerU9;R4&o%cc zm)SMjz1uU#bJlsVKWpIa@U)nQ^fl8vCe2Meq~9r}0Wr=F&gSZ5H5l1IXotJVbDF=e zO^Ml?xIA%U%zEQrR3>?=pc9jsxxnJrXf{w?!g0j476G?Dz%rUIv~OwEs$BV9p;PF` z44+Mxf>R006R-FDl3-{J4Emo=;9r2(HiBb3aZvrL06(imG|j@DAtxyV3h zE5C(Z4;%bFb-!#8KIX?tJE`ZH2ddY#ztn!Fnn3Ln#`8L{7-)ADlK>q#u=<6!Eb;S| zev0?2!K)VqY(SX`fx4aa7JG)=(>!xM=iRB!+kGqg&xRVKCqCJhS(&~#vC^19-wo&a z=X$aNChflwYeno|EN|GGdR z;0e_D&Hh|ptLM5)J(T&t{IEN)($+DlVR~iyjIr~hTIp+{V(&$FtG_{w}wmlKT-gnXn|I1+f_86R&9&V9q<{D)7d0dzgvZX699{*jErhjIFWCz93m1v&i{q=q$6K;2;X zvYE7BY6If7mP%uPslK8;r5~@~rFKw$(ILjdKWH=7(>nNtwZm(!g3%+^_=emA3v~`+ z4{h*H`2~DB&WH8EV&5jTvTW}Ge_kLHekq#rrLiL2YP*R6>1@?;bi?Kpk z>sjxf;aTt9>e~V@p<_-@-@(CHadtuqYSo@@8FN|tR=B{mdw7R;UiioI|6#-QKO^N> z6ZSKUquSy-#?-|RnDlIs@EUO4GQLh~z}oo?yvvCH-#z+m_0P2PwZBs*t1{Uam}2;H zWCB_F3`kV3Q&GB1I z+n8eE2yo1`VmtWFdklR2q#YPP1hIjct?kig8B%m=^)ieZPlK0E0CMI>H1I0yNKgVs z)Q=H4m$6C*Bmd+qkyn9(cJaQ*g-|*A9;W3w$4nJx*u(;9ciE8Pmt)1sT=j zyJ8n>&U4w`lb(8CcW94DF=HtyoEdmBT+cL_w#OYvOtJ5^{6e#eYUiJc3`NrAYW8cY zQdr7%tR{8x9fni?1T*SFU9|SE@V9y&7D^hplRW7-`~>>MHtB|Jg z82%n#gnasjJH;gNJi zLqTpju$N5{GkWv}P}mo=#w2@D)anA*z8%@a}SpC;w#mSLpu}Z!euJ-qM`aHQ_ z*~`NZXCJ)AuDLRtviqLgV4suTGPxnOCz@98jx+@H!NuWMBdtOMxSNUGc0_b#(`SrB z(QD$3u>-J$nt(5D!~O^N;FDCxOrrXPFCryEEA@n$(LU77$Nm6|HEU3VB~&fus-3VP zJH)ZVKA{#~v@_xFaSv!?p7e2~GE@boC&)dFBt>S2_Xb#Bho{rSd$pL$ihcXwn{?3C zCQXjjj-Qn_J9%&P&$S!m6s+_w0I!G@5kDaPHryR}jngs3#-%ZH~K&hC9+@BjM2WO`HR}7{yyxr7h$83wR#Qkc$2tZXh+<8BT%1O z>HU#iCqj{hV7Q+M&jxRz4iyGu?+VmywqFd?2MYWb-Is>b0*|X@`?OU1xDI=^`9oD0 zR^&g>uGaH{)Jz=~YJ%1N6I?BmY%<1l#Z*Vl*KJWX!Mm=Ic3^h@E?BB5R6F{~GrU_$ zgI5~Z4Rs~#+i$XFjBAb1)n*`W%cO4!@9-v32OlCod@VbKt&yK{v*96g4jyG!Ll?oI zaDg1mM_KT($_(6uSKwA}nlFYgG}POZl6Q@fqmv97tQ+`TH9Ukj36n7PKg(YY&*450 zzRPM2ah7S8IOBVo_f$7jX0@oQW?Sh=@V(pspUpzC1A5uaoB_&cW@}+fW@3~MqYQ5N zQf>xvSOny~NlFHW_Y!;qLhu^8&pjJC2JfuJ;RY~Xi-UKA+k^RwgvY&$2e_-%`JXb(WQxLe)t9NH2tV+LK&|kjs3j-D0XR&DAeeZ(#%Mx7EMI znEM%8mWj%gj$$;0=PdZa8dlA=pf9Au*KRf2&*ajv@I%f)S*M6OQWfTmHuz2U35UP~ z?uhszJbL~mZda%VELcVG0(h@z?h<@(1_IfB-nTG(m~J!fwavDrnqzb;v7f~a@I7WW z7Pi;t;H&px*Ac(iLU*gL>dxurYpUR@^*Or{^TA&+;#PvuNC4tM;;+CWz*9IK{>|^h z)Axa?)W@7d6DVUzq*rnl_NDA1(iVz*T8!Qjq4!_Rb z1#4P|S**c7pKGChsNG@c(O=bw;8vEaA8OvxELF#;K4T`p16Kg5+b_Xmn-#GF_&zYw zHu`fW1GV~ux?J@MSg{T8d>RM;;2L0uS&WL=3?^tBxP@_u%sLF*JQo~Q4v;?CnFd=`vy9eBlap-DnN6N6nCu4~@I z{wp5U7Im~H9v-p-s;AUxs&ocXQ$VmwAt53L*({(X8^9VRf|Ge3K6&|&h*;h)fDPCT ze0?Xpu1O5kP9W$-U>iy?mQ(|g&Ie;a;<++0qctKjE*rCl8(2pXW|C6u5a0(R6bnYC zS8{`miABrb0Goy|LT<_rizWxPQi?I69|zf6p$;SFEBJqjeiQ!_Xa_9owa9Y{J_ zfp8$3f!U8D^)q3GB!E$Hqbww%>JH)yNPL3;B%EOO_kfWTN%sR0Uyah_VP}l2ks||B zl@144@hFjJSQz8T#*y>y&w>$tBoCj286o>-Yy)dUqPPf8L3oL3oRg@sx)BaT;VBw1 z!;^hhl$Zw+FF_(?l-=``$g4@MB!VMz3N6D{P%|vVL~FXNDLN< z`ch)T$R{}?=_&_#LSaCiOs~Kt>~Dlo%6oPIho2 z`$ZA{hMbW|GE#3OvVp{mkY~u*Xk3{R(?rrCaXHHVPfCOiiPupgZj`tc5^JLT6Iqm; zNhA;9*G6MxNcx0{A}NjjR^on?HzL8K*eD)$K=NYWaO+aakak}20lYgggals(f(D_NcPoq$#o^a(dU)AQA(!VQ{Euy zl6$0wjMkS@LPhV2zR0smoDtC$*>z8On_L~$q9Qj*i|A+czETg5_Ty7Z^yqW6CLX1u zw4X;M8*M#G%1TQjzZIECzajD}DUyB5l=>idAIYG|MD(oaTe+j?l)R~wo}~Inve93p zcPTabzug^u>i^$wQhK6S@=2sp9r9o0sYEO}Aw8lp3n&ht>N}kGZQeJXAx}&ra zk`wu*^eN>H@~QMqk`o!j6uBSO{OEH^8j5BfrA+QV(iV|?G;gJ4EBBPvqqO|dacVRN zl9nQ^lGA9Uo-%%r_m$ip<*8^`>GLENqIqTSLnQ)oR0@(Fc~;rElDx0zd~{DvvP+XP z%8)0Oag&V2%4n*{N$&jL{)k?Q)JjeyMKZ3FGLdng{FAYW?9V6QH<{n>z~Ue=Z)B&q zu_LwtnX{T;i|oPbdK>0q6R}h96ntV35ud0!3xD%B;DLQZ_VGs|Z^QStn0gVOgHy3z z!WK4%%96Cg8(a!cc=$@$LBGgUus?>?whC(!t7PZD;Qk!xgXdo%5TYB*i@>oqz_0mT z_~5ijuP`sd?revp`;1V^+l3yXMH&ZN_(cZ$L@>FC39G@b6R|+F%jFheCz)8WtpL*4 zfOX{xL|WY&K?0V6!JPtjy96FBXTbQUVF$Hztk!G6IZsEtTQefNRwIsY11z+)U;(Uj zC3Zq9p-G=<#!9mQaXx;;z`cVtXE9xX*vU8PCRn;U><1{o4sL?~&1qPb3ZB#e{4x=I zdOmEdwXoz!+}9`a1o)8TVYfJ9z1)H>n_%1cv5P=EaM#mR55A8=+r%4X8;+x}S{Z$C2t-sSdV3 zc4~qpT8w7}?CemA9T!QgzY2C{Bfk5Q!+w@a})g?z*Sh^ z%CL_5skYcuX={tGwztQ&+XCikx*LQQV(Q>4^>r{cbF}l}hxeFz7h8f_BK!8`(6`t( zG;?&t`eH+eVV|K;KUe!Tc*j;Uh@8b5tOGkFz5|cVkFX!zK7OV!0jN$6T1N-GWkNy> zq9z{0dNvt8;A??*79v{fn0Ku=&sX9zBFbomXR@cqecP4oI_s_q6)?P6O;JZXL`JE)cFEJGJ|Bw1svvkh3% zOjos^U@SKm(*} z1H!|5+C6#RMsJI^$fx(O@R$2TKH0m&%OW<)=i3#;{ujo%F^%?y$CgZ}96x7lT}+d) zoo2#C{uY0ie^=l*e~)#@k-HTT^4SnFq`pv2%*kJWm^osO7Q3 z2`(6BBHt2X<(=qrZmjDj08Pok3i6)(0o6v|!FuxpwDt+o6yXri=*ocTul1JzcQ5p% z`6_(}eXZVeo-WUB?+HXLb|OM;H?H}-b>6ICGi5VvkEt1xGJfNvd1*<>eX%u0cszh3 z^0_lzt2{s9?f{WpsOi#G84LA)XFJhW5ob*QTHUN)X|`Lp#7v8=uypGF&Tf!rL<$3D ze{xX6eM@)=*t{9BPcNuy*j)J1EW=t7eGNW+85oydk~6XLm4R;kD*ON#ZnZz#SL$tr zd`aGXplECTTLO!MOM;dD6Y$QrxJz7#t~~!9YQP+dU!1&Tg5}BW6O&S!;#v){(r#b& z@B>HpP?2jfu*eJAPJNE)uDQkVnF{EbaD?A2)X|c5sadx5z*h%2rrD#ekTu+p?;;S- zPXEi1da7IX9(=zKVK<-$@a*^id%*02C;3=minu|ll*^#$5}+a}d^fyc^gw~Ce1+Z> z-b{a1fb-`D=%6Ri0DL3YYj&kOlZSI$>s@L7d^*KyN!*ySWKvs3c{=v!irry2Eoj~A z2MY&Q4P12I4%=9xuGX;JS`{5<{wt<-#E0<~>}>U+D#y6gHs4;IxH7)UT4lVYdX}&A zrofYWlkaYzjIU%O+B`707GqEg6ouX<&OijvrSPW_?1U=7Cl@}kH{_%6-+GN(7%cP` z``SHao_g;p$l4IF!6&#bn2OlQOV}YU50OdNht4}MgkILph+Q;xO%pW^H zdZ~J8@RB2I;QB*zKkF_L?y*6w**qh*C1$tr4qFW7;A2GiW=n6Yt4)ng7|apL2z#9r`(~4QGhocbAxH2 z%P^0u=jVhg0%`st#DWxKJX+hRozXHt@POW-3xt(vP{tn1U)8iLwwOswPs$N#JF((unB zJHveAxKcW$!ffoyI9d#$wF(K#J! zU)kEb<;3l%JK=W|T8r%AQdzH(h=!>AA;X844{eGZ6&!CGts2}O+?3rEgN9?8E!wx= z@Va$vP+hO>K@$xM=cK`_0po2@b($K3Dtk5co)Nw&?0n!*eVMA$9c%AwZ*EWNRClfP zoK`=rU7|atA7Ff2pP*^;9)(wrQ>oMKI_k|Oj@_(YEtTPE5JC0s0q1Ocmd#~r1naD3 zoQ;kxV4t0xJ37`;yYS$3cd~g-^Pan3-`mD5Jd_eSOv2rMa*G&aoe_ z^GtCT+soMXW9+eZbmd)VyNbK)_C|DbgQ-4dQqkDKmG8JW2;a={?XS5`>GMKngf|S> z6g%UA8H3mNYYOVpKjZ$W*rjk!g$&$G6xvjg5v$6?1lZ-%;1^646k;|*5n3`k#%cMppCCeJLQ zt33ZIHF|$FJ`djKl(*n~et`R_8#U}5w6P~~;0i}An%2?3?e0CPjkBb)5M{$fD$kiH z$WvQ)w6AlI(&q;U_6r+SGj!pw{6UMuN~}*y#rDYNwA=A5gWJbCmZ^3aHd(g>Rt6cZ z?Q%3xxSM*S-~E%uYnc$TxX+YubHDn~T2qpyMmgeG*HPUb+%?Rz!1to&j;@JQ5*<4= z!b6@n&?X;J8oZPJr5Z1;(=)+m3;oTC8Fcp;n)*HNI_gFZaBg4lyya}SO>O`Bp7rkj zmTPzCw3To=c^9_ra_wjGTup{|K01hX?$MsUUYTn86i!Ap{$!>8Lnt^#b<3HUpq>DKJ6q+R6v^5V|rh^(VLGVWS2+fU!+Ml z&4#Jj5H>Vqc3`$?nP#MWeb+q4tL`*!1l8;NbQ;}l`La65yTWKHxlBKG)~-a+)JE`?O760XIrdAI7-Sdz3hJJ@N8%W=yIt z%zc=rHHRIrOm$hhhimm+$)GM^{_jIM)8N_ZMNLDN)tBl)5>B&kxW0DeJKk}QbspjW z#~f2o^>?-(Yun%UQQP76&JM5bb4P${BsJ56cp02%J$SFQ)1n?1SQR!dVnFoRSV#1x zz6F*u@(8b&y5M%(0QW}!HEkV?>(_O`avDs^o33A7PT!xkM~p?*?ZM3*kJEomUZJ}ymq<^eqFms6!862@;U&w# z31uT0Vi|ewXwN%%Jxi!tp7HD>+J4+U&QaMp0|i|Ay*aI?+L}5_>?a*(o%5XI9L4xN zjB5{UTX!$JHN0!1=2}2gpVo-k(XBY3&I{RKTrGXC%*N|rt0Lp_chV4Mv}!L?ajf_J z-Fd_DkaxLOZ{BVV3@YnYA4J5$xR#K=+EkCpAL>@XC4Wp?rICGZm&?(MipElfyj+@& zX67H%R7&8s;Wn@AB*R>wo=$ciFO{n*z5n5)70%v+JfV2clN0+MXDy`JD~W&7P_x}` z&F+Y?CBgog;<)RW=!k=#lZ2l|V|yz4)JD%9{gi-)Ua$8baDVlH#@-3ml^FLn5*w$g z7E1y8MAINcjOI8|q24o(oMWZuj5O8QX%hO?2Fq0Q3SFf1FX@EtCa&D$b^GgP(S zz6_LRVQ6(mP^EUN6qr!U)C!eh1&vITve7$<`-*~zma0i0?_J=Ny_3-LoW>jII*Rlx zZ-z3|9qEX{PdKWxg8X2bEzKU&70wz6>@4r7>FiCNI^DM3R@U{Pv&3H8QEe~r&oMNb zORSz=6T&JX_g3K9TxqI88}_;Jn6WoJoT zS+c})a);hQP5Xu%CC`$slZoY!an-4QhBZ)x5+?x+U=`Pz%ay)Ng|UseCI^0za3401 zofVKtw}2U31%s*fqEzxcM2+e>)TAGw=vsoJw5)5peLHb)nyt(hZ!d)Fv!&~G#{ign@0U< z63jf|cO-ldhoG%d@H29%4C+P{B4?=hB;fy8L@m@wR=!igtHk{t?((N_mLKaH%-}-{Txs$1g;4cljvoT8v-Hcr=cUC8dnH zf%Tt7mxdwap=EFwl2MTK^~tEnQb1%*gPIhhF{ws9av7W{it3h7nJq&nWr5=~1kL)l zXz;d@_YZZwLxuec`_~r76sqiHREtw^4=?HRc45j1|8YCJ+Z4w&p4fa>v+^^Q!z$De zUgh{2#&k3livx}Y#0Si<=38%AORaZN(Fa?-mRfU>X%tH1ar&>>rGoJ`kCeZX zR%sf*wTfWrsBnL1fzzRY=xhWPTLuE+^rmtBwOn~Q+LgIfQ)h86EmRN{KEIn+5%)ih zpKhZrp6hMKUH@qJd^?4^N*~wP&I!(Tu6csP)%6Q?@4H>Mc`m1jr@UR^4)I#=(6DE- zCpZVVr+QUjY2~<)1%#LswkEwDnyds=P8X4ngyjNwOkMJ3_FaxhLt3dp?pNj6; zSjZFE#@$44AGN3w3;e~XxOz+xZdY#`?xgtCS1YI z_kCPrRyhC4-5haV#7+M-bfI6ne6A(#UBqce+zyYW+du3&y-;H0f1ral8yp77RAa73 zhns3CLKRZUGpdBEu-4Lyll>mk9^+Q}IPB41fE92FOzdeXkv;@F!M10j9azNG)^L5R zc<>=1l-;k~eA?^~>Lvaj#~vSOTE`x72iE=jx(9Tdv}aJ< zo{=J@Svav}bnj_-(7o%JSz^8Ca!<|KI1?!Jj|SMFr*;=r$M!eN}ei#QOsy65$;IQkl{{e%lzoa>lNV`Qd~&7|%Lb z1q+~@{U->0Rt|XYEs*(SxC5{lL34Ac`6p^B;0|tQA73q>k)K8{d!KGL-7Pd|1*Yit z&@H43&##YQ(Vgdn{j=`2_N?|Z?sK~|S=xen^(x*vin@@#2&uTsRjFpl<(kJ84Q^6< z**y+%P4T`U@b_=HpGOtZ;bN&rLPPrlYV5-xt`BjCf!xmzN(#?&n3Cl=j^g}NWjK*u zD*k3^s(fb2H0Hrh<#GItG=|uuwK3cU3++C}T{$7z-be@e?!tpLM~MZ=sAMQ$NhpQ3w3W6^Y|IXp?u03&-Q+= zd{2&g)SJM{u4SzZ$p&O+S%r;X7ixNVx#qbUGvQm8_K?qn47aang_HS&+cZaOJDi zxf&}xeieL~&EWr35~w^h!a+fS$u6pJMS1W_ros#y&90loY?DE=+W6O2?ga-WRF0z+ zN%@{$H9jK#VK{nx1UGCiJJM6!GaO>Ro`Jv8f8Eh&A&)YTD;dX7T)R_wze2S0GXBb; zzHRIbLG0Fbz9Fa&Uj$Kqk$c>WS92my0p~q1ejAYm+!~xIW}`j-mlVocTq^~m#F;1G zr=2IqNQx$0S}aXw#{U}zmtE>Dhs&#p2Cu3kx%U_G8qngd6^?d49)8d)HJ+A?kOih= zIxDJxjvtfMo7tIRoHO$>=?hfBGmrL}aY*@IS%ivn84eS>Jiii8J>&U?HBNNz{?WY@ zo|fM;kbXlFfBS;@AL4mjp|=V<`w;f5Pm7Ctn@5c+ss`&1b_5o!_)6e<}u831SNkfdCye5FJ8sJ>s9pp z+3Z|SGNJ9nhHad=^_1)56|EOypf8dalA_Rt`I0t)uM zT+3!E1y&rr*Tb3Xjk+y@_&0|uy+w>!toaF)Qh}dZtv<(Wj*%ac7E#fP^lv6xyT{&E zBK4)37K2V^7Gv@i9#)_09+TTtr^)JZX;eEk4{{0Si~L2z&L87 z4zvgk+@oL{#YCxTXp6IXuE)tRN_l1nSjmqn1DW?>%>O`6rv{Y!ex7(OJ3uD)w~V@q zhZQglHE>@la?$+OT3nkFsdIRg9Ad4VJpJu_zKy66E*Fg8Hby;yDrgleyDz&>oW_aI zW(ljMjQV96v;HEE2VY7hdcbk({~yIWn(lbefsih?d?3R<1p(odQe zqU_Ol_&tm+?|rzm>8eCdidc05C&OxZa(J*(NkV(>3!-vV#cGI$VQrv7(B`%ImNAA4 zd}&mSnv`np;uihK^7!6XvYdh5GOo-3*D?dAmuEP`RRom&-H6CpXUY%4V0j zLe&RUIdqhI(s0m3N!!>C<-UgDXP^SL^NvQ^m@qqqx&SW+Qa@ep15$=HVx zxorjCsUvc!fE4eW&mVqCE^ zY$X02LhQbXGe9_M#PamPP;nlPgOz!jEZM^7N3%|+OXGDLOzD{{tTW`RKMTiErJ@WH*xzOx<1YXoyC zoY|DZ+}_R;FCpS+6LAO;cs4Pt@E%FwYlY6XG z!+n?9rA_jSoblT^({axr6I=y*FzkNr2su$nTF~RE4*Oz-tu$IjPXxfOqLa95|;IM5vNszu`%A)3e}XXXB7O z35T7@FoL#AL}bKdb)ZCf%*NB4V?k8c&k&t2V*O{q04d}QOd}4O1IuOsc~={!PdVNn zS5Z_>Vil+3*_T5nKr>ps|B(Ye?p}hf!PRBOLAZ!lOji_WL;y~|6JXtUkZT;|Z|}H< zxd9?Bz-V7S&30H4f3a$vN4XZ*axFvLwsY1QOU4}bdLDxzpmfZhi?Mh>! z>Aazzej-jxzvwz~3_EYo8#-8jF}MKC0qcE^z23sfu?SbH*IuSZz7D%_5 zx8K1Gr~e%@N8m@3VfSPZ-z*|Z5S{rdeFjbdl=kdyu|&d={*}sL_s7mc$2^dvKFmrT zcXGsa!Kr~qe*(ni73YKCOb^m_;-Ql>CRr>egi#ZGg)`_oqjp~-> zX`(eERe1R6@SCRbMB~UilEDm4qa*yyb;{wjr`k{3U0oBLH*unCV9)8qRc$z5S=&_v zTPCLK1ZsvAV47br#=^OEKB&}R*puSzskXgJqIpc8u<-T$8U_rH3JFiN*wojYW82g% zd+u%RoZ-+bKTA!MLUJMD<1@ zs2=Y3R4|e2J`d;NJ^b*BsY^eJ588h?DR+3j5Ph9;0XL-|(TSqV78fRo@cVDOaFJ_O9~RfU`ags_>b-T&hyTpeIJo z^6q8-l)!QSgX`Gm&Of6mx&VTjn!o;=OpjOXqwaujqZ8lOsHL z*o*94aa~({6j)|SI@f!xnJtm`Qdw zbBr(#*%_y6wn+2!)65sGML10!G2X8Y_YZchw~cDw(YB_ewCizCr9W66YuH7<%9FU6 z|AcSfK()dSi*Gxut(RdJ)e=ut`RJboibTamNk)r&S@UmsF`jb;l8TOqBD!k?S*q*q z)a$3u?W3L-++$r2u#4K+KaGyH4jo-R?@<@~tLrx+z^&{>BhlgXbB%JEU@hc3O1h4_ z)!GvQr6GBJ>idrxP}hG354GON9=BiLa;J-0$Pl)jSX=6aLga9*2_YrK>5 zs2iKc+bxKlx0I-OnQsnENss3#)+SzSupV#Ws67*uAQ#tzAm0#WC0@3F zGNPo<_JG0q>HgOpdpZWS@50|a&i=S^S@XIs3{UuKuJbwraw>9E|0~#s04S- zPV+(iOwADA4)@uvB=q~SoiSY+G;Z;lwb~3rs=2`8poT>3iZe(REG8L+jOY|0PP|Nz zu*+m~hrl1!fyWI2kHF#1Zy>+?nOG8ROr=6CwTJ3+FmrK6S1^^(U@B~X;yFF({Dk;s zDEB-V1hpS3v@3k}AYKRqJX67M;x%iGTLSBYi$f~=Y{w~PC~hJ9_37&O@JNcomwUN= zg|k|z^e1W_lmDdqPo z^Aw8HpZwGDeRu%3y(78_y2W$|UBJ(;VmCR%nX(Pv85P|R4iG`%bpRIgDG_#{X8?%A zw+doVVu2PaE--qD-J{uc%9ml|a`r&#FzKyeFEe`QPsv*8%APSj8KtbeDF?@eIe9uepPcIX* z=E0vUB`z=G9R3w=E&>IjQptVMsv)MBX z$d=&Pg7l@32k#{^Yyzp;Ml2BzGNp-ap#`4e7cKI6U>c?QG^Vli0*dl)h`Ig0&V)HIjrg5m@ZQXZrFg33>^2%k5?=mvdjZ_*of+D1nk+hHr zMuB`3k|EaNU{S4(ClXmLdC4gr=G=4P#rT=@ELuqm)fp#DL?@XjmuzoUB^v_xPA zN$h&(xEc$Y_jMwxZA7yPe2Cm@8JWE3gOSae!ZA|4NTZgXppV2R>I9$R>NFD_Xfe2& z0{$2Q`VfIWvVc2^CEv?P~K0;pRz$gLlg(u z(0@S7GnlP`Jm2r}Z#crof4bxCz-}kZA8<|>q0AxU8bMTA<~v6&{vy6Kr?hvdp4Ax| zj2XC+))+IWDa{AzouPe<85aXnIvTEY7O}qQ9CL;%3L+;EBR-cWn$DAzx$Yro4HJkm z3&|P_h*NI!aT#Ylsv=-NHPmfuiLr-(3FQ-KF9Ty=L=Gi<3Jdv{=qVQm2634&+znou zEbxDD&26la=x%4tt)OHA5B`x_$4m4J4&6>rv3LrR0@E zhz?sggV+T$U*S^n8m@Tn87G;t*dMcv2MtT~6UgD8Cf1q-29QiX*aE&>2v)d&3?d%Hf2RCz@b5ejqDx=`Rm|5MqR)Mt55kGi1|wheJ{KO7 z)0wfM%u*+KyjZoOI$FsHmQo!Ro|wxxUqq*t4DPjos?-S7tV>y$|0P~{jp|bf8P((T zK;KFQ>NjdogUI|}VqRylD|jbIYlaPM2FbJY5py6b$+bha1T2&XrSXPHWs_?+~e{H*po^^U)5t=h>b ztK+1Z;Q3P-r%$=dhnNkb`#>SjGZs|4n7b-s-f!goz-XAYd92)A?*2u-ex3Ui`EVoi z7cT&IS3Q-CU*Q^@pkg?jv(QHeis77$2iZ~o!V3M7*(=tl1baJzIXjKfzYJ$-g+ODB zapowh@RQhgrjjqcgMaP~I5XElq-AQL6|e|4qLBy!VF&eLRd~RP)=@W!z%wxv%oA@1 z`3dbORHpvL{`xFu{bxA5K7tmy9Hg*{aZS!@z2&1v4JV zST-<@1?=8Vo@NH?6MiE|%nxi27wM*vCn_JzVS~nuBYzTD~D)fjPHOUM?3ebCjy1UBCWFIH=G2VF0W^oZ5^A7iauR&TK?*LRO|fvw*3gk4Q`yB%`hxpc?qfbI zglp7(R}kCWhchvqH9$22T)SK$+}|#q-&6FbAH*qh5DXWeFn^rXOSi|6ZgyGQ0}2Du zt-Z}_44+Vw|A+Q>*^&w}ij@}X(c%{dJ<33Lf@kPeq7D&Al z#4QTX%Rc&JsD+*c5r~y?)kENq7{HgaK^H=qe<|#qt5{>H>{;8`r!V;v)n_zCpaaiR zpHtyg1|EV(ek-wD9hGCzt#m2);X(XC=8tx?Z}+sGQEj{r6>%sBr3YDH;Pxm1OeT zW#n%bPCRP-pqjHl3l<+?Ufo zL6>T(HC5wZn$E`tG%jY-VPhrxXoBgm9w%y6+agY`EY3f8JfL7geUrg{*@mNi61a;C z)a7|jffy=-Kg(*e+ZFhDTp<>!W-p2(=FFkGoQ1w>AeH5vJh23JOL~2A0*YRvZA5Sp zRD^Sh|F(g=6)^(o)VDIIb{z10@7BZPJqZeP4Sq}%{FqoAzyqAe$R*x)9VTLXocp1_ z4e`tf`aJA$*LdUP9Y)DqhWo2q|7Ys(=M8&It@MsqXYOx$S2vR`MX|VtJx>*PwX|1N zMjd?!yK)eGr@r)(E5cQ2FA)hEJt~B~wHLHs;X-g1)c5Ib1pHxn6Hjk5ot%>K6RPnh zQaK9e$uHoVr%|asL_I=pV~Eup^FF zo?{-f?>E;y?&)n-*jwbwN11=mF#>+i2s9i6@NYTBs>s8u`;NX9FUCM~4PN~pY9EyG z1c8Y#7KYDPa9S_>ML(WZ{@v8Prc!M$Bua}XFaMO-PD?FUp^BHoM-iv_CE}GlcK;kr z8K>J$7-d;#$X=%EQVENoj;!$f zaT1PUyXd&HM=w^$ecjR7MfyF=q#nB)b=zf-usmGy)7hIX#Eq%c)l1crapI1lYQ2C8 zZYVWwuu#?swH7`DXP2Bni z%lOzybWqDXJK34TIX6KJSOcP4@E<Xgrhm7hp}RWSA*ysEZnI@AKy~eYXSEnIuqs+i)^I!yPf}w&lWQR!lSx^4}kzz z@iRUBE`x~v#P1f~4dM!>@%!iS%4Sv^VgxHW`Rlu1jr>`|^QgejTl8BL{@w?=uWTRJ zxD&0VxXxVa-J;j=vhMfE;2njNweX1$zShD)LU=+X{olJ7#=Q^WZvN;^EVSw3CA7Li zQ7ilwgueDNcPP}f;y%l{{tEv8hr_n;+ZI|-;ruQ7{fd|P8F7C?C*0%h(4)uy!!w}g z^F2CXp@!xc@}KzXA1)BWZK22cp~w3{^a5?`*7l0`?dc9Gey-;odwyQ%f?$kPd(ah@wXn0de8F^ zS0~1}XC!(?Ok81)3R>Jn&v*BXuegW*_gMb%FY*22$%=Jw{r`@H7@eNq*`ul#*V1!E z;u^$%|KIx*-z(JaLgoF(*Tl7oC)48=B7R=f>BQH?XT(T5c`niX{Z{v^68){kQxMwt z9#4;+Cn3JS#|c5)AwS4}Jw6cPio|vFyq~yV;eXNd^o7=0Jachn!tY)Dr1*_Jjtk;> ziFd!uXa4XE5u@3{?7hl&_V@#cYZ32ooyy(_eoGbKKaBSj9S^Qk$tqOm`ro6ndsTB$ z`>d4bcT!p2=5;7F-c(MnQZfOBHFA~wnOMZ%$jThSi8PQkm&AE4oLc%4wZwrJrBgGy zs{R5tdMJ9re)<=s6j&~b{|EOkjsYIwmNZ%g-;X^NM;qQRA7;q&@-)>K)b<|2tvOna zFTOg=uw5SFJL`DMeU$eXox7jb&erZ$-J~B(J$FG*1Wg~^OijAa;a=e!M0C8sR|IdY zObt_0UF?62c?sv|a2ek{Uz2J%o>_gVsUM&+^t~rny&6V`f+FuF z_cU)By`El_KpQw`$Yas$qE`o%=}Q$%R+suu;iS3+ewT{f>=q7O>8ia-3_EMOdM4b4 zT($7p+Dmpk)prJuyhPAYC%DH^PMZvPKUL~-Vmk}bYk_93w*ZyvDo(TQaAb=7d#R_= z@e~#jyd*Cfr7xK$wg_!*8Om|t`kSXpoX2bK$GY9 z5WDZDPLe>4dKTQmbagb2Vhg|$Dyij_66fc@ffLI8T=WP@FgDL|#>ER1gUTk_d3JBn zwW17HeAvyfRIbxa!^!Tw$Ulpda5rlNg&mz;1Oq9atDdXg3Ho;0w-X&aZjf+$_A*OP zGc$kkw-Dd8Q7uj+=Pu_2jz?XZ3)3J5!qA6}Qe?_4^Ah*~yY;5c79S;@1QJ)D|E$PqYTCeS0Onrr?Eud&nIYZ=+;PJU;cdLXsF zM9{Go)<+U+V3sMPP4JIoHye_qT}1CbJT%IlmNkg-Dze%Dt25^1B&kn1MVv z8;+ttWT=d)X3_5>0mgn`G&HASZJcH$L~5LjcbPg(MP%pS>`UTQvhiLrd^#uWqiUi; z`r-0^@u07xS!q}KeK}kQE0^nN;tU;zPvB%S^)g=$D_xwIRXm9_RURl<0V}_nTAqs8 zBAghfbKd2kgoz}!5xoV%xpTpWi(oD8WS0F5%clsXzz{~)zDTcauo2CTjRy;hUdT)iVfOkJlG(zQu zDB?QtO#*#NuEH`9yTL$etJ6U{cY=Rh_i^3CX6dLDqPc<`=0hdXhHzQfNvu%L$2P7m zpL>CM!!9Jw8WC%V*KVQ;;b8G1Sxq<}eY>NF-Q7_{7T@P&EQ@(Juxnl|d_9Y665dY3 z*jH`LK^tE^$By2{9M2+}V3za5SMgKqf#{rD7<1u1QO-*^Cj7(>l|+2Aw>wJNNK8`B z+!KCTCH$^MV2~C3rs(dNMX-rNc`C*HroPO(LcW^DT&>`HXYmy$tLr*A0-pYSelEZF zI#*<2%tR!X!+Vx?M<+dJosD@d)F(y!yozfOk&)mIiDG>FfOz9pjn$Y{P6!OsXc z4RM0@M5IZ4R_rS(R~RaT@Ee6|M>*g35I<>QB-*;Kr!QB2l~E~&ZPLbX5q@1F zE}_oLDChI!#a+=~i_xv*Z)fDEXf>LBxyR{6oc7|5L^l_4l|4Q? zVit-H;})(#WVGV^7O{sI2@yL8FN{2{RCvCL>!!mcGf3=dc|1onxXchS_r%OQ!$$>k zLG7uzF(eGZ2(>Z>}c4q4$u0pJ>eEy%yzh7oo5}kHt@-LA!h!@@H`OIY2Qwdin zMy@CF7EzOk%6smtke}Gh&xq$z#^+iX3o%MPJ!(XJ8NoH=aIH;ZS7)~r%&dXbJ49At zS7h$}lv z9u%rhV26t)J}_|QHeL$%Q_EAR=RR840~J0eJOjo3iF>PM53TP0vx0Zn+5LQmFh?$P z<)XjQ^zQ45;aXI59ox&l$E|jiz3Z z3xXNV$gQFpn**=&CwBZZa5BqL98|*na>B`fLRv1Z2l)$!$GVwL=*9TyByxQ%)M=dX zWbvv2IdgLQMG`Zd=5$NutF_Fp&CDf{!A>ic69d}zvGYd|8aO+eGjf2w$>2aO(hT-A99{oy99CoL zRPa1pkLO|3$H{j7RtdMMFKP+FehmRTAIABW?k)1koU3!;ntwv2@I&otG{*(S1MwH4bEaq7^yzE|isjJmd&&NADr^gy;oSq7NK8h+8OhZP>n?|d0`z;|>c{|aqY zwWfb8WPVcXc{Q0F#tgdizOTCmFX;pQL8{LewOUzGPxn8nm|X2$ zRrY3EsjZk^+5dI@Ks^}l9P55A(ZXp|SW7(z-FHBX-v=`Xi^Sc2f5()zsrRz!=yI`T zSBKthP+ic{qp(*&pYpKfVR^l)gKt|-8%G(=qq>+WUqCrEM*oDi1!&GmoH0t25-}37fm!rrw zlJ4O<+7sz{eZMmho-BJ2G1V;f0r0#OUlNF|p03UwI)v*zZ@D5J1-7t`nAYBRvzxVd zvTn!UEw(i)TFGx17xa2qalfj5Tf%3D6^0lCBP}cGJS*n#pLAn%|CF7$ZtZbZ*r&E% zxO=Smc5~*v=8kuqHe&fOLz!i9pgE`@AkmtL-q3H#08b7#$5iZ$NLlAy!-cB9iA8286owr3#$tGB_PvMMddosFb@6F>pG%7@7vC) zox56tTY~SX?|k0!dHZ2!g}=A9(NJ$b6>vLnY~T&cQggDo*s?m{THr^4GXlI^?I*e% z)kp5t^n`Avr#M|f>E(08@jn#K8_;3(g^#Y{WJ)EfTucq-2J73VzV6HN*d0x_5}fff zTO;u=*wcLK&iUKnO^2^1H)XYDdVZ2OSw{7lao>#ln+A@#Z%cS!=(T{`mK1A|rQS5j zm~H%rZXg`A9qn72&)&*x+H>n}^Z0g?GnA`cX;^2J=r`#w8PV6B4=4>@7`m{xBXmNq zN^6Hci45#>r`Eo*vl2h%SlkSfyW$+T(T<+%IzoqTb_unM{#9!X9~t@^R>Sonp2d}I zm;0b&lRe7jX#f0P#+@TKgRf7zo^a#jjion_H5b|A)b++=fsLWd`!z-Ozpo);V!w%D zLqkP3DWQigt?by!$2man0J+&h~oSFOHWy z27fU)LA$ohG$v?p?*{rU&FfpzryxWhRA=s|&nIiOp^_cu9qPW%dC?xf1)qL;9eu&O>%?|H%Nt!C2)`W2w*Yqjg?N@noB=8kd1 zb`EXxwk~b2?L6D{wJSq0!1>w6iGGQbwNRahl3}})4}Uhr|AXh0Bi**H+aDqB?t9Ir zZlA|dX6daPH?Q3e?EFT_mxda`1D1!3>)jAi9I`yLDYPMUe4lw?ulFtvjty99ij(WT zyMJL7d)2k$JIf{0?*TvK8@L*Xs)AY zRf$yLZ@{HsJiMq>+az3s+HnzDf-koQkLp88J<5=1GDVpjnH&L~lo_7uyUtOL)AngL zPsho&#Cy~3PHS0ur*~6gQ~15bjzxa8Jk%J@x;`7YHZY8?yd#4Lhc@-e@3SdnRPf1w z0Y1@M;CI>qMl)*9D0;{@Qzi3SezDF*U7cac zHqO>PMjxAJl=;*ZbuKTAseH#Pj#xNv`HqXYHN1(|>nPrHxnL5zi6uV8Gx}T3=w^61 zz37Ub*j3-Tt39)I9V4B3=VbFJ_=+lDsroVPSv&v|OpWx(zHMy^THCv>Pi<&LukFDX z8RNg{9@c&b)8snUoK@cE+-IFS=QSKg3LGmLqh-EJXu%RRHccT~jp2q}#sc%})+%cX zJ>ieQ=DAPziM&9%OyrS+mgcy7D9pWUD9XXMsQ7+Jt!6Nsp;IuRHV{dCOC{tuER_H7 zxuI|t2Dwc9E_@L1@7d5khi=7dI!A#bmHY6nf?MFy_BVu>^zgyP2M>Y^R*7@ln4mc8 z8fsL26u^V^x3#P2kW@p=GT8m3bAan3o-8a%YC6*3+KG+ofyZQ-m z!rs@v2CrkD_L5)pg!+EB^Q)T`U zF4voa2L+dOlp_YlY_sjIeK>vU1m3X^1nWgjq2^h<)Wf*;#nxi$RLd6n$yu#s<|^|I zON^z$P9eB4pZ~m;F$(im*Gr08;vfyHW;}PHc zyIZ0{f6w)}XTEEVdyW^rly9T|0DQV~(2AewN?ZWzXQc6KR4tzyS`DWRHO6(ub@Tw= zYFuMnXPBtn0ta~`wPt}82F z@wWNrqiB6ma}Oo(Zq;wFm_9|Z`?#ksRp3atL34f6{W7>FUYf*iD5mL}z8aSJ6x}$y zNN(!q(ZTow{X#e$F73ZzKs+K(hYNBE#vjg4WJLSuNna)uX=p^x;h?sL%I{;+Q94(^ zHS;UpTmCAw$0M*4bls}m${Fyi4%c>R4t>AlG#lJWM4nl2jB@-VyaUl@M1cz)XT(!f z8(=r?Q%!{rKM5SD-c#a})k{6MeCPZxc#`1LZ07V##rrChEOwQ*3e2gHGj_E!jgz7c zEe>5V>4deOyyQc)IC+}I+85=$YLl)IcCg@s3ulZxc^FZT$gE-|bla+6X^p3nx^kx* zm7%&72bw(jM^}?FTz}gEFHRcdPUp$R!tPI2ukyX>GSLsv?0H$*jhZ}@JWTS?r&hhy z{j)zT0Z>XsbU-pNCqeZL!Ef1uBP#G?$Ep#7v5e1z>Zu_(} zP*vwGkPa{=bJbr-O&+>tYcfH-_o{1!3q7^tZNRPYYsAN9wNrk(6$a|T* ztCk($IVDF8Z^gIIb6IoU7vd?^(DBUO7r&Mxp!j^RqVk{9lo5f?k{jW$PNt91W)xVH z<|7Q(-Fd5N8*2p3dyF7B|-Rfdgq%E2tFI*yivK$`8M)K>UDw(>g$lD|6R2EMq z8NH+Twu~~!7e{10+WQ?{ot{xHXs^OWBum#cD+5*Y0^Vu|h_bkvunHu^fb&l^jB}Uo|XR?JJow7OtY`M~3?>!^cdQw!gq!CJ%YNjTh z)%2Yt+$#RA4e~#w%#@?Z_J7jkcoX4!;1lbci*`HHQ^lWilsT$Ra-GlP{)e{EZ+3qP zr#VYmE5%FuT}AjzWO|a+3w4sG(l9&@!tvr(1ddqgUOkFtthWQ;icCiLla8C%wk zCoNE?coi^Z_#Jevs#RY0D?H5;E_}*V8h3$oh}N}jz5=PyWApu_IpD4K?FP-zd*({B z)KoS!3F-`Z{*TD}lom4Ta!sdV(L5~w;yeS3cRJ`#IqWlwTK30jR#CACWt^Qb%b$>{ z6`Sh1<}x!eLS3M-c@NROE5W~*(Mt2))~e_XGt{W^k98b0yqxO)!ZQoJcd$!K z&~wdEq`6-g+hwFb$OiXZ^fBPIVEOfQxoLsv9Hog@?{cq~SNY!a{9XH=yI4c5nKAfK z`#5aEQO5UOPVY$lQ}Co*T6kbeo*eIg&ArQTz*DO@41+w>kTrCG;C*7GY&ao!n8NM7 z!5mA&Nw7(otNxk3V~yTnbaJ}mOi^FZu5gw3U%{&@jc9TpHLVI?t@@Bx=mw|xTA0yv zW&k%au;;B}KAzJY_smrxMSYw?;8aPU;{#;kt2_u(yMasP9V8u%2@@eYvB% zawe|E*QLG6Ep;td^n_&gl)^0fUP)7_*f9k6_%XTOQ^Os->K^Eu$w(^9&c0+_D0$WE z6mmD!x5~fJvSur(JdfR~joudZV#(pD(mX94b}xcE@u*Vee?$L*dzSx_{JjEORa1## z<8RtI$}C@@oWvp89i;im>0sW=)Z5r^&-vTD3C!sVY5}2ed2o#KDO9%7*_)|vc-u77 zz>m(sn@ELe9?OZ=hO%F%hI>nNj#C@8QS5jhYGpLssTy!$Zz%jfG^*4XglY^oLtm;o zloM|>IMNW*sGBt^_&#&s2yFF)`b*VW>^PD1tV`nilkuQE4XPCcL!ulO<}?^Hhr9`3 zi@DyP(4;lM(am5EZuY4-zp~hM6Ew4^UM5oSTSok?sIJ0cDO1gqKOxJGL4kY@$Ab0f zHiOZHrQ-4v$5n5qr;CbpHV~Z`odZ?5zF{E#(L5J=Q?joc;E@SbKAH#trkrmPr`0rX zozJ2gf;P1cRI8p-t4WE*J!Yzs$qBwwNr4FyhObu+_}d&-XImH(#er!x_${f{W^AG#y-PF-XCQGXT=%)^{`UxFK5r`DK2 zO;CYvBn&q@sNf87nRT)cx*?(<~(rO_mY=K(-~+f{5!#>J*t%Xx4`A4OD~+8@93R2 z%{0keZC-4tC6>>E6P92WSiuQ{PxohSI1Kky_%Krq6zWIM7`lL=t{}P_g&Ekp@UkWG57P;=TvT=*_Zg4c*eg*N6qtiqGzF~qxQ zUY;WId&YB<9%F-G{7Rl-N~~s(ewY3;?FZU1`dtRUNoy?)$PZW@Fg~!rT4fw*1fQ_( zqEF+O^!gd0+DI*?0nfE&xR`xhztB_Y6A+&lKqnpU_dGj6)N+W7%GE`vvL?$FAf?x( zFXb&bKqND}(l})k={~lU==C@EE`0M+a3@4d2D2`h9y`l%(YuGX^B#IDfl59Hi);?+ z1R(p?ZvkKWM~%d!G&nOHryL)mEqV@xt%o?@47>VORC2$%N4i2doqc+b z;R93}bcQynt?L692Q3U}w}uDKv$h(S8awf7+io69Z`)K*aHpsddH>>mn+)Qfv%q0; zeuFB1jUv&(=~22CJ})gq%}^q#wGWeDd*zF02|mUxAWb9OwC7NLC?Y!U!n^xjv|CSh zGZhQLV2oYs?YF@nuiy(c2GsZsSkQ0aNAwY$`R2o0oapN1e8!!t`b0O>pp|pzTR&F6 z(tHsIGHt+ltJkWv?g2-xvD^q)Y#FTYEmgvoOr*YP^9>;*cniMWMU+U)B2+Igpmi~L zQ;DfX{*f;QYftIk)jh!MAEO_n@2%Yq_n;gcf1fJGKSVjfY`g?)KjSrBFQ?w=B`iM9j26y@z8Ov6;#&yy8sHc*BLp({oG^*#5r4}4M z9)r34Feu|G{Z0M-Iv0wOIQVbX#2UAJop2_Hpe7FCD&TtJHhTjF-9j`;gUR(Kx}Joo z{E2giYXG^P4^Q#4jtS(1QC$`G_lPmK&^3;z#vdttu3KtsFgKVzrbJ5(s;~@WGajKs z1Ix{a^?%d;1ZVGAZJ`8rfIZOS9gSvmuLnmWZx))N|Ix9q#XpyPahdS+;(Hgv0{N3} zfc|Yfbr^5HQi_897JXBv;byd#HD*RRJs;j<7aF4{VN!p97VZ;9{DkAL&R<~?j&y-{ zFrUZr8L#7p{dN1RuBlWJ%BkbPG{eEsVaPN~XlhGwsjD>&H=Z`H4*WVG(R@$;u5KX} zfcuK2q$?-~eIZN_z2%W|rV~dM|w{Ju0&eL*d+x0kizb6lvXJEj16uU;ez-4yyO0?lWnsS~!xU zcLEzrNWm8!?7G44(B*Ai5BjyR{Kxb(#zq=v# zUvZi&MG@nNwfz@X|2hzEbeU9Y_Jb|FM0dCuU>wV-)QJ9T)pjqHk2O5aM^R@cv!frN z8@d+O`3E?C_0o^khZy=Bj~GuIJNYveRmn$&L56p6E;|qYy&hJx70%2|s!f;SWZK9i zgdfs0^0}S(3|xk*SwRF>$vsTvV#C+8XeNA!wTFnfnd!j2oCC8R(jhvUB&x zA7>;dMgV(M+K+W_;`^6@ z#=L}V|Fqd^so~?aITZ{%-89(Lz!R8b*rmUs`xAZ9&QOhc8c&l%YDjZo&9JPPx0ww-D( z0k1pV8IR9?V@EaKm|ME?*_qJ{P~ow`;dz-e=qq;23HXtQfe_V#sMT7m)_Cg;OSrY& zvf5H+&NI(pruR2JY0NTw4k!2&{nFI35e~WNw2@0aT~uSQQt`~|t}RA#;);r5M7L6G z6*a^>un%@0d#e$8_5- z_DDwen5&sC+@-#wWc@+C`GlqoU4szez(D+*i2lgz?x5{;t7JqbvA*mWizh7Gf&sy zpn3<6^jPX&OTeloqWuo;nr5%1CqNaw3>!PDI_f*hJ6*QfU5g!m$5Ue;NJR`f)fUY> z`L@=i-^$gdni}xon`*gXxoeqioyWCCS%v%KZALiGl84s2!g$i)*6XNreI{SV-Gs~u zRBH~51i?jEK)v=V?3x@Zi>EnzQ(;JpdFnxpCNL=n3azc2vVkZl-zJ_phU-N;E{GR! z+N{7Wdy}m<9>M+TQ{d?+=v-jegH4R> zZ8S?$E#a0TbCs#iXr{KO(Yv%pI%=*3Ex8VMa)9VSbiXU5J|Bsys)YGk1NQci_c)q1 z_=CiNPqAYON7E%-?Ev?Ws8RaSNpJ;q)=jLi{I1BZ<#waJfO%WdxwJE@v#zs@I?rs! zY*g{z;-l4s_taF>!k?fa9>mp}QE$vKCE!uG+OihRTFZT~0Gjf1je3$H^RLt$b#! z8^5UrQ0jEFZ5zx+t~SF`f^TH1rGnk8%v`{Zum%m|Afp)%jfXk$w#)gT-!swvHi09J z=6VG$TyQupqbf-tVic@`@2P)_^|b>Xj{}A1N@A}5tgrpd+QF`0oU_q371Am6Y}Y&@ zv{L#@C(%=-(bkSjaZ=ZM$1&>Df#fK2*lRbUzj~Uk^)6;Ypy7n!b56DfQw+I;){7(K-u;+r4wtr&~G{B!3-cHIYwbxxuHpGTCr9GB*c z&adeU{3GrNJC)5`ujoZo%<0*i%Kewxzj5CGMvsA!#!C7O?=Y<~m2sV=oDCr;NrW3o zrr|f79pAwroY+Uc4w6(1Qz8e3#70;`=U}c;A7h`jv3D)&p0_JG6Z|M_*Dx2Jc2K<0mRy@^&=51XPHq$Pzc(}vRNGA?da zAc~Knx;aNJ?G@eEx+VJGnaewH)Mzps;}qUySZcVb573XJ$L9s2h`V@O{Tr47PDg0M zuX9FBVuTeit(S!=1+9mQ(^dwlZ^pM}Gag~Xlzrs4fsFS)PPU^&EH$`MCE+-dL43NK z`hG1+mYw7$`^YuxaNLT(#pngM1HZ%L@@P4pVA8<>+|J_ZD?CQd)=_a=-+VU~QJ; z+!G5b(L$D22-~udkryXNI{U^lDi(98CZ*vMmcs8BI*e`nYZbUwd^fWuneP%hjd(sn znP1f?IqJyaYrFr<=1&`cZ}8%+MQ%2W_n*m1H^9^poW4ldPlwoDohb8Xk+nC14&Nf; zy3X}&2Q`_?pRwR=S2b4HEYHKad4ln8gOgK;?)gRH3I%3tIxpeG*Z?!AjmSdq=Qc9? zH}YGGVIIBA@4Jk~I2Ios3z5b&p3`2iiOuXYsl*%uK?Q=y_tW|JMpX68X|%fW-0={u zxg1VrDNlDY=V}^!DP}vbWKJ*(Ow@E(NJ02l;fxG#E|&EsuwCKHmPu`97$aN{(pd+O zNpw2BjIZaSZYEJBSoR~hVhUbS46eJ|QRK`-otKQ0W)(P8GzexQD?%{*vbiJS;+)6t z$JLIP(89Mcu7cGhlrn8xc?B~h11zDOipOT+6!APt_*)PPn>6nDD66T0)t`aOS0?O? z(LAM{-bT=wa#W5q!r=V{Yi}yZm6M!r6=OaVZ>NQHvYja#$&^09yYf%k2Z@d!V@^f! zrwfFr2o>cmqWbF~$J+#Nn4PDLCm^aISGnpqb|BHiHU;MNBv8fEXnu0JjtHti{K-|t zQLP-vbqxf=!jqEg8-mwY7Wqsz=*ze4sO+i41Ak%tM=NyOA$Mq1=qTNd*i0jnPJ&~< z8J^Yxd?&?>x8sa5QTqt1;ZNG-+J)M?@^rM;cTqul$x`#-Z>go1QR&VFsoRWCX*Abe z3Bze1S+zKi<`O^Kz{txv>r5bZFCX2XcU8twX^(3{BY({uaT{9AzIez*OJH z7#DSOkz>g^(liOIz$lytCd>V_f7X6S_XMwe7S7mq(4K4LOs9RV+nB=<NDg3~BM*@r&!1=xkPTvY}r*BnmM zUEq30P<=k?*+Y!Ifp{q!MYx_5i;nEfrs?d^r@7J^wXxf^_Cfid@?8|>e+QKsk7M&a z5TA$1_HKjv%p(%LBHxyuz!Bj^zI#D8${9uW*TxktV|Q*N2X~@Yi1YjL+%0D>Hh^7B zV=gw}E>h3>eukLuJw`7Z)#j`4$DZPO_jezovVH^vX9aj>Epv4%pWnwW9UUIsR zej6E73TK6ieb$1@?g7@#5R}Lu$8aWxFoLyYP_Lr589)^MU!Lk3GQML(dHZoYx!|1b zxarzO9@xiq%r%fZ5^fAT!N;;W-4;+e`I#*5BFf$+Ix{io9>ah2?-{1ynvreXrSFC3 z_quK-HC1Cmeo2oMy{7FSmjCrlY>G;ArJx1&jPMoAIj0&H6|7$LAO| zz2{BP$>YRwZ-85!#}mQiItX?$#<`0;>Pb+R=g5sDR{Jz|+$7Gc>&(E<@G8;Z)4vNv zY>}yx{t_0&NUu{D{+M@qhe3LifRQF z1W1x_Q~iLsHdcSikZsB`S6DWo8&_MZtlEGkbDgQlRKvA)8b;y8I7K@XHWca%yo`Qg z|GvuH6`We(;XR3U`3>&h!o%e_S5()XQ}idl9DwTfO}86o!I5BkPk#G2I^RLvPwYpKqcvpXVLZ7nK*QeR$-M^OcNf@~ zm(1~e*V?Yd_DK7>&gJ$gxa4iI7u(J{-gPx_CmEn`vwY?1bMizyDBSu?dX%iSr~`}P zI!y^!8@MTGd2m_K@W4%EVR4r2<}hO)-5K(kB(mBv^ft}x2o}c2!HV-UkMF>bd6kY! z0;gR`-o1e<(ZG2bPcA#sb)KtPY)`kV>ECDWZ0sxpBNE-Vv|V32Pq?z!lb<5)-Ki=d z{ya~o$2H`yYpquT`v(^U>w`;!W(PO-(uXV#nix<>&Xr+aWBge6EO>S-tB=|^t`Og| zBZYvZ{02I;gv{v~;`xu<9b8WsIKclQ>RsTXtkd`ZIiH7_VK@qesHB#ptol{gWsN1) z$XFvIBQqmY7fnqqT{Sh&shOLmuA1>RN>*8Qjg*R%HP*+oErTWkcMLIq84w-J3sm(GWs=>ZK+U00*Y;(Ax2Rdt_a~xXRHu?&M4C80o zxnzDuD{+$rS<}lSq%VSBu;%PTrQ$sG@h_0+eVJ_Vc>3)b=}{t#4_J-vLzCmYw*b|_ zEBLgG>0aK`Mx0y&zwi}zj(;t$-p##jkryi;(Y~e&XkCVMbG{Y5P5b8P9nt#e8i(F_ zz`4*_<;aPG`C}exQ5%|El|M&-l!B)@KaM~kZ+`qtFxM2^ZYFfMU$V3e-niUxvOe2?Px zV|@Q3Ki|UDZ6S+`)}wExH`i0_&P3tP*>$MrhWE0sv8S%Pz&(j9=vn&qso)k_p*pEM z!@d#LpD<5|+GcBUA1ou9fP}PnRBHx5 znAJ{iKBRv{jms^%U5OyBr-LVj)&_TAAK8%&FvE`p-(?ma@7)O`_JQxoLf{P5tE8HR zBV}>-h0f_c)4*Q}@eNbE!v5Xh_iH71rEnc`42uji$?)cyCPpRM%c2i3Lt5=Sq8pqo z(MzJs>^0U(5R6Xaa>IRE&Y#2`S)A#=k{>0pVTCMwqlbI@6CmL&N@&KfrK6ZK>o zj)45B0v_KaZyElwwLSNDeciRo^N#-jK7SLtS9)Iz4q_cTNE@m{^K>P~@Aa3ps|@ok zduE z4diCKRMLAo7a6*TS^YP{_kSOqORaaCd$jA79(~vS-OD_u0t-DCx?btt>wOO#+8#Q@ zK9n1|ty^y1XVB^Q87D+V?5&O+&gSS&N0Z}#bF;HLdWl_W8)rEQM}DyW4Xq!Bf+5t2 z3Xhtz$*au9_x;0z|KSW1j{?oFf%l{)p;8@6Mi1{%e)=ma4=4RbU%sav4>VjbyY$^d zVSHD6N?_HOd7qIo$SpPTn@>k3F;~^*S%$YvlcFX$nxk`K?vFVUUF|4~p6ql+ue5KD z%0iFuZ$gO()e1{kP2L z1*ZCIJb8Gl4(?9v(s$*$fAP)qOuDY{n+z#Hf{yNzR(s z&9Q3d8fME(=Mp&D-$ku5PuBl~^WXcLL>N|wf?oemjH`k8b*6bQdVdYPOCEX)H5uOo z$;1$uFODoA>wAUWc|QA3kc`t3UyEDa(~OhvTX!dR89gCiv3q#e`L1rSBz2L|Y#{>h zD~gS=ww$OdW|uh)t;5Z+BNHm)8lqe5n;l8f<&KNCSnEcE2Yt0|aQw0&hdA|gFg`z_ zr33sE@BsQT@DLHhQMpAri?u?l_;8E%QoDPjsBLO~zqI?})^(eNpU}se>HW+ZenzsqzcRg zTU_QT?3vVcs$=(^;*R2Ot#_WMz58URtLK=%KI}#xa~GM8kY<~8jB}=Qs=YgEu)V=K zCSgocb9}WUBWhmMAzNuwfSu(tHFfON=57kBsf)OVnyu2;54_FfPbar~I;g?ZIFZU@ zs+#dH$PyfFVxc{Xdh}s`=2Sa|6Uk(6zU$)MjqTIha@)^!X?>;MPr9;jP0aAshGU6P z{HzPls#C0~vCg<<&VcPyRI{TzA(9+PY>GZ%ttC5LXDzjCG~QGdk{N5_%(z{;=1=rx zde*qNc_*;fogw?^M57qaK74UEwPEFWL;R!|$uY?>GNvW5w%>u+ zIZ*|sBL+P#+EWa_)ou!Vaawu<)u}Pyk7?dScM2YUKj95B0>(!(j5KtWP|chTGWaB_ zLm8YlMI4r(oJ;*XX9WL3Zxu*Heosf|^!6LK3T};SJAY@nYdx6!qMqWO4gL{|RDBa! z@-2q{>OGd6SarYB#PM-!SXFXjI^(7#IAeENe+G&ENV{3PS+hrZT3+VM_FnUiCx3i^ z-F-Ayw8Ym=ZGV4OysB_@BnH;!0Gz)+f{%L-d9r&aQ)< zM>;d_o@!5N%e=L*ZF=Vv?<(|~+dQB6j^YT~ZP{x#Mv+H0O|tKe-<9O*7azYSdZO)s zeP&EjY{<6H_(x6xP4H0eD90*Z3})jjS>c)C8$%ZOqTlMB<-X?qK3Iu!VlC10VXCHA zswe1P#Jgt>r{8k*XPV2PnC~h^prU+;z2U_`hp*f{xXW{=pd*AU=(x_r_Kwzzt%2Kl zu7uD~Rb}L7&cU@2y|KcvE3PKynyuBmFsdMR)~!3M;e2wQ@+MHP4{|yBSA1; z^Ov{_dbZ%8PPI8zg$ato!~%;W-I`2Oy`|ptwYHEuoTU=_>-+SHnwu!hjwgR@BQ`!4 zsPPn`Qrp_S%2kJx_{8pucjtF1?lDGYH z!%JZff2BqpmW;DUy&I;@ZsV`IKWdher3|RwCQF4@W8^{nxqrkzqKM4QvA`JLP|quz z%IDy@=0Ptz1LU*F-R*suwKvhX#kIC;OJ}F6KD6DCVa+vnXsOB#S(MAoGh-Jfw8l81 zZtMQ2zNTGfog6bJCL`)EI)&mes7RCI0qsilVf4M{c-Fg1y}y(8f^2;l^nmf80(pYrq;ulUw_W{Bg6QF?5)iAR^~e3or;R_CuhwHuRL9 zlAhrFb`gF@oA0f_D)J#eGT(loM&Y7wOOWhpWFu8T8QwKSgk7QAI*GGjsir34loO~y z8)q5l*v0AW4gEItCDq%y5{rX1|D@qx>bfvkyIidJl2z+wxJ0%E=b_cVUw&R7v>6cKlqkTGlItEY%-bp@GI7&H}JK(LEhzCcvp0ln~9s{b@F8OHj7sROB zh&>BDZQlPy8yKB0I2YUC=e!zND`zR=^!Y@4>#Y%!n!3Cf^tI-}j+rr%b2AR4Gqfwn znQ6&ku15=TwcqP!~L>`+$lE zuYMBBpO3(3sitogAg(RsyXWFQGo2H~doV&uh^t@W^xHsQFhZui2~=b!Q9}dSSBabh zjwA4>J^>Sq^;LS(;0P4(==P*}M|-RAt*!(U9Ph6WX|JU{_K&y7+mw)rDPdmkrcGu} zj0SSFF^s!Wy$&3EsQP`3^XwNeU*Hk=C&ME-<1It6x&z01$+d+X!6LWAvxfH6Qd#w( zWKpRNT1$tu-ZI>nqnoVnG|zLC$4*LUj+^GFH|6Mm)!E?j>^9XJFTj!g1|OGm>8;fNqLb?o!G(@CwNtU92VdMzTi)gDx)U!H3mJbg?%ElBO)!>5 zQN08U5Y+qvcy0$O=_ja4T1czvsW$F*7kctN>pj(;4o02Me_A@M_^not{^V%$NaIUf zzt6bLI>yoD+!iw-dSO(HX|3To{ROhBKat;h9)@sn=os0WG31CYqsN{~F6%=w6lWPf zov;IABKgV#=jgzEC0~`ov~K{zu^!V`wi_)^-?scg{EvJdCm+fA^fZf$63Kf zF}jY!5uOYBmJdEPJzND(_&Dvm5*mcAu07O9>^+%y?id`miT)k5>ms-BfP5XtNPDw2^;##n_{}S33ao^jnQm zU4TndGxg^O$$v!?oBRYrbt6t{A94jg85bIK4h=XS-2;p3JGffkX)ojI6Qx%hrWiia zPttvick6YKB@HXyadPyFsq%l1t3KL0{#LUpjAl*GXT9??{%S!$szJw^`T9-z%`E$f zVWaVNet>nNji}`so*^^gye#o3y;a_cFyv19jxjslK@avh)J>}*&#F~=YRW9tmO}Gd zBIJ$6q2|rj9D9=EpsmbukTJNDYP^4{JILQEsB#$;J}dRG+x-=t^_S5T+|MrFpN#su zTtNo8;S6{PGhzC?PUbusKGz@Ep9JeVsGp+$2UUR$uPkv1xYJJ)3Fv@sf9>KpnO|H4XpSFVam-Rq7eQde}y4&693t2o?SS;?Dg)# z@8u={O??*?#);aoTl{CY0P6Hf6sPtAs}dmc6Wdl{YgYJZPX zhWZMXtHycql{23&YHXQ`#+>O_56D|fC4 z44Ha;M)4|nnDjl_?tb)N6rX>AWp%;3km^YoL2!Q_pxqbABfX3}(O@u`Ez%w`-)X82 z?O@|Ja~74Vd8U#0vtFUHxsj7atgV*F>4I)HZOj4<6b!{t@PMz#qol!%(-(NBB#@j{ zWHHLogjb-vL=6q~NdJaG+mGu1sk+~gC;u%h$B(tI<8mGVvs1!W-47Q}Ll5Ti|CeBi zbua)l8VN$1`* zfTqp{N529J)YQ9*qD2h9?{)T|!Em-Wz+wKxcLXO9m47O6#%1`D{n(|1GV>wW;ZKo= z+D|=6rRpBI>-$XAaG)koWB9VZ#AvlljLNZHv|gcpx&!~qE_8@1V8}Ko{gIjw&aC7% zALJA=3YQ$5)tQrxpiwoU0%|5lQwy;YOml#C4fy36lw&<)xnD&sb|z{PCsg$ygDZM* z%owW9uA|^$flo7=@mfs|bq5|4wXh`CvD)q++D`VL&!C5lF+vPV=Wvhlk+(T7HzG)QTc|B?1sCC2b(;0HfwOgGs3=T~ zg3hv$eA0SNEPT9=sY|&Jg;wEv+mG{P1#F3%WC%3iw$mBILV@%q?1;I1xIr$r!qn&w zu6LaXZ#6jXvz#QU1mce1YDmA10NWE9_|w7l9we{*0j<1@R)`M1>20bgUiD9a8~Gh- z)Kdg^8dPFna6Bl$V_*?zkSjv!so+4Zu!J_?Uwd0$YpgU+vrdRg!y9Fv;Tl!({W#Ar zK~W`9osEK>IF(RAP>!+~NPQ z0t_Mlsg|?R!v6u6gkpG9G4OcGaBx^i{PP*B?>pcVOW+Y`VLtzckB_KQN>OEjSd9e( zEvNU=K)U;M@8|aF!PdZh5cj|h>s7dIDnLr99b(T}N>x-g+@8ne(_r!9q&k>9<1tQm z{ouWxWW_q;j|2M~$-LbV$N|ymAp>+Ia3t6gYEXWKgWCYk|9?kw<8}RVxaF1TcKl*# zHrK#wy+FSoqYmji-F;LZC{*Y#<4L-Ov9z4L?Qw7bGu+DCFwkDWWly-RCc+aE3P>`U zl1uRALYjrN_9G$-D<|0d*n9p2Ys-TU@e+772J{Twy_~}@nVr`e(^=@>3s$!R_K$}4 zS@>SKEiY!3QV>(3U&NXym@2KrA3nyHaOe1j{=2`|f#fO96Vx)0@f%35k7TruUj?;W5f(1F zg3JFq&MT>$o6X$4b09rOVPXrv@Fsd@DS6E@(D?)C5w)>K|B3v_FPfR$*S22N`LAI1 zPingHI;sTm#}(WyAdMU`OmgKl!Ag z-k5Hl52wAwcnW4~yzvZNz#PLl>RNt-Qlvq7i>unnxh00#`l4DqGCS{BvnBwW7PwC%l8w|4pi(-iK>?K|jj?nr@s3Pd5kN=_6J)DaEfy3Ae>d?lq%2u$(qHl@5Pdk5!J&sdLqb#@xKI;2&$uAFr`=$WB7Yw~| zDs-}NzkZy1un=9T&F~dJhQ;S5udskCu)wd5p+a~xEP_ppW8pyUV?6DlMTc3PKZIS= z%3}zTXEHN+6Q|@VPFU|TyALzJ--H_}?$%a*VhHVo;Ud^gTexGduxAUW+(vkmcCtQ; zt?}`OO4+UbrgCoGH`6Ip3vrGXbViE4iN4w9L)>A7*aLL=B=&vI*{s zID_I&(aW80CTEX#FFSlS&nm)YoYLyy*flXmG^`IxiEd|yr*nsjISF(!RxWW4*b853 z7$>_GuqX5JC|`g>zydspOJQ%l0Z-;1=<0uhzSa~x75)f=>JOY8yUFL)z?&;o6;sJx z#9Fq3)05ytHgYdc^sdK^oKnyVW1X&n#}UJwTEJgJg*9BrygnL2uMw0W1NDbY@Q8QO z*pSIXq3O+@`U7!ZDSOu*>Pf4@i*UDX42|Uko`8Zwk@B!Snp(qFT7Q)Hp{1~%j!FvR zt0TcOPR>PSEJjcrBpiJ6Bge_wcSR)lvz3h5S2&C9QS4`aZbBb6J3NS5_~`(sorIz*%?=jaxL|tPjlz1IJ?dnQRo;PhZQVp6k+d0jOu)ap9-`(*lMN9Qas7aSv6ABGI!=%auFTG4`r;OIJ$8P zn~MUz%=xnjZt($Xx$DDQse`7*UKv9kGasHT+SK$-EnaR*QAAikgf<&L>HX4vs&*<7i+cuzIdk zCoA3s|IT83B`eES*&#Ws)ohBihJ=H;s2%@|7^Io1?60-sz?+rq_Io346@1Z1rs_}B zt$&H{BCKirP>YqQV}h2*kw0l3rqy>YtT>WkLF%_q=xk|n{_XS zztDxYYY`l;CnE};X$aS%?R;FqDIG29L1?c09K3{cPMfcSzQ`nM|4h9&lHUPSrqyuWI7pe5mP)`gfB0AmhfFk@y?FiPtG}ZJ_i{dab`0q8dh`^r) z$4Cv7DZACjf`_U9UMtt*4dAD;E>js$_XKJbU#S)7V2en4zH|xycf7H~J2b+}?vm;{ z73luR3(G`aUqbGz)LZ|$T_HorY-zDWF-R|K2@JCt=>*lkXvdF#u zMD;Z7WM92vioO-r;x@H7ArE2wxTaCFM?SB<#JI{rV_}%$b9%X2c~}}6!S#iC^0el- z;`88%$V%-?K@_S{ik{90c$|BXM(+#F;1kT?LdJBi^1sQ?*#7Uh8$bO=T$j)(rFKCY+17H{geFhALHV_LySjd!f(N7Ium|%Gu#@=zGZFWGWzGY*GPGh-MCK zZdSals6_AN6gdbrj8ar?GHF|@lpje^?UZ+-h4PlP3B8#Xe0082A7=yrxz$CfK6*N|hX%9_~_YMH{e7)y&Sb6nwrlRLEL4P;yi2JBVFBAO3hQ*HKIj zt*9BS4a3Ahb*74V4h452oi46qE2rTVARwjT>zws=!G74q?2kdcrCQnRpA3(ZI8c#J zRGX~Cfrb0C6vdAkbawJs8|v7*?7j1@ifE{S(RY=d{#G~-{GpM0l>RvRKFcX$1&S4| zFbOKzU(g!ou2!@2Ehhhw#d|f7;TeM7Z6&PUWSGwTdvjaW)UnKG4{YSbvZpsUfTEGm zKjS3l=T*WXLn!YL63_BF8OP2lcFrQ+b2az+DDSg~Jtc;I5gyt?!N1QHKq$=$U1S3} z(Cpq-4*mpR^(pb9htF4iYQcT36jju&2~V37aL;!T!wb(15x)yPT%oHgeBXpG7&rx= z*YWiZo)Ox*LjU&wud3h*DtQOt&{5I*Zha0HUvh7R=fzh%gx>Bg&RjxUShx=e=D*M{ z7CO#C*|*O(A%_3?hUfjg%E&^QSg08b-o5Z35RL^xU0HkxePZ!@7TP(C7WL=%#2piU z2I8*>SCS#z#TZ^Kbd7}yb)W8W7q1a62g2V#xGRWf#QWKMf7*8i3SK9C7le{?-xY|f z1jy*UTJbr7SBQVY2|-+$_$%T&aRuVh_nShqSm=CK_~YhRCw`_ivnB zF`If{DSo4r_Y)8CH~M^EgqFAHpFSn?zLC^7&cvM-BVK$b98UVip}2eE{)vyiku8{_ zBHJeJ?+ETl--q~4D4UPqe{r|Pof6-V;`<@oJMnXLc6e9OYvT3df1#Kzl*olPdS6eA z-s}6zeLX3@7QZD%gt)T*n@fGpA;L98jQ75oD#nF)i1!pDO3a(SS7qhD*Aq3%$7bk$P@g02$~u#Fl~ksSB}78p^>aX9%c|WIe zl&Y!oE+)!t3|~?oK_BD1{v+)Vk>$Z2e+GC{1SD-Q^)>fs(E?Lcg-3*TlT%nkB;F7f zr}I+H+hp=~5gAoP3UTKi6FiMd#9~f;A!^=g(VO~0o=+ZfX=JooWOif828dnU4XY`c zxaJRdnLI+)w^o%*l$;svPaNM&>_1-?UICk^He44jCZ|_N)-)Xz!~){iJKSpp_*#GB z<$8P&6-qnjX|NfxMO5e#ab!?i_@wG-B7WvM8B(g!h<@u(HY%eUOF^!D7SUu3`EZdl zRWR>`*1H1j&l+%^M)GXq_*?jmfG)M6yZS90n{1-RHy|J3eRJdrZW^LHZ zig=Ux`#3og9F#dl_^BT{!5+7j_ouBy*jLE~;EhQpbR7Ly3eE;@LAJ1km~t`s8sQMg z>6}+BO7z;xznIkAg{Gs{}=z28yKaF=Z35i}-d`o5;_d50$i7wQ*p zI@&)oQPj!wuJkMer;YMYN5i*6<1!Rk7TOltm)Wwc2h1(F^@$QyZD85zw1O3vC)U|B0&a~ZcUuK^d9Un8*ImJ;P)o4Dc ze_FjY@}B%s;EcBheb@}J?RrnS_XFQFP6Cw)H^sfYdtYauL)j5%PrNh#&IcVSo#(omyIXn|x$4lO2zkc%oWVL} zy1B{G6gMzs_<%J7D(`7dY;j7aU25DsLJtK``5%)s%rjiA$;GwH{Q?D^W|zZV3eP(? za<8tya>7v)x2<1QznSql(WOxbO)%LaSIA%{kS)tY!L6EkGM)Ky8eOh){{N66|AzV& za(l`|YTqvCZi8`jlmA{E86-RbT^S5Z$%_nEGNT^l=RbCF4)G$lqFW(Gv}V9q$zRdY%0AXa+;a}r8nfh=&wQH zb|`tte`>9oJ+Kr{2RDE*elD$6Ox4Dl*4t8JrzOt5?BbB%M? zx@V#tRpY6Fy|c)LtG%n$Rp8P329nQzP&yL07XQq_>Xs62~eOfAV(p;s!{ozE=jc7f5al<288J$kOG>{B=0+C&g;f7V>)th6R9J{Pz z40X!SgU#*_x@UDSa!vO31d37cdqzLqm}5MK23fIE<~mRLFH#})I|=tdWg47|zmtzg zw=`mth32&yl?s=b_8$x^4QVXTixBOk+&Fqr+NkOjrL-N^_s<`hY5eB-cV;M)+w>iNS^*Mz!g(?psx24~c>o+51aN3T6&G)EdQ_Q==;-a?6SV|S(D$_Amg?u#M*L%|8K~XUQzqVGBIWq5xQBYq`~9K!H5KM* z_EL181vkl|*)OGgl6v&rUw6g#w0Sk*e`*dIGp!e*PFOQdGqhJDrvhqs5;L@*rwlEr z?66h+5$N9A+KeLKF|!t^GORn|8{X*w@-%ypyyu@)L zW?|gmSgXyXvxo8WaINZo#dXX#F1$fkXWix~cg(a8wGP*3MK1eFT>F?01w9>}LGmqC zs$rZt(|W=>%lw*do#Kn&N>5|=_^#CME%3{i%d=6Id=*sOq0Lau4G)uk^j+vV}Wzrqx}>^~b^pe(hFkMkuB?muTx z^T5H$<@SJfi!Am0(jI#|<<8pf!NGrO>R3H@#Y~Bt<6LB3tU4;;l1p9DXzzBpL!Ds4W1tZ2E~0kpD~MF{fcpe{-&y4UhiA)9_n(sGGXmM zDmN;AP|tz+_FKGiV1idHWbk-K~c(9)^*%=Aw7-JrgXS|@s9 zzx;Dx3bW;$cbo4qX@_Q}tuAg(zv_Ez1Cmmw#3q>(%9s3Gx{B{??{N0K?+0%-kD<1z zHKxp&8Ku>}C)4Ndi0c>6H$kf0jTddDXrIrCYB0^z)`u#+^*#AL+g)qCN8qG?tI09u zqHI%d^k{$JEb?jaYj2@vhIbNGHlNGo5fnuJt-)Gc?Ib zaW(a<>aKFFl|M1mM7t8I?kOJ_KVVrxfn|Z}g}~r$&z<7CyIi}2f6|m$E1l}t#+WJ5 z2h26fmtnaN^&IdH54I{?hEujwsx;@=rkmc^6ov=*H@oZII=@ri8abhws(TCk{RhoX z6qoOWB4C{FN^in8976nHKx-f~Qp`_h!TCHYZI^39*<`m?gFvi6Tkl-pln+*?JDast zuy4n4s`?GwmQyeV&j*H}6hnnp;Jgp+Cp&t#`*-qG{b0L3-j!UJwxfSiLbD}B{b^8g zUGA#uiSvIM`LBMCZCZ4TV~Q=&@{aCK_|U#!{)=wiy2^@xn2qs^?hnEtu_-{8lP-Qassik8hkR zV7cNfjju>vpW2YLFnX0?sPZWZrAY6|Kz&%Ntu9QVgI#!u|GCNatBGW`{W{~97NJ1q5!+W+bo8Smn{N$f|~wLjVd zb~bniNakiRyp^~jxt0ClM+wKH-%#Cf0JcFR=T9-N+o{vVyAO4sOE4NokxOl$PTR>? zKy?J&zg*C}#jtJ*;gYb6(KDmq?lh1d_*@DTD-1yKZJD zFVfa_X*!CTPDzfm8&VxRVwS||6L<7;#?P}Cn8%=CIM`4O;+_Pk2+&}Zsvr~1$Yx;}sewh8o3^~T-rbhH{J=TR$$|$8(YboIT zvmJiZWP{l-kXp}6@Dl4mT@^SxCUA$FspSvCuAV@h?L;EVGbq{)h9UPeoX(d~syZyq z2fY`x(tE2I zg?4_!9V!HC$^$JBDnK{G8Oq;lUe*1CeqbY<-+Gg%BQLS6MAdhqZClhrEAAoYS*Eox zo<|zqfMp-huGD-5;&TG5JwH?kYvf^K^_K(v{FOwz8StmK5pkaL=Ad^={UOoIf2lJc z3)5T!hZlwik4>~fgR;_4vWN08EF*i$#p2S?XZ+v6Qc@o$0#NrP+&ahTRjt>?>0hKJQw?j;(Mab6ox=K=kNRYZC56$H zZ8B4#bc+2entH!q;FVnJRrvt70E=&edHEA3=QFI#;!M-wO+!O)CcD!#n3&X2Q)~YW zqkBKs^tmj_#V}$t+`*+}#rAU@b#Qit>$=b{DyM~l$y*8kBMbhFoov`3*h5>n_Ac%~ z0bI!n*1R-&@;Z5?ELON(d|Z!YdXPjkfH)T*Sl8IJgk!6-S*HmY$ zH+}%iJk0OBuDgud=Sp=N8L=v|TEgpLgiO5!jNO;~XMDR@4LeYry};)QsQn&dU3`pc z{UKBdMBoFesbOekC0GGRv4$~yhdUbs17;kRv5ZkhQWN=^Dn`+Kc-2E_g@!7~QDpWG z3te$|W+=4MayO&T&^xz;e_}R$c?oQ}RJ{;p{@-89?1>~=jEc_5g9{Yz?Wg4 zl*wB>Pj>G*GmCYe>?~YKw8kpP?9QN?s)yQ{Z}g3ZL&j_r_EMN9<4_h?Geg?(VaYRg z;*Ix){x)39KWN+FEbW9xh;u1fUo_9;AvnL}1m1>|9`X&K9rvM9c^_Oz6Kvj30{5dd z;Y8tLKRwfe($2e#uu+Vjo!oyn_d1#NSEzaA!k;f<9*bUb^W0JTW(Dn7&B$tDWEImR zuksv4zY_Ul=2&>F&3u$5Wm>6iNhe}%a^0}ktJ zF|W&**)p@_aaywu^yCul5vYhjMU%-B%T#!8Q}5FJ2_2Rh>?CT~bFGH)#*^^r?lJsWyC zxlhqI!|9t;M%mfkp4rHbK8k+1!%P`RD=JuZuQLON!YRpO>+)z;;h|jNVji=(?_$5qAIVucfq zJmsu+d;VYh@4$pk>}}Tq^7voTQ+>FW2F&hVYM5g{14xbo0=syboow>uRBKcL(I82riZIT@b25o$ST!y=SI&PDc-ea6HEv zfEqk`zd)&g&pI2PZ9wxc zR4p%}&w4~}=482ubLA}B7uJ7IZE>_dhl*JT@ka`~&zG=n=k~7KUvbY@FrQbmww`89 zj&Lp$%H9vd+8&0M#|W6Ei>QYe^JxgrA7ypOX4Y4+HsBgVKL}q1;T)a^0$Vob0cmI5b21KFrOq_7}K{{~(Vr1J>#dIHxd3&?4)CqnD;?fR{Fi zopwH0Q7!W$y_Y`$6Oj{TGgUo@8GUYYGqXe+KtKxZ?`6DT~KA9+^CZZ|+c@&Fp<9n|Ugpf0{PV z<*o^rYr$NoVbvFip>R%~&$?L&ax@6M?oIT*74+D%u;#DA9sWJlT$|yiR=}+NH=n_C zK-or9EqIk$!38k%{IFvdgE_VG9>uIz*TI(;a}5?&(K7CRJ}nX|l78;O6b_lunQJmxj z2Hc;Y3!cO+IHOfOs(Fa&6T#~c__W}92z>kh(C!nxzfnWa|IoYQ3SNb9I^PQ0JzLoOm0&4|-_7s+sn7`=#X}&QeVDxXd*XWR z^hzIJLwKZ%hu~p|&wYHEujsLwUao-P&U_0;L9|XhC+ep9ey8uJ;<6buN#Zs>dTu^akkgt#X0_eGD2X9a&k z%nWh;;@|)A76fykk4GaI1cG%So*6|i3)FQd*s;*?6%6vReBH(FHV8f3Jk|5)h0aDp zatW#lEwH@jhJ~W3V1jLB-%${|3eIp0aluh$`U*z2pOe3xRd5h{YD%P(J*TR7hG%lO zYZ>3U%nl1HmJE-36Mnfu#q$o&RI?*@u^v=0N4D}<*Lx<^z#0{*x~;?~^QrEiMeLNz zu1@}f)jI|JJeF#R0LbM(IWzo=%BFHMj(eD!t+Ydo%dzyEg8f#EBypDor)Mm!u7sb~ z$U0ch{2$9YwvKD6Wi3wT+WqW@cWBRXcB6XM@&e)=AN9};oW|dyj-VAjbUthPyF?f* zz4aAMJokm{l0kjhoh$G+`c~DZ{)hIqzR_51$|r6s#0Pn#@khN%r_{Ld! zoG;z1xFwup_Cb&1wWdycar))(iDo#LpsOx6!o*I(a*Pl(qU(1Rw7y-u> zk7&Nr9RhvMv`&s{vTQd_q~3BdE-xF2+V)GRAydb(myx{_m-Z@pViR16R(S2JVfx{B z&1r}e8A$)9?0| zmx=!jp-lA=<8a$N=lb}GiAf2QW0bao`uA{7>;&&y=$jl^0JqdZE$921HAMS686kLh zDGzA8)cUuY4&acw3XQJ&@XDlKh#0Yso!St762IAX)Rbh>f?Mn*GIfMh&Z9-s#i^m5 zs7?{7Bon14BcDnI&uOmeEp+>XFcs$D$y(*PLY@C~@_5(C}C7kY3SnzAfpQ61K$wbZgNf@9)DPss@9QK%UHdRC& zaPG^MZ&IbuF8@v%jsoHLU<)6TokZ;iEw6a=66WFZ+71rg=g8Hj%sfwRbb_3D^8%755vb*@wr@={ND7Nh!7Q750gSH^XJ#cGqZkgL|cSr~Eke z-E*)|Hl3hS}lE1Vsf9Nt^_gik;%XO?>-nrKa~ zb~3>yymP>S{z5h87)F)9XPsBCLFx)Uj_gnRJPC(a{YcvxLtITUG9yBNmP~17LNDi6FeEtVLzPY%s zb;BW9sa{QuLw}e}OX0OQ;I0 zg8lqs{P9vts9J$$H<~L?QlnF-7)|a*3%4SR@V!4B9FCr4yHrV4 z&C{G?3UG;i2pzx4-2Zk^mIgk$-OE^e=HOT`he~cLB?CgA^e^aQ?DCx_!*4>6&)v(Y;DiCKZ+E8z~Y@2ASGhfy{uZjtK11r6|aJy~{RiYBM8OM?JuoEt!7Adk;`{A?K zsy=4d+osmw6q_CT6tx>OS$LOJ7s{tr>@fTP;xImu!O8xas65Q{ctAO~d9>ckJTiEk zfdTMx;0a#!s$c0ddh76gU*u|+UNJbME8-8NOc*$Pz?cN1Wu5BJf#Kb09hrA)dY%g4 zWoo`=6F&1E>p{bGrC$CfI6}7IOcm5uS#0(i`$Fr>`f}w_oSidJXB-$v4ljVe_deB0XkpFVey0g9QgF6RYkIJcfvwd?+#5pl~eUwU{8y0hDcVI;5s&a~9 zO4JEQs-xL*LpMnIpltG=@GSAZ5X_8RS1I9$ysnGYrr`pafC@T(iPVqf5%K*I|CIiU z7Or(mP*k)&B%vc1?hkiA6W-=k{Pvy--X(){4&A5CzBy!&Ci{<}A##B{@&#|H_oD9; z6w?l~Yy3I*fv*F-W2bAgr!+j@+!|BUZ)*Rk={p9L#T_!_hDuzgP_xUwQ|>AZ{Y77A zuZ+owIpCCnOlFypDiAtC@YG1RJvtej`ff~{z_#zwORg9)mAsfB$Z>fAT%l{3;u_JFo zWg&or+5NbZ+$GER1{L|j-9s>uM$vmUp|LW1A4q&7tJaHwm72|tNr`(?1`ntk)O1gh z<4N`V{^gxtw++30qch6ir7Sm(ce-L)q7!Wm#(dT3;7o6Y_bF=59@m^QDs7V-v9?y@ z$Lf~Qcm4v;IJd*=4crP3g|*gCy9TAC_Y{7rr{1IzV<%p^DJTLCRX0&J_lh(HJ_st( z=)~lOc7V?nQ-P5g8H+OMuPDKmqUF2{WU~j>i^$+T1U~x{Ejoq6%3yv5f0kLMz;F3n zK!Z!PF!A193)gV|tdYC)Ad?ECLHhX9IA7szZ zCG#WJh>gJj`<%7J|@@TPJEN9$fc3}D4(@~PJb@V&`Az;KXdBlfeQy#ByEne zE4w|TJ2Gyix24dQ6{?xms+iiiNihp;TMXYurujDZm|f?*i{Kl+Vq6A;DGA4PriJ8vwWg-@X| zTg@8qv^rLu#WTfd_>T8Y;Vkiq&l`9U9nL(mPfh4hZwaQ-llO7XJBT7&G8L+KR726U zcSk-&NA-xm4aB@lwavCN;r^7S{^bL9C9kwyRo?J^aJQ)K+O62TM(?G_Tc$P6(zy88 z8IDGCnfk$Cp?j5UruTGkr=mxfX;~7r#%eY$K(*^@U!iA_yVCPqU`03=h4_D=?9ryF zq9W1_=3ht@>Bc2GU)6-qYKxo-@_&jsJcqm7g5#*He4cF2URc-#p&xnX8e=R2oahNw zwI-^^UjY&Q0tMbb!&}p{=YP+6(axD5TOo6t@QeczLWo&DRA?8vQpAAlQ72TRoQE)oRd_EK$ zMO|FIVjVhoe^PHndC4C7g45D#Wc7c;s4|f&9*?Hkc}`Jhsb)VLY$tF01USVLWGJto z;)>rkEbeUTN%zZ-1x@~(z!k+l(@IBm?EOhA?`iL+cc^rSWtS_v<3f9E=Q58wbi~kP zpAb{$>~sXIf77``)0m@C{vQKFLVHz7hJ16S>Ae0)%`Amkp2vy)g72~5o=~%5f%*WE z>t4-hlmKdoO^ababKPXi*D3FSWG0~Tu^WW_G)@|1kZ>_ihiiGAN_VQ!ncLG>k?EHbt2elT|`fXS2ILQB!ec{dA>qJ)r8=siqSc&pdF zu#Bu?9w+=dYL)X)Uck#p{tqa)U`_uDMmEit2oC-MF;WyrOc>tcX&40~U`%WTuW^EE zcgmZ>OBoOE!QFb0DC}(TaOjwBs=X;@bL^z}N%8CLvD)KOz|+)yzSHcw9XPBo>A#CA zbF$J|-!k^oy{X(O|2IfxUGcQ~qV^^IU-T&3!{BqtX7nCL$>?fDw!*OfA9&GM@cDk4 zj2yMdoM5hV3cCuky;89{RDxFcH>j1hN=;O5{-8=hm8MKpsJsp$zZ1-N1QB~iSh$D2 z$9d;hoKq8tZrg}SzV^RFj{h-o;}Jd%fdFhIg7}b+QD{KV5A6YOX(YmYkF3KJ#F%bH zlWAe}_}DS=w#4Caqb=_#=J>07vN{iTt?@R5j2fGHj$^%Zy-jWECYIGu)7mL*qB;Vn z3f(Cf)hF=bErBEbeXxnzi)YC-+~l!WMcoLQ1$#Ifr@KVW4eiH7DMQuuiu}-8kk^Aj zJrQMt;#p#XOK|!#neT(b)5$p0;)7WVqWA+NcDZ~k7{>Vxw@TRXN8pwZgV&M_SLH#@ z)5q`&ok|3`8xHWJaAdxb^z?T@cmWFy zyynk#UFa%u4G(Tr{7YMARy)Qxu2@d$H)|Gf9XG@I^v^uvswXwi6P4tKQSs((6;sRk zC;I3Ts6-Yns$F2D)%XtPDZfOE>@w~x>(%&2hE~B?n1x4jP1vp2rOKp&#K0O^(VM}| zCC??(I;sjI`#BrLgQ<-n`u#6E^KslyJ_N_u0@uZabL$jVAAx+H1Cf3ip5b7$JYc@S zcFCgtf}AebHJ>|lFt9%qt)F7MKe{ZYBECItlI`D`VWD4q67B=nd~?HhR2TI5Xss=? zPBEZoqE3J*wUfHukHMKfhU;!o<-;A?$?h^sE(_Nx*ORCJz4|Lusoo@>A0}!@i4g{9 z18}qd$ZDxUVe~8voo3G0qhQUaQ1@J*oJG}W8QHcS>f`HU%dA&bUj#3EHsY@s7q}Dgs4wW6O!F-*MtT$#k7^uU+o-wt8ch?W z_F3w{cY|OK!rKhI176KF9AU1iz9q zp&-B3KsLV%);e`N;iu7hC}p?E$K!TT_*0^Wi>PgV=eMD@k2f^uz9Derz7LK>v3We_ z!pV%?cBvlheLGpHjVL+^-N@6yW;qS^Tq}9cqR@jxAGmKwca$EuuAK?H;+tdjj$%Vr zWTJnOYgTu@`?&lY4O&J9r)i7vq)w~O15L1^o!Nq}Y>)n1O^vcCyjB_vo_ScxgFW;| zZ8r+c)vB@Z^quIit&*1FPLD?tsOVnJm-wXAQlH(7V%cco-$mpv_kid>jWWWUAPl=A z{gqFn=u|^bUPUp_!^o)B`}m81ZR8@w$h`lnm_mMd!0gU7liQ{aJW^RB0rT2 zU^Mk7BlHZeaT@NKp;t|17idHb%Kld)%S{KIO>rmUmc)&89xzv^21+gNL*4DJRB69* zrN*PZAC&zb-IuTgN@1_gLmj!ulw{bfX{J{D1sH_C`Xkbpk=3fl$-zCZUacG#p`wb@ zYk!mpa+!J0vufs2Rkf2!qRCFQ%!z6559$l)z(Gs`FF&NPca%lfnE0mJA`1;Wx;$J z)T<)Yj3>=7w2MBcz3n_2iB<&?j;0giBM+S7Nff-cf=X3zCq=gAdDV1FWt=Z@MqIoj z#nP?KjJz0_L0tDj@Id5wbkV?GS>yAl<9|f+qCOuk)p9uezgOpn=aYG8ChIVltV)d1 zLM8qX=@UlVPFh~AT7phbg=#-a4?EfWR!}ns6Bx(VTz=9?zO-H1BEvS~95)OOprMxw zIuxv@05oD9%z+x(<3sWPLry?G&Or5?$LqZ+18Nbhhf1*wU(vc0W zF<;RXwkV#{H``NV$2&(yH5p?Bj)pBaYolOhvOv=lOgO=N5`8KVUt zo&v=L>!cpQOg?!!dNf%eM$d!#D8P;`k>OdvHGqC7D#Kf=qh5bco`lis-_w#mNT zxDo}!vfk{?+uAHtRs<`48~8_$KPzPC9uz1O!e7AR)8Os14-eu=`oC*#DRD9}8a z*ZCRkZq5*J;XtSg`RWe4PzKEC>53hiM#~&$ZFGa>&w49onjHObPUf4n3Uo!z5@AMB z^&m+xuv&(NI*EJq(rof_*H9)Uiwiq^E}85UqQVnOR)o-p!60X}*8`^|p^fzz>gKqJ zqJ30GzH1i_1;R^+YE9}ramE8tDCJB$7A&lbY^Dr)H-s^tKwpi5IX@26p$POXn|as_ zW9Amw;Z3wrD1M5OfW8%3&;;(3h0N<%a?(SnU*&wpDnPX^dK2_8IfbTB8F6zNF~n-s zIOUhH?+56vMXhnx+n1ZH+5=P%kEc@e6m4Ed4Z`5S9Pc*oul~cbV81oVZxAOv6<86< zRs8`LM>YAnO^O=jT=ny4re&hqJtXupPA*SSb8v_2`kj0by2vS$zbqS_4l7 z<}Q4Y<3s`t>^tjt=M-kHP&*fVjda?(8is^W?r^hSW`T`0fnrwTIs8}s)Tk5oHP%!5 ztDM%U>oJ#E8jas-auq&u3~y0aXk+)bgho)4HPP3N+eBBm4)n)P9Gieb;J?9e-_{Dn z{YB`-{C}jqdt8+Fwe~&d12Y4|Ng#@fV$`TH#85-s)DlY#O{rCjmRMrZYBsT$s#U92 zt@;F`IasC2R%&QMlx^6C7;1^7hFZ2E)>vYRK?7Ee8br>_40E1&ziVdmvE9%6{`ESa zk<;Aw{X4GTI$Y~oc22$~W=TO_;*Y3JruVFe$L0u(yWfW`x!LauUIf>!VTVW+)EjsL zOUP_a#46p!zJ8tfYYP~a5-`JWvF2-tl2w7USjo&u#<4`S!#oXD%yKn*brg7yn?#0N zIdA84lEYBN+*n!H4XkDt@-TzFEtLdTvhJ@ zLJfL_HTDi|7|RKY|%%#X1@|4K$ z*u&Z5M9%yb7S>qm6vAi9j#|bkN}epMgS>OHyDPaS4!JB$Cx=)GuZw&_RyA?PNCmb6 za2=XZiv_J5$MxmJxMhbJh$OeN#=|`pL=2HzupYuFnn--14LhrV)drIda$0Z|t%^me z-{~@`Xi^!mQ1^kxquU>`#8PcJZR}JRpsD-N(hb3igF9_dFpC^{6f5aCC*Mx8{w~4@UTcccKSeHV(74WW1?J@?+6m-} ze+LG~&B_JA71a9pF&5R|VKx2;>sXHmAV@_vq#<(1KonO)z0L2DMLz+K>_*P}Cpk?^ z(H}QZd+moH=bAwbP)QkG{UdK4Ij#FbZC#DOWdpl94#}5IJg*R+eE~A906f>T;I$92 zN{M9!>cFqv#+n?(Xvz7ujWt}!YL@w~=QOKi=Psk)Qr&X_r$tjtPhRA6ayr?` zg)(sVrYReh$(qxK0jpGu^}&cnYMY-iEYh^&7kor20Vt*+g=|!NJL0o>zvX!b zYm=H6o;+U%k~JGYPCF8~5r(r}%<={}(h5NkB(u&N(fZ$Ig-T_|Mtmt-!P%*4_crjs zM7Pl5h)S|Eg(s&7Pg5?ld=K^m)chaA$%<6CBe7Mglwqn5s5(upBgw|JIB_KEJHHg^m6beRV{U zxhvw9ZZ%fxdz$|wyLhr{1D0n8c^gjjTO0A{G3d92RK3V#mA=mNv*6bv%K#MC5#*u| z4$x;oAvA&DP2`*qq|~$MS7OB(I8o-)>SE3%`Fhb`z}X2Z7CabUPITY~Ft+c6kW4Gs zd5^KjKc&}_Z%~bFlH3UwV|9a(6wI2`tX_?EWd}t{#!N`kE#vb$L)umqYe2XYu40X~ zftz`k^>aiqS!L1og4_3#leJJg*H9E;G8Gvg*2Tj_@e3?mzh)8Hy7kPV68*IL01~Dc-Et~x zHI-hMFlJ3MW}Gv=Kq+>dpr(qLjlIlDKj(2Cdt@rM<2Gb=I;*yVK4x(8^n-aY^P0pS zmlL502`5Ol$&7JnsK*-1G1u1^D-L93FA@22;JM7 z-P*;({a0zfj&@zm8WCi0MqsOVB$C(WS&t>Z9v{Me_g9`xoL@z(xXp}xJsym1?w$`y z2i8Q8wpNvBT1RNP%-lWhBr<6=GG%N)>Tl=c*-59(IrL&1ve&?RsAX-+*}Fke$eoJ# zUwERjz&**X&EY(TiHti;Lz=~Lw^7_Hi`O=gOZ}Wq@Gf$Gi?%0>O@beLLs&~W2`|ES z+Ct6uXgDo@PK}-(5VB+R^9-lJWN*|y2RrMF+R@r7bshc(Irl(kg39<7_R{-!zB>Yrp=sG8`zmJh|#}#t`r1j2@*w4<0j6| zcCNRx!^z8IcURJHE3)(mZ7JjgJ%Xf2h9~~p@D^NyhkAi#oc5&dDE`vtb^k>T+|M+3 z@zA8>Z-VKIb!+EIePEH+V}t*Zp0;=e8{xzo`5O5qf}(1JKffB^t@LL#65|LObq&&} zmAOynd9|EH7ioc@)I$6zXxh&rU*yxms(z1F_k%53#tIhZY6YvJoH>{3<_p-9f|qRM zJQ6ghNa|YVax)QrLCHSL4$elmGqdjnVQWPXyTcfyah>RKJDKxTo|{MiTNE|Q6u3-Y z(d+^Zw_Wo%?5@{!4q+*zckr z3rkrJJt$$O1lcS2xdzT*!T*K#y3AY{kxHg9a$2yW?cAY;I|;V%D)*3a6nyMv#$y^I zpBBff8$M2#IWU%&X`MH2+^#`6~Rf=@k%iO#`3uOic zaW6>T&AbGwzcRG$g{?}~=EuCG--7cL{lh?C1&tcV4iU~gL2UXMwHoGJ`u{QGB^g=I zGGe7XV;b|F%vffJ##3?{M8ajUYM!MH^7}Gm?i1Lb@J^yXM>=4>% zyV8dS^#c7*2RWHToAJA_vIXH;$H;c0akVp-W}Y;k*Adz)*m>FKi&*p7%&0IC3P0)w zc3B};-U+RfZCvf+B+cdIv$9?rz;PZxPaj1#OdV2T8@+9&9z5D2btKwlook=x^*y+a ze~V@H2)sG(q0cnp3ra%*W@F!{l84iZ3<%X1E|oK;f2cWp@Tken5Bu+(nq3oChvXStQ#*gn7zLizLTLBv~o5BPeLuz4nlnpg{7;&X$_=caUxdW~-G^ zauLN|4Wmti>SJDw^ib^8MaWW%EBU{(l24af4jl zJ;a?>;Nz*{%qU@%y}|fzW+nTuCGbDf*Ams*5RP~!r_ct@;38%s%y(GNUSJ2)=GDjx zD^fFw-^gB-tj`x1ammCHgnO8WPmu7E7a%x(;mMPHgi<~WhbPi^FV{%+#v)#6+{wqN zNG?Yjzg@&VB>y3You7$pX`+SMjQuO*W|UK3>rZM6JMPcaai}9FB}a7?kIU<<=vuC> z#jd`MM9IhUdn}|A7qPbru-C1~_A1UEFe*I#9-bOO^o!O3<0$y^y4-VI^(jeTX$x$1b{v9wPN3#2YERUbRS0JP}l`e}NSa zlWr*Uq@MhWnh^dugWX)gPpkP%Ce~OkpUJ_NdxsU>hJ_460HayO9ruN73>o!So>I%Z zo4IhYqRp(_YQ|VNJu4XTQM5liXK|c`qCfcf>uT1SaHPLL+Z)ksHzQ}#u*}y4R$!L}`n$^sqSbZAy z++Ln0=ZdTd$pE+)I)^hE;atX7Y5@tiUjg46I4?g7tr$7Wt7yedu&$EHA+uA&>G5vp zNlmn=fxU)To$GVB4(?=TWEm1pw4r)NBiwUm;FDk`vzUQ?kk1aPfsH>IZD<3TLRFkqamuN9OiS6-wVWZZhx{gzyVi`} zxiWy~l-`QASv2!GcPwMK3-lkxm$gX>q zF{nnbF*DPJtQe_{(ST&^<}S^wq8ohQ!af&oi)2U$Lu(B)Br8h1c47rhU?pU8=QQ4u z$1P{jr_6`2KMKEOVQ41aVLdeS>JGhHnU@BnudLH|v4wK^RCY)!aWSL97+V`^aWx~5 z&fP2cSr{|-^28i|SHW|n?o&2@6`N9WjXz~P@O`pQcd{c7(UUbiC5_X$jgi^ONC;8{ zlnU2N-$e(kU>t5@>n&m&cXBTom(oy{M>}h;9U1o)W6(-F$Fc*DumcNN!>zRZ4c6Ev z?1M%4JQ|q~SpSjUd7OKP*b^zVGMjgK$Wv9qS}&xJd9=fTUQ^53h0&c6%A%E7XbYlS zRP#iMx!h!2t&F4(4&gP7T^3qeGclA;VGFNey`-~7>`2^EFj_b8gctZMmO2^&em(5= z?EldbWp9etxg<0mutdVrSc4Uoi(Ho6E#Voy$f(pXj#aFod`_Zro}J4&siQR;n1Kpr zQ!+rs>ni%LFbkwJJ5s?XhpP;XbYp0hm^pn7=-`vt<*(xpyUpD9BgJ;o(`xjHT1F-x z-Su_8uH?NInP&$h(ulYI0BfODp+HknkSmwV+lP+0nwBPVq8D?5WaGgXsU@Slmh*5D zd$pEv7fU9WwP9sm4=}rJWJZ)QQ`68L>u8tgc~jwDmQzC3O&Kd5|1I)mB{F0U_m*6| z{*ZS=JWU&rabhuu?mw0BsHXQ;Mp*bh@3F?p@dCA?_0@9SC(P|zXb$hN#_zD-Kcg>l zN>wsCHH>2#*GlH~9rO*!2oq`5ADZ#K>`dWd&Bt~*!U&3=Tlh}!v@-T9L%S;*N!!jk zszmbc#3D~!sg9=igcag&6^m(TI|aza&5D&U;v2{Tf0EgeGsedpKg-Ej z$4cGI>kzBVhsG&3P#Jr2C+qz!^qviPdwra_IkdPv^xG_$3L22k1+2_8=1^=+$x6j4 z#t)UMg#VUty^7zso^v}DcGzv)*~}_+G5&kmBV8>1II>lcX{w8O zhB|l(5(ppsGW8bqT2(u&B6Y<0-=GDu4;8^e_C5Sefz4o2nnRI5GgpvTi^l1rwHufj zJ1gr5qnXO;YsDUE_16Z(v(t>1imK?L8J6t8dKh5M_^>OHS#R+9Nvym|Bm=oxNO7X! zXnY2wOF8rL26ON(GcLJ(^^D(RNQiAn!h9?qACdI~jE7VPIu24|Ea-_9pkH=ki&9Bb zaZwqsP9hTXH8Q>5)2svcc297K=uvgpIg;&HORLBeMKk|IwN9V9Pt zXBAQwzv8?2=@RifB{9?66scGj_26_4A#>5CKwcDq+=?X!$P1R^E3n*4z~YGi`=sXx zx<}Bv!}Fr^hW)p$=XL71eIw>Yx0#mWM}AvZ7vZI@RI%Yb%^SXa`;5U%`$Ts$wIzJP zqwZ+xp2zu*E54(RHAI3L`-VDG)yDXKPyI0!Sil_xenck3m!6tHHW9NpkQqCX6|WPK z`ERQEl#{D7pIH~lbth;>BJT`N1%pH-ZTuoN$d2JiWjbq0n2>9NlgNF03*1{iqq>2N z0TZ~0|8QLZcOFSBW1c&b`e0vBHEXu(Z|?7@yJ8lPdM0T})F!-#6RCvOVww{1U)nh3 z6Ta<3X9o0x$DFnP0;Kz1@|v99PlKy8ztu@qq2GdEO9cyBjZdaVwU4Swqu}#;-@6ku z4H!~YKl!?`lEpt!7UdK+hDC+4v-Dq!2Hq0WalPKiT-8coRT( z{Tsgc55ZvR$+7wkTnj_)P4N5`u>M?NrDnN%-M`gVCRU7}Ic7?1Y{aX&#k!XaFTsN{ zOP{NZ^5)o1cgJ_f4sP}4k+W5ye8La*lKLr;darRmxQef7vZ!QI7tAA5avhamYN@^Q zvil29TENBbS*t{P2X6RXfw#~Yvhn`jBlfu24_`EUp%Qtv!5>SW+a@2VN@8{gsQl5u zN<6>{{tIkP2gytN5s@MjQJ*jIH3z_@{00BI?DHqs=WDzr^kik=15HI-Yih-$Rmr;| zf2-T4nXP*XCY19!gW~(1D`&VcO7X|7wMrDvUY8GHn;><2Iq{gF83dVKncAkEf;{QP%Nh5J2V+^CTK4rlj8|KC0J zL4C~92@@ZwOW7S~GDl5jLMqx{fo%a$-sL(mJubY!a0S0M`a4~{C#?I zG+zT9mm12C%|?cT2LS;v0X^a>yL>9^^zZHnY960;J?(jt2$a?THxP6%f`O0yCca<0 ze{Il?9s&X=P;WAg-ZyFc_<18HTMinFj8Ud(W(yU>|ExCn%k4Y558t2F|Ee2qe$@-X zphpE3wF3$th)DRv;R~|| zTR7*QBwnufoOAsIiBiP43!Bj%;9xJ0FOXEkhja18EiwB zxCv(HyVOVj7I?f=^o`_T5t%HjgvrHy1YEepc3VLNuE4iMEjO?`g8NwkI{YL|NxyVW zVZ^^hB=8{FFXx;uyJGRbAMvFHHefwTm9_uW^^KS}u6f+LQA^_MtZ9)wmP=99)+yHV z$RZ=`#rAW(2?IvwBwrPJOoFG}k#5fzQh6o?l8hQaBO8>HHGiV!;Kwk_ z?WLCVCSPJesst2*pm~zD`xm77Utq*b#4|p{bDsLObKnwnvNoQ;N0A8s=~ObS+rT+c&J5ujQD&4SY=^GIYwU37$OLxd^BXWg!R}kwtn?CcV-F??<3^SE7Y9C`KA0Dwu;mq>?Gf>q+EszX|j8T=F>PI?sYn ztb?I)w{wX6lwwejT|VKCnT9tJc6&6Zy=dX$`M08Zh$g>)(@p_$PO|5-@wgX*Gy4me z<$b{yz{Y${H(fuQ3WW&~m8KT+Eb&**XrTJ*YeDsR5`ZK!kXdTqH&!om2 z=RdgM@B6%;1(^IQWk;pW2CXCzzH?YKNN!@15W9NFjHUSz3(J=za;K;mQV*r`6= zg4-eQ5nb_*w4a{h*?yN;-*|Sbjoz=IeaDfM{|4*VM|iwDxXWhl;==!U!%sb4tdT}6 z-hM?j_}P&#T)kbfk;iAs4%-5M8%Dkbx!9 zz&_Zl`alz@FE$hz)6ok2te%0$TTx|JQ*?#(T-2G!%a$AFBy*N2Jz@p9?;Ev#Y`ytt zldvuM3y{nSo-1HZ?~<3>L_PL$vRl)KGGXgeQqy}Pc|CuCeKj9!tE_JrC&3B|g39GT zj!pRv7G*K`xdL8=_<61Dw~s=*_BGbjCT6FZaetC4kDxVan6nBzlfnomoN(foDZwvR zqBK&o^ct*>9c1<7P_eYqvOKbfs?rytI-};nKiPEv-FzddCBN=G{gR$%`RQ%{24s&Pm{B=16k?y<%Up1L`GNvd0;CN zk@mu!CKgCP(H81{(5pJmmSxxz}j3zL)eK= zay}ZC+WQac`2NCm**VX-kDTLdDywgSGx07g@Qy`+1Nr^ zs>$kgWO>Xb<9H(MXz4IMw!rwf%iL<#G6&h_F4I=3K^%jB^Dr}E)G9UiR2!6I;d9Ll zQ%lF;>@T{;`I$A zc(rgeP>cAI_8BT1{EWUFMO*V4MpA|2HRD8Mn_)c+s6W!bs`~)+PpNvDszUiX3=h(W zLm;64N^bp+;5YasdV<4w*ID5_1;gO4U`ZSd)fxT~>*+}(A*{@-cdDwfN?s4`&?d0v z&0wyj0!J(7nS%8tynHhD?LpvFTj6n3RSSrBQrxdWgcVP53usVU$R$)IWUV= zQ-Gha29KfOe;09n+{D5Z-&rkQr%dMI5D}^t>hYL`X`7Q1#ukv9AmQ*I7UG|o24kc6 zruX855pR~n6~sSOfTNtH?N$X@zcjVGg#R|OHi1*!?emA2ts0Xfc< z?VM%Pu=ypvcbomWfmIW(r|d+V6i(AbR+Grg*RdOJFau+eSFf{U+li_y;3Tf22NHoj zgqK4>>)`ev_S1+LKsdI#c{dR`AR9@1OZe8Bd2S)QWE3ZPJ+{hTd=}#G7k^$hQQOHO zzpgN$2|r;nZ}tLz6U2bTxrO6NYSiST86RRCTJTbhqS{e99_ab>eJ%dk)qKrEYf8n7 z;^K+#gq|r;0)<#K+-Efsa{*5i&N(vVLib;Zr*Rp75qv_prkr4I!ZolyK`MP3dfmZS z8s=Nl(1*tud-0e`#9G*|#K&5TPfKch_4C^to_|%D&YH|t-66{LeV>6nAt%$fd?H8o zA{P%JAvM$^T^q>eE+vc>*lOmmpDLK>2qX_RSRTUYh1FW4xn_qx!EcdYK@5TujMX3r zp+w||R5X(ai?HGyVZT_JpIqK|IPn`fW8lDMwUT8@7Su`44A0C^o$0jLi{%+-;G|}2CAY5jV8@|Rc-gp!i-aK=KDQNtO{sSWa8)4}zgZD5= z1-dhc=Yd?RJ)G5cemjV)Gy2np+r|uYXXNylRdMyPoskzKa^a_)W!!4;z{;_g znp9K$L$1ka93Iz1-;5TynIhnY^a1|_@Z%SL{r2LQ>MArTwpFWsUzm< zKTtO?=68?-7xCDQ_x#Q^0p@teP>gFoycX|svaCV+>Cr)6Cr8P}IsXTkZNB1+9fxnA z)N#?Kw{7h|+PAgW*zN8z_T6$8k|_*>mwJ^p5?v(2+-fb0okjNGmJyBCJanj9(@qXT616?5Eb@#YO9kSK3N%u&>esGSu2s%b>Q)79ZT4C2ps$4r%v!SO-quCv#~5oO zD$VPnwnuNb?zU9Z>nNj&%;c|W3Y2@vpNfJ9B83d%ORzx1z<_j~@2@zokhQ6%<$Xwt zkMMbc?FTu%2aevm&Lu;+R9dJXT-txQXGeE&|7BMbbz=n^)J(PO?`TgLqsspfTjmAPcKAM}32D*J(FAvv>=a2Rbt(|rd1 zlwXpqI0kEE1lr3y_YC|{UwA%;rDHVy0zVuu<@RiQ%Fy3D+X7`|dq066IVpHkS&H>@ zO}E5Q2sdsNoW+|Ws00<*7na>rEX^a>6_S_t8qd`5*9H(k1?Y%P@bBm1L$1Z!2Fegr z4g3abBYtTMn4;w%jJ8u_={IN^%R!X7HP_KSEMPv?slSe%(jG#*WN`AgqXiTawX7#z zUd%b&&pA}iNhOgp4QSDeM4a}bO9;!dFQgkaqi2M5>^ERK+eur*hA0f_u&MOTh9vib zlX)L2;tl$?m7eUOr|*+-U?7h@2c4@049#ZlJ&p6v6^cEMz zg5v;$C{+_a;q_y5?`Cbj&Uj0k zkupf{H;}6v&@SX9DCgt!K{&o$gRI_B=WTVRn z3Qn}CI=ny`A)PPv{GGl_(s^)8T=;sQ)g9n1xaWiMH)bF!$&IxL9EvH8{7KLa!f*9FGOB&JoG_*_D zOkW3y(S*L+Ph8qctCM-jR|R_FRdlaPKCzizkln-@YU1i`oTyKRa12-Ji*TAQW4uy0 zQ3ZF~NGpWZ^ngUMLNBrBq}}qWqQAm#+CtBTLp_=ESYpVToW;USD`h!zUT0^hVG@*R-uL6F~KtG+#-}{N`+@}3+ zaJ5wD&O)CNmQ!Iy6%9)`Fom5{G%v}Ml;;Z5sCZ^-XpP`8BqkZgJW8Cog5QcxQHd5P zZ$S*y^1raj3MNXJOXV$*%P>Ytm`vsOqW#JD!htHi3-gx>mPD90!@RA-yq!|VRN5(; zp?sB}!+K(7NM{dMMYV>m5+tw0R0YM0`WNbxFndbQl{`zHD^L84r_1xhtq}gz;g_(n zhWS_}_cYAQI{Z1gcK9Wq`t*O^5_uJV>2PkBFqR4e5zYwgDQSlxG{VwCVMq-#kqTR{ zv|RW}1(__oqy}bK>gWm^tz0F%slrSg{wLh4zR*a?cu5YKFtdiMnTDSh{$DsuA6z5j zAr)SQ`Bm!1hN~hAGpsx>{Fau5d2nUygAg>3t znZxrUV6>OFk!8hg;LnGiBt$^YmceTyJnk|>|9ss)|195!?=-xYr1jxi-_o1#Z^P>= zJX7H@36HYOoQ#xwUf7buBOuQb1fY!n@HmF&S;jFuI)ZnX>fYgY4X@oWzpw01xmH>x z<0Q4a!&Q%E?aB2r@^|>e@II8!g`XJS3o@dzjs-g`OvE3DzRHY+@7a$OpiT-FuVD78 z_-QIQl@{c)89&1ttb6IrR6JU#NNLGcA{&Do*>rH4De8&%cyAaU(%BWCyQ|QAcMTnL zyQpBe8?9g;m6ks89ffP=B)af%#Z9t$AJM&{t_WU)iRag@V;-NMou*o#HmE)c;(MSL zQG(CszeA>Mvid3vHe{;?sqcuM@{WI7FkRV5Mr1tITJsbdxL?}HwIXs4{!QvWY=Et% zfCxvkZ)YHrT)O`vrXE2xh2Lqb)nN8PES4}v4fN{(lDL7BFA1-m@W-#^yzNGZ$PToC z$LWSu=^nAs96UaajLdw_alt0Kf`U)8gBX8OafnY7(}D*;xRd*X$C2qNoZF4~Q1*sw zX(|#ag@b(~I%f)*1KnVkQW?=vAesBYBxG>fp`UQ}e#%+e9*`6H6YPx=G>x5{^W{OX z>&y_g9D0lfteFB`SFo@KGL*?`L#F3(mRiA|SiuA|pgkQzpKA7}(W4Sxdzs5j&iFWR zPb+!$V~XcEp9OnfMojTI(kv4X3K@^+O=N?>yC+D;R5Ye0yaWCCbMyIKq4E=z7F(Vw z!0-yDk$o(g4+UrwE_@38_$5+6{?+0|+8N^an1s$yi(gS{7rh`^z4+a|*tx0b)obu{ zOAQ(srAGA9V)V^sbUfiwChLHBtX=i*C_Uo9ZhKh{s7?| z+7CzRkHLyu7;1onWXizl-uP~R=jnT=?`3pl^fge|@eB96esIcUxL?AHZ!<2j=&ix% zU~K7#k%=!RsuRY?&yBejX^UtxOw>oiCfW>=6wVO;KZs@4Q4@HVqkQO5V)d!MYB)(> zP`#jjmN|4#3*kNOVz>%RHJ_1NyEk|oo4agihwXBIPj69AcGrZ?oUiuZdGq#5ce6VW z_e>wSMZIB4xDs;%a@}_*38v!GM1aCL|4a8NopNg zom@6%?${fnD@N{4+!*JNsxyr@M(C2%hsd}s_MUR(I!uGH{WJO#1`ZESaP&C69;z~h zurtw`2;FIYpV4HhG8IyEaR4PXs^;f{8A`%m@^+`rqAd^hXMbN?v1zUJndyY9~O zy?OSZde9`*zt*iW3P0-1xW2@Rv|wpU!nl1Y4WnOts3UH=wE(2Ud)gFri9!>2+Izy) zY=3Rwa_`n2WADWN`L^Vtm)!<`9exEPb+zxqGm~tJHn*8?5RG}scv@#s-wd2{ci7kW zAMP&bJa{+h%lvG-Pg zbt(IjcPEv`os61hI&T=Ki&lRUyy5G0FC0=2o?&hecAe^p9k^(J+oi@c(V%D|!fesJ ztebDFG?zuLi+U!q*!+_58C|7vk`IRJf$ZL6_bna!Z%??H(?0OUt?L=L)9+>V)DMn# zW2>mHXlH{B8gH!|F?&?mn0@1Jq~1uKOTFdFq}EtVO7cozU!%R6ob1_ z-p{;$seAlDrQ;#kJjq5?tX6G@?`gcD!Za^(w$%}}FEZ2IWprz;$}!$b$EALEcYf!o zyPaQd`p27J?D^u&f9!`W=3uwUHpOiXR;%3FtcaS(*tm;H%SZ1{X&7IVS~0$F?9$|= ziIp*{A}5=67?j#<U{UY+t~c**?Ru$q(WS@X}@8aW@Yea?k3x{-i)r~j)S)^-^ggoxHiz1^yP)G3c4J9=|f|Eamw## z3ycGnIWct!Gm~FT36859Up9W?xW>`Wq?vKcqbit##oB6Rs{g!u*HEeLwZ4&EX(S(n_BSQFD| zX`x#8GEKK)y>F&FdI+}0-h%G=U6;Eq_vYGmJCA{aX$X|5e$Od6(b!;~jX$jkl*kgx zOA-H}+ot-y&+Qm9aH;#+J>}h^Tkf{3tM1kd*I&DHtTV5-!qMu>QC`=4sV_7gjWUf` zJnF{SrQ^#|m!?i1H)V7%u`PCTFT;=J3Isi8omIBNzG>a@UAMZ9_U7ASTu*?8 z0X3;CSAVLRu1|~bTh<}nYOIqZyI>;vM772@-x1rtr~BBwJ$D8e`*WXP{#F!Hi z#RGSTcKFh$1~4^TgT6vP&1(;bxg|ik;_MPMbEd~HvE+6ev)QG;AMBZ zW6r>t?iKgl_XoO9^iLl8!ejFlb4tF02lqExr(vmOmUUl@H|7=Uq3$=NbUN-_zP{u0J)iBlntL zIeB?vM@(sCvuOutzn|&HD*xt5ciiZ|)N`(TeRp2Z8J_(b$c8<5%8LBu+~s9mt!Zi0 zi!u7x#nD>J8iP?&790;EXaC@ezT*1_@4Wesq^p-db6L9JJ9E^$@vZc_YFyK3Z_@S=jd2@!dZ9Tz!lN@QZJvC`n*%3$r}a+gyVPGlnCWPAYM!CdTo+xL(3;$k(wZ`HY}x3xQC$y}CDzBUib;+<95GIpt9;8l z*V$n+4LbXaKsx3PHV^KyuN$gh#U1x$vuCK$9C6n?-{LoWi9XNM|6c1;woqkv|6peC z$T?5L6P z7ay86di&_=QSdk?txI@iL|gpg*mJDcnR>0d$p5x$ox|dYcffASdpZ2#3!NTU(DMuE%S;ooaWnLp|)rY$|rrD&N6%PKyvT=u9F>?ZaZ#Wy1T#o zwShMWlkD4v#(3<3E$SJD7E`r#PWDXnE%8w!W{tQ3L)_B1D^ZoE^ZE}o8fB4R z531#J5J+>Wz^aA+`n6_k&%&0un%L^^HFs2Z0wz!8&=z~);N`xp zJ=ynjzsk6K@?J_`aIn;l{>8bR8+=dwjG@W2&>DGa09sFY#v z7+l`(?%8@j{@&SpX+1@Qt@a%1ke_uG!Ckjb(?VS0Y}BlngjjVR^_a{VW7Q??Z)AMh{g?{Mm^O$_^`unjLghz@jz(A(|N$)A} ze3PB?9XAHueHXg3?swil+E-{lN9_K^p>pR)Plw;GLI)xqFg+^DIx8wUsy4bV4n|DW6fu(Ns7(f@w;G$;CLl&^b%h7&*6M}7b%tp&Z3sl_$xD^qDJQ_ zP#am)S1ufQsrOd*31sa$kPb^>^}gUd;a=fitGcG!W87p)H|Lpi%vUV8tyAJE;-|-Z zV?Y>L_^qjxI+`O57A?#%f#b029i%SpagcA%P-!>D6>xnEUdRu@NF8M6ClaMtLIuz( z=+v{ETF08f!~Ki<)wYGsFQ~^LpP%Pe`;v)VxwT6SM~&+vUZOU{c48oH(S0#fVg{nB zElNw9xzMaO?Kd9OKddb!Hc9>=Gu%R4EP>G;O$6^%_lGe0pYVJMYTN@_U=ERme+M~s z(sc{o>?&%BuFgOSJ1N*l@=BCkHiXH?M&H9k2hj-lJE;TezNn7nmzejS*9* za6O9}*B;$&GW}PA(HG==C05UE@Gr078E^LO1$&kcvZ)^b^fs)O<00!R4!+MW@(L2b zv;PC8#i<~qUJ5a1j>4Dj#ZH}ujgyQm2U-n2uXg1&Vx24110eYpXdY)DreM7|K*fDP zRKSAYxm|Tnb(82sx9SeKi*jC-#O#-WRmSV|aY*TcRb zl~R7_8Bf1eyk~^)WnFOT9tWwuhM3VdtgD*90&M+5SawZVq$%voZ{WR;*Dlfi4qM=s z)}f8j4Zu=)SNkVo5{rpRd`#r_Gg@aSR?>(k_8mM9tFci9-FBPk$cOMdo&>%5mHRZa zFx@=@KASP%_(0%NRZ;5q{s0t2KBMt&hz;da@&fYlQ`e}{;dNgL(&BOAte1Ii1pa`p z>1OGAh{ygHQ4v2@9IUU{ag&MEEW?*QACy87`8?@ZmxaW+^TA*2Au96+`ag!yar4a6`cH(jB8oe^ljelvus@w$liJONtm5S3M)AR5{S`nv-J zUsnia_DyEz9il98pl+n9a2nq923CaNvm~#p0ee0V8Rb*v5XG;9S6#`oCW8<7Ha6{3 zywmYSiGoZ6`=ubR;|ig4476=5NSz#FB-`)` z)X;){#vu#eKowps4STKt{6+)*eV;Jjfk`<)UnQ#1%)5m+OdZdDo%XHd`GV$vClpUe z10D#;C>x7UOL8G4A3hC#ju|9`#KkW1zhurzB(#OsJzm0jlTNfkazHEa_sSD zIU{*#Yw&37CDL|?*BeAj!tX;oNpch=8?T6$u!rP?Vi4g9UFMK)O#H9no0o_|4RHs_ zv@2tlN_k7=e*dr=;4SOM_hw@~3$y3FHJp2m(6!w%# z#-H3*{Fq}y&lGRFJV%fVK1-TKkQAGr^@$XPjfiNBcB`g&JKH;Wj4a?mJtm1MEvaXQ`ny?o}aMyJKQ#D zk32IR1(6nqKOgq_%T>}ZX_4GRJ|`m^ZfE$O;m^z6>!ioS{go$% z?-agc_!Hr|7=E7IHTU`yGe9H+WX+n^7L>E9`sM{{Qv%MxaZ-Xg`fZ6x^NrhdGeh^ zEaVg6rw!jP+-v!}Jau@!-EKUKb$zy9lZ4*ylY%KV1=`rx|oQ^WG$!Tg8io7^cZ zfko~POTGtBlzWDM3g0C>UUHSptL*0Czso28_djWiM1#X}B;4-s75_T!!`BTzSMD!- zKk^QbfUGQ$)Y8T~{4X&>$&LObbXxU?bOj&(OQi1}dW2;43nJ(a5h4T8oLn@SI)19< zcP=!uZA37rHl}{Ux7@K^y~DU$qg9{OC2Nj=VSWb`XNzK*Dv$WY_cUu&h43ZM(j~%D zxldE8Dxfypaes^Qzt#E5$9&(=1hg*2Yo3*XNxpfmQuPgg1$tbGB8s{gb!4=b1e=vp zeW#tm&-r~%5}a>uc>Dc(6$>L{EY}wK8-fe< zRe`5{|EV=A*9JCV@2(-^0LBh{2OyDPG9{`Otk9IfeSeQARD0km9NI@<6+f<65c~#N z-eW|G;@$b#_^_yUA?-6?)MV#_xpol!}_P{8j zo>sCTVQm6uTFZ(k2)xVZvK2>^8paqd4|L*{yp!M?5XSaJ!8QIHzA|#5H$%MZpir{@aXx4(BNv@g%NY0JHQHzJ4;7v#37? z*A`=x$@RkSmI6ycBlZbeE_G8@D1^mJfdxk8VV<876lSKi@YNJ58i=PJLBm!MInyYz zR2A3@la<{nn6d(U$*d)h7CnA7$lo~SI+(nL;b$$lcpo~s9eZf6@-f9u|8v0x)n294 zPPc=*6V?SQdCytoH;8?-kBpuY)(-XydH$I}s(dS~Z~<)tVK*6T!mNP9j9d+2i}MAm zi91dpGTaWv?g_j~1z?Md;Q~m)F2AZ&s;6nLvfiqLDX^m@VI9`8=cs~Bj{9v`hfC=3 zbJ!wr%FVQ{KyquTK=KKzN7(9971S>wW?aSJKf?mwtNIMgcM_kOB=Kb8skMqeD#^6@ z%%J{CnUAaBxy;yprFWd??^yuB;+gGRiMM!Et0#snb}hi zmwpZ$_EoHx4b%YX@U_DEzf+k;G(8i^C@ex!!+ZgeYAYk35_}g#Uu`f0qvS`J6$Yoa%r4&$P%@xnv}nX?dv&m4-?5}KF&&Orj|=C{i=lxY=gq=ujZa&Up6UT(L6$( zzXj@Ih0p(eayH-bjt#(rN}gId_efLD_do6Ndcg*$9u3v_$yH{NH+BFI!27=2{zipM zSxODCo#-qFsOR$sADN~g3^I^pE>#8<}C(PlE6J zMtXlhR_RUF%`zf3U(=SWS@)bHuj6CL#J|*__%;k?@EC&0dk9>w*jeX#S0*r=syA@UokYo73&j*nhBiN6WBe4tonOAUGg$V z$=YK7UF7tbtJ|&tkBPkA%>LZ1{gwX5WI)DisjNc1>n$)>bh87R@we3DLw`~CI1$u~ zfe(F?I7>$1`^}_&-6$n|$FTZer}pi)$Rl1s4#79U>Yt^C?@#ebQacoEvS7ux;xT@V z%&XhpG;(75ktJ1R6jx9yc&YXw{anLh!)e_Bp6%C_lkv(e)V*fRGi6201aW;9E(hW8 zSP6DFmAR=V|LZ2Q_I30Esh^!nen1vEdhLO~cu%?>CCB3WP>*A#^BL*}9l<{wOO3!2 zuJf+h)Iy$3S+PzhSbOqLy*?_kNJb2Gf{((jdx0FXTU5_>sXn2G%8S|=hBmyk`%F!y zAUN}xU<1}^+rj5c8W}lWI`_{5EH^7wBk1Vl)d5{{eGu#vL2ks#+dORH4 zGpSOTJd{SIRH>4hHZ)+bu+Sz_Rl@1ZD#uG5tXDdyJ$+NX8BUVpR1(}s zeqp@xnrjoYFdc;cENaFrA9{xBthcD?rgdJUQbjH`BWrDywpv?@eTnNxV7qz{mh@!H zl}ICA|9OTvy2;?CK4mpnvHvTcspevdhu&c;;j2}uLn&VvJjCO9P-dg3*6{?00*eBZyZP|k-2j=w8 z>`!uZ`0l6=8sCgs7q@Q2yg0H+jDGce)|>)qRImJjtK5wZEq04*Vt!{|&Os&QmF zGVRTSuMM2&SN9+6z0~v4z%h5TGT%7MT0bK8q1wcTm}zFeCM__7?D`#K22`liBa$O$ zN5x0^BMZz!`rYb&bcPJ&W>t@7xHunhQ-Y8D7=8 zIP&x!l-+Q_@t@mVa z+n~mOPTy>8N~lNI5MqZD!Skw_$ zbc?7yo)4;UD><=6 zv&aOx(!0xEqS#_yI%47I8>tl!N2TcFPwNl(ytdAs|$xo>X&iT8rdj&V_wra#;;GTm}SahAHKlkewrZ?Pu^di9H=>WL@T#7>Wz zZfsGuQor*r@TnNp2ldI8GV64!!@Sk_YhA4>HK2!+r~od7z5YHTeas)V^-ht=?DdQ# zlg8%z1M%H5TXyg1&a)lAc6D!6hOu8{?;@F0m zE854rg#*_+4|ZPY-{gHnJ10`=U_KvrG-{@Ck?OYhm+lq5Ld6~R=lUGW(rBvqS|%E= zfVS#~-{&a2FHGSvGl2Q19;~oIjiWtFbBG^`G=?89LLS()C)0m2D-XUL)sOUBs9ISbdj`DLeO(rncz*(el$sejo0y*hl$)=q|9I?mgd`*YVQ5p02a~6?Q)|bprSzW%q?HlWnhJxy73>Z+yqp zeUCOzcyYw9^=hxSuk&v1-E%z)-CFGxOKaS|gd6c^qM{7#!N0jrxF&gXvE$kz=0s1A ze?InBgi2%uc4CLTC~lQU=x*D=J2~&t{I*NzX9YFydKK8d>b8ydvA3b zJ1%^cb$>>`d+2kw&fVi^9+*J&-t>;seHp>Uks}{kI^o8n`m~9YY7>5}KjFUHv+3^P zJ3G3UyKZU9BOBr;Cfed1QG@!M0q#zGyfxrZw?>SQX&e!ZJ0F!Fv0GagJP+H*NvGOd zOy*%NR{kf{hW^U6+&PCDqQ$-ns+NCUlcRbeFzBwZ`Md9S6n~Z6d95eYKGO3TS@bVD z+Xl{dP55g6-MrpHf0DT_sqEo-X%lB`oIEe_=X$EkclmE!zSY@T;asM1N6k!}KI(Q- zS?r34T*ce2?T))cUwT}M>BfuE9r3&4E?T#nf3Gd^FL7>h>~@Z)(#}}L@2Nq56~?PC zJ=dw;J{GB*N`Q2~3Q>x75; z6GT8K!>{aPe(P;V`?9)|@87yV(33mZ>{v+E@2TEZ&WVFZd)P-`&A;y+nr(O{A!^*x zM_Q*(pSmTv!Som3+yQq-`fY88X>fILzG+^3#Y5!}ZHd=L8nu7$E_XzOTrCfNM;mRv z7-NfD9KAkbm8L4_^gKg;j>A=Uf9%BZ-Wzkk1A^|zB%q`j`;)U zdL6y92a0Wz;Z%uq`JMiu3dedI(xJDY=QKI5MLp|=zNQVvc8uOWG2zjsNA(kK$0n=F z>~G#*f9KSlW8J;(w{`01wh=XnHSwKM_w_A-<<1s+geyJpxW;Bkj%tm4KDO4fSoaRB zQL{X=;DpKuB&q*I=3p+Kq&2?N?nhm}arJpN2cJb(ouDzQ^OR%#Iv07MHluyJy~)uD z-bqIt?q)~6-8(pOux(&Eb#nc^DSdgp17Huo?*lbFV*1$UCrx~G>D29GFGZCGiw7rk zTJE0cDC%$aEi?q9$B)<*KL;khcx{|Ni9FM%y>{hU{cTg5wL12d=qS^6bz)!-(tV`o zY2P+QnPw?issoBtR`lbp9_M-2RJ;1Q-lV>>u3|$le&!hG!)4QUPn|ioCNe2lZcDnK za>sEuzHh(xjIPSMd&I1S88NL9pQxYo?}k6m5XKYH2K!g z1{a{G{ZjX=X0yWSz2=%t1>$3FyWgt%9oW-(hX2+q3!ZYj$+(_4*lL^Og5iif^qs+0 z?=9+BNB5<5ujtC?9X}X7^eZZUf7es%xX^#L_foHZAi{maxHwrqdE=DXQyV6aOfEJ| zv+wWB|0@6P;m&-=pVem~`{Jt-8{!MhdsR<>LtO=5Abg^x*)daM_eIZ$S|3rOn&e&R z`jvZ)KUKL)b3wOIe_fNK_>1QiECn}stkuCK+V#eItg=_NX2q-S7TcMDda73b6$I=` z_18frWe1PBqwOhu1>M=*DZM#^7lvTTgPr0h?i$CHKF9q%_YZbW?w&jFu3}_77~w~P z(>fkqI?)zOHJQGQjZxZL*rqxd zaUtrO<-9?qKc`!#$yUClSf+_GOgC;ajMwi|mjr(9xd7r{>wMFrQO?t;jkyu|5$E)K zRbA}vnXsuXcTM%0sg$-%rBOCf&;AQ`lHT(}?<6u%^uGJ<+M!M0Xgj-4bkFZS++Ss@ z9{M?4MYZWJptvW$+uwK2Y+Zh+VmpKLa&&XQzJ_QC*mZkh#%8i%=@sp5)7%z4?U-xmZ9ezj-eI4 z8dapeS3jgbrT;Bx^|TNt<#@6y41o+hxFvY4a$IGL9pJ_x{O_IgX6Sgk|w6=CoY}jOnoI`mwJLN zx@SVqmA-jSvnpt8GACQoO_93lK$I_6v60=g)f8*(h+Z0bO;=A1>JF-{*DFrx{H9sf z<<{G#?`Zzwi*(gF_Svrtt?+-Q(HqwrCmPP^o+3o@5!KpupcVW4!aG|Rq$U@Bi*JJ5 zfw2$vq7PtmyGI`ngW&BOd~LAEmJe1iWvB-Jp3B%%UToMi{nz^21{=VWcfpsrHKt$b*Q%O*CtSaF?eO}@$;hAz{Q`9(IJFi$S8J8q z)uX{JrGRmK8pfe9Xp^tu2i^c%+df`9>6r_Liad}&iGdaVLf=ooLsvR-?V0vn_8faO zOa=4p>tQR{?btG8ah`C_bp003iP#1eSnVgBm*9eM4^^rYMr<6nZ~QBx)rppSblkLp%_hqY5?c`<62`DOi8RVMiV27k8lKeT?x9myy89fH?TxlZTe@BAXmB(-${bf5)kCXX zKlKQ^&rdvK+`lF71#GUZ(O(+%;^^5aOGb@}zhe2C<{zF*LlJIGz^&e7Fhy+9ZVRmO z)&>0PC_`Gre6!JVG~!QMcwDH7n4|bWTOCmWW9Kh)esvXNEMC=p>Q|^wD_+*8@qfMH z%?{+M{!{xnt4*V-1G^!W%#J9Msiy2B>uEC{=S=*CgrKf5hcd zI-|>zXC+UJFSo2Wl&LjD18%Ap>R#22)Agt~2L@fgbl(Xs*0;hhmPeK0$2B)$teK`7 zP|wuQk7zZ1P1{6drxh>r?=`>HKBZlam+rG*tA8x0zHPxzRiCM!#d9oK+n)wuG*_k2 zr+S!oo(pa!=by}Muw@3k?UH$a9AC+5;v89E_%Z@-kz@BGJaq4}*S3OT6bbiB_&erM z$=&Zf?Ys=0GTyPz@eF&cbEs!%p7VlpDgFC73{*><3!O3WHtZXE+PBB(jGy_?()e?c z#l|JN^ZyS~Zv$6#eg6NSb3UJQ%i&H?Kp`O|A!TaHl#Ce_Ic8+!$jr!5p+l!mOP!jS zlsGMA;?fCAmSkisnNhJ~jv18}78W*a(!@cbp>p7y1LywypBMhVkAFNgz{BD5zQ5kr z^}2z>un!HA1%|(?Pv|NF3NA3GHOmdF(6rL)*XeT5Bm2AlH~5AvG%P}M?_nx&<*4+n zF)Rh8e^?_9T@`42XchF;$<$_}$-WK^f_QD5;E1)L|4=QKiin1Mek?ACWap{YT-3CH zugqcJOb)(5R?tdCtBcJ0UB3UJuNOq&Fa)G(I9S!AC?$GaH(c+zo=4YjIjd?Xd5ADF z65?S$3jLBlqxAO)N+JIR*;`CL^%VKlv%!q8>X_h|7sBPXIM!Hf$_Qz&_L|qg@5#o6 zUG=vH%9S4sZvAK_nk=&`cvT@zBClG9CY(w|8Q5yZka33pc9g$g<9<7cKWYl2saYc5=zD)lj?iMpmn`>r}x--XK^TAhKt+B@|N zjDOXqQybCf&Zx_o+d9Ko{okpIwNjm1rggyesv#TSLLCDACKMzZ$oq?1Y}QDtQsF|n zV45DJ?skTXSOgrle5wi=RFlx=pkC6+>9rS@XFQy_f5Ic|;CcH&S^g8m?Klw6dtg%R zL&fuPlt*rmQTc+={SJ)scv?OO<&me!h=()!6L{rJPL?Co5BgBz-QztQoMm1L?s}H_ zDg9hSpE)P=tFZFW_2xw|`P}GShf+=Z7dhunp178iI#*kRZuAq%CB~sny9^)nc{tj? zgf~#5E)qo}7jv);oG3^GbSO+{k%9b0`z(j&Por(Ma5tq2Dh}`6Bfb?4;0)u*muAxT z-C)P-sk@0gQ8ft3X4sr>F|yNqJ2)ZNf-&`i!+wAf=5z#m+lSNKEb=82Jd;8Fe(7F_ zTBQg7(3i;*2nPCb)RBJ)5<2J!N4aDetn5dqG@mv*Lsy0T-f)XPgcAcr%&#~L=+W@eVP5l}35RluG+NybH=R_ zrSrp_N)OO#(FmS`=IC1f;CYJH-iP$a=lt2^nZufnLQilGvz**ERjzj!bF|s1*3Z`b zLs@Ci8NV=mKy{+s)M!btCR^Jr@0mkTEt+Ne(m2()#_&^pvYMpifEp;I#+Aj0##3$H ztjPj-9Kr5e&x)On>V2|u9u2*{ppO@$UbhVNaWS(a2|PjuSm$W8Eva_Uh8*zL9k6}V z!KsLg+H4TRI1TetqD7a<>ARVhU!Zq0*#&Cotrqloa+&R-1+KD2FZx>1;dzIO=yvvX zQ7*j?m7#c_0=oQDT&|9T=|?*p#>q64!ye$X!{`saPt+U3Ph9k6qoMN_xNA8<_A+13 z@$7%2N^_5Xtl>Sw5Nfgujn5nBp;7j0KIZcOvBuA+Fa8V6{}RyePT12As~@BFXaoOZ zqRp99UByX8g?k{LA&-Cu(7>f;?IkhsWbf+l2B@A^J-b$p*iVrqW{3Af@HqIDCE1=-zC7 zF>Ak&$aMm{|7G{fNJjTqQnwpi%ZKD{1{o)_sU+q3NjUl4%y4 zk32ZX{if}vV$)>Pe~jaeOX-0{`g7_svQmEob^ioa(n_%I!szb=k01_SxY*O?b+l2Q zHXhFdAH4O`{IQWS7LW#qEMYM2M@ zVB*DfI+piqr(%>(`-goUum^TYQonj^2i__)p zC|&*?#>dB~Ac2!&o@c-%&m&&m&1`K$YhdrlI2G`uStG5t(e7TD6)Ak3O|Pt@?afp~ z6KI+Est7hskT^N)bb_7Fqy6$RjnlIS%vvpdAY6DCJs|U`gj;I_QQKSWAfkO6WG(C= zwz-=VFofOkOFlklwjUY!ILvq-VNSeH9PlO4fZaRM_wy0_L@aa2Mz$J1P)z~Ik{;Bj z_38z5%vZt!dd~QX@!#l{7Ma$X#+inUCylQf=NpGO{cfeod=-7JG-km4@Ge&Cde{@Y zVFP6IybU9BRvgKC7-5F?Knk4ocD;oq2*D%dY=?2WzbUGcC^4l=hBP`D6{ zs0w-@kMU20@1a4txR9ihUr&^lvKtzcE^j z^9_F+>4P;ev{on=P^hecfn}oyRJ6yJF=sn@&+3u(7cyQs?1Hi%M9>Q=em;qu<>Cah zf>|QEz_KEUZkY2|KzGKo3(Oo@2{Snfv-n71MO>l%H}zKp8`r~%5bwGOq9-{aOw62>wSDu!Fr|yy*wyDdTUAi-?7O&t7;td%-hmB=Mc>g%TCGP!-MP3=j|A zX!ifg5rwdub3ctX3(sddA6ZnasMgc&&JpZb4r9NcJzp^HE^tkfKbXl!B5f8tuDAh- zhnJiznY7%+Pl^LqHv1u2E5NW^ncZwMCNM-<6eUH ziTIPsK3EGDRWQBcWh&0xqSAYlIU}ep(M)O{nfFif?$Mlwb^LSz{b-`~xVLZ`M$>*- z|HXpvAsPtbMEr<$?jlB7ON+PiM?6Tqto*~&FH4#6O`tvFL7|-{a$5n%?G|`;&l3-w zq}3moy3uQ%Wxf|~>1*V@zhvDnKp#iTt>;M)vnn{P-K@z{PP_tUbsJ~W?2#GR&YAQf z&(h85B>JVI37ko$L9~rKM%tgv$Zcj_G%$+o^g%Q+UKS%;!WfHp+!Z2XAH5;FVIut@ zoFX)G=p*s;Ek%h(`kF2Z{EP{pXl!c%ke(z`3?U5l;1sto@t`L>o{w{Q4*o3 zM6aF9@UtKT_p{GEsh;Bw_aF+3pYZW5%)+bi9A6;H8LfYlbIu8t=^jRSHDlaD{1>Zj zU73M6#hifuY`}AY_=Xwgf`Y|E$4UxPjrBlA^wb6 zygGU$su7fC3h26bIL{s;nv?xocCL>`PObIyX$pIFI(>GNqiF?ml?-z7XUaLMng0L< z^c1|vtNQDD0|=$R(S}=zRaSyIdXhM;j1ep2`NdP(MlbK?H0j|^b(%e@euVwj!Dph{ zXX93?azhb+MbYhBK^{RIT0rlZ@pf%TyQv%qnNFi*st1YYsv^Wwqk^hn#_Ce#p5aq z)~o1ZzN@ZelosHA^bM+?zGK!1YhbFSKT_esr0!pD6boP}0w<0T8k+Vl4lXh00MCTW2qwKsz;5+-lny8Gy zW}=W*kj}|4d#l-jZejd>YWTu1pR@Nr#33`-g^a`@AJF2Huweg041Ndut{#W69_|`0 zG6b32m6kDL39OC!k$qQoN0qZt_MmpZgAXfnG>aLU!mP_@{jB44(iasY^HMY&EBV z*-WdNcn?wU6iv(nd{pxtnJ1$EDZbkoe28{=1E;KpdD#N4csFC7t}Q_Ayo8(NW;onF zGGR}`X09Tm(lw%uS`PQS3)XiHt#5+=D@@~PGPqaBKg4qLn9gl^Ez#!&V#R9as*jOr zVua$R0QkZC`%lVqh8GM#@gKK9_ zJ;A!p5UP$k_5MZ;+>I2{iM6@l_Q) z%Nq6w`N#zwt>N6kFBPsi4%6I`^H2~j<}`s<0H%63>(4>cxCo9rm7HK1J=((WDk7hNODoUROg^%c=V=cJx78PrGp2=A zApXti%z%twbD*3*hjD`;AHf~1itJ1@%5LePfUAkoeSx$5OLU+F{Qg={wLbEET`)iz z+38p@oGYD-b34zN%{h~W&M`b;Z8UvTL!RWEC<}9+FF@t3z`s}fm{RGVf~M1XMoU9p zGX;G*aXv1Dr2_9hP=U`@g})RewvCvqf_8|L>|VyWCI}i9ZbqCgTQl3=%A7%C8#FU6 zZ^0`lzr$YUM%Dl}`(dRCwXP&;32pucDjsW5#FMHMig46cyHPejtohor4K;(0JkvC= z$Nj4V({NM7u17vu8z=wK#gx{g1X5hO- z#r6og?#pzzy$0}S1nZ9KpC(C{7l)-GOzS zV{QIqs%7XgYm$}40h4xxQsJwClSBls8LgKyeUWiRAn1)#n=}r;3sevH2Tis1Ew70i z*j8T^yGj#3jgu+0#nC9}-*A;_{;sU|*3oa%(9C~K_bD}^dsNw-YN+I#a@Xp*luumi zP*$4dsSoxjDc-&0p0|6`QGgl72kl*udJlNxluX^T?z-R_wUyKA8CX0;_~EFqe2VE4 z!Sy$B(jE5Qp`QpsU8`P0sc;oq`#Ibz+hHNq=(+;K)_tCzDQ@LOcZBX?^*)~?Sf<;}XmoL-=S#~Ca>+TuB$uFD~2ttEmh z)3)$*4cey+W8DWdE7f*?4YiPXdf2Kts8iuEJgn>)b#DDP$ac(p`3@a#~Cv|W5>);&p1VWV+oYYU~nu7iG#b!;=yC2?Dk*^2u z)aP9}WSRGYze!dzw4ZxwmB;v9(=?GvCb{Je?2zP(wBqwD9IPaz6Awz1BEUbxpCGfa znLAd2kEnqvUnYF4ew6QT*EiyL{R?nyGri08`Q(Xely-jv9JmD5!dlSa5&neWX!UMS zwQ`sn$_D!52+#AeavHo!7mS}GpAVdO28hxcrA-@f%{31DY=ITz`73>HCC{J8`g@Gn zqDZqj;9}+G`u}Y3diDpqls!Ify5z?<^Nc0bmQzvB(ZH)%hl<)_Joc{Xwu6um4oW65 zaEl#fwoRce)nfZh8m-u51W~ zc)!p$avpUC_4-}jgrLQ+$y@1nstx|_-i5}yJ!KlF@{zYE_`SNF-MaG+qVt$g z<^J4rP?fvJVr{fkC&@P#_;a;-Wv;u6lQ0^$&thL1Yx5CYIxWVi0^NI5aYU;WwR!lClstiBQU#N|oFv}Q+6#rRW zDM$xB%BXS76%CBJM?H7wR|Hlus@$e{%@6LM7^Z1{?cQU0z-I-4zZiy439-Qu-)cqT zpEJ5$p4Uw8xi#8v4bQrg$yR>qeT2Q^uur9eygIO2JDbxBPbJRemSAB3|8`cz!%7Oe z91k)#GWo5gysm(igSQ<$i&GJ~dc2giuXs}xRr|K*R((6KUajD|6L?rj%p89C4s@>NUFq{D5)|FCje^Nx3+9zGL1(rjLVPAsP#84y-&sFEDa^dD4@ z``)64x5L*6Cvycjhc0-Yn>9T=c?(R&_P~=m)aOyK<5S>hUS`Ru;c2e&o4TVpLHW>4de{Ol-?z zu)5{%3%r+&eib$4Y@T13+vjNKSw?jt@7csx5*0`D&gb~;qTVm{`sk6LD&Tcf_{_>X zckwR~{B$CHkw`0|$q$_;+7Vu*{FLY#hysaZPGWhFXkK%HCl_vc3NuK)C;4T`pHAW5 z{!a~BG)hFLK)Ckue6V`?ZB|+-3NE4neU6`!ER5(Eh!%-x5G3*wH+56^fkcV(k3djd{5M9WG*z1ysNAyd1A?D%kx+n zy_tMRdPDLiGG_n(bIEH(nfs;|&o#zb#!q@y-s7eY!dZSpC+{S$m6;*mPvrfj^`g)% zBP{QEGk+oTKz?3&Kvt8Ctc>za1&W(}B6()%LFol)<4wf_nb9|YhrDAa|0=5B(pq_~ zjH-;gPPiHBAqf6H^pzsgg~d&&F}l_=3XxT(M)eJ9U&Q+Y$$C4F&I zX+3wFMTKbko0IY&o7EovhQWd@1QlZgXZvzGGlGLQl9bxf5bUX6sm5H zrR3V?F*oBxkXR8Y05p;vK27C-8yCM} zB5m}MOy+njuc+nSq~b0$Gf`@@ak@p1^l&~o7-{#-y>KF$9C>g42a8uJ!KN(9k{aeF} zU;!_Kn%RlAVHG%nXx%mHPrK1DkP&L)c}jS`-Ka=xX4XmX=JThAo~$FQg-a#a)ES<6 z?j^W(rr~OwfQ#8SoI4YS67ag(ZchM#d3xXzTeCAi;MTWUQbWHDn;SMOd`oyignq?vGnArZ9ouk{M`XlE? zv_~!&vv5p#m|MSs2)@yA6fEQmk>yp>clu6H zd9C=P#ryVo4i4wqU+K@gcJyl2mHeyg`;J{JxW4O#{zl>T?CW{gci4u#)uG*En{R2p zW&ik)TPDP8iCBbtm_Kw;#8+Vp49|KpQ70ehyA1nmpRLgG+3@Go+71o2IA?g!!eGVKY0oMTs`rCa zm~6=isRgGq-gJvTL)(o<_dfUhVLYv!dG_4?%h$|(+8?z)=3T$nHh-XXpm=aUdi`4* z3Y>ug|2Htq7LD5<*E@DgOjSg9$R*Rg#-k>u^(CuI_ZQbjXO=zd#)&?~7V2mm`la_g zIsXvflfmnHuVrQUmeCEdlVXgcmqulWj)i6Oi29(?6j<#05A384P}<3k{f@U?8-gE` z%lWDLcj{?>(|>I|W?pOFWm;_B4(r#3c8JmU8LaEeLmAGN!B+<^*^IUw*R!vMUf1`x z4cP5*j-Y+IBhz^uZRbh0NC)`osGaeP61I$88oMy+1?zLhB`EcdH|wqSmW!JG&Kb7c z{%@}@zOicXWmk&t3Gy7N+!|xGt1OcvtD|FLqT*V{v_)oFe{aaw7NIYaN?zr86jxiF zuMBn!nupQ<4b=^6DQ(3FgyzguCZ&NRYG)Jx+vn6(3?EEoPN39Jxhqi?S zwUr#Ov;X##qeWb0&v5?Mg|DJ^r}_gprPo2zj-^_;2b9hnd}UV}TF7gpcyPMCz}{`9%=YM@@0oeoKV*lcDVcr~0Sgc&)$D{)4O6mxjWGO1>>k zH{P-hCExBbN=z_%>8Smo&lr%;18s2{9e9<+0<()J0^3ZnU&+ z216aKjs)g;ChqbV9ouauZJRuO<{42dqnD0u8dDzq35?gHrbDKFa|XWT$?63B3bLHT z%p~*Bi*CD@>M0quM?Lw0Hw_1^bHe*0H{!&8I|ClXfa1%Smu)WF==Pb0Z?>}|p2lp$+oe?WXwM0&h{35J8Y;<^fsL@hry%gCP zVKG(*`dJ;3!$*e~y8r9x@b2|r@*ab^7w&1)JY^^jnHRPo{F!hg+5uUX7*m`1B~l@0c_HgVrkY=eqq1Lz$1jYO4032$K{e3l{Vn*|kG#X)Pkj@q2=1k3**$_5 zNTps?Nz8l>y|F~p5T5lsgjV+RuK92z8*rdZb6$0>L8ar=(9+?Z!Q7ONIc8RHz4V&iGfoCGbHaSu5@nfZ`W77Zd1?shsIj~S%4)i4FYBXy*s3?|Dl}#nGJWGU_OZNI^Or65) zd4q~sJHKHsk#8>XS_f6Y-9&6X)c3COc{>$J!CU#L^qHtw;9f-4x&%z=rjZJKAGPU! zQiq=nhHNVs$xC=riz@L3eqtT*rRWixsJAs!d3uZ*r563+WniCnj(j|$Y~la=`OXvk zc^I7gL3DxJh>{DaVx&{2JWWiuo80v^s_j&Rc*f}?D!jG$b6crTmV)f8K%rQ4V|uAp z3Pvn{L=kvzpp&|$;A*55twdMXXjKdUCsRv}u7nD-V982(rvhqv4OC;R`I(MD0=+Ev zgo*UCXc*^$Rn61ZfC7m|5hWLldKIctcT!!wHsX0Le&*+>!e1btUBYPeP@7F81MZ?$ zTgq)Bk$7CgjcFZGv3M_%MWc$4Njq4he6|j3is0Fz`Hr|C#z~Ekn~dZvrqTAb;JI;F z=KtBelibGS?s9?Ki_}P>8I?|cs*N^wQe753Vh8Ufu8UUsOEBV9yyFVu`PIBy@}3j5 zwakpgRP4^Pw`sTyl=Eb@d`H}~^LXaBz|?QxGf_6%%RZCOGn@ts<6=cL@|{xpMzWHE zwGkcNn|E<5-^==cuNF_j4m1T8YiG;-hZ;4?%$^UenfKbD%J;8PBA z(-AfEC%}N+sD7{Kd3Y&5XJ!82-UI`u5IkEY8Z_Kc$Ujb}iukd*1aIa= zh8e~`7|i-daL@S@Xc%1Sl?&Rpn1w4?Z)Olr@2XWG7(wcz8gv!}LZ>nsY;S_n!#&r* z4Uk+tdD}*i`r?mjf@$#(SnFc&Q}TJduNTDacpRB^aHIZ**X}WJ_eb5M*ze+;Nse&) z)&aXM(rF63uHR}hT5>|Sg>{Dn!4i)(egofWljRljpy36*UVlwjOpZ2{Or}FyMwb2> zcg0R}lLxp%j8-n|Uj?7P7I%R!z<%7Je~>#<5>NdL#j1;jC6h>PW;1I!gInS%?!8Zr zpckH?4=-?g?___==C-7u_Ol1Aq4Dm6s0%HH6`AU+b?k%(KEQcji`L6FM>7%O_Q6HA zeFK|3=sH-pg^n~#3Um<&+HzeX z@lT2F8Jt=&Egc~(Ax##={5<^HE*(4w=6F8))n*jprhs;e1-V)emuoYqyVdB)Ww0+w z4z@IK#NW>T6GV?{9uY~^P(Hdgb&iXUT1Tj})~SU__rlOT7?0&cFF5^>&lLmNxF!J=|%tsMuc7)q#GOipFkq zj9NipAND=~=JYq7-=pfV4Ni8tquoB*(eC($-gOV{$N54&Qk@DX;aWRdMuXFQr_JM{ zUW!>dt}kKo*w)Bn<`P}I_eb3Bgu@hpdRzqRLBGTaxs00W7X53cVM}vp9c-KzjrZu6 zq0N5>-sEqhivKQORg>920)uCdw;Nt8s32zf8Rq#F@)V2Nv#SI}rCUe5eT_LioB1#Y zHpmWGG4G>?f66`9RSP3C0mYwOKDr%gL(7M&;MpgmTAknfv?Kj29i zXutN%Mq#iKj^lgoox{_etL&!->jwAO2k>x77|L-*qGr%PFx8fMJ?mP5eZR6adP2ga z#BCFTx6THT|&Rt{gwxNyUo<^QR;59;qx-l7h%nX?96Np{_Ar_r=)O1)E1r9bY?P zYI5zw;e=Tcxw@}hYX?5KezpIlp?2?dYHTKWe#iX}uyzx@)d1O5X=coDYS(&=W#=V-$HchREar)E$qmrIcH3)P83tAF1CAN zjKP^Q!V*g1^xqqZ@XzpU!KLk>eUp8?Lra#W-Z|ZIY#<#?s!y(e^<&aiW#F>fH#V3& zcT&fs+S{t)hb_h4rooIGnb%JA&vi9{99WE6=L&5pDsU46%hX!S zEYe>I#K6|Ue}bqYl(V>8m(R>l;PXh8_qu+(x!Vi^*DxCPYX+)W@nF|XcD5>I{Y%=E-ox|D{OO*9iWi-481U)Eelh zKr0%GC*W7Y$MO#2L3xh-K0Q!?Lg4T4bq^h?agIj)Yxz*MfkWkQh64Kz&JH(vC*NP& z)R%rO^u`C*k6kNw7KSt?ZJe?*wL0nEajg+AssHhO>nvaw{9rg9{@v1{R|m7mUaSkm zsU7CY;gd&IMR`YkA4&t2H#z&};Co~vZ*iaR16LJ#a`WBigQ0ql@dHkqmyK)m7hp#= zXia2BT|`Z-JRy9rfZ*^Oc=k5VM&U9PlVj)!$Zq`+8I6zFtDkcvIXecH*uL%G*Pqm1 zh#uWO+oZu)24|s7_e)O;kj3$7my*3>Ux<9q6stLkuHe&TF=0n>>UjtDkehkbQ?1)% z{wB;Gxh-;Pc$#&O=`+KN#!Vq#gf?3CD_?pS4wti+&l~>0y~kTDRk4?>~jaw z`j6fy?k~1E(ZRZWV=FVnKDa`AZ%p{4wv^^uLPqVg7!2ptHEIufzu{hw`?KM-gZU_| zRS#J;ql~E`-IfV>kG*Ngu!cuo;zoUMcv8rn`rP0&?_}7#C(w+%gwIhk>d}epd)M_a zHq7DXQO0fR1y*1kYod*~?+9oIFK$$idiVGi)3@0mI4X$h64{$!R53@Ud1E|thZoxq z+1&l>ZExGg4a_1(w`pM9z+~Iy8*TOr%9hw|$qOcSjNKM_+VUn@v&-Pevvjl3C|Wvn zbYS$2skW2D6`ITXxyH}*e!Mito4*JTiGC^inaCxfGfZ0)ctYHB+Q<`M;;bJS8s!Rl zyEQx26NcZyELm%wqICOSvsrp4ATLN{4L z6?RFqRhqenKkDA#pU7{S4HnUdD%ndR!x4>UHP$&iyDLI33OQT)W?C3$%q?jEai>DttO>6UCNx>nCBUp(oi< z&jvU6U-3Nc4tipV@^e5*g4_V5^16PNrPJ#G0abV+!=^>O1F9f%HM zfgJ8$tGMsBf|`aSg-_g91|xaP6UJHivaYpc^2B!hL;a{R*ic+>5NGFr^v{HolSSU3 znR?e|Zmnl|mtGW5E`sRU%>9$O!>-px%Pvwc`lV+K8R;#3X=SI^k*!sCO7->zXzMUaii zCr{Et?Jb)+XEf(eF1Jt>CU!KZ-q&#EpF<_$cl31X&?R!5&5mlPo>|g3^xaSb3aT%n zYI_ouf#0H+am-cZjn#f(m}@>kcItIl*6ks2Vc&;lheVmafNg}Lto}(5?)C7AFLHaL5J?i@3EurN6HJM8AdX_e%XvR9B|M78b>B6^)S=cma38(ktLhu7ojw zN(KnJB9QA2m&@F8Tb=wDj_rEaPe}|9Uv#wdLF)oX1fE(aSv{g=`kn$zkho8go z-{e{Ao#q4O=uPxBp**n$PQjh(?fMVcTfZ=kzO@VO~6qNg>EKG$$IMYAiTUcoJC2T^$hubeot zS}M4Mbb_a>BzCl;6)zaQGO%(hxL4)Civs7T1Q?4l(1(?%%in{3mWy0L8;Ddr*)xmb zTYZwcMRBS_hSv@MQe1FUGC=W+L!_zdGc#{TDVMZzlVCA$i=rBR+JZ27e9)V-0S2PqJ%n`bY=JMiqkfja0}F2f9(2_|VsZ zI!Ah-7abThHhd;B?wkSvc+;$iU$X{|p!VL#&AJ$t@k~BLtpUaFRm3=N_=P>!$S$1A zDK5?#K0$kfWyvHX53UvrNT>E6@JHA3yK93!-A@hQ=`(cGnQt{HudkpjFTworG4Iak zlGMv^6{&FWv4#28!wDyT_cnUI4bE{Ey>yMt=^I2bWtv9xoyvo~!ifjNd=ZXN3G-_r z8LBDtekXU$VxsfA@s*$LTg@4*a=*CGd!J_lFnP6!7Sdx4u!wkY6eAB1Vy!MnAPFE$a?tpXbgkCu4m=bQn)fUt@Z zI5ARuURG@(CmY&t;)g&6HW9XGBW;6S>kH>b+wV&x&LR^Xh-I#IFcu(QKs;w_@rod3 zzDBR4DoMDnh^D|y7$;6OUCGt7fhle!<64Do)CQ2_UFiIuCuep>eb0E*_%WK0S)8q@ z+A8&R{Q&c%7JSbWx;S#NcrTJgiU6OoSX(Dm4>T{o^j?EgR6y?qM|PtER(B@4K34oC zWOuZ|x^YmeFrke008RmofxCUb#xwRm?kQ-KL~!fPU{9@pP1nHtSjpGRZ#YYaNpNcM zL|Rt#|4VV+RAB_U$WVhj0hyhE!qO;17teG7U!Q8!3NFCLNP;Q4n|3)+4TuhQY0q*( zyrfo9e5c`0ts)2dKlj(36wt-J%xij5tMa$-~^@VHWQKIEM(ee(HT*S zAf|spH_9;95Us;$lN(VT>Z~6t35<0exnJS7S?M<`oQ&h%Tz@XRc{8=WVN_G{$<;Jd zGpJxr{m&DNwxDRQTwyhId%N-c$q9yl)cBJt+=ovD-V$}ZGLxBDQr@%i}PHQu{ zFBOoFz8|-X#njkp(YO7niV717fY)%~PSsu0bP;X5!S0^O=v4;dnWN>pmf&0L`1$Zi z%iybg&HCNpo6he{W2Lcv(LyNTJf>0xBQo6g9QnL%Dy6a}^XTX3z+2unAIL`d0-d-D zq07Opn#xQOq?kk7#Qsu3yZ~NWTDZyqlrUq)o)Y^xpQYwzY+%Ih+h@i8Q26=_?oHES@|lO`;qWu_i!G7 z!ROhE!PNW*g{FAlaqhcsz+Ioh+R5R@P)Kx=#<_Q#Xhl@Iy76eL;RG*617dFgM6dQ? zcA^3H&MwXXcx>94pu(ed=ZTS$)izB#HHX#sM|Xhb5}nr?-N(wm)QyT6Zy6I=eKWO{ z4b16AcBq-`cg?8JHE>2$P+!}?vt3|ZYlt|e;VWItQ)yU>)XZQFR1iVz8eud~WHb!! zdECK1arJrP8Iwx?JKQP9Q+19GNPVq>I#v-eNF}Z&a4_ka)Zh`Kz5}e325Onr+9#-4 zeM3~IRUg6c;txcyU%_}Di#GdSPTyrj7GLTAfF_5NO4oaa`EZ24GkmAsg9FCTQ2K2q zYb|$7aZsHOI?%`I9|88Sj4B_IlCR(6A&>qw9vnqvLvV^FlDm(;JHYPsdjID+=`s&5 zAN~%t_q*BaquEy#=F!oZ%Kjz>sc<#8-N&WsLDoG6A>3y1oT(l^0aK z-p!f2k^0~wL%z9??9(yhBWf)*`B37FF0#{k%)z!mIdPVT6S9%WZa*Geg+u~g)ZtTn z^WER0_V^pm1Keq%*f~dmpz@>jy%ZgC{Lk^Ge%Ad1N_&GuJr8gipF-<$x$6*d_IT>> zc`#_6QRkQ@n2qLr=3;Y{Wq-(}kPOQr(;~w{eKkCHJ?B@KBK4o;R0YTK$3k84St>yf zs*z|GlyeF>h}bT1(<*?E3SS7uw1e8QgS+4x+*mF#=6e|XJ@nCUsRB&`?QjE)#~C=0 zkWU((#hv?Ylnqe`cmEfqKo8z6w{V+H zni?Y9IkIxo*x8EwxS{z!2Y2!;x#tLSNWb)cnvQVJ?l3hMsH`dS&Y9r5l zm3ziQ{SC6#hOKSI!)b#)CNxT;I>s70NO#N`K^Adf819sK3K} zCt|z{C9-1g9uOnThqImg?U~@rvYjuW)tZdjY2DDlp>Kxb_$&>qoRO^C_d}(+Z^9SF zw#Mv?JYoKXl`_e=#PnxFrxLGC1f_D!)92G@-!zc-vesE97$3&(`lp6NCdEA7s8xMn ziyNqT_u$F%I``ZQ=&m_X^N3TN;`M~5eHJ%~XzJqG>;VZpV?VdhFLB}?$KPY!)x*o3 zefD&+Zd&KUp(SJlt`3>;efie;6wWXE$SUs|eBXD~`ciDexH+S%tn-Yw>jw=BOp3{@ zU#7j}Kjc}?+-OHjd%HO;w8$D_{t0(gvwpG(9I1J^p&kw-ks%T1UhWv*!Q#)>RkF%A zaK2qYy`U79QUjO`Q38@Za9J<~uHxgKuigLlyhZ*k#`Ds!*|`IK*yX6|f6of99zHeH z#IqGSs&S?{J@}dJ#K2U~u(>>DVM0x8Z^S0^7kan;26yAXsYP(X-uIkv9UXq(Q>d9| ztP2UX9yR~OfLD_Kq$xe5!4hUX3wIKIHSQJ~^f#T{oa=~ua0Ao*6oo@GzDABj>e9wYhoj-iHFIl6)ddk85c)5>!ii1)I{KG=( zZHruKLraEc4gc9S$^EVCcGuFO^^V@bLxbV=^;ArgscVqi9+>P|8xkD5EonG@PEZWeHFDAUW@uoPcUzKM~zxau6`txPOo11nuAinXPH{?{jCMC#%I1 zX%1?Y%Uv%yL+ti}eS_`xY-*)x_F)jG3vB;(=Y=dBH)}%cEiXmqhuo?+g9!eM;bRo# z5Bma~S)oH4hmZUBtB1&?7F(V(BvAi+-Oy;Uha5BiQD3S>kB$n#5jcDaoJid;sHX=S zaAf!k_>e9zkSaHw#pL}S22C$cioN8H4esB$FM4yBCk_6u(b?ZP{LS$1UGt<4?f#K| z+3(D8#MpD}ePHrZ@eFDm>>nKEE-)+cQ*W!iH7Zt%KTs~w&_*&e)!J^~Z-@+9K?Usa zKCFu|SB1Q4`OIL`JwgMp?e1V6cYU%d%*%51ye?NG z@q{1~Hj|4=BMOc67n4{22#x*P01767BHuChQ^N~}zj93`PXC-8$v^xF`Mz2z`=dc< z4j>0IJZPj(Z@81g79{qjOqyUH-D|GG^XvD<9p?E)n=Ui3&zFq z|9IAWanl6jR1Z$c=!^00Lo0Z;ZwkA@-M%l~i-u1-(}uns{>HTcOm!x>4rRbTu%1f! zIQvT1B-4cWV9L(hLdI?jZ#Nw^7J;ChXK-lXU_09fPYqnOKkd%bh8Y#I*|(ENb?O@| zRbg@A2_b*bujUpnjPp!lSUed6m*|~v?ulLCB+e_Sh7iXUlO=IcS59PChz)#5&gJL6 zz1*#y;HyL`FJF^2I_?vl-2A{4Dkt9%ft(;udcr-%~GnabjS=xh(L!F=&}!nQst( zxWB2dn5(UuP_}ie9n8!lC?_!t%i3U zL4LM95Y4lrH-m5ME;6CT+~g}jP*#8piy{(4gNZenM0Nq>UErd3j^|Y}>O*9HUv+;+ z)Vh?u|C4JhdF$W1EmUXyL=2x%2MoCH<(8vW>tfcVY)svfFe&02LyvMs{Z_vmx4zAA z-u=UUgGX&S4rY+yB8af@#%Dl~tx;b!rCF1#tIP}a)3r_9-QVz|E`YX8J!i$2#G{)) zpo1nxX|jp)B8U9uI!@{fxFIeBqp;ogHqrk(>@3}6?mDN%+Ejb4-7meMx< z;HW6`HtqoPxFwvSCY=;~)|23H_J`a5iDOi;Wi4zKlma=0A2;R_RUR{L(VKKOcHv>F z;(NKbM1!)P%$@s^uQ6B)CQe>`5ESEM?AKChv@-k2#ek~_a$6}TJGzNBr+5p|WJKAG ztVI*&k7xo^_&#NB&1DC_I^=gwA6hj0yz75dGj|akJBaV{snZ>1j~4{RGO};`(SC7g zy`#2GTyy)PBqeHx>0|PuT2x!l>CS*riSf?IHz#}WOZU6VtCoWyYfaVY#k2%RD`Dtf zUe}+8A-~uc&eK%-GT>tuf#*8PhzH!60W|WcQdx1(Jau&hhyClwhk`DK zi{HV0DU%xt=qh5{XmSwk>;rcgnxgwB45y46e=PiuD2bmYPc4dH7d4N2uMdSfl))s& zV%*+VSuUEL`iZF5IQ;e6f1!nTk79*k|4)yOdAr!x%=+AoTZ^2AL+;ao4DR+BDCD@v ze^hXCZs1;>Os%9VX!pKH4eW02=jg6^cQ87K$%b{4L5k)CEN50$fJvLi8E}#M&f}b0 zkFs7SgOS@-w}lbt_ObDkb7I z`ICmyVci^bJQh4@*k`tzZr69CYJl4U(efSK&{|O{@wi`h#dz%akWFJvJnH=sUs?yu zoPRKGojfa}Mz#C^d8Z~a{itq{Ke*`q2y}D_dS27m53DGuJxo1eFIcx0PBOf=(ObC- z#D1P%r5^D(wbD&!yT6Yg&Cl6ST;%mdlXw}~${bh#Iou$Ua0g#Y40DMp9XdS3JLm#3 zf``e8cWP_Gc8+UEXp5N~e#m&9H8_#y5~O`QQ3xJB0~wBwypJez$)Y`HtX8H|9eRR% zo0Dk&W3^H9miKpflDzF{LZ1Era4c?-AjR`PiGj1wE&O)chAl=LPN=S(9frI6=rCf=cH!up9u z8NXHPEud@ih>ayjo)bWKi9Iq$bkR^;=w?-_+}=+IKGQcvE+1PT7ZbJ0I>wl$WHGy3 zXqAvl4~+MwIVTL}I&qlMEipW2{EPlcGW@NK$qHqS`Wcnt2%O))h6@wrZS@}{1Mrsb z2Q=j`p{Kr^nn$(zFDeR^V4dY&Bx-kAlcbnMZa&Z&Ahnh_Wk+itRSJBEMp1+z( z+B=L>3L`U{d0J17w{)bHE5HN^pDz~dN<2TqNK*$0j_3sTq8PfBs_j}TvlsCN$bq3! z!z_@>ZVu>n)MQyLDwR^n{VpS)ERF#UR7K+0!7p-7Zq`Iu+Tdr_M5S4uHr|iNt5lR~ zsEzE^b_Kq0)!6sgw+ydBv+PZDi>K=*!ndwxC%>$I1rGc{J?D&HbYp|~e((1G+&jnp zmFqw54sS1&09@Qr+sq;gFJ^BjAzPKjYR9jgZ1imATRe(?^*kl$5whBq`<3{W#sc4iMf1Kfu@u7RH^9}kPjDiNx^I!W;AQq zV>=nsgX{^bVWOzK-v@>@5#gglqN+kq!E8II{uT5>8a0|`n7f<2e&$%Iu>DH(xjN8o>Xj=jEe>G?0C06WScBx#>;LX%}OHk36;j^=Q+lFM!Gx9%!jXMXLu2A+Qzyt5UojJ0T*-$TB9q48djC%L*O za1W37*Rb|alSh1&xbJP^k|V@?Z)uXzmCA0g z{5O6kw7r-9Ngq*vPA7L*Nd>-=(KI`?kCl2E5x{lL6RZCR?9{OTXd!^hA{-J!Pf7`H}YJQ8B zs+I2VN)`8rb<|C|d73Hgev-)(_FD(z)y|HfvLfox3#}y!9zox5J7q>+;uW3T=9)NR zWG)M?3+EN~n@pZib{k=n#j_8qe$nEc;wu1~^dT!&p-%oO)r7m5LxMAUfYV4cmZQJ` zWzut-IZtJb->06q)%OqOq9r%gHw@y!p$S_5ce*t=7e#ywK(Jy*yb^? z(|x<;TB##tE(wJUR?G(G-s3wz3zelrl87Od#q-*-2EEHnWnOSigdl zy)>eBY6F!qlk@cM5lwJW{m!Q*vb5ec|Rlv6c@i$VWXQ0+JlQ$P^Q2SCRy zX7@PluOn8_aJm+O+$8bRTe54lk zxhI_3-upxm`K;CocBg3iq?wsoN{d91d^P(R=Q%9N266=1x+ghPtJJqm7pXbrTi(Wr zPr(zV-<%D%^=0@u18Rh>hMiHcv=PCHcnkJ{hxB?L=N>za8+kjkbT9AjqtO*OmCp9z004NhPsi0DGjh|N?(uW@_Q5Orp9`iR17B~b%z z2&_v{J-2aQSc&7`0=*`B{_RxI-eBMDW)-%Bs;~tf1N|XPoJ<%*8>n3i%W)NPQan3e z4&OP#$8qwJ8~9!#x19oN-fMYn7GK4)iq?Xi5{_g(>uLii8=1cuX!M@Zo#)RPSh_!h z<+>Wq>k6XwIzCo_a+{9&w4QbQn4qG`$%!9A549IGB494Q0WvMfKC+!rC}Kv&5C@13 z^bs_YQ$TMBCR}dwWg{_@ICzQn@RRgL3)uis_7P1VaY>Piv#0?Ht8^kyKZV*vE>E4v zzsRdZ5$YU0P{#;0@Y9`qG=t4A;bR^9y|@gtuqTMJSPFfVMFlgL)^nqv-DL5p=$b&> z%K|$%&|&x6#&k?)S5qf#P^&1LGz)?*r^o!k~>Kl8)db4b<%uyhqN67EdluT~13x zr%q+W7BfD@tcGl&VHr`;Dia-`Ce}td?c|i>)pdMTOWY_rM)I#go0wKOoOS$+jlC|8 zND1T&`$+>&evTH%(@)_y$dgaxNpI>~iS|P23sfkk5+xN@QB%xn)QG79 z1&lV(fB^#pOd#iECUc(ofA0Cc^J4q%|GM7Hm1Jgqr{}q!^Zk79@0Z-}XT%n+_BFI3 zc`d|>v$@YgFz-wGmA`_Vs^oVU_L15lEL+0hydIB}^t~k@AHcJqQ?{Pf^AUC+B+~o4 z{ATk?B~tSYv4oGqB%%YaT!o#o>9d=uIc20y*wE@o0oXm}4X2sQRx+0=XS@*Q11mOY z=Or2-4XnIH7eSE9q7Njz`Aet^NAN*(V}mi2J_s9fRZsZ|hm){&DO`6-8&KHS6iW#2 zmvD)l;mk(rcl1@y)pd~~5QBTzOg z+hTIKlJ)&QcF>zZs`WG1P34uz%p#k5@8r8i?68zL_cYr48+fwuwlMRTHn^QUzDl^n%NvPx)VcAkq(>PX`jY`hd3W%05yN;DCHUx>B9J<=13ol;2G z3!pS$v*cX`PdY(67tX6vN^1}8UbwsNCuiN{btQF(nFO}kLT$*RAJ9?Cry0TW=?6OL z3F>JXjr?zPH=h-92fbJUIbBHZt&vYzuurgyruhMA^8!Y( zOL%dr@HU<#e?^)tj9&$K(JCol@U*lRVN1P03Xfp<*2r450^~^xvt)^|naHfrPhOWX zUpYiSp2Pe_bbO9P+HMiGaUS`U6xZ@zMsQjUZQ2@HN3~#)Ci1+KoQsNrhf&W#{SXGa zYWgPOixfSuWs#cXV7F;J?PLkFkfW4B4I^?jC3}gU=maHFOn#f0|K~6QrBKRyiK5@b znadcV8{n}B;OnWT2N!mWUA&Hvd$2atfE?CA<0wV(-PG~|i%O0blcSZ4%%zOYVDcEZ z-@psejPD7ieEKbz-zk;xO{_ZD-OS#8WSj)j&R5&+|P}-u_CYm4Fk>0JCD+{ra zs7@C!_MD-=m7Z%csAWk>HhHvCMn%-JdTQ=MMxgbiGKGAjPD3eo(-%wpeLtlxO#3=Y zx`w*ChO6?i%LQs~1Al9{La_DK5^2jFiac!IS4tIW%ksaQGxpG;#S1QH6LU%pNQ|6s zVg@6s5tXz|DNmWD<&zRAD-(6HgI+?~tB&+{)AKbmYSc3Cswbr{&>L0o?hrkBE$0ZQ z%ZbQnRKO@x#rT|vMY5>l<>bE*>tzsC&`zH`nr~$UIz+$K$Q=vm`(-XBbDvsr{s`&s zjEt=T%8|ubq%Llu&)J71mQqFqkrAl`du*W$^RZ1Gt+x?7=oph)Xv@NR@(6RQEs?%g z=8+kscW$KB%;lN&q;(7b-{5Zp^>QicYb0mQ3`XOii}Xpu*%Lz|04`0!o4cvTpm`P$XM%O9(alIy_ykH#;!~B z<5Dm2`O-rQLrXv9mj%}LN=SNS(Slje`Bp}&ZoES~$vsRY;VSHKA~c&k%6wFmF)rc3 z{ha!qMZFDBqD%3(F2Jf!QH$#t13U5Fc9Uko0XEVbwUHMzTeI+zI}dku!Q~$}5FyQKci#MO>2tK-CHQWH zjajtTg!j3Q5fINA?QkA8XkdPGgw!W;{&;MW0jH6~v5lwx&L$_Ku(*^_$-(%<*v@sr z_FG4(9>IR+@EDpvN*7{#S^Mr`UL>oFo#c_6G7dCQYBCS6r)7$kM;4-j#tP5(n*GeoR z^J?KcmA%h2QYR~`4019Nn=d1c!mcE1?tEG=Q7n}0f=G!h#un^F;jzz9i&9q_uQINX z7K^fTq%7{|+A{7We65Y7p^ma$3axBGGvMP~W z31co?k7%WzFTl zE+vstm%1r-6I)1aI7c3Y1zI?h#CkH9$lzDub3VuYh22?Lc^fH1S&7KnRqVE%oL|6h zb=c8XjdQqhSO4NXzwk~}JYfeVomFXrho$_k(E zUXF^ks3&3}SqnJ$REZUfA{LRcxPTN(A6v@%V$wemTNd*Si}7JV_cv zy;s&!4Ux~Hp4C7qgl$;iBo@|oVX76MimcJZHz9n+qp_3lEGtaQ!e=a>gd<$|s0$*! zz4RwCM-W}q`#GK%v7|65-%E~!59V{q>`Wvz!hI%!a_;Sj~mGno#x9y=GNDCG= zXyG}QS}W^iSruF-wZaZ249n8`MP*XXlKZFeUbv-&XIN^Ni5gMKGo{@2lK)O}tN2l5 zRVgJU=gGCgzb)PwsW))*^Zs*wxh-jz+9K<1;nSAygsWMu5^D%ow_GnT@!-f6Vlj#K zkso2ymh=fHurU1#ley$TUb60${!Dxw!Yh4-)SZl^K-NFVE|OFEb&@*C*@cL8g!{RkU!{x`d&&Le zYGDaiN>biSuEi@RX%TygeI;#5+QpwBtZvdyWHeB&P+F~gFYhGBQU+2M{EPgT_riuQ zbwg|*$7MY(El_yOmE236kh8>6;;)lDD!vS{m3$*MlK1kGG|4w|y^=bygyfa zALI%pH*$^GMEK5PsGKABm-|Rsl>937OL?Xo6{d5!Qn{%Ec#LxqiIvpDJb|^CV}{{3-wCxRjBkLXIgGP}20j=ScZBYCokV$#ZT$QQk``m1B|uDP1W~r6$XjO0AZjPL3;gl_$viZ$4KnAnB9)-~KOc{kCnCbV;j` z@0FA)_mOL&_Y-SMtCF+jH=1HO{>>7Q&&pNG5xJ*QFXSwFMStXex6izNj&i*`Px7mz zM4ln-PD!PtLfWHpO|+!sUP?=nnxouZxqtMza(*;_azwF;JV9EY+(+7~QV*nMDlJ!O zm2z(MeC3^3UhX1Kl5e8*M|rYRT1x#<&R4Ehaxdq|`N}o&Ny&AzZ>K$ael*2$f90u?Z#k;uLdlb~E=k=#_OG{duH0Wqw~`Av zH=0(thjRaDk0-~X=gD``ALR-uXC=RKrSxfXH#zHe`sIq-_Loo5C(0F3Yen;|9FP80 z`U2%@IYY`y89$)mau@k5f4^x5l>Eu_qUlxc z_{}So{L44Wx&M5h=;wd@uiQuZtJqcUbGyG+&W;{eKFPdSTA!pxDSJ7tJU{wQ?x6Jb zN}8jks?5if5|R2WN0q0^J(b$8%&{aj(Yz?<%Nfc~)Hd=-?xB>KQa_cxK6+l%Vv<%V zTc!Od7LqHKJj-*V_mnIC(XJ`~PfBj*HENIOHOlAP^;@o&`$W%EQlq3*jz}ux+HYPb z-zirq-z(=SDOQdtIgP$oo-KE{eVt;De@uV$*-?u`^DZT(lu$Gsx6hEfN(n1%Nv=@l zrqLcqj!KDsvpCXOWKAV2DrvXTIu*5xoGU*{ z`l9I;dqziirTr@FA$fw-t=sJ;a`FGKKS~b8l2IEdmnk`k=JA{5rHa%4@6J~GV@ow4Wii;eJAe~goXGZWPN^)Ijnd;=oqVdmY0LPA2~H zM116#e3oxet&M!j;Y>M46p?ed;&FVgDDjeaD=8?&Pb^-2iQ5yTVj{bX?773loP?%# zCfxX|;P>AIZfGGGi2I+0!Am1K(^BB@S;a{`%|e5$}oYf~T=B+{Av| zPIl(XK+y=h&Bt(d!FmOD`hMab+QIG}_U{1URspZq58=7|A+H~U%X$f9=3H=@iR{gG zgJ|QvAhvbvnl1%>mjuJvllsg0-x_)i&l|O-S*C|f3t%gK)l`o5{4E$848~)IMMUvE zp?`%)hLfN0)twg3!ZK?c)HI>*LxuB`as;Z z@#+J?KMvIa;u=7>zX!H(BRoFu`BoE?wGo76h=|Z%d1rcm>YYxc`bAGHu{B>4fs;l) zy8>?n`$HM*`KGgzTcy1NzObJgkHb>A!?f8vKBglk9CIROMND-}S4?}%$e0TF3Rjqf z_w5#pZv$|oeGW@T6}#G5#AKDAudtBaz+6zgFJYS^_IVGI>K~H6iCEMPx@?H3E<4!B zz6hIJAB=AAfcpFiNPeyVGmvk+-rtedJK%Hp!1G5K|AJ^kq!V%Wkb9cvdDJolefJ=7 z81qn0JAoor60BhVt%n%_-I~p&Y38vp>6Wb)wKZsIwXCr0<$bQDJf_RsV1C&gHr<0l z(L;uRfz9n4HRcR>!S(DKqQ*e2ZDGgK0Ul2F+{NNz>q9U)ETHti$M1J|jl{B};0R0b z+d;4HfJH9^2f%Jp>?5u+9tOkLVWc?jp5xv>j0OuzKF8f}dB>4UEm*b6U}3l)woE5n zNZ;0LV1a$j{JMD?DXWcXw$xeoTNhZ9tZ~*piyOU--7!1NCFaHEsphFBA6yjsL29E= z4tgRVC6kHx;Pb(-IwKJpd(e9r4R1mkHvX6wY$|Y(jPxf{U%y9+ufg{5p-(WVD}480 z<6jYX_zF?BH;4|Nz$?rB0j%{KTw`2uE_e&v$wa%@sULkrY0VD95=Y%Q3DWW!ybs6B ztIY?@#u%-ofi&A}H>?H39u8S*EJtIsF)ik1^mr`h38t^H^ARno$>2Ifi!4zMYJ^r! z{6Bd11~3ids3#f0vOo)%%E`oNtn<&Mh1vaG#Jjyp^w-~fZ)4-1`X*rI-+AM`=dtmf zun;UF8e8q&Fq{K-X`ySrYtFC+zLNhSM#oNj-5ltE0o_Ct&1`rR2lTfLGmTSCQ_U4I z`Icg<*IHupTJ^Rh{;tB_`BuF(&oVb=HN2rASPIQ>>Rkt$wO1p2AJ=J#SHKsTLe1>o z-yd8D((n!H#Dm!NhgkVL*f~rL_BG-s`-qvGh{eAq5^@JTtwz`v(9(jncP4h;@16$Z zLOv-z;hH~uC+sAveIJ7={Tg)oE+y#qZr zx(m94eLwg8%6QiPP~0c>X|b=_*Ctk_%ukze=iTY;soRqtvcCrVdD!#`EdRSS`vNCi zBL|Ixsjdxhi2RoJE4-Z+KPU~&WkVyp0d>^q9Wkxe1jk5NhiAp+#auM3r3bk}4+y(Z z@Llg0!{MQkgWgI6u@TuVuo*wsBSMyN&;O>Ff z`YZa5_3Y@LGH`Ejg4t&M2)>bj*RP1#?>I4{C3(|`q4;|o>9G}-*Ug)(L0ePI4c)%b zdCwI0W8Qt#!2{myt{P|V@N?cn!A_!Co)#`eqYpODCd(#AU;KnPvu&(-qiI{rO>33a zVoKE%1lPej`FAhu`JSu8?J%b_4s|(~4}Z(^n0JCV$^DEg+jVvL+nza|4KOI=4Wa8b zaQDEwa0tKHeg0-e_kQ;jqu2U7Llye&zcbEpG^fnDhrKiJY(BspXL~_EhZ9pzja3J^w=Kj zSyPraK)js;yiEoi3j;7lXFE3yqfg}ht;gls4&Uw^_XzJCPu6ff7AzQicCd9|NB{U< zb5GK(ijK=)mkfTcT>y{#xL`r>EyINPrqq!m>qpFTT(-@OYmS=~UmL$QF3*ywZHKY* zZFmz@VWU5PXh(1JK)k0oSjtY=!Qc-ADXL!?O0CQ77RNKOW2{M*Y}*va*0>JG2HPal z1E`S+X5n+McPfmj^z^Q57;C4xKcTf(xr~GE{=R|s!3D5a9Pho{ZM)_A zdh?Cf`qI?emsl(5!ucgzJ%38;ke#AT#~ zrnpb_o$uM?yyhDR>%fP^m+te|st1j!mL^M_^%-jxJ;7A_l=uyC524QSHGKZ33@-h8 zO$V9?_YsR~@P5lP)BQB;B#Q^%9au1w?VbdWv2Z@*_nqxG4#F(of2wzU_tlPy8!vY? zg<7qx=C6E<+=c2o$Jq2|()Y&~#WcihNT^D!PJSYOmA%aDKoO%!y$O8t;b5Ul-MhQb zJG=mnvR|WSH^uvDAXoDhx=XX*1`eCDEf-_Q#<#~M(OweZ1u_!r@jIf+E};kY3wS?{ z!=LyT+J!@d#e-Qxd(ri%ca{z=fOC6H-#z`W_Rk&I(Km3j{Kk%sGUq1!0W)}-;ZM9d z7Jo`#T1%4G3fr%3MrwHErV$71<6~Co&uBl_{>Gr!zoDA%8R~zr@8U4r7@^DVdj@lc zrufEa{|MJqmHMdWdBez7vSUAs@=aQ z_=>LDGRM|tS&RZnwRM7h8p@3wmJ;(l##i*qVf89duMEBJ+u%M5i}QR~k|zx8?mgT6 zNpD48Y47=4-pexm`j}~f?(d5vC{rlZ_c}5dC{&VlM{jQ$H&NhE1al&KVuHm#$$nd)PEyF6^DML|A zHH;rFdyg&EY9}UOqj>_OSDX9f!?IvqP9E_U)K)5Wj_V5eaurNt=@xMz`(Br2QkNqfFTq^95QNYf+^*t!@g=AFdb-I`3iD zx8Id8FwocF{JYN)ddc7GEeYgn<{Mf}6(|$r#yo3@v-Q}H*m7(|mSdFS`}zTWD$4Qy z;yKEAkm|hA_u{P;of|vHcZ|7~-&WFI(s$E!pwD|Vv-f4+B7K?p3;hN74&&^0^blvo zKjE+>G$n;ocBN>O_S##m+pRS*E6lfGgWDK*j<)n{V94hkPI4|l>2iwuKRt_3ME$jQ zPXN9R^$eKxLYuaOu#8@nD^RvWVy$b`|cx_|&G^|j+^_t1mw!X|>b#K1ul4P|InZ&jSM4?rZ|l)_j~wjs?`M5>4bJ2u z&5@W5wgdDm+buTxthhx9eeqR}i?Lf{Z`fb8U$(tyx&fn`mw8b+YK=yC{U`c=1((kk zFh&o<=W~$xeK~UukYhyOJPOX=MvV4$^q-!BtDt~&L1$=Uz~Xy|an}Zy*YUo$dS36f zq1e|tw4<+V;Jo`Y|I&~VHmKrYx^6D3rLo2rjeU&RpV(J9CfIAO#kN(karRxdQp?{c z{~w^E_^7T(Es?t=pk)2T;v1ROZuF04HvBOWgDS9*MT{^fz*fPz2pX=Nwd|wtXOE_h%m%b+A?8406%0s)fXj-wd{I>2-h6;0mWnIiY z%zaBNLzY_0i!q-Ooi*0@TjN>%6(Z4vZ?6O<$N-4egTWGJSgoX_j&!85Rt5*gSX98; zTu>Y^H-c_V1@mDdR{LXSe_h1Iy#uR>g&3#r!Nu`2;_0;<*~6;oVdmj=A-oD8LYEMy zeu5aF>%@&zY1D9WUu6Zh2;RTPVb@y@L$w}u3hF4a|BryFH?dk?LTo`ZSk7)(OS6~- zt_MZ<0?5Y~K!i4cMtT8+hm%#ejx;WdV9kZ|c_XWpiL8G3VxP#+8eAs zJ7D_RN!s2CjE+RVEd|p!2Fy(wBfpbKvKsZ6$Qpe-sNgrXX1I%H>x2hzYvk`DxOZ(t z@7ECX;{hqXhEz3yy&Mnrb`2hfORW8KnRD+0F=-{PP@-M(h|g|gR(qOfzCm=-$HX~* zNStCC5y1HzUyRMWi5$)YZ&=NYHwQmN6>VoU=(1Xshq5UL-+O^saJD9&_?m@8p;UwB ztt2XAStRmA;x8^>m6f1eQ$T7h{gtxob{ko<$oQy z@8{c2Uj3A+C|wKpK_O@@!Ee=o9yy1n$rSOo2+nFfb`s^Z!#qWhaHk{hY9oIoj!Md8 zCFr3woGV_L0#YJq7x5>F=c$uaNkq;OFf&!W2Z7QU5zPh}=W`dtwjqn91Mh z``dUC@q;N|KV`o{u2Z~?a!2v&$xEIuXb;8Dr{JOFsIn(5-baZrisC^O52>;nAURPy zcZyeDa6;m@R9=!t`9|5{Qg&7(%}OfdzVgg}PDeCNinS$m3a&~?huB|Ua-F=yUoAg^ z4^z-l;v-bbKq(Ie!6-E`jqeo%lvqmqgOV4?o#a&Uaf**nJeUfWO7K+jNido6lJCWv zDc(*=pLj%-t8e>crH)B0llqoOjgYiR9g%lRq+SNcB$a|FQtYPeNkqNC;;EDrDXCIQ zLhLUH0mYB0JXg+FUQ)(Nxrhf>o-er*#Fvs!#X?g1#QIVu@?QBQb`={)Et58;)HT_m zRZ3RyVbS_2v3H^fDM-P7>>(J%dx=s2$4g&K^mMpZy#$uylOU=+#3f6-bshZ%9K*Ei zEs?t2N$hF?9tyFZlz>DbevS|I2&?|8P`27hjGdLa?l@SJ-iOK7hDO{`n3bD|y+}l< z+!|r*UINA)Y%=<4T>*MtjSh_}6Mjc}G}b3O!Tw(fXMpb3f&PDqsDkT6XsnM!qY?40 znaBF7n@HRHiNhDny$(L3N;G9lLKo0p{1?Lv!(x>%q{0UlssoQMjP>u~>*)^WsR}jC z@Ml#p;-$lNyq*4dA39Hkjr#xzze>4yyJr4YEH+~TZ{2>9Dq)O;( zb78?BtrE8EmGGxL0ykMXUgzS_Vo;F>d3q~~I1=;F4L9;KbrDfsMd~%-R`TwEUstWw zsp-?GlWQoqB2qsZRazDEjPB4H7(>7)s?WeJy%IIHdg8P$fbY&=+{h(TqljZQu#!1p z7FrrQ1g1a0*p*0MJDIvwM0%>p$0f#+dKEE~!4n~sS_enkTHSh0Jsf(VN#S)m$|@Qx zJxKf$!SOI!>*O$# zt=HUT;699f9~0%>q~Tk+R$YaRiPsIZP~Rz z@q8YhcZnKv!2NJA__c45u8lbST}0_T=UquU!hRF1?5NPFzN>xE-=z9nTY$#$f9v5$ zMx$V!dShS@(cf#BA1?O85f(n+W4@>Vi9Y~y&p7y!;^2Q;5}wT*Y9A5d!njfv{0LsJ z$NZV$XSDYQn$_F2&Hh4W7?1gP>6WQ~@A&|Ar>xYo&n`s*Ip$N``%xb&V*Uslu~i(4Om8YqL0ai)zTX9G<$A z;I9mMaQWJ_aBQg_(Hw+D{9|8%rWDne4KPE!=g0Goy5Z7LkIutv<#8-i9~2GydGI@| zB$~*|9QBa@Jf4mB0f)%tuj zES}oZKt^a6<%k5p51VXV?Zy^Hk$b)zhvb%d-KzX@+uUctx82(1*TkZ6%C= z(+!E_^#j9rjoMvmPE`HIQ)JoZZ&f{|t7WeDZQVp>sc#uF)EdtyQ$cWq=QoC-!0F&Q zO@H_u|5lwFU-GcwJyaV)#*jBf<e~9?)07NLHyPij~+);(#?j_N(!?{?%U}+^SuGPbpv3NaW&F&vLz0 z^`L(#I|*BYhw<%yN~xWJGp|9jEpSu6j`eJZ=|1mL^xyZvIn^Crs7dg9h$TA|$kNQy zj`F9V0sL^FFs#=tU{B*oeX=hK?7e_debM+?To<_RXxC*L_D} zEem;mZuH}ic4_nIr|MPZA=&Nv()W=b-VM(4z~|AgSr*C& ze^6T$8tbbtjSp#j4;e3oZn%$`KlbMZzc#v24td_V+`U1ICP=W56@xaAPHlS!?#Uw6 znh?7Vnq{=~Zra;RSmMvxiNRXcN;Pv$O=F1oA^d;Yp}pFT{-gAbM9`>-Q&(?MtqgA0 z7BcE|hkl?P7nr9#hy}pnhbmONGza|-b-yMhm>IsVeZ#+p)~-XjU@5%N!iYD@zgEi} zK5!BakN14ts2I)-!Uhz|(@2!(ehrahnqPXiYB%W8hkK1yyr1*JS=u+K({F0s{_XT( z3q#8^zwkC%77drGer>$vsil7m!>;fnt#C#)QO+iKfZ;?7W}z$433J5zT8RwJ(v1#2 z>o*ur1WN*c&~ITBnH+pl|CldF%W5TP(@MNywdQSK4m*;QJZB9{@%NxA5>|zTO`{fm zmg|fp$$>u^Ci=6(59rvfQFUs6-%!*`-*f-WT6@g<8^eeG4)}g126w8e zsLL6l8HNO3tLjbdhgiHCZs!bEx$S|y+J&JrFc~^oSrmk3=}&py)K@d2t*4J|f_rBv z{FBSrYsi6%X}NC5*U$J&geq(CH-ZbX^BY0pyWxpFN7;UtKHTV!(+fvVp{fi<;XR=Z zx*z*hs*iMk^QCJ%?AUIBW$*$KWhkYqRQ$bP^jZa7t|Vy(Gf zTaTLZU3yb+eP|wILSA?rE$(n|A-WC?Xc8RJj-y6ApzHFNt5<3N!)wzQs+t3d>VSG* zpaqq&cLKHHOWF!nqe1PX;kLka?a^>suz(mncs5AM66Rt{)isQol`!%+;DM4A+Z)U| zU_zplWcRcQJfbNi zw)eU^gmzLaJ6E&E)oH!$)%iUYjRi{7odhz z8}d+IsjPhqncFln3zg`YB4#c`R)(C+Z?DkLGG`92hc}K`GHMMm<8U%n(f;NUUy{Lm zcs$osGB-(M#td^j^V&S&{bOal9{pX3W+M)Vy{t3nGL7MyHqL?LMO8@j8$Abmxlhq2 zzz&L1&eHH2UV@Xo$9&dl$cg}vpq2H zT2;-=KVLuxHIaTTl@V3aSk645pYqzmth|j`TO!xyGaIjg1Dn4(`Vi`{9Z^M~z#?mBOBZ zH<4DCLRw`F2kRu~il1mZR%boP8e}e-UAvf}Nt{?SYB@`&Z2{_M5zp~RuSqT?x@afX zYT)Wi)KOtnkXW>8EGKO5i#Zb%47I3+ksU5yRxypFyh_*&S?j1MDHTufz}&f>r*~7Y zg=hUB^{bIuPAn;PQzE-0deRhLNlQT41&cmTtvnh&0b>OUO|10_u^Rjnp$lAlj+8WF z`2xJoTj1nNq;AOhiSWlYkek``RLm)9<7?=38W>6XMe&VxPtS^<^$cYv@rguc!)Q>) zo$^@|)`kl3zt&OTiy2uj&`yXmW#wBNYNFKY_{KpQ3cq71p1KpEWn6_iUGQFd@{*9G zD1}y^NO{bopOhLF0Qo^prc&ypWf`M$J)@(Pva}%BekuPd(I}^e=V9q0YI8QDp+wi81iR2kPgKWh zq0e85P82cpjHi|AMAi~3@io_jNx`$HzQlOg&wl>+&}`Ocl~}ln)Ml}7`F_V(Ra&pcN_W%tSOhMU-njpXBm>gQ2fj|874(oXW-;p&+KtaXs>QHN>p+U1d`eRgekX9sAo_FDjk@3t-wfmU@NR!~_z~FEMu*n3E`zCwHd&7M z;Yt0Y;ey~Xqe|r+enS70_K%)@!EYIKFg*XvxDKSp|Db32nmY|GnM!8cf7aehOM*Fr zl72?(2=8IG^t65jyAB(eeFh-*aWum3&_vNWlb-CfZpb9RA3`PShHFO`UMwR#>B*ryQBjb{w z-5mIh;#ld01D|Vh)aA5{$#AE=L|=#RmmVQEq=~+36{xwpuj~ zRzJ%qgYppU<5*f)En4Z}@yEbCc!@r14<3aE)`1>IG#&k3KP<8EUGRMXFW9~G1(X9G zo+5e)9X_2j(gv#!y(00~%xePVXk6$deW#NdKr!vl%5{5K!%NR%QWw(;mhwbmF3_*W zmdqTo;R-m!OaPTR%DD=U(i!@`R(hauT7gOa_6|}hMsmE^K=CFY6p#{W478h z*ayJRWX8^n-<~`twI+FNVpUwGeY|ajeXDCTc!iC(77VmI zE1X9L8`wLX2Q;!itn+P$%D)-v`Bf1n?u zX%7Aj9fPZ&zb2q|QHb_}d3cKFKG?IRZ1eGf7P2FCLpRen!!*ITNxy_O_+nbnp5Uv# zQ=ZGi?L(LQ#`oN~b+z-Yj*GWm>tE}v9%>ut8?1J{>>Wdo_?B@qr7}J?D|SZgR!2vY zCABSie_})2F2^E!b*v3_uCb<%^aUDS=$1D`T}Vc>t*CF8oREX=nub!9u}qAqma z(-f@M_8ML_HOJ&wZ(5gIZ^Smlca2C+u1lfSWG!Tk$Z?C%AMn;6tFmakXks{@)%x;RQC2d~F6qWyO?Xg8 zLVHi&yFHoRXS-h>sCG4>0@32UIDEISk`>J}`d=H)np$HvS?5|W+vdbgNSdB>B7SDv z-Hx#!+RlT)*k=9{>baSyAqsP@F4TtR;%_})xQ9J2gOc6Ney0^4#79^!z-p*|3f8K5 znqqiZgaP7Fdbuw6M(^`(a*rR*b>$AZL8Lv~`*P3m{-drZ-OGmyT^|fT@0o~?`;6vk zRG+t*%0P796SK;8B+ebLjSoBK*tgrBwq?gIx7AsO%&((z=_I6REMKkIV%~sfwvvsMmXRJSK_`C5f z(@4`j#^0lDHv{F|DTccYKho>bdlZHu!FOL_-Xk1Z#q{)hVC-4Rh?qkHcuRbk!?>DH2 zhZ$}PVb&{Sj~CSe?j2CUdJcc3_!GUT;`|wPyYK4f>;FstpZY(8P5A^}=2!H;LvQd$ zC}KaLzf=D?yWbC>T)Y8g#2S3n_4s<~Nm~siTmw(nVOX$sV$es zpZ9Kl??K_=XWrj};=bjb0;c<8aHfYqQLljiu@c6=N_I3(GUH*41s}f)1jb&J)%NOw zD24wXWnxkEoMD(lOPz!HId(+@aYf;?=bj)Lm-F25zua~W3?+V z%%;GkH69Q6zia=D2IJGZ|3Me!`n9!i<%^n*00WbHQ^kAScciYjpqY)##u z7^-;ELa-LuL>pM~g?GRs*~+}9iB}uSA1b&u9U!*Hb502e5qwv&x&iemteS$eVCBF( zuZ%WW&ul6M9;$gTtJN{8chXL*aN${DcuZ$c!9kh=AW=?G7xuB5nMZpS{w-m7dI`LP z%-UtmDk&5WBKG{i%CzvjYG%AI(0#&uyaABp&q&Li^%`fF1 zc%WDt%Z|OQPuS?3?Duste-^ghMqc%iS0kTW=r^uY&m6?K2vgsqtU{MC z*Q_ROS6FolpNFtLm6N)a*zgdx5I#joV>>TN1Kw6%o(N-G2j4X_t3J(H!i|M02_A<+ z?jbBMva2nuHZq%Ck5^A&DP723&?DGW*e?Z{Qoyfx_arT%m9jAMx02P9g4Z}nnO-1u zoup58z^#mQg3Xvr2}w%JS%0pik6lBp+Cnd@{M|`v_VC)r{}&?P3W{Yhb}z;H!gMIC zL$WU}$P8h+5q6@zv>9PgjlP6aPS|ji^snLjExdN&fvMqjl&b_yB9TYpLlJL~AO^TC zYX`BWf+G?3Nx`xR@0SwS#C|jH3nDql=e?j>gda!N8-gql78XI02ogekYJy4-3q*NP z1+^gP6=8rs-Q*ia+w+u&P@QSMn&1ZC{Rj2ex`X&JH_R&c6N zw4wi92%l;?7=;2fnXA}QyB=Yae2N}zJv^G1=tc7BO^(uLJDGnukssn+J0LF0Gz-$z9PFEoA$_Hlz~n}hnk*Wbz* zyO^Anfi(2s&kE2duuq8pYaw%r#es>;1*dy9fuk?(`=TewPk9Pl+xm06U%&Z9 zM^2}=YjJ-?|M8x}?$W-@;lfao?ia?z=20=LEyrV@iPt8((`Js^HFDaBhZ4pnbR^uJ zI5ob=_9K0N$c=8-2c8jVD4B=H4ixkp?>RPL8$RxR&%YVXjnDi`)OQ(%%)2ZVmU^?` zG~3K$P16l}-8faUKigdm;AeIByPXv`p1oFbz3kTN-Z8y5yDNIS27cphRKI2_w`SU& z0A=5iv_JLeoi%qI&D@)Ick-fyBk`*e=ESF4XKUO1|Ak&&s%ta)nXmUB>#n#})U~>I z!O&RmPFl`5_Jz~6rRFDW$L;9%*=i`g!kA?9L&hIz(}HSu!qDa3^EWr%c=y_CZRRV- z+jDRBbQN{&=sMm%!P~FSi#cXD#x=#&C#||;>ZnQL2-!9iZK4xVBXE0YYY}_7O9p4o{)A2>@@z~3;4OYMT z3&UvjSKcws*ZaTdKHGWr`i-maUOwLXa@)GjF}>-%$9gvQPj#25&DN*mpGoXUY)Y;f z*?i}wj7g*Ce{0fRo9?hDx*eay=Gms2^VQ4UFAi+$E9+a)bD%q`E0?moar1I-!O(Q~ zHh)v-UeyEI38uAJaC*W$3C;1AxT%g^_AhMrm^(F1!RNitfU+;^dAH;2)vU`ex0)~I zUdih$?91!h(fwlYRd>4XZ0!6b|A@Ur>{O53KB^=0zA-1pHr+idb+2O)Z6V1r1FS%- zN8RVTmEX0wJG;B6yR18SE9+KSPqE7ptW{sp&e4xB+-J^>J(^fQVpP&Y@yU){>$9di z4O_J>L6__0z7M)~bk4oL?rP~*#jWdFU%gh=^>W|pzKZ^u;hn0vmI;Y1DNQLOQ>UkC zM~%uzzPn~j)7@j!CdGGIrkKAn-qbl%Pq?4$yU~@`{cd+|&-w1FU3pzuJ?HxeTqs&; zer-X}@b$#@bQtkuwtf=p?k8?=yN-FZ;5G-tFz_eU~RD5u@hq zJ2p_^yyh(muh)E}`;F0W>cIcADsEN$C`Yx`AF~auWY}wiGrdQKQu{97D*pQNwW6!J zU#VN)Z7sRJqp#LEG*CMDci$%ceEZs@_T-T%Po_4c&Cghy(SB!h`pA)66Wo@td4<_v zELA_~FLO@nKi*$GB#}P3eXkAd8cKJ)<;@2Fkgko>ziezI0&9X}jw5KZST=$Vzr#?i zIUTygyL-^v^Lp3uo5ya%T`O$!UV814`ufej4Nl9DfAFu~0bNb(GfCQ%gb~IO?$jBh z?i;l!JvqJoj_QOa%aNGzF(E^1nN+F}ha_ z4Q97xQS2A7Ikr}!bDPai($d=lpdehugD>|M-a2r@-0r&Ky?p$u)UUVobqr1G@9Y1i z_l&N==8sEA>_|A0RC7n)ollRtFMavQp_Ka)&f7Lv+bw5Jji4}Ib+2-6aOMnmI2Q~| z=uaLjB0}yzyy1ZCe9My^U7mDm%zoLGal zn0dOU&>$Ml%Ta&M9Bvsr)_-guZ^-QWrRSjkej;3}!r#~4V=Oki$#Dzgb8gJ)m?@?j z6c?M&h_7}X8T9tKx(;-Xx#7C{?v)p>d%HIeCJ(;YH+N`X;9m``_(|_}+~;UbcqY}7 zemc!QazkqJ2rd2PG>1NRvngHMj5gzw_~!o8Q{^1V062!ozWKv5J;%MTqDICXNY|sc znC3ECwVU1|HszT4lpz4#p%2eyv8!z`sc*rplCNLBUUL0|udj9;8+dWBv~T3Vm)_;5 zdY4(!tv9R_;$|kV8aaa}&rC5U?@wrrn-<^f*d6m1?Gk)^34!nUf9;(){OaI3^6kPy zIR=f{sowASP2ro`MR*n7*Z&?(m={UG%fw_fs`JCmf!}$b7_J+d+5d9)x|`)4J8qPH zy{@}$;N`*N{T~d>@}#MsGZqjN-C(JXtxw#QI^m9RN<;F<5y^?u6YA)vi>*@)OF&mz zQLwZ4y~Em}hQV5ACW^r~P$l)cAM|Yr=Bm}Khd?=2p-<7E^tzjz zbNb)v{@`Xor@r&%t-`*7LG8f!zL7&izD&(DgO=IT>*jauGe)!!ebYYTnGrQf+Y?76 z4JFQr+Z{8YPtxMIVSoNh-)UFY&|GI73h(tUuQT1{C8{Vp)W|ODc(q>pU--_RHT}kz zq^FJH1>6%_;h*X$b50mo(3jeC_SW&6!JZFL3qH`duKyX=Tz|Xj91$xw^+TplVmlHx zB+XB1O-fGMl~|W_bVPOHJ&q*HG?T&bjP9J~O89R+7ryr!?s!k$aFsKgc$wGS<^D4E zwpOrnmZkZDehTq`dR-E6OR3s7*i-qN?>v6!(}NrPU+r1f{c2A+QHpmDjOj}rJmq!- zPoN96QsdPc&1(D1gohFm;-8MMPuQE(G{TtN6`vVvA(o`r9AkW3>qIN{n)hcuyYG}c z-&Nquan%mffA}6`PO}2fw3WE)wdiEbj_9Q9XZIvAIMN^Eoip4pv~A#6?~3002KEvo zd!z40U!L@fAfKLM9=uiCWS$W_$-dV9G*OqQ}G7WZI!hpW}PWXzf)Bb ze1{qO-TvQu)()q;Ho4{#7dpZFE40n$`r)i*pRib)Cj-sqo7|8cqh4^7@pw%jpu96yY7MEo9_2~6(HqghiN-rtvdB1nl04x zal!jpSA6C5x;vcVfsOr}hqe#r4L72Odm3%%)kKGs5dA(Tyh?Z0WQ-XblWl3WPT}b! zGQJfr{CRtiJv+9^(qdYxkJA>ApU1-wqs%HC0~b98_C20<$9O*T!c;)y&f;JxyCPuB zSe1#AvZ(O=-S?PR?QVDOU{|IX|MErJO8H>1bDFoye-NG4b`YVD>Ru$$e42S~Oo27d z?snA2yW?g$GVN3CPuic1-DI6&zJU%|Enfa{sy)Hoz7a&13gf{D^8TdfVc!t_oOm0T zh904>UBlY+I+~{y+&zK4!n-`S;nJbagA1Hzho`&8I}f1c9q$?E_p|m`?O(|baj)Sk zcAVy#4&c$f&oL>k(UIVoYF|YB_)Pm8TUpE}#y{xa2gg_q9_dqj`3dj=toB{=&cX`6 z#_#@NU@<%bC9I8>vYI`LS6PMPaR$n|_uvzsZ{-ZHd#=%(wZs2S>5 zYb*|&48NjV05`ab9D6)@Pn}KA5`yKU0JW>ZK_XqvQiCun|2;3^?i-W72bBH@z?B1K+d!eFb~$-XU1N(m0I2ANyd-#di`It z+3Na`j$8?^!)D*R-bv`^e(L)%9>P+dj>a9HrEdIPdHBN;gP-}=`F4B1P5s#G%6D!f z&(l1|>Gg8yC0_Os>B_3NJuptSOLx(5mB`^4rZJYd*g5u1_T1P_vEJD2lzg+bE2hx= zGvgG)xAfVdAdiOH@By~LA#lLw^L|MWaLqT0m7|5Z{4!J!P=RHIdO8x#wi?^Vdnb8< z>?Je~4LCQU6@6p4&Y3#=pPoJx$-7u*w*=R#zpbBTc*1bEaieLxWqj-u`xCLuEMv#o zU9mN`ENgkpYo=4iImFx7XwaRYPO~x&RG^joDq5RH6uesf8PswU9!OJ2TEKduyP5(W z>`?yQJKy8NZ(r}+%g);r_zdcXz0T>w=RF6}(tMD0FQ`z>UHa+xm!C5nHK9}NPwQMXbKCR##yV6p#h-*4b>IF8D3nd^viyX#w?W9)$4aAmuH>)qkM7aa0} zKw0RhI$M{dKc$~Z9W0DFO~fK|Vz?%HZ2H(ZTeW3W%v7_M{fyrcomsD5&z@d7Q4IHE zy*`xSCR6k6L;-eSfrIowZ_uK_gt9v^ksZ1z-kF~H?k|QLT_vt6*VFD-D2>eF{lmZT zl6H2YTTzk>fV}^UZYC6dv8~8@I%WtSgI^fGZFo~xsVQUx z1xthnqywM1u(2eg<(7;(w&J~inBHtRCGs}AAnVw{fi=dn)?GJz5q6NX?mK9&`NJO! zU-o=RjXjLdy&bLA&oyVX7qqYFrm}aF3!h1oWwm9MHOZD~v)P)h`z;;BId3x^Fitl- zKnj+jQjr4cnn+Fhk7w~e{TLez5rrUnv>Bx74cfz8q7MDONm$@{&os|oc5u(TY^30n z=M~R!Mh%~5uK&mE6z-uFJWecw9yXnm)Pvo|LUT6!FIB|%+pNS%TZ?QRmOk@X^J>#q z#^2Bi)x@?m;F(l`EjSF@%ZK34!zk5F#O9*eSH@n!Xmp)N!SwJoBfS*x;ORrz?IS#Q$`w&TIyh=(c4a2 z+bm;a!lrA+YsNc_@9W2FYw$g*h(pN<=b%oW&+A^weg#}U2iap8itzlDgM8TxMqoE_ z(vM*Wm;03a3HKPHS4S{Xk6=W64rZKBeV-D4krqq|ULj_so*ivH@u71Jcbi7Rd@`4~ zZhuS?Y&dx^p47&)nS91qQT2S1-Q9k4qEm?4RN*OXA)2xb&;NSbMgp2}8>yRpl*4z} z=b4BS@;2YkeF?t5df!5A{55tIx5IWbpT7GOcC4QByz2cP+R7$o>U;1{7OAelXp#;; z&kfx)!+#i$nO-&f;f-;d7nqY{sw^!rn;9#n7|*gZbyc4ZR-gb6V1xP%c2%>PNwl&T zyB^)sx7pc|`p^Q;Q3jTIiPoA7N6h!&w|P0jfABZBEgtg@(~p0}-03^);H^VfcLjTl z`RrH}q6Ar`PKWJhp>~n(BmEPGS&Wf0Ob1NIfC0}ky=ZD9;`r+bAJdQUjV{++$G50r zXQx;_8V-Xdyp~7s;uq6rcLa(kp>eeNA#^Ou=rR1Hs+_&f!|Z@1Q*U4mf@x^6;NCua;QU>mflI+spkc+zkK=B4-bH}tUuIL*%R}luS#lWe24gMVW<+aVOcifagn{7%pIsO-HJn#N;m1?1H!F z3~E$Qu^+ceyA``tf{u7V-^fg`32mR#>T~Fm*Q1qL!fs)JIj2=~hD2KE@9VY zJTc1>cYA?7qfT|LW{YaSW-@rUH?^6tAXRJ8=mEQ3uYLyg(loG)IiPAf)z`HzQG@c> zM~djXs>?wlnpE1rda(3cm{~qf>`s2DTD=8?Q*GG9o*Q~NjCtvrwBU#SPLPT#gO!?( zDd|czb6QZf{=h-vzqWw4uO)Jed@?sd(^XxlNhIE@j9r9sJonA2L^bHYa1I*E#p)bQ zh8nbSa5)TRY4rAGU|t`^-~G8pu&xI|%Iol>SBK8y!WE1E~bivvrvhw1;H0c%zt zTmXib9evQ(jYKaw(HNh{&SD-2)B+;C%2j>-eoZM6@cW5W=w^+Ez7sg8H1$FDKnlSf zZ`Z5=jamz?!2xy}%)5F)u!cRsT+qi7gDgrgjT-a-LpfR^3qsjiPiV4#lkP~kCD5{rqr#XtuV6`ueC2Y&go>wV{N7SvMK%!&ACXys4U{q4vO2x)+!$d<3Fy1xnNZ zuDcX|JFrFbU|=tJ*E-pk(<}smxm{C^ceEc5c9kYS*sdzj>?H2+aZOoBc8h-&I7S?NK#>jjK>)601~(oxk$Esa2^`f)D7{5CexgjQUgmc5MmKFWDfVL5&gL z;!vN|{MGxW7WFy5p7@)c!IheEq3hw7D5)xSd+>4Catc}JqGkr_JfAo=cA8n?oDRLO zBmRT<#TM`!?9kE&dxBRq*(jL(NZ%3A1=~#1yjxUH>fiHUXD|0U2!^0`$d`pS)&2g* zi8^`0W7e%#|I+&-T?x?{-Rczqk0v|(Q~x|&P6(|zjg<(RM0L6Uc|#N9gH3lLc%9ur zy;jHmk4L>ebP2TX5_LY<#T~@D<*0w?yQksW9maUFmf#o_=u$!H?wjfH^;1W|pM>Tn>$9$`Fmo$6bzcxtQ+Cm(_70`%f z_$2Ezg+!$_sTT*P81L|$1lu#ocZRsr{2)87>_q0V*SADlOdnA~e$%NFovZ`}0}C>f z8dMSZNZYFVz8xOauZKi)!-@A|uF648kX@q|)q{c0iS7WC$Uf3Lfh4Uva60s~Zk&IWwpsm0&(r#o z;jerf^r_)>{;P&h{ROJ;Yukv7>c4sajF)M zt*0Njm-<8u4z0OReHvUvmhRp_1L$pG8OR8Jt}Vj0aXJ(GgX6<-x^=z{x}qSeKg8pd zvNzW5KcPCKecOMMn3`Yt3N?-4VxsZph4zE%>{tK9zevYUtA7=k)omPgs8z(-906tc zrEfoRP#MHNe&s!<&D1?Je9SnSNYZ@bKgWU2%~GR?tEpr!u}Aw0-;1VSdRkQ%^Srni>8x?CiT6liCDZ}Wu!&#RFCp52T%T*$7k3Q9`gOS{%GiJ z|9|Tyt9E;TZXBojt!IWIQ?uXoC&NN*i*uvJ?yn6#pdZa%<9c1QehO)ry@LI~USY5*2%R+J>h>JvO8S6CtV%0sq=L{)9Fj!#YzZ0lqlxz;xqg2Lx zO!X)IA3fW(W3+DHcJSHEYJy`l#o*P6oo770Ww_z`#89esxnDPoQ_b?cZff%%3jD!f zVU_>Belnhcy9|f@OEjN{KMg#fJxYvqV(3o&HvgmHf72}v)PcPx=7Wf~hF}Wui6{Xx zbGVna&rosL(;f!V8W%YU+rosgtIRT9Q&S$|x0MWMqsnrbUKD zh7LMpnt0)U?q~0F&i}jl&FB9)A5BeRU*31UYd!0Ep2ezJCHP%$8tVK{DlZ9E%@(xK zRl48Hdi|$zSU@mVIopgQ{ZGn;rUHCs62#JAtDM2g=gn(3p!-6zuUkn5)XdkMpgbY& z)O_L_Z}?hPSXW!LLwpaRIsRN;iE{af=L;!J^A!=bowv@kMeazWEs!?}j$nc86xwm2 zl!OB?{VR1Rz5kMmQEfA=l9P-1ZvMw43uwfAyd)>eiC7JDNeBsmw*a%iDb%Vw)T*G@!b^0HRk7 zU*LORk>-qkl6$9irar^jYMda~5@8O@@MmD{{p@|wuqYtNx24Enmj5yCdNEz}@<4?) z6s2f73XSK)N!}Ro3n9f>XekE@txk&sFb z?BUH-`#XdrZ>Enzd3;EW;cHZ}VX$Q8mBRQLJ*e~1f$*@{z&0~4&)ZIf?{pWuG_j($(zW6wlGyyivk zKtmffAG6ZPCg%trdp?kU(M7p0OLe*tzEoY0@Ub@y4&6G1d3oXvSG1|Y|F(CX@hdq! zumdflnsK0`3wXt5B`jbu-s{<+tKoF)Kxwi#SfedahU-+P0CiPd67tY!E2vDr(~+wg-ihnmxayo9-TF{*u|` zLxoGa%dS~4-u~jfil<|;HVma%9vXKex|@eY0c3lFR2K-9XLD|kA&*=Pp8A5k%=o7F z0)C{=%Q>3Aix2wpv`>iJe3|G47ilBWh-RWqU4Z^^0cw)7nl|OHQd;m!`3n&psj>rA zey^w3&;}a&o*`OU?Q5{icV%g&8cUrC!m9?S<9#VpchhS|i8F_ok^{SnnWCtD*>;0R3Ev8U- z!%1ela@sr05U;G%mcgjh~z|f#G5tUJ$Qv8a8Rl#M5|VJ|;aXABRU(q`}FGtnOAH z=SmDt)i3l!2{ka*o>6vC-A1Ehhf^LP*W+B;Ac1NqgG_&bga2p4*UF;6%09K(OFDd4 zIFUc~oz;ZsU+`6;?+}!~i-Y0oexkfDRpZL$6;!obwN{qPIE(iAqokF=U%i{er^Wwz z+XJiEv&H^!YKIo(C0!nRlP)<)8zmZ)eE%`M(SMhmygqPLN#}Gw7yLvXu6vgK)ur4t z6e!=K%)b%X;hJJdXO*8Y#`zB^o5Y}eC72+7i67VthCli~Q7-D6P?bL+mH01!2~^@& zIgbu0-bcD=aveTjo!&Ay%GVWmph}{c6l{~%qD&I0dZ}xaWwKMc6wKvZ&7`Vo*E#$X z$Q%w+w@=hJ`d9d#7E^RDqGBr3-SnyW&`5t3Rab$3xKJJZ)~m3aqJwp*!(2mh>YwQNLJ zGsBaoJ%_spCmJ=LhPrzTIfz<&*5U<{FFQ2Dg?83Q28!duGJ36GF(-o~I8EVKn&Wc4 z<`*FoOoron6`sZ=KVxpJW^v5SIcvPj|liUa9UNNn=00Zqoq24{^~5+ z=wdYNaoRQPBp9P|C92C?fg5P>wg>V!DfXcMR%5k-hucg1pG5SMOqSzK6%YrXgGbgL z*aM!T;`0NvjS384m{*+G8VAwu9Ccea9Wg7Kr#Njf*>N&8&3b<|PyIM=Rn;T6_(#H} z-$HheY8}35GwSYm6i-8SbsD!)g)c-JJiB%j--p;eaqK5G&+jJkX8Dt~;V9}f=;qdI zz&&*TTg$Xg z{!E=buK@>%>tll|R0Kx2j%{3_>Ua~w{tN>-Sr3|G{e*tAZ$cl-?k|G^#chU*2Mr_Is#C76($uk5G{D^uy56-arSb&YawW7JfUY*jcK= zJihLJu18fkrlR&$JqW%da_-0PXfRlCdLRR~Z#{E@QPs01#(?}S;%}g9Bz~`ipIQN` zp!#5;*YdSX zc@jg}$08MJGYDOLU;zCTnJ&a(ITQrE3)R{0K{ks(;(w2`bv$VQr%D01YBKpo4yfEY zofA*Y*Ywj3lkkl{Wvn-CHrJYC;KL-D>r5NrNQ^QKGS1gOCT+lhYLW1hE|t#f0OCj( zZra1-_q-##lR(yw;Oae-{_lC$Q2IROjxt9qop_TYv-g|s`d+Wox4!~@vK2jns5_<~VT?8w86PxNSt_mdHh;*O&`lw@vs$XK zZ;7(KXZoA86~uRyWP*vCi*wH*e>FM*x=>upM!Ap*Hx=}@h+fQqzG?3oyvIfMT`e8C z0seTdeJpH&6L=+0a_?~F+eh_C&VG3NH#*Dh^^RfQ9J*KW^pRTevkpin^{>Gfkxc~_ zzqK;tmSm}V} z(tUkFE>IF-+KU0VG5`;*ZkYD*R8Z5D-_yC;EPshouLJ$!54H!JOU$+V-J$e__*T8pj z)7|=!&dmy6le~`{sexYKU1d9N5*kjIeRN66SYHjS_)qa_9p#(mJLb9QN_TplXIvEy zd-ss;R>yzo_bh`|J`& zTLO!~y5@0KG2?>l?Q>#dkWNphXF5Lcs~mo=F45l8`C)g6w@tq4skC=Eo2ai+bq6@X ztF^bKz2*(pM&e=zO2iSN)5E5RPY!ijH}Jbta6=+Yf70Kk3v{pkJFxnBXdH9VH`TJ1 zKF6;%pN`~e_+i872_9C~a~-)b{|_rk0R?A?I@VbxCy&y%GqZSZH&F~v^WC8<_c9L2 zLvaOn!tiPBHgzT2i|Fir=+-*_yKFQ>vDC z&dhX9p>9sd9F#+;Xuc28Yt5zCZGh=EiXI}aNv47~;hpd|JY8cT=UH_39wHm1>3suMzSK=Vhh!QLTj5C|~*~S$fXHPti zF8dAo&LN)R?sDfQhiuPw3}xa(P49`GW(PfDzs`HcodxnZ4;|e@;#<;(#tciUEe|GT zDlXbXLu@EMaJ26VEqFH4?pW?j zVRk|RUiBxuO@T4WY|pRc@Nc3&Um|Xh9yGL?H`&T;f<>^bg7?137He}_i!52DM$=jI zUXxS0O<(c@(InQ8SM5ZILo zEqHj4A@8JD`YD}#H6Fjp)Pz^PKfvH03X|lrE0!2D$lhevy1cGz*!kBTtMRYf2Dhz6 zc~;jU{R1`NZ0V$-(o%sILaoA@xQQ8anFc6LmX zss;+!IhE8br5-PZ46@r?-feZCT1s^mI6wxPj^o>U&bn9>6%{DS6i$vu>HTk1zC?Ha zD@x1LsMc-n%P>(dyJosfjt_hD9pUcjuGZeWJ?CBf=;f|v!pBOGs=w*=hJT7rOXE$I z%-$S=Yf$IA*k%Q_L?KHi$2wb9tDRS_$^Q9yEOnQMQW}m&;z-PTb64uPnkb!hr{8 z8u{l@s`yy4{aHlHmx2Spu@i|8kJ5cU#6+EV&Z1853!K~mGCqoH*G1P*R_qQm6%F=v zcC9nkz1&@BPwzeNs-!=07_~`R;0q~;)4@{x2a?-3J;WV$*_H?A_q@#(deP>#tTs(& zuF@b_-~II$aCCl6{0H1Ng`VRUJP*cDFIwOWv?!^(;eG6_0kG;?H4Pw;hgq}D91 zy*2i?=z^O(_ z2-E#tykBpV&gm-YqM`_-!#~*HEPt+)v(M}BvX11(daklon?n?6V6Ae?q>TY6bI?L6uOdObWbN6 zbImQrclCPx2-DP%tD#zJuVJ>a1}};_YoqCXPSWoUq2@xPB+jGSz6aK04cvi9un#NB z&So-Mw-QUutU4mapvl|9HLrwa5l5X{&RRkX+86nXIkQw8{Y!ZroUqTCv18}lf1GOQ zNpc1~lawZT|IAFHL?#kVcK5gk!cy|PoUH8~?kP-ITEg^)zj*8Eg6-E0K>7NG(5v5U zYGe}5GUG8*DI8gowac`bI!T~j$~T>cqo>nP)-RNHF&)H4r(`h|Q!%yM0x}%jkvRA1 z;Dd6hK3MUr2)u-Jm0bHgw9*RCdLB`2C^=W5zW^O{1RRjf@VdIt_6exdj`s1XPB6db zAiDhVMC3ntmNHlAP0yb__o2`4aL-_B&o`cjJgeN}VS42%ztEwG)!op})7ix7IFC#= zEi)xk;gwnpmbcBJ=6d)%^?XK|_8MmD-S89;LFSsh zNA~zQ`S-`vana~5By0aqkNH>I8I@B0j}qzz%ZxfLn7Gs~r;@ za`?fHrLT3EI=c*1VGV4yM*m7qa3j2>FPSBm!-<&-H@1j6MV-)g0M-1X^*|B!;4(^K<_o4`8RN&UHtePSa;eAUxj`K#n_Kd^4* zvwr@}&oe!jCL$_)q2Z)A7qg*;v4HzQr}GC-(x*#Fca% zLAybQHK4B}dFDs^=Jyry%=6%8J%ieF0y%v)(I%9P?gjY9pAyy8iDEZd{gb_SJU_tZ zyy`j09W3OtlssyZM@O`u>iO8K_U_V&4HLkyVHb<@_5VQ!deR7@VQ%HBE6vrMo-#A? z68mPK4Ktl#Zr)hqD~2CnbX}CZqM!amqHYb(eLL#oTPT||x%OnP`4&h}HRn`0&PMTl zo)j%go6^YVD9kcdMY|0q&^CUJLos(^z!4{pm2;DdVcE zT=`XI(D~grJ)ii#qVk@Ho_Pyv=Lz9H=}oG#0-ipZKI^4TRneHccn>L-SW7vx+pZ09_st``${oA~qjwT((jk&>G#azNf%Px!Gl5MFl%goWkY1l-bV4P@p zP5%R~66!<@`bn@Gi(r7&g6^rl1?sb38OY{z)sRUSlcO4Oj6T3Fr4r^j8$dO3xNtro6S9a*? z>1SnfdM4AOTS>*GIy8v*v+p4jjDz>y#0nv2p-xpNKkn}1GT)UAto&)*Q2{@ep-(O7 zyMwP_(+*+>EcN|?cM!o__!i#gd*~0;cQJ#X-{OuYc|&j=3H3!QTLR7aoLA`*g~!CJ z(isCDYUVU^CeJ+=UTunXhb74p%6+Ij+XR@c&9F+B8bS;UIR*ZSPI{3pNjnDgxdLpa zklO1ySE8<+RarS;;CT8*+P79H6Rlw_6GB&yy9 zOB3wk7V{AEW>cGSi17{L=0Bu`;w+(#o-h9FbOV!cwKb5>eo5^93n%_4@1K}U_#fWz zAK?@WM8&V+t=jp~#m^C35j+>3_)co)CZ&Zd>rx_FYxHSAkXrhtuYlNaKhJO{Ozq*n zhxl7@T>1Jw_mYuR`+3BO%Ky1vgn?ZEmBtYw4MX>GEiGPPE-4=T(> z<{*l`W5y9^yx!Em#k9)>IC9~i2_8N#km=7?RGjg@-aiww-{gehbmV+r>h*!*{s{86 z4^%0SD_672yX5fMoWqCEG8FSsT~+p=cUS{rSi+y@vl{b2>5BUPOm*yC&);pOBFn*n zxPhKQCg-B+MfZQs8b`*<8= zCY~qX1WuR1^&aP{27~OV-bzWF6DjyCq!V`&i3DxbiY+|vdMd(d?jWCgR__Q-KUlOn zv#kn-@M0>*bTXC(`U2OWDwSeW{E z$RX|}A9{-H`5|iT3i7!UdJft6#}(mqFhO478v+~r2zN7s$ngg2>R0dMXjz_z_ngg2 zVG@K|t#U<1>MwO&tb})*10I;(cON6Eyj%JIhp7+`fxWBI&BM|M;cMbi7tDnM75C;N`z-ACCy64Tz~ZJy&)FQ$`#9=v z;_B2Z)Np!u=#>uO3~a}dUhSXM^Xw1tcT{IUg?Et3DY*~!_ziF{sv@!#)u&jU=B&<@ zR-Iq+xq}9>$ZkHW4^9I;rQ_U<+HJ36HbsIi32bSuG#O>lRATE}hTUX$dUQ1XaOb?u zo%9I1$T8OAt+N>B?*-851>~TqAVe+X68qSt>sU`}9NJ7q+Q>dNlIN}ITP3QmPYTaD zmv^G#iMiC9#e5d?$>GnF_zdQ~sedbt^IrA;*-6%N7N?tAycGq%ofr_56nd8n@N7~2 ztZ{*-Po=uFsBYwrpl}j`3uC=-^gpY{IA@nEys-LgF9|gc$V2hW; zO`thnka>G>vWldOrB01iSjAy|dN6e#(4zv=x$Zy9r-o-<|35S;vG4lV(92WZf?yNz zspP+9?CCxHX$gJH_5c6=7IBX%kA5eArxx_DnR{yIo;)C^w?NVr_743!qQ8}TN}cF5 zk54ff5M-il5f2k)yHww%`hm;PchBqjc@b|kgSVK>yG-F%nMCMfo(nF#tWS~e ztojqp;rndiJ(lxP{fXjOiwfOU+`-TUL@{IRBk=+bT|2}j;v-@Q2+IeHl5ofU4ozmGaiWAVy5&Et8W8!-xo|3j}mOSqWyyB%unP$nRFY~+6fO+ z)^dBF?{gt>=Qt4vjxP6FLrpS^9j-cwrgD#Trum?lT_40emZxN*Y}PV z^u4>xzPGoi52LCiW|wp2dBjS*g^7yA#PtHA`#yfxOym*SkptKRVSGG9AtP^6#kz@! z+8w%I@XpiekLe#o2RYSnihcDD)?kA88FS$hSbN`r6zsx9=LX$Vg(!1{iX@$Dw(=h4 z@XmiCDyiPg1$;7jiwZke^)mXY?=)1Wu@_JSt3G1*ShEJV5Fs3WXOJ2faWv*F*74cS z9SRB%NlJ4F8H-z)NbPZ_skV~n#m0Y8 zaL)`W#RKTW(32sr*$S*>mTl%-(!e?h_bGHUo3v^j&7>E0>ebji#)pXig7a@Z_S z39&@Hr-%-};2Q9S_zn}eCmRpa0XbpHH{Pdzgb6`K{?%kbLwJ58d20tanfCCERcz{K zPTpp@8O(MbuEqtdv|=1NyL>Nt9{@`m?hbd)z*Xr8Slb5lsA;`#ciB2)Ixlx8xZ(q+ z^)o}KMhzLTHo7I!9_F{6VBJI;9|RGZXbJ_vfs79sNrO+ha=yj)+XPy>9qIyd+LBtT5+Uv%v-L)n{l^{jYnM(<7<2H}?77mf*WG z$cgpyi)FYJRjnHJwv2LR0;I|$$ZIHIZnxof@a&!oq!3pH`YNu! z!gbsvsOPLv2g{{z_k2ZdbI|(^(@tkH6W4BE!SgxRb*!_n;~JhfsU5#{X4#*Ve~@m6 zwng6^JpLZ{z|zQ+kdI9R4U_d3^1o4@>ZHIEF^^8DE#ujTf207;9N# zZL#Dqxp*6xrYInSyI=QrGUzb)7|>=R0P1kN2K3R>n-a zZ^Wp^5t{~g*vhqq@=5Qfs12T`{!Z1(%2H37BbC|c(eirj2Sk?)TWmgn7xC{OxMx`#Wfm}_3z6RY_Z)qMyugdT8DRZ?z}sf1@Qwa$cI-z|g)U2@A*(!~Vd6TDuhp=>txVz(Sj(fC zV^&4JR*i$eUa!gRC2}P>m$)Xn=elC)ByQ|X?p%$#)zyxn9jESY?0nvvWSu%Bc>k2q z58l@{&=#^m=u*aenjG}MJxxl9-z{JCTy-vVe(QT$Xtl^;JNoVKH#>ZJNSAq(aiMer zB>OWmfvTWeewA+UK<~G537nC?=`F^CR7nr(wK#(8=WWN4y(fVf)cgM+51}vjsV`0; zbD>Tt39j)Qg?AW zf)uW{k-{2^P??fNNF}FLWb9YB-M`4Gz{nQ=jt&8oSD!U@vMs6B2BC&DUkpAmT zYc#Si2amV0-LJXO#xmRO6W>f+bDj&XFuoRA9X=zxG<;(CdD|J|1z5}Vq=Gf|#T)Uk0y|3q$V<-deF?;y@SAA9EnhnR%$%82rQ8*pS^9x}q1fmZT3(+&>f_-usfJ{opR zv@8ZL=yn^1Tas+0wq(n1(k-TRjrYFl%Y!w!hiMaVe?U`?%IoOKrI7_?5(k=?DSgJ3 z;B0YZz#NcqZAt0M?uzLCtyAhKx*h&o$8RTk%FWZ`qsMiPbtg=ZJrn8`Cnyj{->g8zh75QaYKD|wv% z97>LXlFRV68TGyOXPm4G;S>K}Y4ul-L05w=#lw;s$ux)}R)2CJpSSQHOoy$g>u)m= z{XG8fNv^cs)150iv>n6S3+|k0OTPI{$4k0FvD3$Njq4iij@uCNXG5Gn#j&wt;LrL5TSItb#FWS>k>z27OtXS#Jg3}+@?h;8@innY^EFqwhKU+l zVU^TsEVSfWW*ezugVDI?#o;BmhDx5P&FJZHf1pcu9kh)811Qup@_5fKXOp7`?}dvv zP-OP5=zgzjSoeor8#}V@9JzV9ZL4d4*v$Lul1r0n;&((1G9FeW*QoB{?P;AS+^TQZ z%D`>B&fjBx*KgLD5!I1(cpycDb(!O}$2_;4N4%xMMD|b;(dQQLuNvL>)8Z+EV5zp& znjesc2M>|^;Ux(>V+_5-cCzyf(Bozt8%pqcjq^X_yY239Nu2xFTr*%S4J8H^_N?wb z*mIY8?x*g0e;ax;(HS06c3>kAsXr#yUZ>L7ZsnvUFK?oBl3&G`Y$) zyiBYxtIM>+SI$>Ar++B<@$)G7|BK(oQu?iv+&y@y>?Nk#dvolKy}Nrp?*6!YZ&yM4 zwL7DF*BOTl8!JY*LAJ^?b`?M{KYHxn|uGv zhsGxc<0||8Ah|pdJ>RrDJ9aoDQKl^pzU8TRp7i2WYpf2N+W&6 zrr@mwS#Km(pH&_0d3%?%gNzw?R7^BJE4HIRJ`6L*5v*siMs5&w^u$>#)ks+aNPD78=N_IDoZ{Gl_kqvv+`ooZ*5E%d(TF_Rvsxwj?in5o5o z*OAbb*=e^khg0RsyQ}Q?d9DQB)DI15ifD+s9$6FKWUkkJMbv9YDHS8C^F`IU92;@Q zM{}heVQ4mwGT+qSMoXruX7=%R72PH3n6oI=c7w(319fh~^#F}3Yk7ww#)+Ghqs^gZ zri9&oxjU`P-SeG7KK+lQLYmEu@N23**^xhKoM))DS_uy=L$Co4D0`3eB*ts z!?xZtq<@ondtf0IMU(wO?|P7oPH!a)rx$!<&^1mmZVFuz*%GxiBEc5WKY^=4sklqH zf%CNnl{-5IuI4^@JvrVPX<%R1=ObZ_t`OXNjaCvnr~ucaKtsW%Y?MEtV?4=w%=0oX zRN45A{m?D+eAo-S4_0zP@3HQSUDrCI?)G&2>fLW^iVcl--@9Q*dCa7+Z=?q0qI)Ym zgpuUgg52X;=-TT3(Hn`1s>VD!ye6_Tazrs^+CuZd+sG1_QZ>Gf=4;ztYm z_Bp~XD!Kdh4@q0;Xj-YIFX>j{YCn`|ar=$#SML)Y?z>w04i`3GGd= zXE@sIqk2E=THU#^J@u}(^Q5=cJb%EBxbgSyh?_B}HT;;d7>)fa-orA=6c{^yq}Ds) z+2G#j8y|Y^E$R0|Xsk78{GQIhKR|Flpz_~^#_wzS2<(L}9O^Eh6Zwhrq(^!{ zdP>+2#<75GsE*1%4t>E<|JV2|O~GYrISigur@QaXPVGtU4dQ0i+$;AC>%NL3ZA$yw zJsXvL^Mg@K293BUI@S`is$aV08v2z;7;tp|nO3};dbO7s9%H>}fgQpiV}&Kz_M6p> z@9YTkgT|NPqf7#)ND}f<5`IokD2~cD9==o^vq70N!j)C?b3OG;6D<8EJkY7)$-1}U z1Gn7$vNMAx_jdQju8+G$@ci0(M)gMYzTFeiozS@r-E!GZDeWa6$mvY6b6GgsXiuUh{QuW$Jk=B}dQZB~lEG~3EnwyS&>h+{ zqGxi?r0yBr>$*pEU+fNdZbqYY*1REPVn1B(`duU2&!I9I0b?uF+-aDiUx;I2Z6Kc2 zGMYI$<)}!F^deH|RORv=*T8m(5*JGUlC-enRcvQ4DpT};6;p#ss(!l7pa z_+BKq-hSP3;U73Ry(Qg)E6@kz(Q3zdi&g_tmLNQZqmh>RC%+SB>Dt+|XxTWSw`*&` z4u|4LeHOL5>ayPmWA74<_c5GjSKwN-!?VzUuBC(O)`2fqf>4%#=vg^g)UMQauX$wQ-U_P!f3piztiQMq@uza+U#Ib})%i1P_|pa4@vOd1 zfI9ti4ZVk5ygwDWRi`G7;iL8}_px78#!NNd>a`jxP8H18g{x!*or>fN<3Y1Epo&dg ztB3nkYd>}Bz;U=t>Ab50^rO}UbLoK`VkI*{OZbAyGCPpZ8F?!(8a~%Af#=Xpd0Z_h;x(4-H23%YtZ*&ZQ zV-M;v6zGgO1D?)1sONnsxB-np!!-^Sd8Sh@4q^Tm9{ES~APK1?!~N`ECa$}Tj6M)dJ%Jn3XQt9w`h*SV9CJZV(n zxDzqKl{beL6+~Rsvh3t;6X=Gh?{GBNz7i#*$ak7Y*g%0B8?vj#0^yiK_`p1 zA5W)X4Qsrcnlc?0Ao+^_B_j5Bbe~Ez73}|=T9GRf`LC*Ruv3l@YDFQRLCP z<@XifKErkMw0QCIgw)xSzwm?#`PB_NQAxbNcKVeWTx&UdHyd73DmbSdCY}IZZ&%cw zbOsF0Y~1PICR6$o9I6-LTuh+i4Z_sO#febCRb(jYK^4J22;;?#bABGYuIb=uW_l7! z;iZ*|Pm^I(v#YlANx*GnAL}$77TrA7GafsA(>C(yKB+1@b>`&_o>(nvmE)Y0oB6&c znb2f&&v0qMuT#*Bq4IG`PMdSMv%-<@m``??gr5Fh{a!<+snVQcNrdGTg;HRbZGK3S ztp--E6Kv-ftvdDbS@xB>i<$$2{VKa;BRZoNKXW`Z#bnq|!7%>_FRTtRgsEKX;JCa1 zPw+8mtxylIXfdqpeStdvNM)MuLGSyXDR`m{hYLChk2Wncp_ZWXIqO^tGkY?#04};; zc27kul3?HD7~)%iCd6PYG$p{|FjxxU){KHtR&A{cnH*AJnPI%f_c#m=ru6lJ7Vu`8 zz|BX%KT=)qM3mIe3Xe(C^)CHXc*LLKzyBY@H^vC~fGwu6U{_k9iT=i6_`wtW8?Ooj#YSI>S|wY{ZhqkX)uhRUN0 z_HUL&FrP6tqk;;B>)C2GgpCMGvem-8ehw{bBh{Sh@s&ml$ik_~2P5ir96i$lHOzK< zOnjKD{Eg20aM2`vqmRHp+GY)!_Zlxt!*T0c48wLcy_;xO)ia#fZ+Loms|#IG_zzEp z-FcJlN`iZr3%u8L!ad3x;d|42)t!I?>DZnrJylE*_$R)rZRR|3u)W6J#&vKaWs?mD z)=go*g_K%44Nr(kTJ_vmi9V`@GjtER*^|EUsQM;)BXI4r;>xiQzq}si{k#UldA@#< zaih7_l3^KUI;np`+^1zaWH2v)_JI@mYk8yZg+3>~h3=C$*Us?%=zX2c<9RQt8ut?S zB)DLwsWgv~Z(d|Q;)UGPWS{6u(~d{^R&U9%O$xz0xW zAaIHej&;6$cmQRX(&032G`_F@7zJUyDcd?ZbXC}~&^l|b=_Bb`^s(8T5_pKgHyG)E z2Bi91?=gHtElL(T{c`qAnRt#4))&Ha%tZXzAefV2$Oxv_^rdLRG`Nn{63>Wh_?IvP z(K`w5;wk(L2f^Vg1n1p|c4-FFEA9cw{1a^D@6lwu!SCn0?mAZ6YkFHz7A6I52saI@ zVFL~^U54@f7H)?t%yVs{P@!$L?KKZGd?XCeW^odyGdGH^Hj0Hquvo{?3ppyMG2LW6 z%)GVC$=c5je@Za%=HbrLa~x&rZ2i{nW$M*J}e4hqD3BRL?4`@=V5t$j{n1PaEl^%mp7>%J7K#{!E^X^c(AX~ zIeeeKrPmqd+z*a%*^`A&&->DJW0on}7=ZQjfYhnqXv(r~#i2dH>Smt8zwn|K>2%ca z^HF*o%|w>FFxm1Ik!}sK8xKKsiq<7Ew-nt_T*L8%HTdXDd?an4TU&!tq>eLZWw4BQ zpR0T>4?(Mm(h3guSo**b;6Mf<)t|lJz^0svVq`R|oG$S65VTh3UBg{5uH~+Uo^R=k z>=I7t)q3UuTy3AGqcFpeiSzFco7VcaNoVLl3*zCaMNvZ!$CGanXMGcybR3){Bi+SD zO%*w)28Lx0=k@|!l=u(nGAx_F=&y>uzzxFXg*BeTd7g>)-e4;5&zTS>)0?u$(M&s= z#%_8M_uvP;{osI}=84{f`LvNc`58Xc2XI+uxGRZSWw7nJ)PRx(_}HWN$mQWx9U} z*Qk`9f?tMfJy~5XRhyA{Rcm01W>8^03EHRn#||c;^af)F{N}x;9}POamM64s@6)f621?8En!iu~H!9KPf&%M(0@s(xlRUuH4&d$1Ay>IU-U-r0Hdjh4 zh}2zzcTNYQCwUsB>|&rLJ5w%t+?)ZhJuWbD(8h$ur^I>Gm0Hm&zRUE(88Auy%{qPpmhTSuAp6OW6UgQcP!F}! z^GQG>HDu21{-MDwur$)?!_?qqQh^Ub9xA#_vQTvn*#*Al0QR_=1tj6b zqt4E&ZdlPtEQpG*vYjpmku?1B6-Ian6{~Kj1q!Is{ZZ%JXTQ* zxkwRAqdew$Q+I(}q@n~=IfGTyT5;@il_k9<-~rFsLe+DNQ=nM09i3Vw(I69CqJ+9% zeV!#K1Rd0&>b$O5x+JQrEg&s}@%XR8gZQUl7Qa(HnZvl_0a|sncdIHss)1&{OFaz3 zNHjuK!A#s+%7Zs>?$i(+;oxxxXwSIQuj$0&SE~6`i2>g&!Wn5Nl~Wj}_W(Q$%eYGq z|60K*vlz$no%l<_EW@Lt#m`g+oJQ6&lOEp}cF$0JS+jAn9WGP^Ta|WAg05PV!JXY; zUc}izx0ZQ_c%M~*mEQ`Kz`bnYl>R`>)xPI_7Dt{5nj)qoOJWA=IR}Jb0p4Mkboq1^ z!dPSIPg(ER;di#@y3|m|H^L%N@MObjD-g!} z-NL!MoyfHk7TX$D(O_I>L^K{FsgDyuxFV?m3zhxB1-e6c9`T&;8~jSSDlQRRoef{g3x|TTW@s0~2CdMla~;B1&liLUp2#EOEF5G9XkeQ#%l13mY=*<~ zXvaq~&VN-ruH3@A2G3N~?t8#({-QtbAM1NsdJb>837WOcbZ5RJPd+2KO8 zIO<;{1eFraB{&((YsLwtA&`Uv=^Ne~LM$l67R_ko7hM**na>ra&r`IGnIQDBIoE6b zl6c*pi4WQ?ag0WgyS3@O|Ml7k-xZ-!`@QcWF_jLzN7q81C=>6(eVX;0lo|Z9!wBih`_#42At+ifIYtfHA<80$6Lm_zo9R_{%RCAdspth2(gsMIc# zFA0tQD*R(JISF5o$q{gntK_6e<=lDCG}Uul;}u)+8+=YoMBQD3Rx^kf-M@vPubbNF zb>AA?9A!B!a8lpjyNKD$H@%C3S0%Tc8`vaP;8#=`oFj~reFip`Js)#P> ztguS-%P|%Psyh`Uv=a*Tf@U{w5Xc!}_~&GM$lmflA>;*aRh98#85bK3>n) zyp9WXnC^SLcb{dZxK{Y1?}n}dPvDhQAvt*G{=`(m)xIC_NEQ7O{(^6K8qL4LDZI_| zz2<%noy=a}t-!;2ltTXZ3_p7{xCQ>`TSL_QML6T>fyetg-rO6+Ue6ZoXW{`AgHK7Z z!54i_JV%fF4=C%58|B0PT{s^8L4FMP!vpeV2}X+FiQm~x&na9Z|LTc0uGb{F;A&{6 zdp~2ZrGZ8+52%+tNfzO2=G0!{K%TC zP!fFJciQj=ISiMvX5|ujLYw?7oKlczOd)kBn@pAwvIc-igI3^C;S&vZqXdVSuRfdjPHH@bvVcW5@VTRwpd3zX7$7hFUTKA(MqK-!LMe6ZLnn= z^s*~ZQITzsQSZ|I*Sl0-qUrB_&(uuybX(XNctKt+Hjr;Fm)yP_@)KOWs0P}U--%g% zkG4L*R9hH!8-4#Z3?L8n3v0*~wz6-JYASVM{;I$wT@mhMbC?Yu@5l9xz1RZs`iU~a z@Uf>1Z@)y{5V?{l@~tn1oOz_uAzI;KPBy%aztCp$+g^)&1Xt1Gz#5eQt@6Kd32asH z*{^m}|_im3$DOf=sAtHJ4R)Gm>}_f-a#OZX+hfbR6} z5O2^S|Aoo3z1}3|APmBL&usYBo9j18*ELf;Ug?7Fq^CwZ-7v-5=pS!r!T)eOo{%5A zvo+HVBjDe?j#uY&?@-HRZ>thvn(R#tykYp9K4FaI87fJ;WvuTIZ=4wbxB!199FoJd zN5RCmGoPjap34fw!MT22-U^&@RWxcg%1@II)!?6R~K=%`PNr)u& zB@6i11t#O4I7Bu{8QP)FBc=<%rNo~tnu)%F5^lYCbth_naE~y5g)7r}sVcbIXVy1p z2FdgF@%VHw4OurvLGy{X@-A55*WBM5QiC(xn~nPdzk0W_ik|l^XQm~dM8xV6=BKWZ z3pC4xkxEkVU8zaV%1S89)nJSCu>@$VNG1>f~7#QXC+>u|qr zqHiB@z6pI`h@mX_p|1+ZNT#vSJz?rK`9~tW^DBYPxTsbJC;FnqbGjO8n|Gu`ip6)= zRP6uSx7s+8xdH2$Gmy%6JgjN<@5D2@8&AYy-DtdkR_gLul}@1oF8v+-q_ZQ4<{HU$o=YSOP}+9lf3{e9uL?49gwNz+}z$Dl7^`lCtiQ2jQ4E9ww=vE3> zG=Y34o1CwXXnBR)Rqc=A8$<6(Wv8y8+f&CgQ+q7xzf(Cn8T8TikUuE=?ks4%gYTS4 z^teT|$s*^q2J^Y=a?}v}dE(vFd-d8#IGkDJHcfso91!PAeE($X`E07?Zu*p!WWZ|2 zvX;87mYt+dJ37u?rP3`C!KiAe;y7ifMjh05sshEsbv1#sYUnJc(=%l<1z6H0I(F&g zT-8)esp^b*DjZy0xywuHcR=}(KtFUOUp1ENDy9lO$Jd)fZCXpGjVI2Z;`4$7fx=g7 z=le4C9cRigvJMZ=H;yY_$@QR4rQ>#muawD?O9DxyR^WYAlAlz84Q6s*xqbP^1#*=b zzOFi@p0}ZX4XU$9oLkiYqwuW1<2ptDjXEuzDFi&}MmlX7bl5m8$o!0~nQH!hGZhTe zN4VErJl%Zytr@(-3cB!XsIXoF<6*7_PhG9ijr1N~;_vOCJ66eF$f28L9lCBlT6n_ebgH^8i9bt%tx(3_tKsTO zxQ9&EKoXjwI=X)2RF*x87ge?YOrNs(0I1M9khy4F6Nll#5dl*42h=;$Ko+CreM}*q zMU9g{C2&W72Da5P*z*PEEOVo2ohgX2*9SMo4^kTgd%hgzS~rZ38*pKY;Uw(m+7g5k z-7Pxl>#1sL$&@aj4BJTr5dB5)ccOi7z~H%xCrpYn(W!NoIhHwE9UGiioy(oIIGlCi z&UMz2?y~q&wci-dn!~NQTH$ilj8Zy=DNr%iWtN`}8BAeNwPC2=c%G_0>nzm`u8T}D z*(5!r?}3G7*1yXnuMU({s$bzT{Xc~~B1ALHh&5n59=d6=ywUd-y^1m5-rGdtdR!2%(ngA<1y|Yiwn3h8ob9^w=sx zDy;MM?P!2L^%cPKsKc!)60Eub$A@N^VQZ!9rV2ccmf;N0Y3MYJFiyesC}4O*TCd4h zM#K8cg(DK*$I%F*56RRp;xN-DbjeifZske39R+j%rpaS*iChOyY+3h2ctzXn(e|94 zy`5Lv*R^}Q4DtwbUF6(od(_m3=CCE9^FwxpTnstQ+`tA?Y%t2Tj9HJd?n!uJWKnYr zpcjHGsQdmco2zo;}qyjx&=P?tKv~3)WMT@Pj^R9B+C^e@<5q zZZaBm`vAP{zVb=#*52sugszQUQ@WdaOehG-9oKuW_hk3-ByrsO(pP94+dnjR|DYw& zW&Ngx%B8s^I-`e%aHmgrs*-!N3#c0>${S{q(z zjW*q7*m_u&2Y^A^ZyOjcL|Dm+`7OK+l-Uauc@WsGoLvoC<71`Y$@CH@u4Zv0CV-DR$; zba>CelfLRKcP?R1$2v0{sZ1U&vKRJrb-TQ92O=j8+%%v%GAi`0aXGWy{>ps6GnFlm2K2ByK2ZGho%R=X7~ z+Er9usXBEMs+O2pN4494W<4H-pXxH2L|%{Ho37b0-+%c0NDpE%o#+O3A5$WUmTQ^Y zSlOG>JF#bD*LC|M?e%_x?y0=TFkpOGjAa;cdZ9?(^Nb&ROF0sIm7{$CmY98yaoChOf>~_|h=tn|WDY=jQHH-Q~^!%q1FPoM=1` z_i~|hf(fLdmMB}bt|Sr1^1c1|7spnhd4@lqrgNY}e=zHo=Qr&uLUa76+mYw(-ZEstlc^o;kz6KjymEMTp6d<#jktMxTB=9s6j&} z-4hZ0TIg8QHPM4+md+k0(RTj>o@Kp9di0(`jTu!*mSvRb@A{88ANv{J!;_@Q9AS!K z0z()2S!;l~E8rB1$+yYL$>+;ZQ@+N|ZQ{8e@;C7o_K^?ek(aJ#o#!d*@gWD1fpb*D zO1iGp`ETGYXoP9|I-1z4oTsI(DrXFzq0UsNm#Lq6`ogn4Gh79BuVZL%S4i8SxkJVe zl*1+(7c&P)WH%?{Ip+v|B``l6Er(sMKY6h_5 zn{e)KCk8aqDQUpfd^`P!wZau5pOGmLsq{&x8&FKqhX|admZ_oNgB}fZs@tzR+3jPU z#M5{31nw#~eV0A`@G@dPCCrOzrqm{)PTSxLcTdILI>FWGoR1IdMR&S(Vc7M-(Q#V` zT(foRpQR7=3u`1(yPxTIFZi7NyQQ8y&0a$+?kIxsX~`x&CH=>6hM9!{!&3bvT}B`j zcZV2G(>Y*E)&GyC^MR|nzW4w6|A%wn9R3L;h9;&gnYwhv95XUz+{PMpu~8#OO&fO6 zi%wlO^(Rf8x?;*zrp#D*iz!noa;!09jTt(1=+J2s69eVXIlzH);QT-5_k5po>pU<97v5Q-eNt!sP2v&Nj3Qx{)DX`*qE?@D~pfjXa63 zu#-w&C&3Z5qYZTv?OsMifPtD$FTmfJ0Wli^Vgo)69He4gGIw=5^k87TW@?~lh!U*8 z7ScsFc0V?KyrS$&ZYZ7jtDB&Rdnfm@*3WS3Wlxq&Bu~b8^q|9x;zj;3US&1W1`m0c zdHxzmF}9@GrZuM}S}*I`IAz$iq1#35a$R_buVdoQL~-C-nhh~6<}%X*$SOu6f!A{L z{z7+G_YLh)<;LJ)^1K?utyEY+PQYqVL}i_gcuEB~08cU={I`Pz_)HYUk2a&_fY6KQ zcr$*KSrHvcu)sReHt@-zz2=78WJG`8ORWkxchSV&iEJXWBxdXx&xq%H1!#v73%ef;+m2S+NMsf5P;-siM?4R{PT&vz z|0Jg79{N8)jN&sO1vhcmW>IxKktl#IL^<=A3pB8E4mt{#>v56=*wl)$(u{ zS>_GuWF%{9zznE`=3A(wX&~bJ1@#NGAs%r-jaFd4eHIV*LU`5N!KbK?1ezs&;3A$V z#8%mZ_2vztS~I264_y0Q+D!?3!K^+45$YnRAiV1xFz`O{{L}kAqB9P0<^@lz z6Z~_Dq5ozek?gmm;1BiH39q6_Of%oqc4I;6QP!!`i1NQqy@b)g6z{*hnTjd8E~AsW z-7jlfz(09ja|P6i71}e}pt=%U&JF6RmXX2MNH)<%qR-P6`>~aM7wj648L)**+BsL((ql4 z0*gKoEb#a?tRFi9fA_k5-Qn+Q=SiHX;di>PYZ}#SiR^e!>($zQD8*~mNDM-{;wZ7n*-+q4a)RRHc%Bed`JhBEGR2q>M3y2_~!V3}5^T8KuC*o$5yB0gM zNJMa+Vq!qj1D}KVDM%gvDM%;K?8`(p-u2EW_GV!uW~38}-olMsMbyQ4q_*97m|J5W zO1hTZXzei8YR)Um;qG9VaLTdqgpd2*^8VFdpm>hR!YxLN;ijgYNS3u|T(v}7T~Sl( ziCgyVfQO9WbZnbBpg2=)TX9`=me{dsO|^;|r_8GdIWnCLmg!2+hR6(-IFB^qO9rH3 zAC@<6FFqaMX55OstvCpFI~gfEk%G#JXelI0?Ih>*UFJ!h;lOSpi~dO@+(M${c7TlX z3ts=_+wD*1Ti++5@$JAKaxV_Ex^r16YMKTA6le`C)9;FRC+)K4$Gk)=oCCgJiXG@A zR>{rbq~M5mwl6)DtbUI8=6~xGbbls7>)Y(qAn}QU01mb{wM0s>oJd4`2ladR2R&dH z=BujVt5?`j@w~T0=Glm?OM`W#gV+K*vsn3v?@}hAr}Sep%O|R8C;2~EMMHatd*6cg zpMqDYgqYz{cu*S>RV6b$&P)ZT&=1O)3C;FLM5`SlUhPLjf=QNWWdsecD}qNlKUj!_ zTLJ>W4*Vxtb3=TGwZ+_J_>FcaHAzbF(MXj0PGS#UL9?jvH;1pQhnVSCiM3n+mP$1m zSiJVv+IX#ztg?KfZL5%xsVIX)ksbJ6VH1{+Indu5 z@XkhZRvO4mYFA|OTa&7u$Sy44^ebpFE8+K5-1QyoY8g~4`0h8bq6iw{LZXQe2mU~w ztW+oe00iU%@SCu2lo$9XREz25{u89JTX5Moh~Ik5Z=l9zwb>NE!ZOYDj(!lo!E%)X z`8cg^`OhW$DV>lM~>ks-Amfds%N3ss$ds#?M`BqN|2n2 z$kEA%6OEDYHm0$YxmgYWOyUf7v18axiMcAJmQ{VoLtM#GteXmaBvw4~WO~Atq&nX< zB664s{3{(kbc50EhZf-EXdo_XVo1~_mDbo{!JT}aD8JXS8Tdg_k!Ze0h)oRp5?Kqu zg5CkzkQL0MIiW33-*R=Esm}U@r8nk;{u|nE#)LFUbj@+HYU)FN@bBOC=ZDX#wh^uP zb&zdmv9r`8*L(=F=~eCX>LTvFPr0E$t0&Shl{~9vu(U;fj8|XR{FXJ^gPck14*PeK z2%vmq_zGTy?AKP#Y&MqE9Iyr(z=2xN-MWqllmc?FGeezND8p2@*@)k-6*RB+6K`x0&%yY6L9|Zd8@W z3|f|1Hkei#E^F(+Q<_2VE!g$RvY}?)q<5;{6U-y)|2EWH$9!MIdi6KWIW5&ebbr%W z@vIz1sxM;w&qK$HA_#0~1gdY`383T)47SK{3$ANoe5jL_?NgecZu} z4}e$bpj8 zE_IC5)ZCHC3Gkixw2r95m%%|w2J7iZ@T>*Y#4>QR7ejUVoRChm&Q4SUL0U~hhxs-@Ssi}Ja73L+Fx#7UB>5LM`>;B7F6OvXGN z&)-S(y4_fWZ!p{G&~q<4R)N*+WB5|oFQ~8*Q7M?n)43fE^8a$4>_QLkhYJ20ctiP0 zOh){&_?Kfh7(=?-*afHY)52-$P?A9fuF=29qy*s%yWodNr2El8cd7qI_Q{X+3Hn{y z)#`jA`=mYM0pP_ap`35hkAuvhZlzIOP28o6iuNF`BMY?(GR>f1Q8jVi(k?yOyUC6Rq!S8Fr>r;UbW)V5=Hl%0b#Nj^0#6U=% z+5oj#6i6lTTXyaO_LR-Ms?a%(;<>VO0ws6yC>DP~6WoI@ko;J(d^*^*Yn-)Qq|Du{ zpWx&^&Yqeh+_;MwRd6>^ZJ9Zqhe|z>JGBJuESDR6G|-CQ^)NNrG|=Wj#bW)FmQ4xO zmR!?316jD_NXKfg6XUEUUb&HR+3e|`$@$z>l)_@s#2Ok%H- z;1@eXM9HH)OO{|EdciVXa_mD1*Wz!1hx0OW>Mrd=;EbQv{|C4?o$RLMBS}rB0(92m zz*KNc{{U7|BeeTxB!4U1(t!L_LwxW>P{OR#VkkydLGr`ri^PfzAxw-;8&czDY+Qmx zl|!US9ek*cI2jkS7L>|s#KG^u&su|3RujquUFl8ePHG8B#%~_f*o8Dfoj&|p?{Zt2 zq3DIow}qMe+2OJ2z`4q9~|*@u0^G8Kltp(!@STW-f@oBZIf{e&EqLU8wIgV%ji zXHbGf4E9<#n#6uZFDOid?l^GdZUmA-)M7+V?F(&2f_EtUz^(m|3NZhH#o{S;)r!rj zg-nNT^s7{)owpg)ac;hU;4!-E+d%GmIX9h*{uVrRKdorsKAVvsy7<9TO0>k;h>va# zI;jEQlL@qipMWTh&4qlZw|qaqgH-}wdXu~RSu7;i;9dCcRrl&D@eSt3fC>lRdz-Nq z%jY7KA1U^$+5q|BM6BZDCJr}P?*BJ=EWbe8naMeKL-FO3H;u3RAfCo6NDbe>X7o2= zqKQT2HW1vX60i~92>umR>_3pH@qzCc*d33P`VuXSd^iIQ{T?#Jw~@mT3;!}=V~f{ZQuX79UQNW0Af8r1CHu+H zTaBeQi#ym1&nO^T`$ICeyyU-qjR^TGRKK&5eMuZI^4?)=PEWxhC1a}tn+jD@p@{$| zp*Lb-S&eTv6A7HUmq-x#Sn=?>VB1^-szEdH^O$E)5`W zIncWE$kEOQHL(XhY7@Sq3rN*}*1P~(<|TBiD|nK77|}JxBbl1j++B9~cr$j023B-G z{_1vmrP2{25&t3RFYrAB^EJTtrAp6Mp6@1S{2C*gh7ChJvz=IIB(M7@+4G!PUaM%S z3tvM4IlF>T>froi{X@!a^*FhtatzE^xjkME7Yn!>{$SUFQIyk>Py@n_DKub+L z(aJZfSV=*eJ<9(jBfN^wTA5#f{VInh`q4jkl9g^nl7Ak(r5dUJI=NTPUTz-=%3i(@NTO_+(vLvSSoIGod?0Gk1xPsmn#pAs~ zI~tyO7)maNhCxqZL~pZN&AeJ7KPUN1UQ)+nt5}mc?J~|x+7z@}!FZKd_y4siZC_yq zf?(XtoMm^K`HAjoJ^!y^UbXC!Waw|@CG)D|C9|z#{~B0($yXP|;7hdB$IYdox9RjL zbDK~9d68E><1S!CWsIwmpK^Xm_!)kO6upT9vaiz`p~m`T+m-NQ4Zt?2hcz+5cNpSy%aXTjbkU_?}b_ zk^VASulbxi$=#RHq2wCe;&Yrez9iXg_|N%0LwetY) zw$0G|IwYC}%CoGaf|{q(#L7(dTn8~iTai!$5;et{S%pks1*OEwdPZy)jI$drX$_I+ z9Yi|`xmaLS#UHhmahZ{LMP3--v zW+5@Ig3njOv1-`2!brYRA+(sseKeN}&S2bfH@8RTEOK)d)Y8WYTxdz1Xve~{p2Z(B zU0F`fNi#mY=V`5)m7Y$2BB=@13Gr3zU{@%OJ!s~&l|5X;%`u;SIm@0dfd0w!VQ1zN zwbf1UX2vHwm5+4t6k`($QmXO|{&I!RhN~F=Tt1~HHNEsO6Nz-Y1)rJ2TCAdf4J*2i z_uH5gIxTcEH$u6!+$RfI%RNX)sq94qoS&Lte4a~d$JwhUMt78_PeRkeeI(vYK`+8T zq&KM+xRSB#$8IP5mio1MRaWa%O&Pa*@aGe=qX!%#yZqq`Dm z)ynyisQh}yxr+Vv@C^kKY=XIx2KTN$9m734;_gq>p{vksr*JhpLwyp)FTD)#jm@m3NC+;T8Q{Fi%_^K#K?|}navKP@mo@~OMtHDrXNk3J zr>(P*XN6PC{z?Yl74}hXUAe=-H)e(pGuN_+v}omfvNLlSj|MuDTRhsEaBR7ggtj%2=jAMB@-nl!a^?q^vCOE5{x9&XR3@2A+da%q{+2%FOlczb zC~>X)rh#%VFnz3fhx-2vV3q-j~sfE=Ghk-<8>< z(~qEZ$<8G~Z{2)L5xf6urSsCHOoHe;EMA~U# zG$L1vrB31ogmbjgoA^Ro*(W(of-aVh{L#u20Y0f=uc@3&yVE&;HGD3*tw`gI{MOD) z#oy6In`@AIsdI|_pgH85V=8g zibm$h8|@qG6mnC&w9zp6PdV(ISUq?)al7(_lqMVmAEgFi1yHqaHW;J zTgQmyTbYr*q8gTHK_ZD%MkK3Bp4mz7eNbK|ZBf-eLVdzvgxcl)>4~gxJ!3E68R1_o ztp0xZiIu#Fo$RTLHR^*FGg)tuX9Gxiok+w78T)D^&~?n|VB{Mu$dlcn<;>#^o*;Lf z6%b9IoOx(~dU(nTtlW#BgxrW^RY)JJki&$wh0BY4q=7Qz6o}R;lp^wpov}%V%Ciy6 zR~xMfYC<045m{FFoSY|-5UThQPS?W7UGyS+I*)h6l3`$_MedV1QJa`G+|TL~)yR3k zmrjcT#+Jlif-+RjKGrgxt+Xv;%Y|l|_)Knf$*ZyR?WlYvd_im_`OI2;Z(?5&KC}u- zly-y;8d(|Y=U@>n<*W$ZtzsNzP83;oj4U@IX$uBf6|_5@R+899p^sd?*F*dZ_GWsq zGQuWkzlA5RGZriJ?chiD*#ONHVxM_~yt4Blxf6sdiv$gJ2e*Xmx^Q@zyKp2kGaz3Q zJx-)IS!XLP$h{;tzt|Z(tgY0UspfOhY{AFo+j4fn%SNWh=f=KDZgDlIvW?F+^Qmz7 zK4#p5X7@1XMC4!5f`w|!Sk-pg64_QViFU$SMM@AkS1P|WvFpM&kFySPn>Et1@EZr7 zsQrvYq@X=KlgAuHwid0hn`fTo6jaipnRc(xANnt+K3$p56QZY9@vrzmJgnq-IE$Ur z{FLmjSn-uuG_W1kFpjzGd@*z{cQrn7o~Vt`CUnE-h35)ydx6i$L}K>B^MpT&&RGc6 ziWG2`zcj2w3v{s)3cW(lt@DI8-bBbf`A?4&PpvY`Vz>(+s%yiBew0KUDx5I6^sbMB?>0d|G> z8~W;D&SF<7p!JhbEnFU*P3YnhvlGdkxE0#Iz!=5S>!PP7c3v!a!hv#`-3?ma!wDCg zr^s$1|A{TAJ@Ri4I&m}oiv;v6>sZBTa%rnP!b78$D2Br4sgc=Q?vIxSgg87y( zkFCr&mv;nTlITaM;91V)YI+l%{xHv^vZK_F64sNJ{8?l>cj0}t(Vi!apJFB5gbgx(Q&c+qVf0eyJ)WU za0m_U$c~FmEgcFMo4xQW(f);BOMX)mYjT|THbVCsp-GX5#ZHK&gz+`gnpshRO{^9g zxxo8LP{@AfW@m14mc-tDg=eJBnN&cOl>@PuHaBvwn09iuW%c$j?TUia+y8&jjhqVKLhY<kbcAX0?r20K|l(W6D06- zU>RdRR3o{YB14OWcZJ@CFK(pWRDL?yfv7D|^oZ5Wx1Ddjz$-Jd_o5ewY$lO3HPG)3 zo|nB44uxHddEH{9O;D{=fD>(t%mbduWNo>VBYZ{t5~530ah65;6D>fr+5Mc%QYhF# z1Yl$2UN41C&hUD(M!HSt7gvMD_^PZ-xhnwA;r!_3Z5G2#1Zz z_fq}NgN$>HJKoG%h_rlxHfo{DBu-WZW0aYS9#%nC=MAWYJY>c%_Gi(FE^waE?^*i` zto58oPa=yDHx%JX*|c~AE27x^Qkjw1!{z+cFgM}NBEei@Yz9ts01B;VUQ*Rg@OVXo z1kWMD8)_I)1MNS|T~ns)XEn*>F)pdSte~sPOXbI5L>5MKgDtcE8&m6+ z*5}YqIh;ynA~N>#*eIo5f|)0J*oRhbxC(Ybq=!1rc?t8a24n;Ph&k9`*j zN$P0}#Y>IHQd-@}teT-+v8muYptV*;FSf~UX1NjWgvAzmu7*=nGmCd}p+?D3MuTKl zqCGv#9_)b^h?*H`0z+2(3_*NFrs-C5NXrvdW@w zi=DQcHK|};!nvcCq*}%&o*%I_^Mxu|DGh2xj5N<8}01b(8M*WDQ-xo#piA;S~ z$i?_s|HvH|UGGb@At$hiofAGU7VQg+%gTBPjf%Wf!1@VY3Vqizda;yC=6%!)AWv!d zZqzF#-l08wU$ldh>`F81B=>=t_RATe=+@8jU0DT*a**71u~5kBiT^B}w&a9H`JmXI z#8OE%3!lr)CYC+fqh`KuU=(?r#yPBq=-r|-i0wga4Ea!;tea?fV#O?Ed{&+p`?kpD z>AW+CzQqzEYb*9B(V^z>jM$Lc=`)?bz@%gDZIRDKeia#6JO|eq#azBAwoK8L<}-4! zUx-ZJ$2(&4%VflI?>RW}TbZ52GaO~t#p7`)LL(x1x3aHt-t#yqVjJvc_GcM^$OsYv zBHE4IRnod})A_XYFrUaxT@8Ilxw6Q3qNh}|wsNZq=PcqW(cy$2$Sp7QAyx?Svx@aa zWYmWlrC3D8Mi#Ypij`742R-y9{!p=$igo`i;})q-Jf>nh63P*)sfMTX>02zzW?B>; zmdkrl`=qR#_+T_pI9!$Un(bCIEq-^0=z^<5W96dKVt0{ zJDAv^#Kv)*c7;brEyQ9)6aTS%N37#w1*djICa;k_Mq@So=TYSLcGZ8Q z^-D#4TQXCT_C@E9jzGK!vcJOlWbNeL=-8v*j`sAWH%h~^@(pXG-{^DE(a5)CRpp+D zuCq{{v?q3J*&CU6l$N6XL}@_UkiC-mNB3T6K%SQIh=wH|A(@>#Cx3rApXdyt9=k73 zv+P22?c{mkEYV#NiW6^4^uOp>i2NR1UHPu;P;?&A?~C90%iW0HS#pM=bB)e6I$xnip(n9} zeA%C@qs;TmlOm&()@3wN4iJ4JIvROTdXhbqu?juOic4Q2!^%pE)h_y+{4M-N`jzLU zU1=w}D$<9vBx99%%4hOlbTq=@qo2qqzND5fpONRHJ1x&k`%&H@bSWz+@5r2Fm7+8v z`!D+_r}WEJleLtVE4tpGnM%+&c1| zZh8^PRQAOTr;gHflvhSMh)}<@Df<+qC6STj_L1*NKf(jVKO-LC0i-?YL8L?(zgRZN zC1-_1PL}nRdB{G?9J=|YnRf(b`Z_m-$S)@oxeK4Pv#KJ3HT%jnjJ%~IaaaC>1Sgfvcya2Hhv>}CxZ8HtMShQfVFhu!` z;5lJS;*&LKzM@TFWgtIl622~@Oz;Dy9nboL7LdrzzoPl1TP`b#SRiNM2 z5gRBG8I7!l$Y0CRT$+{KB*7O{9g2KVyxss!xPxq;8v%&}ZwVh?7A=2S7{Bv|RAVwCa;VBoAdr2%K5umW?5ag+k*A9aTt0Z1Bfw$$}3TsItgr z-lZ-LYQlvesO#l^s?6G7o61H0c&`<#^T?j>>z(H z3oO$OfhsCAl!15pq51|n^wnyrIgw9%i^#B6Fw@qs!$@-EcM|6v61!@h>ahPX5oy%6 z@pr2xG>g3}6=yX`M7KYqGk{j{nkGjT@~kFD_DQc+vq<^F27_J^!n&`9pl_WHBKvovc$w`6_8w!q)i&`PjI-5#1o z4BRSWx*e)5{>{q0>c9HlR{T+O%(qbSXU(U9HDH()5@)`K2;3ckK9wzSkbLZ3X#07M z6AYOH`jx?b-e+|6M9`wCDeYvi?b5UdHL3v+TZuN=F((R}bVx(a|C@p7V67Gg z=lGZDSA}l63u7J!&j^%gddvq)uq)82+KMeIPx+4TftYjdGR0l}Oy9xa&voDQFA8bZ z2SI+cgX$XyW|7B`;lEC{r!BB26w)5@_NksytNn`59jzO*h8j(ovXt!5-)rCU^{8sV z*KUAwj0VQEKkzn(=IMQ&dMIe8q9l-|3H!6j@9Fa2(P)UOuTm`6eByaY3z}kRje0dm zbyo1d-wkxD%7TTe4?wm(?$09En7ZVlW=6X?SfwN$GUB;A4C*%6+?>!GaF+Kp-ih^^ zRHcDhx1jVZD}yP1o1u|v*MC$$sy2fJ{Isq<{D^ObHdT4VmqaeN285zcBG}uhBd;KT zXDd8ldhpL+6?cZbped$J>J8hp*PLJ16vh;fSq#^y^PEFI=&e9IQT*eeeoCc~4E=~# zp$aIr1m4r^_9@lvg6d3$_n7AKzO4>Z9}hgCKTgI{cF1PDGoexZSidEZ;rpk)GxUltq#yM)gtuunkQFOf zREs@pwfmLFeD$iAH9zwdcCtvp@0blF*3fnrWtgPQa^Ktvu&Bt?LZ^ z%l8BQbh0i!Bxkr~qQg`c{5yE6rHWnd*QjPa#dBRf4c^fk7^D}4GJrjzMd=_0b}u_T z;eJnBuRP)X3%T%%!WI7QhEtw)a_$!fu804kdC(_lZRZ1JN=1l>e{fbGV@?IiUwUhK z{@=dVkj3zb&p@uw`9M5%vEL2kE2$ys-yEjxpv~W9SmK=yO2+h{gDRqL`a%EYcIi_7 z##0<~#M?oAgnH$6a?ih^NeWQ~-cS`@?46=dRXzmfWwz>5kcnHBt3zb8P$kz+{@8B+ z3z`;qz$SL(QH{;_Z4HR~s_p(6WIwC@uTtIfVuq2fi)bGj%vkGHiZ zUybHXYOSuF#L)U51-Ry>{pbMh5Z+83)%kP3H(xD4>GJ# zy-fE#uX~m=ZufgYX`ZF}lmGYd(w^B{%n;^fE1;?G#q%03)sooC$ zQJvx2O9g}5?lQxku+w)I*b3(`T#i1W~yOi44EPH}}hiS=JxB53a@KzL! z>R9(L^p4Q4JkRK#Rlhp%fWAjn>$ZpI8SnFK4tmMkc;8*9It8|CZm26}gWo$@X=)Cu zd{v4JW2}JN|E}>%#v9k2tUR%>4p2;TJit8JwYz;l> zDT)aPe(3$VZo0x2$X5PY-4NbM|UHifxm%rj4Qh@%%)uP%7Qs<=fgufn@{qc8kPBA&>IAkfU3AViK9OW zZaWsV@Yi(rcszYXzwVCa26t#FzSQEt>X1adq>#;*r)mnh;hK%y#;wZb&`~|MeNdvA zzfn6)y*+>>5Q!@@u$Kx9$AfQhic85mc!N2uLBg$~Mryw55-3if$b>}d&w-QlBkhwJ zjD~c~x0y;CdxF!TI*HV1L(ixp6Z%3p6Un#`ZJ-SfdmMcrJG7dyfzbngZ5JZ|XA|6q zMPz-}vd5e%{Bj!tTR}|(#~gnr)nSqNtKi6zcjQ1e%?Y(3x#S{EKZW1#IQmB`()Cs} zm@4qOHRLhZE6b?nU{@U${|53V`JrG3O$Yh?0@D5(a?z`JJw?2J9@)m5LpkVW5_uv1 z9r5y56>CCxia}slhaQrqCI%SYCR@1^J*S%cSNtdjykYCecMG6991I=D-q{n9NE~}e z>fYF?l~qgzK%=q%1hWS6(t5#?5NxMxX5T~WbCLY_fE7U|D*2|@s97g@Ww~SwXpq6H zSev8Rg10K)Mt1a|SrFe#Zree8p*zC?a;+4~A}W@&Dn*y=!TyWoR5eSLga+1v{3D+D zDtIc@XINu1nyTbP6@mt3Ak(#i{OU}ot%`h4KbeSbTZMq z)`6t+7tJqp|IjZn{2NrwINfh`^Qb-eRjM<+q5adqNF&&FUE=*u=kmwB0`DZW3U zJw7GD7XL(iO8l55KJH>{mU*giiCzZ|MIV(U&y(v;?k0YDJGuDf!H20x{Z}e>>FMwP zdAyS?u4d{gs~s~&lSg-q>>ExSE*dVf-RZw`*J$f-76;x_&oiF0)FsYKTAbLCuq5H3 zM1S(yl=I2%#HSOsS({DYKxUfHzTFZ}7d0zaEAfYb#83#Gx2tApz)I5GubogAgiDdH zzwbNdshzYr-D5{aKeE3+xbJRp-}YW@@AjU&o7$ejq0c5SgzwSUSXLx=rFNvYr}U;g zk@nowx@is5I#XvS&rWKJOV&C;CHz2;Zoq0;2{m>G{^WZcyr4F)vZv^e7}}{Qm}NSm z-^@Mj1k+-~b9Hi&>!M>~c*o$)z74nE|2*er-p$IJ=eqN5AF@5^`G&gJv@~%vZS>y6 zj2qM3>GP&l-@78CA-yGKUUFT+4buYU0q3de(S37ITeQg=0dpoB!O>0Ttl-8eQiBC4)r`_&va^;PeIe} z@qWJV=DyE2^c3B`+~*y32K1(7Q!X2tc*FJ?^gE* z?@XL?Y#B)&Eg5x=-F7bupH=^!s)PN;MO6K)qjp(a%mJg5x{!DEGq_V8^S?Mb({Xty zyZ>}wPTz^%b6-^4KGTzTJNb*-x66Bv3>|a-M7KDhb816o#sj+^Xqf%XeNE}lO_>+} zVqBSNso_!WH`Mvze|vMC^Bf1jf_>4EdQb_ z>P8YS0lmG_89VIlpXf{Ot^8uv7oUA`v-hpO&wBTLG4aLCfqd6$r7^ZWrG7@ooMjJ$ zvodC7+&eWTY;7|a8$Z-`bGNoA{^WnvUFX{F+`!6~JC}M5;Y;dCeXZ$o+-sJ2OS3sI zCZwOK_k!hFg_KV8Z(wk;cr4$3_3q&9S8g4;b*ks~7nOavcW(FXzO%))YqDEeXg)e+ z?Tl?%)pKWmWmD!u)1FJd8v9TEcd0tM6&u!d_$mt`T9C*vS-d*E<$J-Uy ztVqx`nO?R$V|@m`w8^|ozW_T-QSkd@c0b}9^1M1Z)tNK8a;W6)hA(1!#y*ej`RH~@ zZ^fMjw&aOs#i(g!a_zKdW;J9r&1r&io=#aCH%~teEVklc7g+rcZUiufki4lXPEEYy z{^Nl~;cWE~btMbU7h_+J>osjK1d)U0ge?A_cu&FUHn{ZT<)g{=+x-=NpWXKM-0m6d zxq3VMj@J(2ovO&(p4@wH=dAwOEAHQQ-`@20#2k}R+o{N=qDC<|9)bWii~Qq8>{vzA zt8r32_g1h-8C3gpH;k?3*UYwt{|)e=sr4v{#cK{sS0d9|z8&Eg7}{O=Z|$ z>AtI;uGkrT#s45!2A>A-nX7)Q+it8ko#%wiGk_f#emL-?r^2;z%;dN{x_xx}$ZdO$ zEq~yxzD2joZ>8NT`(n}1YZETT4U=igri?av)X&biuk{{he1&nHDjmGsfBJGl=ao0L z?-)u=wwQMewVE5)1pn#1;~sWD?kfszRW8*Xp~mfk*k+T{P^Tdp$$!e->P#HFJi2If z-^i)q{NdBK?E@G4wtR7Cf8hT?lpcz=Tf!@AM)J~X734KC6#Bsu03Zc0g3W_%md&HkRKOZ;@#kS z-H|u4+n#L89*Q44)W6{FjJu2O=H6Y|f5^6b{FpCawZXVJz9#jhX&D(c_by4Fn%bB! zEq0&bg5cH=(GErpndF01v;Bo`5CpYaRj)+FfZUFEgIYj))z4E;_CAm{wy8DPCcoo< z$NjqV+~|VgS+-e&r|ypR?e06%H|x&WozDicMv5nrsL0VD6OPYF?U*`qT3dSSJsnfp z<8xxS8=urY3o`RY&^Km)P4R2pPxW7?Lggm40~;n-;m9#FkfN%nfEA=d=z0*1b5&$5 z1|RX=nOxx1J66(T?BIs}+jn>0ExEhp?&bbzwjGXb?t`IkXwREwCbUm!PD`0;yQePo zM$!$dGIodYCEZ>SUmhjLe?ohe>W{wy1O4mTKZ18dBoa4&OHjjU-$P}tHtObr*@;Y~ z2+t41cnc>w$4-yrL8r-s`v$TH&h&rMe|g}up_L;o&hPsiigUVIW=s6y zJEb`>1uU#WW1Rli;43@__WJ+mj_LoUKcjzNw@rJ6ITAGhp2JDBqXz6Zn~`f?~XfbV)pp!qsHMGwzQ$$gP#oK4R{A~hGyAwN2^_j& zr9N4oc+paA&WU-#uvGt3DjAOG4$xnjK@YCR1gIfrslMqT!mk8ef;DipjYx2Fk)RiV z%Q`3UFK@N`ybBEP(W}E{oQ#RVS%WJF(}uJ*?XchR;$%v|th`sZ$aK>(JJFxCDS2x0 ztBLdCYvanyvtu^U;}ZRY`g0(-&o*u`78n;B?qS}40hd$io#%rC(GE_=8aQAx*{j$# zgKzrZ0dZA1Q87N#F=OO_J_=;~nB(C(q#Hc-*WFT48#RZ5kvDW)dwnRtIfOX9M` z=J<=2thn{&eK8k7j1TD_G|U6d+!C`R=7{l(;h*{^b=jIcsC6SaC_AxEIKW_OMmp^U zm+0^QW7G_OnI6xNAA{p4hj-Wx4Q&{DYbf7#+g>qR>>PBz9k@o#sEdYtvp3FZJs)2e z-)Mc`vLbHK?2qXJktA%8s^5iF(N)LTjEjvGh8X=z)Y{IaUhD#_a2h24xkx>NP_!`c zpCE?(5|s1&iC3Ka#$FlyZ1@vht-DYAELI>qdC<>F^tDI8 z;QK2mUT=XQc*gx2DAP?yk0%_5Mo)}v8Gg%NU_W8sFezO3g}Kd?XKIge$2h@3E-~3mS!SDQG$uD@iSY~lOS(&7%eN|b5}AA$ zG^6xjDHtT*^Zyv^#zC-9t3eL$7=L1HrsMsQSBB5o581cdV}~n7rc!lz*uBtSf(PKw znxE-gjTNRMb0e=ob2B{6Y3ei`<)_|MY`#NPgICQ)a}hHyqsq5V|FZT&PG$ff&v`VO zYiJcS{WE=sygvcuvvXosr22HFqhPdPWXJF!`!su{{m}6Hqr1nOCr)`j_U{aBrl#Cy zx*diN=Ym8kV`&n#VY(s2XY@@luw7^stV=?{^s(n-wP=UOK zCNLk{*yGfdJ^*&@7?Q2c737&? zRAZW7>z<_gd2vi{Oq*$jx!ycxZjSYGaE(P<;e!yD{JYVqebIgt|#0f z@4JD`VJkM&`*rW@`@x0WZYnjmn4gQi7`ryEF>aS7XxV0c+WL|;!}^&e7?&M4Blf82 znDJlwr*&sF4iGfEs5SU7C;{VC^?uDgHesCbJ1x%F$2PFCD>)}#TbXU8ZM*Fid*z6J ztj5{SOh5I3N)_%=UQv6sztX!6rkIt?Q_FhZrn15&Yn!zp-WGo&er9~Xb({6FWmDX= z*cwy2@s$2UZI8MQ9o_*_*em`=e82F#=+2$oHj(Wz(dUbf!I9zNL3@rJPR`l9Yx&B&vLZ1PKdXp(OHcNO!{DTQM z61ozXBvmKPPSPi>NPH>5Z@p?+8&?Ao_|NqBX?s*eK1X=0%6FefOQrVnE{n5kYy~qd z8+m=W(*BX{^&u_&uC$HWKN{KM*iJ=-#>tO7fA_fq*T4nqQ9TMqzT2=Urq?tpwmz=b z@<99(2`dsi5|1WzC3PgtOlnE&OxP9wk!4z3ow+$?w&7Lnqgb7u4TH&p6<_5|c27*S zxaypz$F?|fMvskbA3kC~1n)~8N*fv;x@>=av0ba&o$i z?wA|qr{g|jg_k8fkvKK!p`=?$qe-nvU5T?3x!0{{;XNo49!+?X*|x_{Ggm@SztzT4>ph3r zo&?)E!+X;WX1;5d)5)nhJ^Ine;P5N<4N&l+A?48Rp&j~x-zYm2@Q!)CbcKkB=sk?BrQ$yC#VzZ ztz&UPbDn93@frGE0E%`QRT^jdP1No%4V~e|3Agjw_=96lIEqJ4jg$|wy0(d- zoT1l;&e+ZkmyMot6pxp4IQyf*x%v&t{g9LJwfgG1HM0iNL@@_)IV#^>5dsv zVs4lUVo$}@SeC#YpH8f2&P~u(Mp9>DMxrhLLF+C~(gM?IYBi)$Ns`PGsUi!t31euMFR|zhhV0lkM7J@5m8H;&`X?_Qd_3N$;Qh z8>v5Lr>@=enx9~aaT*&6kBF0H_p@l zS+g0ev(?-STc`$g%=d5ae|xIknNJvRE*sNUgr zyu;nlFtNhD)cbF=w9TNk4Nx|=4ss%hMbrN>|HYR+C)=QVp1I|zbvr@JES^( z*is$0*StIC6~hxCR8PbD0VX7MIBuc!|CH(`$&*fJc+88Xam(;W_EYf8vB8|dkNV5+ zR^HitC%M1ewrp$+9p}1cyQ$w=pA=5EOtB)mMPoZW^SX7kj>;a zK55YD-UHEcIT}qZ^{94YJ3dSVPA8t{3hIk@f~2;FxEP5AZ3IDj$W!JzH5zN*GU)9u z?=R?|=sz-0KA1Q3if!NUJ7Yn2U-%vU49kt=ZBrL#?4laTgZHdW8jk(BzFV0}Wgon8 z_#jrPZfnlzml&@Zmm13r4COSG@`?ZxO!NLJ(~7kv~R3h{jro6g$=T zRgZFVw{xH4BKK6W{h~d0__N`{k>JS9(W{R9@fuf#_j-7?-WxlbI5q7?`pfs$r>CTL z#gE3kM&-G?o-FrAZnfVN`YuSr8Qjzr=3-Nsaa=d1*+!3xKua^K|Ds+_jr1OT=u|+b z{=!MBe;xM=LhL`Ff!W?H_YIeGYzvyuwBhaay>j#!N4aBg^x|l_;}`2eptPgOac#hn|pJKUc0vTqeQEl?a>@)mIyN9~q9|Sh|7y1ulsVPTWLYohK>Pzs}xj&m|rk2X9 z)Y-Y^>UD*kQ^yNNlZRvNm9{NISBH{EHcb|)md0hI*3M{~bL}f@XEopRTx`Cg-t~$7 z^`TS4Q=Pm0^_qI~Wy_f5>A0XNL*I;SuvPJt`cdr<^rOZmL-Bvh+WN7o+P7K?ejp3KDEVkdf zDf8OwC7Dapa?A?l8F!+?XumzY#kt+TTs^~dHEtgFXFJ%%4ItH0;aQceoz#sQEa)8t zF~8AWp`LsIy=_l$0ac0w-|S%M?SS5Q)$?1=gWix=O$?8f$c<0DEv}oRZ`qVX@1un; z7(OuagySj_)q`Up@2~Woi4UgN-q$~S-t5t7H{(i_=O$;2o*LR>+dlS@_o%8M=Az|} z#TdKWI9tC-(*fG;G|g#!qOsIiNd>tF3{Pvb@R{x;nniN2B!2EF6?Og#T3q58N3Xbs zFVUmSfmb~oT;V?8I5kw4ht74{TTaby5D_&qG)JjfdDDHwQ8~1||4#pzq4!5R#+>83 z$1}#Zk8H3%;P}{AXk0d>@!olvOJ?7?uVKn7hPVAU#!HZ~w>w^SQ+-|A7E?>LozJx2 z(yrAl2KziOCO$Slw#YQo@Eqr2D_F=C>{W3`4%B;q9A4%ShX zcbDe(TBUYCMg9q~IZM34``-wXFQxiD)~hkKoth#m{r7uv@S`M;9J6QJPuufHO0YHN zAnR9-6pYLqU+H%mucfS*o|3t2R>$-glQZ;hdCRB_w{pbd{4w>XyYcq_68-*Bb(ZEA zRLFTb_KZax|CZ&bNu|qHmQW3=5`5@3B**8pm#7Q2m6`xQ!ecZa4}V9fT^WycYaQ5a zuHgQ%A+>dk{TEe=P08)|R?nQBnK8XC`IJ5- z(B^#K;U24-Q2Qkc?>+SzwLzs}?jIQ%V&}zgN@$C}L;bNHqK!Tf&tCX%L^=FF{blVp z)MdnyY=D11?mr&{?E>ub)hY)ZwuM--Js@DN;~NLOyCxQo6%8*KS}^#|(4pZBN6FZc z@qFj|WA8X-xYmSirq(GhrMKeic`)Ur*jCj{f0ldi3*sI1-0w~ z#B`GF8*0Hn+6-E5IT)@~^CPY^6|eXk;COZrYxM*F6wgZ&o#XSU4R{gm(m$5yY;@K; zZ#ZYTa>m1sGH1VkTz5OplJw-1SEsz3&}*8|wkzB4VcS79E}~Y00ZYk7xM-7VDORk; z*j>mOztU4P0lZd``Ylvh`Y{~nAQskqywmM)ygs}Vex%&TxCMx)1f`s8Jn~Ib6}PbM zUiCcaPMhc$pE_1PwvWn+z0OKkqidzB-&yOV#-MZ2WJU0&x?J;Tmf6;txKWeI@N*qV zyi^l<27m4%(1w%n_BVj!|2NGE!)~euPU_Zc<3Z~EmS(B$gnk~i6@RI}qirBsp%Wi; zH`P{&4hE~a5)0FO(4qgz`Rl}TxQ^A^ioEiHZ;3Y!Dec9{Y1n{rT?G@{CgQ1Cwtk`% z3(ctW)_A$=U*1Z^quQV73yo!<38xqy(ogE&GN`fKZ8v^h_Z~hcJKpm}s_%l}{HUb?E5@@7neyV~kmu9Dx)5k3%2iae+NK`V zEY|&sT8HI^Ck-?87a7U7wO3S^z->MY>aS7rp5|H2de*4{kIEWkmJdOseU@>4Tm2M% zd8@<#60ba+s9dR)bDkKA4sd?M?BVmoP!NXA48`UV*3VWiYDY%Y&b$zwi|Kvrh3 z5Ly(x+I(nc8YieM)XGZ`be|;xVGsY-5Fv|BPYn8ePM&1qZH;6V_JfgJNF=6EZY%h` zE~=9msPbioS6nATs{?%2o!GOpm1aEaw}_}XjPF~JP$YjbllW_)+FE+aAtHcSY-G`T zUd=?l2paBcYEl#uxlus;!cNX?2i`0z90q+)=@C6ATB5_?Mh3Ya}nNkUYh5 z#wpcrC4V-Gfhzf;f^Je2$<`B8J;`1dL{dRwoy&a}&Gm@p21v${V1FW;5bZ9xu98C_ znO>Q!f*=eCR$LU@M$nW6pD&*li};a@F~KdA3~I@M6HJ`>e7caf1lcs2`y`S6lIihf zCW`bcSp$+QD7o{3Cn89!QS`nj0-VISOO~0SiJj#$$z79|WU5`EHOX6$EF{UIklY5z zwv&tt$#j$%M=@pv31U7!lJ6{8fPz|;$zPHwAUQdbDIvKL9@>?>ktjBX;I;^|px{mj zl84~uMMo(AMl-MEGuagxldM-Xvmu)QAsGXbD<;`ul8+}yU4oe^wHlWzfkh*B=1Qw zilTWe(G`+>v}k`(v{%WNmlcuxXF&;@L+_HSCfOJ1tde9+ME{dD7HW{(I9Y$m$detC zo#qmu7MGiqdJ(#Mzi zc|sqOR}{^KkxwNOluQ(%C0TLlOLF9eDg>2K@>C>i zM(B4sTHvi9I9k-JycRO3($r1imBCBs3+4C$ts%)z5f51@9Hfx^`Xpk%$Y~}bOfr{u zMn-cMx=;|wbd)D0vr$fEnZ(Gmih}S;aB$c#`ZcX>KG6s+P~GDZu;ZBm1?II+}}^UoqZ02eJBEb(yk+ zd#jDQX0x#5%q9LTkEr&l&~ZgBl|(kv=1!`~w}y7YA&Dd7-mb^DN>&clb90ng%6A#- zoA7jUgYef~Q*`4|AN6I2Z)%E^&cO4;xxLGnmV+HTpL)m#6^la4!&YS(ISoX?Daqj> zk~oL>=VoMqEMnFgL-nl4aiR-BnnBZ_&Pek?Aish1kqZxQV_e3i2ZWKceNmw5g%Bjj8(UC4;~D5gV0?^1YRRmsrso-Pd$Xta%_|p z$cE|EFX<#2Y!9`E$Rr}}ty{@?Lc(Yu$HqalnJb*7tRhl$Kh^Emfj#o7I#c2FB@rE5 zrfd(D2GDtkmkSU}TucrKb--2mswCy>fmPvet2NNEf$W_NWHm_qd#y(A3kSZX6>Px| zeCyS(YHqvpl$XF7t`GgpyP41bras_J@h{cqs&cux_k^|DYr*Nno`Urd3@A3M3WFer zD`!xpbhrN-$RoXq7;n3Bg|5<{6#S!RivFs{iA4B}-sZbAu_2~P)jrXxC@|i|FYznA zfjk_s7p|weAaRltM$MSHW`de@I*n?FA3CD)UN13emcaf{2X><40djiOkN8jM8bd?G z4X;t>D?X;qZ~%n!ROO4Fw*&7OkE(y76&|7^R4e@# zHK;xaWEeE$Bz;}$mP`<8gr5%_^EKlWYE@;sJA&D!E&iW+7VFO|H~5b$@`Cx;LH^-g zk8E+r{ine1v}JyS;#IO~e(I@=v4gZ+ zSC50i=c4K}O>$rp(atrRziCf+1;JPY|6V;gj1yhklv7LvVtSQ2AcW2H(B8de5*5c~Zhup9lJhl%*HTxp63~rO|=AlHHv(SyHubJSC96a-5aP-8Ktfa`N7J7&EqZ{GSA|njrBJvj98;z%02_@%49{p|85{om9DCwCTb~q9i z$bu|%<8CrK$B_eBhd%suK#%Qk7Y~omlg~I2j1VaK=jS|}5z#OVYSFVwD0WQ1&=_$xk zpCO;&!c%U>uL1@$Gk7<>TTK5dU`a`1pMZhNS4mDaSZiq82CfAb7&dzvxxd0TSF3zS z@g;T(`J(Iz?bIe_AhB~`un-)CC>R&^Gh(aB_83j1zB{mxvnKmT5__o@JCoYP4eaqS zD^Rb*m|(9+ok%HnyNK(F7El(jV22WoV~?uheU{+AqC0{BR#Qps+l%$gtFE}%e99KhZrwUughMW{eJ2(|sK~bQI zj1i3$Z2Ne4>k`-tuX2uR-x6;cndxh(INN~6l!Lb7B>R3ZEz6*a@hZE#0)0rK+{Rg- z33&bJFx*iAJyk}(EoD|`v&%0<53S~Yr@;PPgT9dir{r$@u^Z_5Ag1XUGH0)o6I9NK zMxn!@hr_NxJ|8{cW%M$^>R&>}x-d)Tprdc#*a@tO>&(*fz#_P=_R{+WRCn&D`c<&^ z(3Hpy7bK{!1GQv5XDLAe^tZqp=p>Ilh(ueD#`6q&cm-^lx3Cto$bT0+KdH*ir(Rcd zuP8htx$uC!Lw=!^-TW>xv5I##;5%V7xW}*gMj0b?$d`e(vjUrSKY8l)%)OszeHw#u zFH(LjT2C|i)`D6r+yH_Cl}jdk6z3`nhsLy!I?lDsy{oK_wcJrP^0tz?=M~(eti@bD z5vw4RG0LDyc^mDcG8{G+^l-c};5#xyxkvD3$dn?B`Wf_Or(z+jhV9CeXp(|aX=83y zFq^4$pmy{GJW`A3_q{~Gipi)OP47ifTXBo}iKB2&!GlLd^KiUzE4Vj!ahSi0{9gvj z$n^s$HZX^k`Yg2$Xz57Q!+aAtr@X6L#E!km^RyQjJ$G2tAcz(sSqdc z9{<<|*WfkLrx%ndf#tpyxO{&G_fs`ms+Yd6RB6cxOl9v%;<%;k zj}Mb0)ab2*U#s1J2)+bidJw%{tS79B(RpIifSzAtb+VMqe+msOSwr)WNX0jdD9>JW7n z-HL3Occv)Q`Bpw>J?c9|eYXaAJ)f$q=haK`vn#0dp$;2f(N{dN)I1A56PUilAC}Ob zGOUs^WUH_zWZ;L}2P^YvDk+?SRJ3Uu-mAOplq2EEJjo7qRauC3kwdLxCYtSjs(s>A zrK$wtxn<1bX4%z{WTiyljoQyNsp@hh#vvFz1P}5Aypz>r?i$t6!e*%6PklZ}v5aXt ztGX09o$J(FV2b>Q<{|Am zc%4>iuB#Uz^A0iNchm1w^TOpJ{1mas%M@5NuXEN?uTxWFC$@pyNvj*Q)XdhZ;N@FOMCK79sE^RLuhl<<@$74Os;;VA)mN!w*{!@xj`>>lu+j8gO27&4 zS2a7tT4YTHI(v1Xk(`3j^zdEQ^>kaQEgUk z5E~FV2p%zVGnr8ne2ay60S)XsDd@ySlc8~dqy zkEfpK_G6xJUB7kCbiVHT&iyo>Pw=X}YBI85I`hru8CJ3OTh zm>PHSXoDl}1w8bh!k7IdEX!lv>*1jL!fp0^M_&3b;nBY1K1jPpBh9v9ms1%I@7qz| z3HH1djCCgLneUT}U!#0WUt%c;%?L>~>kW^r`ipn~S8?1`G~iNYI-Z0)d}fvGnAhR1sT4L_R{vzK z-@;>q?>(v|BRwy;w{n*sy2;6bx%}q?{~dw7ZXb0?|8o7x^&nh$p{}W}S6z2pyJ*=- zBGS3cxB&Vy%za4od~%2n(>J5pOR^PC@HdzrSZWFCcRw-asO z4HiX$<`2483=@qrnSnd;qAw)!{}dkn=QT$4QZ(sqG((964QIEDM_CPsui*s4g3vMMhR;+EVIlv&W_xCCJ>>cj+d;U+2Iu-ZHB*U%E^i^ zLi;Es*S-SZ#~5}Ri%O?^!&qt#m~sstX^qH*=hZ!YK40^$Izo#UU-b1upW-sC*e!p%71xOzWAP&en7iTnS{PJPVqe1)+v??mQ%(dx~1w;t+|KrR4=^R5o;G zYBiCza~Y>fG%qS&vDuQ*%%?CnYlz3pL1K;b9l{T?$(!X%B~miSH^Fnmxxf+PXr&6T z(9!B}IcAVQJj(I5 z+Rr=|y*C?e_^zxL@6X+*oo5{D9E%+Vj*oeqb6j(rc3fgUKIhcAdR^b*$u<0fMY*B5uEid?@0az7O4Bc=8m} zbXn%okOLvjrr$9av+%^c32W{L+GtI;V!tmJ4QvvL6uyl*MYZx#Vun|# zbEpEjN30vsk2eQe&^^=8SC}1W<)c)OtH-GCVqL)b$X=7itUNvt2To`8eFzivd+uK$ z6C>Skx$ZdcC)?$$V~yiQs>E7p;VI`3qKFgOg`YSMsOC%zl&n}fc$oF04_EGWUnKLmdN^PnPmv7ip&txq3_h|Xt6$Y=ca zPjm{#EzPx%25is`Bmi_wU(%REmlee zqf`eEZ!)+}8O+n=ASlnrva!&g7NW{kST}+#yOs!=;MJGlKg#vCu~UD7*A@nO?)W(E ze&79T^pHNr_%K}RA5xj|vGXD41I`zn5iXUx3yq{6U91ibx&=9IV9qPpRgaP%p64;D zr-u~}nMLJkrg@sKR=HH-QCLmS68+h$*yCBoF0&lgok7;`X1Ig?d-;@vQ(@WDqC_r@yx=t?9Myb7x(aJj3jm)!q!z2Q3T z>ZhG|TxR+ql9dT}JaT6=yZlHvV5?v!mzMqXA%;6wNuKLni#oMAkZdzB^l zRFZjKjig$mu2SWa39<<0*=^YJ7qPa|7)8OpZ|6!i_!P6Sm^8{Hu#Hx*M+^HQJU0Gu ztkX?smMtC&yXF*T^e!?{kCO>G60Y6D15D(PF^B(y$IqDok?utI5w{-fi&Y+n=O{Tq zC+Q^%`7BLXZp(cWU?kt5Of~g|mqgCA`pn7OvSNT`% zMDkHi>xYn?x&jHZMtx25kv3YZ1e2u}TjdjPGCGBY-9cvQAq6=1tivr>JCCZPz+zj7 zt(k`XQ3UE+9)1-kJ7G3nyj(>wtxZK=fFA?hZ8JNJSS!`&3rEof8W_oXeC(O<66a!b zdT7aG$nIKVc00&8$-@c)qX!LdoX3O4y=S1stC-D^%;RxL@O1QBawyQ-E6A8U>u*;s&>qo$Or6@7njZB|^)qC7J*E8{ zR$K)hWwL@edo4baD*V3L*l@zAQ3}@D0sOsJu~6f|vXjh+`T-v9ubH>vKh4HY5wAcT zJ8c!ZgrE~o=G(DEJ;AB-#ljpvl9^lYJwb+NH*2?#mNv5>U9aA=gO#*{9N3d& zvg|@K_q$$j4+Ftr2}icUD7^ZRs#4Q3Yq|BSkYT3Nx;5%{n22Ss ze*)g>daCU|gq3=OFH><*{j$!ae@M4c`R1bI=x1s$AE+mXt*nO461TyfT%wk?R$7)D-_@33r;fpDFH$979i=H3`P=B% zVk*xrYtHFJBHt$KIUk1R1kLN(P%`@-S66{3vX*;F!k)QHq#+L-y$b0E`wF|+-)MJ; zT0$AjU@xYovQg>H-3lB z^g<(Zx)zIFtoJ=$q7h^{H_uFOimYCO?*A#Z+>LN`YtW!eSY6v# zRR(bKju=`)E`?69=#9Ul&T6XTnC-tKXpIQt-~JcVFbsh zS86t*@fTomfhdC3pO5B)<>UX1o$X=#a^%h^3seQzGLM6Vm8>KyhwP=L_%24_;i~59 z@vO5Icy$ls8EnUcvzp!e5VIUFCmw$0q_5uVLHZbw23Gz*$rY;kU4 zm$OqH;B%YQ@gjQMa9#avCl7c^4+`i$pw9sR){a zhpgRiQ#_)6QTrpt{25INIDa+lTWd7yV21wyo9-gJQX_X(FOfIwL&+qnLa)dm`WKF+ zI(#6~4pRxbPBQI&9$#QR_FFu^mcYZ}=f#5B%U)Ma^k#sA0Y6t7k)0;wr3Vkr6ePlG z#%wdzXbzsYyF@9_D&y+FS zyBVK_?2H=%sj4fwDk|?Y&94}T;0Y;2dZja}aYT7a$*CZg%>5l!6cJ|+Av5@0wCeff zDz~eys{gDhCR;BGt6V|kDVAQfpi7BF7j}#^Y*-Ec{0jVe7tlj9(7zIR2qW+!y#DY5 z(>?;mOB4dzk<0S-Z{Nk?c+521dOB zTfyO3kL-JeU2TVF7#esOexxr3`b3XDkw-S5*SC$izLdy~f}R)l_Zr2!noZQul$ja~ zNAy#5e*>Sb1dAYFRmC0W6aOkFpTwpp#1i_m#;&=D=dMtF7Y#Wb}D^AjqC$s3|2 z@_S(bamk5{65r&0q!+mw?9LZpaBo2)TgX+PrB&%@DjUf7+e-!hm+S@#Vy45n3a=*IBj;)c7H+4V0q2oxrk?5Lr>~)8+{~YYt^_~WP|H+I$ zj@A7$_H@{d`4X|GRlboJgbi5*uNnab}K8f4sz%x8?6^y-BMaufWN6l zahWJ;J`t8}cr-6#i6+qU)#zkaq+a&GeH4QanE_g56}Uu6cse9rD%wvWx~u4wN7?N70AVH}%Vo@Y$LKypt zv}Ymmt`%Q3_z*;7VCUq_*;M-%qrnO5jPNhcC2q4H&D+b0PFLn4?}Ry8*qVe_sS4Rc zOrLMCClafg!@OtzCz=w=91!n-lQvc}(oI;I$bMvC4Xf)AUbJ`cspldqC9=>+HPH_I z0G}cqng%qwzq8Bl_+O8cjD8G#G?#rbcVPVIqW8{aE@g5)3-PRWkk7UncTwN3)BgdP zc9ksnN6>3_YA#|w-9<`ek_%QsET;!Q{MYId^<^S5QTV9r>PMIlKLNiJ21(=z`3}q# zL0&H6+#=n`9#tmLS624V!}xk6Gfnukgc0+3VqmiG*7(0BGPsoWkqSnd6YqBo(PJk) zA$mZrzaGsdiMfS@p#6&&^`p%HcBFR(`$Zj@y$mo+gi$oQc#ofpPeBfk%5lzI?w;TJV zn{~YwxnBqp#bF)>_Wnv{V-0?I@%qo<7-3ZxzTU&kf+^T{Fboi>8;OieM^=b_lEV7m zJ0SO~kmU8)Uv>0H+JMjdFyp_3#~k*86WA<2NB({2F|*^z40s>=rGcIUDT|#s4oQ^2 z3OEct!W(+CVXA(a?h5|o*EB!ju|;Fl{uNBjc=GFDt)dOjun$(LHmg>s#;}^n)KEQ2 z4tbHou8OdcG=wner%Ea2L>wo0zl5k={A@poP74 zKJy`iwgovbgg-{|PK7O~hG)qPCU=b95?&U`?iFUz!*~v(SOqoogK!${=ZveE6XmQI zCmL!oJ4`z}je?m2)*pIr9hzV=YuZLlSJHqq_&b{aNHnEdG(kP~tpo4bQRKsi^v*l< zk_T^Voi7zBu$T51BU?3$f@H#PQ+%enE$B4b1={vbS&1bPyBSuBm1fN1>6W zAkXHrLzc5<*W$@M!45wPt^YE6{01~T4e|&C;p2TQeRVlR*x(|p?hG_rB-anrR>lV$QnH$e*ipbR=8<59lG*+^c z=$}|Je$r_7Zfw+r^hcCx1?w2Z_yKG0FzvO#vv!Mdyp9YiX7sBWf5|c@Gnbu1xOk*b zau{{Nrx#Y|Jm!Vu<7^-fAZUdedUY@VS1>oM*o@+N6^3ESre96397b23%MP)KS)PIZ zRK-pig$C7y*Y07~N;XzR7c%qj=nj4K%bWo%BbS}!FfzqSPs~9s<)dxXlT9H#{tVYB zWw#Jy{947=sx6wKS}j)3B26-Qeh1L|cWSO_d_(|^>J0RD(1D4WRUpwaSQ&fCMMz?l zoAG&-5l2g=N2+<`4fM%wdS@IfW;Z=AvOso+ATzdb!U;E(WW5W6?*w{7Diwr*R`_v7 z)6>FEn8#-iv$ooiEt35|n&V`y*w~Y94OkeN$c_EPZ=-lvk*8U#8LzLFH6uRBOlHDf zw5$#EL==+5!c58Ivns3%i7b!g=q!3OA3IG!|AG&Vyy&l+mSf;>zg5cT8L3HlPDtE&?l=|ElFq< zqtQI0m>(Bt_X%1bjL8P~+A8{|nwcXxl}XH~9KN-mJ-i*OM|Asbtl`T@^LC!4zCqTn zWYyQB<2)-7Fyv%6zjv{%o>ebWzXq0f0bY<|X30V_zSFU`GpG#Pz$}r>b;+GkU=5yN zy=>-u!p$L-3?dtZgHD*sgIvS6=oR@?ss_vG4_U2JuOb{%!mZLx`-QJcY9yq7MtGy7 zu0|xWaHeGr^sVeH`K#(6l09&o*jLFPzq8tct4xGeN4nh))n1ETMP2*o+I& zHdgcAGel_?s*=&Vu9FX$!freUncGa?)DFnN48~U~Wn>jhKwlA-E~zb&N)_RKl-5cW zLpcxOJ1^qCHZuccRY<$%BWHviF%5krgF6?_>RYUhI`+nr0ZUn)VE~{<#C|Ad_3UQ8 zl<;>o?=59bWb(Y4=eewnaz2rqiZS#k7`e;}VQ;%W;M0?uB;i^X?%66d-EOR;I$F}g zt7e`pv`jK1rG+VY{6yD)*Mr#2QvTb9E_s;u%J|*Du2YQND6jL8x4EpMbYyE3dbA+C zHZtcfuuB}J-$iylOz(^v*i~c(NoHO-vqxCc#9mv-J*Kd>5_rfn8HnhxLrL|W6 zj^vTVJ62W;{3aaN&Nrk_oA`DO=MwS!|jpq>!*6E@d4`bS9Y9DHU%bSfwHyUA>A*1+oJ0CRDg&4EiBup-jha-1 zNp0F>W{k{M(L-Vf-j!NF*>5D%Vz9zZYNVuwO7QJx>B zALJ9MlpD#j@D2$hVHzt~q854lP2+RnF%wR^asTUhIg`96)nigSCN*xt^CUefwUyGp z($7*^7OZFEP+rNW!dN6Vc+#g*Q7L+k+_B_`205+dzhHf!RI>${mV*3MQjsTBZo=3m zHG485!W(>x9{-HL@~PAXN{?LTmAsZZIkM>n#zbDpXR@crZ?GmZ$fp$KOp@`F3b){O z6c&~o!qPNMag^#wpxx1vY^Scv`=>KASar9BFD-;FF$e) z`ApgwJd>O!*dt=2%DJS%P>z)mmU${aaxIye!H1l0u*c;)@~y$o2G1J&mYi9t3*|py z+mdt3v(zezb|F`j_eDYknYx0<29K9>%K7D-!D|MuGkE>MyOPi49CG%-`;jvQe}erp zcosQQt~}Tia^=BR$$6z`zVDylnFsG9c&5Sg%3nE)FmcKEvqliY>$t^5T0QC<&@mAsa7 z%cr9I$lMERd2$B1lfmoB5y3V~tK^>Lo&WtOcpW)oaP;ITvCIb_a;3pJ8f<^?+TWl1 zf1daI|AKcsc;CUh4$htbc@Kl%7(7z`mw6yZ1xH0b3H}WB``~wi?*=~&9u-^@gQFQd zmmC#rhy0cP`o0apGyI>=!7F{=Z*ulvKYV{T!8QeZ_y5+v-;comyyyS(ivNG!8N6Fr zg(BO7Jtxm{*5EpocL&=o@5;~reox>3KRDyRZ`a^Ea!jxfr3{{QRMU_TFD?f>om2hS1go9}xl*!zPrL3TTl%fS`|$42J7Fs2F{;^4ae zULpps8I-cJHw4=teIU{&c%IU4ej2=<{3rWBum#`GCpj{BSF(oX zTXIbB3PH&dyoOv?e&lSy`<8x?SHb^+-;px~?>u<)|Lr;0AHnZ@f4<5pKa z2iy1kr}8Q|2Ejg;^9A>w;0Oi3|9#88|K|7iFnIRhXQKZGdv@^s!4}Hr^4{P*%UPu- zf;|@8E5#EgjJ{rGJ5?=wYQ+Kovxf>n1r_3{D%jt#7o=(urbhH+(G;(v*@_p$LXFF3 z#AA|Zh3IN>MbWdpMAtJ|NAS53bzDNWKsjFf3tVS7+T96s`Gx4(!Y?@ozp4S-K+pk% z5k~m1D;Ni`ozSdn21bdfAKI^ln8w%m-E5ryGumLmuOPmSAUAaqHiPsKb%z{Ni6$wn zq1K5xVa2A8B4%8`wS`08OTE@)MFP5dA$Qk}U0BWCF`DS~;_Vc6X_-mF^ViMY&cPa? z#*xQWykW7#2`&>Q6AzAP8RAFZz+H=XYsG*D`z0+<5MLU{8OBh*I+t3<>cC|stoHO{ z5_Yr!i*zs6a{`rWC#eaXpeV-%YsA`y+1{@~$DV+NWaaEcEU=(gP_K0oFIF{HUA!_2 zTg!^&k*SngOlp+K0ZH|L77*KAD#aygvlfdkA5Ui$pJyuH0fk_R&#HV^ov-Y3Tj9@| zplI-J1{r9Is*tSc{eCB2(lqSG&5Dbvb}CGyCoM|hQEl|LP^Cl-EBz&$enh+cKhdPB zPP>X>iYQl&_PyZFRJ^QDS4D#^c2?I2dytKcjHlFlz;s-0*sVI^2JJ@Kq1i?)VKddX zyOpD1s~JfKs8qB@`M0Pm$ir{4{`!?ULMo#NriIg__a^R#( zRke6+FhEt26;Tps)f^_@aIQaG+oE^_6!{B^WX&*Qb<+)Nl{;X*8w1~*QDX_bpiDjOO;~{@yc%;pLo8}zOHKqBl|j)f3@D~z-;Yt?`U#K zwv#)$1isJrJO99?nF|x@Dz9Brj`vFg2UImHAqm8&Ib;C6Ozrodw2O#S zWO2W<^k*H1eA9Gi4cDACwisg_SOEWar7JqLiJk+DXN~G%UnZGR_4v7r`B^Yd(z9z5Ubj#n^w?LNyQ`lolFfUO{_^x)5a)Nh+ z%jP>1wpTf#H;sIb^Lmr3)45Z7NYSH>ao6LAKI3>o9jfLGSMFRxg~}HAx%-6oHWf}k^|br{pnX2D$GzM4Ykda!Zr%9y zq7+*_d&y@CR}+KsSgFsyipRs@eMtL0l}T?IY8CffhwxY#0{3a_J#F3u^){c2`hv5r zVwDSK%MAb5DvfxKlu2IkXk_|}G>hPjYw%wO=j%Nd|rdp;I ze-+$O_-`wGf7Ue_+Ilv5Ki9VVYm7U3pLSmf`wthn1l~MLu3~)e546|G=^5@w^yp0m z>X)4#dOFn83}ane9M{NpF{&T-MtZI)7itv#_dMIx31DcLJsMaG%hci8m)-xMiZF=^ z@ZUIBtLn_psslZ3o;1t+K$9((sCmAs+Xt(w|4En%rH;75eBROL9;$y2uFQPJW9}4k z1|z+v9OjTS5n zm$n7Y{b5#Ci`k&FXc`|L!5OLO0r)l-!TZL8csJFa-tiD@OLzE=N zU#D=vs9MRGj`OA3#v7J_M6jc;!Si5vlp^5HhUv@#=i+%SyjzM9o?o~=(_GV)DSt*3 zv`{@4+?7maX75$yun2=ctaraJLf3!?d#L^p89gQai~AEJEQ+1BXNgN>s8)Gl%X2>M zDK{mUcX>{O3~Kcs2w9+o$Jw?``H0r-`ms~$k(_}FKU-cI&wP7~Jzg#M3 zng93t2LD3tVY+LCXSgcMQtK`5PlCtH7y6`U4ZI@9eXmn*SE4!w2FYWd2egUSL%wLo z0_5V)uDf6%1_#`#9L zj<{Eb{D;1h{Hp}pL)!V|vQ-<~{a-oN)K-o+E?4~4dDwTsm;jq2fOG2PJ47HO#|7=_v94mvfA2 zx~`h6tTC#efr&^AlsP@pr}wDbS|V}=#TG}cFP}_u(MgVaPr2gB>C@>mlzDJ@%_sU` zKx^?`?#|H;eW9iTi2ze(`0K(^d~laRC{{)HZu2 z`|R4WrXRwGyx3Xo`%)cd+@i+M>&psg)wO=^R=t6`;WDsP{^iPHZ(pQbq^VQFEB68wm_3-j(@niT17sJzg*RTA000* zySq)%L*#ZYoEJ-ogEB3w(=l|;$%iMN#~JJc`~sh0b`bq({ZRmB({ z(jEvH$XBR9&i{kD?rn;6i80_`O#-W^jC$=UL{YlDQ@}8}s7_KI_bSMkv5~bfh6qI+ zk*6eo5xVOOp4m?Dm_X*wA>y4(KI<)xxu7fqajBlWD5ef?oUdHOX z5qFoT*_-G&;(Ci$D~N=!<510sR*jFHNF%#m2la{V%KNm(4CSUs=-$g7}F8`bTB(>$Xu;~H>Qu;te24Hw;e@})6O@kM4Jx2;|%A2f%G_) zYR~hIg8p>-a9c%RZSPxP@viDiu*Y#9@HzLtO20IKO_f1(^MGcA(HI&NJ~liuygs}< zvOId{kg-DwqUJ^Hw3dYyQIY$tzF5#?FX|jIE0;83_gg7;l8j) z>-cbeggUY)Vn$fK@s#dCeWJcdldEX+EO3_DU$Zsz<@BBfMLwf50T!MVm}x%mI^F$o z_ss6)J*#ZRj`6Nvx|YDNx*MJBP0cw&A8Z>&_*$-n%?l5UaKL;~9bpciWGx754V@QK zXNj{k8Q;`}lj--YdINQ1n_0O%j^U1J131o2{TVM=*^&aCyc!>5K{vS!;Dz{u#H1YhLl(X#+lU5&oF$W zt*0JpnsZ9O(xGy_@5MMbmW<*Q;fw;iX2NSD@ zmnKFI9UpEs4%Pmi-ML9|#u?VTscU-oj-IaGU43&vojun3wj&RQo3+Hdx&zOG*a`=R za=G_as&=>ckLlBPD(@a_?zxfHvc79d->8m`JD2XwgRAa5oHXCU?qv+E8aC~L3lGF3 zsfVo`R(xOB{rdY$<1a+)GF{`2-ql`K{@k6?cdb*|)!b8S>+72h*J4A@tIqeSJ5EDZ zl2?NERjiqdMK#lac%JD7R*jpk!FM%~GVh38<$k-PiaZuEbxau^Sq&gyUKmxWcw zPaD-WV%~jq!zvP*l43@5C6x?)F#K)fA9WG>N43`jq0T_hmCi_b1nT?RdeYz($nHyY zWw5Jd5iMUE*g_S>N~#=lJ+qwC998!8-pr13Em=2DUANzPyQ{5dO52V*J#EXJrRqpi zgYlehhJHr)mirn;6^yL7Z`QB}6ZA>d_b(XU6}ODlFaz}4XK?h4cV+f!J9l-J_ARz2 z_QrK4bPczSaMxi2S*aIT82C&jsJz?!6Wtw-nT~4PD_yPk67HP6nR_dxy|yQ;drs@t z)`p(6z_0W>$O~Vl%QKCQ2^&!`YGY#J(Ah)F@9Vn1>Hd<0(x`Atm_cpm)c-{NGjD-? zIy@a0dNb|ewsYOlUG+Ui$9nHtvWK=(Z?)E6M1^~k;;83y#{pX#JP&6(_q6(M_T1Rj z3iCtXS8eO>c6HD7{ZgN1I?7V(UDY^`ai(4^;5O_fIFV3+%hW3k=bcdlzr`>DIv znp190x%)=<>F!tVZM}P`>!|k++Np-$=+EdXO)H}ui5Ko)d|%kMz@BAgjAVZj9+Oh;maQ7m<6)Q>rVIpdNqhVfA3=$L$}t*526l}QB&>*5-t=2^&$-Uz_Dd~SZl1f5(DGI1jjl0mr|wSej`UQjE@}TvZsI1B zdB}`-&#=g$4-TCjKRdDXzRHA(VOxgGw0;%pgKOsZ`YEa;uL@a8MxeW=zp?K%+uP2g zwC*>q*{+w}dA>^4*;Bq(wf%wh3j1q zXk2P+byH>J%3Ij(PbDP|j=N znmXrvi)zF5?uo8W*JEDrXMB;~JFc0|S^d?voSux%-1dyN^LOHIo@&;%eFZ4 zlJ9q}ax6XA&|O=p=rmLF_l_r*ocRE?oiDp?_t*D+)pfq}e8&yAa>H+)xw+)t+dY|G zarc(DIr=UKdcfLvRJ~HWGvq)_VSH)q_~==&FUI#JL?$d7Hg-q@94p1)GpvQ?A8N_P z!E&geZt+`qHU8k-=f3F0a^v6c(;|FuuMD`u((S;U!2% zJJ`{Zd?W69R?F6|Njfc-BgNYdQ8o*=Gcm5)l)X7`ZGo%yh=!FytC$P;FSP^pas`KnG~{-lW?4$JBEirwV(f^R`_q z`i=JS-TUsfw%l$#-_P@4R#mUY?dSEz56jZ8~;s$1P*W$6cSdMRx@HO8m<-_v_|q|5N`> z=;9bpTtW2Wh_>jWp)ba7jC(D*2oASl@INiEI!x!_rD`&tG0p@lBp+X6snojo-eVo^ zp_;26%lHmFU&-t*8|`O$a@&oqS$FL%SMHo@K6mY0~uI%`9OI=VQ#-1LIpW4vwIVg8kVsk)iGY%LkO zQv&B*KXqsMKpgh|n>DH9vC+|E%kEj;+0&-H`^N3Fw^MKLYkIqJRnytlU7h>dPPOjq zPV#CsZ-ROBfUe$BFr;AE;uw8IU1TwP<$}1e(P0sbBi2QZ=b9zvOk;_8UPuepl~TJ< z9ggPwi27pSr_|+D-~nIn9p?TMYx2Bvx^qnb1vo2OVRy~CJLmRS%^A(DO}no4TuX0x zt5b_+o88mo>Cub_HGiz(K*;PNlj1$G6JgRa4{eC=i;IaW53jM7M~siG3a>Elg9EG8 zQf@xORky31;DQZTUr@ZlEK2tg&+t9ye$)9X`;5s|)bHqP>)q46qa(Vt_4fK((YI1= zUb*UOOu4CT@9De-cSd=Cn(~b9y#AcN(bN(#ZD{1sHBrkVUyN-VRv71qo*kJOv4Gh- zJA70~p0U}ev}}cA>b!2FW-~SGAK*RQ>uZ1)Y%MwXC%jYPdO8Tw)>K!xJ-2U^EwVSG zW69mqx6j`?ed~>z+1FAUFWs2i{&v@@j`f{dheg>BQ(Tx~j=3+w5vNBV*a8a8x?zFX z!sxM)4@R_Eo8T%P9+GXG2%B7+DG$b@4>Y5w(u~%6RSmwMc{0J%cKT0(r82@@kGweU z`3g4lW?Q)Jh0mT!fzjIzVTgd!$HG0 zx+l~Zlw07nA#aqrevMz445~n@$OREY12xiV*Ma_2dy8#uZ)#@-w%IGUjx;ZCPP;k$ zyM5oa-d=@v+}M+9TkrXcwgS%32Gg~W*P=>c^evC{gfEER7^5B%pw$~g4}>lYb#d)P zT71UvmZ2Zc-2zP()rzB3pg9v;*yUXcADS26MfN}}tHGa61^K^RJ^iV+J-u&qFX{OF z?!nu+&EGWV-MVz+(sys&NN(HLI~+E@S&p>8Cc}Djnt7}FWr#FH=(gb7B_EjU@AcPF4{Y;~re;n-#7{6UFM~a| z3Y?J_T-)vSy+?Ycb!WA2ZP|Bg{jJY$-@d)?M$dQWZl3G#^i|u+dzC~sMi_Ho52-h8 z44oPs7FRT6R`^bw1fKVd@11v z1EjAVhECmVA{SqiPkxJNVn!gB*kdJVrgMq$YQU|mq5i@Ea?}QpD&BKnao(_FX?Abz zy3w(`E$!~LJ7;fa-O0I=d27kdMI zz9n*cu!E@;YC%H3q8 zrcw7GNVpfclTSb&IPaR~7}j^K3%#;yZb$3AQ7va~kGV7EZb-}9%+KW3iM{3h9e9;) zx{K5e#&pYESdVK$Rz|5~+M-5;msm$cmPF2r?6M|=bL%g;{0c^;=V9(C1IeNeKW`fG?kaMLgPa3ZcuT=B09#4&F-8;9K8n3C%{AV2 zzq7!e(^u7-*E^-Ftn|2bO)%tzN}fPTc>FNhc=f= zxEf}%WLR71uV-LfIe?@+05V@Gb@EhI;wc*ky2X>8W3Jcm7;fzEf#=rKo7Yl#)rYm1r0kc(iz-+h@wkCXIn8RFSc`@7^6FKCq zFs0>8=tk?}u#}KA*tgFcF6rMeBpN3fT3Krs@%-Pf?a}NdL!jQ@4E9PAxdwt{T>_SV z2aiXodbzA#23G54m;qpT0G*=G`w2{ci?LyU3p-AYy~mbg&$g%8=fDGey2sq(>8b18 z0psPa&fJbuow>co(6&~!)%SoM2AfipuToiWoE1JJ+!Y#TPOvPFm_1}x)XLB%xVt8X zRe{tHXVIHx>n~}4qlZ0Le_K-q;;KfqPV=aSe0at-i|W-|)RD%KU$T_S-F#|K9@Tt+ z7s^gG_p?-=j8TH-O;!CK?;f!BdmIlr-*(P&M#GRC-apB)((zn>nk~gvjAyj4ucG&@ z-j=>B+mYUn;fbv8onv2R8{U=O2{*R-tiD)z$p5(374}->wXkyYCX+U_DRRaTd)Pks zi{HlUTwyv01KN1Q0&NF~UD5RALiKSF=h8uxxJpE((l5xX&x0?riq(2q<${TPpfZ$F8&k=;SxXG41vD`g7UVhSZ(tyL$axAJxI4~BX6lvxGW*B1K=s}xjkE%orAb1`Exh##y3=Q9r0 zqc=5w*1k?n?I^0GH&bzUUgmp>usK+ zu)hXrysW<#UwaH3T?KeLGi>{Mp6hPtNwy{1PQ%HaYa6NjFP%Y^=t)&(m_6acB2q)T zjLGIT5f8@JLthwUx)xd#eju#cbP!Gdm-;o@?OMC`UD(@dsXQK|jHWh?`cfZA%*v;b zm4YFzVAg!e`%i%8aT|P(ziP^;@?L{1h$12(^$ATr4_PLkxL=`$5>7Rz-#HnKjXw8G z(A;LY4!T~473#MjCk};0d=y^vG4^llN_z!NRe5~}Y(63fjj**3>rLn{v>)l|=_qhK zplvg3R*oZQBP*mfVsZF_&=&JLc)tQMVNnZ0Ukt6b7FpMYR9h;|b4@e#Pib$!N3dPH zNSz$W^=jb8C|6};we0bJO*MZFNZ(WZ-GL9(KhR!eXWU60`*x~%KcaHKikj^lGL_=V z*8w+{IL*^u4Ry0Sh~M;q2R9DBt(%|$ZK7H{7tP_C(+m!s*?q>b1`9~*xDD&pa$9+y z(bm*g+gH)IT==;9j`VKr&9K$mw%ZQ&EVp(0zR>?r+wD(7pKG*^jd;*n#csLIIx%Ka zOp&!Uq|n+DzAm)hk`_{L$u*jEQWf`m?KSOB)DwL5u()pnJu2Y+1qfN$Fo;ld;D6sM z%qdTR$xyB;1OKf9+^A^0E=$3isUr7fEp^GDY9WQB*0F-BlT=V6CX;({oEpYV-%r87 zk$U)D)PsKklGjYvDc39_S4Cj%ecoTve*rmEW!q`1>>Ja!zBdc)JepQT_NMhV+A8{s zY*TxxZ6_3y3_s90;kVEm9M)o(zg?D8b5GcTn3C9)5i3JyT8qMOhs_De4#~7cnuhAW zhI8xzW?Hy9-52R;feXZ`s9@gw2Z(fDe-b%b`BdWC{7K-nZ3k~*KiC)BHJ9*tJW3s0 zyXs4@vyy3Bwo@5zV*Mmd;wK>wBedR9|LaeP6OIj{kCOmG((w)A;Ps_Iu8+HG1>wy1PVJeddOU zNl_J87IsTr_<yq19gisy6j>$lz$lQ3HOkSxQ^a&(Z16wkortBe z!;=lm&3lD%f)PuhHnJ4dqjDPN7JsbLUV~FGGgM8+K4*K8>TIm5_1#!a<1X`un@z| zO;xiK-sly9C7v+1!y_DWOa1lUqj*Hq$XEn>f~vb*bk_N5kZ!bgkUyRP-RW^n4+v%- zfs4|vsK&egB^Zeyd9m_p!1%PF-&WGH3STy>5Vsqe$pmWfpY%3@NV3E21}E_wFta;} z)*2o27{zkf{I)Y@CALwvxwa7dV*6|SUt%x7SNW#%H~7$I;*0sjz14HdUFn~ptF|z+!m{&}pXso8ua zY*_fL&=gCirJ2Y?qTwa|4Qz>(TCI9M94QwAcgZr{P1eW>&}ui6(Us8el^u4-2gPpfb%cz0CUl14x!XcICU{W@Ej^epH1V(iOajt}coZUs>3uh30 zssxqgA~RWJ?Nwfb844<0If$(aRxPNTpvYx`d;`)l688f2?diUH?rYNkLPi}v?78fi z^#jtk1rNHQjvr%A_qx7w{gxHn>AVEy=TKVMO1tMV+7FPiu$W$$$tOqf!9Pu0<5QSn z4}u!~FrJBe#ox4_gVX<+s#SB!*k}pClTdEn17CM?s510|B?ErwQRZEY@ay`S`a8Nm zl5w~hX_AWd49|?;fee`d22cfAE$JXyq_g8>kqO-ZE{4l!Sa^aaR!2T}H+7Q3LCjc+hM@2ZVw?k1pp!&pPLK;DxM-8vO$A45IWs#9 zJAD(oS|s^uG45k90FOW;`jhi_w01W8>@wDE1if+1IUZEv9iW(dh(5dz{@72wX3*!7 ziR(_+b$~Q`S=FI^!{{(K;xTB$!jR$4rB|3BgBnFLC*|J|ArJhwPr8i>}8o}RzF%)E)N-|61Kth#vU#C~{ ziP+auEt^iJd_H;1f*V7|H3%2W(T)TmQ1DRpGuMC29XAsH+QfK2?D++4|2vVXCkJ9h zdbbM9_E*`Re#tH@agC{Dbj+g7W9gqAAi_q{TRX^73ozCo#DO7U0TWS6#Nu7J7(Uld zgPC=%d6cCM`CMzx1!FDMTxza1*O*3_F5$zuqMgb-U!+lh^0}71@-saD7ZuzG=Xh-2wVg!y*xaSjdk$@5ciEd-_yNiI77C;9Mf3S z5>p&}?7K|sO?{@hrV++rjQu&CPWKn>KS0_kXP?=t%v7d(P$N$<-<$>tEO>71T3~q*C6%44;FxxB<(sf}J}DzueM+ci?*hcR)dH!)AI_m~Kiz zE?7Q*f+7fdg5`Q>05fqe7-D*O2(~f{mhku~_-Vsf1OJ3=Ad&wbqpc%&M@D-WI?Fp? z3)a(XkMV6m{mnr$tpy9W4BcWI{)$+%x^6`(QtoR|bZ%=@WJAm|JZMZr5Bb=z-l#Sm zG)^~8H98D=`1bu`x04t0Bkhkg-Czve;{KbM3xa~Sh>^=-Cwhk+C=YxND~LB5`X&SH z4nZTaQmda#+b$y$YW|bm3ay?y%uEUoA$)KD?A4x-4#o=T~0z(wPvi;3U()A;uZzggJ=Y?s6Dl3t?PN$^O;`6Tzwr^vkzUDM*f%k6;ONZY&M!QlJOnmL3GxtX;t#hm>)dN+=? zeFNWqo$p^^9SLb6#0`alh

q;ifAib&TaGoR-<<#3-mcn+V840c ztD~KL<4sQA2+wxnyJz`}9pk$9Tdq3`{AKbsy)}Ai-whF)jU#mZ*-2bhrK0f}9r(NF zyz2+wAJpp%mBuDw{Y=(e01ciA!mb@0LmEofcYDA0ZUP~a<``j-cb#u(Xvn&m_FG=# z>zdd4wS*1nqwTjKf>qc5w6_?~`>&pS+wrcTb*MYvuU9V!-xsyE&jNEkifzBb1lmR< zfsfez1U)~y6N57~nfl{eJ1DMhFl~j(W8jAqh$Y_+*Y6RK7+-k~xvFgkyZvpm8guFs z8WVoYXde`z?Kd@Ia`d)ns}W2k48mwm6GqSIp7~uZj>kNA1Zs4{!@D9+grysf=|<_( z41XcMSq|FBhtT9u1$^?6D6)J4QtC_9W!AeQ#n&)So>0c45WNl@Mw9n|`(>Bjewb*= zEzS9j=NnG^cBLc9bSbVb%oMpY1^j6DJQz5GQYLkb% zFC%t6>#wpJ1`bx7%cr!-9JFL1DdV!F5llh6CFRgOXcHrdBn;60}Pua(3C^aPv1*~jWQzQ-2>KUnP;r~ zlq<`=hq%=-s8z+Zq&DO?pO%g7TNgGXtgTOzVFuB*i{-bYBGVk0YhT>Cse7h#mN!$L zVjL8aVjijUXx`AK60PH3+6MJjxzVR_$(^ZzIqI{zI?es;+3JZ%X9n-+)lby;WFoHI zBkX(+gY%m1sDf{n*S^1HOY_RchK2-ZlDQ=8h;dG2Q{({MT4GQ?iBi5rIl+Cz*3`4u zn&dd``%Kk=QrihI)$`E`8HA$PPbg-k$zsuQT<$L=@*?pa)DNO$cLavt<7${KTAN}@ z(8sAW*0M^}h*S9c=Hmb{s$!R0{*))`%%$$AR;`F#4T>DITexIl! zz>!A;cKFtL#)If{xau5-Emu0$bYyolG^MwWMPqgz(XRUth5DFw1&o|DqD5cP43^Cz zBHK@BelGF*m5oG;Hi6iB6y$5M(x6eYbGWR!fx5*Xl((x}`2J_;Ja#iu%Vo{5Kb{1e zdle>o4hmaOd0zEQbpOp&Z8yUDh=CV$xM^|c)9%^m!QKD?KbBMJ2GuU@OtaoRnRb?; zRx2Z(n@VU6Jp@LhTwMYdcN#gntV~x_ za?;E7YdwcutuDVa*_m#S>KWC2w8Pw%*_MV*;2%Lg=fV^zWPkp(a;o+;k!3a!|9b}e zpRJgYbHb z$cFjwOZvKZI;T2z+A~-OW_Hi&oY}p>HWT%oOWulr59>;4u$|TM3cLN=P{W$exY))% zBpHYdKGT3bx1w zinq$P4W@}-fSSqIvvUlH+%I|c5v#;1uZ44ibDNEV1XnHsjxBnu;HbF+Y-nX@vgug z34cKeN+!khGoiJ(jXse{iW#(72G5>BYh^Zgu@d@+u#U>e!vl;$p(3%I-yB{=^!lcR zYD+Wg{2jd(&0$=TMcwC950gkWkN>N96fl}H$YC0viX5Dzc9&B>XTS!&Lh2$FVcE^# z^Eie=qlr9-J}c0L0>vmS<>Eboj}!S6R(kP$ zfm{8Yvf&&rc`{3B;xJkQ8;q{iXDJZ}{)~hWuobpF)&L%8h7!@oj;p6zEOS`U88v zAMzE7dx=tsekIylq%Y-8@~sM$9qI0p*Glvm@gw;gh2oH?M^T%SA3|UyMcLZvF_pce zM)ZU;z2&U`_uU1eZ7_Ebcf5ywpTqA~#>0F@#NE7C#=Ha`Sw;+S#csy>KDn6)!j;TP zn^`NDfcG6Hix1|@^veF4CiG0r>I~UeAlLhW+g}0WEsm%X`MyQ|5*Q`ub+ST~<8>&5 zp_Qf(SXQ*L(4YPheYRDBDZwg#lXnX%*+WDj&cdRULR{l&aLSj!E9Ws^uL_u$kB+d? znCSZ%l%WE2a~jsVc7HOn=?|c&o4|8N6NyzQt!820&V@zsG0}am1qS!76vd2-VlcRI z^xFt*TG3$H*E4J0!+gy-*T!kyQ2uzkzxKFw5f<$?^hP+62g*T zaj(?h#hK+}FwuJDhwAq=b2RhV39i7qq11I5(qPYS*MF%t6V(7-7v6|SxTtTTKwLo$ zRicTn3q0U0cC525CZ6Qt_M@ndxH~5}?;u{HT-ks<$cRSeRsCzGYSRhbc{~RL<)5RW zpCK=XPx&GC*w@iCo`8~yRUd9#ra!Nit4s=65N;_9qDA1dw_@+l2Iu_|r^knUW84|G zmE9{l>@Df&1n=*dV^2eQpVaeZ`2OZ ztCRpyTL4$v_Gtrjcnc3RW7=Lrq z&HT@}pRM-qkc}!}T`}3(Q7ND?&+pnQs)Dd^dutmG7$^>~TjM53;wfGNhWT z4SThps3VBha5sL6GJj!Ec-6MTd$SSG2?nr#k7tA3+U=)2d+XIVlA7Lhx1*?jUQJ9) zeWYp)I>(j9Ip!Yr<5Lt}Xesn}eeZ*tto}(GZcK;8JDhm4#im-L;O;)7sz}x%Bhl3i8?|jHY(!yWJE*+? ze>B!R-Klnr^bo&YJwZ3qd?kE?xy!K6m}ssxZpY%aR+$=n9=`66?*B$xN!cA3?ny+0 z=XO`6!lNlRltd1V-p1J}$5?FKV;HVop+2WD1-E!o9OH@iNW^yHeI&4ZEs**AFS;+< zXIau(_ctW`mh#I+OM1?6X#96bAGY_m9f!o5ZEnVRF_ZL#C}diz`q z&oB)!{H9G*l@cp31!eZNjyGH{d-DA0a=Y?9?B4t2-}@Fg3$2IR*EOa#p1*0ok@X{-N58D#a9{ zojF`3j(CT{vq*jjZKILV;eF>u%?M9{(|V_-NIqXS8}-_izK6UkIg2dwO(DKlA~R#Q zJl1D$UG2$k*Ei)i>KiQ$*|rFj!2V0KHgZPz9Q7wCeP4#hVPiJN@8<3A`m5_3PkSIi zwZ#w>SsODlW^))ZLG|^@b8rL8IR7nT{(lR`bS`n}?js6HA-oV-V6JnSwV`cY!?NaO z4JR6gxL4y9=+bQ?I&mG!P{hMk`-rBUgAUTGo&}s1fAvHJi9%o~j3gdwLVic107(8a-2Ws`nFcwyLFbEG-Su$o%Z zDXwukvT+W32i{L=;A764m)QyEf|q=CuKhi!od?<$HmjSvdf!-r@(F!rfk-A8tb*N!O$8*XT@oS=C41NVSOGGsd zdK+D7*2vC0q9)fhDy>E$-uq-@;moFSMx92f2jhjCgU7p4Ze#Z!9c=gSlkeBAG-jB0 zhNYOV8jfoBDYMXw{zCarbiAjr?qml(Vy*ffi%paFA#b5$p=EEUx;>>OxqYegDypgX zs>kSwv5Hk>3DyLa!mS*KBHpUry;mkDqIi{7-(adSRT{I5hxIdAKlY&qjx7N;`&-1` z+lmMJTYQI)z&D(SHGWU`y7tAb>Ft?zr}rywp(4uguqr+{8$<*ge0hUfyaeZcE$!?&%3Ng3-|O5~JSOI_#Lwt_Z` zEgvP+9rA;CxKbG-Nm%7lSjRV_s+*!r<@`R0w%e)vS-Z*LHPjfk7>;XWRLv-wrg6$n zg>PI%9Hnmfam&09d3uP48c*I!dNx^7MAU8PGWe3UXsoH#I?8z&)tHy$CzW5QpT!T_ zjLvR0XZka$EojUT1Hce&c$fH-XUI1Sb|_^hDesV#vKt)c-w#^k0hF+I+R}TBJq2B7 z+a_3E^o;R*>shKiueK36@g%xMDcB+Ag5XGkY3l-;fb&%a!;6Qsi9c!-aoIeZKy=pnj| zYPISNyJb0eiOocPpG|!GJ77cqh-C+!uv2Ys>e<_^M=3tbu4NVf+_#T$agKOrpRkgZ zu!>IAT2!O4W{J5v18d+2y!2yH5?1PFqWka(N};`!v|(e9LK$$fYqPBi zor(Ca{hfZ>dDlkQH(tRvz6Xt&Gpa|iSSwIepRDP?`tln3^3M{PGZw_bNW9H|A?8aG z_iceCUxJOM92JUeY%kgVb!Y*v^z?UE*vqYxtb>SJv8!hSkxHNTBoYIUvlxA_f&D7+ zRJ141jVi{<^(?IN3&gJ(q6?tFeHg{#`%z?Yqq-bJS+lTgrg9#b#UA-vPosOBYl5Q` z&FM)!+q)8Nf1wNju8Xi7y@QIt5IFO7Xh2rks;$PJ@fNe=Ad#65x>xmT4ZI+~rd$DPpf9MJ zv+6S_HyvUmZ6u|!_;Ti{r-Qz^6P(T0pj4hzVpn7Dt_o&h2dW~b(sA%SeOxVQC8*im zI4u@?fA?H!r52U$a>`fDCA50>XunjirLH!rU&2c`kR8M~oCwx= z4|ra7nVsXE!e1JRdi)@FfA3qFD(sBRL7kcIj|dMk?KFVY*(0K=bX!sC7k&tD_G_D2*(uQ(zd$>@ooyYKL$TQGx& zRGmbXeZV(3a59+A%(enF8A@5|0lHHtxv$Yz8}f)?vt3sJCg)}CXKK-(OND*|-cgW9 z%t1l)r~44`c`nhKlkMd_$+kMj0_RT0YmPfzZE&8$eMLmRO;VUS>0MWKs84A;+WBC7 zYEi6j)5Q{Bda^E7_nh{9&RdDhaW~NP@nPl3;hc4q9oH|Or_doSbe1~uZ1b&2_KD63 z-~|$#9@ievV()r?G&(WqAXC0ZcjXPu>%>SX2RE{v+!M1yw-C(DG0hinxg+rlm{_5! zIs5xyX#W5=`5y0Z&p4ErjuU4l&zfdC<_JUcd?^a+PkTN_$GRO=()kf%))Q6LJ2gkLyYyL_+vwN9Ao&>)#44YmRx=6cyIlfD*iHBXUIrSijj#x|S zy<=SQXcUz?p7VT!wrq9))l4k1;M|mBQA&RgTlyjGB>i^g{S&(3dLt39CZbS22S2=z z@w@|jDW_cGJburr_#^E0!}y0MqE=r^{D95&aqKUX998z2u7`*ab%dC8AIPflZ>(25 zLHvkMu+ksW?475+!>Zsw!@jJ~nX z7{^s>5izT0xFU%Pz1jYp`|l{e=5qf6Sv6V#dgWBSG}ozzIhq3fc4D{W=#})uN&2bm zI)ZD7}4|K3sp86Z=_od_}aW(P*mKK$tuL8t*Hg62J2) zltCssBZ+KKXtz7Ypy-lqyDqE`-aPLn?DfUibB>@qu#FYrYfw@{wY9n`eGTJi9VoF{ z{Ux1NHyON>9jsF#d!A1*h>EPnHhKW2KyVt{Yh{X*(05A5FcZhcduumubcfTcwu~W=M+zn!bHsw zMpl<@i#}W5r8|UfZw+w}&eM->SI-77wv90`3k$mm&7C}dzVGjhu%qtxnQbPc!x!fm z%9^v9SV!^B3GP3E)%ues+xLOr#9ld_lXWCIQ{b+c!@{v)EJkC`s9#AtOeNyb8`}TU zJj{-x2qpa&6sxeIl?8Q_Ws(1G-}hiK`=e~!;oR&Lq=~Xda{O(H;W{*hK7{a{V4qdWS)t&CpC`Ez+yv*l(qx43LHHPCC){ ztXRqJW9FzZTB2kk-dnKF~)NkOy(xf$3!pt0IN@FFbQu+9Qe}L7#l<+ zAd*O)E=r%PkJX9^&!61~D8&TV zcH)>#bofz9eZw8)*#dTVme+_TO07R5cprN4!QG-V$kCaA(oyhC$WsRVX4-rlLP9hLY$LIQ?>ovz_$9t@=&$)kg z|A19di*2xixUiS7_-CW$awllNd7wUC)gB;Xkd~2nlsQYQ8>@XCo7&^(6eprB2UZ4e z-%d^ulTgfP13S9L`$tbFkpMOBInMKr=iDRMA#8B?9RlGv0{`b?;!S+xyNR~XX5!9u z%VY53eW?BmYCA)*XbCme0{vv2ux(A&J*~Z{evn<(<0?H$emX@rHs3{P=adGv`qwZv z+lXAy2}1S_R~h)QUqCreaHiU4I*+*(V0QPo7vl5#ioUHQ;>loiK8`3htImQs77>UX z+5&wzNTfERuJll!Q<){_qD=BI%1UYMo_%<&+Qq&HRfq`xeatkXO-7;!)7Q1lKGiYN z{V&fPXNP?bsl|Fb(Wv>x1CkccA}S47cF-PSSLH@Ns2Aens>i!m z<3EjmaiD*o&*_czt>hG=MOkK$dpfZ|Vo|l3<5JqDy8eVN(n?Pl_1eby8a=r{4KdhW zl6Rn9yqefxQ+0!wIjp)%`gCKBZjR=tHV7VIDvDIoROirb^HKlIm(*oC-ow-A$P7kB z<1z1i|NAbzZH()cz$3l^p7}(?%S4kSDfogskzQ9U zzZa}xo#ur05LU&#;MLO%0}KV&erKqAh=cMtt6nOaK7-{)@JkG2Cw`ig^4PPk2aE6Y zohEL@LagrD#K`%@^A&w$jrV<`k!*GxU~jw9v(sKq+U5T0K)_w-{Lw3Pc-~@fb_R_6 z{rJHr>PDL~P>QQTA2!aIV?4xsd_wmGSc7A%^o5EV#?fV=xP@ALx4(c{ayNSOudqIT z?p+Dae<3>Kzqpq%%dd8nqU2eFFFw{vOd{`Q$F80@dn)nbO~jBE+R**gzkqU_WilFr z`ZA+F>`GX&xy7{DxF5t7EC7(JX|nZ{XJEjDT67cM>-)Ix7Hy zL{<|KL_YUT@@ui$@3dL1qbz$mXIYX^xVma>>b`2b7Kl@pF}qJAR_7Y99))2O!}Z~F z!mgVO!z^gR&NqzG{egM&EyX2STJRgt+GE(wz5yr({Xy>SV6?^YeyVZ3X zjrK`SdEG%BXt8ilnqAu?@62zz(mJ{!t+A40FG=4=92ZvxbC%^v0w;8FOL)4eEnn<-! zTw9<##|gU%u1Gwn3>k`5PkAT0M}Zqk0>3V|FC`vTzGZ0l_KwuHWnEjmbzrM@FzWZ~ zw9ym#q?;>E<@ytvWRSEs*sHDd^#_sI>i*pS3Z92M@bs*b$El`r8mQ7cnLpk@zez*1 zuG=(;3YF|hR=gTHF$w}%{(9Eg`9v;~VX;d`6|Bp10uF}7ndYdlm!fvpWQpot(|xc* z-qGT$Q*AOhG&cFWnvp2+&kP%l4)deRPyIi83Wzqdn;5Pl3QivO!0(9Rx0L-yxUK@f z&?PYNZ))FD+ffip#j@k#4E+EIjwOuFua(W%IWB{Fo(D(eIx!it@S!dDe~UFH!(p`M zShjS{>PqVz-7&heq2omRq4r4o+uF8>1?GjEL`Fugjj4=WW;m->$wztC+LNt`ww;bc zP6cCgfHz5D)n%Kmnzj)!Ai=cGP|jZTxF$t$1b^K^VtM_Mo%y%yz3b&GRL_AO8^ZcH zKv%$;-^ea0j`)~)%q2avNt&&|IvT}FVz+i(?>gH(nmA$O(M3Gfoi9VdyU#8#H%Z2$ zL~Na)OIEfA4!E*y%X)5F1Fo&U2x7H5* zsnnGlOU$*VN&4#`?b}&}p5#1N;O}%FcF*@M$0ocQ?JtjJF=wYqhMPoleNkJA-SsWJ zMX;zqbqM9E2v|M!?Ellye|rV|{U%h4Gi`%<)^wb0TgJ$3Y2DS>&`yk_e$!)Dgw+~$ z!h{$?RM?T)XvH{ZZdZC+OVd^dt|%AvYM^Xjm9 z<{)cJ0nr)t>TB{vfikc%Z`l-&=2 zDog8#Y|XpACSvbwL2XXP%7o%3@z{=WCUB$jScS#FOFV^RVAxNzrnSek z{h!Lt1vttoOThh1Ixh%fc0kAoBa}oyiI9D-@(K3St%+9y0->6)|c+aFGFxN?dSd1`$F!Nw-NS{qFrv&fe+(TI;4O{rLa; z-^aP>nkxVXxOtzF{k|Asb`FOGHOL7H$@6Amr%- zT#3ntl8z=e!MRp2$JN$+vb}%E`FbC5<<^GQhI@y97d+Zk=eyOLO*HkaE4j_jx>pd- zzQD5lru~D?{@riWUr0NVblmR1^J7$GKq%y0a>eV;>>O)(iSzb_HUUq|{$gj;-DvWObm?8-=~VXtvI7Ffutf^p>Fw@+K@K zZ%N+m3Xmyyb9Y3wdDzVqf}QWOuaAw);H33=8}@wckShQJg3IJVGslvvZplG{@j zrA%?ncZ{@N#5>7CGv*TY5L%quJ0%Q_pabCV=jafFz`zeWbxLiFpI_F>UG{fm8j zyWR~hjecw&O%AvNa9&yVugF}mge;^-@S~VZ{@@RT<0B)m50sebUpj@hW~Npr-{Q)! zljl3s5Xr}1O`?9)i7hC+=m}0@F9suBd7WP3=@)yyYTeaR!Pov zYeldjx(tcW0j`y^OZ_f!8YjbFp*x*n9Tfd8X)&>`0}eby6Yoe`kvtb)ACD~^ zjaF%7Fus=SSy6sY6*X|A>F7h7k&XH~mv^4(ScJ#hH1FZIjFyY|11<6fQn&P36%*V}FLvs)+e5H%9V0`n0_7>CNsNP1v7Wm$p6mh6IV(IMR`Ulr$W98^0S{V0Fl1 zd51WkTO1zFPRs1Q63@CCIBUOUFO98XXCBCjZv$tP%U1DRyO&Huh4@T7kx+-7cnznQ zA;hMw;S^pRaYUA4-H0C{@$wJ(=XX90cf9et_dVXWS6g#h@Q=Fkyzj2wCkB-FUEZr8 z?Vl39At0S0@g3R*nMC1YRsRJXKSP1Pw3&(lP(#L+(+2G1aS)2S!Vu>AFR`i9_ ztl3Bo%dqQMiWNl`nvdDx;er0F{)@hgonS)8U)x@4?%%xCy}k8p$K2$;14n1A{`c8j-|v2Zzwq4c6-7d$PC^W%L?0TjzbB%@U^Om_6@!o_#62T&qo(x z5mQZeQ-3rM53}vAl%yzmNoJz)^<$Ij!9rH7Njr$j>%$iM60+A{B;JlSjROm)!Dw{C z^mH@!tZm^OA{Xu?e)v)UMc@987kt_H5*6Wz5NXY5efH8`Z&h*~*@5|55O1ALHk_O0M+rG1kn2g@$8Rjt8e^>l0!vEsKljwbvGY3}U_8>p=wzmW{`V4<7k zoRSU&nt`NjTX=WiV&^Kd`dWS2MCQ+G-PrPWYhLq607iKN**$QpC(Hzckm$nANw?u+oE4I>daht~WUnXHzCMn?DB zK5-sGR+LPv{u9Y@n7T%}(8&Sd1q5J;%3+7hGBPCxelW zBH!V_vBJd9W_?-TRMGA?ex|ysP zPx+7e@}L8Ky>|A3kJ_wlhuqe-Qp=b4qTJz{jpx#0=V$l{Y=s{Q*i%9$$kh5~&>zKO z)UpM;>YdmHH;@tHU(l<)V?Wky<7*-M<#b}}R$I&1$Lpgr*k#|t*D3=!F1l`_Zl303 zZpFT99@tsY9W9$IUqm*aL-vv!H0ORir`Ormlj%8|`o~D#c}{$9`>Q%J|8@j?o0T#=X@}(~CztJkhVUG$OiQint*MNccmh_R z^O?j5Qol`Bu`fT%>2W%-y9ZhwFHd_I)_G?UvQnrv8%}=#(0He zBK8a=j&$yyVJ&139lsI^>mf$xjqJ4j$Upox*@rXHA`buV7EhnnmX`dMNzKJxi-jDy zNbhb+cntfuUF6NZmCW*K&K+oyw}n!P*&1diFN=+cEC`*zo-B))!0(Vvcpx!&2U-{6 z`THnZl!4ZcXgbtsB`1rI(YjQTwW^TluIQ%WWzIpBQ^A0B^o!Fs{jTJSn`pvw=y|{% zMxxr$y=SH3OZ9B9sVf(qLXvOScXrQ4XltH3v*m8jj?iJKrPIEg^WHM&=ftt@N;<{4 zIN$OGvhO)~-loF69Y>G1BD5=56REXkAhC5|Yk7gls#Vr|kO$pEgymZF8RQFu8`yv+ ze3e+oV|Tw2J1xAbkRs=e>~_c5gL`=A29gsyB1PVv{E zEj#0F_FV8xYRhQd>%Qo2>dFH8o8Tf(qapm9OhLU}k<^XJ>_5a@z)l>E#9$~+t%Zoi z!TZCPqaJdQ6rx9v?4&j9@DtIVA9S80*Lpg3PuTL&@=QlLtLAkq8glR&$itfdGS>Y0 ze9N+Dk(Xg2r#CD3;UJRs0rc2J4v&?Asp1p;1+v^s@_NO{(6;)!lWkcoR`jn_ zgIoqa*;udmv7{02> z*bw9cZ;-XT6FK@xG6cPZ*ZW%Um#xpX9Bp&@`i4r0)ITWdfith-jI^Ct_ytK(XC3YO znX}V)WxwU9w5^N{K^n3L{mNWqmX#8f78{qwSOOV~S0(O~CrLdeA5RH6@`fo9cN>Y-M3Ru6HOmxMn>tNDk(L;goQYsn{m z*t^=3)3(~Ps`F;_J1>XO-;t}@g+%cQ=W6njHxfnQ2wE5`61d->kKW*LSyMT;%*NWK z0t%jqUB%1rl-r{9L}#c)#v-}T>+p4Ldgk`*ih}_{W`~pAmf5gcr!`Ixgy8RBQesM=y*I&pQ{6R1j9)`B3 zKYTRV3D`%SSSz7Vgi;;FqwxkT=CHGYZ*hQc5>56TJNzc%8+)ST(UmkH#aT&K2saY) zKAcgAU4W+kE5}8w03L-3d9hQo!ZB2%J9JQTKN8krBt@lgr`w1VEFKai(X)8ne1Oz= zS=X|z#jN*gUo$kc22J8m{Z9t&VRZ~Ih1fi;%!K` z2HRJnfoni|TY|sO5DW45k&3OsdOZ(0_d;|>Batv>gNe(DFns{p^D}z)A3$=7^AqTh zm%XVL&y;!ST5cqc!BQj(lYmSyuQAAD|AIF4U-7ZK$?xs7c0J#j>B}Y7!tCL{3=}U1L4=Gk9J?rN^PGl5E)Gm)PQ%N4%HC-SNC#Xqoeo56r|?8toCj5m0ra zx1@^>6#J=z_%BVxZe@dG89TTYsn*Z&ten~%WuXL2FCqs1Ce~dhvX33uDXicuItB{V z8*QEV<7Lvn2)WyTizfV6ftT=EdonnJ5^KX`mx!)oFJ1=~8;3mbAS?ZLEV^C;7H^Tq zyaidAn{`o#XHG7Xzx71um)$hgWF)lQ&y0Sw$PCynR=a&2_sq4z1$#3hM1jSg5a5N%El5PE?2-Slc5wpU~hWzP4b0j{Fg;+f@QEi^Lgx!_u%r3xRc z8)KX3)gW?`Z^WYG?%>~u3-leaZFaL}KPL}{EfDqpGqS0>h)wivXaRc8{+vVcguuqr z1w4+hw+K~`oa5tI4}WC+_JLL$0d8Kr1HZxUZY2LDNU3+yuZ6bhaIrZ+ZlUM}pdRC~ zo;byxlMn7o1oUF;e@39s8$`7IVc>r>@J!$#uW!{IG%lE5>^`bah_U&tx^{-Sb-+J6Dj+*f%}4eLp#V!fcIed zYxGN^^C^U1n+-Luf}flT>}O)#lf`OV%r0|~wOxYcmCV?5Y{xRl2{94;=?~tnq$lTq zM-jB)0{i1E_J|s2x(k^@3EG}|`~#cA&k`f(Ke1hY9H@MV<x9>!wmzwoEMpFZ3l{1m#l8vp+x$jF9~ z=ib9gD&S1L63)4lK1^jLmtg%km`q#Cp>&e@s~l?d0I}}U7VkmDNGwG-GHApPZ z;fG0$ky0@8U3~F}ghS}w7O@xn1lx`;f&;*h4?=^w{YOXPy=ftn!%&_b!`_|^Me=k* zAP0KygWf;F=$~c|O6b7@_^~!J1JhX@73_EE-K{&|o)=)9SIwHphEv#veB8sRMz9NP z#-=8P@$aMku~_Ur3`Vp8`C_cfhM|9KV0~VWF2&=FXa&eGPLZK-4s@dg&6ST+_XOr( z95X-C(H~y^Gw9d^W`8Ffu-jV6$onD7F9hy8$?y=y=4}m{0?{QF;V9vK~h=?#tGp(C*`$@+#RAoycDY)2if?LRQCa@`ydj zUI%@W{iFm+P>OWp9P^F*0PFT5v=gWBgdGnzcLFadY_0Iw?Ty_*1KzsDK%$`=+iIC@ zk$M$^arvA92EoBK;ypbFoX)1@1K@ZPtsVga2WWLN97Kn$nh|?ga|Kv2??b0D7QdjS zK-*2G@JU$ap26d378u`!UB;VOO1+53@Lcd~N%+rbl}B)oGYhM;mQz|8wvOVRJA|FR z5qp$5thq3I+hnLPRy4d%<}{FICl3%*VLa{1cn3j;*T7w}PJneflz1JmPUUR*I=PyP zBKeVFz&r%c*37>H+%JY>JcT@CJY3m1Z01Yxof|!mydaQ5Y(q+r%c<5)?RDT8_TW&Q8DLWx*j$fI#V}62 z_#2>Y$c5KF4sTq9zuE=7yt1it6LJneaXIpV=~T{7dF(gY#2`z>)~W^zom{xq2Fi=3 z$-&NC#@-B1j}1f>baoD-&d1AOVE3yZuuo?f$f3pcK=(5It@ylE(5gjvUU;@z;+|Q= zj*d}pgBv~umXnEyR~@fySj%BS14oZn2{W6={^CZjv4K{H(y9mUt&W~1|p#qhRcOHJsl6fPCac5C~CVW69-*SMb_&QI9+Am~2Dk=Xl(lg1| zFpydbn5o%}VGX0+MBQ)DLN)LEz_vG-NA(Gwg@OU z7+Mr>Yw?a1Ur_Phtf#JOYTN@A*xCK6;#(EJ#Ur?y`w}}+a^z?p4$T&!KBq%yc|>=; zW2jpoID!`BQ@mx>V^+O#)jw9fV@;e&saO0-FVhcwskg8AsGjF#;{J+XuK1~{U#j@J zia)Bx&lPW6@llmHnHpnV0TLryu4}uB!8%?o zEq6`5Qj`42>}z{+&&;WQDsb3JR&S`;VcdBvu9J5u)gc~#iQOKbhw zKOLFCT1G6R(lKZ)SNC66PrP5|uJ*9!6AB3(m%K{}oo{JJYv_q7@$nkC>!}4sQ$7yk>*1ySHZ|(6A1bAHuYWf!{`mH!4tYwUDxcTa z+VhIQL|>+-Jx4Aj^}2l1dgI@;SLS)Mx(zl-dDCxsQuF92y^7bVt;J_nYS%xhH(uBG z_rsJmxP0|9|M%$4^Exs!KJ8unT}#CwD&-XW%vy~@+1!g?F>S;@uf3=38cYRt_(z-^Vk&^LiJ$s;KR zF=|{9*|Buj(s9T=vqN3IX9|rKnlHSap~?y+^SnN*PxssxOkYgf`jqxUMrxoYSELPz z`mWDwpOm`G(+V{+A~Rm|{P*FBwyn>ae)PnYIDB-*q)Z$_SL3zZQ)u^`RVlCan|?^S ztN-MFe7slRlV3VAvt9&aO%%nH=-NTTz{tj%0bE#dR8#k z<%$`V!85@ngO7SmpOb6)+|{&7X{KfqQ8>yKDJj|emSs%l@7ibOx;&}4VrEO`TifY54t-Wi3s&iUgI`)+Ymz#D_%9BZ zA1W=^4OHcu8O_z@e@s0xD#6aHpOn7I6@`_yByAWz)QsIgCJr-s%0NU?WU$QKiP!kUaEjm2-V08c`H}C^j?xvms&lR*klM|B#A}gnQj?aE8U%u79^>V- JHTmt2{SWqU=K%l! literal 0 HcmV?d00001 From 1300e764335ff2ce612ba2546c7a0af4ca4db932 Mon Sep 17 00:00:00 2001 From: uranus0515 <110005227+uranus0515@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:27:30 +0800 Subject: [PATCH 09/13] GNNE-1714:Fix/regression bugs (#1117) * revise bug for ort.transposeConv * Avoid change caused by reference types * Apply code-format changes * Change equal to similarity because of rounding * ToArray should be called outside the loop. * change name to Camel * change var name to camel --------- Co-authored-by: guodongliang Co-authored-by: uranus0515 Co-authored-by: FusionBolt <59008347+FusionBolt@users.noreply.github.com> --- src/Nncase.Evaluator/NN/Conv2DTranspose.cs | 95 ++++++++++++++++--- src/Nncase.Importer/TFLite/MatMul.cs | 12 +-- .../Evaluator/UnitTestEvaluatorNN.cs | 6 +- 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/src/Nncase.Evaluator/NN/Conv2DTranspose.cs b/src/Nncase.Evaluator/NN/Conv2DTranspose.cs index 626043680e..56ae681279 100644 --- a/src/Nncase.Evaluator/NN/Conv2DTranspose.cs +++ b/src/Nncase.Evaluator/NN/Conv2DTranspose.cs @@ -27,24 +27,91 @@ public IValue Visit(IEvaluateContext context, Conv2DTranspose conv) var stride = context.GetArgumentValueAsArray(conv, Conv2DTranspose.Stride); var outputShape = context.GetArgumentValueAsArray(conv, Conv2DTranspose.OutputShape); - // [w:[left right] h:[top bottom]] + // [h:[top bottom] w:[left right] ] var pads = context.GetArgumentValueAsArray(conv, Conv2DTranspose.Padding); - var outputPaddings = context.GetArgumentValueAsArray(conv, Conv2DTranspose.OutputPadding); + _ = context.GetArgumentValueAsArray(conv, Conv2DTranspose.OutputPadding); var dilation = context.GetArgumentValueAsArray(conv, Conv2DTranspose.Dilation); var groups = context.GetArgumentValueAsScalar(conv, Conv2DTranspose.Groups); var kernelShape = weights.Shape; - return OrtKI.ConvTranspose( - input, - OrtKI.Transpose(weights, new long[] { 1, 0, 2, 3 }), - bias, - "NOTSET", - dilation, - groups, - new long[] { kernelShape[2], kernelShape[3] }, - outputPaddings, - outputShape, - pads, - stride).ToValue(); + var inputShape = input.Shape; + + var outputSize = outputShape[0] * outputShape[1] * outputShape[2] * outputShape[3]; + float[] outCache = new float[outputSize]; + Array.Clear(outCache, 0, (int)outputSize); + + var gIC = inputShape[1] / groups; + var gOC = outputShape[1] / groups; + + var weightsArray = weights.ToArray(); + var inputsArray = input.ToArray(); + var biasArray = bias.ToArray(); + int inputIndex = 0; + for (int batch = 0; batch < inputShape[0]; batch++) + { + var outBatchP = outCache.AsSpan().Slice(batch * (int)outputShape[1] * (int)outputShape[2] * (int)outputShape[3]); + + for (int g = 0; g < groups; g++) + { + var outGroupP = outBatchP.Slice(g * (int)gOC * (int)outputShape[2] * (int)outputShape[3]); + var wGroupP = weightsArray.AsSpan().Slice((int)g * (int)gOC * (int)gIC * (int)kernelShape[2] * (int)kernelShape[3]); + + for (int ic = 0; ic < gIC; ic++) + { + for (int iy = 0; iy < inputShape[2]; iy++) + { + for (int ix = 0; ix < inputShape[3]; ix++) + { + int outYOrigin = (int)((iy * stride[0]) - pads[0]); + int outXOrigin = (int)((ix * stride[1]) - pads[2]); + int filterYStart = System.Math.Max(0, (int)((-outYOrigin + dilation[0] - 1) / dilation[0])); + int filterYEnd = (int)System.Math.Min(kernelShape[2], ((int)outputShape[2] - outYOrigin + dilation[0] - 1) / dilation[0]); + int filterXStart = (int)System.Math.Max(0, (-outXOrigin + dilation[1] - 1) / dilation[1]); + int filterXEnd = (int)System.Math.Min(kernelShape[3], ((int)outputShape[3] - outXOrigin + dilation[1] - 1) / dilation[1]); + + float inV; + if (ix < 0 || ix >= inputShape[3] || iy < 0 || iy >= inputShape[2]) + { + inV = 0f; + } + else + { + inV = inputsArray[inputIndex]; + } + + inputIndex++; + + for (int oc = 0; oc < gOC; oc++) + { + var outCP = outGroupP.Slice((int)(oc * outputShape[2] * outputShape[3])); + var wOCP = wGroupP.Slice((int)(oc * gIC * kernelShape[2] * kernelShape[3])); + var wICP = wOCP.Slice((int)(ic * kernelShape[2] * kernelShape[3])); + + for (int ky = filterYStart; ky < filterYEnd; ky++) + { + for (int kx = filterXStart; kx < filterXEnd; kx++) + { + int outY = (int)(outYOrigin + (dilation[0] * ky)); + int outX = (int)(outXOrigin + (dilation[1] * kx)); + + var w = wICP[(int)((ky * kernelShape[3]) + kx)]; + + outCP[(int)((outY * outputShape[3]) + outX)] += (float)inV * w; + } + } + } + } + } + } + } + } + + for (int i = 0; i < outputSize; i++) + { + var biasIdx = i / (outputShape[2] * outputShape[3]) % outputShape[1]; + outCache[i] = outCache[i] + biasArray[biasIdx]; + } + + return new TensorValue(Tensor.From(outCache, new[] { (int)outputShape[0], (int)outputShape[1], (int)outputShape[2], (int)outputShape[3] })); } /// diff --git a/src/Nncase.Importer/TFLite/MatMul.cs b/src/Nncase.Importer/TFLite/MatMul.cs index 59bbfcd1d7..056d2bdee5 100644 --- a/src/Nncase.Importer/TFLite/MatMul.cs +++ b/src/Nncase.Importer/TFLite/MatMul.cs @@ -66,14 +66,12 @@ private Expr VisitMatMul(in tflite.Operator op, bool isFullyConnected = true) : Expand(Cast(0, GetDataType(GetInputTensor(op, 0).Type)), new[] { otherTensor.Shape(0) }).Evaluate().AsTensor(); var matmul = MatMul(lhs, rhs); - List outputNames = new() { GetOutputTensor(op, 0).Name + "_matmul" }; - matmul.Metadata.OutputNames = outputNames; - outputNames.Clear(); - outputNames.Add(GetOutputTensor(op, 0).Name + "_bias"); - bias.Metadata.OutputNames = outputNames; + List outputNames_matmul = new() { GetOutputTensor(op, 0).Name + "_matmul" }; + matmul.Metadata.OutputNames = outputNames_matmul; + List outputNames_bias = new() { GetOutputTensor(op, 0).Name + "_bias" }; + bias.Metadata.OutputNames = outputNames_bias; var mm = matmul + bias; - outputNames.Clear(); - outputNames.Add(GetOutputTensor(op, 0).Name); + List outputNames = new() { GetOutputTensor(op, 0).Name }; mm.Metadata.OutputNames = outputNames; return fusedActivationFunction switch diff --git a/src/Nncase.Tests/Evaluator/UnitTestEvaluatorNN.cs b/src/Nncase.Tests/Evaluator/UnitTestEvaluatorNN.cs index 385dd863fe..74296b8cfb 100755 --- a/src/Nncase.Tests/Evaluator/UnitTestEvaluatorNN.cs +++ b/src/Nncase.Tests/Evaluator/UnitTestEvaluatorNN.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using NetFabric.Hyperlinq; using Nncase.Evaluator; using Nncase.IR; using Nncase.IR.F; @@ -275,7 +276,10 @@ public void TestConv2DTranspose() PadMode.Constant, 1); CompilerServices.InferenceType(expr); - Assert.Equal(expect, expr.Evaluate().AsTensor().ToOrtTensor()); + var expectValue = expect.ToArray(); + var realValue = expr.Evaluate().AsTensor().ToArray(); + var cos = Nncase.Tests.Comparator.CosSimilarity(expectValue, realValue); + Assert.True(cos >= 0.99); } [Fact] From 21eccd21b945050054509690b08c593aae0ec2d2 Mon Sep 17 00:00:00 2001 From: zhangyang2057 Date: Fri, 3 Nov 2023 16:41:44 +0800 Subject: [PATCH 10/13] Refactor infer report for k230_benchmark_test. (#1119) * Refactor infer report for k230_benchmark_test. * fix python format. * Remove update_trace_info.py --- tests/inference.py | 151 ++++++++++++++----------------------- tests/nuc_proxy.py | 141 +++++++++++++++++++--------------- tests/test_runner.py | 12 +-- tests/update_trace_info.py | 92 ---------------------- 4 files changed, 143 insertions(+), 253 deletions(-) delete mode 100644 tests/update_trace_info.py diff --git a/tests/inference.py b/tests/inference.py index 4585ba89a9..a3e2a5cb3a 100644 --- a/tests/inference.py +++ b/tests/inference.py @@ -20,11 +20,10 @@ import test_utils import preprocess_utils import socket +import struct import json from test_utils import * import time -import subprocess -from update_trace_info import * from html import escape @@ -123,6 +122,30 @@ def dump_infer_output(self, sim, compile_opt, infer_dir): dump_txt_file(os.path.join(infer_dir, f'nncase_result_{i}.txt'), output) return outputs + def send_msg(self, sock, msg): + # Prefix each message with a 4-byte length (network byte order) + msg = struct.pack('>I', len(msg)) + msg + sock.sendall(msg) + + def recv_msg(self, sock): + # Read message length and unpack it into an integer + raw_msglen = self.recvall(sock, 4) + if not raw_msglen: + return None + msglen = struct.unpack('>I', raw_msglen)[0] + # Read the message data + return self.recvall(sock, msglen) + + def recvall(self, sock, n): + # Helper function to recv n bytes or return None if EOF is hit + data = bytearray() + while len(data) < n: + packet = sock.recv(n - len(data)) + if not packet: + return None + data.extend(packet) + return data + def run_evb(self, target, kmodel, compile_opt, infer_dir): ip = test_utils.nuc_ip() port = test_utils.nuc_port() @@ -133,13 +156,13 @@ def run_evb(self, target, kmodel, compile_opt, infer_dir): client_socket.connect((ip, int(port))) # send target - dummy = client_socket.recv(1024) + dummy = self.recv_msg(client_socket) target_dict = {} target_dict['target'] = target - client_socket.sendall(json.dumps(target_dict).encode()) + self.send_msg(client_socket, json.dumps(target_dict).encode()) # send header - dummy = client_socket.recv(1024) + dummy = self.recv_msg(client_socket) header_dict = {} header_dict['case'] = os.path.basename(self.case_dir) header_dict['app'] = 1 @@ -147,141 +170,77 @@ def run_evb(self, target, kmodel, compile_opt, infer_dir): header_dict['inputs'] = len(self.inputs) header_dict['description'] = 1 if self.dynamic else 0 header_dict['outputs'] = len(self.outputs) - client_socket.sendall(json.dumps(header_dict).encode()) + header_dict['cfg_cmds'] = self.config_cmds() + self.send_msg(client_socket, json.dumps(header_dict).encode()) # send app - dummy = client_socket.recv(1024) + dummy = self.recv_msg(client_socket) file_dict = {} file_dict['file_name'] = os.path.basename(test_executable) file_dict['file_size'] = os.path.getsize(test_executable) - client_socket.sendall(json.dumps(file_dict).encode()) - dummy = client_socket.recv(1024) + self.send_msg(client_socket, json.dumps(file_dict).encode()) + dummy = self.recv_msg(client_socket) with open(test_executable, 'rb') as f: client_socket.sendall(f.read()) # send kmodel - dummy = client_socket.recv(1024) + dummy = self.recv_msg(client_socket) file_dict['file_name'] = self.cfg['kmodel_name'] file_dict['file_size'] = len(kmodel) - client_socket.sendall(json.dumps(file_dict).encode()) - dummy = client_socket.recv(1024) + self.send_msg(client_socket, json.dumps(file_dict).encode()) + dummy = self.recv_msg(client_socket) client_socket.sendall(kmodel) # send inputs for idx, value in enumerate(self.inputs): + dummy = self.recv_msg(client_socket) data = self.transform_input( value['data'], compile_opt['input_type'], "infer")[0] file_dict['file_name'] = f'input_{idx}.bin' file_dict['file_size'] = data.size * data.itemsize - dummy = client_socket.recv(1024) - client_socket.sendall(json.dumps(file_dict).encode()) - dummy = client_socket.recv(1024) + self.send_msg(client_socket, json.dumps(file_dict).encode()) + dummy = self.recv_msg(client_socket) client_socket.sendall(data.tobytes()) # send kmodel.desc if self.dynamic: - dummy = client_socket.recv(1024) + dummy = self.recv_msg(client_socket) desc_file = os.path.join(infer_dir, self.cfg['desc_name']) file_dict['file_name'] = os.path.basename(desc_file) file_dict['file_size'] = os.path.getsize(desc_file) - client_socket.sendall(json.dumps(file_dict).encode()) - dummy = client_socket.recv(1024) + self.send_msg(client_socket, json.dumps(file_dict).encode()) + dummy = self.recv_msg(client_socket) with open(desc_file, 'rb') as f: client_socket.sendall(f.read()) # get infer result outputs = [] header_dict = {} - ret = client_socket.recv(1024) + ret = self.recv_msg(client_socket) header_dict = json.loads(ret.decode()) - length = header_dict['len'] - - # recv result - count = length // 1024 - left = length % 1024 - - client_socket.sendall(f"pls send detail".encode()) - recv_data = b'' - for i in range(count): - data = client_socket.recv(1024, socket.MSG_WAITALL) - recv_data += data - - if left: - recv_data += client_socket.recv(left, socket.MSG_WAITALL) - - detail = recv_data.decode() - + ret = header_dict['msg'] if header_dict['type'].find('finish') != -1: if self.cfg['infer_report_opt']['enabled']: - if not self.dynamic: - # update trace info - model_name = self.cfg['infer_report_opt']['model_name'] - infer_result = f'0:{model_name} :\n' + detail - trace_file = search_file(infer_dir, 'trace_info.py') - assert(trace_file != '') - update_trace_info(infer_result, trace_file) - - # roofline fps/mac usage - estimate_file = search_file(infer_dir, 'estimate_fps.py') - assert(estimate_file != '') - - mac_file = search_file(infer_dir, 'mac.csv') - assert(mac_file != '') - - cmd_status, cmd_result = subprocess.getstatusoutput( - f'python3 {estimate_file} {mac_file}') - assert(cmd_status == 0) - data = cmd_result.split(',') - assert(len(data) >= 3) - self.infer_report_dict['roofline_fps'] = data[1].split(':')[-1].strip() - self.infer_report_dict['roofline_mac_usage'] = data[2].split(':')[-1].strip() - - # actual fps - fps_pattern = re.compile( - r"^\|total\s+\|(\d+|\d+.\d+)\s+\|(\d+|\d+.\d+)\s+\|(\d+|\d+.\d+)\s+\|") - buf = io.StringIO(detail) - while True: - line = buf.readline() - if not line: - break - match = fps_pattern.match(line) - if match is not None: - self.infer_report_dict['actual_fps'] = str( - round(1000 / float(match.group(2)), 3)) - break - - if not self.dynamic: - # actual mac usage - draw_trace_file = search_file(infer_dir, 'draw_trace.py') - assert(draw_trace_file != '') - cmd_status, cmd_result = subprocess.getstatusoutput( - f'python3 {draw_trace_file} {mac_file}') - assert(cmd_status == 0) - data = cmd_result.split(',') - assert(len(data) >= 1) - self.infer_report_dict['actual_mac_usage'] = data[0].split(':')[-1].strip() - - client_socket.sendall(f"pls send outputs".encode()) + self.stat_target(infer_dir, ret) + + self.send_msg(client_socket, f"pls send outputs".encode()) # recv outputs for i in range(len(self.outputs)): - header = client_socket.recv(1024) - file_size = int(header.decode()) - client_socket.sendall(f"pls send nncase_result_{i}.bin".encode()) + header = self.recv_msg(client_socket) + file_dict = json.loads(header.decode()) + file_size = file_dict['file_size'] + self.send_msg(client_socket, f"pls send file".encode()) - recv_size = 0 buffer = bytearray(file_size) - while recv_size < file_size: - slice = client_socket.recv(4096) - buffer[recv_size:] = slice - recv_size += len(slice) + buffer = self.recvall(client_socket, file_size) output = np.frombuffer(buffer, dtype=self.outputs[i]['dtype']) outputs.append(output) + if not test_utils.in_ci(): dump_bin_file(os.path.join(infer_dir, f'nncase_result_{i}.bin'), output) dump_txt_file(os.path.join(infer_dir, f'nncase_result_{i}.txt'), output) - client_socket.sendall(f"recv nncase_result_{i}.bin succeed".encode()) client_socket.close() else: @@ -289,11 +248,11 @@ def run_evb(self, target, kmodel, compile_opt, infer_dir): if self.cfg['infer_report_opt']['enabled']: self.infer_report_dict['result'] = 'Fail' - self.infer_report_dict['remark'] = escape(detail) + self.infer_report_dict['remark'] = escape(ret) prefix, suffix = os.path.splitext(self.infer_report_file) json_file = f'{prefix}_{os.path.basename(self.case_dir)}{suffix}' dump_dict_to_json(self.infer_report_dict, json_file) - raise Exception(detail) + raise Exception(ret) return outputs diff --git a/tests/nuc_proxy.py b/tests/nuc_proxy.py index bdc647ef68..3da68bdcca 100644 --- a/tests/nuc_proxy.py +++ b/tests/nuc_proxy.py @@ -24,16 +24,18 @@ import logging.handlers import serial import shutil +import struct import time import toml from typing import Tuple class MySerial: - def __init__(self, port, baudrate, logger): + def __init__(self, port, baudrate, separator, logger): self.s = None self.port = port self.baudrate = baudrate + self.separator = separator self.logger = logger self.timeout = 60 @@ -87,9 +89,9 @@ def run_cmd(self, cmd, expected=''): expired = False self.open() self.write(cmd) - if expected != '': - data, expired = self.read_until(expected) + str = expected if expected != '' else self.separator + data, expired = self.read_until(str) self.close() return data, expired @@ -102,7 +104,6 @@ def __init__(self, name, cfg, nfs, clear_queue): self.username = cfg['username'] self.password = cfg['password'] self.working_dir = cfg['working_dir'] - self.separator = cfg['separator'] # nfs_dir self.nfs_dir = os.path.join(nfs, name) @@ -120,33 +121,54 @@ def __init__(self, name, cfg, nfs, clear_queue): self.logger = mylogger # serial - self.s0 = MySerial(cfg['uart0'], cfg['baudrate0'], self.logger) - self.s1 = MySerial(cfg['uart1'], cfg['baudrate1'], self.logger) + self.s0 = MySerial(cfg['uart0'], cfg['baudrate0'], cfg['separator0'], self.logger) + self.s1 = MySerial(cfg['uart1'], cfg['baudrate1'], cfg['separator1'], self.logger) def reboot(self): - # reboot after login - self.s0.run_cmd(self.username) - self.s0.run_cmd(self.password) self.s0.run_cmd('reboot') time.sleep(30) +def send_msg(sock, msg): + # Prefix each message with a 4-byte length (network byte order) + msg = struct.pack('>I', len(msg)) + msg + sock.sendall(msg) + + +def recv_msg(sock): + # Read message length and unpack it into an integer + raw_msglen = recvall(sock, 4) + if not raw_msglen: + return None + msglen = struct.unpack('>I', raw_msglen)[0] + # Read the message data + return recvall(sock, msglen) + + +def recvall(sock, n): + # Helper function to recv n bytes or return None if EOF is hit + data = bytearray() + while len(data) < n: + packet = sock.recv(n - len(data)) + if not packet: + return None + data.extend(packet) + return data + + def recv_file(conn, case_dir, logger): - conn.sendall(f"pls send file info".encode()) - header = conn.recv(1024) + send_msg(conn, f"pls send file info".encode()) + header = recv_msg(conn) file_dict = json.loads(header.decode()) file_name = file_dict['file_name'] file_size = file_dict['file_size'] logger.debug('recv begin: file = {0}, size = {1}'.format(file_name, file_size)) - conn.sendall(f"pls send {file_name}".encode()) + send_msg(conn, f"pls send {file_name}".encode()) full_file = os.path.join(case_dir, file_name) with open(full_file, 'wb') as f: - recv_size = 0 - while recv_size < file_size: - slice = conn.recv(4096) - f.write(slice) - recv_size += len(slice) + data = recvall(conn, file_size) + f.write(data) os.chmod(full_file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) logger.debug('recv end') @@ -155,15 +177,14 @@ def recv_file(conn, case_dir, logger): def recv_worker(conn, target): # recv header - conn.sendall(f"pls send header".encode()) - header = conn.recv(1024) - header_dict = json.loads(header.decode()) - new_case = header_dict['case'] + str(int(time.time())) + send_msg(conn, f"pls send header".encode()) + header = recv_msg(conn) + dict = json.loads(header.decode()) + new_case = dict['case'] + str(int(time.time())) target.logger.info("test case = {0}".format(new_case)) case_dir = os.path.join(target.nfs_dir, new_case) os.makedirs(case_dir) - file_num = header_dict['app'] + header_dict['kmodel'] + \ - header_dict['inputs'] + header_dict['description'] + file_num = dict['app'] + dict['kmodel'] + dict['inputs'] + dict['description'] # recv all kinds of files(app + kmodel + inputs) cmds = f'cd {target.working_dir}/{target.name}/{new_case};./' @@ -175,70 +196,71 @@ def recv_worker(conn, target): cmds = cmds + ' ' + file target.logger.debug("cmds = {0}".format(cmds)) - target.infer_queue.put((cmds, conn, case_dir, header_dict['outputs'])) + target.infer_queue.put((dict['cfg_cmds'], cmds, conn, case_dir, dict['outputs'])) def infer_worker(target): while True: - cmds, conn, case_dir, output_num = target.infer_queue.get() - separator = os.path.basename(case_dir) + target.separator + cfg_cmds, run_cmds, conn, case_dir, output_num = target.infer_queue.get() + test_case = os.path.basename(case_dir) + s1_separator = test_case + target.s1.separator ret = '' timeout = False - # exit from face_detect after rebooting - # target.s1.run_cmd('q') - target.s1.run_cmd('') + # try to login serial + target.s0.run_cmd(target.username) + target.s0.run_cmd(target.password) + target.s1.run_cmd('q\r') - for cmd in cmds.split(';'): - ret, timeout = target.s1.run_cmd(cmd, separator) + msg = [] + if len(cfg_cmds) == 0: + for cmd in run_cmds.split(';'): + ret, timeout = target.s1.run_cmd(cmd, s1_separator) + msg.append(ret) + else: + for cfg_cmd in cfg_cmds: + target.s0.run_cmd(cfg_cmd) + for cmd in run_cmds.split(';'): + ret, timeout = target.s1.run_cmd(cmd, s1_separator) + msg.append(ret) # infer result - dict = {'type': 'finish', 'len': 0} + dict = {'type': 'finish', 'msg': ''} + ret = msg[0] if ret.find('terminate') != -1 or ret.find('Exception') != -1: err = 'infer exception' target.logger.error(err) - msg = f'{err}'.encode() dict['type'] = 'exception' - dict['len'] = len(msg) - conn.sendall(json.dumps(dict).encode()) - dummy = conn.recv(1024) - conn.sendall(msg) - - # reboot target when exception(it is likely that next test case will fail) + dict['msg'] = err + send_msg(conn, json.dumps(dict).encode()) target.reboot() elif timeout: err = 'infer timeout' target.logger.error(err) - msg = f'{err}'.encode() dict['type'] = 'timeout' - dict['len'] = len(msg) - conn.sendall(json.dumps(dict).encode()) - dummy = conn.recv(1024) - conn.sendall(msg) - - # reboot target when timeout + dict['msg'] = err + send_msg(conn, json.dumps(dict).encode()) target.reboot() else: - msg = ret.encode() + # send header dict['type'] = 'finish' - dict['len'] = len(msg) - conn.sendall(json.dumps(dict).encode()) - dummy = conn.recv(1024) - conn.sendall(msg) - dummy = conn.recv(1024) + dict['msg'] = msg + send_msg(conn, json.dumps(dict).encode()) # send outputs + dummy = recv_msg(conn) for i in range(output_num): + file_dict = {} file = os.path.join(case_dir, f'nncase_result_{i}.bin') file_size = os.path.getsize(file) - conn.sendall(str(file_size).encode()) - dummy = conn.recv(1024) + file_dict['file_size'] = file_size + send_msg(conn, json.dumps(file_dict).encode()) + dummy = recv_msg(conn) target.logger.debug('send begin: file = {0}, size = {1}'.format(file, file_size)) with open(file, 'rb') as f: conn.sendall(f.read()) target.logger.debug('send end') - dummy = conn.recv(1024) target.logger.debug('infer finish') conn.close() target.clear_queue.put(case_dir) @@ -254,18 +276,19 @@ def clear_worker(q): def main(): # default config config = ''' - ip = '10.99.105.216' + ip = '10.100.105.239' port = 10000 nfs = '/data/nfs' [k230] username = 'root' password = '' working_dir = '/sharefs' - separator = '>' uart0 = '/dev/ttyUSB0' baudrate0 = 115200 + separator0 = ']#' uart1 = '/dev/ttyUSB1' baudrate1 = 115200 + separator1 = '>' ''' # args @@ -295,8 +318,8 @@ def main(): conn, addr = server_socket.accept() # recv target name - conn.sendall(f"pls send your target".encode()) - info = conn.recv(1024) + send_msg(conn, f"pls send your target".encode()) + info = recv_msg(conn) target_dict = json.loads(info.decode()) target_name = target_dict['target'] diff --git a/tests/test_runner.py b/tests/test_runner.py index 60948ac2f2..3aa5ce0777 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -81,12 +81,6 @@ def __init__(self, case_name, override_cfg: str = None) -> None: 'shape': 'N/A', 'if_quant_type': 'uint8', 'w_quant_type': 'uint8', - 'roofline_fps': 'N/A', - 'actual_fps': 'N/A', - 'roofline_mac_usage': 'N/A', - 'actual_mac_usage': 'N/A', - 'result': 'Pass', - 'remark': 'N/A' } def transform_input(self, values: List[np.ndarray], type: str, stage: str) -> List[np.ndarray]: @@ -247,6 +241,12 @@ def cpu_infer(self, case_dir: str, model_content: Union[List[str], str]): def import_model(self, compiler, model_content, import_options): pass + def config_cmds(self): + return [] + + def stat_target(self, infer_dir, results): + pass + def run(self, model_file: Union[List[str], str]): if not self.inputs: self.parse_model(model_file) diff --git a/tests/update_trace_info.py b/tests/update_trace_info.py deleted file mode 100644 index 7cf107f01f..0000000000 --- a/tests/update_trace_info.py +++ /dev/null @@ -1,92 +0,0 @@ -import re -from enum import IntFlag, auto -import os -from typing import Tuple, List -import io - -ITEM_PATTERN = re.compile( - r"^DataItem\(\d+, \"(\w+)\", (True|False), (0\.0), (0\.0), (0\.0)\),", re.RegexFlag.MULTILINE) - - -class Status(IntFlag): - find_titile = auto() - find_time = auto() - - -def find_titile(line: str) -> str: - title_pattern = re.compile(r"^\d+:([a-zA-Z0-9_.-]+)\s:\s") - match = title_pattern.match(line) - if match is None: - return None - return match.group(1) - - -def find_time(line: str) -> Tuple[str, str]: - time_pattern = re.compile(r"^\|(\w+)\s+\|(\d+|\d+.\d+)\s+\|(\d+|\d+.\d+)\s+\|") - match = time_pattern.match(line) - if match is None: - return None - return match.group(2), match.group(3) - - -def find_items(info_path: str) -> int: - if not os.path.exists(info_path): - return -1 - context = None - with open(info_path, 'r') as f: - context = f.read() - return len(ITEM_PATTERN.findall(context)) - - -def update_items(info_path: str, times: List[Tuple[str, str]]): - if not os.path.exists(info_path): - return -1 - context = None - with open(info_path, 'r') as f: - context = f.read() - - cnt = {'i': 0} - - def update(match: re.Match): - i = cnt['i'] - time = times[i] - new = f'DataItem({i}, \"{match.group(1)}\", {match.group(2)}, {time[0]}, {time[1]}, {float(time[1])-float(time[0]):.6f}),' - cnt['i'] += 1 - return new - - new_context = ITEM_PATTERN.sub(update, context) - with open(info_path, 'w') as f: - f.write(new_context) - - -def update_trace_info(infer_result: str, info_file: str): - status = Status.find_titile - title = None - item_num = -1 - times = [] - - buf = io.StringIO(infer_result) - while True: - line = buf.readline() - if not line: - break - - if status == Status.find_titile: - title = find_titile(line) - if title: - status = Status.find_time - item_num = find_items(info_file) - if item_num == -1 or item_num == 0: - item_num = -1 - status = Status.find_titile - continue - - if status is Status.find_time: - time = find_time(line) - if time: - times.append(time) - if (len(times) == item_num): - update_items(info_file, times) - times.clear() - status = Status.find_titile - continue From 338ba1070d7d6ac7848335ca8c5b2c2c3b63a3ff Mon Sep 17 00:00:00 2001 From: huochenghai Date: Tue, 7 Nov 2023 10:13:25 +0800 Subject: [PATCH 11/13] Feature/cpu (#1019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add layernorm * pass reduce * add comment * add layer norm test * fix layernorm * fix layernorm * add demo2 * fix build / add view * update layernorm * support layernorm of llama * fix build * add demo2 * pass ym * pass demo2 * fix onnx external data importer * fuse MHA of llama * add more cpu kernels * update MHA fusion * reorder MHA weights * add demo3 * add demo3 compute statge 1 * fix build * fix __tdma_all_sync_apply * add to v35 * dump const * update demo3 golden * compiled * support multiple output compare * fix MHA kernel * resplit v2 * fix MHA kernel * to v26 * push * fix double free * fix mha kernel * fix V35 * fix all * support rmsnrom * fix v22 * fix v22 v10 * pass v28 * pass v43 * remove dump * add other part * pass all llama65b decoder layers * pass gather * open 32 threads for demo4 * update binary/unary with external op * fix codes using stdlib * update kernel inputs * Fix gather * refactor cpu cmodel * update demo head * pass graph to tir * fix head main * pass norm case * update demo names * add xpu source gen * Fix head kernel segment fault * fix cost evaluator * Fix head kernel cos similarity * decoder layer pass input layernorm * Add uanry demo * pass v30 of decoder layer * fix softmax * Enable ImmOutput * fix malloc * remove debug macro * Add ImmOut * fix tdma store * refactor cpu runtime * refactor method table * fix cpu test * refactor auto distributed * update cpu test * fix rdata * update cpu test with rdata * fix typeinfer * add XPU Op layernorm * update layernorm cost * fix cost evaluator * fix layernorm * add partial resplit * add rvv matmul * add codegen of cpu gather * add concat/slice codegen * merge * add codegen of cpu softmax * update slice cpu case * fix slice * fix cpu concat * fix cpu concat * Apply code-format changes * fix build * Apply code-format changes * add codegen of transpose * add reshape * pass reshape2 * update stackvm * merge * fix build * update compile * fix to slicing * fix negative axis * fix matmul evaluator * add NormAxis * fix ToSlice * fix matmul * add GatherReduceScatter * fix ToSlice * refactor auto dist * fix boxing partial to slice codegen * softmax support split on axis * add conv2d cpu kernel * disable outter split on inner splited axis * fix binary distributedtype infer * fixGetPartialCandidateNDSBPs * pass cpu conv2d * support dilated conv2d * add mha pattern * add combine reshape transpose * fix mha fusion/ add rules * fix rdata map dispose * add xpu reduce arg * Apply code-format changes * add VAE fusion * fuse VAE * support xpu instance norm * Apply code-format changes * add reduce arg * Apply code-format changes * fix to tir keep vars order * Apply code-format changes * add XPU resize * Apply code-format changes * fix resize cpu kernel op * fix Resize * Update layernom op for test * Apply code-format changes * fix conv2d kernel * fix boxing with reshape * fix build * fix pytest compare * add gelu kernels * add xpu cast * fix swish type infer * support xpu expand * Update layernorm rvv codes * fix binary broadcast with distributed broadcast * support multi outputs * fix single output * fix new linked section * fuse Unet * add cos dump * fix build * speed up onnx external data load * add typeinfer case for binary/matmul * move matmul rvv to kernels * fix conv2d kernel * fix Unet Fusion * optimize dynamic onnx * change fusion counter * fix conv2d if split is partial * split conv to conv+bias+clamp, and add xpu clamp * update fusion merger * fix slice with negative axis * llama-4-decoder pass (x86/rv64) * text encoder/vae decoder pass (x86/rv64) * fix conan config * Fix cpu/test compile * fix cmake config * fix synax err * fix synax err * normallize axes of slice * disable module cpu on windows * donot split softmax on axis * add softmax kernel test * add rvv instance norm * clean modules dir * add rvv clamp * Clean modules dir * fix match result * Apply code-format changes * Clean Tests * fix csproj * fix buffer schedule * Add unet pytest * add target's commands * fix buffer and memspan hashcode and equals * fix unitest * fix unittest * fix test_cli output dump * fix command line * fix type infer * fix format * fix all test * Apply code-format changes * fix merge * Apply code-format changes * fix merge * fix runtime build * fix kernel test build * Apply code-format changes * fix use mean * Apply code-format changes * optimize dot dump * fix merge * fix output when test cli --------- Co-authored-by: xhuohai Co-authored-by: 郑启航 <597323109@qq.com> Co-authored-by: zhen8838 Co-authored-by: lerenhua <2532375005@qq.com> Co-authored-by: liuzhiming Co-authored-by: liuzm6217-jianan --- .gitignore | 4 +- CMakeLists.txt | 3 +- Directory.Packages.props | 5 +- modules/Nncase.Modules.CPU/packages.lock.json | 295 ---------------- .../CodeGen/StackVM/CodeGenVisitor.g.cs | 14 +- .../CodeGen/StackVM/StackVMEmitter.g.cs | 11 +- .../Targets/CPUTarget.cs | 11 +- .../Nncase.Modules.StackVM/packages.lock.json | 19 +- nncase.sln | 2 - python/common/pystreambuf.h | 1 + python/nncase/__init__.py | 2 +- .../nncase/kernels/stackvm/tensor_ops.h | 12 +- .../include/nncase/runtime/interpreter.h | 1 + .../include/nncase/runtime/runtime_module.h | 2 + .../nncase/runtime/stackvm/op_reader.h | 7 +- .../include/nncase/runtime/stackvm/opcode.h | 19 +- src/Native/src/kernels/stackvm/tensor_ops.cpp | 14 +- src/Native/src/runtime/CMakeLists.txt | 1 + src/Native/src/runtime/interpreter.cpp | 11 + src/Native/src/runtime/runtime_module.cpp | 13 + src/Native/src/runtime/stackvm/op_reader.cpp | 4 +- src/Native/src/runtime/stackvm/ops/tensor.cpp | 18 +- .../runtime/stackvm/runtime_function_ops.h | 4 +- src/Native/src/test_cli.cpp | 68 ++-- src/Nncase.Cli/Commands/Compile.cs | 291 ---------------- src/Nncase.Cli/Compile.cs | 217 ++++++++++++ src/Nncase.Cli/Nncase.Cli.csproj | 4 + src/Nncase.Cli/Program.CommandLine.cs | 26 -- src/Nncase.Cli/Program.cs | 149 +++++++- src/Nncase.Cli/packages.lock.json | 39 ++- src/Nncase.CodeGen/CodeGen/LinkedFunction.cs | 1 - src/Nncase.CodeGen/packages.lock.json | 19 +- src/Nncase.Compiler/Compiler.cs | 17 +- src/Nncase.Compiler/Hosting/PluginLoader.cs | 131 ++++---- src/Nncase.Compiler/packages.lock.json | 19 +- src/Nncase.Core/CompileOptions.cs | 5 + src/Nncase.Core/CompilerServices.cs | 21 ++ .../Converters/ConvertersModule.cs | 1 + .../Converters/PointerConverters.cs | 22 ++ src/Nncase.Core/CostModel/Cost.cs | 4 +- src/Nncase.Core/DataTypes.cs | 2 +- src/Nncase.Core/Diagnostics/IDumpper.cs | 2 + src/Nncase.Core/Diagnostics/NullDumpper.cs | 5 + src/Nncase.Core/DistributedType.cs | 68 ++++ src/Nncase.Core/FunctionCollector.cs | 2 +- src/Nncase.Core/IR/Buffers/BufferLoad.cs | 28 ++ src/Nncase.Core/IR/Buffers/BufferOf.cs | 2 +- src/Nncase.Core/IR/Buffers/BufferStore.cs | 33 ++ src/Nncase.Core/IR/Buffers/DDrOf.cs | 3 + src/Nncase.Core/IR/Buffers/Functional.cs | 2 +- src/Nncase.Core/IR/Buffers/MatchBuffer.cs | 21 ++ src/Nncase.Core/IR/Buffers/Uninitialized.cs | 4 +- src/Nncase.Core/IR/Callable.cs | 2 +- src/Nncase.Core/IR/ExprCloner.g.cs | 39 +-- src/Nncase.Core/IR/ExprFunctor.cs | 20 ++ src/Nncase.Core/IR/ExprFunctor.g.cs | 61 +--- src/Nncase.Core/IR/ExprRewriter.g.cs | 96 ++---- src/Nncase.Core/IR/ExprVisitor.g.cs | 224 +++++-------- src/Nncase.Core/IR/IIRPrinterProvider.cs | 8 + src/Nncase.Core/IR/IRList.csv | 7 +- src/Nncase.Core/IR/IRType.cs | 9 + src/Nncase.Core/IR/Imaging/ResizeImage.cs | 2 +- src/Nncase.Core/IR/Math/Binary.cs | 4 +- src/Nncase.Core/IR/Math/Clamp.cs | 2 +- src/Nncase.Core/IR/Math/MatMul.cs | 4 +- src/Nncase.Core/IR/Math/ReduceArg.cs | 6 +- src/Nncase.Core/IR/Math/Unary.cs | 2 +- src/Nncase.Core/IR/NN/Activations.cs | 7 +- src/Nncase.Core/IR/NN/Conv2D.cs | 6 +- src/Nncase.Core/IR/NN/Functional.cs | 9 +- src/Nncase.Core/IR/NN/LayerNorm.cs | 10 +- src/Nncase.Core/IR/NN/Normalization.cs | 6 +- src/Nncase.Core/IR/NN/SoftMax.cs | 2 +- src/Nncase.Core/IR/Op.cs | 22 +- src/Nncase.Core/IR/RNN/Functional.cs | 2 +- src/Nncase.Core/IR/RNN/LSTM.cs | 2 +- src/Nncase.Core/IR/TensorConst.cs | 2 +- src/Nncase.Core/IR/Tensors/Cast.cs | 2 +- src/Nncase.Core/IR/Tensors/Concat.cs | 7 +- src/Nncase.Core/IR/Tensors/Expand.cs | 2 +- src/Nncase.Core/IR/Tensors/Functional.cs | 4 +- src/Nncase.Core/IR/Tensors/Gather.cs | 13 +- src/Nncase.Core/IR/Tensors/Reshape.cs | 2 +- src/Nncase.Core/IR/Tensors/Slice.cs | 2 +- src/Nncase.Core/IR/Tensors/Transpose.cs | 2 +- src/Nncase.Core/IR/Tensors/UnSqueeze.cs | 2 +- src/Nncase.Core/IR/TypeFunctor.cs | 17 + src/Nncase.Core/IR/TypePattern.cs | 5 +- src/Nncase.Core/ITarget.cs | 22 ++ src/Nncase.Core/LinqExtensions.cs | 32 ++ src/Nncase.Core/Nncase.Core.csproj | 3 +- .../Passes/Mutators/FlattenBuffer.cs | 40 ++- .../Passes/Mutators/FoldBufferSlot.cs | 51 +++ .../Passes/Mutators/FoldMathCall.cs | 30 -- src/Nncase.Core/Passes/Mutators/Mutator.cs | 6 - .../Passes/Mutators/UnRollLoopSequential.cs | 8 +- src/Nncase.Core/Schedule/ScheduleTypes.cs | 54 +-- src/Nncase.Core/TIR/Buffer.cs | 217 ++---------- src/Nncase.Core/TIR/BufferLoad.cs | 40 --- src/Nncase.Core/TIR/BufferStore.cs | 47 --- src/Nncase.Core/TIR/MemSpan.cs | 105 ++++++ src/Nncase.Core/TIR/Ops.cs | 18 +- src/Nncase.Core/TIR/PrimFunction.cs | 12 +- src/Nncase.Core/TIR/Scheduler.cs | 29 +- src/Nncase.Core/TIR/Script.cs | 96 +++--- src/Nncase.Core/Tensor.cs | 6 +- src/Nncase.Core/TensorUtilities.cs | 7 +- .../Utilities/DistributedUtility.cs | 144 ++++++++ src/Nncase.Core/packages.lock.json | 18 +- src/Nncase.Diagnostics/Diagnostics/Dumpper.cs | 6 + .../Diagnostics/ILDotPrintVisitor.cs | 6 +- .../Diagnostics/ILPrintVisitor.cs | 35 +- .../Diagnostics/IRPrinterProvider.cs | 19 ++ .../Diagnostics/PatternPrintVisitor.cs | 245 ++++++++++++++ .../Diagnostics/ScriptPrintVisitor.cs | 63 ++-- src/Nncase.Diagnostics/packages.lock.json | 19 +- .../Passes/EGraphExtractExtensions.cs | 7 +- .../Passes/EGraphExtractors/Extractor.cs | 15 +- .../Passes/EGraphExtractors/SatExtractor.cs | 8 +- src/Nncase.EGraph/Passes/RewriteProvider.cs | 2 +- src/Nncase.EGraph/packages.lock.json | 19 +- src/Nncase.Evaluator/Buffer.cs | 9 - src/Nncase.Evaluator/Buffers/BufferLoad.cs | 42 +++ src/Nncase.Evaluator/Buffers/BufferModule.cs | 3 + src/Nncase.Evaluator/Buffers/BufferStore.cs | 47 +++ src/Nncase.Evaluator/Buffers/DDrOf.cs | 2 +- src/Nncase.Evaluator/Buffers/MatchBuffer.cs | 29 ++ src/Nncase.Evaluator/Imaging/ResizeImage.cs | 45 ++- src/Nncase.Evaluator/Math/Binary.cs | 133 +++++++- src/Nncase.Evaluator/Math/Clamp.cs | 29 +- src/Nncase.Evaluator/Math/MatMul.cs | 195 ++++++++--- src/Nncase.Evaluator/Math/ReduceArg.cs | 66 +++- src/Nncase.Evaluator/Math/Unary.cs | 63 +++- src/Nncase.Evaluator/NN/Activations.cs | 26 +- src/Nncase.Evaluator/NN/Conv2D.cs | 110 +++++- src/Nncase.Evaluator/NN/LayerNorm.cs | 99 +++++- src/Nncase.Evaluator/NN/Normalization.cs | 49 ++- src/Nncase.Evaluator/NN/Softmax.cs | 24 +- src/Nncase.Evaluator/RNN/LSTM.cs | 5 +- src/Nncase.Evaluator/TIR/Load.cs | 7 +- src/Nncase.Evaluator/TIR/Store.cs | 16 +- src/Nncase.Evaluator/Tensors/Cast.cs | 30 +- src/Nncase.Evaluator/Tensors/Concat.cs | 110 ++++-- src/Nncase.Evaluator/Tensors/Expand.cs | 40 ++- src/Nncase.Evaluator/Tensors/Gather.cs | 78 +++-- src/Nncase.Evaluator/Tensors/Reshape.cs | 93 ++++- src/Nncase.Evaluator/Tensors/Slice.cs | 29 +- src/Nncase.Evaluator/Tensors/Transpose.cs | 48 ++- src/Nncase.Evaluator/Tensors/UnSqueeze.cs | 35 +- src/Nncase.Evaluator/TypeInference.cs | 1 + src/Nncase.Evaluator/TypeInferenceVisitor.cs | 68 +--- src/Nncase.Evaluator/packages.lock.json | 19 +- src/Nncase.Graph/packages.lock.json | 19 +- src/Nncase.IO/packages.lock.json | 12 +- src/Nncase.Importer/Ncnn/NcnnModel.cs | 2 +- src/Nncase.Importer/Ncnn/ParamDict.cs | 2 +- src/Nncase.Importer/Onnx/Concat.cs | 2 +- src/Nncase.Importer/Onnx/DataGatter.cs | 6 +- src/Nncase.Importer/Onnx/Gather.cs | 2 +- src/Nncase.Importer/Onnx/OnnxImporter.cs | 1 - src/Nncase.Importer/Onnx/Softmax.cs | 2 +- src/Nncase.Importer/packages.lock.json | 19 +- .../BufferScheduleExtensions.cs | 25 ++ .../BufferSchedule/BufferScheduleTypes.cs | 77 +++++ .../BufferSchedule/BufferScheduler.cs | 215 ++++++++++++ .../BufferSchedule/LifeTimeCollector.cs | 168 ++++++++++ src/Nncase.Passes/DDrBufferSchdeulePass.cs | 200 ++++------- src/Nncase.Passes/EGraphExtractPass.cs | 2 +- .../Mutators/IFusionMergeRule.cs | 8 +- .../Rules/Neutral/AddRangeOfAndMarker.cs | 1 + .../Rules/Neutral/CombineQuantize.cs | 9 +- .../Rules/Neutral/CombineReshape.cs | 58 ++++ .../Rules/Neutral/CombineTranspose.cs | 51 ++- src/Nncase.Passes/Rules/Neutral/FocusFull.cs | 9 +- .../Rules/Neutral/FoldGatherReshape.cs | 8 +- .../Rules/Neutral/FoldLayerNorm.cs | 54 +++ .../Neutral/FoldPrePostReshapeSoftmax.cs | 38 +++ .../Rules/Neutral/FoldReshape.cs | 23 ++ src/Nncase.Passes/Rules/Neutral/FoldSwish.cs | 41 +-- .../Rules/Neutral/FusionMaker.cs | 4 +- src/Nncase.Passes/Rules/Neutral/NormAxis.cs | 115 +++++++ .../Rules/Neutral/PrimFuncMergeRule.cs | 12 +- .../Rules/ShapeExpr/GatherToGetItem.cs | 5 +- src/Nncase.Passes/RulesUtility.cs | 38 +++ src/Nncase.Passes/packages.lock.json | 19 +- src/Nncase.Quantization/packages.lock.json | 19 +- src/Nncase.Schedule/packages.lock.json | 19 +- src/Nncase.Simulator/packages.lock.json | 19 +- src/Nncase.Targets/packages.lock.json | 19 +- .../packages.lock.json | 19 +- src/Nncase.Tests/CodeGen/CSourceHostCases.cs | 8 +- .../CodeGen/UnitTestStackVMEmitter.cs | 12 +- src/Nncase.Tests/Core/UnitTestDataTypes.cs | 3 +- src/Nncase.Tests/Core/UnitTestExpression.cs | 14 +- src/Nncase.Tests/Core/UnitTestMutator.cs | 3 - .../Core/UnitTestStringUtility.cs | 10 +- src/Nncase.Tests/Core/UnitTestTIR.cs | 47 +-- .../Core/UnitTestTensorUtilities.cs | 42 ++- .../Diagnostics/UnitTestDumpper.cs | 19 +- src/Nncase.Tests/EGraph/UnitTestVrp.cs | 29 ++ .../Evaluator/UnitTestEvaluator.cs | 6 +- .../Evaluator/UnitTestEvaluatorBuffers.cs | 2 +- .../Evaluator/UnitTestEvaluatorTensors.cs | 4 +- src/Nncase.Tests/Match/UnitTestEGraphMatch.cs | 10 +- .../Properties/launchSettings.json | 2 +- ...nitTestPytestCalibrationDatasetProvider.cs | 84 +---- .../Rewrite/Fusion/UnitTestFusionMaker.cs | 13 +- src/Nncase.Tests/Rewrite/RewriteBase.cs | 52 ++- .../Rewrite/UnitTestDataFlowRewriteFactory.cs | 3 +- .../Rewrite/UnitTestEGraphRewriteFactory.cs | 2 +- .../Rules/Neutral/UnitTestAddMarker.cs | 4 +- .../Rules/Neutral/UnitTestCombineReshape.cs | 51 +++ .../Rules/Neutral/UnitTestCombineTranspose.cs | 41 ++- .../Rules/Neutral/UnitTestFoldReshape.cs | 18 + .../Rules/Neutral/UnitTestFoldSwish.cs | 3 +- .../Rules/ShapeBucket/ShapeBucketTest.cs | 2 +- .../TIR/PrimFunc/IDataFlowPrimFuncCase.cs | 48 +-- .../TIR/PrimFunc/UnitTestPrimFuncMerge.cs | 27 +- src/Nncase.Tests/TIR/UnitTestMutators.cs | 26 +- .../Transform/UnitTestPassManager.cs | 12 +- .../Transform/UnitTestSubstitutor.cs | 15 +- src/Nncase.Tests/packages.lock.json | 19 +- .../Nncase.Targets.CSource/CSourceTarget.cs | 34 -- .../Nncase.Targets.CSource/CodeGen/CSource.cs | 278 --------------- .../CodeGen/CSourceVisitor.cs | 317 ------------------ .../Nncase.Targets.CSource/CodeGen/Interop.cs | 140 -------- .../Nncase.Targets.CSource.csproj | 16 - .../Schedule/CSourceScheduler.cs | 22 -- tests/importer/onnx_/model/test_llama.py | 127 +++++++ .../importer/onnx_/model/test_text_encoder.py | 39 +++ tests/importer/onnx_/model/test_unet.py | 40 +++ .../importer/onnx_/model/test_vae_decoder.py | 40 +++ tests/kernels/test_concat.cpp | 9 +- tests/kernels/test_gather.cpp | 19 +- tests/kernels/test_layer_norm.cpp | 4 +- tests/onnx_test_runner.py | 16 +- third_party/onnx/packages.lock.json | 12 +- third_party/tflite/packages.lock.json | 12 +- toolchains/k230_cpu.linux.toolchain.cmake | 33 ++ .../Pattern/PatternGenerator.cs | 4 +- .../Rule/RuleGenerator.cs | 1 + .../Nncase.SourceGenerator/packages.lock.json | 12 +- tools/stackvm_gen/IsaGen/packages.lock.json | 19 +- 243 files changed, 5115 insertions(+), 3322 deletions(-) delete mode 100644 modules/Nncase.Modules.CPU/packages.lock.json delete mode 100644 src/Nncase.Cli/Commands/Compile.cs create mode 100644 src/Nncase.Cli/Compile.cs delete mode 100644 src/Nncase.Cli/Program.CommandLine.cs create mode 100644 src/Nncase.Core/DistributedType.cs create mode 100644 src/Nncase.Core/IR/Buffers/BufferLoad.cs create mode 100644 src/Nncase.Core/IR/Buffers/BufferStore.cs create mode 100644 src/Nncase.Core/IR/Buffers/MatchBuffer.cs create mode 100644 src/Nncase.Core/Passes/Mutators/FoldBufferSlot.cs delete mode 100644 src/Nncase.Core/Passes/Mutators/FoldMathCall.cs delete mode 100644 src/Nncase.Core/TIR/BufferLoad.cs delete mode 100644 src/Nncase.Core/TIR/BufferStore.cs create mode 100644 src/Nncase.Core/TIR/MemSpan.cs create mode 100644 src/Nncase.Core/Utilities/DistributedUtility.cs create mode 100644 src/Nncase.Diagnostics/Diagnostics/PatternPrintVisitor.cs delete mode 100644 src/Nncase.Evaluator/Buffer.cs create mode 100644 src/Nncase.Evaluator/Buffers/BufferLoad.cs create mode 100644 src/Nncase.Evaluator/Buffers/BufferStore.cs create mode 100644 src/Nncase.Evaluator/Buffers/MatchBuffer.cs create mode 100644 src/Nncase.Passes/BufferSchedule/BufferScheduleExtensions.cs create mode 100644 src/Nncase.Passes/BufferSchedule/BufferScheduleTypes.cs create mode 100644 src/Nncase.Passes/BufferSchedule/BufferScheduler.cs create mode 100644 src/Nncase.Passes/BufferSchedule/LifeTimeCollector.cs create mode 100644 src/Nncase.Passes/Rules/Neutral/FoldPrePostReshapeSoftmax.cs create mode 100644 src/Nncase.Passes/Rules/Neutral/NormAxis.cs create mode 100644 src/Nncase.Passes/RulesUtility.cs delete mode 100644 targets/Nncase.Targets.CSource/CSourceTarget.cs delete mode 100644 targets/Nncase.Targets.CSource/CodeGen/CSource.cs delete mode 100644 targets/Nncase.Targets.CSource/CodeGen/CSourceVisitor.cs delete mode 100644 targets/Nncase.Targets.CSource/CodeGen/Interop.cs delete mode 100644 targets/Nncase.Targets.CSource/Nncase.Targets.CSource.csproj delete mode 100644 targets/Nncase.Targets.CSource/Schedule/CSourceScheduler.cs create mode 100644 tests/importer/onnx_/model/test_llama.py create mode 100644 tests/importer/onnx_/model/test_text_encoder.py create mode 100644 tests/importer/onnx_/model/test_unet.py create mode 100644 tests/importer/onnx_/model/test_vae_decoder.py create mode 100644 toolchains/k230_cpu.linux.toolchain.cmake diff --git a/.gitignore b/.gitignore index 2dfd485fbe..5b1e72c18f 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,7 @@ artifacts/ *.pidb *.svclog *.scc - +*.bin # Chutzpah Test files _Chutzpah* @@ -306,4 +306,4 @@ cmake-build-* *gmodel_dump_dir* *.ipynb_checkpoints* # Auto generated files -# generated/ \ No newline at end of file +# generated/ diff --git a/CMakeLists.txt b/CMakeLists.txt index b97e01e62c..7ac7539a47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,6 @@ endif() if(NOT DEFINED NNCASE_VERSION_SUFFIX) find_package (Git) - execute_process( COMMAND ${GIT_EXECUTABLE} describe --always --dirty --tag WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} @@ -274,5 +273,5 @@ if(BUILD_TESTING) endif() # Modules -#add_subdirectory(modules/k210) + #add_subdirectory(modules/vulkan) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3b3a93c2df..bcc1830198 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -49,8 +49,9 @@ - - + + + diff --git a/modules/Nncase.Modules.CPU/packages.lock.json b/modules/Nncase.Modules.CPU/packages.lock.json deleted file mode 100644 index 5dd73e5bda..0000000000 --- a/modules/Nncase.Modules.CPU/packages.lock.json +++ /dev/null @@ -1,295 +0,0 @@ -{ - "version": 2, - "dependencies": { - "net7.0": { - "StyleCop.Analyzers": { - "type": "Direct", - "requested": "[1.2.0-beta.435, )", - "resolved": "1.2.0-beta.435", - "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", - "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.435" - } - }, - "Google.OrTools.runtime.linux-arm64": { - "type": "Transitive", - "resolved": "9.4.1874", - "contentHash": "Z46ndZcZa2Lt5b76xU9kxVYbPLg/LfuMufhUVsu3Qo3L7Bibf7WXd9j7RRldjnuv8RIHWTqb0b+2FwwMxs0c5A==" - }, - "Google.OrTools.runtime.linux-x64": { - "type": "Transitive", - "resolved": "9.4.1874", - "contentHash": "zGeDb8FuvP9HXjrsU7krVXtSDFpR+DUGNEsH51k94jL9tzf2vWYI8+WUBRHZ/cGe50dpLr+vIjfcNo3gFyOpkQ==" - }, - "Google.OrTools.runtime.osx-arm64": { - "type": "Transitive", - "resolved": "9.4.1874", - "contentHash": "Wo0ZfDaH6DhiQw0jZm4HWJm/oPGPpWNwOLUz+EYaoH3MLtocSxItHGQj/Ta3HyhXnYNOv+TliAH8L+8RCXu/2w==" - }, - "Google.OrTools.runtime.osx-x64": { - "type": "Transitive", - "resolved": "9.4.1874", - "contentHash": "IAfGgKR1og6vU87axK1d37Ak/4jy8B4NMoElovG/KZc/2UY+cJEAQDA709UMegtI4lBhuxTWFNUiHQYmRIB9yQ==" - }, - "Google.OrTools.runtime.win-x64": { - "type": "Transitive", - "resolved": "9.4.1874", - "contentHash": "fUs5qDnZA6itygolcX6nPuachQkY9CVvQbakIzIiRAWKcaj8umQAbFdGwbkyzp3qp34BKW5mtPVsmMyfQBBjOQ==" - }, - "libortki": { - "type": "Transitive", - "resolved": "0.0.2", - "contentHash": "svfuG5mxGY/QC/5DVheHOCELmdSP90RtxQ73j23KarPXZ9ZXW+7v1l5J77hGDyQbEh1BGrnGgKBlyn76RauGHg==", - "dependencies": { - "libortki-linux": "0.0.2", - "libortki-osx": "0.0.2", - "libortki-osx-arm64": "0.0.2", - "libortki-win": "0.0.2" - } - }, - "libortki-linux": { - "type": "Transitive", - "resolved": "0.0.2", - "contentHash": "b04LWD4lgGy60tys3hPFhnUpgWDM6dN5r1PI7GOcPj8VupXCaI70LKNQ5/5twbDE6rkowOGanVTw0S2wBGBqBQ==" - }, - "libortki-osx": { - "type": "Transitive", - "resolved": "0.0.2", - "contentHash": "O6Q9GLULkDkZEPAZJVKLPH0ROXGVOE7BxuddgOcHNK2oiTEM7wIRnzp2OIlYgLpaOLyxJMisbGOhtWgdzt2Wng==" - }, - "libortki-osx-arm64": { - "type": "Transitive", - "resolved": "0.0.2", - "contentHash": "4Qn2dirJmRicnUG945oWpq7HVGwgqCKKxYPMISv/MRvmpZBbXrZ1cVvRaF8WwTu4XXgfKTa1sLv+i8zLifUMeQ==" - }, - "libortki-win": { - "type": "Transitive", - "resolved": "0.0.2", - "contentHash": "HAoROgAKn8XBun11X43HZuspKlo5JGy8/OYw5IUPo7FVh5TCaPrLjGmyGYYZ2dqLlv31yv/b6s254PIRGn95cA==" - }, - "Microsoft.Extensions.Configuration.Abstractions": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "qWzV9o+ZRWq+pGm+1dF+R7qTgTYoXvbyowRoBxQJGfqTpqDun2eteerjRQhq5PQ/14S+lqto3Ft4gYaRyl4rdQ==", - "dependencies": { - "Microsoft.Extensions.Primitives": "6.0.0" - } - }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==" - }, - "Microsoft.Extensions.FileProviders.Abstractions": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "0pd4/fho0gC12rQswaGQxbU34jOS1TPS8lZPpkFCH68ppQjHNHYle9iRuHeev1LhrJ94YPvzcRd8UmIuFk23Qw==", - "dependencies": { - "Microsoft.Extensions.Primitives": "6.0.0" - } - }, - "Microsoft.Extensions.Primitives": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==", - "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - } - }, - "NetFabric.Hyperlinq.Abstractions": { - "type": "Transitive", - "resolved": "1.3.0", - "contentHash": "WXnEcGwmXfa8gW9N2MlcaPNUzM3NLMwnAhacbtH554F8YcoXbIkTB+uGa1Aa+9gyb/9JZgYVHnmADgJUKP52nA==" - }, - "StyleCop.Analyzers.Unstable": { - "type": "Transitive", - "resolved": "1.2.0.435", - "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" - }, - "System.Buffers": { - "type": "Transitive", - "resolved": "4.5.1", - "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" - }, - "System.Runtime.CompilerServices.Unsafe": { - "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" - }, - "nncase.codegen": { - "type": "Project", - "dependencies": { - "Extension.Mathematics": "[1.2.12, )", - "Nncase.Core": "[1.0.0, )", - "Nncase.IO": "[1.0.0, )" - } - }, - "nncase.core": { - "type": "Project", - "dependencies": { - "DryIoc.dll": "[5.3.1, )", - "GiGraph.Dot": "[2.0.0, )", - "Microsoft.Extensions.Hosting.Abstractions": "[6.0.0, )", - "Microsoft.Extensions.Logging.Abstractions": "[6.0.0, )", - "Microsoft.Extensions.Options": "[6.0.0, )", - "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", - "NetFabric.Hyperlinq": "[3.0.0-beta48, )", - "System.Reactive": "[5.0.0, )" - } - }, - "nncase.diagnostics": { - "type": "Project", - "dependencies": { - "Nncase.Core": "[1.0.0, )" - } - }, - "nncase.egraph": { - "type": "Project", - "dependencies": { - "GiGraph.Dot": "[2.0.0, )", - "Google.OrTools": "[9.4.1874, )", - "NetFabric.Hyperlinq": "[3.0.0-beta48, )", - "Nncase.Core": "[1.0.0, )", - "Nncase.Evaluator": "[1.0.0, )", - "Singulink.Collections.Weak": "[1.0.2, )" - } - }, - "nncase.evaluator": { - "type": "Project", - "dependencies": { - "Nncase.Core": "[1.0.0, )", - "OrtKISharp": "[0.0.2, )" - } - }, - "nncase.graph": { - "type": "Project", - "dependencies": { - "Nncase.Core": "[1.0.0, )", - "Nncase.Evaluator": "[1.0.0, )" - } - }, - "nncase.io": { - "type": "Project" - }, - "nncase.modules.stackvm": { - "type": "Project", - "dependencies": { - "Nncase.CodeGen": "[1.0.0, )", - "Nncase.Passes": "[1.0.0, )" - } - }, - "nncase.passes": { - "type": "Project", - "dependencies": { - "Nncase.Core": "[1.0.0, )", - "Nncase.EGraph": "[1.0.0, )", - "Nncase.Evaluator": "[1.0.0, )", - "Nncase.Graph": "[1.0.0, )" - } - }, - "DryIoc.dll": { - "type": "CentralTransitive", - "requested": "[5.3.1, )", - "resolved": "5.3.1", - "contentHash": "E3zclUh2CIBks1t2uBD1k18pyGFJ1YSKCrbCDbB7qCdl2RAB+k68AyDpjeplhF1ot2XPV82AgyCWBXMf0ggL1g==" - }, - "Extension.Mathematics": { - "type": "CentralTransitive", - "requested": "[1.2.12, )", - "resolved": "1.2.12", - "contentHash": "D4mn5Cab4ztPLJ0V8uMErDrO/Y61098nwrvyIOLZymVAYOQcwP1vomVWKbTagf1aPU3cX5Q7adZtQEQwOy6XEg==" - }, - "GiGraph.Dot": { - "type": "CentralTransitive", - "requested": "[2.0.0, )", - "resolved": "2.0.0", - "contentHash": "ThvS2mQVveSkTMUm04tMbRYzu1XFPV8xBHISrUMp02APjhv9IRbLu3v3upTPCywORx2Ds/c6AqEUL1WU6kPfuQ==" - }, - "Google.OrTools": { - "type": "CentralTransitive", - "requested": "[9.4.1874, )", - "resolved": "9.4.1874", - "contentHash": "jqRoI+pYlym+fhoU25u+13oti5h+772bllQ9zDitTVMclDXVTiG6pxzvmYO74wnADBMdpb2SQlgiNQxoNk5dlA==", - "dependencies": { - "Google.OrTools.runtime.linux-arm64": "9.4.1874", - "Google.OrTools.runtime.linux-x64": "9.4.1874", - "Google.OrTools.runtime.osx-arm64": "9.4.1874", - "Google.OrTools.runtime.osx-x64": "9.4.1874", - "Google.OrTools.runtime.win-x64": "9.4.1874", - "Google.Protobuf": "3.19.4" - } - }, - "Google.Protobuf": { - "type": "CentralTransitive", - "requested": "[3.19.4, )", - "resolved": "3.19.4", - "contentHash": "fd07/ykL4O4FhqrZIELm5lmiyOHfdPg9+o+hWr6tcfRdS7tHXnImg/2wtogLzlW2eEmr0J7j6ZrZvaWOLiJbxQ==" - }, - "Microsoft.Extensions.Hosting.Abstractions": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "GcT5l2CYXL6Sa27KCSh0TixsRfADUgth+ojQSD5EkzisZxmGFh7CwzkcYuGwvmXLjr27uWRNrJ2vuuEjMhU05Q==", - "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "6.0.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.FileProviders.Abstractions": "6.0.0" - } - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==" - }, - "Microsoft.Extensions.Options": { - "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Primitives": "6.0.0" - } - }, - "Microsoft.Toolkit.HighPerformance": { - "type": "CentralTransitive", - "requested": "[7.1.1, )", - "resolved": "7.1.1", - "contentHash": "TRnvDpZPXO30hTOtjfLw6Y9BtTKtTpzk9lefeh4RMCaUihWrVKQR454nYH4/mMJAh+LXqfAPyk0kfkJs0Amopw==" - }, - "NetFabric.Hyperlinq": { - "type": "CentralTransitive", - "requested": "[3.0.0-beta48, )", - "resolved": "3.0.0-beta48", - "contentHash": "oYUhXvxNS8bBJWqNkvx5g8y0P/0LtyqS2pN0w4OWjVDNWEpLbdbvPy9w/9z1n2PrqIjX3jxUsEnoCmxxGnI3gw==", - "dependencies": { - "NetFabric.Hyperlinq.Abstractions": "1.3.0", - "System.Buffers": "4.5.1", - "System.Runtime.CompilerServices.Unsafe": "5.0.0" - } - }, - "OrtKISharp": { - "type": "CentralTransitive", - "requested": "[0.0.2, )", - "resolved": "0.0.2", - "contentHash": "q8j0yR5836Zhv9WB9BFkQt1UaEFyibq8bqJcTiULlILF6/sz8z7Wy2N8sgYdDKsdW25zncIz7j6IDbKM5ynePg==", - "dependencies": { - "libortki": "0.0.2" - } - }, - "Singulink.Collections.Weak": { - "type": "CentralTransitive", - "requested": "[1.0.2, )", - "resolved": "1.0.2", - "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" - }, - "System.Reactive": { - "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "erBZjkQHWL9jpasCE/0qKAryzVBJFxGHVBAvgRN1bzM0q2s1S4oYREEEL0Vb+1kA/6BKb5FjUZMp5VXmy+gzkQ==" - } - } - } -} \ No newline at end of file diff --git a/modules/Nncase.Modules.StackVM/CodeGen/StackVM/CodeGenVisitor.g.cs b/modules/Nncase.Modules.StackVM/CodeGen/StackVM/CodeGenVisitor.g.cs index 04cbd48a7b..314930a8a9 100644 --- a/modules/Nncase.Modules.StackVM/CodeGen/StackVM/CodeGenVisitor.g.cs +++ b/modules/Nncase.Modules.StackVM/CodeGen/StackVM/CodeGenVisitor.g.cs @@ -1,6 +1,6 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/18 下午5:04:31 +08:00. */ +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:08 AM +00:00. */ using System; using System.Collections.Generic; @@ -59,7 +59,7 @@ private void EmitTensorCall(Op op) Emitter.T.L2Normalization(); break; case IR.NN.LayerNorm top: - Emitter.T.LayerNorm(top.Axis, top.Epsilon); + Emitter.T.LayerNorm(top.Axis, top.Epsilon, top.UseMean); break; case IR.NN.LeakyRelu top: Emitter.T.LeakyRelu(); @@ -176,7 +176,7 @@ private void EmitTensorCall(Op op) Emitter.T.Cast(top.NewType, top.CastMode); break; case IR.Tensors.Concat top: - Emitter.T.Concat(); + Emitter.T.Concat(top.Axis); break; case IR.Tensors.ConstantOfShape top: Emitter.T.ConstantOfShape(); @@ -191,7 +191,7 @@ private void EmitTensorCall(Op op) Emitter.T.Flatten(); break; case IR.Tensors.Gather top: - Emitter.T.Gather(); + Emitter.T.Gather(top.Axis); break; case IR.Tensors.GatherElements top: Emitter.T.GatherElements(); @@ -205,9 +205,6 @@ private void EmitTensorCall(Op op) case IR.Tensors.IndexOf top: Emitter.T.IndexOf(); break; - case IR.Tensors.LSTM top: - Emitter.T.LSTM(top.Direction, top.Layout, top.Activations); - break; case IR.Tensors.Prod top: Emitter.T.Prod(); break; @@ -289,6 +286,9 @@ private void EmitTensorCall(Op op) case IR.ShapeExpr.UnsqueezeShape top: Emitter.T.UnsqueezeShape(); break; + case IR.RNN.LSTM top: + Emitter.T.LSTM(top.Direction, top.Layout, top.Activations); + break; case IR.Random.Normal top: Emitter.T.Normal(top.Type); break; diff --git a/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMEmitter.g.cs b/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMEmitter.g.cs index 6e2184c5ea..b6512a344c 100644 --- a/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMEmitter.g.cs +++ b/modules/Nncase.Modules.StackVM/CodeGen/StackVM/StackVMEmitter.g.cs @@ -1,6 +1,6 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:30 +08:00. */ +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:08 AM +00:00. */ using System; using System.Collections.Generic; @@ -723,10 +723,11 @@ public void Compare(CompareOp compareOp) } ///

. - public void Concat() + public void Concat(int axis) { _emitter.Write((byte)100); _emitter.Write((ushort)11); + _emitter.Write(axis); } ///. @@ -841,10 +842,11 @@ public void Flatten() } ///. - public void Gather() + public void Gather(int axis) { _emitter.Write((byte)100); _emitter.Write((ushort)27); + _emitter.Write(axis); } ///. @@ -925,12 +927,13 @@ public void L2Normalization() } ///. - public void LayerNorm(int axis, float epsilon) + public void LayerNorm(int axis, float epsilon, bool useMean) { _emitter.Write((byte)100); _emitter.Write((ushort)39); _emitter.Write(axis); _emitter.Write(epsilon); + _emitter.Write(useMean); } ///. diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ba77ae58f5..2f63e02be9 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.CommandLine.Invocation; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -21,13 +22,15 @@ namespace Nncase.Targets; ///

E#S@!rB~Yw#(} zN^uK%K`A_uwx?GY?P#GEp@EbIoSDPFa}2%^wZcScCvFG#4cpxt65JR=5}c8GaUa;jL5Wu&ML=2sy@o?>M5AO*|NdYaEmyf1;)R1!$Y8`x;qB86v3Ifmvx z9B}3!8h;1eX`~z>qR6vp3om^J&lO!|l$64P5YASDyEj$HLI=EV4bY9op?Q{511vM# zdOC%*dYRSj5Ic|&^qMoEP3{0|?HHJYpAFEKwi0#aZMgFi7+u0Ed(8bQyUb0XEKA2d z%j!1(KaR8Z?M6!40|X959n$`^Ge9X(B&iQF5CR!RjT*~Mf zrBxUGgZS>MH9h%JW*mHw@%C-v226!N&aI^>Vx5f=rvxlhqV0eyfM&JCZV|LQbe1%_ z+d({th4q&v-h+^32Hy=PZ4c=cCUiYKv*Hg*Cp&JnCV zx=|zBU6Wm4&p3+rerMs892O9#+%9J(8R3S}Ql6+S@Z{kz*kOyUOXVdhA9&FzeTMW^5fBYv(gxJe=r-Q@m0)+t3> z^qjwp-59YrAv+9$8Q4eQ^F9OGBe545d%aEvQ9K*9YTxwCYHx-iy3w&7JlNda0fGr zb$6L{aK`x&XXaHzNB%+N<=CG^j}h0fjks#3h?07n+W_`hf*Un`ub|CGexAi8|ZJv>8@o}t|nSUwf~A&ZhtekiMXgQ zW_Mw8=&?Jo)qddh)1Vn1 zc7C>c-rp9vIhL}YT$R0!j`Ka|y_Ivh6KJ(lM6lbAW#X*$dw4V-;OzdtvHpW!qo@J)iQE&} z@xqUzu5-_7lh;lxm5bH~)YoCv*t!uj%C0BJ?Y}Xw7cFZDJtJ#XPTz)INc zzn<&29&^jAcUE4)$!NW=Sl4Alg-+v9{*zdj{U-i>gApfQm+#?L){ymVlzSZ9$_1+` zGG5pbndHuA9jnHub0)IZdAMNZe))=9Sog=zTi7L52k=8VTB!CnuuB?cKF-*;awqV`Y|eYey*p8l zPhgqfVm}=j%oC}G=q~4Tnf!b4zlgn=+j6DE`k&~<#1ZSYtM_>0_FpFsyNlN2@O>NJ z2LBdoxBIbWUntyl-HmRW{SGl{UXE{${(0o~mHWI`qmO|G+T>SQ+nf>*T#4zxn4Psx zfge6mcx~mFduO~E?eLw|pMoUaz|G?mImO;23IMC*5h$&)w{W8?VZW7aXHS2UXxa^E zMb#d+FXGQf@4nhdRMIop_U>M3~=t?Q~hx~^T9+~Yf|iw<`#nP*B_6Yk*L6WlGU5nbYK zw=chFt#i|yH_llP}k7i|xg$f3Py){Ayy#-u*#c;qUO)zT-+Ve|>y` zeZd1>ACavz_kJd;^XGbY$FF-|kFK-d#h2%VPaK)3efgsGJJ-GI-JH!@x5nG-r?6G_ zMc#@v=MP-o;eIi`#9rgPS8e?$TH-h>m;6Ga)~X~Db=I$BH8_p`;6?wTxMkmeWe5Hd z+_7>#Slxnc2h2dHHb2EKZ6{jLfkl_c?nZW`P)gszSZemqE$!z!8Xb0Xz89W0nVku4(YVty+ z9m^+3Kj+_sg?|hCund;9LGF~d`vc&nE5`UVs}3<=xs!R5yMudML|1k;*{9hJyg>vh z#Y`GQ;ui|J+#9S+UV$5W!goW10EyNa<@Q#Vdyi!xuP-X*Fe~0H^0mKE?xD#NPrMl` zHYXSx&tutV(8)j*(~O8lxED+Skje7<(bGFvxdRQ7 zvXf|%gVZ^KRlUR#LTxiz>==?g$$oDP?UwswP&my#RQN8N3LdlUApTlHK$<`o?c;d` z^k<011TrYQgnIS?El@Q`e9%r>P{-aijYfbSkRIU#oF^5aC+s24>mVa7!+q^Za`%&Z z6b_YuxviQjnTLm1(JM(?$H`|Fs*jUPXlX)YT;NGLToOK6DSy{vFA?G#@vV7s2CvW* z`%H`dW)l>T;#t%|OUAixO&O$ZBUc8R$IyXO0YY6bySr*qd$h))PN8@nBi}Cc+!0C? zcBybEQ(#aC7fpB>W26)cEc=N7RYWLmLO~OT+>CmHx{2YxJB0wE z_6y5mn*8heO=xT-mp8{v#A_H~&go__}MQ0Q91Yg-`o1kZ)mBizaqG|WJ80{Vp4 zV{mwd6(=maELJCR$AhDiVvDkg;TY65Nr^%coTT10)K)_71)hjvVblqUZwPsvHj$Hyk%S|#mqDPQpsynwSLL;@-(L7bN4j?uarN+(`-&`O~SaifkL z!ey6TZYF4%$pij9PbZNT$qZ*@w4;O+-IOOT2wP(WPw- zkfl?+StQLebP3PPq9=s5C&a=^-i0$;125#`mO}2U;Xw(%tml&?Pw{6`d`t6QOI~56 zh{r-sX0_t?22xh@dmaD9;~Jhc@X3@WSrwYFa1e!rSVnGxODL>IgHx$-AUroChx%Ir zt;_U`13si}tCV^21yPnY^~(V%y?QIeV6SiPHWQyI{~5@dyH;Ht-k_3ThoS z@5DMrY;jsY$qJC6^b~XT80a^9sAE2ll-b+x+e=}I#ODl4(rK#=U8)S7w4Rw_8>g7Z ziA`T>GdHo)_hFUkWSwZjcGHJ7cAzlD>Nu6Jz;a&0T0}3~lk^L#5xazLRvS(r?7h&{ zOgdH<=GJ;v!xU&Or_dr6@e}L9io3?1VMR+8>ak+Dd~W9KXa}n`?y^`F>w?*B0Pc9K zJ(I|G8k*+W2c+$7IFH20yynup3#n&O$WnwFqSa4NB1g znVIZUXq7~SXMCjb@!p3=B32`*;?fM;2`^zH#kd6a^AMl>L8{^g7q`)B<)N7Aig2^`EsgH zta!Y><^oKeEHTEeOT4fCPV!Kj-IDJhhIbEp9p0mzkh3JJdJ*G*e*tTlKmGxH32__3|P+%3%~06OS~0TJdQ9_W&=z4GtMsE#3Mw3FOGzty zf9ajo@UNO2nqA6yqSakU@H^2p%AlnS&Pzv`rA}#u`Xp?9VGB$1E+w_J+YC}(LwdD0 zOY7?RoC!3HWvIwdzj!PCWDK6mCR9O>%^*1&X{YqT6i94X%s)yR?>pk$mU@DhG-8|H5_Rv|~M36(M{nF_wgSkN`0>s87 z-34lq{@=uR#h;O8o#b;JR7np!9F!pKOuH80A4@BjUM?;O(OH^{be%FNo`gc3o=QI( z=0EE*>4izRiyG?TzEH|DlrBVP>9Er6q@OI1S{Qo5GgoYmMW|J;sB~%V%A2NEvqMms zS<(srT6{IV#E<+ldY$$V(wr)3o3t-Un8Dzdo*@)%#gLKyC>`bkJW_qyM=Gau8#hSD z9VqIOmaTS66Ek|6$~KtHk}=h3v@FS%+9iBRqr++LR|&$5mWHKoTA54={i@bTSCg(O zITW2ruM<}3ONIhP9FPvBmo#RTuAQdFKso=V`3YB3w2OPHQ*}u<)A&)FMOmn)i9YFD z=4D!=7KtLYT@*@hQ-1A4lv_^>a-?u_RkvPduc=n4^~!B_SK_C6iH=|VPx%Z#wDUCW zHl>;8D#4&dnqLhsO}VP4_`A_ejXtV&8hm7vLUb1CDwbzbt7d~VsnUfYsuZF^|3V6b zFTo$_&PpM!=sl!Dy=3w$SrKP7?Cp=16;fe1C`t^C#d^XNVVm@eS`l(0Y=1FFSc9S5 zv^f0LJTqmO7K>iBQ;4FHef`s(LhaEn+DB_145dmlkl~8nO}kCsiz4L`p0trw5(MQ) zX2Y5dpTch{p~!cAQcm+SEe?CE*iQ9@%KK=ae6*dax%ltHX+wURT1}b{mt>xs($qRh zc2Po%REQf!2E%m9rS_ZAt@2cl;YRpbPnFaBGwlh#Yu*!wRg<`BehpKavh~bJq^a9b zYHBfM>#y<_X)$RG&wpO=hu)Ol1`l4-9oG(Y%T@hCIDiJ?2@FZk1jn z;o9|^)E`b)%<+pqh13|%Tsys55ROvu{MxDX)Noy8nQ}wkgr)1behH;S-@?*N>x`_M zbYW^!mgGg_Nwb6*wWh`}fAKw(c+K^qK>1BtQ=3^COuvLPu^Drwrm)9~C7MqkABUzL z;)rQ+F?~q+CtDh}GHj=*%al;e|H)6PK`jxzVbp>m_rm(j*fY8H?AqmiSdL82u)Z+Q zweu-eF-6#O;gj%tSXTHRw%C0C|5AmiKmJ}U{g=zBRN|9X-f*0Lyv$$bd&oQU9?DxN z#~~ji#~+n(wbuL^ju=y$dPeKLRt@czi}O|}b4I?+YVq;5hTrwB`c#&A7XG5=Mn+VB zkJ MjYd;yk#nx~e+}8={r~^~ literal 0 HcmV?d00001 diff --git a/examples/audio/tts_3.wav b/examples/audio/tts_3.wav new file mode 100755 index 0000000000000000000000000000000000000000..deb31e9a5915371290856b76e6df86c0f9abf78c GIT binary patch literal 61244 zcmaI83s_ZU`aZnQhkfRxqLMOdMn=Y{8TliJ9CFCWK?nU&kwXp|nwmOlYHDg~YD(%T zsbgehq>K?m4&#uKL#9TChK?E%A}D9}`LNF4z1x4U`G42-T_5Y>wD(%?`F-B!x$pbg zGcuofX1tRiW~a}}n78Q9!yN=ckhtD@3V*C42#RnJf6D)Jz619V1cOU8e1;2`Y7j)P z5yDU3FF(WYt9Uhp_rq5&u1;LHja&ccYR46S)K;d@;X#GvOle6?r9GR`|aV zen-j;q3rvmNB)ldynko7ZINffzg?&+{CwvxZFQmUaQh>z4Zk00<^2-EcZUBCzaP0X z+|uyR( z4!1Kr>i_);-xcmh_{m7U;h*9E!)?D`UU;q|y@@;-dHw$V_s6sImk|x$cfTLu9!7c> zUP<9s!)qX1_J7NX)D*(|u3uh{{1s_Wqz#dJ{@dpJwcl@b_^xoz{@b^3|0ClUel6Uh z|JD~CukbpIJmJ9TgvV$Qz9ZNDR~)~T7#YRzGvTlM;}`Bl+Gj?^8QkMKM9-;b=D zNS%?_!o9gauK(}-5P2@rwn+QJ^;mvsSEL>Pt^a=i?$27JPm%J%PeuL@_aU+a?vL61 z(GLF(ufF?j3D+07?zcSh#Qk!^^+ZaIJQwLvq<7&~h3!eC?C^8pJHz#c$30x${r*P& zi_{RFwg3CN-;(es{P!zdQsgUqZ}^_@&+y;jwH2-*az$SI|Laerlt^ijyYH79?pb6{ zL~09{5*f+x9ry1I{|@&fygtJ#InuB2tC4p6cf2Cw6n-*%-QNN2MmwxzWS@uaWjicp z_?5`(uK!~}BX@*f4`02%?25=U;rk*b+%F^2`>^G`{mZ>!D-zjf_pM2|b>UaSSERhi zJK-n7);jXseftx+E8MC`Ig#fge@A-x-*WEn>ifNm{2v~dutg5n72eI^D?D2F=j{Gm z-Cqxp`HYNfWH#>qjI621t_ttZ$ojv(8t$)y$OuPDi~Jp~;r^<)U(5Y6!*dp~(Eoks z{#@Okt?<3!ckj<`WH&}i2-gt#OF%i6U;YfYGu+DXec{)_>puL({Z$)TJ^y$ABO?}> z|EtFL#s%ZD5&m6=zi#|8a^bOyoQ{!JMXY(muHIj5kvd|D;lErD5l;~5xSqiE81WEZ z8-&(G+VI~Qik!*eGJUA$#xHf&8dXNAQEcom)*H)yxfUCX@q4|o20vHgox?^M+IAJI zf+2>Y{2XE>afqlT8VMJ{kVDBNGL_6Av&l^SOvg2nj3piTX2=$zjyOSV#SzkoWI5+ULc8Bdi` ziS!qA60?uFz(lc6v2)qk>`eAimSsO@mN3c8E&2ue0+mJ8qZJ>Z_nQoto~D1I8ET%o zQTa-~BaM(IhqeW~0viHJ0c#*NP!X6Jd^Gf$^sW3G^}Lp8oF`^b#q`(A$Lw32iJ!!e z%#IJS@JVv3oCOcLX!57QIq8)#QL>O4edk-fwkVm!QtQiCx{ z^foQ1HYt1LX_75e9$4bv;7j+>xbl7Fem3|(=%m!9sM-rg6H!KW(M8NcR%ajQ=5p_I zgE`YL-|w*>vtO{EvQM*qW*hS|bC`Bfhp7~50o=Aa%)xHrI*~*!CJCy7ilU#TOXv;s zlk_sGf?P_(8!NRY?EeTltTfDWNBc4T`U7n3zzi(9FQt+H~LfNVn8%bm}6~zo@ zZ?X%yW^OWngx}8}0>$W8aJCC#y`xhWp^^0s5QhP{dejyX?9Ta)q3{3hqwy+ zukDqu)~n^|D~)V23qAjmv2mBUmr#2p_fK{dQ%1gEEK<)&Hv?YJ zbXS}+s;{8?Quo>3uHM4Fb1ttp(?2crtx}`iGJ2^enQbgBWQ#{kYb{pWB>Qzox#PTJ zT+~VXb<0lkYVittfvO^^v=hop@>J<`=q35L%J0-cx`miW?x5VX#+>EG@jvh{3MYj# z{JY!|CWA^f?CM*gLSN7o<#hMtb;Nb%cE@+6bh-N`x)o1C;FOeuxX98(-KHF{gj+?ZNNt974wi|`b?mvm_xmH$Yl(EQ+;z{21(Ns;rFkd{ooOqJ4Z zb_8at$5dfnWYNq?=F`F)ZY!0d|5i>8>~U50=Jzz;Biq)sAG|lA-Fh#(x6;|;iVyr< z@oA0ZO?Hx4XYR7~M8(9OkFATV8Mu7F=GcWXMbXP`X48-SlWYc2sP31WgQP#tm*6Y* zCHO0YyW|yWsa`~_W$$s7V!7q8t;;sY@uFj=ZG)xO^bThsx2V^GG49jdSshXBl|M%{ zyIb6?QLT&a=J$_v$9P959mE0hUA6+Ve9bmH#v4~XU~c@_p=E=M22~7j#*T@~FwJDk zsaS1Qu*Fy4z2-XUKJIq9N`03?XG6bJ2;v4cmwk<|H6ON3b4-k}$0@Nh2k>#z9GRBW zLJj$yw8fj#pWJctc6_t@X4lm{*H+(>Zbh}6?J|2xz0b)D=xIWMm})DDJsev;plQgo zgrXtN2g-(dmvTC?9-%@Y=y62A2+69(&gIqOg)YCz;(RyGFI=w}>|mUcK^TVZ+Ua)z{t4X{`sl z6N0(=VyeYl9W!h|aKP9h10S#tJv_APf%8M1gXtk_2i7|do8RTjj8*=0XIF1ZN9LWg z?X8^&U3;AS0%N3H{TjPV*l0RxkBQwJXCBZpcuqoXLhaD{Aw}_%;&$3H#bwl`&{TI{ zcUD{B&#ku-uUG!K@yFaNH!r0&=KfrG_jr#Un$720TB7z1DH^^rA((Lefzlz@2N?sW z4j33)5iQxbS>l+Xx>*_RPP>#BhG^EbN9~Nxzcj<=hUXmo4G$-`A_;)@h00I z)pyEOsEoEx8#p~K=7A-@YDlP#e{q1~IP564?J;@9UD*4xxoRz1dLxwW+|#k;Zr;7^ z9T&TjoqMDZagFV=wnewatQs(P(6G3on2xy31BzqoV>ZY3#7&IOw)BV(atrjgLP`E3 zu7iDBdbf0^bv1Xa=;*t9?atYjr9X=|w*R!Wr^`9b^(8$yc5d9*L9>TWO6Z8U$Fy1# zv1^X-I=hyc!u`O%LglD$hEn}EopGJ5ckg!J>?v^03>9c5Zkv5t)RdSxgXRt_jolaB z5ao=z?ik`wqN?%tS$m!NjBtVOP^X6a0*c4!n&vv_>Tzv%p6+e#KGS)v?e3k#+sCh` z-s9-E5DLE#^0hU^#4dbKD$@y&b@oF zr`5U3r3EijW5rraTU5~iBW_CcoTxI4U9RP@<*=hRy2LR#Dkl1-HD7#+DbUVH$Ak6W zEuIeVW`C4_gy)Q_&{^4;*S@DMzPa`K#U_le%jteZZHO8(s3>~MpdmxX#*MQd7h7S= z9_BXV6q?C7g?G6s-LAYQuXbm2r*t*b!iy2fOv(#KD4&%S3|H|69@99sO$-)I@q_$C>9x}Vn^`vy? z_Dyl^@pzSgGG}?KjgO6qSsqm#H9ac9dQfaLZ?H|Vt+mgMULL){k!C$3bW&B?tFjiX z^=}ANhD?%2IwHLo9OAv%cdV=H?zz^}KPNV=Zp!RR^_+6{D3fhPjx+XQuqh?cQ*0N7 zYuq3CX11AWWgce7376O^W3~E&Z1r92%kOFJ8{t~&c`+0sj_|uo#nzapX;E=@(N=4T zvPhOJ>qT2r)UcSE=%%PdTdC<`P9Qd_Ta?MtuS2@@kn$h-ZRuRF)|2j>&~vRlsx|G- z&6e#y72H1QtnwP(t<-!+HCnPErY%ed&;7}_D){MIq1Tl6L)NB@WV z4|$_ZtIw(16-B-ntoBqokM?ft=xdK}zt+03CB6Mitpym^0RT)YqWzft3Sz00;^q<+@wFw$NE>mN>>wKGK<7X;&D;5w3rW@-If&F zLi;Mm!lOpy7$nP8HKGyH=IoP?UBktbPwrlOzy5ss!yDkORXde)3C^yp#zvI{3pSb{c zkUs@~ZHmw!+D-lZhpa@Wkel=oN+4vD?ny@_vvNYaYOIIA2wG9_6g?qMDf}neQ=1pAjz`htxmG(?W@Xbl)-05I610bq;hb zat(85!jsJP#|8EU4hQFkqNSIlKguy`me#BH5<93@=vL-=_AX;(OW?Iy=@LW^BE0o~ zGS9Nh*(>a?S&`k$NOT|lJ^du}IDC5@ewbHKa8(NZ@cPekr?`*N%l-U0{vCcg|0}MD zsij(omHKe?KhpGIoo|$Pzh|uHsOzR{hYR%cVTEg|d!1*IHw~WD9%;L>5-sr&HSpy&{jwrfF;B4% zBJx-&bYP_(H0?LliG|`KF(?c~{fk5zCnaqfXsQ&ahzA8)unRw;B~uU`j1kWAQ~BxK zawd%$P7qkXyetJX{k5K;YbxfX(3Rq>>`!zaa~^bdIVZYKVzye`iQal&a`08jkXuy2 z_<&eR-XISnhS^8YW9G6mxwrYV!UVJ;$!wTfOiN9frsLvCu~HmnGQ>2~YSU=bQ85NB znJpaUU*g~9|AZ)M8XwIs;rz@kY7p73=V_~zKS|c$3g0kKhikjb>>lSj>RjzyhZ!2- z>T+(!Y8mY|+~>XZ{-WR+X`Pa(F^F?B5y`zmCDSD0F+V#IQOS0(z?5bVnzGF~<`%Rh z$CPc_g0}Hyr^##DZfX+8pd~+JZhk=IbO-Ztoagwzv6tyvWT_F-t}AA_Bv|ZE@lJ8a zxQDq%yRJC*I3=`WtSiPvV}y9O(_Q7w@J9vZ&@s79jYZ7;8L^&>r4r~U?2WtZYjho8g)4#!hz6be(n1cFlB6a?N%fah-LIbEkP~y(xY%SQ+|*{CmV% z79*P&N&3hr`Ud?J#$ycTe2-X*`f|*5<`w2Va|K4A!4!}2Nia8=I!rO972*i2;&H-H z7>8s0Ow7s|#O@z6m#K7erBSQ3C}ZWjK{l`*C-6A;BKH~B6xUd^VzO(5s~IucFjtFf zgr^wep#|PY)bl^;C)#bj&LGGV@-j6GR^?6ZBELw$&M+04d9nmJLb4+i~EQt#U}>N2H%v1C|{|mdNOdWOfrF5K$kENaFh6r!cno#WSG_=VqIZg zikP*PzxmSXZYv z&oj3u@3W)fp%;}YZ@>v`Q|c{*_0z1LU=4rBA!1Zti%4D$j{`SMJt}c8XZn-)wiqL zVLuiItiG^C9pTx9$mfh}ta}CKWx8u3M&p!wh&RWV6nH1}sDaekO%u!(1z=E zdIjbqohrwPCUbj)9-F$NYaS6m@@P&+ML!aU@N!^F`z z?46($)-cwi+LC=K1RB52RLz~6C#if{* z*k{_8*e2L&EJwwS!U}OL|0d3y z*)}?QTHK_#j_B!88P-$$T4n;hl^BT#tksW`qer`&bUWo%;!p86j^A8;_jvD_{`sLC zP?Pc0RBpPkNG!IlbyP)}V_RbU_Ko6$>~cD&PLw|ae)u}h|E0>D&@@EMQ$zodA0nP+ zKIJ-0O_owilcmN&+LqW)+4-oF=-E*_?V9DH@F4pNm0%puQ;eW$mP-R0-L1XHI`i-K zwTXAO+{tcE`njuz_6_kKP*X6Lyvc7}Zog!kW{a}cnqy4c#mCut?cb`PJsisPzbPG3 z9#hQ9AU$N%8g)b${UA3}-Ty}k3uy~?)Z&4q0h{vV`MQlnTMJKA0Z#3a>nP8f}oXExWz zd`@3i>wTL8`=xUDLJRbKVj0ktS(J;sO`qb<@a3io$5cn2ImNQvGDKJcuin6k+6~O+ z4F4t0?!DwnU^#1WEg;*p*Q7${RZxc<1lGo7dIMcvJ9S#*c=mr*r3PS^nr4jbQK z+GT3y>_RmE0#ibbC#Ff~-SwWq(*H_7NFzx89{$*vcj|)kG1snEahQF<9-jdrpQ9X}gJUm=%a+ zMV6%>=JHMFO^V5ForDM`j!mJ9>6ht$(htDXIxT$4eMy&7FHkjPDOpN=j5&Bjn=UQ% z^PXX_wv%A>^LwpbQcrxx(YwEshnSk#7IA8f+wv^E81vdr&u4o{a52PO#6U~o>o;mC z^z(G3m}>cnPed)pgeLK{kZU<*OE$m9712@j=jh?17?~V#GG7ckbAvn&tbR9HMOnz- zDP;k>ceE?pdCYYbHaD&ROz-OcsC(?aPNTq*NUt_c9bh&uAUC0{vzSz1+_lsJ{g^bt z-{u)3y-a<{Jj*1Q)2#2asic+37IV#$#4RSxGR&0793sD_{p_!~_u12Yk1&&6Lmnck z$wo4r*g>R`OSL({81Gp3Y3C8=G5CgUE^GhEp5q;r9dgKtyKBTr(T(P3nOmfboWZ;Z zyE2N(BOg<4`pf;NrH_c0=?}RU*tE;cH}v!DMZONvM};NL`ZoV1O|maD+vs!bJHYQB z;L7Q}z^Y5Y4eubTjBNc~sm===*qPWr1|I1;SXqDn-QNAZ`Mv*EYQz_XM$=APt@swl zkoEdS{!=#Ia8Vx+FNO{W21xbVW5ko}5%`2HhK)MMuR%vtfFa`zNsX68QHV z_a0~Ce$6Z(7Z@te?DKlA_L#CfxY4uPd8~hh^N6d~Tk9R+sskRM&^_Cg%q-!)VOWdR zdW=tI){`z`Em#c;n3D$LxV$;ILiJHBGn+q;_h) z7qaj2lelNt7pN238qJ4z@VMM9T?%A-V_cd2N1aaB7H_%N>6srW_r&$h43cyz_=?rQ z4&G-o$p_iVTq(JiTu&vD%gD#HBhnyp3g5+EV#_Uwwn@TGwhgD-Xdp(Hxar(xb`8}_ ze#{=>udoZ5C)v+whG{6PT}%F`GThf6Dxt znaX|1K97iSAF~6!+=SIoOTNVoge_|_l_4tsD|-Y{&*y9Kc z-7uaYt{N>0cD*E_H!D2DJX>6q?osfY&U^Oxs=PJc(^3`C@nmu^`!HKeS=nodW+mn? zY&l&A8-EC^ejjs6Fq=jTV@yY2|378}{3P)lKbebTI$^VY#w=<+*NU-up59A7rjC#} zsb4b7XF_FxMc&capW{3&o;d&fV1|FG?_%JO(jSxpZ36b$H@L2WDQX5E@HxARy}?A$ z6R27A*X$hjQ*J-Fl(F#q?!Y6z#y!P;0S@h-^i`awYr$UZzTm+)YiW9$^13`d zxYn2FONL!t;;#xwKC8dM-{BeUnIAd<+(gxvQ6i(!JDJa!+3a7qE@mJ7A!;tA9tV2% zF#91pT^K9g;dMksmw?+OG9sM_>st+*Ig2W0SlVYSBhx5Fc{{`_6Xn*>cDY}=89Wub z5~}j2`X3CP3H>$nR_JiBIXFT-p`~fRQE%%fz-wlZBs&dW+hADqN7*FIp+xs{LlCbw z!IRN|EW{zA{~LDGX|9CIBW@UoAM`9@6h^0pT1u=m{-J)LY?OZnCb30+Q~EIUVQ8k* zEd}NA$|7~0GD-do))Ox;ReXAtRt>a22Yb+hz4stcjmGx>oKSy|evkOKK~2{4 zwRP%8$_aIZaz}Q6(@7^>T1Hssq305}sW0d(tf6PwThszdCDVvU5T(|WR{9(GySGs1 zKKh0+lsEvsakswJSU@f|W*DCkx%k}^cUtg*;s*kB&HIP_IfVvSyH zrS8+pjjh0$vcN0nYX!KM>y27{ma&rfh^W?LF?NUabfSQ; zkTal=I8Uab9r?s{+;>24ho4o8lW-oqr)09-NJcyqUNJr}b7@8z`2{@>tr$w=6CY^n z^?9(ev1A6(%*50Cj7QWHS~WROug179pwfwYJ&lSb)??*V=qt6$STEzXEX-pCIMpmz z#vMkz`h-?P9HxBwCSy0@Bj*_w;sRZ-7iy{@GPUFyqm1sQzEYI!>iQPbz4`>#>6Kai2y+p4k*62w_4fPJrmSi$hAFBNU_OC@*Pw2on6PR`* zQ_EL}$tTooax)Xf_^=KeFuuQ0^Rz@T`wdWZNbq6@5sRVms5UmL1?oKbRJFt!@_8sP zW>6Kzw`!)oORdp1L67hlaa)h24uFlG0UgPD?A&yHDDfn{6_|qsyJ(lvs1y+AVOtYu zmdT{5vHnk>{q-mz8@6%*=02O8hxy2WRTz(z8lLw9+IZOTYNCZKP}wno22Eko4QYuut#W1TuwRn%lC zAfBc7VPBNMGtbe7YwHm`&cGV!G&VsSQ4M4Q|3P(R00D9kz53%&Ih0c~jCuMKh?=sr zecz1X_e6}xHFaN z)X!rcYq7tos9vaYGRP;H6mlT+G6UrUS{C^yAO`!W1F*1x&_`-4qPBdfeqg`I98%Tu zU}0}VWm5{|Qy8q20wNP920@M|78@z#DtZRpOJsvHN`r+F>G|B(%x!%DJkFu& z3}Y63m6=6tGMcpS;5SqmJDI1!H*5vQ_#3rWD>CxPO*p;Q6W1{-0#2HGeF9m7)xQ|K zCYQ)GW>5@Pcq&eieMTm+)o4@y7oLKui*zxQ34gdiX}~T{p^o!c;2-YRCm4m`tbVWV z)mKvUV0kZKXC-0Q)&PNT*GFozqb$Z~jFYp{BTYEH$XZ^eFCsl|dB z*-KWTZ#c=IpczHQl7wEb{U~`A!hp7czsQx+DaLrMTl%+>s$a#rOh?OpP;%hal)(Rx z=$&*US%ovyWvtXcQTM8Y;9vO=)vYIDsSDWWvBW0eq;0T4zm>m}KhuVjNr;Ks5g|OG zZB!4TjRG`Fi@}6^&wS0eaMp~K=7v@%4!xA>VoY2T`GGn_c|xhysxkjn#LGZCm(wBg zI?f6oG;J#MXmzmN+4^VNFr`gP!ct&9v1 z7|OrpEoz3515I2txdvnX0BZ+2yh}LDoY%KN=Vz#U2`GX18>qX=NHAtGpR&bZ{c_1C zv?1~v((hzh+An=47ecGmNiJv7sh{KxzALU3;BWl?qtXgBA6{N9X6KfnlEt)-T1-no zR8pYgdx3hAohYt0uMlSfZG9b#?GpBL_DgVn53>V+S^Jq;#!|F_z`2@8p@oDM+BS!A zPriqk>m@l5njG3Aw<%vK$G{O^>m$1hI=A<>`hEH;W-8wXEzL}U;!+UtvvfSp@$bMS zRp?JM#{|16-g?YV+Ul%*wxIb#HUvAp8)w@NaulPpC%HcM7VW@JvA{nYq;(J}U~dD` zJE7~!XNU&w2B!rYeEFVA*Qnmqp47es-w*P2rJC??b*7!>bHYt-79yxLsux^sslFIK zWFn=}!-SJ;&kBbBlstABJYOJoPwC-GiWED2j`lMJz0vgxE+{;pgXa1UzI-$ z`JsVa;?MSHcn4_njCA9F7KA$V#@s4Y($kD3pV<@Dps3}_HVMBH7pm3fM8B=VKx!3}{b zpWP$&UF%8giobXBPG)mKZ-b}|BJB&=Y;&S*r+ublg?SoRY{Un;dMkUUhb%xTTqHne zZmz8+wlTIkYN2hv=_zKb)}pqt}FB2mDa-B7h7h#=0}emRUfTO)l^Z;&bX;jQ*7_DpDBhns<*OlqPz_2 zRt{0p3(RuI=s}GGuGyEG#)_YEhcJgXaN5ra7JGLFOZCUWR2O3vzsVH}lfG2HgdHX+8O!Y3=UTvp-ACk}oEH+plKYw~`r_*+Ytlo{#AK5$=<{ z$NOf7nzXs01m8e;AAQB9#7v3OtQDph@ZU%HyX*pbw|+4g@82#*VJ&D(9WadLY=-HY zHN||1OQ2h{CzRhRKg%bB^&V?qTvy87gtmgVoc8^#XM4-y%0?T}&(hGe3>Y%BZm`v& z>N7*M>u7I5|6Tt}vfH2IuJI?(&DNTjj_54wL~&ABO8{2$47bLZ1WYGcZ-xiF5>~p7 zBbwI7Se#%uU3998}ivuy05t}x$9JSPFF&^cza{>YWMsh_R(X+fYd1LjISL$ zC)RIqki(?&zAMgD-*44_%b8xWugNRXW9(7(Ldy|h6w}3Km=n;SaeRgGom^sgxD`Nv z2&ztBpl_zn@x!17N}&Md3G?#zaLyorYEZ>t9|v|<~#9szLKklTpy@F<2x?K zo~R6IiayIXDSr-K0uONl=*-#B-9D-RAM!MAqVPDI4i!LK>oS2lNt2R0cIxvi6bA^}3BOf~_7;?1T3h%`dn5Eu+Pm>|fa!kvAt= zwu_f(7a4*V(g|E<6czTq3lOn>rJe*DKG35%*Cu6 zmBloOaqL{~8#;;n6-Nt7M@AZJiN@KqS$py^Hi z9Vm9LgYO#vZhi)x3#|Y5$~&P&KC`>8XI*DrchcR|o2hMok{8o|kdO6cdh5)=csjbu zJkXJ8Jt9;CNuOgaGlj?(LnD+?@bMhP8EGG|;FpaV^z*Q@<%s{cbwWdd+^tZiQCq2v+10 z>$ccB$3->=5yR)qLGfeuI#}Ybiq@%Voy$IQv31NrR^#HcKvSegtlYB)uv&vCOP66{-WTw71?lnda(BlLH@g>_!EkMk(mR=|F;zC4gHAIf~gjTJQ z!FAB*CHMK;*seD3TTprzbQgA3>lv2ARF1B*Uoc;@Vg5DCTILCQF#;Jv{Mmq5@QrP@Msf@QumXKsH=UyL_P+w9NmD0C9=kuwaZ`jWDe z8qEKixybfeYOUw^_qinVVMlq)8QU;KM#Z)Y+fi{9c*v{J)m$giaH5YVo&Xd0m{zXW zK#Be_G7gSQ%|OO0-PYc`-sM5Fe6~Ngr_?Klo3QO>J~B=-r__{(9{h!que~( zu_5M)?UZ=fa@L+^U2XOX7lEx>83HU!F}2<(P==}n;P+Q3j^bxVcbd!6NYcqQtjZumA^QI>AT8H9j0gt;K ztl&(pmD#{-K%~E3Z^OJejQ>?kp$h*h@GgtI@%^Xz6c6ny=u7RL=x+8;lCA|)18+#* z8`Z`dU1ZJxWu68VPo1gE(G*>ezTCA>jIMG_h0=2@7`fBJLS#;S43D`87+?wTz(eY_ zP=miDI4-!wbI?ilUvWAy+t>Q8^k#U=0;T>b!2rC>d>~m>#${>%xPZazej(X<%D&28 zV_jvfcg&Aj;wZM}SgOs%=3OEOW^piEN>w3u<`5#yMatPwbzqyH3{3Fu>O0$a$$1n! z{{Fs{-YxE8eejlIYqaPc2qbj?PKj@qMKqG9WUCJSfREx z<%tv+k0fR#m>nq7pj&uOo*S(2Tm4?|QCGY(#W@qYo(cW?`;NLAJiEM2zG=Z}&^`WJ zod=)#Dm|Eeo}I}bHqVAF?6bAn${o|9=R}W#p7ok_r=CsSeK=eoXPP;`YnMS*NT&>Pjr`>hXE%}xQ<^(>3R;EWO0g|zbewh6;H<4d0O4fz; z`St|+MEkg?4bf|(_SteQqGc)gvV;6Kb|=u0%Rr?p`Z{QGb_Q4Z+k9=_bk8O58L6JJ z?n>uHXQO93c$>IDbKqoftgNUF`Uk{iVA3x`mr9$HETZ+2b(?h}Jir$nt86vaL`xF% zNR47E_=RVH{BA--J6!)j39AW{gZqMO{bgWC5`0PC0#CAc9W?1R{v3aje`fGX=nZK& z;*wNj4bcvZX5-sL@G+)hOR;s1wbUlr%B<;@R5LOpgr(wpTs!>?B2X3gXCi8Q3~~KY zXj`v@t^|twDS?vUK6tZ<{!)K@U?7m)7k3++xw?hwvj!Dlchk)k9BH~IlEYy?GE0JJiJlu=III#m-zsdYh{0b;*N5FoZ z?FUZR)SgDs}~z-uruSKUQE}9z{a1E{wxROMap)W3Kb~bh(`*6#ta3vJ`nnw z0xcJ@>I|S#i;YIuO%m#;S;$8+ai79>c$2%!yv5%JE80aTLLso1EMf+cW%_zD94&oM zIrQ~Vn$_z+AqS-xxStmo&tqWHa`hF+MH;C(m67^Lb%F7mdR`x?Uk6fCqvw(#U|l20 zGT?PS*v-pS60-);)@>T-3bh!@xl#(zDsa|^z|-tQ*419D-%a{(vPoU}%RDW`xqB79 zObtAu6VUF&8U@IrDIf~eeBv{;5uCth>KAOTI*)1t8eQ)~1Wp!mP7m(Y9Fbf8sP;EM8*N>@efLlo0X6&ZwagRa#?I!iA@6yVb* z&;#Wps8kMU1X&+OA<0JVJg~S?@V00*AETqFlQ=Fkn_Y>~{f7BL8q8(@`Pf9RPdjm79(HC8A~O~rx#@ZZVnp>7?C*Z!_7 zrWw?4fiGICFQES;Z(u({d5cLyd5Zau{46~loUE!pL@dw`ksoOSS*Tg4dM%4+A-+@Q zvu(=r%t}oq5{;|$P<1c4g!xfABm6ZK$CP4LUZ5XQ66tjPNqV&^0F#adM!N%SRtjAO zzT+tdag8uh9z^V;4=K0k@v1-#RG(sdQB=5bhcg%RJf5Guk9vxW6hi=JJe6vvv|j>R8uRF zPqdQoDjS$B@;*k;&+AvI#n{bmj6)k%X_N8{U1R(t*NU5iv$+N>N1ZL~mpUl{I9)cP z@iLvkI`A{i!FRZ9{g6I`ek0V%VSO4Q;CsL(z?IaY-dsHgT+?=S33ohrfk`D#N#}%} z0Ymsv+NuxZUy^_w(;Z3;@hSV4(91yDRvKT^4~FuEJ<1crRw`XDdHHHFH+ z_%v#@#BgAlBefTpC$!?ercl`dKB@z#;3go*&74C^H%h5SRb)R<2gqPXMIJ6xff zqrb%$`(@4a{`rwodM1YNb0oUcGLV&2E0XwahS?7)`Khfnm!y<#Mz;{ z%pho__o7$T)KI9ScEJB0C~pzo2rXe&f_wNU#8=u z)e5jHdARQ^mkS&HG9ydTT!&gg_QGC2McoPSDrMC6)@`0Dh8uNO!Y? zrPJK|p}#S;M4`g*PWfrDs)azWd-Y_jm{*V?v=@1U%M6C}s*ke&Qp=4-Vu$`QQ=lND z)ZpdwA|<6#pR$|%i^L}7U+QV$pnL$h?i!*TJ9|ClRi9xR!40K=6A^ylLT`chsRH*= zp_hg6Y%POasjUSQ@EUe|J$%>6+`n+H9KwuD;m%4vfU(p_WcEPYM<>|CVQg@o!W=A&$z)3}GywKT>j`>&_wS$<^SuKhfs1 z6D5^8pw<&JxzoWN(6Qz~tMrjN!HA`*wIn)EJp`8GWhg6lDJ*cF$GM?uEUfJ^qm1lP zUZAtI%P7ABd~B)yJ^efN2DMj>1s46fY^BSfA#5Ov!6!Wf_0mwioqkS^r&X#@?FF{| zg5jspH|=rmi1ZE9ssBsv1EZU+ZPjm(f0WR( zCgg;G&({|ryL%aR3lZWc7@wD!|50~?7pc`=rY;*FsAW(dl;|VL;TkwXpyWG{-Etl5 z&uwrP>tW~9pwMhVezc2Zps2nMM!y}L$yTg2wAI)~t34xI1;{I{D(M+?>it^W)s$N|0A5Xek;C7Xc16J&#aov4E+NnA%_Sd>d7QN(c0j-t^^8y9!wMi zu00DZwW?X5s;C3|Y$2+_?N*bu@PI;K5#y<4WCr}R%gi&(m%u=uW?$qsFsqmdsi}!t`P7I#v%W+N*(CT&dC-m_(=`#3H z1|0MmA8C6CFWEnJv|{!8#<)8!IA1D;#4{#W^vJPcgz9@(yNQXjH?CQ5n!Gtjc8cnx{D+U(!bo6&k1yb_gX`2fNJYW<|MCR9xvK?M-H;5v&=6#&RhEt`JKZjEllIqg6$yi83QTc z(hvcFSI-28y9TFBDpY!#5VZ|PPRyrZ@5;$s_;K~xaOlc*2By0D`Y&QdN&dv(wH~tl zv~!4ZGQ5)lNxpx}OIVS~ATF`Hfa#SGolkCUsiO3lZn#S7W&HrRxK@@kJs-q5p zlYK<<=}WanMS+SX1&n7YR6Ol82lTWUnL+RHS;!81moEV)7z^!wtTskk9&GcT=||>! z&)uF>XNLD^UsQX3S8Bi$S{Qsk^iJrw^d)@|_~~S>##(AEG{@QL=on-qmfD6vk=TN) z!p*?rhLT&2N5HhF!q&7XBUQ7SuTDS|v>s94HmJIP<_nNnFdCQHlxIpsU#wtZ9x%)AhnhpRgda-5G5mbnDo3-G#!4@> z&5xcG6K|oBefAnShF&TeSxbWcoXP|F%vW-tj$5R(sL&-qXK?|!2bFw1Fsx(?Ypu3s zSeuaTaNRUpa5Ew7#fO08#RZOdj`n5tUh1g4eX4y=Ut!O>j`Xg+u1md30)Lddg6Fj* zj1@>pqj=pk&R!o|7qvXB>bGC9iS|ZIfPWAxK8qZw*TR+$)pY0sUzO}~y*3gfq8hJ2 zyFCxtk50jBF140ha?KkolYs!9x9$NtG?g!*^PpPVBa6W-U!gmv?_#HPukuc1%f$|| z|43(LM_*sIv(R%)`ayX@y+H2~rvnk7&3*P_2OYg3`kGC$jJ20w->pN&-3|H>u^7r0 zMZxDjAVU968mQze12G!8&_Q$|!|_4j>+$BbmTG*CLV_&;S-2A{HORRcBP`^er{@_t z+P~z({_AdJCH5wCuD+Mvo^+@A-j&{C$ZL-3&vv)DkB6$Y3S%F07T8BOH$~iRo8qW< z?28^_?=hEHT1->8Ztf_w{#nE=mBBa z3#{@Vln9LZReHZLU6=-xy26}eIc&C~Z%%U^a#w&%(h_uq4Acfby-wS!t^D+(s_CH|R##DES=(4~`5u+QeE1Cgr5XMqI844~O@<_uGT@E*{e zFQ6!l1%I<1+O86#8l3+E_!Te1hjvj585Y{wcah~eNnD2mU=vwQltG=83lt^?>bjN0UJ|G@eHkeJX?}{h z9obQX@_7Cy{ypGDYp4TIMBS1963*iH9&w*@NnkGHI~R2=?LOMu>NH#x z-mYLDa@oN@Xf7!5Zosej6W=KI2xcJ<1{9wc+mJoxM3!e0^0EeV zpF{s$2d}MHKcP;-%B_ax+wN-ai|a}1O6gANUDUVKRqxG%rKhC#L;ni|KTB)WE67)n z5mz8&A!k+*j^OiKY@EdY1)mzUmeHU*|B?}zQhYv+j~os~XgX9X7pP~DfBYBZ{M(RC z{Wd=NXDsr3rwax&&1O?JGJ`8QmMx=l2#emL9F$svGXtXUh&$Pt(tEl)r#HWEb>B2s zp|`;|Az%u<7c$E~C>ESn9Yis`mVK0?_#?tp;Ux0RPI6ty74Kta0N;NQs{O;z&ptzs z!fE5c=Qpjv*>(fDI3iBADad-J_{qpHKL{1~Fys&J7si6~{}k%V1bR1^f!`zJY9?^$ za$nHHyIY;7kxRPKbJwK4;L|Xg^Lu34HoH++Wd_e&p#rfE>t{LDzXpyi2_eo>XrxF40%xtM-Y0p!0$9z}ulo@=H(-uZQZ3 zfk(d*%u_jfe1m$Pewujz8A1y=89TijwY`VB{t55yeeB@Ba31b;jKXN}m~Zf5?AC^i zxhp_+m*ahSN6ca{Zw{bYU#W%4JMigV3?%qh_~QK4z|<3crM@^{j<43&;_LDKKb*Y_ zSe0ek{=N45#YR9xB_$&z<25oe$Cr$Zj2v>z7Y&UZGBq+XGc7fB)XdyvYU(H%nPX&R zwV~5YwF} z$jL144$zV%Sb?JDq>Hqi11JsP*gk-)CE$(jl?BGsD6uV8~!{bzwEJk)BVMqtNMZDtC2@gE6s{xP!#v84X9u3fobBPqiG{-f8Li$u_rm^Ij_6^$f_3289E(2Q8q}-Ohvt7v+^IxzrjuaFSG#O(2it= z2`AN4c!{2dTQos?lvWzU+%*ms`qPY@Ojd)Uor>|>oHRFFi-5ERsM`t!9mmmj%)wL<6Z4g6auC)4?nJ+!#G*1`c8R6Q6kR< zBRkl?+dITF&b7iJ_#^SQOxsoKRca!cN3pF5o=2Je2zqEUU5DLPuv<;ot8zg>S7Lwn zDK?M`*c)n#`UK%!ojhT$1FVy|nCr?qeC38Md@-n=J=vRw+o|EV*%R!gE+$hK)5(DtCC||U!?k_Uj0n@#F*c?%XU$Yw(9QF+ zf4YHw+w`G~m4A!(DUghZIh|EA79u&DueNQr$??stuwS>&bGRJ`oCjS$xf_|y zRdAk{1hZv2EV|FqGmDh*tZEzCa~;$?hhAJbbA><;HB$Ns%npOJjhZX4agL)E@*T6K z8x6Y-5U;skjETT(fQqb7WNmwXF-ZQh06HW)fv(5m?#&!0Wdd)C3G zUgz26`PCEa#TLWq;Tw1%AHY-*)}NhdMdC*jI92%(?4vKS+cLAd3$2w7ddd@|`XFoW zgJ>2eqpk7>M#DKio2^V>rmRq`r#5WNyn@ey_e_?7y>2}o#t4)emv9zr=d_;#$0q2j z^?mO97#7p_Xf=KgezF==Yc!u^fK}e@zvS;1$i+t3&WU|9YLSESt9_16A9hCerls<0 z^xZf`CWw15_nas#b%UrcyAphapG{F$~h@E{tHFO8c&K02Ug*P!3 z1xew-ED3A{p_?1X3S^)rnaU%r_gMz-EegCtK91m7YX(_6n))pamg5-^jEI4hFOU1a zgDzzSekU92&R|NO41xx)82W}T>>V{cg!S%otaUrE4K5-d*=RfV>s9&`O4C9+vm#j9 z``8CYb~%qFpt4^gm#^S6na1z4oCjv}&MV{qCInZB$7Wh=AK#M{H6@)-p&Kgw3LWBp z*p&uT>cQ9zhq05tn}>KN-W3m_=PXo4+hHx-ASdVGK~zveVKXV_-i4&FjmKtw3wd44 z`=xwZ#x)gOdxq8#wIM0AhIC4W#;Bx$EcAx&ptN`MSL8-|M3bh_9+i=@q-iP?Uqudt zimXtK4JpfBCQsLR=NdVyZN$iIhcp;(17qS<+g@@Yq`~lJ=|gO;NK_u7nzTNDEi0vrzt|{75w@ zY5NL=-H=YJ(0~=KEmD*6A!%BH8UJs zsorrKEf>4i9A?mBxN_aB&56t)OB81mk1H>r^rTmn!0wM{w*3UwaG~l=RMjdJkE_=* z_fC>O12;~0S#T_FT>WyUwo>eZzeS zJLMc#Bac1qvsmTwV1uSoM|rYE@I9}C#Q!Jr^y8d9;}kkYJwAp6aAnUjqc4DOQX=;& zQ^k$_Oc=Y zl*30Lfv(9T)k^&ubY>1|W7Kl^XY@Y*5bt59x2L6}u4QF&WaIwd4mM`C&T4zvlci5H zcSX$UH!p5MWWMQ$VLmqH1dtu~cqcn<_FU=SWSQ)^jqL?wub^qJD%nPeZaTLsM30CSDqXSl6jA}Ximv*>%x4W9GLwZi2 zcDkn3-LkIvY(q?IR-3uCRDP*%FsdSUP{M@Ra$}upF#1P9)VdbfQ*3G0T+61eEPJiY zs-GM&JnB;PfT)?~r{Sm{=iaIKF|&Ptai7L_@T+IH>?roZ3{za#(Xb?x{@&z#dZ!vr ztpB(x%eDq(+Zs!FPebRfj;z*6O*st>4M!VC_Y_4I4J?Ya#+Sv*!{dWbdDc4pjz>IK zoNb(dx~!8>kond-O20F5a&$#>VT?Y!4&9C&fjBt6rRc{l1G)GW{sdK^kH%`814?ae zgSl_!&gTfMM>kVItkDF3n(XfQDnO0`^MWBCVsj-&sJrb)oJP2-@2?m zr#Ysju5ponZ0yy6qXyItE+6o?>Tj;&7I%9^&st}sb1as$Vcu8WUA_&63R)wkR@*1b2}8aFq7PGV`o0#jNb-Fl$a(y`52Y;SR9 zGh5u_{nVSRNHSlFdM##pj4~=p{}wp=65CYAR~}hlG%TPP^=xcbk1NONijBpl1aqQ! zh@nMEns^#+;)~2=mUGFAKu|V$N4e(O%X;Q_7j&ewBsHyT^55LvFyA*hcGQ5XfyK9% z4~){SaD;cSY)S4sVs% zL!4|jD4$h)r=E;fP?RANY@b29Rw=y653v6p2*&j!%6Zp7W37V~h}Yh_40Ph8*0{#h zrd@RfjV}d)(R2D2-qv;J<^gWiC}(baT}w{aBug@y3>Osj0V9a!Pc)m&N5h?wYa>(5 z7Zh_`D{T4JHk-qBpR7SW2Ft@ZPEk`C=Y6zF>~bUYS2Wv{CU_xvzQKOMY3V?v`y{NE z&wa^Qf6{ug@g=I;Pc$8BNN-JVS=o_pD86m%pw|YiNSGJB#hcz`Z`$6J(^X{a<8u1y zImPMZvsFdLnSCnpf^0Ltt1=qcI$$CY`YSQn#cXhu!sZS?nsV6kk(NB5F^ zH_S$%@{YH*?X ziZi3l(iqb+(o$>t(SzMjT?uYwf~L|uF|sY)CI~vMqJi`v0K}$JRSY z--=!EJPO!q^?dXxma0L*fe!L9Ud!Pl{|(#Wqi{{P23C4jJF@I$*1cUTJFc{?YhK)( z*_z+tw&k0`6QTxfh%X(KWI}1K{a|Bi%h9e}+i-Vb@G`u$Gm0Kf8EEhkkypZQ8WO>G z9kHE8JFt(dEO16KU-z1+$g~$_lC`QB^*pS1{joYtQ{{j*mw{BOz(>0lg_iH|MWqIR z@Q(Dn?y5z>b#GU8*TVKg*ws6la+^NzRrM{7ZyIpv_NfEj)Rs7MnpZaLX&qp(+Xi`G zP;OM;hIMa}etg87KIg-$jc;n_%O<)qt(Po%N51c-Y_%#|Uu2BZzlUbV0_-ZuDmlDi zJO6jc3$QOI!-Gmy&gB%iO?ef@^DN(XcOmHBN>s;JcDviv&E{5l%lW1b=j1*^2L<~p z2hA8b%TVZuYi()px7_SWbbRDHqw23wV^tibU26`H+!mn@%Q2i(?(%H2)>wkpi_V?F z&8kbf(I~8K*IgvPJ5g`4Qy&k)9cMSlZYCL9yB-aU8JgSGoR0mmo{Nrb`&ygP+SWa@ zdsFB2j+O13TDG^8Sf_`L>3<|{^KByr#)NrX$J;M9<+U8|n(5f?eM#{;e7KW}sp>Z4 znDDu1mu4HLsP6Hdw%1tVY$x2O1K%mNx{ued#0q&DZfUfvM1C1v zp7YqBo>0Dpwm@{?Zf}P(#h#AF&VJPX_I4(>pK9OJR@c<(s)-CAIC-FQP*L39G!^zN z+9bYRX?x0@0Ltfe?R8bM@?EvnSc+#1egfQ^EuJR0X)_(Uz7qMJnvvjVN8*)9!bWx- zYx*g*LX)I2p=E`&31(#q3My{)UrQ9VAU|^aJ?_=c1@<)SI*YbD8L!Xc_FWxUTHPJB zvYRn81}q&g_O`Z2mrQA`Yf5U4>$KPpdX~t-G#k-qi&Sg%>M(!UYIC^Z52_`BmtFg8 zS@ur1O7>6XA>9EKA|0A%l^G~=z+>T@eVy}ey`l<@rAm1vjN!{*A&QhHc*8FR;5>TH zVsDvc-DEk89%Ej|%C>_Yds_YNpUM`*jvL?{5I*p=$PZ*WU8ya3t;f5UIv?>Zhp%v5 zbp$5raQ*P`)!~Q3at#*Ms({~Fg|^KgZ>{V$>T45Tgf}&%@Qd~Evvy)5dKB%^Gm7Qx zPq%U2*5h+{Rv?S9g9(Mma4$PSyA=!iC`*+ktBbu;YkpH)%LBd{kwg04j31FOFD5|| zW6NrbZ{5{($@a0gTP7@wA!8}bt?dvt=)&E2T5PbsXk`KJ6 zT;=xdJ-aN!dnQ?f@SSoxPj#j?oo}m@jq6j>_jH_dKrk{*e!z09^-$Z&?s&&Q-#ON( zCzL6QbE+JDocVm%Qu8Ilm+F+jOz_~7-FKjz5f6T30{+Ag;eLkU6LO=Mr6!s{6KEnE z>U1tvGNE*QQgMg;Bjys67TC*9ciHV_*hp*fC>*k!?W}9dYB#qQbRE??qeewn$ITc} zVqWJPVTo%^>e${h*O?ZGRNbpNr^-SEsPzgpGuboBDX5KX#uih-H!Udp*p)?`HgBC zVSXpVJ$^#9ML#0k5*}yXZum@{!+yQNdC;95$WXkio}-zd+phg9Ea^nWG)`?r%4w(s z3)@%%_V)!?)Yf9N6v)e5*|Q*^szKLFTz2$fy1kzb*CwQ(Po+_gv|}BGJK)#4*upDz1_?S_I}k?`2*OZMzbr72hFvVRXam| zH+#9)n1*VATk zW(RJ7CW$~vAW1n$8*l7@EthFJuI-1$+RN_eJp~~5W@rxJ4|`KnL~pG?sc#!?aj$AA zyS@^9&1v|~9%V-`4gJOhwB)8>(O!lI*dF&>{QTQ{4p?$}YOQ5fZTH2FoR*gM(Scn$ zxp`)Ed0bg!iekL82Axm6?Jur({BJQUf$j*NmhaVWFqN8P!ZOT1YpdlZ|I3~Syp?dj zpH=nMyiJ{)RK_#^$(c?1tEaH;sPO^T$Y4|BU3>;)k(l94Xx^6lQT~FHy#ej@0$YAh zby6Y+LD}i*d(fwrU=u;^)TC}^9 z%qxx0X?2Rhfm#q--5>*>RDFn@S+Ko?kLqj2$p*Zb|3s@O8_R1pY63;@z+9|7Su!{+ zoM&NxI3Eh6|1 zy{;Q1d9re&w%RxWb#1ref{y*buqAK6?!5p%cYPutd6 zZ}uEyH6DX*U34HyM=#XN&1W_1gCBW4t^#kq{}t3AYOqtS!s?Q!mw9g( z&5GMJmvq^N(}q2|&(xPuMnfq|_L}m3^w(~%!j#Y&LdS=m%B->vt)ddF-0RslC$j35 zW96;sRTR40znuPM^eR9+tcF2V>PT>&bRIG3 z3J82_8W_PGZ0Qdx!!$>AD-A>Sec8umqO6buc5H*{Uc65hd`~vTXzc$h*sI$az2%%@ zWuR`cYs2R7;cHAowIYabAqRZxR^K1tK3#Fb>T*nVu5iBZnh5_Y&XL=5-lfAT*sYj> zcS#L@W-AIMf>m@xb_0J*1^qP>4U%)pp<1ne01EeaaxN$XovlU{U^Tnce=xI`u=43x z?=~yH!z0Fios~|9U2_RFvlV{@-U94tS5Y$q@#MYRTj#psJnB5*nCUzZ66R%BvSR}t zx<>aB{1x4@+cj6zaNh8zeFdgF9bFlLvIR#B#~KJsXnKl#bK5C*7un2E3m@L z;Lsj&*TICF<2vTlkdM9Y74|ChNGs(zARDHGjjci#{vI^hYKUKyhCT=AW0b!Zq22j7 zD3jy(=@zK*W8w*VoH27=Jp*0I1?)spK{8~+N&hEawQl@FIr4iz&8MLSwF5izHmshf z(et?nZlTiqgJ(D@AheSOW8yg{Gn2Qpa^CxUB%kI;3)*NFap zIk=!A;c19<+Fex<`Q)Y=>7Zl=GJ0QTr4hp3nwVf%HjM zq_GRnX57Dm3g>k7aq1yV^RUXt907xg{onndWRuV-ncW-5Z8vIcuP{Qod+Wajo)Zi| z{tgagf;V~0+vU-Meh&97X6^_l53pxI8Rz)Q1KaS_3gx`YpwNiAL8P-Lv~tp!p=$6B zY?V!d%ltTN>R<7YKf~PZQpn*sJjn_#w0Sd8!}*9gt{=KQZ=sKs&Z@c-pYd)ydqsg) zn8BC(3s6m&<-O1QvDXVD`X}0q z&sZI%fyS<(U5dcpT*iA5uRPs5EBoL-G@+hzL!QUlvV=99@k8CCI6!UY_&>tyu!Gq$ zntAGDkl;UhpCT_de<>dF%jhhnGghjwsG5WqMP5c7M$7MGM#=zl)XXWhP$dToEA)Ym zpz5_qk*rKY%~3G83g{d8oQ5j!xd<(x7g#eZNz(-PVp3ojp6Ut|(S(L#HQwM_-)`8e zAE4ccY8h<9GQ7uec#o&xZS7{jO96-#MzIeXZ*6p3i=B{8Nl<<-1fMgNKbBX|yf>5<;M z2w#>AE?o^uJ6SL;GO#{ZqueL7r-W`|nZE|j$J78wF4`mmU#sx{mf%yXM=eBnt89$_ zCNSzBP!1n@JPGu#7nn^{^lO>?j$ZXIPBwV5IAh_ZJ%WeKh813L7>JNTYE^-9QjDWq z@gM|XwUTxo|4pbF$moqGlo?X+i5K6`vr$lTf^`{K7ikDJ-+pi_`jToV7rvz$v=X(eK)W3$f59G3vZwB_lYQ#I`9T3 zDgSyN`zXsQQdv)ZV&TTORn+Uzmb~R6hhr7 zWcBWj2J?9|={!KKq>-BV>86c@clt7aPts;Zqzn6veX{V1w^N(qnr>d5gOGo*5ZyNj!()I_SP-n)zUgpXTzQzgH%ga8nw5wS-jNLRFhgbffLU*Ulkc@WW8 z#Wmtbcz~s7v%Son3t6;g~`;X@9^ zf|dN{;u8@OSCmU!BmGF}iSLPfNZw)Tp2Ax!enj1f+LUr3P!`R-=?HJNXb0(YQLcL0 zPPB?>C6Tu9-b?;>aR*US@mlx5Tdd{?Sx>E6O`F1{^V zPWn`QE~O|wlX4_-FMRFdebGClh~(mPQD*V!?_y1$Rs66KIQ`{KVyL!>O8i8Mto5w$GkT3jpA6P|W~ClEbGycXXNr5b9dQ2L=Z zk-XtP_Y(CfN+#|m-9fxB(wD9dl~%kfe*WLDe_w`B{fc%JX^EB<|3m#EloxRq z6S)`TNcxV{UgEmYYw=9VZzwm?cn;OC^k0l{@hfWLf9Flg@vXnNJ`dgT|NA?)zWw`W zw`%m(J*BdVJBr*$bt&~}spm*_EdGYtTij7xFCJ29#5nU#PTFeTlqFH67|F|2scY zEv}QMh}1&g52YjBS4uB5a-}Mxf` zR;2U$a!NHXJ`q3SYSDtyx5PW4`-(D#0y?^~Zp&qC>i`lWP-&^M*jLiY=mPQ3g3>;LyX|Np+0>Mzu%|Fay@ zJ*6Ck(wF*&xL#Zv8beZRh*?1FD5Rg?-%C7~N*yX!=&I0vslSN!47F(}XQ4iEtM$d* z#C`s6_maLP?kdvz&-p+~>(&*aJpJ!>59KNJUrJ5nOI#`54dqF?Lfl7s7An8AzDTRL zluxlzi$`epA=Q`|Bho%ld|#@eP`{IET$EAT?@6=Utulu`m)b{+qg(TZlxC>Mh+K&r zO5YH>8?ieUy;;03)#k1Hiu9!GLo;EhCPOo^_^!C()H3gus#pF|x=JyfbisYgg{CcY(oDwQ-e7H;)3sU|`_Tn1`u?r;QnZCs*WwE46!)L^34I=F zBk2y}d!ex>?kc?!rI0=e0Y6Iy)=5Ib`jr^etw@`sP{;1Dq1phos_+E>@H(lz-`pTYtqh(j)YqbWiCtDq3Cq{^xuk z%?#rGdfKkJHxE(?q*93I(%2I1@t@-;)E>9mC^QdCGne=)Q1v1gqO78%q5dmMD*8gG zZbXhltr4pKTV=U*wYZlU86sE0RuMWGitkFJUVKaZh}smXO0%axtqW}ZPpnHn0+3=IwI^dORx~F$D02!7T|aI52qA+Q~{4kSXI)nVTkj+ z#P^tj4JsPzV>foNDlA{S1M5LOqsPGSK5UYgV4K8YuX=^+>tUSaV{MYbgDJujBzT_X zMBRAvk;-=b9?5uUv%#IDW7SV40-y|B=?|#r(wMMfeu1sn51Ph6Osr3c+!O$(=YZ|B5q*nO>ZwG$ zi^R`TfuAW8Z(n`a)HH%f90R{>I1I1rq^Hq*g2Kx-7-mVz<#_8a zVL3^_|D2B(?J8QNeNpcgi(`@kE0l@vaN_xm=; z#>sF7zZ7^x6cw+5wa*Z7Xt38M2iK!lnhnOmN^gilgIX&TA!)Ni?gQ@MP^;fgyRE|q zvBqce>*%L90w+oDDphnN7kb zodybFAl7~nYbY5z^T+;4D6MRzcSfPdanoz{{7p{Lfd7Kw|)F}gPuDBB1w+FFp*#J zi0d6I!heq95!Tdp?CO)K^I^URyxWO$^SXN^ed>hkcGpg4yfcAl_jl3GW8F`oUHK>C zV#=_{V+}?xNFd_IF***w)6dhLfI&PI#m!oxUk724w!*AFPVEOk62zl{QjDKsT5vfm z&F8_Uopl~_s-4M>a=3{>IJ>DfC3-%Sh)NdkJODfLe>`{4oALrXWc5TmZva)+sQFnN ztFP2|5gpfPOfaqizicE9Y7?=xM-Y3pvzHr4Tw~(F?ZZb{6By`w4i0%do>VvN%c`FA zo@~q1?(L{<1MTmbWlOPdbB=ZY(c|@25piM(42WOhIBp{D?Mi*7F%#8|22+N4wJFaS zXG}9>p(5}TCAtV3_`et{XDHJ^`8gu2{^nlmyxNn|;ci*eoZ0N|+};!4d7xWwx7dz3 zuLa6c#wyVD2R|y}ZEq)Dg3e#ds2(~=7FY7-I?I7zCkl;QSdsRJi9v4Jl}Yi<{Zkiy5MDyZdI~E z@P96Ps6}~JlLNl{XWC@4Rspj3UgF|~qw0KGeh+rwBVY=if#G?tI+HjpANYUr{@Fdj zvEOp2^=xBy!=&1{-_oqr#HD%^e^RmHG|{k~y4NDcMKq|3gOmLK<4p*bfEsC%MXI~; zaa>UwjWt9{7;fl`9>Pgg22lmCYb@$=ROp%m^JQ_M(B`V9!Xw{-0`(&QQ=YT5S6y3N z%euy#+Fe(3n?^ay6iVa6>e>9tLC8X(@x~Bc|;VQrR%T#P8lKl6A@q?t|D7j`@zfTPlq*TUwEVikwl*@nPmZxGqB>$Nk-r8+ltk8rzpg2_rM4dKthEkur^}-BMPYAf zTT};ibIqtlMOB#h>SPL|r_>(jGzEztry7Vpaxsd*kE>@HizziMpbodDwWM^WS_gQN!YvksY~i?(#H;78#^bWxKF0(K_dDc)RybNR0Uie?Rjm7yUseZ zWkU=leSqGC4K>sl=SJ)euhP#}>Cljjff@d@CsAqCozSEzGr%{$uQ`S`K{h(=!({oi z-ehOF{itP6yS-&r!>Jorem!*UQ1bvo_`s=sD~NJ%+Nja^ShvH7P@v8AkctSY((Awp}>&~mTa{DfiN%1yn@8dMv7^Mr8U#dr;n=)Nf zAai<$g99W~g__p7+=tE%u*1Ir3fF`6%X`(maTtv|DjO zHBnlfF{uctQj)$EHuJz#k3sJ?T}cd85g*Ih!3 z@MrgduDxCP)&|F!;86X}sPS>tQI|}cjROqFQL%al{=hr_{zM@8#`m~#ye`++rVnG3 zWU-g1RQv(mfN2U<5R{3t(stT5-dSZm+_kdvd{a!_@oV|DX^P7JC6VF%st248e^~t! zS`&BsPooF9(>B~vWi5p{Ri~O7dAi@o=)$lX*-E}!wr#cv35#x;>8NSY-<=*o&V4Y;^P)!=*`N-QHRA}=| z`wSCBj8{C}m%-`30F%$=UxGmCHE#~M)M3=;#j!H4_)t^Tzlnni{Vq)iut#X|GT0)MzIUsES(M zXMEh~up5EJo;LP0xqhQ(6xi^cj)#1c1`QQFw@s5H|SU0+a{dgUwV`=$9GLy3N! z*G~T)N(7f7+65qy27@`E*Rm%qLwD&L>i%U

 public class CPUTarget : ITarget { - /// - /// Gets kind. - /// - public static readonly string Kind = "cpu"; + public const string Kind = "cpu"; string ITarget.Kind => Kind; + public (System.CommandLine.Command Command, Func Parser) RegisterCommandAndParser() + { + return (new System.CommandLine.Command(Kind), (_, _) => DefaultTargetCompileOptions.Instance); + } + /// public void ParseTargetDependentOptions(IConfigurationSection configure) { diff --git a/modules/Nncase.Modules.StackVM/packages.lock.json b/modules/Nncase.Modules.StackVM/packages.lock.json index b69bdc40f8..4830d0bbf6 100644 --- a/modules/Nncase.Modules.StackVM/packages.lock.json +++ b/modules/Nncase.Modules.StackVM/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Google.OrTools.runtime.linux-arm64": { @@ -103,8 +103,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -134,6 +134,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -271,6 +272,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/nncase.sln b/nncase.sln index afde606dc0..065feb6959 100644 --- a/nncase.sln +++ b/nncase.sln @@ -44,8 +44,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.IO", "src\Nncase.IO\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Schedule", "src\Nncase.Schedule\Nncase.Schedule.csproj", "{8E0E0672-0F96-4EF1-BDCD-D31F96A3DF73}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "targets", "targets", "{A2590531-71C5-4326-88DD-6A9DB2EF0A2B}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Targets", "src\Nncase.Targets\Nncase.Targets.csproj", "{56283378-06E3-4C6E-A8BF-7BD85C92D42C}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Simulator", "src\Nncase.Simulator\Nncase.Simulator.csproj", "{901AC17C-7B53-4B10-A2AC-EA7AEA6DC614}" diff --git a/python/common/pystreambuf.h b/python/common/pystreambuf.h index 178a041e0c..27581c1d75 100644 --- a/python/common/pystreambuf.h +++ b/python/common/pystreambuf.h @@ -1,6 +1,7 @@ // https://gist.github.com/asford/544323a5da7dddad2c9174490eb5ed06 #pragma once +#include #include #include diff --git a/python/nncase/__init__.py b/python/nncase/__init__.py index 3c6aa2b69f..cef1b150bb 100644 --- a/python/nncase/__init__.py +++ b/python/nncase/__init__.py @@ -298,7 +298,7 @@ def _import_ncnn_module(self, model_param: bytes | io.RawIOBase, model_bin: byte def check_target(target: str): def test_target(target: str): - return target in ["cpu", "k510", "k230"] + return target in ["cpu", "k510", "k230", "xpu"] def target_exists(target: str): return _nncase.Target.exists(target) diff --git a/src/Native/include/nncase/kernels/stackvm/tensor_ops.h b/src/Native/include/nncase/kernels/stackvm/tensor_ops.h index e918f22f25..84cae4e387 100644 --- a/src/Native/include/nncase/kernels/stackvm/tensor_ops.h +++ b/src/Native/include/nncase/kernels/stackvm/tensor_ops.h @@ -1,5 +1,5 @@ -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:29 - * +08:00. +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:07 AM + * +00:00. * * Copyright 2019-2021 Canaan Inc. * @@ -78,7 +78,7 @@ compare(runtime::stackvm::compare_op_t compare_op, value_t lhs, value_t rhs, kernel_context &context = default_kernel_context()); NNCASE_API result -concat(value_t input, value_t axis, value_t output = nullptr, +concat(int32_t axis, value_t input, value_t output = nullptr, kernel_context &context = default_kernel_context()); NNCASE_API result @@ -157,7 +157,7 @@ flatten(value_t input, value_t axis, value_t output = nullptr, kernel_context &context = default_kernel_context()); NNCASE_API result -gather(value_t input, value_t axis, value_t index, value_t output = nullptr, +gather(int32_t axis, value_t input, value_t index, value_t output = nullptr, kernel_context &context = default_kernel_context()); NNCASE_API result @@ -211,8 +211,8 @@ l2_normalization(value_t input, value_t output = nullptr, kernel_context &context = default_kernel_context()); NNCASE_API result -layer_norm(int32_t axis, float epsilon, value_t input, value_t scale, - value_t bias, value_t output = nullptr, +layer_norm(int32_t axis, float epsilon, bool use_mean, value_t input, + value_t scale, value_t bias, value_t output = nullptr, kernel_context &context = default_kernel_context()); NNCASE_API result diff --git a/src/Native/include/nncase/runtime/interpreter.h b/src/Native/include/nncase/runtime/interpreter.h index fc91970657..a8f5814d34 100644 --- a/src/Native/include/nncase/runtime/interpreter.h +++ b/src/Native/include/nncase/runtime/interpreter.h @@ -73,6 +73,7 @@ class NNCASE_API interpreter { options_dict &options() noexcept; result find_module_by_id(size_t index) noexcept; + result find_id_by_module(runtime_module *module) noexcept; /* V1 APIs */ diff --git a/src/Native/include/nncase/runtime/runtime_module.h b/src/Native/include/nncase/runtime/runtime_module.h index 354d747cbf..194dd3b1f7 100644 --- a/src/Native/include/nncase/runtime/runtime_module.h +++ b/src/Native/include/nncase/runtime/runtime_module.h @@ -58,6 +58,8 @@ class NNCASE_API runtime_module { result find_function_by_id(size_t index) noexcept; + result find_id_by_function(runtime_function *function) noexcept; + protected: virtual result initialize_before_functions(runtime_module_init_context &context) noexcept; diff --git a/src/Native/include/nncase/runtime/stackvm/op_reader.h b/src/Native/include/nncase/runtime/stackvm/op_reader.h index 80372463e4..ffde6669bd 100644 --- a/src/Native/include/nncase/runtime/stackvm/op_reader.h +++ b/src/Native/include/nncase/runtime/stackvm/op_reader.h @@ -1,5 +1,5 @@ -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:29 - * +08:00. +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:07 AM + * +00:00. * * Copyright 2019-2021 Canaan Inc. * @@ -837,6 +837,7 @@ template <> struct tensor_op_reader { template <> struct tensor_op_reader { tensor_concat_op_t operator()(NNCASE_UNUSED span_reader &reader) const { tensor_concat_op_t op; + op.axis = reader.read_unaligned(); return op; } }; @@ -964,6 +965,7 @@ template <> struct tensor_op_reader { template <> struct tensor_op_reader { tensor_gather_op_t operator()(NNCASE_UNUSED span_reader &reader) const { tensor_gather_op_t op; + op.axis = reader.read_unaligned(); return op; } }; @@ -1055,6 +1057,7 @@ template <> struct tensor_op_reader { tensor_layer_norm_op_t op; op.axis = reader.read_unaligned(); op.epsilon = reader.read_unaligned(); + op.use_mean = reader.read_unaligned(); return op; } }; diff --git a/src/Native/include/nncase/runtime/stackvm/opcode.h b/src/Native/include/nncase/runtime/stackvm/opcode.h index 5c17c82894..8a5225b54e 100644 --- a/src/Native/include/nncase/runtime/stackvm/opcode.h +++ b/src/Native/include/nncase/runtime/stackvm/opcode.h @@ -1,5 +1,5 @@ -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:29 - * +08:00. +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:07 AM + * +00:00. * * Copyright 2019-2021 Canaan Inc. * @@ -190,7 +190,6 @@ enum class tensor_function_t : uint16_t { gather_nd = 29, get_item = 31, index_of = 36, - lstm = 44, prod = 52, range = 55, rank = 57, @@ -218,6 +217,7 @@ enum class tensor_function_t : uint16_t { squeeze_shape = 81, transpose_shape = 87, unsqueeze_shape = 93, + lstm = 44, normal = 47, normal_like = 48, uniform = 90, @@ -614,7 +614,9 @@ struct tensor_compare_op_t { compare_op_t compare_op; }; -struct tensor_concat_op_t {}; +struct tensor_concat_op_t { + int32_t axis; +}; struct tensor_condition_op_t { bool can_fold_const_call; @@ -658,7 +660,9 @@ struct tensor_fix_shape_op_t {}; struct tensor_flatten_op_t {}; -struct tensor_gather_op_t {}; +struct tensor_gather_op_t { + int32_t axis; +}; struct tensor_gather_elements_op_t {}; @@ -685,6 +689,7 @@ struct tensor_l2_normalization_op_t {}; struct tensor_layer_norm_op_t { int32_t axis; float epsilon; + bool use_mean; }; struct tensor_leaky_relu_op_t {}; @@ -964,8 +969,6 @@ inline std::string to_string(tensor_function_t tensor_funct) { return "get_item"; case tensor_function_t::index_of: return "index_of"; - case tensor_function_t::lstm: - return "lstm"; case tensor_function_t::prod: return "prod"; case tensor_function_t::range: @@ -1020,6 +1023,8 @@ inline std::string to_string(tensor_function_t tensor_funct) { return "transpose_shape"; case tensor_function_t::unsqueeze_shape: return "unsqueeze_shape"; + case tensor_function_t::lstm: + return "lstm"; case tensor_function_t::normal: return "normal"; case tensor_function_t::normal_like: diff --git a/src/Native/src/kernels/stackvm/tensor_ops.cpp b/src/Native/src/kernels/stackvm/tensor_ops.cpp index b9ea85d40b..2cb452aa3d 100644 --- a/src/Native/src/kernels/stackvm/tensor_ops.cpp +++ b/src/Native/src/kernels/stackvm/tensor_ops.cpp @@ -47,8 +47,9 @@ result nncase::kernels::stackvm::batch_normalization( } result nncase::kernels::stackvm::layer_norm( - int32_t axis, float epsilon, value_t input, value_t scale, value_t bias, - value_t output, [[maybe_unused]] kernel_context &context) { + int32_t axis, float epsilon, [[maybe_unused]] bool use_mean, value_t input, + value_t scale, value_t bias, value_t output, + [[maybe_unused]] kernel_context &context) { try_input(input_mem, input); try_input(scale_mem, scale); try_input(bias_mem, bias); @@ -124,7 +125,7 @@ nncase::kernels::stackvm::clamp(value_t input, value_t min, value_t max, KERNEL_FINISH; } -result nncase::kernels::stackvm::concat(value_t input, value_t axis, +result nncase::kernels::stackvm::concat(int32_t axis, value_t input, value_t output, kernel_context &context) { try_tuple_input(inputs_mem, input); @@ -132,7 +133,7 @@ result nncase::kernels::stackvm::concat(value_t input, value_t axis, try_var(strides, get_strides(input_tuple)); try_tuple_field0(input0, input_tuple); auto dtype = input0->dtype(); - try_positive_axis_with_rank(axis_value, axis, input0->shape().size()); + auto axis_value = positive_index(axis, input0->shape().size()); auto out_shape = concat_infer_shape(shapes, axis_value); try_output(out_mem, output, dtype, out_shape); auto concat_dims = dims_t(); @@ -293,14 +294,15 @@ nncase::kernels::stackvm::flatten(value_t input, value_t axis, value_t output, KERNEL_FINISH; } -result nncase::kernels::stackvm::gather(value_t input, value_t axis, +result nncase::kernels::stackvm::gather(int32_t axis, value_t input, value_t index, value_t output, kernel_context &context) { try_input(input_mem, input); try_input(index_mem, index); auto dtype = input_tensor->dtype(); try_var(typecode, to_typecode(dtype)); - try_positive_axis(axis_value, axis, input_tensor); + // try_positive_axis(axis_value, axis, input_tensor); + auto axis_value = positive_index(axis, input_tensor->shape().size()); auto out_shape = gather_infer_shape(input_tensor->shape(), index_tensor->shape(), axis_value); try_output(out_mem, output, dtype, out_shape); diff --git a/src/Native/src/runtime/CMakeLists.txt b/src/Native/src/runtime/CMakeLists.txt index b892beded3..f92450b6a0 100644 --- a/src/Native/src/runtime/CMakeLists.txt +++ b/src/Native/src/runtime/CMakeLists.txt @@ -54,6 +54,7 @@ else() add_library(simulator OBJECT ${SRCS}) target_include_directories(simulator PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries(simulator PUBLIC gsl::gsl-lite) + target_link_libraries(simulator PUBLIC fmt::fmt) target_link_libraries(simulator PRIVATE kernels) target_compile_definitions(simulator PUBLIC -DNNCASE_DLL -DNNCASE_SIMULATOR) if (DEFAULT_BUILTIN_RUNTIMES) diff --git a/src/Native/src/runtime/interpreter.cpp b/src/Native/src/runtime/interpreter.cpp index dc5839fb44..fe69b8a071 100644 --- a/src/Native/src/runtime/interpreter.cpp +++ b/src/Native/src/runtime/interpreter.cpp @@ -246,6 +246,17 @@ result interpreter::find_module_by_id(size_t index) noexcept { return ok(modules_[index].get()); } +result interpreter::find_id_by_module(runtime_module *module) noexcept { + auto it = std::find_if(modules_.begin(), modules_.end(), + [&module](const std::unique_ptr &p) { + return p.get() == module; + }); + if (it == modules_.end()) { + return err(std::errc::result_out_of_range); + } + return ok((it - modules_.begin())); +} + options_dict &interpreter::options() noexcept { return options_; } result interpreter::entry_function() noexcept { diff --git a/src/Native/src/runtime/runtime_module.cpp b/src/Native/src/runtime/runtime_module.cpp index 4b5d747bbd..66acaca61b 100644 --- a/src/Native/src/runtime/runtime_module.cpp +++ b/src/Native/src/runtime/runtime_module.cpp @@ -189,6 +189,19 @@ runtime_module::find_function_by_id(size_t index) noexcept { return ok(functions_[index].get()); } +result +runtime_module::find_id_by_function(runtime_function *function) noexcept { + auto it = + std::find_if(functions_.begin(), functions_.end(), + [&function](const std::unique_ptr &p) { + return p.get() == function; + }); + if (it == functions_.end()) { + return err(std::errc::result_out_of_range); + } + return ok((it - functions_.begin())); +} + result runtime_module::initialize_before_functions( NNCASE_UNUSED runtime_module_init_context &context) noexcept { return ok(); diff --git a/src/Native/src/runtime/stackvm/op_reader.cpp b/src/Native/src/runtime/stackvm/op_reader.cpp index 901f0f6125..4cb7f0e36a 100644 --- a/src/Native/src/runtime/stackvm/op_reader.cpp +++ b/src/Native/src/runtime/stackvm/op_reader.cpp @@ -1,5 +1,5 @@ -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:29 - * +08:00. +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:07 AM + * +00:00. * * Copyright 2019-2021 Canaan Inc. * diff --git a/src/Native/src/runtime/stackvm/ops/tensor.cpp b/src/Native/src/runtime/stackvm/ops/tensor.cpp index 6f09a7084c..3172fd0f90 100644 --- a/src/Native/src/runtime/stackvm/ops/tensor.cpp +++ b/src/Native/src/runtime/stackvm/ops/tensor.cpp @@ -1,5 +1,5 @@ -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:29 - * +08:00. +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:08 AM + * +00:00. * * Copyright 2019-2021 Canaan Inc. * @@ -207,9 +207,7 @@ result stackvm_runtime_function::visit( dump_op("concat"); try_var(input, pop_value()); dump_input(input); - try_var(axis, pop_value()); - dump_input(axis); - try_var(output, kernels::stackvm::concat(input, axis, nullptr, + try_var(output, kernels::stackvm::concat(op.axis, input, nullptr, module().kernel_context())); dump_output(output); stack_.push(std::move(output)); @@ -491,11 +489,9 @@ result stackvm_runtime_function::visit( dump_op("gather"); try_var(input, pop_value()); dump_input(input); - try_var(axis, pop_value()); - dump_input(axis); try_var(index, pop_value()); dump_input(index); - try_var(output, kernels::stackvm::gather(input, axis, index, nullptr, + try_var(output, kernels::stackvm::gather(op.axis, input, index, nullptr, module().kernel_context())); dump_output(output); stack_.push(std::move(output)); @@ -683,9 +679,9 @@ result stackvm_runtime_function::visit( dump_input(scale); try_var(bias, pop_value()); dump_input(bias); - try_var(output, kernels::stackvm::layer_norm(op.axis, op.epsilon, input, - scale, bias, nullptr, - module().kernel_context())); + try_var(output, kernels::stackvm::layer_norm( + op.axis, op.epsilon, op.use_mean, input, scale, bias, + nullptr, module().kernel_context())); dump_output(output); stack_.push(std::move(output)); return ok(); diff --git a/src/Native/src/runtime/stackvm/runtime_function_ops.h b/src/Native/src/runtime/stackvm/runtime_function_ops.h index ae6944ef59..351b758b88 100644 --- a/src/Native/src/runtime/stackvm/runtime_function_ops.h +++ b/src/Native/src/runtime/stackvm/runtime_function_ops.h @@ -1,5 +1,5 @@ -/* This file is generated by tools/stackvm_gen/IsaGen at 2023/9/5 19:40:29 - * +08:00. +/* This file is generated by tools/stackvm_gen/IsaGen at 9/20/2023 10:17:07 AM + * +00:00. * * Copyright 2019-2021 Canaan Inc. * diff --git a/src/Native/src/test_cli.cpp b/src/Native/src/test_cli.cpp index 7f703a216a..3e95be5a64 100644 --- a/src/Native/src/test_cli.cpp +++ b/src/Native/src/test_cli.cpp @@ -12,6 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include #include #include @@ -19,6 +20,8 @@ using namespace nncase; using namespace nncase::runtime; +// constexpr size_t loop_count = 10; +constexpr size_t loop_count = 1; #define TRY(x) \ if (x) \ @@ -34,8 +37,7 @@ result write_tensor_buffer(value_t value, std::ofstream &of) { } result run_core(const std::string &kmodel_path, - const std::vector &input_bins, - const std::string &output_bin) { + const std::vector &bins) { auto kmodel = read_file(kmodel_path); interpreter *interp = new interpreter(); // auto dump_path = @@ -47,16 +49,16 @@ result run_core(const std::string &kmodel_path, try_var(entry, interp->entry_function()); - if (entry->parameters_size() != input_bins.size()) + if (entry->parameters_size() > bins.size()) return err(std::errc::argument_list_too_long); /* create the input parameters tensor note the input tenosr must be contiguous */ std::vector parameters; - for (int i = 0; i < input_bins.size(); i++) { + for (int i = 0; i < entry->parameters_size(); i++) { try_var(type, entry->parameter_type(i)); try_var(ts_type, type.as()); - auto input_pool = read_file(input_bins[i]); + auto input_pool = read_file(bins[i]); gsl::span input_pool_span = { reinterpret_cast(input_pool.data()), input_pool.size()}; @@ -66,21 +68,40 @@ result run_core(const std::string &kmodel_path, parameters.push_back(_.impl()); } - try_var(ret, entry->invoke({parameters.data(), parameters.size()})); + double total_time = 0.0; + for (size_t i = 0; i < loop_count; i++) { + auto start_time = std::chrono::steady_clock::now(); + try_var(ret, entry->invoke({parameters.data(), parameters.size()})); + auto end_time = std::chrono::steady_clock::now(); + total_time += (std::chrono::duration_cast( + end_time - start_time) + .count() / + 1e6); - std::ofstream output_stream(output_bin, std::ios::binary); - - if (ret.is_a()) { - try_(write_tensor_buffer(ret, output_stream)); - } else if (ret.is_a()) { - try_var(tp, ret.as()); - for (auto &&ret_v : tp->fields()) { - try_(write_tensor_buffer(ret_v, output_stream)); + if (i == (loop_count - 1) && (entry->parameters_size() < bins.size())) { + if (ret.is_a()) { + auto output_bin = bins.back(); + std::ofstream output_stream(output_bin, std::ios::binary); + try_(write_tensor_buffer(ret, output_stream)); + output_stream.close(); + } else if (ret.is_a()) { + try_var(tp, ret.as()); + auto o = 0; + for (auto &&ret_v : tp->fields()) { + auto output_bin = bins[entry->parameters_size() + (o++)]; + std::ofstream output_stream(output_bin, std::ios::binary); + try_(write_tensor_buffer(ret_v, output_stream)); + output_stream.close(); + } + } else { + return nncase::err(std::errc::bad_message); + } } - } else { - return nncase::err(std::errc::bad_message); } - output_stream.close(); + + std::cout << "interp run: " << (total_time / loop_count) + << " ms, fps = " << 1000 / (total_time / loop_count) << std::endl; + return ok(); } @@ -92,13 +113,12 @@ result run_core(const std::string &kmodel_path, * @return int */ int main(NNCASE_UNUSED int argc, char **argv) { - assert(argc >= 4); - std::vector input_bins; - for (int i = 2; i < argc - 1; i++) { - input_bins.push_back(argv[i]); + assert(argc >= 3); + std::vector bins; + for (int i = 2; i < argc; i++) { + bins.push_back(argv[i]); } std::string kmodel_bin(argv[1]); - std::string output_bin(argv[argc - 1]); - run_core(kmodel_bin, input_bins, output_bin).unwrap_or_throw(); + run_core(kmodel_bin, bins).unwrap_or_throw(); return 0; -} +} \ No newline at end of file diff --git a/src/Nncase.Cli/Commands/Compile.cs b/src/Nncase.Cli/Commands/Compile.cs deleted file mode 100644 index edb64f2b3e..0000000000 --- a/src/Nncase.Cli/Commands/Compile.cs +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright (c) Canaan Inc. All rights reserved. -// Licensed under the Apache license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.CommandLine.Invocation; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Nncase.CodeGen; -using Nncase.Compiler; -using Nncase.Diagnostics; -using Nncase.IR; -using Nncase.Passes; -using Nncase.Quantization; - -namespace Nncase.Cli.Commands; - -internal enum QuantType -{ - UInt8, - Int8, - Int16, -} - -internal enum DatasetFormat -{ - Image, - Raw, - Pytest, - Random, -} - -/// -/// Compile command. -/// -public sealed class Compile : Command -{ - /// - /// Initializes a new instance of the class. - /// - public Compile() - : base("compile") - { - AddArgument(new Argument("input-file")); - AddArgument(new Argument("output-file")); - AddOption(new Option( - aliases: new string[] { "-t", "--target" }, - description: "target architecture, e.g. cpu, k210")); - AddOption(new Option( - aliases: new[] { "-i", "--input-format" }, - description: "input format, e.g. tflite", - getDefaultValue: () => "tflite")); - AddOption(new Option( - alias: "--dump-level", - description: $"dump ir to .il, default is {0}", - getDefaultValue: () => 0)); - AddOption(new Option( - alias: "--dump-dir", - description: "dump to directory, default is .", - getDefaultValue: () => ".")); - AddOption(new Option( - alias: "--quant-type", - description: $"quant type, default is {QuantType.UInt8}", - getDefaultValue: () => QuantType.UInt8)); - AddOption(new Option( - alias: "--wquant-type", - description: $"wquant type, default is {QuantType.UInt8}", - getDefaultValue: () => QuantType.UInt8)); - AddOption(new Option( - alias: "--dataset", - description: $"calibration dataset, used in post quantization, default is empty", - getDefaultValue: () => string.Empty)); - AddOption(new Option( - alias: "--dataset-format", - description: $"datset format: e.g. Image|Raw|Pytest", - getDefaultValue: () => DatasetFormat.Raw)); - AddOption(new Option( - alias: "--model-quant-mode", - description: $"model quant mode, default is {Quantization.ModelQuantMode.NoQuant}", - getDefaultValue: () => Quantization.ModelQuantMode.NoQuant)); - AddOption(new Option( - alias: "--calib-method", - description: $"model quant options, default is {Quantization.CalibMethod.Kld}", - getDefaultValue: () => Quantization.CalibMethod.Kld)); - AddOption(new Option( - alias: "--pre-process", - description: "whether enable pre process, default is False", - getDefaultValue: () => false)); - AddOption(new Option( - alias: "--input-layout", - description: "the model input data layout, default is empty. eg. NCHW/NHWC", - getDefaultValue: () => string.Empty)); - AddOption(new Option( - alias: "--output-layout", - description: "the model output data layout, default is empty. eg. NCHW/NHWC", - getDefaultValue: () => string.Empty)); - AddOption(new Option( - alias: "--input-type", - description: "the model input data value type, default is Float32", - getDefaultValue: () => InputType.Float32)); - AddOption(new Option>( - alias: "--input-shape", - description: "the model input data shape, default is []. eg. `--input-shape 1 2 3 4`", - getDefaultValue: () => Array.Empty())); - AddOption(new Option>( - alias: "--input-range", - description: "the model input data value range, default is []. eg `--input-range -100.3 200.4`", - getDefaultValue: () => Array.Empty())); - AddOption(new Option( - alias: "--swap-rb", - description: "whether swap the model input data channel R and B", - getDefaultValue: () => false)); - AddOption(new Option( - alias: "--letter-box-value", - description: "letterbox value, default 0.0", - getDefaultValue: () => 0.0f)); - AddOption(new Option>( - alias: "--mean", - description: "the model input data mean, default []", - getDefaultValue: () => Array.Empty())); - AddOption(new Option>( - alias: "--std", - description: "the model input data std, default []", - getDefaultValue: () => Array.Empty())); - AddOption(new Option( - alias: "--model-layout", - description: "the model's input layout, default is empty. eg. NCHW/NHWC", - getDefaultValue: () => string.Empty)); - AddOption(new Option( - alias: "--benchmark-only", - description: $"benchmark only", - getDefaultValue: () => false)); - - Handler = CommandHandler.Create(RunAsync); - } - - private static DumpFlags DumpLevelToFlags(int dumpLevel) - { - return dumpLevel switch - { - 0 => DumpFlags.None, - 1 => DumpLevelToFlags(0) | DumpFlags.Compile, - 2 => DumpLevelToFlags(1) | DumpFlags.PassIR, - 3 => DumpLevelToFlags(2) | DumpFlags.Rewrite, - 4 => DumpLevelToFlags(3) | DumpFlags.EGraphCost, - 5 => DumpLevelToFlags(4) | DumpFlags.Evaluator, - 6 => DumpLevelToFlags(5) | DumpFlags.Calibration, - 7 => DumpLevelToFlags(6) | DumpFlags.Tiling, - 8 => DumpLevelToFlags(7) | DumpFlags.Schedule, - >= 9 => DumpLevelToFlags(8) | DumpFlags.CodeGen, - _ => throw new ArgumentOutOfRangeException(nameof(dumpLevel)), - }; - } - - private async Task RunAsync(CliCompileOptions cliOptions, IHost host) - { - CompilerServices.Configure(host.Services); - - // 1. setup the options - var compileOptions = new CompileOptions - { - InputFile = cliOptions.InputFile, - InputFormat = cliOptions.InputFormat, - DumpFlags = DumpLevelToFlags(cliOptions.DumpLevel), - DumpDir = cliOptions.DumpDir, - QuantizeOptions = new() - { - CalibrationMethod = cliOptions.CalibMethod, - QuantType = cliOptions.QuantType switch - { - QuantType.UInt8 => DataTypes.UInt8, - QuantType.Int8 => DataTypes.Int8, - QuantType.Int16 => DataTypes.Int16, - _ => throw new ArgumentException("Invalid quant type"), - }, - WQuantType = cliOptions.WQuantType switch - { - QuantType.UInt8 => DataTypes.UInt8, - QuantType.Int8 => DataTypes.Int8, - QuantType.Int16 => DataTypes.Int16, - _ => throw new ArgumentException("Invalid weights quant type"), - }, - ModelQuantMode = cliOptions.ModelQuantMode, - }, - PreProcess = cliOptions.PreProcess, - InputLayout = cliOptions.InputLayout, - OutputLayout = cliOptions.OutputLayout, - InputType = cliOptions.InputType, - InputShape = cliOptions.InputShape.ToArray(), - InputRange = cliOptions.InputRange.ToArray(), - SwapRB = cliOptions.SwapRB, - LetterBoxValue = cliOptions.LetterBoxValue, - Mean = cliOptions.Mean.ToArray(), - Std = cliOptions.Std.ToArray(), - ModelLayout = cliOptions.ModelLayout, - IsBenchmarkOnly = cliOptions.BenchmarkOnly, - }; - - // 2. import the model - var target = CompilerServices.GetTarget(cliOptions.Target); - using var compileSession = CompileSession.Create(target, compileOptions); - var compiler = compileSession.Compiler; - var module = await compiler.ImportModuleAsync(compileOptions.InputFormat, compileOptions.InputFile, compileOptions.IsBenchmarkOnly); - - // 3. create the calib dataset - if (compileOptions.QuantizeOptions.ModelQuantMode == Quantization.ModelQuantMode.UsePTQ) - { - if (cliOptions.DatasetFormat == DatasetFormat.Random) - { - compileOptions.QuantizeOptions.CalibrationDataset = new RandomCalibrationDatasetProvider(((Function)module.Entry!).Parameters.ToArray(), 5); - } - else if (cliOptions.DatasetFormat == DatasetFormat.Pytest) - { - compileOptions.QuantizeOptions.CalibrationDataset = new PytestCalibrationDatasetProvider(((Function)module.Entry!).Parameters.ToArray(), cliOptions.Dataset); - } - else - { - throw new NotSupportedException(cliOptions.DatasetFormat.ToString()); - } - } - - // 4. compile - await compiler.CompileAsync(); - - // 5. code gen - using (var os = File.OpenWrite(cliOptions.OutputFile)) - { - compiler.Gencode(os); - } - } -} - -// Validate null in command line parser. -#pragma warning disable CS8618 - -internal sealed class CliCompileOptions -{ - public string InputFile { get; set; } - - public string InputFormat { get; set; } - - public string Target { get; set; } - - public int DumpLevel { get; set; } - - public string DumpDir { get; set; } - - public QuantType QuantType { get; set; } - - public QuantType WQuantType { get; set; } - - public string OutputFile { get; set; } - - public ModelQuantMode ModelQuantMode { get; set; } - - public CalibMethod CalibMethod { get; set; } - - public string Dataset { get; set; } - - public DatasetFormat DatasetFormat { get; set; } - - public bool BenchmarkOnly { get; set; } - - public bool PreProcess { get; set; } - - public string InputLayout { get; set; } - - public string OutputLayout { get; set; } - - public InputType InputType { get; set; } - - public List InputShape { get; set; } - - public List InputRange { get; set; } - - public bool SwapRB { get; set; } - - public float LetterBoxValue { get; set; } - - public List Mean { get; set; } - - public List Std { get; set; } - - public string ModelLayout { get; set; } -} - -#pragma warning restore CS8618 diff --git a/src/Nncase.Cli/Compile.cs b/src/Nncase.Cli/Compile.cs new file mode 100644 index 0000000000..e7f65a5303 --- /dev/null +++ b/src/Nncase.Cli/Compile.cs @@ -0,0 +1,217 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.Linq; +using Nncase.Diagnostics; +using Nncase.Quantization; + +namespace Nncase.Cli; + +internal enum QuantType +{ + UInt8, + Int8, + Int16, +} + +internal enum DatasetFormat +{ + Image, + Raw, + Pytest, + Random, +} + +/// +/// Compile command. +/// +internal sealed class CompileCommand : Command +{ + /// + /// Initializes a new instance of the class. + /// + public CompileCommand() + : base("compile") + { + InputFile = new Argument("input-file"); + OutputFile = new Argument("output-file"); + InputFormat = new Option( + aliases: new[] { "-i", "--input-format" }, + description: "input format, e.g. tflite", + getDefaultValue: () => "tflite"); + DumpFlags = new Option>( + name: "--dump-flags", + description: "dump ir flags. \navailable value: None,ImportOps,PassIR,EGraphCost,Rewrite,Calibration,Evaluator,Compile,Tiling,Schedule,CodeGen.") + { + AllowMultipleArgumentsPerToken = true, + }; + DumpDir = new Option( + name: "--dump-dir", + description: "dump to directory.", + getDefaultValue: () => "."); + QuantType = new Option( + name: "--quant-type", + description: $"quant type", + getDefaultValue: () => Nncase.Cli.QuantType.UInt8); + WQuantType = new Option( + name: "--wquant-type", + description: $"wquant type", + getDefaultValue: () => Nncase.Cli.QuantType.UInt8); + Dataset = new Option( + name: "--dataset", + description: $"calibration dataset, used in post quantization", + getDefaultValue: () => string.Empty); + DatasetFormat = new Option( + name: "--dataset-format", + description: $"datset format.", + getDefaultValue: () => Nncase.Cli.DatasetFormat.Raw); + ModelQuantMode = new Option( + name: "--model-quant-mode", + description: $"model quant mode", + getDefaultValue: () => Quantization.ModelQuantMode.NoQuant); + CalibMethod = new Option( + name: "--calib-method", + description: $"model quant options", + getDefaultValue: () => Quantization.CalibMethod.Kld); + FixedVars = new Option>( + name: "--fixed-vars", + description: $"dynamic shape fixed vars, default is empty. \nset by `n:123`", + parseArgument: result => + { + return result.Tokens. + Select(tk => tk.Value.Split(":").ToArray()). + Select(tp => (tp[0].Trim(), int.Parse(tp[1].Trim()))); + }) + { + AllowMultipleArgumentsPerToken = true, + }; + PreProcess = new Option( + name: "--pre-process", + description: "whether enable pre process", + getDefaultValue: () => false); + InputLayout = new Option( + name: "--input-layout", + description: "the model input data layout", + getDefaultValue: () => string.Empty).FromAmong("NCHW", "NHWC"); + OutputLayout = new Option( + name: "--output-layout", + description: "the model output data layout.", + getDefaultValue: () => string.Empty).FromAmong("NCHW", "NHWC"); + InputType = new Option( + name: "--input-type", + description: "the model input data value type, default is Float32", + getDefaultValue: () => Nncase.InputType.Float32); + InputShape = new Option>( + name: "--input-shape", + description: "the model input data shape. eg. `--input-shape 1 2 3 4`", + getDefaultValue: Array.Empty) + { + AllowMultipleArgumentsPerToken = true, + }; + InputRange = new Option>( + name: "--input-range", + description: "the model input data value range. eg `--input-range -100.3 200.4`", + getDefaultValue: Array.Empty) + { + AllowMultipleArgumentsPerToken = true, + }; + SwapRB = new Option( + name: "--swap-rb", + description: "whether swap the model input data channel, like cv2.BGRtoRGB(im)", + getDefaultValue: () => false); + LetterBoxValue = new Option( + name: "--letter-box-value", + description: "letterbox fill value", + getDefaultValue: () => 0.0f); + Mean = new Option>( + name: "--mean", + description: "the model input data mean, default []", + getDefaultValue: Array.Empty) + { + AllowMultipleArgumentsPerToken = true, + }; + Std = new Option>( + name: "--std", + description: "the model input data std, default []", + getDefaultValue: Array.Empty) + { + AllowMultipleArgumentsPerToken = true, + }; + ModelLayout = new Option( + name: "--model-layout", + description: "the model's input layout.", + getDefaultValue: () => string.Empty).FromAmong("NCHW", "NHWC"); + AddArgument(InputFile); + AddArgument(OutputFile); + AddGlobalOption(InputFormat); + AddGlobalOption(DumpFlags); + AddGlobalOption(DumpDir); + AddGlobalOption(QuantType); + AddGlobalOption(WQuantType); + AddGlobalOption(Dataset); + AddGlobalOption(DatasetFormat); + AddGlobalOption(ModelQuantMode); + AddGlobalOption(CalibMethod); + AddGlobalOption(FixedVars); + AddGlobalOption(PreProcess); + AddGlobalOption(InputLayout); + AddGlobalOption(OutputLayout); + AddGlobalOption(InputType); + AddGlobalOption(InputShape); + AddGlobalOption(InputRange); + AddGlobalOption(SwapRB); + AddGlobalOption(LetterBoxValue); + AddGlobalOption(Mean); + AddGlobalOption(Std); + AddGlobalOption(ModelLayout); + } + + public Argument InputFile { get; } + + public Argument OutputFile { get; } + + public Option InputFormat { get; } + + public Option> DumpFlags { get; } + + public Option DumpDir { get; } + + public Option QuantType { get; } + + public Option WQuantType { get; } + + public Option Dataset { get; } + + public Option DatasetFormat { get; } + + public Option ModelQuantMode { get; } + + public Option CalibMethod { get; } + + public Option> FixedVars { get; } + + public Option PreProcess { get; } + + public Option InputLayout { get; } + + public Option OutputLayout { get; } + + public Option InputType { get; } + + public Option> InputShape { get; } + + public Option> InputRange { get; } + + public Option SwapRB { get; } + + public Option LetterBoxValue { get; } + + public Option> Mean { get; } + + public Option> Std { get; } + + public Option ModelLayout { get; } +} diff --git a/src/Nncase.Cli/Nncase.Cli.csproj b/src/Nncase.Cli/Nncase.Cli.csproj index 3070806ba5..17e370d4ca 100644 --- a/src/Nncase.Cli/Nncase.Cli.csproj +++ b/src/Nncase.Cli/Nncase.Cli.csproj @@ -26,4 +26,8 @@ PreserveNewest + + + + diff --git a/src/Nncase.Cli/Program.CommandLine.cs b/src/Nncase.Cli/Program.CommandLine.cs deleted file mode 100644 index e88f691647..0000000000 --- a/src/Nncase.Cli/Program.CommandLine.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Canaan Inc. All rights reserved. -// Licensed under the Apache license. See LICENSE file in the project root for full license information. - -using System; -using System.CommandLine; -using System.CommandLine.Builder; -using System.Linq; - -namespace Nncase.Cli; - -internal partial class Program -{ - private static CommandLineBuilder BuildCommandLine() - { - var commands = from t in typeof(Program).Assembly.ExportedTypes - where t.Namespace == "Nncase.Cli.Commands" && t.IsAssignableTo(typeof(Command)) - select (Command)Activator.CreateInstance(t)!; - var root = new RootCommand(); - foreach (var command in commands) - { - root.AddCommand(command); - } - - return new CommandLineBuilder(root); - } -} diff --git a/src/Nncase.Cli/Program.cs b/src/Nncase.Cli/Program.cs index 2d3e40c803..0ef25c0565 100644 --- a/src/Nncase.Cli/Program.cs +++ b/src/Nncase.Cli/Program.cs @@ -1,10 +1,14 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. +using System; +using System.Collections.Generic; +using System.CommandLine; using System.CommandLine.Builder; using System.CommandLine.Hosting; using System.CommandLine.Parsing; using System.IO; +using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; @@ -16,12 +20,155 @@ internal partial class Program { public static async Task Main(string[] args) { - return await BuildCommandLine() + return await ConfigureCommandLine() .UseHost(ConfigureHost) .UseDefaults() .Build().InvokeAsync(args); } + private static async Task RunAsync(string targetKind, CompileOptions compileOptions, DatasetFormat datasetFormat, string dataset, string outputFile, IHost host) + { + CompilerServices.Configure(host.Services); + + // 2. import the model + var target = CompilerServices.GetTarget(targetKind); + using var compileSession = CompileSession.Create(target, compileOptions); + var compiler = compileSession.Compiler; + IR.IRModule module = await compiler.ImportModuleAsync(Path.GetExtension(compileOptions.InputFile).Trim('.'), compileOptions.InputFile); + + // 3. create the calib dataset + if (compileOptions.QuantizeOptions.ModelQuantMode == Quantization.ModelQuantMode.UsePTQ) + { + if (datasetFormat == DatasetFormat.Random) + { + compileOptions.QuantizeOptions.CalibrationDataset = new Quantization.RandomCalibrationDatasetProvider(((Nncase.IR.Function)module.Entry!).Parameters.ToArray(), 5); + } + else if (datasetFormat == DatasetFormat.Pytest) + { + compileOptions.QuantizeOptions.CalibrationDataset = new Quantization.PytestCalibrationDatasetProvider(((IR.Function)module.Entry!).Parameters.ToArray(), dataset); + } + else + { + throw new NotSupportedException(datasetFormat.ToString()); + } + } + + // 4. compile + await compiler.CompileAsync(); + + // 5. code gen + using (var os = File.OpenWrite(outputFile)) + { + compiler.Gencode(os); + } + } + + private static CommandLineBuilder ConfigureCommandLine() + { + var compile = new CompileCommand(); + foreach (var target in LoadTargets()) + { + var (targetCmd, targetParser) = target.RegisterCommandAndParser(); + Action targetHandler = async (System.CommandLine.Invocation.InvocationContext context) => + { + var options = ParseCompileOptions(context, compile); + options.TargetCompileOptions = targetParser(context, targetCmd); + await RunAsync(targetCmd.Name, options, context.ParseResult.GetValueForOption(compile.DatasetFormat), context.ParseResult.GetValueForOption(compile.Dataset)!, context.ParseResult.GetValueForArgument(compile.OutputFile), context.GetHost()); + }; + targetCmd.SetHandler(targetHandler); + compile.AddCommand(targetCmd); + } + + return new CommandLineBuilder(new RootCommand() { compile }); + } + + private static CompileOptions ParseCompileOptions(System.CommandLine.Invocation.InvocationContext context, CompileCommand compilecmd) + { + // 1. setup the options + var compileOptions = new CompileOptions + { + InputFile = context.ParseResult.GetValueForArgument(compilecmd.InputFile), + InputFormat = context.ParseResult.GetValueForOption(compilecmd.InputFormat)!, + DumpFlags = context.ParseResult.GetValueForOption(compilecmd.DumpFlags)!.Aggregate(Diagnostics.DumpFlags.None, (a, b) => a | b), + DumpDir = context.ParseResult.GetValueForOption(compilecmd.DumpDir)!, + PreProcess = context.ParseResult.GetValueForOption(compilecmd.PreProcess)!, + InputLayout = context.ParseResult.GetValueForOption(compilecmd.InputLayout)!, + OutputLayout = context.ParseResult.GetValueForOption(compilecmd.OutputLayout)!, + InputType = context.ParseResult.GetValueForOption(compilecmd.InputType)!, + InputShape = context.ParseResult.GetValueForOption(compilecmd.InputShape)!.ToArray(), + InputRange = context.ParseResult.GetValueForOption(compilecmd.InputRange)!.ToArray(), + SwapRB = context.ParseResult.GetValueForOption(compilecmd.SwapRB)!, + LetterBoxValue = context.ParseResult.GetValueForOption(compilecmd.LetterBoxValue)!, + Mean = context.ParseResult.GetValueForOption(compilecmd.Mean)!.ToArray(), + Std = context.ParseResult.GetValueForOption(compilecmd.Std)!.ToArray(), + ModelLayout = context.ParseResult.GetValueForOption(compilecmd.ModelLayout)!, + QuantizeOptions = new() + { + CalibrationMethod = context.ParseResult.GetValueForOption(compilecmd.CalibMethod), + QuantType = context.ParseResult.GetValueForOption(compilecmd.QuantType) switch + { + QuantType.UInt8 => DataTypes.UInt8, + QuantType.Int8 => DataTypes.Int8, + QuantType.Int16 => DataTypes.Int16, + _ => throw new ArgumentException("Invalid quant type"), + }, + WQuantType = context.ParseResult.GetValueForOption(compilecmd.WQuantType) switch + { + QuantType.UInt8 => DataTypes.UInt8, + QuantType.Int8 => DataTypes.Int8, + QuantType.Int16 => DataTypes.Int16, + _ => throw new ArgumentException("Invalid weights quant type"), + }, + ModelQuantMode = context.ParseResult.GetValueForOption(compilecmd.ModelQuantMode), + }, + }; + + foreach (var item in context.ParseResult.GetValueForOption(compilecmd.FixedVars)!) + { + compileOptions.ShapeBucketOptions.FixVarMap.Add(item.Name, item.Value); + } + + return compileOptions; + } + + private static IReadOnlyList LoadTargets() + { + var loadContext = System.Runtime.Loader.AssemblyLoadContext.Default; + var pluginAsms = PluginLoader.GetPluginsSearchDirectories(PluginLoader.PluginPathEnvName, null). + Select(PluginLoader.GetPluginAssemblies). + SelectMany(x => x). + DistinctBy(Path.GetFileName). + Select(x => PluginLoader.LoadPluginAssembly(x, loadContext)). + Distinct(). + ToList(); + pluginAsms.AddRange(new[] { Path.GetDirectoryName(typeof(Program).Assembly.Location)! }. + Select(basePath => + { + if (Directory.Exists(basePath)) + { + return (from filePath in Directory.GetFiles(basePath, PluginLoader.ModulesDllPattern, SearchOption.AllDirectories) + where PluginLoader.IsLoadableAssembly(filePath) + select filePath).Distinct(); + } + else + { + return Array.Empty(); + } + }). + SelectMany(x => x). + DistinctBy(Path.GetFileName). + Select(x => PluginLoader.LoadPluginAssembly(x, loadContext)). + Distinct()); + var targets = (from asm in pluginAsms + from t in asm.ExportedTypes + where t.IsClass + && t.IsAssignableTo(typeof(ITarget)) + let ctor = t.GetConstructor(Type.EmptyTypes) + where ctor != null + select (ITarget)ctor.Invoke(null)).ToList(); + return targets; + } + private static void ConfigureHost(IHostBuilder hostBuilder) { hostBuilder.ConfigureAppConfiguration(ConfigureAppConfiguration) diff --git a/src/Nncase.Cli/packages.lock.json b/src/Nncase.Cli/packages.lock.json index b438ba9cc6..56eaafb112 100644 --- a/src/Nncase.Cli/packages.lock.json +++ b/src/Nncase.Cli/packages.lock.json @@ -33,21 +33,22 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "System.CommandLine.Hosting": { "type": "Direct", - "requested": "[0.3.0-alpha.21216.1, )", - "resolved": "0.3.0-alpha.21216.1", - "contentHash": "zP8QEUH8dSUYUHdGk6k71kOJy8uFgEPZG2RfhA0cMjDH3/Jov5AjUNaxOvpSNHh+ewu8eIUCYgV8+fEkCPyNlw==", + "requested": "[0.4.0-alpha.22272.1, )", + "resolved": "0.4.0-alpha.22272.1", + "contentHash": "x9JhHxBLxlKyCIZADFYC8q16L9yGHdTakrLFjHabwR7Tk0761aTexiGgMTIS744HGuhc8pk9MoLUzsr/TlRfMQ==", "dependencies": { - "Microsoft.Extensions.Hosting": "3.1.5", - "System.CommandLine": "2.0.0-beta1.21216.1" + "Microsoft.Extensions.Hosting": "6.0.0", + "System.CommandLine": "2.0.0-beta4.22272.1", + "System.CommandLine.NamingConventionBinder": "2.0.0-beta4.22272.1" } }, "Google.OrTools.runtime.linux-arm64": { @@ -344,8 +345,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -362,13 +363,12 @@ "System.Runtime": "4.3.0" } }, - "System.CommandLine": { + "System.CommandLine.NamingConventionBinder": { "type": "Transitive", - "resolved": "2.0.0-beta1.21216.1", - "contentHash": "Nbv/tW8sbOKN5T+4SSVBMdk4ADSIpJpY4UHMsj3VkcNtOckIT4iyzagjF+W5FEh2YBRvmvVQijOTIZbUJ1+1aA==", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "ux2eUA/syF+JtlpMDc/Lsd6PBIBuwjH3AvHnestoh5uD0WKT5b+wkQxDWVCqp9qgVjMBTLNhX19ZYFtenunt9A==", "dependencies": { - "Microsoft.CSharp": "4.4.1", - "system.memory": "4.5.4" + "System.CommandLine": "2.0.0-beta4.22272.1" } }, "System.Diagnostics.Contracts": { @@ -696,6 +696,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -937,6 +938,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Linq.Async": { "type": "CentralTransitive", "requested": "[6.0.1, )", diff --git a/src/Nncase.CodeGen/CodeGen/LinkedFunction.cs b/src/Nncase.CodeGen/CodeGen/LinkedFunction.cs index b3dbe692e0..bd4d97cafc 100644 --- a/src/Nncase.CodeGen/CodeGen/LinkedFunction.cs +++ b/src/Nncase.CodeGen/CodeGen/LinkedFunction.cs @@ -15,7 +15,6 @@ public class LinkedFunction : ILinkedFunction public LinkedFunction(uint id, Callable sourceFunction, ulong textBegin, ulong textLength, IReadOnlyList sections) { Id = id; - CompilerServices.InferenceType(sourceFunction); ParameterTypes = ((CallableType)sourceFunction.CheckedType).Parameters.ToArray(); ReturnType = ((CallableType)sourceFunction.CheckedType).ReturnType; TextBegin = textBegin; diff --git a/src/Nncase.CodeGen/packages.lock.json b/src/Nncase.CodeGen/packages.lock.json index b618b5504c..fd39ebd2fc 100644 --- a/src/Nncase.CodeGen/packages.lock.json +++ b/src/Nncase.CodeGen/packages.lock.json @@ -10,11 +10,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.Extensions.Configuration.Abstractions": { @@ -53,8 +53,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -76,6 +76,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -138,6 +139,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Compiler/Compiler.cs b/src/Nncase.Compiler/Compiler.cs index 9895f6e0b5..afd0607878 100644 --- a/src/Nncase.Compiler/Compiler.cs +++ b/src/Nncase.Compiler/Compiler.cs @@ -88,13 +88,15 @@ public void AddPreAndPostProcess(IPassManager passManager) public void TargetIndependentPass(IPassManager passManager) { - passManager.AddWithName("ReshapeMatMul").Configure(p => + passManager.AddWithName("NormAxisAndShape").Configure(p => { p.Add(); - }); - - passManager.AddWithName("SqueezeShape").Configure(p => - { + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); @@ -102,6 +104,7 @@ public void TargetIndependentPass(IPassManager passManager) p.Add(); p.Add(); p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); @@ -124,6 +127,7 @@ public void TargetIndependentPass(IPassManager passManager) p.Add(); p.Add(); p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); @@ -157,6 +161,8 @@ public void TargetIndependentPass(IPassManager passManager) p.Add(); p.Add(); p.Add(); + p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); @@ -168,6 +174,7 @@ public void TargetIndependentPass(IPassManager passManager) p.Add(); p.Add(); p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); diff --git a/src/Nncase.Compiler/Hosting/PluginLoader.cs b/src/Nncase.Compiler/Hosting/PluginLoader.cs index 73f10f367a..014ab29fb1 100644 --- a/src/Nncase.Compiler/Hosting/PluginLoader.cs +++ b/src/Nncase.Compiler/Hosting/PluginLoader.cs @@ -19,12 +19,14 @@ namespace Nncase.Hosting; /// public sealed class PluginLoader { - private const string _modulesDllPattern = "Nncase.Modules.*.dll"; - private const string _pluginPathEnvName = "NNCASE_PLUGIN_PATH"; + public const string PluginPathEnvName = "NNCASE_PLUGIN_PATH"; + + public const string ModulesDllPattern = "Nncase.Modules.*.dll"; private static readonly string[] _builtinModules = new[] { "Nncase.Modules.StackVM.dll", + "Nncase.Modules.CPU.dll", "Nncase.Modules.K210.dll", }; @@ -42,67 +44,16 @@ public PluginLoader(ILogger logger) ?? AssemblyLoadContext.Default; } - /// - /// Load plugins. - /// - /// Plugins. - public IReadOnlyList LoadPlugins() - { - var pluginAsms = GetPluginsSearchDirectories().Select(GetPluginAssemblies).SelectMany(x => x) - .DistinctBy(Path.GetFileName).Select(LoadPluginAssembly).Distinct().ToList(); - var plugins = (from asm in pluginAsms - from t in asm.ExportedTypes - where t.IsClass - && t.IsAssignableTo(typeof(IPlugin)) - let ctor = t.GetConstructor(Type.EmptyTypes) - where ctor != null - select (IPlugin)ctor.Invoke(null)).ToList(); - - return plugins; - } - - private static bool IsLoadableAssembly(string filePath) - { - using var fs = File.OpenRead(filePath); - using var peReader = new PEReader(fs); - - if (!peReader.HasMetadata) - { - return false; - } - - var metaReader = peReader.GetMetadataReader(); - if (!metaReader.IsAssembly) - { - return false; - } - - // Is reference assembly - if ((from cah in metaReader.CustomAttributes - let ca = metaReader.GetCustomAttribute(cah) - where ca.Constructor.Kind == HandleKind.MemberReference - let ctor = metaReader.GetMemberReference((MemberReferenceHandle)ca.Constructor) - let attrType = metaReader.GetTypeReference((TypeReferenceHandle)ctor.Parent) - where metaReader.GetString(attrType.Namespace) == nameof(System.Runtime.CompilerServices) - && metaReader.GetString(attrType.Name) == nameof(ReferenceAssemblyAttribute) - select cah).Any()) - { - return false; - } - - return true; - } - - private Assembly LoadPluginAssembly(string assemblyFile) + public static Assembly LoadPluginAssembly(string assemblyFile, AssemblyLoadContext loadContext) { - return _loadContext.LoadFromAssemblyPath(assemblyFile); + return loadContext.LoadFromAssemblyPath(assemblyFile); } - private IEnumerable GetPluginAssemblies(string basePath) + public static IEnumerable GetPluginAssemblies(string basePath) { if (Directory.Exists(basePath)) { - return (from filePath in Directory.GetFiles(basePath, _modulesDllPattern, SearchOption.AllDirectories) + return (from filePath in Directory.GetFiles(basePath, ModulesDllPattern, SearchOption.AllDirectories) where !_builtinModules.Contains(Path.GetFileName(filePath)) && IsLoadableAssembly(filePath) select filePath).Distinct(); @@ -113,19 +64,22 @@ private IEnumerable GetPluginAssemblies(string basePath) } } - private IEnumerable GetPluginsSearchDirectories() + public static IEnumerable GetPluginsSearchDirectories(string pluginPathEnvName, ILogger? logger) { var directories = new List(); // 1. Environment variable - var targetPathEnv = Environment.GetEnvironmentVariable(_pluginPathEnvName); + var targetPathEnv = Environment.GetEnvironmentVariable(pluginPathEnvName); if (string.IsNullOrWhiteSpace(targetPathEnv)) { - _logger.LogWarning($"{_pluginPathEnvName} is not set."); + if (logger is not null) + { + logger.LogWarning($"{pluginPathEnvName} is not set."); + } } else { - var targetPaths = from path in targetPathEnv.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) + var targetPaths = from path in targetPathEnv!.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) select Environment.ExpandEnvironmentVariables(path); directories.AddRange(targetPaths); } @@ -135,11 +89,62 @@ private IEnumerable GetPluginsSearchDirectories() var modulesPath = Path.Combine(rootPath, "modules"); directories.Add(modulesPath); - if (_logger.IsEnabled(LogLevel.Trace)) + if (logger is not null && logger.IsEnabled(LogLevel.Trace)) { - _logger.LogInformation($"Loading plugins from {string.Join(", ", directories)}."); + logger.LogInformation($"Loading plugins from {string.Join(", ", directories)}."); } return directories.Distinct(); } + + public static bool IsLoadableAssembly(string filePath) + { + using var fs = File.OpenRead(filePath); + using var peReader = new PEReader(fs); + + if (!peReader.HasMetadata) + { + return false; + } + + var metaReader = peReader.GetMetadataReader(); + if (!metaReader.IsAssembly) + { + return false; + } + + // Is reference assembly + if ((from cah in metaReader.CustomAttributes + let ca = metaReader.GetCustomAttribute(cah) + where ca.Constructor.Kind == HandleKind.MemberReference + let ctor = metaReader.GetMemberReference((MemberReferenceHandle)ca.Constructor) + let attrType = metaReader.GetTypeReference((TypeReferenceHandle)ctor.Parent) + where metaReader.GetString(attrType.Namespace) == nameof(System.Runtime.CompilerServices) + && metaReader.GetString(attrType.Name) == nameof(ReferenceAssemblyAttribute) + select cah).Any()) + { + return false; + } + + return true; + } + + /// + /// Load plugins. + /// + /// Plugins. + public IReadOnlyList LoadPlugins() + { + var pluginAsms = GetPluginsSearchDirectories(PluginPathEnvName, _logger).Select(GetPluginAssemblies).SelectMany(x => x) + .DistinctBy(Path.GetFileName).Select(x => LoadPluginAssembly(x, _loadContext)).Distinct().ToList(); + var plugins = (from asm in pluginAsms + from t in asm.ExportedTypes + where t.IsClass + && t.IsAssignableTo(typeof(IPlugin)) + let ctor = t.GetConstructor(Type.EmptyTypes) + where ctor != null + select (IPlugin)ctor.Invoke(null)).ToList(); + + return plugins; + } } diff --git a/src/Nncase.Compiler/packages.lock.json b/src/Nncase.Compiler/packages.lock.json index 639bb6a9bc..f22a606140 100644 --- a/src/Nncase.Compiler/packages.lock.json +++ b/src/Nncase.Compiler/packages.lock.json @@ -49,11 +49,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Google.OrTools.runtime.linux-arm64": { @@ -350,8 +350,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -674,6 +674,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -885,6 +886,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Linq.Async": { "type": "CentralTransitive", "requested": "[6.0.1, )", diff --git a/src/Nncase.Core/CompileOptions.cs b/src/Nncase.Core/CompileOptions.cs index 5d7b3cb058..e5d1b2874c 100644 --- a/src/Nncase.Core/CompileOptions.cs +++ b/src/Nncase.Core/CompileOptions.cs @@ -119,4 +119,9 @@ public sealed record CompileOptions /// Gets or sets a value indicating whether is benchmark only. /// public bool IsBenchmarkOnly { get; set; } + + /// + /// Gets or sets the target compile options. + /// + public ITargetCompileOptions TargetCompileOptions { get; set; } = null!; } diff --git a/src/Nncase.Core/CompilerServices.cs b/src/Nncase.Core/CompilerServices.cs index 5d1bd6cb83..9c666bcd21 100644 --- a/src/Nncase.Core/CompilerServices.cs +++ b/src/Nncase.Core/CompilerServices.cs @@ -73,6 +73,14 @@ public interface ICompilerServicesProvider /// false for save const into bin. public void DumpCSharpIR(Expr expr, string prefix, string dumpDir, bool randConst); + /// + /// dump the expr as csharp code. + /// + /// expression. + /// file prefix. + /// file dump ir. + public void DumpPatternIR(Expr expr, string prefix, string dumpDir); + /// /// print ir type. /// @@ -468,6 +476,15 @@ public static void DumpDotIR(Expr expr, string prefix, string dumpPath, bool dis public static void DumpCSharpIR(Expr expr, string prefix, string dumpDir, bool randConst = true) => Provider.DumpCSharpIR(expr, prefix, dumpDir, randConst); + /// + /// dump the expr as csharp code. + /// + /// expression. + /// file prefix. + /// file dump ir. + public static void DumpPatternIR(Expr expr, string prefix, string dumpDir) => + Provider.DumpPatternIR(expr, prefix, dumpDir); + public static string Print(IRType type) => Provider.Print(type); public static string Print(Expr expr, bool useScript = false) => Provider.Print(expr, useScript); @@ -583,6 +600,10 @@ public void DumpDotIR(Expr expr, string prefix, string dumpPath, bool display_ca public void DumpCSharpIR(Expr expr, string prefix, string dumpDir, bool randConst) => _irprinterProvider.DumpCSharpIR(expr, prefix, dumpDir, randConst); + /// + public void DumpPatternIR(Expr expr, string prefix, string dumpDir) => + _irprinterProvider.DumpPatternIR(expr, prefix, dumpDir); + /// public string Print(IRType type) => _irprinterProvider.Print(type); diff --git a/src/Nncase.Core/Converters/ConvertersModule.cs b/src/Nncase.Core/Converters/ConvertersModule.cs index c7e5d4a9bc..3b406a8c88 100644 --- a/src/Nncase.Core/Converters/ConvertersModule.cs +++ b/src/Nncase.Core/Converters/ConvertersModule.cs @@ -28,5 +28,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/src/Nncase.Core/Converters/PointerConverters.cs b/src/Nncase.Core/Converters/PointerConverters.cs index af274fab5e..134c773609 100644 --- a/src/Nncase.Core/Converters/PointerConverters.cs +++ b/src/Nncase.Core/Converters/PointerConverters.cs @@ -30,3 +30,25 @@ public void ConvertTo(ReadOnlySpan> source, Span dest, Cast } } } + +internal class PointerIntConverters : IPointerSpanConverter +{ + public void ConvertTo(ReadOnlySpan> source, Span dest, CastMode castMode) + where T : unmanaged, IEquatable + { + if (castMode != CastMode.KDefault) + { + throw new InvalidCastException(); + } + + if (dest.Length < source.Length) + { + throw new ArgumentException("Dest buffer is not sufficient."); + } + + for (int i = 0; i < source.Length; i++) + { + dest[i] = checked((int)source[i].Value); + } + } +} diff --git a/src/Nncase.Core/CostModel/Cost.cs b/src/Nncase.Core/CostModel/Cost.cs index b989507457..e60414a613 100644 --- a/src/Nncase.Core/CostModel/Cost.cs +++ b/src/Nncase.Core/CostModel/Cost.cs @@ -204,6 +204,7 @@ public static UInt128 GetMemoryAccess(IRType type) { TensorType t => (UInt128)(t.Shape.Aggregate(1D, (acc, x) => acc * (x.IsFixed ? x.FixedValue : 1)) * t.DType.SizeInBytes), TupleType t => t.Fields.Sum(GetMemoryAccess), + DistributedType t => GetMemoryAccess(Utilities.DistributedUtility.GetDividedTensorType(t)), _ => 0, }; } @@ -229,6 +230,7 @@ public static UInt128 GetCPUCycles(IRType type, double cyclesPerElement = 1) { TensorType t => (UInt128)(t.Shape.Aggregate(1D, (acc, x) => acc * (x.IsFixed ? x.FixedValue : 1)) * cyclesPerElement), TupleType t => t.Fields.Sum(GetMemoryAccess), + DistributedType t => GetCPUCycles(Utilities.DistributedUtility.GetDividedTensorType(t)), _ => 0, }; } @@ -328,7 +330,7 @@ public static Cost GetActivationCost(TensorType ret, uint macPerElement) } // cost for op similar to broadcast - public static Cost GetBroadcastCost(TensorType input, TensorType ret) + public static Cost GetBroadcastCost(IRType input, IRType ret) { return new() { diff --git a/src/Nncase.Core/DataTypes.cs b/src/Nncase.Core/DataTypes.cs index d1a24255e7..0b59a5696e 100644 --- a/src/Nncase.Core/DataTypes.cs +++ b/src/Nncase.Core/DataTypes.cs @@ -114,7 +114,7 @@ public static bool IsPointer(this DataType srcType) => /// datatype name. public static string GetDisplayName(this DataType dataType) => dataType switch { - PointerType pointerType => $"({GetDisplayName(pointerType.ElemType)}*)", + PointerType pointerType => $"({GetDisplayName(pointerType.ElemType)} *)", PrimType primType => primType.ShortName, ValueType => dataType.ToString(), _ => throw new ArgumentOutOfRangeException(dataType.GetType().Name), diff --git a/src/Nncase.Core/Diagnostics/IDumpper.cs b/src/Nncase.Core/Diagnostics/IDumpper.cs index 0e07109232..cc84bf46cb 100644 --- a/src/Nncase.Core/Diagnostics/IDumpper.cs +++ b/src/Nncase.Core/Diagnostics/IDumpper.cs @@ -42,6 +42,8 @@ public interface IDumpper void DumpCSharpIR(Expr expr, string prefix, string? reletivePath = null); + void DumpPatternIR(Expr expr, string prefix, string? reletivePath = null); + void DumpModule(IRModule module, string? reletivePath = null); Stream OpenFile(string reletivePath, FileMode fileMode = FileMode.Create); diff --git a/src/Nncase.Core/Diagnostics/NullDumpper.cs b/src/Nncase.Core/Diagnostics/NullDumpper.cs index 7212fc7686..3120fa25e5 100644 --- a/src/Nncase.Core/Diagnostics/NullDumpper.cs +++ b/src/Nncase.Core/Diagnostics/NullDumpper.cs @@ -46,6 +46,11 @@ public void DumpCSharpIR(Expr expr, string prefix, string? reletivePath = null) { } + /// + public void DumpPatternIR(Expr expr, string prefix, string? reletivePath = null) + { + } + /// public bool IsEnabled(DumpFlags dumpFlags) => false; diff --git a/src/Nncase.Core/DistributedType.cs b/src/Nncase.Core/DistributedType.cs new file mode 100644 index 0000000000..efe52395da --- /dev/null +++ b/src/Nncase.Core/DistributedType.cs @@ -0,0 +1,68 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; + +namespace Nncase.IR; + +public abstract record SBP +{ + public static SBPPartialSum P => SBPPartialSum.Instance; + + public static SBPBroadCast B => SBPBroadCast.Instance; + + public static SBPSplit S(int axis) => new SBPSplit(axis); +} + +public sealed record SBPSplit(int Axis) : SBP +{ + public override string ToString() => $"S({Axis})"; +} + +public sealed record SBPPartialSum : SBP +{ + public static readonly SBPPartialSum Instance = new SBPPartialSum(); + + private SBPPartialSum() + { + } + + public override string ToString() => "P"; +} + +public sealed record SBPBroadCast : SBP +{ + public static readonly SBPBroadCast Instance = new SBPBroadCast(); + + private SBPBroadCast() + { + } + + public override string ToString() => "B"; +} + +// public sealed record Placement(Placement.DeviceKind Kind, IRArray Hierarchy, string Name) +public sealed record Placement(IRArray Hierarchy, string Name) +{ + // public enum DeviceKind : uint + // { + // CPU = 0, + // } + public int Rank => Hierarchy.Count; + + // public override string ToString() => $"@{Kind} [{string.Join(',', Hierarchy.Zip(Name).Select(t => t.First.ToString() + '@' + t.Second.ToString()))}]"; + public override string ToString() => $"@ [{string.Join(',', Hierarchy.Zip(Name).Select(t => t.First.ToString() + '@' + t.Second.ToString()))}]"; +} + +public sealed record DistributedType(TensorType TensorType, IRArray NdSBP, Placement Placement) : IRType +{ + public override string ToString() => $"{TensorType}, ({string.Join(',', NdSBP)}), {Placement}"; +} diff --git a/src/Nncase.Core/FunctionCollector.cs b/src/Nncase.Core/FunctionCollector.cs index 9ed2a8bca7..12655d25a1 100644 --- a/src/Nncase.Core/FunctionCollector.cs +++ b/src/Nncase.Core/FunctionCollector.cs @@ -17,7 +17,7 @@ public FunctionCollector() public HashSet Functions => _functions; - protected override int VisitLeafFunction(Function expr, Unit context) + protected override int VisitLeafFunction(Function expr) { _functions.Add(expr); return 0; diff --git a/src/Nncase.Core/IR/Buffers/BufferLoad.cs b/src/Nncase.Core/IR/Buffers/BufferLoad.cs new file mode 100644 index 0000000000..dbf3427b6e --- /dev/null +++ b/src/Nncase.Core/IR/Buffers/BufferLoad.cs @@ -0,0 +1,28 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR.Tensors; +using Nncase.PatternMatch; +using static Nncase.IR.TypePatternUtility; + +namespace Nncase.IR.Buffers; + +/// +/// BufferLoad expression. +/// +[PatternFunctionalGenerator] +public sealed partial class BufferLoad : Op +{ + /// + /// Get the input parameter. + /// + public static readonly ParameterInfo Input = new(typeof(BufferLoad), 0, "input", IsTensor()); + + /// + /// Get the indices. + /// + public static readonly ParameterInfo Indices = new(typeof(BufferLoad), 1, "indices", IsTuple()); + + /// + public override bool CanFoldConstCall => false; +} diff --git a/src/Nncase.Core/IR/Buffers/BufferOf.cs b/src/Nncase.Core/IR/Buffers/BufferOf.cs index a3bb033275..47a2541c1b 100644 --- a/src/Nncase.Core/IR/Buffers/BufferOf.cs +++ b/src/Nncase.Core/IR/Buffers/BufferOf.cs @@ -16,7 +16,7 @@ public sealed partial class BufferOf : Op /// public static readonly ParameterInfo Input = new(typeof(BufferOf), 0, "input", IsTensor()); - public Schedule.MemoryLocation MemoryLocation { get; } + public TIR.MemoryLocation MemoryLocation { get; } /// public override string DisplayProperty() => $"Schedule.MemoryLocation.{MemoryLocation}"; diff --git a/src/Nncase.Core/IR/Buffers/BufferStore.cs b/src/Nncase.Core/IR/Buffers/BufferStore.cs new file mode 100644 index 0000000000..2d8e86cad8 --- /dev/null +++ b/src/Nncase.Core/IR/Buffers/BufferStore.cs @@ -0,0 +1,33 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR.Tensors; +using Nncase.PatternMatch; +using static Nncase.IR.TypePatternUtility; + +namespace Nncase.IR.Buffers; + +/// +/// BufferStore op. +/// +[PatternFunctionalGenerator] +public sealed partial class BufferStore : Op +{ + /// + /// Get the input parameter. + /// + public static readonly ParameterInfo Input = new(typeof(BufferStore), 0, "input", IsTensor()); + + /// + /// Get the indices parameter. + /// + public static readonly ParameterInfo Indices = new(typeof(BufferStore), 1, "indices", IsTuple()); + + /// + /// Get the value parameter. + /// + public static readonly ParameterInfo Value = new(typeof(BufferStore), 2, "value", IsScalar()); + + /// + public override bool CanFoldConstCall => false; +} diff --git a/src/Nncase.Core/IR/Buffers/DDrOf.cs b/src/Nncase.Core/IR/Buffers/DDrOf.cs index 8657d3f417..116e019d5f 100644 --- a/src/Nncase.Core/IR/Buffers/DDrOf.cs +++ b/src/Nncase.Core/IR/Buffers/DDrOf.cs @@ -17,4 +17,7 @@ public sealed partial class DDrOf : Op /// Get the input parameter. /// public static readonly ParameterInfo Input = new(typeof(DDrOf), 0, "input", IsTensor()); + + /// + public override bool CanFoldConstCall => false; } diff --git a/src/Nncase.Core/IR/Buffers/Functional.cs b/src/Nncase.Core/IR/Buffers/Functional.cs index 54cd9f59cf..a2e3507a5f 100644 --- a/src/Nncase.Core/IR/Buffers/Functional.cs +++ b/src/Nncase.Core/IR/Buffers/Functional.cs @@ -41,5 +41,5 @@ public static Call BaseMentOf(Expr input) => /// /// create the uninitialized buffer. /// - public static Call Uninitialized(DataType dataType, Schedule.MemoryLocation memoryLocation, Expr shape) => new Call(new Uninitialized(dataType, memoryLocation), shape); + public static Call Uninitialized(DataType dataType, TIR.MemoryLocation memoryLocation, Expr shape) => new Call(new Uninitialized(dataType, memoryLocation), shape); } diff --git a/src/Nncase.Core/IR/Buffers/MatchBuffer.cs b/src/Nncase.Core/IR/Buffers/MatchBuffer.cs new file mode 100644 index 0000000000..3cafa7f595 --- /dev/null +++ b/src/Nncase.Core/IR/Buffers/MatchBuffer.cs @@ -0,0 +1,21 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR.Tensors; +using Nncase.PatternMatch; +using static Nncase.IR.TypePatternUtility; + +namespace Nncase.IR.Buffers; + +/// +/// MatchBuffer op. +/// todo maybe need united matchbuffer and allocatebuffer. +/// +[PatternFunctionalGenerator] +public sealed partial class MatchBuffer : Op +{ + public static readonly ParameterInfo Input = new(typeof(MatchBuffer), 0, "input", IsTensor()); + + /// + public override bool CanFoldConstCall => false; +} diff --git a/src/Nncase.Core/IR/Buffers/Uninitialized.cs b/src/Nncase.Core/IR/Buffers/Uninitialized.cs index 2f529638b9..42564bbc4c 100644 --- a/src/Nncase.Core/IR/Buffers/Uninitialized.cs +++ b/src/Nncase.Core/IR/Buffers/Uninitialized.cs @@ -19,11 +19,11 @@ public sealed partial class Uninitialized : Op public DataType DType { get; } - public Schedule.MemoryLocation MemoryLocation { get; } + public TIR.MemoryLocation MemoryLocation { get; } /// public override bool CanFoldConstCall => false; /// - public override string DisplayProperty() => $"{DType.GetCSharpName()}, Schedule.MemoryLocation.{MemoryLocation}"; + public override string DisplayProperty() => $"{DType.GetCSharpName()}, MemoryLocation.{MemoryLocation}"; } diff --git a/src/Nncase.Core/IR/Callable.cs b/src/Nncase.Core/IR/Callable.cs index edd6004539..16fc9ad5a7 100644 --- a/src/Nncase.Core/IR/Callable.cs +++ b/src/Nncase.Core/IR/Callable.cs @@ -17,7 +17,7 @@ public abstract class Callable : Expr /// /// StackVM module kind. /// - public static readonly string StackVMModuleKind = "stackvm"; + public const string StackVMModuleKind = "stackvm"; public Callable(string name, string moduleKind, Expr[] operands) : base(operands) diff --git a/src/Nncase.Core/IR/ExprCloner.g.cs b/src/Nncase.Core/IR/ExprCloner.g.cs index 214605c69e..855ff4e22e 100644 --- a/src/Nncase.Core/IR/ExprCloner.g.cs +++ b/src/Nncase.Core/IR/ExprCloner.g.cs @@ -1,4 +1,3 @@ - //--------------------------------------------------------------------------------------------------- // // This code was generated by T4 template. @@ -57,8 +56,7 @@ protected override Expr VisitLeafIf(If expr, TContext context) return expr.With( condition: Clone(expr.Condition, context), then: Clone(expr.Then, context), - @else: Clone(expr.Else, context), - paramList: expr.ParamList.Select(p => Clone(p, context)).ToArray() + @else: Clone(expr.Else, context) ); } @@ -141,31 +139,6 @@ protected override Expr VisitLeafBlock(TIR.Block expr, TContext context) ); } - /// - protected override Expr VisitLeafLogicalBuffer(TIR.LogicalBuffer expr, TContext context) - { - return expr.With( - dimensions: CloneArray(expr.Dimensions, context), - strides: CloneArray(expr.Strides, context) - ); - } - - /// - protected override Expr VisitLeafPhysicalBuffer(TIR.PhysicalBuffer expr, TContext context) - { - return expr.With( - ); - } - - /// - protected override Expr VisitLeafBufferLoad(TIR.BufferLoad expr, TContext context) - { - return expr.With( - buffer: Clone(expr.Buffer, context), - indices: CloneArray(expr.Indices, context) - ); - } - /// protected override Expr VisitLeafBufferRegion(TIR.BufferRegion expr, TContext context) { @@ -175,16 +148,6 @@ protected override Expr VisitLeafBufferRegion(TIR.BufferRegion expr, TContext co ); } - /// - protected override Expr VisitLeafBufferStore(TIR.BufferStore expr, TContext context) - { - return expr.With( - buffer: Clone(expr.Buffer, context), - indices: CloneArray(expr.Indices, context), - value: Clone(expr.Value, context) - ); - } - /// protected override Expr VisitLeafFor(TIR.For expr, TContext context) { diff --git a/src/Nncase.Core/IR/ExprFunctor.cs b/src/Nncase.Core/IR/ExprFunctor.cs index 4462f8d2cc..2d19dbc1b3 100644 --- a/src/Nncase.Core/IR/ExprFunctor.cs +++ b/src/Nncase.Core/IR/ExprFunctor.cs @@ -102,6 +102,13 @@ public partial class ExprFunctor : ExprFunctorResult. public virtual TTypeResult VisitType(TensorType type) => base.VisitType(type, default); + /// + /// Visit point type. + /// + /// pointer type. + /// Result. + public virtual TTypeResult VisitType(PointerType type) => base.VisitType(type, default); + /// /// Visit tuple type. /// @@ -116,6 +123,13 @@ public partial class ExprFunctor : ExprFunctorResult. public virtual TTypeResult VisitType(CallableType type) => base.VisitType(type, default); + /// + /// Visit callable type. + /// + /// Callable type. + /// Result. + public virtual TTypeResult VisitType(DistributedType type) => base.VisitType(type, default); + /// /// Default visit routine. /// @@ -135,12 +149,18 @@ public partial class ExprFunctor : ExprFunctor public sealed override TTypeResult VisitType(TensorType type, Unit context) => VisitType(type); + /// + public sealed override TTypeResult VisitType(PointerType type, Unit context) => VisitType(type); + /// public sealed override TTypeResult VisitType(TupleType type, Unit context) => VisitType(type); /// public sealed override TTypeResult VisitType(CallableType type, Unit context) => VisitType(type); + /// + public sealed override TTypeResult VisitType(DistributedType type, Unit context) => VisitType(type); + /// public sealed override TTypeResult DefaultVisitType(IRType type, Unit context) => DefaultVisitType(type); diff --git a/src/Nncase.Core/IR/ExprFunctor.g.cs b/src/Nncase.Core/IR/ExprFunctor.g.cs index 642b2709e4..188aad4659 100644 --- a/src/Nncase.Core/IR/ExprFunctor.g.cs +++ b/src/Nncase.Core/IR/ExprFunctor.g.cs @@ -1,4 +1,3 @@ - //--------------------------------------------------------------------------------------------------- // // This code was generated by T4 template. @@ -79,6 +78,11 @@ public partial class ExprFunctor /// internal protected virtual TExprResult VisitTupleConst(TupleConst expr, TContext context) => VisitConst(expr, context); + /// + /// Visit . + /// + internal protected virtual TExprResult VisitMemSpan(TIR.MemSpan expr, TContext context) => DefaultVisit(expr, context); + /// /// Visit . /// @@ -94,31 +98,11 @@ public partial class ExprFunctor /// internal protected virtual TExprResult VisitBuffer(TIR.Buffer expr, TContext context) => DefaultVisit(expr, context); - /// - /// Visit . - /// - internal protected virtual TExprResult VisitLogicalBuffer(TIR.LogicalBuffer expr, TContext context) => VisitBuffer(expr, context); - - /// - /// Visit . - /// - internal protected virtual TExprResult VisitPhysicalBuffer(TIR.PhysicalBuffer expr, TContext context) => VisitBuffer(expr, context); - - /// - /// Visit . - /// - internal protected virtual TExprResult VisitBufferLoad(TIR.BufferLoad expr, TContext context) => DefaultVisit(expr, context); - /// /// Visit . /// internal protected virtual TExprResult VisitBufferRegion(TIR.BufferRegion expr, TContext context) => DefaultVisit(expr, context); - /// - /// Visit . - /// - internal protected virtual TExprResult VisitBufferStore(TIR.BufferStore expr, TContext context) => DefaultVisit(expr, context); - /// /// Visit . /// @@ -250,6 +234,13 @@ public partial class ExprFunctor /// internal protected sealed override TExprResult VisitTupleConst(TupleConst expr, Unit context) => VisitTupleConst(expr); /// + /// Visit . + /// + internal protected virtual TExprResult VisitMemSpan(TIR.MemSpan expr) => base.VisitMemSpan(expr, default); + + /// + internal protected sealed override TExprResult VisitMemSpan(TIR.MemSpan expr, Unit context) => VisitMemSpan(expr); + /// /// Visit . /// internal protected virtual TExprResult VisitVar(Var expr) => base.VisitVar(expr, default); @@ -271,27 +262,6 @@ public partial class ExprFunctor /// internal protected sealed override TExprResult VisitBuffer(TIR.Buffer expr, Unit context) => VisitBuffer(expr); /// - /// Visit . - /// - internal protected virtual TExprResult VisitLogicalBuffer(TIR.LogicalBuffer expr) => base.VisitLogicalBuffer(expr, default); - - /// - internal protected sealed override TExprResult VisitLogicalBuffer(TIR.LogicalBuffer expr, Unit context) => VisitLogicalBuffer(expr); - /// - /// Visit . - /// - internal protected virtual TExprResult VisitPhysicalBuffer(TIR.PhysicalBuffer expr) => base.VisitPhysicalBuffer(expr, default); - - /// - internal protected sealed override TExprResult VisitPhysicalBuffer(TIR.PhysicalBuffer expr, Unit context) => VisitPhysicalBuffer(expr); - /// - /// Visit . - /// - internal protected virtual TExprResult VisitBufferLoad(TIR.BufferLoad expr) => base.VisitBufferLoad(expr, default); - - /// - internal protected sealed override TExprResult VisitBufferLoad(TIR.BufferLoad expr, Unit context) => VisitBufferLoad(expr); - /// /// Visit . /// internal protected virtual TExprResult VisitBufferRegion(TIR.BufferRegion expr) => base.VisitBufferRegion(expr, default); @@ -299,13 +269,6 @@ public partial class ExprFunctor /// internal protected sealed override TExprResult VisitBufferRegion(TIR.BufferRegion expr, Unit context) => VisitBufferRegion(expr); /// - /// Visit . - /// - internal protected virtual TExprResult VisitBufferStore(TIR.BufferStore expr) => base.VisitBufferStore(expr, default); - - /// - internal protected sealed override TExprResult VisitBufferStore(TIR.BufferStore expr, Unit context) => VisitBufferStore(expr); - /// /// Visit . /// internal protected virtual TExprResult VisitFor(TIR.For expr) => base.VisitFor(expr, default); diff --git a/src/Nncase.Core/IR/ExprRewriter.g.cs b/src/Nncase.Core/IR/ExprRewriter.g.cs index 4c8cece3f2..b842c110f1 100644 --- a/src/Nncase.Core/IR/ExprRewriter.g.cs +++ b/src/Nncase.Core/IR/ExprRewriter.g.cs @@ -1,4 +1,3 @@ - //--------------------------------------------------------------------------------------------------- // // This code was generated by T4 template. @@ -92,6 +91,12 @@ protected sealed override Expr VisitLeafTupleConst(TupleConst expr, TContext con return RewriteLeafTupleConst(expr, context); } + /// + protected sealed override Expr VisitLeafMemSpan(TIR.MemSpan expr, TContext context) + { + return RewriteLeafMemSpan(expr, context); + } + /// protected sealed override Expr VisitLeafVar(Var expr, TContext context) { @@ -110,36 +115,12 @@ protected sealed override Expr VisitLeafBuffer(TIR.Buffer expr, TContext context return RewriteLeafBuffer(expr, context); } - /// - protected sealed override Expr VisitLeafLogicalBuffer(TIR.LogicalBuffer expr, TContext context) - { - return RewriteLeafLogicalBuffer(expr, context); - } - - /// - protected sealed override Expr VisitLeafPhysicalBuffer(TIR.PhysicalBuffer expr, TContext context) - { - return RewriteLeafPhysicalBuffer(expr, context); - } - - /// - protected sealed override Expr VisitLeafBufferLoad(TIR.BufferLoad expr, TContext context) - { - return RewriteLeafBufferLoad(expr, context); - } - /// protected sealed override Expr VisitLeafBufferRegion(TIR.BufferRegion expr, TContext context) { return RewriteLeafBufferRegion(expr, context); } - /// - protected sealed override Expr VisitLeafBufferStore(TIR.BufferStore expr, TContext context) - { - return RewriteLeafBufferStore(expr, context); - } - /// protected sealed override Expr VisitLeafFor(TIR.For expr, TContext context) { @@ -247,6 +228,11 @@ protected sealed override Expr VisitLeafIterVar(TIR.IterVar expr, TContext conte /// protected virtual Expr RewriteLeafTupleConst(TupleConst expr, TContext context) => RewriteLeafConst(expr, context); + /// + /// Rewrite leaf . + /// + protected virtual Expr RewriteLeafMemSpan(TIR.MemSpan expr, TContext context) => DefaultRewriteLeaf(expr, context); + /// /// Rewrite leaf . /// @@ -262,31 +248,11 @@ protected sealed override Expr VisitLeafIterVar(TIR.IterVar expr, TContext conte /// protected virtual Expr RewriteLeafBuffer(TIR.Buffer expr, TContext context) => DefaultRewriteLeaf(expr, context); - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafLogicalBuffer(TIR.LogicalBuffer expr, TContext context) => RewriteLeafBuffer(expr, context); - - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafPhysicalBuffer(TIR.PhysicalBuffer expr, TContext context) => RewriteLeafBuffer(expr, context); - - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafBufferLoad(TIR.BufferLoad expr, TContext context) => DefaultRewriteLeaf(expr, context); - /// /// Rewrite leaf . /// protected virtual Expr RewriteLeafBufferRegion(TIR.BufferRegion expr, TContext context) => DefaultRewriteLeaf(expr, context); - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafBufferStore(TIR.BufferStore expr, TContext context) => DefaultRewriteLeaf(expr, context); - /// /// Rewrite leaf . /// @@ -430,6 +396,14 @@ public partial class ExprRewriter /// protected sealed override Expr RewriteLeafTupleConst(TupleConst expr, Unit context) => RewriteLeafTupleConst(expr); + /// + /// Rewrite leaf . + /// + protected virtual Expr RewriteLeafMemSpan(TIR.MemSpan expr) => DefaultRewriteLeaf(expr); + + /// + protected sealed override Expr RewriteLeafMemSpan(TIR.MemSpan expr, Unit context) => RewriteLeafMemSpan(expr); + /// /// Rewrite leaf . /// @@ -454,30 +428,6 @@ public partial class ExprRewriter /// protected sealed override Expr RewriteLeafBuffer(TIR.Buffer expr, Unit context) => RewriteLeafBuffer(expr); - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafLogicalBuffer(TIR.LogicalBuffer expr) => RewriteLeafBuffer(expr); - - /// - protected sealed override Expr RewriteLeafLogicalBuffer(TIR.LogicalBuffer expr, Unit context) => RewriteLeafLogicalBuffer(expr); - - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafPhysicalBuffer(TIR.PhysicalBuffer expr) => RewriteLeafBuffer(expr); - - /// - protected sealed override Expr RewriteLeafPhysicalBuffer(TIR.PhysicalBuffer expr, Unit context) => RewriteLeafPhysicalBuffer(expr); - - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafBufferLoad(TIR.BufferLoad expr) => DefaultRewriteLeaf(expr); - - /// - protected sealed override Expr RewriteLeafBufferLoad(TIR.BufferLoad expr, Unit context) => RewriteLeafBufferLoad(expr); - /// /// Rewrite leaf . /// @@ -486,14 +436,6 @@ public partial class ExprRewriter /// protected sealed override Expr RewriteLeafBufferRegion(TIR.BufferRegion expr, Unit context) => RewriteLeafBufferRegion(expr); - /// - /// Rewrite leaf . - /// - protected virtual Expr RewriteLeafBufferStore(TIR.BufferStore expr) => DefaultRewriteLeaf(expr); - - /// - protected sealed override Expr RewriteLeafBufferStore(TIR.BufferStore expr, Unit context) => RewriteLeafBufferStore(expr); - /// /// Rewrite leaf . /// diff --git a/src/Nncase.Core/IR/ExprVisitor.g.cs b/src/Nncase.Core/IR/ExprVisitor.g.cs index c296f5f7e0..dd974ec60b 100644 --- a/src/Nncase.Core/IR/ExprVisitor.g.cs +++ b/src/Nncase.Core/IR/ExprVisitor.g.cs @@ -1,4 +1,3 @@ - //--------------------------------------------------------------------------------------------------- // // This code was generated by T4 template. @@ -104,38 +103,31 @@ protected internal override TExprResult VisitTupleConst(TupleConst expr, TContex } /// - protected internal override TExprResult VisitVar(Var expr, TContext context) + protected internal override TExprResult VisitMemSpan(TIR.MemSpan expr, TContext context) { VisitOperands(expr, context); - return VisitLeafVar(expr, context); + return VisitLeafMemSpan(expr, context); } /// - protected internal override TExprResult VisitBlock(TIR.Block expr, TContext context) - { - VisitOperands(expr, context); - return VisitLeafBlock(expr, context); - } - - /// - protected internal override TExprResult VisitLogicalBuffer(TIR.LogicalBuffer expr, TContext context) + protected internal override TExprResult VisitVar(Var expr, TContext context) { VisitOperands(expr, context); - return VisitLeafLogicalBuffer(expr, context); + return VisitLeafVar(expr, context); } /// - protected internal override TExprResult VisitPhysicalBuffer(TIR.PhysicalBuffer expr, TContext context) + protected internal override TExprResult VisitBlock(TIR.Block expr, TContext context) { VisitOperands(expr, context); - return VisitLeafPhysicalBuffer(expr, context); + return VisitLeafBlock(expr, context); } /// - protected internal override TExprResult VisitBufferLoad(TIR.BufferLoad expr, TContext context) + protected internal override TExprResult VisitBuffer(TIR.Buffer expr, TContext context) { VisitOperands(expr, context); - return VisitLeafBufferLoad(expr, context); + return VisitLeafBuffer(expr, context); } /// @@ -145,13 +137,6 @@ protected internal override TExprResult VisitBufferRegion(TIR.BufferRegion expr, return VisitLeafBufferRegion(expr, context); } - /// - protected internal override TExprResult VisitBufferStore(TIR.BufferStore expr, TContext context) - { - VisitOperands(expr, context); - return VisitLeafBufferStore(expr, context); - } - /// protected internal override TExprResult VisitFor(TIR.For expr, TContext context) { @@ -270,6 +255,11 @@ protected internal override TExprResult VisitIterVar(TIR.IterVar expr, TContext /// protected virtual TExprResult VisitLeafTupleConst(TupleConst expr, TContext context) => VisitLeafConst(expr, context); + /// + /// Visit leaf . + /// + protected virtual TExprResult VisitLeafMemSpan(TIR.MemSpan expr, TContext context) => DefaultVisitLeaf(expr, context); + /// /// Visit leaf . /// @@ -285,31 +275,11 @@ protected internal override TExprResult VisitIterVar(TIR.IterVar expr, TContext /// protected virtual TExprResult VisitLeafBuffer(TIR.Buffer expr, TContext context) => DefaultVisitLeaf(expr, context); - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafLogicalBuffer(TIR.LogicalBuffer expr, TContext context) => VisitLeafBuffer(expr, context); - - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafPhysicalBuffer(TIR.PhysicalBuffer expr, TContext context) => VisitLeafBuffer(expr, context); - - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafBufferLoad(TIR.BufferLoad expr, TContext context) => DefaultVisitLeaf(expr, context); - /// /// Visit leaf . /// protected virtual TExprResult VisitLeafBufferRegion(TIR.BufferRegion expr, TContext context) => DefaultVisitLeaf(expr, context); - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafBufferStore(TIR.BufferStore expr, TContext context) => DefaultVisitLeaf(expr, context); - /// /// Visit leaf . /// @@ -353,182 +323,168 @@ public partial class ExprVisitor /// Visit . /// internal protected virtual TExprResult VisitCall(Call expr) => base.VisitCall(expr, default); - + /// internal protected sealed override TExprResult VisitCall(Call expr, Unit context) => VisitCall(expr); /// /// Visit . /// internal protected virtual TExprResult VisitFunction(Function expr) => base.VisitFunction(expr, default); - + /// internal protected sealed override TExprResult VisitFunction(Function expr, Unit context) => VisitFunction(expr); /// /// Visit . /// internal protected virtual TExprResult VisitFusion(Fusion expr) => base.VisitFusion(expr, default); - + /// internal protected sealed override TExprResult VisitFusion(Fusion expr, Unit context) => VisitFusion(expr); /// /// Visit . /// internal protected virtual TExprResult VisitIf(If expr) => base.VisitIf(expr, default); - + /// internal protected sealed override TExprResult VisitIf(If expr, Unit context) => VisitIf(expr); /// /// Visit . /// internal protected virtual TExprResult VisitMarker(Marker expr) => base.VisitMarker(expr, default); - + /// internal protected sealed override TExprResult VisitMarker(Marker expr, Unit context) => VisitMarker(expr); /// /// Visit . /// internal protected virtual TExprResult VisitNone(None expr) => base.VisitNone(expr, default); - + /// internal protected sealed override TExprResult VisitNone(None expr, Unit context) => VisitNone(expr); /// /// Visit . /// internal protected virtual TExprResult VisitOp(Op expr) => base.VisitOp(expr, default); - + /// internal protected sealed override TExprResult VisitOp(Op expr, Unit context) => VisitOp(expr); /// /// Visit . /// internal protected virtual TExprResult VisitPrimFunctionWrapper(PrimFunctionWrapper expr) => base.VisitPrimFunctionWrapper(expr, default); - + /// internal protected sealed override TExprResult VisitPrimFunctionWrapper(PrimFunctionWrapper expr, Unit context) => VisitPrimFunctionWrapper(expr); /// /// Visit . /// internal protected virtual TExprResult VisitTensorConst(TensorConst expr) => base.VisitTensorConst(expr, default); - + /// internal protected sealed override TExprResult VisitTensorConst(TensorConst expr, Unit context) => VisitTensorConst(expr); /// /// Visit . /// internal protected virtual TExprResult VisitTuple(IR.Tuple expr) => base.VisitTuple(expr, default); - + /// internal protected sealed override TExprResult VisitTuple(IR.Tuple expr, Unit context) => VisitTuple(expr); /// /// Visit . /// internal protected virtual TExprResult VisitTupleConst(TupleConst expr) => base.VisitTupleConst(expr, default); - + /// internal protected sealed override TExprResult VisitTupleConst(TupleConst expr, Unit context) => VisitTupleConst(expr); /// + /// Visit . + /// + internal protected virtual TExprResult VisitMemSpan(TIR.MemSpan expr) => base.VisitMemSpan(expr, default); + + /// + internal protected sealed override TExprResult VisitMemSpan(TIR.MemSpan expr, Unit context) => VisitMemSpan(expr); + /// /// Visit . /// internal protected virtual TExprResult VisitVar(Var expr) => base.VisitVar(expr, default); - + /// internal protected sealed override TExprResult VisitVar(Var expr, Unit context) => VisitVar(expr); /// /// Visit . /// internal protected virtual TExprResult VisitBlock(TIR.Block expr) => base.VisitBlock(expr, default); - + /// internal protected sealed override TExprResult VisitBlock(TIR.Block expr, Unit context) => VisitBlock(expr); /// - /// Visit . - /// - internal protected virtual TExprResult VisitLogicalBuffer(TIR.LogicalBuffer expr) => base.VisitLogicalBuffer(expr, default); - - /// - internal protected sealed override TExprResult VisitLogicalBuffer(TIR.LogicalBuffer expr, Unit context) => VisitLogicalBuffer(expr); - /// - /// Visit . + /// Visit . /// - internal protected virtual TExprResult VisitPhysicalBuffer(TIR.PhysicalBuffer expr) => base.VisitPhysicalBuffer(expr, default); - + internal protected virtual TExprResult VisitBuffer(TIR.Buffer expr) => base.VisitBuffer(expr, default); + /// - internal protected sealed override TExprResult VisitPhysicalBuffer(TIR.PhysicalBuffer expr, Unit context) => VisitPhysicalBuffer(expr); - /// - /// Visit . - /// - internal protected virtual TExprResult VisitBufferLoad(TIR.BufferLoad expr) => base.VisitBufferLoad(expr, default); - - /// - internal protected sealed override TExprResult VisitBufferLoad(TIR.BufferLoad expr, Unit context) => VisitBufferLoad(expr); + internal protected sealed override TExprResult VisitBuffer(TIR.Buffer expr, Unit context) => VisitBuffer(expr); /// /// Visit . /// internal protected virtual TExprResult VisitBufferRegion(TIR.BufferRegion expr) => base.VisitBufferRegion(expr, default); - + /// internal protected sealed override TExprResult VisitBufferRegion(TIR.BufferRegion expr, Unit context) => VisitBufferRegion(expr); /// - /// Visit . - /// - internal protected virtual TExprResult VisitBufferStore(TIR.BufferStore expr) => base.VisitBufferStore(expr, default); - - /// - internal protected sealed override TExprResult VisitBufferStore(TIR.BufferStore expr, Unit context) => VisitBufferStore(expr); - /// /// Visit . /// internal protected virtual TExprResult VisitFor(TIR.For expr) => base.VisitFor(expr, default); - + /// internal protected sealed override TExprResult VisitFor(TIR.For expr, Unit context) => VisitFor(expr); /// /// Visit . /// internal protected virtual TExprResult VisitIfThenElse(TIR.IfThenElse expr) => base.VisitIfThenElse(expr, default); - + /// internal protected sealed override TExprResult VisitIfThenElse(TIR.IfThenElse expr, Unit context) => VisitIfThenElse(expr); /// /// Visit . /// internal protected virtual TExprResult VisitLet(TIR.Let expr) => base.VisitLet(expr, default); - + /// internal protected sealed override TExprResult VisitLet(TIR.Let expr, Unit context) => VisitLet(expr); /// /// Visit . /// internal protected virtual TExprResult VisitPrimFunction(TIR.PrimFunction expr) => base.VisitPrimFunction(expr, default); - + /// internal protected sealed override TExprResult VisitPrimFunction(TIR.PrimFunction expr, Unit context) => VisitPrimFunction(expr); /// /// Visit . /// internal protected virtual TExprResult VisitSequential(TIR.Sequential expr) => base.VisitSequential(expr, default); - + /// internal protected sealed override TExprResult VisitSequential(TIR.Sequential expr, Unit context) => VisitSequential(expr); /// /// Visit . /// internal protected virtual TExprResult VisitRange(TIR.Range expr) => base.VisitRange(expr, default); - + /// internal protected sealed override TExprResult VisitRange(TIR.Range expr, Unit context) => VisitRange(expr); /// /// Visit . /// internal protected virtual TExprResult VisitIterVar(TIR.IterVar expr) => base.VisitIterVar(expr, default); - + /// internal protected sealed override TExprResult VisitIterVar(TIR.IterVar expr, Unit context) => VisitIterVar(expr); /// /// Visit leaf . /// protected virtual TExprResult VisitLeafBaseFunction(BaseFunction expr) => base.VisitLeafBaseFunction(expr, default); - + /// protected sealed override TExprResult VisitLeafBaseFunction(BaseFunction expr, Unit context) => VisitLeafBaseFunction(expr); @@ -536,7 +492,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafCall(Call expr) => base.VisitLeafCall(expr, default); - + /// protected sealed override TExprResult VisitLeafCall(Call expr, Unit context) => VisitLeafCall(expr); @@ -544,7 +500,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafConst(Const expr) => base.VisitLeafConst(expr, default); - + /// protected sealed override TExprResult VisitLeafConst(Const expr, Unit context) => VisitLeafConst(expr); @@ -552,15 +508,15 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafFunction(Function expr) => base.VisitLeafFunction(expr, default); - + /// - protected override TExprResult VisitLeafFunction(Function expr, Unit context) => VisitLeafFunction(expr); + protected sealed override TExprResult VisitLeafFunction(Function expr, Unit context) => VisitLeafFunction(expr); /// /// Visit leaf . /// protected virtual TExprResult VisitLeafFusion(Fusion expr) => base.VisitLeafFusion(expr, default); - + /// protected sealed override TExprResult VisitLeafFusion(Fusion expr, Unit context) => VisitLeafFusion(expr); @@ -568,7 +524,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafIf(If expr) => base.VisitLeafIf(expr, default); - + /// protected sealed override TExprResult VisitLeafIf(If expr, Unit context) => VisitLeafIf(expr); @@ -576,7 +532,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafMarker(Marker expr) => base.VisitLeafMarker(expr, default); - + /// protected sealed override TExprResult VisitLeafMarker(Marker expr, Unit context) => VisitLeafMarker(expr); @@ -584,7 +540,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafNone(None expr) => base.VisitLeafNone(expr, default); - + /// protected sealed override TExprResult VisitLeafNone(None expr, Unit context) => VisitLeafNone(expr); @@ -592,7 +548,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafOp(Op expr) => base.VisitLeafOp(expr, default); - + /// protected sealed override TExprResult VisitLeafOp(Op expr, Unit context) => VisitLeafOp(expr); @@ -600,7 +556,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafPrimFunctionWrapper(PrimFunctionWrapper expr) => base.VisitLeafPrimFunctionWrapper(expr, default); - + /// protected sealed override TExprResult VisitLeafPrimFunctionWrapper(PrimFunctionWrapper expr, Unit context) => VisitLeafPrimFunctionWrapper(expr); @@ -608,7 +564,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafTensorConst(TensorConst expr) => base.VisitLeafTensorConst(expr, default); - + /// protected sealed override TExprResult VisitLeafTensorConst(TensorConst expr, Unit context) => VisitLeafTensorConst(expr); @@ -616,7 +572,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafTuple(IR.Tuple expr) => base.VisitLeafTuple(expr, default); - + /// protected sealed override TExprResult VisitLeafTuple(IR.Tuple expr, Unit context) => VisitLeafTuple(expr); @@ -624,15 +580,23 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafTupleConst(TupleConst expr) => base.VisitLeafTupleConst(expr, default); - + /// protected sealed override TExprResult VisitLeafTupleConst(TupleConst expr, Unit context) => VisitLeafTupleConst(expr); + /// + /// Visit leaf . + /// + protected virtual TExprResult VisitLeafMemSpan(TIR.MemSpan expr) => base.VisitLeafMemSpan(expr, default); + + /// + protected sealed override TExprResult VisitLeafMemSpan(TIR.MemSpan expr, Unit context) => VisitLeafMemSpan(expr); + /// /// Visit leaf . /// protected virtual TExprResult VisitLeafVar(Var expr) => base.VisitLeafVar(expr, default); - + /// protected sealed override TExprResult VisitLeafVar(Var expr, Unit context) => VisitLeafVar(expr); @@ -640,7 +604,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafBlock(TIR.Block expr) => base.VisitLeafBlock(expr, default); - + /// protected sealed override TExprResult VisitLeafBlock(TIR.Block expr, Unit context) => VisitLeafBlock(expr); @@ -648,55 +612,23 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafBuffer(TIR.Buffer expr) => base.VisitLeafBuffer(expr, default); - + /// protected sealed override TExprResult VisitLeafBuffer(TIR.Buffer expr, Unit context) => VisitLeafBuffer(expr); - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafLogicalBuffer(TIR.LogicalBuffer expr) => base.VisitLeafLogicalBuffer(expr, default); - - /// - protected sealed override TExprResult VisitLeafLogicalBuffer(TIR.LogicalBuffer expr, Unit context) => VisitLeafLogicalBuffer(expr); - - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafPhysicalBuffer(TIR.PhysicalBuffer expr) => base.VisitLeafPhysicalBuffer(expr, default); - - /// - protected sealed override TExprResult VisitLeafPhysicalBuffer(TIR.PhysicalBuffer expr, Unit context) => VisitLeafPhysicalBuffer(expr); - - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafBufferLoad(TIR.BufferLoad expr) => base.VisitLeafBufferLoad(expr, default); - - /// - protected sealed override TExprResult VisitLeafBufferLoad(TIR.BufferLoad expr, Unit context) => VisitLeafBufferLoad(expr); - /// /// Visit leaf . /// protected virtual TExprResult VisitLeafBufferRegion(TIR.BufferRegion expr) => base.VisitLeafBufferRegion(expr, default); - + /// protected sealed override TExprResult VisitLeafBufferRegion(TIR.BufferRegion expr, Unit context) => VisitLeafBufferRegion(expr); - /// - /// Visit leaf . - /// - protected virtual TExprResult VisitLeafBufferStore(TIR.BufferStore expr) => base.VisitLeafBufferStore(expr, default); - - /// - protected sealed override TExprResult VisitLeafBufferStore(TIR.BufferStore expr, Unit context) => VisitLeafBufferStore(expr); - /// /// Visit leaf . /// protected virtual TExprResult VisitLeafFor(TIR.For expr) => base.VisitLeafFor(expr, default); - + /// protected sealed override TExprResult VisitLeafFor(TIR.For expr, Unit context) => VisitLeafFor(expr); @@ -704,7 +636,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafIfThenElse(TIR.IfThenElse expr) => base.VisitLeafIfThenElse(expr, default); - + /// protected sealed override TExprResult VisitLeafIfThenElse(TIR.IfThenElse expr, Unit context) => VisitLeafIfThenElse(expr); @@ -712,7 +644,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafLet(TIR.Let expr) => base.VisitLeafLet(expr, default); - + /// protected sealed override TExprResult VisitLeafLet(TIR.Let expr, Unit context) => VisitLeafLet(expr); @@ -720,7 +652,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafPrimFunction(TIR.PrimFunction expr) => base.VisitLeafPrimFunction(expr, default); - + /// protected sealed override TExprResult VisitLeafPrimFunction(TIR.PrimFunction expr, Unit context) => VisitLeafPrimFunction(expr); @@ -728,7 +660,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafSequential(TIR.Sequential expr) => base.VisitLeafSequential(expr, default); - + /// protected sealed override TExprResult VisitLeafSequential(TIR.Sequential expr, Unit context) => VisitLeafSequential(expr); @@ -736,7 +668,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafRange(TIR.Range expr) => base.VisitLeafRange(expr, default); - + /// protected sealed override TExprResult VisitLeafRange(TIR.Range expr, Unit context) => VisitLeafRange(expr); @@ -744,7 +676,7 @@ public partial class ExprVisitor /// Visit leaf . /// protected virtual TExprResult VisitLeafIterVar(TIR.IterVar expr) => base.VisitLeafIterVar(expr, default); - + /// protected sealed override TExprResult VisitLeafIterVar(TIR.IterVar expr, Unit context) => VisitLeafIterVar(expr); diff --git a/src/Nncase.Core/IR/IIRPrinterProvider.cs b/src/Nncase.Core/IR/IIRPrinterProvider.cs index 3c267771df..f411167f2a 100644 --- a/src/Nncase.Core/IR/IIRPrinterProvider.cs +++ b/src/Nncase.Core/IR/IIRPrinterProvider.cs @@ -73,6 +73,14 @@ public interface IIRPrinterProvider /// randConst = false will save the const into bin. public void DumpCSharpIR(Expr expr, string prefix, string dumpDir, bool randConst); + /// + /// dump the expr as csharp code. + /// + /// expression. + /// file prefix. + /// file dump ir. + public void DumpPatternIR(Expr expr, string prefix, string dumpDir); + /// /// print ir type. /// diff --git a/src/Nncase.Core/IR/IRList.csv b/src/Nncase.Core/IR/IRList.csv index 5ae3c89d18..ba9dd8033b 100644 --- a/src/Nncase.Core/IR/IRList.csv +++ b/src/Nncase.Core/IR/IRList.csv @@ -11,14 +11,11 @@ PrimFunctionWrapper,true,true,BaseFunction,,Target TensorConst,true,false,Const,, Tuple,true,false,Default,IR.,@Fields TupleConst,true,false,Const,, +MemSpan,true,false,Default,TIR.,Start;Size; Var,true,false,Default,, Block,true,false,Default,TIR.,Body;InitBody;@IterVars;@Reads;@Writes;@AllocBuffers;Predicate -Buffer,false,false,Default,TIR., -LogicalBuffer,true,false,Buffer,TIR.,@Dimensions;@Strides -PhysicalBuffer,true,false,Buffer,TIR., -BufferLoad,true,false,Default,TIR.,Buffer;@Indices +Buffer,true,false,Default,TIR.,MemSpan;@Dimensions;@Strides; BufferRegion,true,false,Default,TIR.,Buffer;@Region -BufferStore,true,false,Default,TIR.,Buffer;@Indices;Value For,true,false,Default,TIR.,LoopVar;Domain;Body IfThenElse,true,false,Default,TIR.,Condition;Then;Else Let,true,false,Default,TIR.,Var;Expression;Body diff --git a/src/Nncase.Core/IR/IRType.cs b/src/Nncase.Core/IR/IRType.cs index b8aeda469f..a4311aae13 100644 --- a/src/Nncase.Core/IR/IRType.cs +++ b/src/Nncase.Core/IR/IRType.cs @@ -139,6 +139,15 @@ public sealed record TensorType(DataType DType, Shape Shape) : IRType /// the Pointed Element Type. /// the pointer tensor type. public static TensorType Pointer(DataType elemType) => new(new PointerType(elemType), Shape.Scalar); + + /// + public override string ToString() => DType switch + { + PrimType ptype => ptype.GetDisplayName() + (Shape.IsScalar ? string.Empty : Shape.ToString()), + PointerType { ElemType: PrimType etype } => $"*{etype.GetDisplayName()}", + ValueType => $"{DType}", + _ => throw new NotSupportedException(DType.GetType().Name), + }; } /// diff --git a/src/Nncase.Core/IR/Imaging/ResizeImage.cs b/src/Nncase.Core/IR/Imaging/ResizeImage.cs index 088651b511..ae48d48831 100644 --- a/src/Nncase.Core/IR/Imaging/ResizeImage.cs +++ b/src/Nncase.Core/IR/Imaging/ResizeImage.cs @@ -21,7 +21,7 @@ public sealed partial class ResizeImage : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(ResizeImage), 0, "input", HasRank(r => r >= 2, "RanK >= 2")); + public static readonly ParameterInfo Input = new(typeof(ResizeImage), 0, "input", HasRank(r => r >= 2, "RanK >= 2"), ParameterKind.Input); /// /// Gets roi. diff --git a/src/Nncase.Core/IR/Math/Binary.cs b/src/Nncase.Core/IR/Math/Binary.cs index ead10ff710..f61f8a8704 100644 --- a/src/Nncase.Core/IR/Math/Binary.cs +++ b/src/Nncase.Core/IR/Math/Binary.cs @@ -20,12 +20,12 @@ public sealed partial class Binary : Op /// /// Gets lhs. /// - public static readonly ParameterInfo Lhs = new(typeof(Binary), 0, "lhs"); + public static readonly ParameterInfo Lhs = new(typeof(Binary), 0, "lhs", ParameterKind.Input); /// /// Gets rhs. /// - public static readonly ParameterInfo Rhs = new(typeof(Binary), 1, "rhs"); + public static readonly ParameterInfo Rhs = new(typeof(Binary), 1, "rhs", ParameterKind.Input); public BinaryOp BinaryOp { get; } diff --git a/src/Nncase.Core/IR/Math/Clamp.cs b/src/Nncase.Core/IR/Math/Clamp.cs index 9f14cf287d..b8409f375c 100644 --- a/src/Nncase.Core/IR/Math/Clamp.cs +++ b/src/Nncase.Core/IR/Math/Clamp.cs @@ -21,7 +21,7 @@ public sealed partial class Clamp : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Clamp), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Clamp), 0, "input", ParameterKind.Input); /// /// Gets min. diff --git a/src/Nncase.Core/IR/Math/MatMul.cs b/src/Nncase.Core/IR/Math/MatMul.cs index 51d5615f1f..fc74e211e1 100644 --- a/src/Nncase.Core/IR/Math/MatMul.cs +++ b/src/Nncase.Core/IR/Math/MatMul.cs @@ -20,10 +20,10 @@ public sealed partial class MatMul : Op /// /// Gets input. /// - public static readonly ParameterInfo Lhs = new(typeof(MatMul), 0, "lhs"); + public static readonly ParameterInfo Lhs = new(typeof(MatMul), 0, "lhs", ParameterKind.Input); /// /// Gets Other. /// - public static readonly ParameterInfo Rhs = new(typeof(MatMul), 1, "rhs"); + public static readonly ParameterInfo Rhs = new(typeof(MatMul), 1, "rhs", ParameterKind.Input); } diff --git a/src/Nncase.Core/IR/Math/ReduceArg.cs b/src/Nncase.Core/IR/Math/ReduceArg.cs index ecad8e95e2..2afd43010c 100644 --- a/src/Nncase.Core/IR/Math/ReduceArg.cs +++ b/src/Nncase.Core/IR/Math/ReduceArg.cs @@ -21,7 +21,7 @@ public sealed partial class ReduceArg : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(ReduceArg), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(ReduceArg), 0, "input", ParameterKind.Input); /// /// Gets Axis. @@ -42,8 +42,8 @@ public sealed partial class ReduceArg : Op public ReduceArgOp ReduceArgOp { get; } - public DataType DestType { get; } + public PrimType DestType { get; } /// - public override string DisplayProperty() => $"ReduceArgOp.{ReduceArgOp}"; + public override string DisplayProperty() => $"ReduceArgOp.{ReduceArgOp}, DestType: {DestType}"; } diff --git a/src/Nncase.Core/IR/Math/Unary.cs b/src/Nncase.Core/IR/Math/Unary.cs index 820572437e..20d6b3fb03 100644 --- a/src/Nncase.Core/IR/Math/Unary.cs +++ b/src/Nncase.Core/IR/Math/Unary.cs @@ -20,7 +20,7 @@ public sealed partial class Unary : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Unary), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Unary), 0, "input", ParameterKind.Input); public UnaryOp UnaryOp { get; } diff --git a/src/Nncase.Core/IR/NN/Activations.cs b/src/Nncase.Core/IR/NN/Activations.cs index 46df70d241..1866e65220 100644 --- a/src/Nncase.Core/IR/NN/Activations.cs +++ b/src/Nncase.Core/IR/NN/Activations.cs @@ -154,7 +154,12 @@ public sealed partial class Swish : ActivationOp /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Swish), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Swish), 0, "input", ParameterKind.Input); + + /// + /// Gets beta. + /// + public static readonly ParameterInfo Beta = new(typeof(Swish), 1, "beta", IsFloatScalar()); } /// diff --git a/src/Nncase.Core/IR/NN/Conv2D.cs b/src/Nncase.Core/IR/NN/Conv2D.cs index 43c1b2fced..0607870481 100644 --- a/src/Nncase.Core/IR/NN/Conv2D.cs +++ b/src/Nncase.Core/IR/NN/Conv2D.cs @@ -21,17 +21,17 @@ public sealed partial class Conv2D : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Conv2D), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Conv2D), 0, "input", ParameterKind.Input); /// /// Gets Weights. /// - public static readonly ParameterInfo Weights = new(typeof(Conv2D), 1, "weights", HasRank(4)); + public static readonly ParameterInfo Weights = new(typeof(Conv2D), 1, "weights", HasRank(4), ParameterKind.Input); /// /// Gets Bias. /// - public static readonly ParameterInfo Bias = new(typeof(Conv2D), 2, "bias", HasRank(1)); + public static readonly ParameterInfo Bias = new(typeof(Conv2D), 2, "bias", HasRank(1), ParameterKind.Input); /// /// Gets Stride. diff --git a/src/Nncase.Core/IR/NN/Functional.cs b/src/Nncase.Core/IR/NN/Functional.cs index 30b4005388..e8a44d8a38 100644 --- a/src/Nncase.Core/IR/NN/Functional.cs +++ b/src/Nncase.Core/IR/NN/Functional.cs @@ -34,7 +34,7 @@ public static class NN public static Call BatchNormalization(Expr input, Expr scale, Expr bias, Expr input_mean, Expr input_var, Expr epsilon, Expr momentum) => new Call(new BatchNormalization(), input, scale, bias, input_mean, input_var, epsilon, momentum); - public static Call LayerNorm(int axis, float epsilon, Expr input, Expr scale, Expr bias) => new Call(new LayerNorm(axis, epsilon), input, scale, bias); + public static Call LayerNorm(int axis, float epsilon, Expr input, Expr scale, Expr bias, bool hasMean = true) => new Call(new LayerNorm(axis, epsilon, hasMean), input, scale, bias); public static Call BatchToSpace(Expr input, Expr blockShape, Expr crops) => new Call(new BatchToSpace(), input, blockShape, crops); @@ -103,5 +103,10 @@ public static Call ReduceWindow2D(ReduceOp reduceOp, Expr input, Expr initValue, /// /// create Swish call. /// - public static Call Swish(Expr input) => new Call(new Swish(), input); + public static Call Swish(Expr input) => new Call(new Swish(), input, 1f); + + /// + /// create Swish call. + /// + public static Call Swish(Expr input, Expr beta) => new Call(new Swish(), input, beta); } diff --git a/src/Nncase.Core/IR/NN/LayerNorm.cs b/src/Nncase.Core/IR/NN/LayerNorm.cs index 2dff32f440..2474f44fc2 100644 --- a/src/Nncase.Core/IR/NN/LayerNorm.cs +++ b/src/Nncase.Core/IR/NN/LayerNorm.cs @@ -21,19 +21,23 @@ public sealed partial class LayerNorm : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(LayerNorm), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(LayerNorm), 0, "input", ParameterKind.Input); /// /// Gets scale. /// - public static readonly ParameterInfo Scale = new(typeof(LayerNorm), 1, "scale"); + public static readonly ParameterInfo Scale = new(typeof(LayerNorm), 1, "scale", ParameterKind.Input); /// /// Gets bias. /// - public static readonly ParameterInfo Bias = new(typeof(LayerNorm), 2, "bias"); + public static readonly ParameterInfo Bias = new(typeof(LayerNorm), 2, "bias", ParameterKind.Input); public int Axis { get; } public float Epsilon { get; } + + public bool UseMean { get; } + + public override string DisplayProperty() => $"Axis: {Axis}, Epsilon: {Epsilon}, UseMean: {UseMean}"; } diff --git a/src/Nncase.Core/IR/NN/Normalization.cs b/src/Nncase.Core/IR/NN/Normalization.cs index ba91f13f83..2b7b74168c 100644 --- a/src/Nncase.Core/IR/NN/Normalization.cs +++ b/src/Nncase.Core/IR/NN/Normalization.cs @@ -61,17 +61,17 @@ public sealed partial class InstanceNormalization : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(InstanceNormalization), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(InstanceNormalization), 0, "input", ParameterKind.Input); /// /// Gets input. /// - public static readonly ParameterInfo Scale = new(typeof(InstanceNormalization), 1, "scale"); + public static readonly ParameterInfo Scale = new(typeof(InstanceNormalization), 1, "scale", ParameterKind.Input); /// /// Gets input. /// - public static readonly ParameterInfo Bias = new(typeof(InstanceNormalization), 2, "bias"); + public static readonly ParameterInfo Bias = new(typeof(InstanceNormalization), 2, "bias", ParameterKind.Input); /// /// Gets Epsilon. diff --git a/src/Nncase.Core/IR/NN/SoftMax.cs b/src/Nncase.Core/IR/NN/SoftMax.cs index 686696f032..919f8ac76c 100644 --- a/src/Nncase.Core/IR/NN/SoftMax.cs +++ b/src/Nncase.Core/IR/NN/SoftMax.cs @@ -33,7 +33,7 @@ public sealed partial class Softmax : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Softmax), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Softmax), 0, "input", ParameterKind.Input); /// /// Gets axis. diff --git a/src/Nncase.Core/IR/Op.cs b/src/Nncase.Core/IR/Op.cs index 07d4bbcabb..2af2e39fd3 100644 --- a/src/Nncase.Core/IR/Op.cs +++ b/src/Nncase.Core/IR/Op.cs @@ -12,6 +12,12 @@ namespace Nncase.IR; +public enum ParameterKind : int +{ + Input, + Attribute, +} + /// /// Parameter information. /// @@ -24,11 +30,13 @@ public sealed class ParameterInfo /// this op type. /// param index. /// param name. - public ParameterInfo(Type ownerType, int index, string name) + /// kind. + public ParameterInfo(Type ownerType, int index, string name, ParameterKind parameterKind = ParameterKind.Attribute) { OwnerType = ownerType; Index = index; Name = name; + ParameterKind = parameterKind; } /// @@ -39,8 +47,9 @@ public ParameterInfo(Type ownerType, int index, string name) /// param index. /// param name. /// the param condition. - public ParameterInfo(Type ownerType, int index, string name, TypePattern pattern) - : this(ownerType, index, name) + /// kind. + public ParameterInfo(Type ownerType, int index, string name, TypePattern pattern, ParameterKind parameterKind = ParameterKind.Attribute) + : this(ownerType, index, name, parameterKind) { Pattern = pattern; } @@ -60,6 +69,11 @@ public ParameterInfo(Type ownerType, int index, string name, TypePattern pattern /// public string Name { get; } + /// + /// Gets parameter kind. + /// + public ParameterKind ParameterKind { get; } + /// /// Gets this paramter's type condition. /// @@ -90,7 +104,7 @@ public Op() /// /// Gets get the parameters. /// - public IEnumerable Parameters => + public virtual IEnumerable Parameters => _parameters ??= (from p in GetType().GetFields(BindingFlags.Public | BindingFlags.Static) where p.FieldType == typeof(ParameterInfo) let param = (ParameterInfo)(p.GetValue(null) ?? throw new InvalidOperationException()) diff --git a/src/Nncase.Core/IR/RNN/Functional.cs b/src/Nncase.Core/IR/RNN/Functional.cs index 07ebf7f8d6..571bb2141f 100644 --- a/src/Nncase.Core/IR/RNN/Functional.cs +++ b/src/Nncase.Core/IR/RNN/Functional.cs @@ -19,5 +19,5 @@ namespace Nncase.IR.F; public static class RNN { public static Call LSTM(LSTMDirection direction, LSTMLayout layout, string[] acts, Expr x, Expr w, Expr r, Expr b, Expr seqLens, Expr initH, Expr initC, Expr p, Expr actAlpha, Expr actBeta, Expr clip, Expr hiddenSize, Expr inputForget, Expr outputSize) => - new Call(new IR.Tensors.LSTM(direction, layout, acts), x, w, r, b, seqLens, initH, initC, p, actAlpha, actBeta, clip, hiddenSize, inputForget, outputSize); + new Call(new IR.RNN.LSTM(direction, layout, acts), x, w, r, b, seqLens, initH, initC, p, actAlpha, actBeta, clip, hiddenSize, inputForget, outputSize); } diff --git a/src/Nncase.Core/IR/RNN/LSTM.cs b/src/Nncase.Core/IR/RNN/LSTM.cs index ec5b9802b3..4bdd60f223 100644 --- a/src/Nncase.Core/IR/RNN/LSTM.cs +++ b/src/Nncase.Core/IR/RNN/LSTM.cs @@ -5,7 +5,7 @@ using Nncase.PatternMatch; using static Nncase.IR.TypePatternUtility; -namespace Nncase.IR.Tensors; +namespace Nncase.IR.RNN; /// /// LSTM expression. diff --git a/src/Nncase.Core/IR/TensorConst.cs b/src/Nncase.Core/IR/TensorConst.cs index 64b1fd6442..9e651978ed 100644 --- a/src/Nncase.Core/IR/TensorConst.cs +++ b/src/Nncase.Core/IR/TensorConst.cs @@ -146,7 +146,7 @@ public override TExprResult Accept(ExprFunct public override bool Equals(object? obj) => Equals(obj as TensorConst); /// - public bool Equals(TensorConst? other) => other is not null && base.Equals(other) && EqualityComparer.Default.Equals(Value, other.Value); + public bool Equals(TensorConst? other) => other is not null && (ReferenceEquals(this, other) || GetHashCode() == other.GetHashCode()) && EqualityComparer.Default.Equals(Value, other.Value); /// protected override int GetHashCodeCore() => HashCode.Combine(Value); diff --git a/src/Nncase.Core/IR/Tensors/Cast.cs b/src/Nncase.Core/IR/Tensors/Cast.cs index 5345cac153..1bc618f786 100644 --- a/src/Nncase.Core/IR/Tensors/Cast.cs +++ b/src/Nncase.Core/IR/Tensors/Cast.cs @@ -20,7 +20,7 @@ public sealed partial class Cast : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Cast), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Cast), 0, "input", ParameterKind.Input); public DataType NewType { get; } diff --git a/src/Nncase.Core/IR/Tensors/Concat.cs b/src/Nncase.Core/IR/Tensors/Concat.cs index 88a22e4376..cbe4861bc3 100644 --- a/src/Nncase.Core/IR/Tensors/Concat.cs +++ b/src/Nncase.Core/IR/Tensors/Concat.cs @@ -20,10 +20,13 @@ public sealed partial class Concat : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Concat), 0, "inputs"); + public static readonly ParameterInfo Input = new(typeof(Concat), 0, "inputs", ParameterKind.Input); /// /// Gets axis. /// - public static readonly ParameterInfo Axis = new(typeof(Concat), 1, "axis"); + public int Axis { get; } + + /// + public override string DisplayProperty() => $"Axis: {Axis}"; } diff --git a/src/Nncase.Core/IR/Tensors/Expand.cs b/src/Nncase.Core/IR/Tensors/Expand.cs index 3b74de6740..91a0d53e26 100644 --- a/src/Nncase.Core/IR/Tensors/Expand.cs +++ b/src/Nncase.Core/IR/Tensors/Expand.cs @@ -21,7 +21,7 @@ public sealed partial class Expand : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Expand), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Expand), 0, "input", ParameterKind.Input); /// /// Gets shape. diff --git a/src/Nncase.Core/IR/Tensors/Functional.cs b/src/Nncase.Core/IR/Tensors/Functional.cs index 71c0bfc51c..d20f69cbf5 100644 --- a/src/Nncase.Core/IR/Tensors/Functional.cs +++ b/src/Nncase.Core/IR/Tensors/Functional.cs @@ -70,7 +70,7 @@ public static Call Bitcast(PrimType type, Expr input, PrimType newType, Expr sha public static Call Cast(Expr input, DataType newType, CastMode castMode = CastMode.KDefault) => new Call(new Cast(newType, castMode), input); - public static Call Concat(Expr input, Expr axis) => new Call(new Concat(), input, axis); + public static Call Concat(Expr input, int axis) => new Call(new Concat(axis), input); public static Call ConstantOfShape(Expr shape, Expr value) => new Call(new ConstantOfShape(), shape, value); @@ -89,7 +89,7 @@ public static Call Expand(Expr input, Expr shape) public static Call Flatten(Expr input, Expr axis) => new Call(new Flatten(), input, axis); - public static Call Gather(Expr input, Expr axis, Expr index) => new Call(new Gather(), input, axis, index); + public static Call Gather(Expr input, int axis, Expr index) => new Call(new Gather(axis), input, index); public static Call GatherElements(Expr input, Expr axis, Expr indices) => new Call(new GatherElements(), input, axis, indices); diff --git a/src/Nncase.Core/IR/Tensors/Gather.cs b/src/Nncase.Core/IR/Tensors/Gather.cs index a498d38984..012a0de053 100644 --- a/src/Nncase.Core/IR/Tensors/Gather.cs +++ b/src/Nncase.Core/IR/Tensors/Gather.cs @@ -22,15 +22,18 @@ public sealed partial class Gather : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Gather), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Gather), 0, "input", ParameterKind.Input); /// - /// Gets axis. + /// Gets index. /// - public static readonly ParameterInfo Axis = new(typeof(Gather), 1, "axis", IsIntegralScalar()); + public static readonly ParameterInfo Index = new(typeof(Gather), 1, "index", IsIntegral(), ParameterKind.Input); /// - /// Gets index. + /// Gets axis. /// - public static readonly ParameterInfo Index = new(typeof(Gather), 2, "index", IsIntegral()); + public int Axis { get; } + + /// + public override string DisplayProperty() => $"Axis: {Axis}"; } diff --git a/src/Nncase.Core/IR/Tensors/Reshape.cs b/src/Nncase.Core/IR/Tensors/Reshape.cs index 2db6d16b89..571fa457e6 100644 --- a/src/Nncase.Core/IR/Tensors/Reshape.cs +++ b/src/Nncase.Core/IR/Tensors/Reshape.cs @@ -22,7 +22,7 @@ public sealed partial class Reshape : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Reshape), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Reshape), 0, "input", ParameterKind.Input); /// /// Gets shape. diff --git a/src/Nncase.Core/IR/Tensors/Slice.cs b/src/Nncase.Core/IR/Tensors/Slice.cs index bc58e51ee8..05963ad7f7 100644 --- a/src/Nncase.Core/IR/Tensors/Slice.cs +++ b/src/Nncase.Core/IR/Tensors/Slice.cs @@ -21,7 +21,7 @@ public sealed partial class Slice : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Slice), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Slice), 0, "input", ParameterKind.Input); /// /// Gets begins. diff --git a/src/Nncase.Core/IR/Tensors/Transpose.cs b/src/Nncase.Core/IR/Tensors/Transpose.cs index 896d279c29..211dda9a54 100644 --- a/src/Nncase.Core/IR/Tensors/Transpose.cs +++ b/src/Nncase.Core/IR/Tensors/Transpose.cs @@ -15,7 +15,7 @@ public sealed partial class Transpose : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Transpose), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Transpose), 0, "input", ParameterKind.Input); /// /// Gets perm. diff --git a/src/Nncase.Core/IR/Tensors/UnSqueeze.cs b/src/Nncase.Core/IR/Tensors/UnSqueeze.cs index cbc2574fc3..6ce9247d24 100644 --- a/src/Nncase.Core/IR/Tensors/UnSqueeze.cs +++ b/src/Nncase.Core/IR/Tensors/UnSqueeze.cs @@ -23,7 +23,7 @@ public sealed partial class Unsqueeze : Op /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(Unsqueeze), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(Unsqueeze), 0, "input", ParameterKind.Input); /// /// Gets dimension. diff --git a/src/Nncase.Core/IR/TypeFunctor.cs b/src/Nncase.Core/IR/TypeFunctor.cs index 453cfa257a..f11ec869a4 100644 --- a/src/Nncase.Core/IR/TypeFunctor.cs +++ b/src/Nncase.Core/IR/TypeFunctor.cs @@ -32,6 +32,7 @@ public virtual TResult VisitType(IRType type, TContext context) TensorType t => VisitType(t, context), TupleType t => VisitType(t, context), CallableType t => VisitType(t, context), + DistributedType t => VisitType(t, context), _ => DefaultVisitType(type, context), }; } @@ -68,6 +69,14 @@ public virtual TResult VisitType(IRType type, TContext context) /// Result. public virtual TResult VisitType(TensorType type, TContext context) => DefaultVisitType(type, context); + /// + /// Visit pointer type. + /// + /// Pointer type. + /// Context. + /// Result. + public virtual TResult VisitType(PointerType type, TContext context) => DefaultVisitType(type, context); + /// /// Visit tuple type. /// @@ -84,6 +93,14 @@ public virtual TResult VisitType(IRType type, TContext context) /// Result. public virtual TResult VisitType(CallableType type, TContext context) => DefaultVisitType(type, context); + /// + /// Visit dist tensor type. + /// + /// dist tensor type. + /// Context. + /// Result. + public virtual TResult VisitType(DistributedType type, TContext context) => DefaultVisitType(type, context); + /// /// Default visit routine. /// diff --git a/src/Nncase.Core/IR/TypePattern.cs b/src/Nncase.Core/IR/TypePattern.cs index 6eea450f5c..183ae381f7 100644 --- a/src/Nncase.Core/IR/TypePattern.cs +++ b/src/Nncase.Core/IR/TypePattern.cs @@ -57,12 +57,12 @@ public TypePattern(CallableType valueType) public T Check(T valueType, string fieldName) where T : IRType { - if (valueType is TensorType tensorValueType && tensorValueType.Shape.IsUnranked) + if (valueType is TensorType { Shape: { IsUnranked: true } } || valueType is DistributedType { TensorType: { Shape: { IsUnranked: true } } }) { return valueType; } - if (valueType == null || !MatchLeaf(valueType)) + if (valueType == null || (valueType is TensorType t && !MatchLeaf(t)) || (valueType is DistributedType d && !MatchLeaf(d.TensorType))) { var cur = valueType is null ? "None" : CompilerServices.Print(valueType); throw new InvalidOperationException($"{fieldName} Requrie <{Reason}>, But {cur}!"); @@ -187,6 +187,7 @@ public static TypePattern HasRank(Func cond, string reason) => HasSha x => x switch { TensorType ttype => DataTypes.IsIntegral(ttype.DType), + DistributedType distributedType => DataTypes.IsIntegral(distributedType.TensorType.DType), _ => false, }, "IsIntegral"); diff --git a/src/Nncase.Core/ITarget.cs b/src/Nncase.Core/ITarget.cs index 7ecc6fa840..a72c0c5f9b 100644 --- a/src/Nncase.Core/ITarget.cs +++ b/src/Nncase.Core/ITarget.cs @@ -13,6 +13,13 @@ namespace Nncase; +/// +/// The targets own compile options. +/// +public interface ITargetCompileOptions +{ +} + /// /// Target. /// @@ -23,6 +30,12 @@ public interface ITarget /// string Kind { get; } + /// + /// create the current target's command and parser. + /// + /// command. + (System.CommandLine.Command Command, Func Parser) RegisterCommandAndParser(); + /// /// Bind Quant Method And Quant Cosine With IR. /// @@ -91,3 +104,12 @@ public interface ITarget /// Module builder. IModuleBuilder CreateModuleBuilder(string moduleKind, CompileOptions options); } + +public sealed class DefaultTargetCompileOptions : ITargetCompileOptions +{ + public static readonly DefaultTargetCompileOptions Instance = new(); + + private DefaultTargetCompileOptions() + { + } +} diff --git a/src/Nncase.Core/LinqExtensions.cs b/src/Nncase.Core/LinqExtensions.cs index a4245953e8..b40f14a8d5 100644 --- a/src/Nncase.Core/LinqExtensions.cs +++ b/src/Nncase.Core/LinqExtensions.cs @@ -14,6 +14,21 @@ namespace Nncase; /// public static class LinqExtensions { + /// + /// Get the ranges from range desc. + /// + /// stride. + /// start. + /// stop. + /// Ranges. + public static IEnumerable Ranges(this int stride, int start, int stop) + { + for (int i = start; i < stop; i += stride) + { + yield return new Range(i, Math.Min(stop, i + stride)); + } + } + /// /// Get cartesian product. /// @@ -31,6 +46,23 @@ from item in sequence select accseq.Concat(new[] { item })); } + /// + /// Get the permutation of the source. + /// + /// Element type. + /// Source sequences. + /// Permutated sequences. + public static IEnumerable Permutate(this IEnumerable source) + { + return Permutation(source, Enumerable.Empty()); + + IEnumerable Permutation(IEnumerable reminder, IEnumerable prefix) => + !reminder.Any() ? new[] { prefix.ToArray() } : + reminder.SelectMany((c, i) => Permutation( + reminder.Take(i).Concat(reminder.Skip(i + 1)).ToArray(), + prefix.Append(c))); + } + /// /// take or default. /// diff --git a/src/Nncase.Core/Nncase.Core.csproj b/src/Nncase.Core/Nncase.Core.csproj index 716f4e3dba..1ea4c4f534 100644 --- a/src/Nncase.Core/Nncase.Core.csproj +++ b/src/Nncase.Core/Nncase.Core.csproj @@ -4,7 +4,7 @@ enable enable true - true + true True @@ -21,6 +21,7 @@ + diff --git a/src/Nncase.Core/Passes/Mutators/FlattenBuffer.cs b/src/Nncase.Core/Passes/Mutators/FlattenBuffer.cs index a0431cdd68..b0b41be0b6 100644 --- a/src/Nncase.Core/Passes/Mutators/FlattenBuffer.cs +++ b/src/Nncase.Core/Passes/Mutators/FlattenBuffer.cs @@ -12,35 +12,43 @@ namespace Nncase.Passes.Mutators; /// -/// Flatten the multi-dimensional BufferLoad and BufferStore to single dimensional Load/Store. Also remove Block to ensure that the flattened TIR can not be scheduled again. +/// Flatten the multi-dimensional BufferLoad and BufferStore to single dimensional Load/Store. /// public sealed class FlattenBuffer : ExprRewriter { /// protected override Expr RewriteLeafBlock(Block expr) { - if (!expr.IterVars.IsEmpty) + // TODO: put the unfold block into this. + if (expr.Predicate is TensorConst tc && tc.Value.ToScalar() == true) { - throw new InvalidOperationException("Non-opaque blocks are not allowed in FlattenBuffer. Please call pass ConvertBlocksToOpaque before."); + return expr.Body; } - // 1. Visit the body - var predicate = expr.Predicate; - if (predicate is TensorConst { Value: { Length: 1 } t } - && t.ToScalar()) + return T.Nop(); + } + + /// + protected override Expr RewriteLeafCall(Call expr) + { + if (expr.Target is IR.Buffers.BufferLoad) { - return expr.Body; + var indices = (IR.Tuple)expr[IR.Buffers.BufferLoad.Indices]; + var input = (TIR.Buffer)expr[IR.Buffers.BufferLoad.Input]; + return T.Load(input.MemSpan, Enumerable.Range(0, indices.Count).Aggregate((Expr)0, (acc, i) => acc + (input.Strides[i] * indices[i]))); + } + else if (expr.Target is IR.Buffers.BufferStore) + { + var indices = (IR.Tuple)expr[IR.Buffers.BufferStore.Indices]; + var input = (TIR.Buffer)expr[IR.Buffers.BufferStore.Input]; + return T.Store(input.MemSpan, Enumerable.Range(0, indices.Count).Aggregate((Expr)0, (acc, i) => acc + (input.Strides[i] * indices[i])), expr[IR.Buffers.BufferStore.Value]); } - else + else if (expr.Target is IR.Buffers.MatchBuffer && expr.Arguments[0] is TIR.Buffer { MemSpan: { Start: Const or Var } }) { - return new IfThenElse(predicate, expr.Body); + // remove the all fixed match operation. + return T.Nop(); } - // Step 3. Handle allocations in reverse order - // TODO add the alloc buffers. - // for (size_t i = new_block->alloc_buffers.size(); i > 0; --i) { - // const Buffer& buffer = new_block->alloc_buffers[i - 1]; - // body = MakeAllocStmt(buffer, std::move(body)); - // } + return expr; } } diff --git a/src/Nncase.Core/Passes/Mutators/FoldBufferSlot.cs b/src/Nncase.Core/Passes/Mutators/FoldBufferSlot.cs new file mode 100644 index 0000000000..018183c5d7 --- /dev/null +++ b/src/Nncase.Core/Passes/Mutators/FoldBufferSlot.cs @@ -0,0 +1,51 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System.Reactive; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.Passes; +using Nncase.TIR; + +namespace Nncase.Passes.Mutators; + +/// +/// remove buffer BaseMentOf/DDrOf/MmuOF. +/// +public sealed class FoldBufferSlot : ExprRewriter +{ + protected internal override Expr VisitPrimFunction(TIR.PrimFunction expr, Unit context) + { + if (expr.SchedResult.IsScheduled == true) + { + return base.VisitPrimFunction(expr, context); + } + + return expr; + } + + protected override Expr RewriteLeafCall(Call expr) + { + if (expr.Target is IR.Buffers.BaseMentOf) + { + var locate = ((TIR.MemSpan)expr.Arguments[0]).Location; + return locate switch + { + MemoryLocation.Input => 0, + MemoryLocation.Output => 1, + MemoryLocation.Rdata => 2, + MemoryLocation.Data => 3, + _ => throw new ArgumentOutOfRangeException($"You Can't Assgin The BaseMent For {locate}!"), + }; + } + else if (expr.Target is IR.Buffers.DDrOf) + { + if (expr.Arguments[0] is TIR.MemSpan buf) + { + return buf.Start; + } + } + + return expr; + } +} diff --git a/src/Nncase.Core/Passes/Mutators/FoldMathCall.cs b/src/Nncase.Core/Passes/Mutators/FoldMathCall.cs deleted file mode 100644 index af25604454..0000000000 --- a/src/Nncase.Core/Passes/Mutators/FoldMathCall.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Canaan Inc. All rights reserved. -// Licensed under the Apache license. See LICENSE file in the project root for full license information. - -using System.Reactive; -using NetFabric.Hyperlinq; -using Nncase.Evaluator; -using Nncase.IR; -using Nncase.Passes; - -namespace Nncase.Passes.Mutators; - -/// -/// fold math calc operator. -/// -public sealed class FoldMathCall : ExprRewriter -{ - /// - protected override Expr RewriteLeafCall(Call expr) - { - if (expr.Target is Op op && op.GetType().Namespace is string @namespace - && @namespace.StartsWith("Nncase.IR.Math")) - { - return expr.Arguments.AsValueEnumerable().All(x => x is Const) - ? Const.FromValue(CompilerServices.Evaluate(expr)) - : expr; - } - - return expr; - } -} diff --git a/src/Nncase.Core/Passes/Mutators/Mutator.cs b/src/Nncase.Core/Passes/Mutators/Mutator.cs index 1392ff7726..1f2587d5d0 100644 --- a/src/Nncase.Core/Passes/Mutators/Mutator.cs +++ b/src/Nncase.Core/Passes/Mutators/Mutator.cs @@ -50,10 +50,4 @@ public static class Mutator /// /// RemoveNop. public static Func RemoveNop() => () => new Mutators.RemoveNop(); - - /// - /// fold math calc operator. - /// - /// FoldMathCall. - public static Func FoldMathCall() => () => new Mutators.FoldMathCall(); } diff --git a/src/Nncase.Core/Passes/Mutators/UnRollLoopSequential.cs b/src/Nncase.Core/Passes/Mutators/UnRollLoopSequential.cs index e0a043cc64..241899e1c5 100644 --- a/src/Nncase.Core/Passes/Mutators/UnRollLoopSequential.cs +++ b/src/Nncase.Core/Passes/Mutators/UnRollLoopSequential.cs @@ -144,7 +144,10 @@ public LoopBodyCloner(IReadOnlyDictionary vmap, Dictionary expr; + protected override Expr VisitLeafMemSpan(MemSpan expr, Unit context) + { + return expr.With(Clone(expr.Start, context), Clone(expr.Size, context)); + } protected override Expr VisitLeafVar(Var expr, Unit context) { @@ -189,9 +192,10 @@ protected override Expr VisitLeafRange(TIR.Range expr, Unit context) return CSE(expr.With(start: Clone(expr.Start, context), stop: Clone(expr.Stop, context), step: Clone(expr.Step, context))); } - protected override Expr VisitLeafLogicalBuffer(LogicalBuffer expr, Unit context) + protected override Expr VisitLeafBuffer(TIR.Buffer expr, Unit context) { return expr.With( + memSpan: Clone(expr.MemSpan, context), dimensions: CloneArray(expr.Dimensions, context).Select(e => CSE(e)).ToArray(), strides: CloneArray(expr.Strides, context)); } diff --git a/src/Nncase.Core/Schedule/ScheduleTypes.cs b/src/Nncase.Core/Schedule/ScheduleTypes.cs index 55b89ce8a0..6800f79fad 100644 --- a/src/Nncase.Core/Schedule/ScheduleTypes.cs +++ b/src/Nncase.Core/Schedule/ScheduleTypes.cs @@ -10,52 +10,6 @@ namespace Nncase.Schedule; -/// -/// the memory type. -/// -public enum MemoryLocation : byte -{ - /// - /// input. - /// - Input = 0, - - /// - /// output. - /// - Output = 1, - - /// - /// constant data. - /// - Rdata = 2, - - /// - /// compute temp data. - /// - Data = 3, - - /// - /// shared data. - /// - SharedData = 4, - - /// - /// l2 data. - /// - L2Data = 5, - - /// - /// L1 data. - /// - L1Data = 6, - - /// - /// base addr. - /// - PrivateBase = 64, -} - /// /// the scheduler interface. /// @@ -261,12 +215,12 @@ public SchedFunctionResult() /// /// Gets the buffer allocation. /// - public HashSet Rdatas { get; } + public Dictionary> Rdatas { get; } /// /// Gets or sets the data section length. /// - public int DataUsage { get; set; } + public long DataUsage { get; set; } /// /// Gets or sets a value indicating whether the Scheduled status. @@ -296,8 +250,8 @@ public override bool Equals(object? obj) return true; } - return EqualityComparer>.Default.Equals(Rdatas, result.Rdatas) && - EqualityComparer.Default.Equals(DataUsage, result.DataUsage); + return EqualityComparer>>.Default.Equals(Rdatas, result.Rdatas) && + EqualityComparer.Default.Equals(DataUsage, result.DataUsage); } /// diff --git a/src/Nncase.Core/TIR/Buffer.cs b/src/Nncase.Core/TIR/Buffer.cs index a3a35aac52..570a3d43d8 100644 --- a/src/Nncase.Core/TIR/Buffer.cs +++ b/src/Nncase.Core/TIR/Buffer.cs @@ -267,233 +267,58 @@ public SelectedRange Slice(Segment1D segment) /// /// buffer. /// -public abstract class Buffer : Expr +public sealed class Buffer : Expr { - public Buffer(string name, DataType elemType, Schedule.MemoryLocation memoryLocation, Expr[] operands) - : base(operands.ToArray()) + public Buffer(string name, DataType elemType, MemSpan memSpan, Expr[] dimensions, Expr[] strides) + : base(new[] { memSpan }.Concat(dimensions).Concat(strides)) { Name = name; ElemType = elemType; - MemLocation = memoryLocation; + Rank = dimensions.Length; } public string Name { get; } public DataType ElemType { get; } - public Schedule.MemoryLocation MemLocation { get; } - - /// - /// Gets if this buffer from the constant !. - /// - public TensorConst? Const { get; init; } - /// /// Gets rank of the tensor: number of dimensions. /// - public abstract int Rank { get; } - - /// - /// Gets the strides. - /// - /// This Strides is by elements not by bytes! - /// - /// - public abstract ReadOnlySpan Strides { get; } + public int Rank { get; } /// /// Gets the shape. /// - public abstract ReadOnlySpan Dimensions { get; } - - /// - public override bool Equals(object? obj) - { - if (obj is not Buffer other) - { - return false; - } - - if (Const is not null && !Const.Equals(other.Const)) - { - return false; - } - - return string.Equals(Name, other.Name, StringComparison.Ordinal) && - ElemType.Equals(other.ElemType) && - MemLocation.Equals(other.MemLocation) && - Rank.Equals(other.Rank) && - base.Equals(obj); - } -} - -/// -/// the logical buffer. -/// -public sealed class LogicalBuffer : Buffer -{ - /// - /// Initializes a new instance of the class. - /// create from the IRType. - /// - /// the name. - /// the location. - /// prim type. - /// the shape. - /// the strides. - public LogicalBuffer(string name, DataType elemType, Schedule.MemoryLocation location, ReadOnlySpan dimensions, ReadOnlySpan strides) - : base(name, elemType, location, ArrayUtility.Concat(dimensions, strides)) - { - Rank = dimensions.Length; - } - - /// - /// Initializes a new instance of the class. - /// . - /// - public LogicalBuffer(string name, Schedule.MemoryLocation location, TensorConst tensor) - : this(name, tensor.Value.ElementType, location, ArrayUtility.ToExprArray(tensor.Value.Dimensions), ArrayUtility.ToExprArray(tensor.Value.Strides)) - { - Const = tensor; - } - - /// - /// Initializes a new instance of the class. - /// - /// - public LogicalBuffer(string name, DataType elemType, Schedule.MemoryLocation location, ReadOnlySpan dimensions) - : this(name, elemType, location, dimensions, TensorUtilities.GetStrides(dimensions)) - { - } - - /// - /// Gets get the total length. - /// - public Expr Length => TensorUtilities.GetProduct(Dimensions); + public MemSpan MemSpan => (MemSpan)Operands[0]; /// /// Gets the shape. /// - public override ReadOnlySpan Dimensions => Operands[0..Rank]; + public ReadOnlySpan Dimensions => Operands[1..(1 + Rank)]; /// /// Gets the strides. + /// + /// This Strides is by elements not by bytes! + /// /// - public override ReadOnlySpan Strides => Operands[Rank..]; - - /// - public override int Rank { get; } - - /// - public override string ToString() - { - return $"LogicalBuffer({Name}, {ElemType}, {nameof(MemLocation)})"; - } - - /// - public override TExprResult Accept(ExprFunctor functor, TContext context) - => functor.VisitLogicalBuffer(this, context); - - public LogicalBuffer With(string? name = null, DataType? elemType = null, Schedule.MemoryLocation? location = null, Expr[]? dimensions = null, Expr[]? strides = null) - => new LogicalBuffer(name ?? Name, elemType ?? ElemType, location ?? MemLocation, dimensions ?? Dimensions, strides ?? Strides) { Const = Const }; -} - -/// -/// the physical buffer. -/// -public sealed class PhysicalBuffer : Buffer -{ - private readonly int[] _fixedDimensions; - private readonly int[] _fixedStrides; - - /// - /// Initializes a new instance of the class. - /// ctor for physical buffer. - /// - public PhysicalBuffer(string name, DataType elemType, Schedule.MemoryLocation location, ReadOnlySpan dimensions, ReadOnlySpan strides, int start, int size) - : base(name, elemType, location, Array.Empty()) - { - Start = start; - Size = size; - _fixedDimensions = dimensions.ToArray(); - _fixedStrides = strides.ToArray(); - } - - /// - /// Initializes a new instance of the class. - /// . - /// - public PhysicalBuffer(string name, DataType elemType, Schedule.MemoryLocation location, ReadOnlySpan dimensions, int start, int size) - : this(name, elemType, location, dimensions, TensorUtilities.GetStrides(dimensions), start, size) - { - } - - /// - /// Initializes a new instance of the class. - /// . - /// - public PhysicalBuffer(string name, Schedule.MemoryLocation location, TensorConst tensor, int start, int size) - : this(name, tensor.Value.ElementType, location, tensor.Value.Dimensions, tensor.Value.Strides, start, size) - { - Const = tensor; - } - - /// - /// Gets fixed dimensions. - /// - public ReadOnlySpan FixedDimensions => _fixedDimensions; - - /// - /// Gets fixed strides. - /// - public ReadOnlySpan FixedStrides => _fixedStrides; - - /// - /// Gets or sets start. - /// - public int Start { get; set; } - - /// - /// Gets total size in bytes. - /// - public int Size { get; init; } - - /// - /// Gets dimensions. - /// - public override ReadOnlySpan Dimensions => ArrayUtility.ToExprArray(FixedDimensions); - - /// - /// Gets strides. - /// - public override ReadOnlySpan Strides => ArrayUtility.ToExprArray(FixedStrides); - - /// - /// Gets shape. - /// - public Shape Shape => new Shape(FixedDimensions); + public ReadOnlySpan Strides => Operands[(1 + Rank)..(1 + Rank + Rank)]; - /// - public override int Rank => FixedDimensions.Length; + public override TExprResult Accept(ExprFunctor functor, TContext context) => functor.VisitBuffer(this, context); - /// - public override string ToString() - { - return $"PhysicalBuffer({Name}, {ElemType}, {nameof(MemLocation)})"; - } + public Buffer With(MemSpan? memSpan = null, Expr[]? dimensions = null, Expr[]? strides = null) + => new Buffer(Name, ElemType, memSpan ?? MemSpan, dimensions ?? Dimensions.ToArray(), strides ?? Strides.ToArray()); /// public override bool Equals(object? obj) { - return base.Equals(obj) && obj is PhysicalBuffer other && - FixedDimensions.SequenceEqual(other.FixedDimensions) && - FixedStrides.SequenceEqual(other.FixedStrides); - } + if (ReferenceEquals(this, obj)) + { + return true; + } - /// - public override TExprResult Accept(ExprFunctor functor, TContext context) - => functor.VisitPhysicalBuffer(this, context); + return obj is TIR.Buffer other && GetHashCode() == other.GetHashCode() && Name == other.Name && ElemType == other.ElemType && Rank == other.Rank && Operands.SequenceEqual(other.Operands); + } - public PhysicalBuffer With(string? name = null, DataType? elemType = null, Schedule.MemoryLocation? location = null, int[]? dimensions = null, int[]? strides = null, int? start = null, int? size = null) - => new PhysicalBuffer(name ?? Name, elemType ?? ElemType, location ?? MemLocation, dimensions ?? FixedDimensions, strides ?? FixedStrides, start ?? Start, size ?? Size) { Const = Const }; + protected override int GetHashCodeCore() => HashCode.Combine(Name, ElemType, Rank, base.GetHashCodeCore()); } diff --git a/src/Nncase.Core/TIR/BufferLoad.cs b/src/Nncase.Core/TIR/BufferLoad.cs deleted file mode 100644 index 86081624dd..0000000000 --- a/src/Nncase.Core/TIR/BufferLoad.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Canaan Inc. All rights reserved. -// Licensed under the Apache license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Nncase.IR; -using Nncase.Utilities; - -namespace Nncase.TIR; - -/// -/// Buffer load node. -/// -public sealed class BufferLoad : Expr -{ - public BufferLoad(PhysicalBuffer buffer, ReadOnlySpan indices) - : base(ArrayUtility.Concat(buffer, indices)) - { - } - - /// - /// Gets the buffer to be loaded. - /// - public PhysicalBuffer Buffer => (PhysicalBuffer)Operands[0]; - - /// - /// Gets the buffer indices. - /// - public ReadOnlySpan Indices => Operands.Slice(1); - - /// - public override TExprResult Accept(ExprFunctor functor, TContext context) - => functor.VisitBufferLoad(this, context); - - public BufferLoad With(PhysicalBuffer? buffer = null, Expr[]? indices = null) - => new BufferLoad(buffer ?? Buffer, indices ?? Indices); -} diff --git a/src/Nncase.Core/TIR/BufferStore.cs b/src/Nncase.Core/TIR/BufferStore.cs deleted file mode 100644 index 56d6a9df4d..0000000000 --- a/src/Nncase.Core/TIR/BufferStore.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Canaan Inc. All rights reserved. -// Licensed under the Apache license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Nncase.IR; - -namespace Nncase.TIR; - -/// -/// Buffer store node. -/// -public sealed class BufferStore : Expr -{ - private readonly int _indicesCount; - - public BufferStore(PhysicalBuffer buffer, ReadOnlySpan indices, Expr value) - : base(new Expr[] { buffer }.Concat(indices.ToArray()).Append(value).ToArray()) - { - _indicesCount = indices.Length; - } - - /// - /// Gets the buffer. - /// - public PhysicalBuffer Buffer => (PhysicalBuffer)Operands[0]; - - /// - /// Gets the value we to be stored. - /// - public ReadOnlySpan Indices => Operands[1.._indicesCount]; - - /// - /// Gets the indices location to be stored. - /// - public Expr Value => Operands[_indicesCount + 1]; - - /// - public override TExprResult Accept(ExprFunctor functor, TContext context) - => functor.VisitBufferStore(this, context); - - public BufferStore With(PhysicalBuffer? buffer = null, Expr[]? indices = null, Expr? value = null) - => new BufferStore(buffer ?? Buffer, indices ?? Indices, value ?? Value); -} diff --git a/src/Nncase.Core/TIR/MemSpan.cs b/src/Nncase.Core/TIR/MemSpan.cs new file mode 100644 index 0000000000..f8e537d549 --- /dev/null +++ b/src/Nncase.Core/TIR/MemSpan.cs @@ -0,0 +1,105 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase; +using Nncase.IR; + +namespace Nncase.TIR; + +/// +/// the memory type. +/// +[Flags] +public enum MemoryLocation +{ + /// + /// input. + /// + Input = 1 << 1, + + /// + /// output. + /// + Output = 1 << 2, + + /// + /// constant data. + /// + Rdata = 1 << 3, + + /// + /// compute temp data. + /// + Data = 1 << 4, + + /// + /// shared data. + /// + SharedData = 1 << 5, + + /// + /// l2 data. + /// + L2Data = 1 << 6, + + /// + /// L1 data. + /// + L1Data = 1 << 7, + + /// + /// base addr. + /// + PrivateBase = 1 << 8, +} + +public sealed class MemSpan : Expr +{ + public MemSpan(Expr size, MemoryLocation location) + : base(new[] { None.Default, size }) + { + Location = location; + } + + public MemSpan(Expr start, Expr size, MemoryLocation location) + : base(new[] { start, size }) + { + Location = location; + } + + /// + /// Gets the start. + /// + public Expr Start => Operands[0]; + + /// + /// Gets the size of bytes. + /// + public Expr Size => Operands[1]; + + /// + /// Gets the memory location. + /// + public MemoryLocation Location { get; } + + public MemSpan SubSpan(Expr offset, Expr size) => new MemSpan((Start is None ? IR.F.Buffer.DDrOf(this) : Start) + offset, size, Location); + + /// + public override TExprResult Accept(ExprFunctor functor, TContext context) + => functor.VisitMemSpan(this, context); + + public MemSpan With(Expr? start = null, Expr? size = null, MemoryLocation? location = null) => new(start ?? Start, size ?? Size, location ?? Location); + + /// + public override bool Equals(object? obj) + { + if (ReferenceEquals(this, obj)) + { + return true; + } + + return obj is MemSpan other && GetHashCode() == other.GetHashCode() && Location == other.Location && Operands.SequenceEqual(other.Operands); + } + + protected override int GetHashCodeCore() => HashCode.Combine(Location, base.GetHashCodeCore()); +} diff --git a/src/Nncase.Core/TIR/Ops.cs b/src/Nncase.Core/TIR/Ops.cs index 76f9e395b6..3405cdc841 100644 --- a/src/Nncase.Core/TIR/Ops.cs +++ b/src/Nncase.Core/TIR/Ops.cs @@ -12,19 +12,22 @@ namespace Nncase.TIR; /// -/// . +/// Load op. /// public sealed partial class Load : Op { /// /// Gets handle. /// - public static readonly ParameterInfo Handle = new(typeof(Load), 0, "handle"); + public static readonly ParameterInfo Handle = new(typeof(Load), 0, "handle", IsPointer() | IsIntegralScalar()); /// /// Gets index. /// - public static readonly ParameterInfo Index = new(typeof(Load), 1, "index", HasDataType(DataTypes.Int32) & (IsScalar() | HasRank(1))); + public static readonly ParameterInfo Index = new(typeof(Load), 1, "index", IsIntegralScalar()); + + /// + public override bool CanFoldConstCall => false; } /// @@ -53,17 +56,20 @@ public sealed partial class Store : Op /// /// The buffer variable handle. /// - public static readonly ParameterInfo Handle = new(typeof(Store), 0, "handle", IsPointer()); + public static readonly ParameterInfo Handle = new(typeof(Store), 0, "handle", IsPointer() | IsIntegralScalar()); /// /// The index locations to be stored. /// - public static readonly ParameterInfo Index = new(typeof(Store), 1, "index", HasDataType(DataTypes.Int32)); + public static readonly ParameterInfo Index = new(typeof(Store), 1, "index", IsIntegralScalar()); /// /// The value to be stored. /// - public static readonly ParameterInfo Value = new(typeof(Store), 2, "value"); + public static readonly ParameterInfo Value = new(typeof(Store), 2, "value", IsScalar()); + + /// + public override bool CanFoldConstCall => false; } /// diff --git a/src/Nncase.Core/TIR/PrimFunction.cs b/src/Nncase.Core/TIR/PrimFunction.cs index ea208efb15..2bf94454eb 100644 --- a/src/Nncase.Core/TIR/PrimFunction.cs +++ b/src/Nncase.Core/TIR/PrimFunction.cs @@ -28,8 +28,8 @@ public sealed class PrimFunction : BaseFunction /// module kind. /// Arguments. /// Body. - public PrimFunction(string name, string moduleKind, Sequential body, ReadOnlySpan parameters) - : base(name, moduleKind, ArrayUtility.Concat(body, SpanUtility.UnsafeCast(parameters))) + public PrimFunction(string name, string moduleKind, Sequential body, ReadOnlySpan parameters) + : base(name, moduleKind, ArrayUtility.Concat(body, SpanUtility.UnsafeCast(parameters))) { } @@ -39,7 +39,7 @@ public PrimFunction(string name, string moduleKind, Sequential body, ReadOnlySpa /// module kind. /// Arguments. /// Body. - public PrimFunction(string moduleKind, Sequential body, ReadOnlySpan parameters) + public PrimFunction(string moduleKind, Sequential body, ReadOnlySpan parameters) : this($"primfunc_{_globalFuncIndex++}", moduleKind, body, parameters) { } @@ -48,7 +48,7 @@ public PrimFunction(string moduleKind, Sequential body, ReadOnlySpan class. /// build function. /// - public PrimFunction(string moduleKind, Sequential body, params PhysicalBuffer[] parameters) + public PrimFunction(string moduleKind, Sequential body, params Buffer[] parameters) : this($"primfunc_{_globalFuncIndex++}", moduleKind, body, new(parameters)) { } @@ -58,7 +58,7 @@ public PrimFunction(string moduleKind, Sequential body, params PhysicalBuffer[] /// public Sequential Body => (Sequential)Operands[0]; - public ReadOnlySpan Parameters => SpanUtility.UnsafeCast(Operands.Slice(1)); + public ReadOnlySpan Parameters => SpanUtility.UnsafeCast(Operands.Slice(1)); public override IEnumerable ParameterTypes => Parameters.AsValueEnumerable().Select(x => x.CheckedType).ToArray(); @@ -66,7 +66,7 @@ public PrimFunction(string moduleKind, Sequential body, params PhysicalBuffer[] public override TExprResult Accept(ExprFunctor functor, TContext context) => functor.VisitPrimFunction(this, context); - public PrimFunction With(string? name = null, string? moduleKind = null, Sequential? body = null, PhysicalBuffer[]? parameters = null, Schedule.SchedFunctionResult? sched = null) + public PrimFunction With(string? name = null, string? moduleKind = null, Sequential? body = null, Buffer[]? parameters = null, Schedule.SchedFunctionResult? sched = null) => new PrimFunction(name ?? Name, moduleKind ?? ModuleKind, body ?? Body, parameters ?? Parameters) { // note maybe add SchedResult into ctor. diff --git a/src/Nncase.Core/TIR/Scheduler.cs b/src/Nncase.Core/TIR/Scheduler.cs index 214bd983f6..30e9552c04 100644 --- a/src/Nncase.Core/TIR/Scheduler.cs +++ b/src/Nncase.Core/TIR/Scheduler.cs @@ -87,7 +87,10 @@ public For[] Split(For loop, params Expr[] factors) } // TODO add assert total == (loop.Dom.Max - loop.Dom.Min) // Step 2. Replace all occurrences of the original loop var with new variables - Expr total = 1, substitute = 0; + _ = 1; + + // Step 2. Replace all occurrences of the original loop var with new variables + Expr substitute = 0; var newloopVars = new Var[factors.Length]; foreach (var i in Enumerable.Range(0, factors.Length)) { @@ -96,21 +99,23 @@ public For[] Split(For loop, params Expr[] factors) newloopVars[i] = loopVar; } - Dictionary opaque_block_reuse = new(); // TODO the opaque_block_reuse for what? - Sequential nbody = loop.Body; + _ = new + Dictionary(); // TODO the opaque_block_reuse for what? + _ = loop.Body; // Step 3. create new for loop. var nFor = new For[factors.Length]; - nbody = (Sequential)new Passes.Mutators.SubstituteVarAndCollectOpaqueBlock(v => v == loop.LoopVar ? substitute : v, opaque_block_reuse).Rewrite(nbody); - for (int i = factors.Length - 1; i >= 0; i--) - { - var @for = new For(newloopVars[i], (0, factors[i]), LoopMode.Serial, nbody); - nbody = T.Sequential(@for); - nFor[i] = @for; - } - // Setp 4. update the function - Entry = (Function)new Passes.Mutators.Substitutor(expr => object.ReferenceEquals(expr, loop) ? nFor[0] : null).Rewrite(Entry); + // nbody = (Sequential)new Passes.Mutators.SubstituteVarAndCollectOpaqueBlock(v => v == loop.LoopVar ? substitute : v, opaque_block_reuse).Rewrite(nbody); + // for (int i = factors.Length - 1; i >= 0; i--) + // { + // var @for = new For(newloopVars[i], (0, factors[i]), LoopMode.Serial, nbody); + // nbody = T.Sequential(@for); + // nFor[i] = @for; + // } + + // // Setp 4. update the function + // Entry = (Function)new Passes.Mutators.Substitutor(expr => object.ReferenceEquals(expr, loop) ? nFor[0] : null).Rewrite(Entry); return nFor; } diff --git a/src/Nncase.Core/TIR/Script.cs b/src/Nncase.Core/TIR/Script.cs index 9d9a212e46..28740e43ab 100644 --- a/src/Nncase.Core/TIR/Script.cs +++ b/src/Nncase.Core/TIR/Script.cs @@ -52,7 +52,7 @@ public static class T /// /// The buffer handle variable in the load expression. /// The index in the load. - public static Call Load(Var handle, Expr index) => new Call(new Load(), handle, index); + public static Call Load(Expr handle, Expr index) => new Call(new Load(), handle, index); /// /// get the nop op. @@ -76,25 +76,7 @@ public static class T /// The buffer Variable. /// The index in the store expression. /// The value we want to store. - public static Call Store(Var handle, Expr index, Expr value) => new Call(new Store(), handle, index, value); - - /// - /// If the op is BufferLoad, it will return BufferStore - /// If the op is Load, it will return Store. - /// - /// the op call. - /// update value. - /// new store call. - public static Expr Store(Expr op, Expr value) => op switch - { - Call load => load.Target switch - { - TIR.Load => T.Store((Var)load[TIR.Load.Handle], load[TIR.Load.Index], value), - _ => throw new InvalidOperationException("Only Can build Store Op from Load!"), - }, - TIR.BufferLoad bufload => new BufferStore(bufload.Buffer, bufload.Indices, value), - _ => throw new InvalidOperationException("Only Can build Store Op from Load!"), - }; + public static Call Store(Expr handle, Expr index, Expr value) => new Call(new Store(), handle, index, value); /// /// build for loop. @@ -202,7 +184,7 @@ public static ISequentialBuilder Sequential() /// )); /// /// - public static ISequentialBuilder PrimFunc(string name, string module_kind, params PhysicalBuffer[] parameters) + public static ISequentialBuilder PrimFunc(string name, string module_kind, params Buffer[] parameters) { return new SequentialBuilder(body => new PrimFunction(name, module_kind, body, parameters)); } @@ -224,54 +206,73 @@ public static IIfThenElseBuilder If(Expr condition) } /// - /// create the memRef by tensortype. + /// create the buffer by tensortype. /// - public static LogicalBuffer Buffer(DataType elem_type, Schedule.MemoryLocation location, ReadOnlySpan dimensions, out LogicalBuffer buffer, [CallerArgumentExpression("buffer")] string name = "") + public static Buffer CreateBuffer(TensorType tensorType, MemoryLocation location, out Buffer buffer, [CallerArgumentExpression("buffer")] string name = "") { if (name.StartsWith("var ")) { name = name[4..]; } - buffer = new LogicalBuffer(name, elem_type, location, dimensions); + var dimensions = tensorType.Shape.ToValueArray(); + var strides = TensorUtilities.GetStrides(dimensions); + var size = (int)TensorUtilities.GetProduct(dimensions.ToArray()) * tensorType.DType.SizeInBytes; + var memspan = new MemSpan(size, location); + buffer = new Buffer(name, tensorType.DType, memspan, dimensions.Select(i => (Expr)i).ToArray(), strides.Select(i => (Expr)i).ToArray()); return buffer; } /// - /// ctor for physical buffer. + /// create buffer by const. /// - public static PhysicalBuffer PhysicalBuffer(DataType elem_type, Schedule.MemoryLocation location, ReadOnlySpan dimensions, out PhysicalBuffer buffer, [CallerArgumentExpression("buffer")] string name = "") + public static Buffer AttachBuffer(TensorConst @const, out Buffer buffer, [CallerArgumentExpression("buffer")] string name = "") { if (name.StartsWith("var ")) { name = name[4..]; } - buffer = new PhysicalBuffer(name, elem_type, location, dimensions, 0, (int)TensorUtilities.GetProduct(dimensions.ToArray()) * elem_type.SizeInBytes); + var dimensions = @const.ValueType.Shape.ToValueArray(); + var strides = TensorUtilities.GetStrides(dimensions); + var size = (int)TensorUtilities.GetProduct(dimensions.ToArray()) * @const.ValueType.DType.SizeInBytes; + var memspan = new MemSpan(IR.F.Buffer.DDrOf(@const), size, MemoryLocation.Rdata); + buffer = new Buffer(name, @const.ValueType.DType, memspan, dimensions.Select(i => (Expr)i).ToArray(), strides.Select(i => (Expr)i).ToArray()); return buffer; } /// - /// create buffer from const. + /// attach the buffer. /// - public static PhysicalBuffer ConstBuffer(Const expr, out PhysicalBuffer buffer, [CallerArgumentExpression("buffer")] string name = "") + public static Buffer AttachBuffer(Buffer originBuffer, Expr offset, TensorType tensorType, out Buffer buffer, [CallerArgumentExpression("buffer")] string name = "") { if (name.StartsWith("var ")) { name = name[4..]; } - int size; - if (expr is TensorConst tc) - { - size = tc.Value.BytesBuffer.Length; - } - else + var dimensions = tensorType.Shape.ToValueArray(); + var strides = TensorUtilities.GetStrides(dimensions); + var size = (int)TensorUtilities.GetProduct(dimensions.ToArray()) * tensorType.DType.SizeInBytes; + buffer = new Buffer(name, tensorType.DType, originBuffer.MemSpan.SubSpan(offset, size), dimensions.Select(i => (Expr)i).ToArray(), strides.Select(i => (Expr)i).ToArray()); + return buffer; + } + + /// + /// attach the buffer. + /// + public static Buffer AttachBuffer(TensorType tensorType, MemoryLocation location, out Var @var, out Buffer buffer, [CallerArgumentExpression("buffer")] string name = "") + { + if (name.StartsWith("var ")) { - throw new NotSupportedException(); + name = name[4..]; } - buffer = new PhysicalBuffer(name, Schedule.MemoryLocation.Rdata, (TensorConst)expr, 0, size); + @var = new Var(TensorType.Pointer(tensorType.DType)); + var dimensions = tensorType.Shape.ToValueArray(); + var strides = TensorUtilities.GetStrides(dimensions); + var size = (int)TensorUtilities.GetProduct(dimensions.ToArray()) * tensorType.DType.SizeInBytes; + buffer = new Buffer(name, tensorType.DType, new MemSpan(@var, size, location), dimensions.Select(i => (Expr)i).ToArray(), strides.Select(i => (Expr)i).ToArray()); return buffer; } @@ -294,7 +295,7 @@ public static Expr MayBeConst(Const? expr, out Buffer? buffer, [CallerArgumentEx { name = name[4..]; } - buffer = new Buffer(name, Schedule.MemoryLocation.Rdata, (TensorType)expr.ValueType) + buffer = new Buffer(name, MemoryLocation.Rdata, (TensorType)expr.ValueType) { Const = expr, }; @@ -331,4 +332,23 @@ public static Call Emit(out T value, Func creator) value = creator(); return Nop(); } + + /// + /// buffer load. + /// + /// buffer. + /// indices. + /// call bufferload. + public static Call BufferLoad(TIR.Buffer buffer, params Expr[] indices) => new Call(new IR.Buffers.BufferLoad(), buffer, new IR.Tuple(indices)); + + /// + /// buffer store. + /// + /// buffer. + /// indices. + /// value. + /// call bufferstore. + public static Call BufferStore(TIR.Buffer buffer, Expr[] indices, Expr value) => new Call(new IR.Buffers.BufferStore(), buffer, new IR.Tuple(indices), value); + + public static Call MatchBuffer(TIR.Buffer buffer) => new Call(new IR.Buffers.MatchBuffer(), buffer); } diff --git a/src/Nncase.Core/Tensor.cs b/src/Nncase.Core/Tensor.cs index 6747cddee8..78f3e5e999 100644 --- a/src/Nncase.Core/Tensor.cs +++ b/src/Nncase.Core/Tensor.cs @@ -342,18 +342,18 @@ public static unsafe Tensor FromArray(Array array) public static Tensor> FromPointer(ulong value) where T : unmanaged, IEquatable { - return Tensor.FromScalar>(new Pointer(value)); + return FromScalar>(new Pointer(value)); } /// /// Create tensor from a ulong address. /// /// addr value. - /// Element type. + /// pointed type. /// Created tensor. public static Tensor FromPointer(ulong value, DataType elemType) { - return Tensor.FromBytes(TensorType.Scalar(new PointerType(elemType)), BitConverter.GetBytes(value)); + return FromBytes(TensorType.Scalar(new PointerType(elemType)), BitConverter.GetBytes(value)); } /// diff --git a/src/Nncase.Core/TensorUtilities.cs b/src/Nncase.Core/TensorUtilities.cs index 717274e0d3..79f658aefa 100644 --- a/src/Nncase.Core/TensorUtilities.cs +++ b/src/Nncase.Core/TensorUtilities.cs @@ -323,10 +323,11 @@ public static bool IsContiguous(ReadOnlySpan dimensions, ReadOnlySpan /// /// check the dimensions selected range is contiguous. /// - public static bool IsContiguousSlice(ReadOnlySpan dimensions, ReadOnlySpan slices) + public static bool IsContiguousSlice(ReadOnlySpan dimensions, ReadOnlySpan slices, out int contiguousStart) { if (dimensions.Length != slices.Length) { + contiguousStart = slices.Length - 1; return false; } @@ -366,13 +367,17 @@ public static bool IsContiguousSlice(ReadOnlySpan dimensions, ReadOnlySpan< }; if (status == SliceStatus.IsInvalid) { + contiguousStart = i + 1; return false; } } + contiguousStart = 0; return true; } + public static bool IsContiguousSlice(ReadOnlySpan dimensions, ReadOnlySpan slices) => IsContiguousSlice(dimensions, slices, out _); + public static long[] ToLongs(this ReadOnlySpan ints) { var longs = new long[ints.Length]; diff --git a/src/Nncase.Core/Utilities/DistributedUtility.cs b/src/Nncase.Core/Utilities/DistributedUtility.cs new file mode 100644 index 0000000000..13b2870bfb --- /dev/null +++ b/src/Nncase.Core/Utilities/DistributedUtility.cs @@ -0,0 +1,144 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.CodeAnalysis; +using Nncase.IR; + +namespace Nncase.Utilities; + +public static class DistributedUtility +{ + public static IReadOnlyList> GetLeafCandidateNDSBPs(TensorType tensorType, Placement placement) + { + var ndsbps = new List>(); + for (int i = 0; i < placement.Rank; i++) + { + var ndsbp = new List(); + for (int axis = 0; axis < tensorType.Shape.Rank; axis++) + { + if (tensorType.Shape[axis] is { IsFixed: true, Value: int s } && IsDivisible(s, placement.Hierarchy[i])) + { + ndsbp.Add(SBP.S(axis)); + } + } + + ndsbp.Add(SBP.B); + ndsbps.Add(ndsbp); + } + + return ndsbps.CartesianProduct(). + Select(ndsbp => ndsbp.ToArray()). + Where(ndsbp => IsDistributable(tensorType, ndsbp, placement, out _)). + Select(ndsbp => new IRArray(ndsbp)). + ToArray(); + } + + public static IReadOnlyList> GetPartialCandidateNDSBPs(DistributedType distributedType) + { + IRArray ndsbp = distributedType.NdSBP; + TensorType tensorType = distributedType.TensorType; + Placement placement = distributedType.Placement; + if (!ndsbp.Any(sbp => sbp is SBPPartialSum)) + { + return Array.Empty>(); + } + + var candidateNdsbps = new List[placement.Rank]; + for (int i = 0; i < placement.Rank; i++) + { + candidateNdsbps[i] = new List(); + var innerSplitedAxes = distributedType.NdSBP.Skip(i + 1).OfType().Select(sbp => sbp.Axis).ToList(); + if (ndsbp[i] is SBPPartialSum) + { + candidateNdsbps[i].Add(SBP.B); + for (int axis = 0; axis < tensorType.Shape.Rank; axis++) + { + if (tensorType.Shape[axis] is { IsFixed: true, Value: int s } && IsDivisible(s, placement.Hierarchy[i]) && !innerSplitedAxes.Contains(axis)) + { + candidateNdsbps[i].Add(SBP.S(axis)); + } + } + } + else + { + candidateNdsbps[i].Add(ndsbp[i]); + } + } + + return candidateNdsbps.CartesianProduct(). + Select(ndsbp => ndsbp.ToArray()). + Where(ndsbp => IsDistributable(tensorType, ndsbp, placement, out _)). + Select(ndsbp => new IRArray(ndsbp)). + ToArray(); + } + + public static bool IsDistributable(TensorType tensorType, ReadOnlySpan ndsbp, Placement placement, [MaybeNullWhen(false)] out TensorType distType) + { + distType = null; + if (!tensorType.Shape.IsFixed) + { + return false; + } + + var shape = tensorType.Shape.ToValueArray(); + for (int i = 0; i < ndsbp.Length; i++) + { + if (ndsbp[i] is SBPSplit { Axis: int axis }) + { + if (!IsDivisible(shape[axis], placement.Hierarchy[i])) + { + return false; + } + + shape[axis] /= placement.Hierarchy[i]; + } + } + + distType = tensorType with { Shape = shape }; + return true; + } + + public static bool IsDivisible(int input, int divisor) + { + if (input >= divisor && input % divisor == 0) + { + return true; + } + + return false; + } + + public static float GetDividedTensorEfficiency(DistributedType distributedType, int burstLength) + { + var (tiles, shape) = GetDividedTile(distributedType); + return Enumerable.Range(0, tiles.Count). + Select(i => tiles[i].Ranges(0, shape[i])). + CartesianProduct(). + Select(rgs => + { + var slice = rgs.ToArray(); + var iscontiguous = TensorUtilities.IsContiguousSlice(shape.ToArray(), slice, out var contiguousStart); + var size = TensorUtilities.GetProduct(tiles.ToArray(), contiguousStart) * distributedType.TensorType.DType.SizeInBytes; + var (div, rem) = Math.DivRem(size, burstLength); + return ((div * 1.0f) + ((float)rem / burstLength)) / (div + 1); + }).Average(); + } + + public static TensorType GetDividedTensorType(DistributedType distributedType) + { + var (tiles, _) = GetDividedTile(distributedType); + return distributedType.TensorType with { Shape = new Shape(tiles) }; + } + + private static (IReadOnlyList Tile, IReadOnlyList Shape) GetDividedTile(DistributedType distributedType) + { + var shape = distributedType.TensorType.Shape.ToValueArray(); + var tiles = distributedType.TensorType.Shape.ToValueArray(); + foreach (var (s, i) in distributedType.NdSBP.Select((s, i) => (s, i)).Where(t => t.s is SBPSplit).Select(t => ((SBPSplit)t.s, t.i))) + { + tiles[s.Axis] /= distributedType.Placement.Hierarchy[i]; + } + + return (tiles, shape); + } +} diff --git a/src/Nncase.Core/packages.lock.json b/src/Nncase.Core/packages.lock.json index b2543377b5..b846e4b245 100644 --- a/src/Nncase.Core/packages.lock.json +++ b/src/Nncase.Core/packages.lock.json @@ -60,13 +60,19 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, + "System.CommandLine": { + "type": "Direct", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "Direct", "requested": "[5.0.0, )", @@ -109,8 +115,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", diff --git a/src/Nncase.Diagnostics/Diagnostics/Dumpper.cs b/src/Nncase.Diagnostics/Diagnostics/Dumpper.cs index 399ba10b28..8bd3794074 100644 --- a/src/Nncase.Diagnostics/Diagnostics/Dumpper.cs +++ b/src/Nncase.Diagnostics/Diagnostics/Dumpper.cs @@ -52,6 +52,12 @@ public void DumpCSharpIR(Expr expr, string prefix, string? reletivePath = null) CompilerServices.DumpCSharpIR(expr, prefix, EnsureWritable(path)); } + public void DumpPatternIR(Expr expr, string prefix, string? reletivePath = null) + { + var path = Path.Join(_dumpDirectory, reletivePath); + CompilerServices.DumpPatternIR(expr, prefix, EnsureWritable(path)); + } + public void DumpModule(IRModule module, string? reletivePath = null) { foreach (var func in module.Functions) diff --git a/src/Nncase.Diagnostics/Diagnostics/ILDotPrintVisitor.cs b/src/Nncase.Diagnostics/Diagnostics/ILDotPrintVisitor.cs index 409a4846af..c20883fc05 100644 --- a/src/Nncase.Diagnostics/Diagnostics/ILDotPrintVisitor.cs +++ b/src/Nncase.Diagnostics/Diagnostics/ILDotPrintVisitor.cs @@ -363,13 +363,13 @@ protected override ILDotOption VisitCall(Call expr) _ => throw new NotSupportedException($"Target type {expr.Target.GetType()} is not supported."), })) { - if (child is Const or None) + if (child is None) { continue; } var portName = $"P{count++}"; - row.AddCell(arg_name, cell => cell.PortName = portName); + row.AddCell(child switch { Const c => c.CheckedType.ToString(), _ => arg_name }, cell => cell.PortName = portName); connect_list.Add((child, portName)); } }); @@ -385,7 +385,7 @@ protected override ILDotOption VisitCall(Call expr) // 4. connect edge. foreach (var (child, port_name) in connect_list) { - if (child is BaseFunction) + if (child is BaseFunction or Const) { continue; } diff --git a/src/Nncase.Diagnostics/Diagnostics/ILPrintVisitor.cs b/src/Nncase.Diagnostics/Diagnostics/ILPrintVisitor.cs index 93fa794679..4a447073b8 100644 --- a/src/Nncase.Diagnostics/Diagnostics/ILPrintVisitor.cs +++ b/src/Nncase.Diagnostics/Diagnostics/ILPrintVisitor.cs @@ -274,6 +274,33 @@ public override string VisitType(CallableType type) => public override string VisitType(TupleType type) => $"({string.Join(", ", type.Fields.Select(VisitType))})"; + /// + public override string VisitType(DistributedType type) + { + var shape = type.TensorType.Shape.ToArray(); + foreach (var (s, r) in type.NdSBP.Select((s, r) => (s, r))) + { + if (s is SBPSplit split) + { + if (shape[split.Axis].IsFixed) + { + shape[split.Axis] = shape[split.Axis] / type.Placement.Hierarchy[r]; + } + } + } + + var sshape = shape.Select(s => s.ToString()).ToArray(); + foreach (var (s, r) in type.NdSBP.Select((s, r) => (s, r))) + { + if (s is SBPSplit split) + { + sshape[split.Axis] += $"@{type.Placement.Name[r]}"; + } + } + + return $"{{{VisitType(type.TensorType)}, ({string.Join(',', type.NdSBP)}), [{string.Join(',', sshape)}]}}"; + } + /// protected override string VisitCall(Call expr) { @@ -449,13 +476,7 @@ protected override string VisitPrimFunctionWrapper(PrimFunctionWrapper expr) /// protected override string VisitOp(Op expr) { - return expr switch - { - Unary op => op.UnaryOp.ToString(), - Binary op => op.BinaryOp.ToString(), - Compare op => op.CompareOp.ToString(), - _ => expr.GetType().Name, - }; + return expr.GetType().Name; } /// diff --git a/src/Nncase.Diagnostics/Diagnostics/IRPrinterProvider.cs b/src/Nncase.Diagnostics/Diagnostics/IRPrinterProvider.cs index d396e9de74..9d2b7c87a9 100644 --- a/src/Nncase.Diagnostics/Diagnostics/IRPrinterProvider.cs +++ b/src/Nncase.Diagnostics/Diagnostics/IRPrinterProvider.cs @@ -102,6 +102,25 @@ public void DumpCSharpIR(Expr expr, string prefix, string dumpDir, bool randCons } } + /// + public void DumpPatternIR(Expr expr, string prefix, string dumpDir) + { + var nprefix = prefix.Any() ? prefix + "_" : prefix; + string ext = "cs"; + string name = expr is Callable c ? c.Name : expr.GetType().Name; + string file_path = Path.Combine(dumpDir, $"{nprefix}{name}.{ext}"); + if (string.IsNullOrEmpty(dumpDir)) + { + throw new ArgumentException("The dumpDir Is Empty!"); + } + + Directory.CreateDirectory(dumpDir); + + using var dumpFile = File.Open(file_path, FileMode.Create); + using var dumpWriter = new StreamWriter(dumpFile); + new PatternPrintVisitor(dumpWriter, 0).Visit(expr); + } + /// public string Print(IRType type) { diff --git a/src/Nncase.Diagnostics/Diagnostics/PatternPrintVisitor.cs b/src/Nncase.Diagnostics/Diagnostics/PatternPrintVisitor.cs new file mode 100644 index 0000000000..3b8d5df531 --- /dev/null +++ b/src/Nncase.Diagnostics/Diagnostics/PatternPrintVisitor.cs @@ -0,0 +1,245 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.TIR; +using Nncase.Utilities; + +namespace Nncase.Diagnostics; + +internal sealed class PatternPrintVisitor : ExprFunctor +{ + private readonly ScopeWriter _scope; + private readonly Dictionary _names = new Dictionary(ReferenceEqualityComparer.Instance); + private int _localId; + + public PatternPrintVisitor(TextWriter textWriter, int indentLevel) + { + _scope = new(textWriter, indentLevel); + } + + /// + public override string VisitType(AnyType type) => "any"; + + /// + public override string VisitType(CallableType type) => + $"({string.Join(", ", type.Parameters.Select(VisitType))}) -> {VisitType(type.ReturnType)}"; + + /// + public override string VisitType(InvalidType type) => $"invalid:{type.Reason}"; + + /// + public override string VisitType(NoneType type) => $""; + + /// + public override string VisitType(TensorType type) => type.DType switch + { + PrimType ptype => ptype.GetDisplayName() + (type.Shape.IsScalar ? string.Empty : type.Shape.ToString()), + PointerType { ElemType: PrimType etype } => $"*{etype.GetDisplayName()}", + ValueType => $"{type.DType.ToString()}", + _ => throw new NotSupportedException(type.DType.GetType().Name), + }; + + /// + public override string VisitType(TupleType type) => + $"({string.Join(", ", type.Fields.Select(VisitType))})"; + + /// + protected override string VisitCall(Call expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + var target = Visit(expr.Target); + var args = expr.Arguments.AsValueEnumerable().Select(Visit).ToArray(); + name = AllocateTempVar(expr); + _scope.IndWrite($"var {name} = IsCall(\"{name}\", IsOp<{expr.Target.GetType().Name}>(), IsVArgs({string.Join(",", args)}));\n"); + + // AppendCheckedType(expr.CheckedType); + return name; + } + + /// + protected override string VisitConst(Const expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + name = AllocateTempVar(expr); + _scope.IndWrite($"var {name} = IsTensorConst(\"{name}\");\n"); + return name; + } + + /// + protected override string VisitFunction(Function expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + name = AllocateTempVar(expr); + _scope.Push(); + + // 1. functionv var + _scope.IndWrite($"Function {name}"); + AppendCheckedType(expr.CheckedType); + + // 2. Function body + _scope.IndWriteLine("{"); + using (_scope.IndentUp()) + { + var body = Visit(expr.Body); + + // _scope.IndWriteLine($"{name} = new Function(\"{expr.Name}\", {body}, new Var[] {{{StringUtility.Join(", ", expr.Parameters.AsValueEnumerable().Select(Visit))}}});"); + } + + // 3. Function signature + _scope.IndWriteLine("}"); + _scope.Append(_scope.Pop()); + return name; + } + + protected override string VisitFusion(Fusion expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + name = AllocateTempVar(expr); + + _scope.IndWrite($"Fusion {name}"); + AppendCheckedType(expr.CheckedType); + _scope.Push(); + _scope.IndWriteLine("{"); + using (_scope.IndentUp()) + { + var body_builder = new StringBuilder(); + string body; + using (var body_writer = new StringWriter(body_builder)) + { + var visitor = new PatternPrintVisitor(body_writer, _scope.IndentLevel) { _localId = _localId }; + body = visitor.Visit(expr.Body); + _scope.Append(body_writer.ToString()); + } + + _scope.IndWriteLine($"{name} = new Fusion(\"{expr.Name}\", \"{expr.ModuleKind}\", {body}, new Var[] {{{StringUtility.Join(", ", expr.Parameters.AsValueEnumerable().Select(Visit))}}});"); + } + + _scope.IndWriteLine("}"); + _scope.Append(_scope.Pop()); + return name; + } + + /// + protected override string VisitOp(Op expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + name = $"new {expr.GetType().Name}({expr.DisplayProperty()})"; + _names.Add(expr, name); + return name; + } + + /// + protected override string VisitTuple(IR.Tuple expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + var fields = expr.Fields.AsValueEnumerable().Select(Visit).ToArray(); + name = AllocateTempVar(expr); + _scope.IndWrite($"var {name} = IsTuple(\"{name}\", IsVArgs({string.Join(",", fields)}));\n"); + + // AppendCheckedType(expr.CheckedType); + _scope.IndWriteLine(); + return name; + } + + /// + protected override string VisitVar(Var expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + name = AllocateTempVar(expr); + _scope.IndWriteLine($"var {name} = IsWildcard(\"{expr.Name}\");\n"); + return name; + } + + /// + protected override string VisitNone(None expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + name = $"None.Default"; + _names.Add(expr, name); + return name; + } + + /// + protected override string VisitMarker(Marker expr) + { + if (_names.TryGetValue(expr, out var name)) + { + return name; + } + + var target = Visit(expr.Target); + var attr = Visit(expr.Attribute); + name = AllocateTempVar(expr); + _scope.IndWrite($"var {name} = new Marker(\"{expr.Name}\",{target},{attr})"); + AppendCheckedType(expr.CheckedType); + return name; + } + + private string AllocateTempVar(Expr expr) + { + var name = $"v{_localId++}"; + _names.Add(expr, name); + return name; + } + + private void AppendCheckedType(IRType? type, string end = "", bool hasNewLine = true) + { + if (type is not null) + { + if (hasNewLine) + { + _scope.AppendLine($"; // {VisitType(type)}{end}"); + } + else + { + _scope.Append($"; // {VisitType(type)}{end}"); + } + } + else + { + _scope.Append(";\n"); + } + } +} diff --git a/src/Nncase.Diagnostics/Diagnostics/ScriptPrintVisitor.cs b/src/Nncase.Diagnostics/Diagnostics/ScriptPrintVisitor.cs index 62dac6f69c..ac4b0e3a50 100644 --- a/src/Nncase.Diagnostics/Diagnostics/ScriptPrintVisitor.cs +++ b/src/Nncase.Diagnostics/Diagnostics/ScriptPrintVisitor.cs @@ -216,6 +216,22 @@ protected override IPrintSymbol VisitTuple(IR.Tuple expr) return doc; } + protected override IPrintSymbol VisitMemSpan(MemSpan expr) + { + if (_exprMemo.TryGetValue(expr, out var doc)) + { + return doc; + } + + var start = Visit(expr.Start); + var size = Visit(expr.Size); + _scope.Push(); + _scope.Append($"MemSpan({start}, {size})@{expr.Location}"); + doc = new(_scope.Pop()); + _exprMemo.Add(expr, doc); + return doc; + } + /// protected override IPrintSymbol VisitMarker(Marker expr) { @@ -279,18 +295,17 @@ protected override IPrintSymbol VisitTensorConst(TensorConst @const) { doc = new(new($"{@const}")); } - else - if (@const.Value.ElementType.IsFloat()) + else if (@const.Value.ElementType.IsFloat()) { - doc = new(new($"{string.Join(",", @const.Value.ToArray())}")); + doc = new(new(@const.Value.Length > 8 ? @const.CheckedShape.ToString() : $"{string.Join(",", @const.Value.ToArray())}")); } else if (@const.Value.ElementType.IsIntegral()) { - doc = new(new($"{string.Join(",", @const.Value.ToArray())}")); + doc = new(new(@const.Value.Length > 8 ? @const.CheckedShape.ToString() : $"{string.Join(",", @const.Value.ToArray())}")); } - else if (@const.Value.ElementType.IsPointer()) + else if (@const.Value.ElementType is PointerType p) { - doc = new(new($"{string.Join(",", @const.Value.ToArray().Select(i => "0x" + i.ToString("X")))}")); + doc = new(new($"*{p.ElemType.GetDisplayName()}@{@const.Value.Shape}")); } _exprMemo.Add(@const, doc!); @@ -481,34 +496,6 @@ protected override IPrintSymbol VisitBlock(Block expr) return doc; } - /// - protected override IPrintSymbol VisitBufferLoad(BufferLoad expr) - { - if (_exprMemo.TryGetValue(expr, out var doc)) - { - return doc; - } - - _scope.Push(); - _scope.Append($"{expr.Buffer.Name}[{string.Join(", ", expr.Indices.ToArray().Select(Visit))}]"); - doc = new(_scope.Pop()); - return doc; - } - - /// - protected override IPrintSymbol VisitBufferStore(BufferStore expr) - { - if (_exprMemo.TryGetValue(expr, out var doc)) - { - return doc; - } - - _scope.Push(); - _scope.Append($"{expr.Buffer.Name}[{string.Join(", ", expr.Indices.ToArray().Select(Visit))}] = {Visit(expr.Value)}"); - doc = new(_scope.Pop()); - return doc; - } - /// protected override IPrintSymbol VisitIterVar(IterVar expr) { @@ -570,12 +557,8 @@ protected override IPrintSymbol VisitBuffer(TIR.Buffer expr) } _scope.Push(); - _scope.Append($"T.Buffer({expr.Name}, {expr.MemLocation}, {VisitType(expr.ElemType)})"); - if (expr is TIR.PhysicalBuffer phy) - { - _scope.Append($"@({phy.Start}, {phy.Size})"); - } - + var memSpan = Visit(expr.MemSpan); + _scope.Append($"T.Buffer({expr.Name}, {VisitType(expr.ElemType)}, {memSpan.Span}, [{string.Join(',', expr.Dimensions.AsValueEnumerable().Select(Visit).Select(e => e.Span.ToString()).ToArray())}], [{string.Join(',', expr.Strides.AsValueEnumerable().Select(Visit).Select(e => e.Span.ToString()).ToArray())}])"); doc = new(_scope.Pop(), expr.Name, true); _exprMemo.Add(expr, doc); return doc; diff --git a/src/Nncase.Diagnostics/packages.lock.json b/src/Nncase.Diagnostics/packages.lock.json index 93fabe1e48..1b9a6c1dd5 100644 --- a/src/Nncase.Diagnostics/packages.lock.json +++ b/src/Nncase.Diagnostics/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.Extensions.Configuration.Abstractions": { @@ -47,8 +47,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -70,6 +70,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -129,6 +130,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.EGraph/Passes/EGraphExtractExtensions.cs b/src/Nncase.EGraph/Passes/EGraphExtractExtensions.cs index bbafcac29c..7c0cfbdc26 100644 --- a/src/Nncase.EGraph/Passes/EGraphExtractExtensions.cs +++ b/src/Nncase.EGraph/Passes/EGraphExtractExtensions.cs @@ -26,10 +26,11 @@ public static class EGraphExtractExtensions /// eGraph. /// Root eclass. /// base func cost evaluator. + /// the picks. /// Extracted root expression. - public static Expr Extract(this IEGraph eGraph, EClass root, Evaluator.IBaseFuncCostEvaluator? basefunc_cost_evaluator) + public static Expr Extract(this IEGraph eGraph, EClass root, Evaluator.IBaseFuncCostEvaluator? basefunc_cost_evaluator, out IReadOnlyDictionary picks) { - // 1. set the all expr checked shape + // 1. set enode expr with more accuracy type. foreach (var eclass in eGraph.Classes) { foreach (var nodes in eclass.Nodes) @@ -50,7 +51,7 @@ public static Expr Extract(this IEGraph eGraph, EClass root, Evaluator.IBaseFunc // EGraphPrinter.DumpEgraphAsDot(eGraph, costModel, root.Find(), fs); // } // return new EGraphExtractor(costModel).Extract(root.Find(), eGraph); - return new EGraphExtractors.SatExtractor(costModel).Extract(root.Find(), eGraph); + return new EGraphExtractors.SatExtractor(costModel).Extract(root.Find(), eGraph, out picks); } /// diff --git a/src/Nncase.EGraph/Passes/EGraphExtractors/Extractor.cs b/src/Nncase.EGraph/Passes/EGraphExtractors/Extractor.cs index c15dc50d83..fcf2abd729 100644 --- a/src/Nncase.EGraph/Passes/EGraphExtractors/Extractor.cs +++ b/src/Nncase.EGraph/Passes/EGraphExtractors/Extractor.cs @@ -17,7 +17,7 @@ namespace Nncase.Passes.EGraphExtractors; internal interface IExtractor { - Expr Extract(EClass root, IEGraph eGraph); + Expr Extract(EClass root, IEGraph eGraph, out IReadOnlyDictionary picks); } internal class Extractor : IExtractor @@ -25,6 +25,7 @@ internal class Extractor : IExtractor private readonly EGraphCostModel _costModel; private readonly Dictionary _eclassMemo = new(); private readonly Dictionary _markerEclassMemo = new(); + private readonly Dictionary _picks = new(); private StreamWriter? _dumpWriter; public Extractor(EGraphCostModel costModel) @@ -32,7 +33,7 @@ public Extractor(EGraphCostModel costModel) _costModel = costModel; } - public Expr Extract(EClass root, IEGraph eGraph) + public Expr Extract(EClass root, IEGraph eGraph, out IReadOnlyDictionary picks) { _dumpWriter = DumpScope.Current.IsEnabled(DumpFlags.EGraphCost) ? new StreamWriter(DumpScope.Current.OpenFile($"{nameof(Extractor)}_Class_{root.Id}.txt")) @@ -46,6 +47,15 @@ public Expr Extract(EClass root, IEGraph eGraph) _dumpWriter?.Dispose(); } + foreach (var enode in eGraph.Nodes) + { + if (!_picks.ContainsKey(enode)) + { + _picks[enode] = false; + } + } + + picks = _picks; return _eclassMemo[root]; } @@ -132,6 +142,7 @@ private void Visit(EClass eclass) _eclassMemo.Add(eclass, expr); } + _picks[minCostEnode] = true; stack.Pop(); } } diff --git a/src/Nncase.EGraph/Passes/EGraphExtractors/SatExtractor.cs b/src/Nncase.EGraph/Passes/EGraphExtractors/SatExtractor.cs index 36cf26a0d3..038873173b 100644 --- a/src/Nncase.EGraph/Passes/EGraphExtractors/SatExtractor.cs +++ b/src/Nncase.EGraph/Passes/EGraphExtractors/SatExtractor.cs @@ -22,7 +22,7 @@ public SatExtractor(EGraphCostModel costModel) _costModel = costModel; } - public Expr Extract(EClass root, IEGraph eGraph) + public Expr Extract(EClass root, IEGraph eGraph, out IReadOnlyDictionary picks) { var cpmodel = new CpModel(); @@ -108,13 +108,13 @@ public Expr Extract(EClass root, IEGraph eGraph) throw new InvalidProgramException("SatExtract Failed!"); } - var pick = eGraph.Nodes.ToDictionary(e => e, e => solver.BooleanValue(vars[e])); + picks = eGraph.Nodes.ToDictionary(e => e, e => solver.BooleanValue(vars[e])); using (var dumpStream = enableDump ? DumpScope.Current.OpenFile("Costs/Pick.dot") : Stream.Null) { - EGraphPrinter.DumpEgraphAsDot(eGraph, _costModel, pick, root.Find(), dumpStream); + EGraphPrinter.DumpEgraphAsDot(eGraph, _costModel, picks, root.Find(), dumpStream); } - return new SatExprBuildVisitor(pick).Visit(root); + return new SatExprBuildVisitor(picks).Visit(root); } private void EliminateAllCycles(EClass root, LinkedList<(EClass Class, ENode Node)> path, Dictionary> pathMemo, Dictionary visited, CpModel cpModel, Dictionary vars) diff --git a/src/Nncase.EGraph/Passes/RewriteProvider.cs b/src/Nncase.EGraph/Passes/RewriteProvider.cs index 7642a5395e..07d3416edf 100644 --- a/src/Nncase.EGraph/Passes/RewriteProvider.cs +++ b/src/Nncase.EGraph/Passes/RewriteProvider.cs @@ -36,7 +36,7 @@ public Expr ERewrite(Expr expr, IEnumerable rules, RunPassContext var graph = new EGraph(expr); ERewrite(graph, rules, options); - var post = graph.Extract(graph.Root!, null); + var post = graph.Extract(graph.Root!, null, out _); return post; } diff --git a/src/Nncase.EGraph/packages.lock.json b/src/Nncase.EGraph/packages.lock.json index 83470253e5..2fb3aca77d 100644 --- a/src/Nncase.EGraph/packages.lock.json +++ b/src/Nncase.EGraph/packages.lock.json @@ -41,11 +41,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Google.OrTools.runtime.linux-arm64": { @@ -140,8 +140,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -163,6 +163,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -227,6 +228,12 @@ "libortki": "0.0.2" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Evaluator/Buffer.cs b/src/Nncase.Evaluator/Buffer.cs deleted file mode 100644 index 079cdaecce..0000000000 --- a/src/Nncase.Evaluator/Buffer.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Canaan Inc. All rights reserved. -// Licensed under the Apache license. See LICENSE file in the project root for full license information. - -namespace Nncase.Evaluator.TIR -{ - public class Buffer - { - } -} diff --git a/src/Nncase.Evaluator/Buffers/BufferLoad.cs b/src/Nncase.Evaluator/Buffers/BufferLoad.cs new file mode 100644 index 0000000000..78bab2e920 --- /dev/null +++ b/src/Nncase.Evaluator/Buffers/BufferLoad.cs @@ -0,0 +1,42 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR; +using Nncase.IR.Buffers; + +namespace Nncase.Evaluator.Buffers; + +/// +/// Evaluator for BufferOf. +/// +[TypeInferGenerator] +public partial class BufferLoadEvaluator : ITypeInferencer, IOpPrinter +{ + public string Visit(IIRPrinterContext context, BufferLoad target, bool iLmode) + { + if (iLmode) + { + throw new System.NotSupportedException(); + } + + return $"{context.GetArgument(target, BufferLoad.Input)}[{context.GetArgument(target, BufferLoad.Indices)}]"; + } + + private IRType Visit(TensorType input, TupleType indices) + { + if (indices.Count != input.Shape.Rank) + { + return new InvalidType($"the input buffer rank {input.Shape.Rank} != indices.Count {indices.Count}"); + } + + foreach (var item in indices) + { + if (item is not TensorType { IsScalar: true, DType: var dtype } || dtype != DataTypes.Int32) + { + return new InvalidType("indices is not int32 type!"); + } + } + + return TensorType.Scalar(input.DType); + } +} diff --git a/src/Nncase.Evaluator/Buffers/BufferModule.cs b/src/Nncase.Evaluator/Buffers/BufferModule.cs index 4547718379..a2512b6f13 100644 --- a/src/Nncase.Evaluator/Buffers/BufferModule.cs +++ b/src/Nncase.Evaluator/Buffers/BufferModule.cs @@ -20,5 +20,8 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/src/Nncase.Evaluator/Buffers/BufferStore.cs b/src/Nncase.Evaluator/Buffers/BufferStore.cs new file mode 100644 index 0000000000..81a833f79e --- /dev/null +++ b/src/Nncase.Evaluator/Buffers/BufferStore.cs @@ -0,0 +1,47 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR; +using Nncase.IR.Buffers; + +namespace Nncase.Evaluator.Buffers; + +/// +/// Evaluator for BufferOf. +/// +[TypeInferGenerator] +public partial class BufferStoreEvaluator : ITypeInferencer, IOpPrinter +{ + public string Visit(IIRPrinterContext context, BufferStore target, bool iLmode) + { + if (iLmode) + { + throw new System.NotSupportedException(); + } + + return $"{context.GetArgument(target, BufferStore.Input)}[{context.GetArgument(target, BufferStore.Indices)}] = {context.GetArgument(target, BufferStore.Value)}"; + } + + private IRType Visit(TensorType input, TupleType indices, TensorType value) + { + if (indices.Count != input.Shape.Rank) + { + return new InvalidType($"the input buffer rank {input.Shape.Rank} != indices.Count {indices.Count}"); + } + + foreach (var item in indices) + { + if (item is not TensorType { IsScalar: true, DType: var dtype } || dtype != DataTypes.Int32) + { + return new InvalidType("indices is not int32 type!"); + } + } + + if (!value.IsScalar || input.DType != value.DType) + { + return new InvalidType("value can't store!"); + } + + return TupleType.Void; + } +} diff --git a/src/Nncase.Evaluator/Buffers/DDrOf.cs b/src/Nncase.Evaluator/Buffers/DDrOf.cs index eb6de07acf..86c53e04b7 100644 --- a/src/Nncase.Evaluator/Buffers/DDrOf.cs +++ b/src/Nncase.Evaluator/Buffers/DDrOf.cs @@ -14,6 +14,6 @@ public partial class DDrOfEvaluator : ITypeInferencer { private IRType Visit(TensorType input) { - return new PointerType(input.DType); + return TensorType.Pointer(input.DType); } } diff --git a/src/Nncase.Evaluator/Buffers/MatchBuffer.cs b/src/Nncase.Evaluator/Buffers/MatchBuffer.cs new file mode 100644 index 0000000000..7a8122d2ae --- /dev/null +++ b/src/Nncase.Evaluator/Buffers/MatchBuffer.cs @@ -0,0 +1,29 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR; +using Nncase.IR.Buffers; + +namespace Nncase.Evaluator.Buffers; + +/// +/// Evaluator for BufferOf. +/// +[TypeInferGenerator] +public partial class MatchBufferEvaluator : ITypeInferencer, IOpPrinter +{ + public string Visit(IIRPrinterContext context, MatchBuffer target, bool iLmode) + { + if (iLmode) + { + throw new System.NotSupportedException(); + } + + return $"Matched {context.GetArgument(target, MatchBuffer.Input)}"; + } + + private IRType Visit() + { + return TupleType.Void; + } +} diff --git a/src/Nncase.Evaluator/Imaging/ResizeImage.cs b/src/Nncase.Evaluator/Imaging/ResizeImage.cs index eab837a460..e25db7b8c0 100644 --- a/src/Nncase.Evaluator/Imaging/ResizeImage.cs +++ b/src/Nncase.Evaluator/Imaging/ResizeImage.cs @@ -110,16 +110,55 @@ public IValue OnnxResize(IEvaluateContext context, ResizeImage target) /// public IRType Visit(ITypeInferenceContext context, ResizeImage target) { - var input = context.CheckArgumentType(target, ResizeImage.Input); + var input = context.CheckArgumentType(target, ResizeImage.Input); var newSize = context.GetArgument(target, ResizeImage.NewSize); + + return input switch + { + TensorType t => Visit(t, newSize), + DistributedType d => Visit(d, newSize), + _ => new InvalidType(input.GetType().ToString()), + }; + } + + public IRType Visit(TensorType input, Expr newSize) + { return TypeInference.ResizeType(input, newSize, null); } + public IRType Visit(DistributedType input, Expr newSize) + { + if (Visit(input.TensorType, newSize) is not TensorType tensorType) + { + return new InvalidType(string.Empty); + } + + var ndsbp = new SBP[input.Placement.Rank]; + + var invalid = new InvalidType($"{input}, not support"); + for (int i = 0; i < input.Placement.Rank; i++) + { + switch (input.NdSBP[i]) + { + case SBPSplit { Axis: int ix } when ix < 2: + ndsbp[i] = SBP.S(ix); + break; + case SBPBroadCast: + ndsbp[i] = SBP.B; + break; + default: + return invalid; + } + } + + return new DistributedType(tensorType, ndsbp, input.Placement); + } + /// public Cost Visit(ICostEvaluateContext context, ResizeImage target) { - var inputType = context.GetArgumentType(target, ResizeImage.Input); - var returnType = context.GetReturnType(); + var inputType = context.GetArgumentType(target, ResizeImage.Input); + var returnType = context.GetReturnType(); return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), diff --git a/src/Nncase.Evaluator/Math/Binary.cs b/src/Nncase.Evaluator/Math/Binary.cs index f5f9d3c65b..1ac424b0be 100755 --- a/src/Nncase.Evaluator/Math/Binary.cs +++ b/src/Nncase.Evaluator/Math/Binary.cs @@ -2,9 +2,11 @@ // Licensed under the Apache license. See LICENSE file in the project root for full license information. using System; +using DryIoc; using Nncase.CostModel; using Nncase.IR; using Nncase.IR.Math; +using Nncase.IR.Tensors; using Nncase.Utilities; using OrtKISharp; @@ -42,6 +44,14 @@ public IValue Visit(IEvaluateContext context, Binary binary) { return Value.FromTensor(Tensor.FromScalar(Compute(binary.BinaryOp, lhs.ToScalar(), rhs.ToScalar()))); } + else if (lhs.ElementType is PointerType && (rhs.ElementType == DataTypes.UInt32 || rhs.ElementType == DataTypes.UInt64)) + { + return Value.FromTensor(Tensor.FromScalar(Compute(binary.BinaryOp, lhs.ToScalar(), rhs.ToScalar()))); + } + else if ((lhs.ElementType == DataTypes.UInt32 || lhs.ElementType == DataTypes.UInt64) && rhs.ElementType is PointerType) + { + return Value.FromTensor(Tensor.FromScalar(Compute(binary.BinaryOp, lhs.ToScalar(), rhs.ToScalar()))); + } else { return Ort_compute(binary, lhs, rhs); @@ -54,17 +64,24 @@ public IValue Visit(IEvaluateContext context, Binary binary) /// public IRType Visit(ITypeInferenceContext context, Binary target) { - var lhs = context.CheckArgumentType(target, Binary.Lhs); - var rhs = context.CheckArgumentType(target, Binary.Rhs); - return Visit(target, lhs, rhs); + var lhs = context.CheckArgumentType(target, Binary.Lhs); + var rhs = context.CheckArgumentType(target, Binary.Rhs); + return (lhs, rhs) switch + { + (TensorType a, TensorType b) => Visit(target, a, b), + (DistributedType a, DistributedType b) => Visit(target, a, b), + (AnyType, _) => AnyType.Default, + (_, AnyType) => AnyType.Default, + _ => new InvalidType($"{lhs} {rhs}"), + }; } /// public Cost Visit(ICostEvaluateContext context, Binary target) { - var lhsType = context.GetArgumentType(target, Binary.Lhs); - var rhsType = context.GetArgumentType(target, Binary.Rhs); - var outputType = context.GetReturnType(); + var lhsType = context.GetArgumentType(target, Binary.Lhs); + var rhsType = context.GetArgumentType(target, Binary.Rhs); + var outputType = context.GetReturnType(); return new() { @@ -121,6 +138,76 @@ public Expr Visit(IShapeEvaluateContext context, Binary target) return ShapeExprUtility.BroadcastShape(lhs, rhs); } + private IRType Visit(Binary target, DistributedType a, DistributedType b) + { + if (a.Placement != b.Placement) + { + return new InvalidType("lhs rhs have different placement"); + } + + var rType = Visit(target, a.TensorType, b.TensorType); + if (rType is not TensorType tensorType) + { + return rType; + } + + // assume broadcast shapes are left algin + var padA = tensorType.Shape.Rank - a.TensorType.Shape.Rank; + var padB = tensorType.Shape.Rank - b.TensorType.Shape.Rank; + var ndsbp = new SBP[a.Placement.Rank]; + for (int i = 0; i < a.Placement.Rank; i++) + { + switch (a.NdSBP[i], b.NdSBP[i]) + { + case (SBPSplit sa, SBPSplit sb): + if ((padA + sa.Axis) != (padB + sb.Axis)) + { + return new InvalidType($"lhs rhs sbp at {i} not equal"); + } + + ndsbp[i] = SBP.S(padA + sa.Axis); + break; + case (SBPSplit s1, SBPBroadCast): + // invalid (S, B) if B is not broacast + if (s1.Axis + padA - padB >= 0 && b.TensorType.Shape[s1.Axis + padA - padB] != 1) + { + return new InvalidType($"lhs rhs sbp at {i} not broadcast"); + } + + ndsbp[i] = SBP.S(padA + s1.Axis); + break; + case (SBPBroadCast, SBPSplit s2): + // invalid (B, S) if A is not broacast + if (s2.Axis + padB - padA >= 0 && a.TensorType.Shape[s2.Axis + padB - padA] != 1) + { + return new InvalidType($"lhs rhs sbp at {i} not broadcast"); + } + + ndsbp[i] = SBP.S(padB + s2.Axis); + break; + case (SBPBroadCast, SBPBroadCast): + ndsbp[i] = SBP.B; + break; + case (SBPPartialSum, SBPPartialSum): + if (target.BinaryOp == BinaryOp.Add) + { + ndsbp[i] = SBP.P; + } + else + { + return new InvalidType("lhs rhs all partialsum only can be added."); + } + + break; + case (SBPPartialSum, _): + case (_, SBPPartialSum): + return new InvalidType("not support lhs or rhs partial."); + } + } + + return new DistributedType(tensorType, ndsbp, a.Placement); + } + private int Compute(BinaryOp op, int a, int b) => op switch { BinaryOp.Add => a + b, @@ -149,6 +236,18 @@ public Expr Visit(IShapeEvaluateContext context, Binary target) _ => throw new ArgumentOutOfRangeException(nameof(op)), }; + private ulong Compute(BinaryOp op, ulong a, ulong b) => op switch + { + BinaryOp.Add => a + b, + BinaryOp.Sub => a - b, + BinaryOp.Mul => a * b, + BinaryOp.Div => a / b, + BinaryOp.Mod => a % b, + BinaryOp.Min => System.Math.Min(a, b), + BinaryOp.Max => System.Math.Max(a, b), + _ => throw new ArgumentOutOfRangeException(nameof(op)), + }; + private bool Compute(BinaryOp op, bool a, bool b) => op switch { BinaryOp.LogicalAnd => a & b, @@ -228,26 +327,24 @@ private IRType Visit(Binary target, TensorType lhs, TensorType rhs) return new InvalidType("The Binary Logical Only Accept The Boolean Datatype."); } - if (lhs is { DType: PointerType { ElemType: var letype } } && rhs is { DType: PointerType { ElemType: var retype } }) + if (lhs is { DType: PointerType { ElemType: var letype } }) { - if (letype == retype) + if ((rhs is { DType: PointerType { ElemType: var other } } && letype == other) || rhs.DType == DataTypes.UInt64 || rhs.DType == DataTypes.UInt32) { return TensorType.Pointer(letype); } - else - { - return new InvalidType($"The Binary Lhs {CompilerServices.Print(lhs)} != Rhs {CompilerServices.Print(rhs)}"); - } - } - if (lhs is { DType: PointerType { ElemType: var lt } } && rhs.DType == DataTypes.Int32) - { - return TensorType.Pointer(lt); + return new InvalidType($"The Binary Lhs {CompilerServices.Print(lhs)} != Rhs {CompilerServices.Print(rhs)}"); } - if (lhs.DType == DataTypes.Int32 && rhs is { DType: PointerType { ElemType: var rt } }) + if (rhs is { DType: PointerType { ElemType: var retype } }) { - return TensorType.Pointer(rt); + if ((lhs is { DType: PointerType { ElemType: var other } } && retype == other) || lhs.DType == DataTypes.UInt64 || lhs.DType == DataTypes.UInt32) + { + return TensorType.Pointer(retype); + } + + return new InvalidType($"The Binary Lhs {CompilerServices.Print(lhs)} != Rhs {CompilerServices.Print(rhs)}"); } return TypeInference.BroadcastType(lhs, rhs); diff --git a/src/Nncase.Evaluator/Math/Clamp.cs b/src/Nncase.Evaluator/Math/Clamp.cs index c2da8bb94c..383e2dd509 100644 --- a/src/Nncase.Evaluator/Math/Clamp.cs +++ b/src/Nncase.Evaluator/Math/Clamp.cs @@ -29,25 +29,25 @@ public IValue Visit(IEvaluateContext context, Clamp clamp) /// public IRType Visit(ITypeInferenceContext context, Clamp target) { - var input = context.CheckArgumentType(target, Clamp.Input); + var input = context.CheckArgumentType(target, Clamp.Input); var min = context.CheckArgumentType(target, Clamp.Min); var max = context.CheckArgumentType(target, Clamp.Max); - if (input.DType != min.DType || input.DType != max.DType || min.DType != max.DType) - { - return new InvalidType( - $"clamp type is not equal, input:{input.DType}, min:${min.DType}, max:${max.DType}"); - } - return Visit(input, min, max); + return input switch + { + TensorType t => Visit(t, min, max), + DistributedType d => Visit(d, min, max), + _ => new InvalidType("Wrong Clamp Type!"), + }; } /// public Cost Visit(ICostEvaluateContext context, Clamp target) { - var inputType = context.GetArgumentType(target, Clamp.Input); + var inputType = context.GetArgumentType(target, Clamp.Input); var minType = context.GetArgumentType(target, Clamp.Min); var maxType = context.GetArgumentType(target, Clamp.Max); - var outputType = context.GetReturnType(); + var outputType = context.GetReturnType(); return new() { @@ -71,6 +71,12 @@ public Metric Visit(IMetricEvaluateContext context, Clamp target) private IRType Visit(TensorType input, TensorType min, TensorType max) { + if (input.DType != min.DType || input.DType != max.DType || min.DType != max.DType) + { + return new InvalidType( + $"clamp type is not equal, input:{input.DType}, min:${min.DType}, max:${max.DType}"); + } + if (TypeInference.BroadcastType(input, min) is InvalidType invalidMin) { return invalidMin; @@ -88,4 +94,9 @@ private IRType Visit(TensorType input, TensorType min, TensorType max) return input; } + + private IRType Visit(DistributedType input, TensorType min, TensorType max) + { + return input; + } } diff --git a/src/Nncase.Evaluator/Math/MatMul.cs b/src/Nncase.Evaluator/Math/MatMul.cs index 4785e1e1c1..1f19b64388 100644 --- a/src/Nncase.Evaluator/Math/MatMul.cs +++ b/src/Nncase.Evaluator/Math/MatMul.cs @@ -19,62 +19,93 @@ namespace Nncase.Evaluator.Math; /// public class MatMulEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator { - /// - public IValue Visit(IEvaluateContext context, MatMul matMul) + public static IRType VisitDistributedType(DistributedType a, DistributedType b) { - var input = context.GetOrtArgumentValue(matMul, MatMul.Lhs); - var other = context.GetOrtArgumentValue(matMul, MatMul.Rhs); - return OrtKI.MatMul(input, other).ToValue(); - } + if (VisitTensorType(a.TensorType, b.TensorType) is not TensorType outType) + { + return new InvalidType(string.Empty); + } - /// - public IRType Visit(ITypeInferenceContext context, MatMul target) - { - var lhs = context.CheckArgumentType(target, MatMul.Lhs); - var rhs = context.CheckArgumentType(target, MatMul.Rhs); - return Visit(lhs, rhs); - } + if (a.Placement != b.Placement) + { + return new InvalidType("placement not equal"); + } - /// - public Cost Visit(ICostEvaluateContext context, MatMul target) - { - var lhs = context.GetArgumentType(target, MatMul.Lhs); - var rhs = context.GetArgumentType(target, MatMul.Rhs); - var outputType = context.GetReturnType(); + var aRank = a.TensorType.Shape.Rank; + var bRank = b.TensorType.Shape.Rank; + var oRank = outType.Shape.Rank; + var aPad = oRank - aRank; + var bPad = oRank - bRank; - uint macPerElement = lhs.Shape[^1].IsFixed ? (uint)lhs.Shape[^1].FixedValue : 1U; - return new() + var ndsbp = new SBP[a.Placement.Rank]; + for (int i = 0; i < a.Placement.Rank; i++) { - [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(lhs) + CostUtility.GetMemoryAccess(rhs), - [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), - [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, macPerElement), - }; - } + var invalid = new InvalidType($"({a.NdSBP[i]}, {b.NdSBP[i]}) not support"); + switch (a.NdSBP[i], b.NdSBP[i]) + { + // split on k + case (SBPSplit { Axis: int ax }, SBPSplit { Axis: int bx }): + if (ax == (aRank - 1) && bx == (bRank - 2)) + { + ndsbp[i] = SBP.P; + } + else if ((ax == (aRank - 1) && bx != (bRank - 2)) || (ax != (aRank - 1) && bx == (bRank - 2))) + { + return invalid; + } + else + { + if ((ax + aPad) == (bx + bPad)) + { + ndsbp[i] = SBP.S(ax + aPad); + } + else + { + return invalid; + } + } - public Metric Visit(IMetricEvaluateContext context, MatMul target) - { - var lhs = context.GetArgumentType(target, MatMul.Lhs); - var rhs = context.GetArgumentType(target, MatMul.Rhs); - var outputType = context.GetReturnType(); - var k = (UInt128)lhs.Shape[^1].FixedValue; - var m = MetricUtility.GetFLOPs(lhs) / k; - var n = MetricUtility.GetFLOPs(rhs) / k; - return new() - { - [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(lhs) + CostUtility.GetMemoryAccess(rhs) + CostUtility.GetMemoryAccess(outputType), - [MetricFactorNames.FLOPs] = m * n * ((2 * k) - 1), - [MetricFactorNames.Parallel] = 4, - }; - } + break; + case (SBPSplit { Axis: int ax }, SBPBroadCast): + if (ax == aRank - 1) + { + return invalid; + } - public Expr Visit(IShapeEvaluateContext context, MatMul target) - { - var lhs = context.GetArgumentShape(target, MatMul.Lhs); - var rhs = context.GetArgumentShape(target, MatMul.Rhs); - return IR.F.ShapeExpr.MatMulShape(lhs, rhs); + // invalid (S, B) if B is not broacast matmul + if (ax < aRank - 2 && !(bRank <= 2 || (ax + aPad - bPad >= 0 && b.TensorType.Shape[ax + aPad - bPad] == 1))) + { + return invalid; + } + + ndsbp[i] = SBP.S(ax + aPad); + break; + case (SBPBroadCast, SBPSplit { Axis: int bx }): + if (bx == bRank - 2) + { + return invalid; + } + + // invalid (B, S) if A is not broacast matmul + if (bx < bRank - 2 && !(aRank <= 2 || (bx + bPad - aPad >= 0 && a.TensorType.Shape[bx + bPad - aPad] == 1))) + { + return invalid; + } + + ndsbp[i] = SBP.S(bx + bPad); + break; + case (SBPBroadCast, SBPBroadCast): + ndsbp[i] = SBP.B; + break; + default: + return invalid; + } + } + + return new DistributedType(outType, ndsbp, a.Placement); } - private IRType Visit(TensorType lhs, TensorType rhs) + public static IRType VisitTensorType(TensorType lhs, TensorType rhs) { if (lhs.Shape.IsUnranked || rhs.Shape.IsUnranked) { @@ -113,4 +144,74 @@ private IRType Visit(TensorType lhs, TensorType rhs) var end = new[] { lhs.Shape[^2], rhs.Shape[^1] }; return new TensorType(lhs.DType, front.Concat(end).ToArray()); } + + /// + public IValue Visit(IEvaluateContext context, MatMul matMul) + { + var input = context.GetOrtArgumentValue(matMul, MatMul.Lhs); + var other = context.GetOrtArgumentValue(matMul, MatMul.Rhs); + return OrtKI.MatMul(input, other).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, MatMul target) + { + var lhs = context.CheckArgumentType(target, MatMul.Lhs); + var rhs = context.CheckArgumentType(target, MatMul.Rhs); + return (lhs, rhs) switch + { + (DistributedType a, DistributedType b) => VisitDistributedType(a, b), + (TensorType a, TensorType b) => VisitTensorType(a, b), + _ => new InvalidType(string.Empty), + }; + } + + /// + public Cost Visit(ICostEvaluateContext context, MatMul target) + { + var lhs = context.GetArgumentType(target, MatMul.Lhs); + var rhs = context.GetArgumentType(target, MatMul.Rhs); + var outputType = context.GetReturnType(); + + uint macPerElement = 1; + if (lhs is TensorType { Shape: Shape lhsShape }) + { + macPerElement = lhsShape[^1].IsFixed ? (uint)lhsShape[^1].FixedValue : 1U; + } + else if (lhs is DistributedType distributedType) + { + var lhsType = DistributedUtility.GetDividedTensorType(distributedType); + macPerElement = lhsType.Shape[^1].IsFixed ? (uint)lhsType.Shape[^1].FixedValue : 1U; + } + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(lhs) + CostUtility.GetMemoryAccess(rhs), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, macPerElement), + }; + } + + public Metric Visit(IMetricEvaluateContext context, MatMul target) + { + var lhs = context.GetArgumentType(target, MatMul.Lhs); + var rhs = context.GetArgumentType(target, MatMul.Rhs); + var outputType = context.GetReturnType(); + var k = (UInt128)lhs.Shape[^1].FixedValue; + var m = MetricUtility.GetFLOPs(lhs) / k; + var n = MetricUtility.GetFLOPs(rhs) / k; + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(lhs) + CostUtility.GetMemoryAccess(rhs) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = m * n * ((2 * k) - 1), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, MatMul target) + { + var lhs = context.GetArgumentShape(target, MatMul.Lhs); + var rhs = context.GetArgumentShape(target, MatMul.Rhs); + return Cast(IR.F.ShapeExpr.MatMulShape(lhs, rhs), DataTypes.Int32); + } } diff --git a/src/Nncase.Evaluator/Math/ReduceArg.cs b/src/Nncase.Evaluator/Math/ReduceArg.cs index 5ded2c065f..ad865c0e51 100644 --- a/src/Nncase.Evaluator/Math/ReduceArg.cs +++ b/src/Nncase.Evaluator/Math/ReduceArg.cs @@ -40,16 +40,23 @@ public IValue Visit(IEvaluateContext context, ReduceArg reduceArg) /// public IRType Visit(ITypeInferenceContext context, ReduceArg target) { - var input = context.CheckArgumentType(target, ReduceArg.Input); - return Visit(context, target, input); + var input = context.CheckArgumentType(target, ReduceArg.Input); + return input switch + { + TensorType tensorType => Visit(context, target, tensorType), + DistributedType distributedType => Visit(context, target, distributedType), + _ => new InvalidType(string.Empty), + }; } public Cost Visit(ICostEvaluateContext context, ReduceArg target) { - var input = context.GetArgumentType(target, ReduceArg.Input); - var ret = context.GetReturnType(); - uint input_elem = input.Shape.Aggregate(1U, (acc, d) => acc * (d.IsFixed ? (uint)d.FixedValue : 1U)); - uint ret_elem = ret.Shape.Aggregate(1U, (acc, d) => acc * (d.IsFixed ? (uint)d.FixedValue : 1U)); + var input = context.GetArgumentType(target, ReduceArg.Input); + var ret = context.GetReturnType(); + var inShape = input switch { TensorType t => t.Shape, DistributedType d => d.TensorType.Shape, _ => throw new NotImplementedException() }; + var rShape = ret switch { TensorType t => t.Shape, DistributedType d => d.TensorType.Shape, _ => throw new NotImplementedException() }; + uint input_elem = inShape.Aggregate(1U, (acc, d) => acc * (d.IsFixed ? (uint)d.FixedValue : 1U)); + uint ret_elem = rShape.Aggregate(1U, (acc, d) => acc * (d.IsFixed ? (uint)d.FixedValue : 1U)); uint macPerElement = input_elem / ret_elem; return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(input), [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(ret, macPerElement), }; } @@ -93,4 +100,51 @@ private IRType Visit(ITypeInferenceContext context, ReduceArg target, TensorType return new InvalidType("ReduceArg axis and keepDims are not const"); } } + + private IRType Visit(ITypeInferenceContext context, ReduceArg target, DistributedType distributedType) + { + var rType = Visit(context, target, distributedType.TensorType); + if (rType is not TensorType tensorType) + { + return rType; + } + + var inshape = distributedType.TensorType.Shape; + if (context.GetArgument(target, ReduceArg.Axis) is TensorConst axisValue && + context.GetArgument(target, ReduceArg.KeepDims) is TensorConst keepDimsValue) + { + var axis = axisValue.Value.ToScalar(); + axis = axis >= 0 ? axis : inshape.Rank + axis; + var keepdim = keepDimsValue.Value.ToScalar(); + var ndsbp = new SBP[distributedType.Placement.Rank]; + for (int i = 0; i < ndsbp.Length; i++) + { + switch (distributedType.NdSBP[i]) + { + case SBPSplit { Axis: int saxis }: + if (saxis == axis) + { + return new InvalidType("can't split on reduce axis."); + } + + ndsbp[i] = keepdim ? SBP.S(saxis) : SBP.S(saxis > axis ? saxis - 1 : saxis); + break; + case SBPPartialSum: + return new InvalidType("not support partial sum."); + case SBPBroadCast: + ndsbp[i] = SBP.B; + break; + } + } + + return distributedType with { NdSBP = new(ndsbp), TensorType = tensorType }; + } + + if (!distributedType.NdSBP.All(sbp => sbp is SBPBroadCast)) + { + return new InvalidType(string.Empty); + } + + return distributedType with { TensorType = tensorType }; + } } diff --git a/src/Nncase.Evaluator/Math/Unary.cs b/src/Nncase.Evaluator/Math/Unary.cs index 64c3bfbfbf..95824a5bb8 100644 --- a/src/Nncase.Evaluator/Math/Unary.cs +++ b/src/Nncase.Evaluator/Math/Unary.cs @@ -64,21 +64,27 @@ public IValue Visit(IEvaluateContext context, Unary unary) /// public IRType Visit(ITypeInferenceContext context, Unary target) { - var input = context.CheckArgumentType(target, Unary.Input); - return Visit(input); + var inputType = context.GetArgumentType(target, Unary.Input); + + return inputType switch + { + TensorType tensorType => Visit(tensorType), + DistributedType distTensorType => Visit(distTensorType, target.UnaryOp), + AnyType => AnyType.Default, + _ => new InvalidType($"Not support {inputType.GetType().Name}"), + }; } /// public Cost Visit(ICostEvaluateContext context, Unary target) { - var inputType = context.GetArgumentType(target, Unary.Input); - var outputType = context.GetReturnType(); - - return new() + var inputType = context.GetArgumentType(target, Unary.Input); + var outputType = context.GetReturnType(); + return (inputType, outputType) switch { - [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), - [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), - [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, CostUtility.GetCPUCyclesOfUnary(target.UnaryOp)), + (TensorType tensorType, TensorType tensorType1) => Visit(tensorType, tensorType1, target), + (DistributedType distTensorType, DistributedType distTensorType1) => Visit(distTensorType, distTensorType1, target), + _ => throw new NotSupportedException(string.Empty), }; } @@ -117,6 +123,23 @@ public Expr Visit(IShapeEvaluateContext context, Unary target) return context.GetArgumentShape(target, Unary.Input); } + private IRType Visit(DistributedType inType, UnaryOp unaryOp) + { + var invalid = new InvalidType(inType.ToString()); + var ndsbp = new SBP[inType.Placement.Rank]; + for (int i = 0; i < inType.Placement.Rank; i++) + { + if (inType.NdSBP[i] is SBPPartialSum && unaryOp != UnaryOp.Neg) + { + return invalid; + } + + ndsbp[i] = inType.NdSBP[i]; + } + + return new DistributedType(inType.TensorType, ndsbp, inType.Placement); + } + private int Compute_int(int input, UnaryOp op) => op switch { UnaryOp.Ceil => input, @@ -156,4 +179,26 @@ private IRType Visit(TensorType input) { return input; } + + private Cost Visit(DistributedType inType, DistributedType outType, Unary target) + { + var inPartType = Utilities.DistributedUtility.GetDividedTensorType(inType); + var outPartType = Utilities.DistributedUtility.GetDividedTensorType(outType); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inPartType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outPartType, CostUtility.GetCPUCyclesOfUnary(target.UnaryOp)), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outPartType), + }; + } + + private Cost Visit(TensorType inputType, TensorType outputType, Unary target) + { + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, CostUtility.GetCPUCyclesOfUnary(target.UnaryOp)), + }; + } } diff --git a/src/Nncase.Evaluator/NN/Activations.cs b/src/Nncase.Evaluator/NN/Activations.cs index 002314c9fa..aef4860dbd 100644 --- a/src/Nncase.Evaluator/NN/Activations.cs +++ b/src/Nncase.Evaluator/NN/Activations.cs @@ -1,6 +1,7 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. +using System.Linq; using Nncase.CostModel; using Nncase.IR; using Nncase.IR.NN; @@ -526,20 +527,21 @@ public class SwishEvaluator : IEvaluator, ITypeInferencer, ICostEv public IValue Visit(IEvaluateContext context, Swish swish) { var input = context.GetOrtArgumentValue(swish, Swish.Input); - return OrtKI.Mul(OrtKI.Sigmoid(input), input).ToValue(); + var beta = context.GetOrtArgumentValue(swish, Swish.Beta); + return OrtKI.Mul(OrtKI.Sigmoid(input * beta), input).ToValue(); } /// public IRType Visit(ITypeInferenceContext context, Swish target) { - var input = context.CheckArgumentType(target, Swish.Input); + var input = context.CheckArgumentType(target, Swish.Input); return Visit(input); } /// public Cost Visit(ICostEvaluateContext context, Swish target) { - var outputType = context.GetReturnType(); + var outputType = context.GetReturnType(); return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(outputType), @@ -558,8 +560,13 @@ public Metric Visit(IMetricEvaluateContext context, Swish target) }; } - private IRType Visit(TensorType input) + private IRType Visit(IRType input) { + if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartialSum)) + { + return new InvalidType("swish with partial sum is not supported"); + } + return input; } } @@ -582,14 +589,14 @@ public IValue Visit(IEvaluateContext context, Gelu gelu) /// public IRType Visit(ITypeInferenceContext context, Gelu target) { - var input = context.CheckArgumentType(target, Gelu.Input); + var input = context.CheckArgumentType(target, Gelu.Input); return Visit(input); } /// public Cost Visit(ICostEvaluateContext context, Gelu target) { - var outputType = context.GetReturnType(); + var outputType = context.GetReturnType(); return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(outputType), @@ -610,8 +617,13 @@ public Metric Visit(IMetricEvaluateContext context, Gelu target) public Expr Visit(IShapeEvaluateContext context, Gelu target) => context.GetArgumentShape(target, Gelu.Input); - private IRType Visit(TensorType input) + private IRType Visit(IRType input) { + if (input is DistributedType d && d.NdSBP.Any(s => s is SBPPartialSum)) + { + return new InvalidType("gelu with partial sum is not supported"); + } + return input; } } diff --git a/src/Nncase.Evaluator/NN/Conv2D.cs b/src/Nncase.Evaluator/NN/Conv2D.cs index c26e219821..9ff14e5256 100644 --- a/src/Nncase.Evaluator/NN/Conv2D.cs +++ b/src/Nncase.Evaluator/NN/Conv2D.cs @@ -48,20 +48,28 @@ public IValue Visit(IEvaluateContext context, Conv2D conv) /// public IRType Visit(ITypeInferenceContext context, Conv2D target) { - var input = context.CheckArgumentType(target, Conv2D.Input); - var weights = context.CheckArgumentType(target, Conv2D.Weights); - return Visit(context, target, input, weights); + var input = context.GetArgumentType(target, Conv2D.Input); + var weights = context.GetArgumentType(target, Conv2D.Weights); + var bias = context.GetArgumentType(target, Conv2D.Bias); + return (input, weights) switch + { + (DistributedType a, DistributedType b) => Visit(context, target, a, b, (DistributedType)bias), + (TensorType a, TensorType b) => Visit(context, target, a, b), + (AnyType, _) => AnyType.Default, + (_, AnyType) => AnyType.Default, + _ => new InvalidType(string.Empty), + }; } /// public Cost Visit(ICostEvaluateContext context, Conv2D target) { - var inputType = context.GetArgumentType(target, Conv2D.Input); - var weightsType = context.GetArgumentType(target, Conv2D.Weights); - var biasType = context.GetArgumentType(target, Conv2D.Bias); - var outputType = context.GetReturnType(); + var inputType = context.GetArgumentType(target, Conv2D.Input); + var weightsType = context.GetArgumentType(target, Conv2D.Weights); + var biasType = context.GetArgumentType(target, Conv2D.Bias); + var outputType = context.GetReturnType(); - var weightsShape = weightsType.Shape; + var weightsShape = weightsType is TensorType ? ((TensorType)weightsType).Shape : ((DistributedType)weightsType).TensorType.Shape; var macPerElement = (2 * weightsShape[1] * weightsShape[2] * weightsShape[3]) - 1; return new() { @@ -104,4 +112,90 @@ private IRType Visit(ITypeInferenceContext context, Conv2D target, TensorType in var args = context.GetArguments(target, Conv2D.Stride, Conv2D.Padding, Conv2D.Dilation, Conv2D.Groups); return TypeInference.Conv2DType(input, weights, args[0], args[1], args[2], args[3]); } + + private IRType Visit(ITypeInferenceContext context, Conv2D target, DistributedType input, DistributedType weights, DistributedType bias) + { + if (Visit(context, target, input.TensorType, weights.TensorType) is not TensorType outType) + { + return new InvalidType(string.Empty); + } + + var args = context.GetArguments(target, Conv2D.Stride, Conv2D.Padding, Conv2D.Dilation, Conv2D.Groups); + + // Not support split on h/w/r/s + if (input.NdSBP.Any(sbp => sbp is SBPSplit s && s.Axis >= 2) || weights.NdSBP.Any(sbp => sbp is SBPSplit s && s.Axis >= 2)) + { + return new InvalidType(string.Empty); + } + + if (input.Placement != weights.Placement) + { + return new InvalidType("placement not equal"); + } + + var ndsbp = new SBP[input.Placement.Rank]; + for (int i = 0; i < input.Placement.Rank; i++) + { + var invalid = new InvalidType($"({input.NdSBP[i]}, {weights.NdSBP[i]}) not support"); + switch (input.NdSBP[i], weights.NdSBP[i]) + { + case (SBPSplit { Axis: int ax }, SBPSplit { Axis: int bx }): + // split on ic + if (ax == 1 && bx == 1) + { + if (bias.NdSBP[i] is SBPBroadCast) + { + ndsbp[i] = SBP.P; + } + else + { + return invalid; + } + } + else + { + return invalid; + } + + break; + case (SBPSplit { Axis: int ax }, SBPBroadCast): + if (ax == 0 && bias.NdSBP[i] is SBPBroadCast) + { + ndsbp[i] = SBP.S(ax); + } + else + { + return invalid; + } + + break; + case (SBPBroadCast, SBPSplit { Axis: int bx }): + if (bx == 0 && bias.NdSBP[i] is SBPSplit s && s.Axis == bx) + { + ndsbp[i] = SBP.S(bx + 1); + } + else + { + return invalid; + } + + break; + case (SBPBroadCast, SBPBroadCast): + if (bias.NdSBP[i] is SBPBroadCast) + { + ndsbp[i] = SBP.B; + } + else + { + return invalid; + } + + break; + default: + return invalid; + } + } + + return new DistributedType(outType, ndsbp, input.Placement); + } } diff --git a/src/Nncase.Evaluator/NN/LayerNorm.cs b/src/Nncase.Evaluator/NN/LayerNorm.cs index 4088d8f9e4..b76300efbf 100644 --- a/src/Nncase.Evaluator/NN/LayerNorm.cs +++ b/src/Nncase.Evaluator/NN/LayerNorm.cs @@ -2,6 +2,7 @@ // Licensed under the Apache license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using Nncase.CostModel; using Nncase.IR; using Nncase.IR.NN; @@ -23,26 +24,52 @@ public IValue Visit(IEvaluateContext context, LayerNorm layerNorm) var bias = context.GetOrtArgumentValue(layerNorm, LayerNorm.Bias); // return Value.FromTensor(OrtKI.LayerNormalization(input, scale, bias, layerNorm.Axis, layerNorm.Epsilon, 1)); - return Value.FromTensor(LayerNormImpl(input.ToTensor(), scale.ToTensor(), bias.ToTensor(), layerNorm.Axis, layerNorm.Epsilon)); + return Value.FromTensor(LayerNormImpl(input.ToTensor(), scale.ToTensor(), bias.ToTensor(), layerNorm.Axis, layerNorm.Epsilon, layerNorm.UseMean)); } /// public IRType Visit(ITypeInferenceContext context, LayerNorm target) { - var input = context.CheckArgumentType(target, LayerNorm.Input); - return Visit(input); + var input = context.CheckArgumentType(target, LayerNorm.Input); + var scale = context.CheckArgumentType(target, LayerNorm.Scale); + var bias = context.CheckArgumentType(target, LayerNorm.Bias); + + return (input, scale, bias) switch + { + (DistributedType a, DistributedType b, DistributedType c) => Visit(a, b, c, target.Axis), + (TensorType a, TensorType, TensorType) => Visit(a), + _ => new InvalidType(input.GetType().ToString()), + }; } /// public Cost Visit(ICostEvaluateContext context, LayerNorm target) { - var inputType = context.GetArgumentType(target, LayerNorm.Input); - var returnType = context.GetReturnType(); - return new() + var inputType = context.GetArgumentType(target, LayerNorm.Input); + var returnType = context.GetReturnType(); + switch (inputType, returnType) { - [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), - [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(returnType), - }; + case (TensorType, TensorType): + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(returnType), + }; + + case (DistributedType inputDistributedType, DistributedType): + var scaleType = context.GetArgumentType(target, LayerNorm.Scale); + var biasType = context.GetArgumentType(target, LayerNorm.Bias); + var ring = GetRingReduceCommunicate(scaleType, new[] { 0, 1 }) + GetRingReduceCommunicate(biasType, new[] { 0, 1 }); + var reCompute = inputDistributedType.NdSBP.Select((sbp, i) => sbp is SBPSplit ? 1 : inputDistributedType.Placement.Hierarchy[i]).ToArray().Aggregate(1, (acc, rep) => acc * rep); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType) + ring, + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(inputType, 1) * (UInt128)reCompute, + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(returnType) + ring, + }; + default: + throw new NotSupportedException(); + } } public Metric Visit(IMetricEvaluateContext context, LayerNorm target) @@ -70,8 +97,53 @@ private IRType Visit(TensorType input) return input; } + private IRType Visit(DistributedType input, DistributedType scale, DistributedType bias, int raxis) + { + var invalid = new InvalidType($"{input}, {scale}, {bias} not support"); + if (input.Placement != scale.Placement || scale.Placement != bias.Placement) + { + return invalid; + } + + var ndsbp = new SBP[input.Placement.Rank]; + + for (int i = 0; i < input.Placement.Rank; i++) + { + switch (input.NdSBP[i], scale.NdSBP[i], bias.NdSBP[i]) + { + case (SBPSplit { Axis: int ix }, SBPSplit { Axis: int sx }, SBPSplit { Axis: int bx }) when ix >= raxis && sx == (ix - raxis) && bx == sx: + ndsbp[i] = SBP.S(ix); + break; + case (SBPSplit { Axis: int ix }, SBPBroadCast, SBPBroadCast) when ix < raxis: + ndsbp[i] = SBP.S(ix); + break; + case (SBPBroadCast, SBPBroadCast, SBPBroadCast): + ndsbp[i] = SBP.B; + break; + default: + return invalid; + } + } + + return new DistributedType(input.TensorType, ndsbp, input.Placement); + } + + private UInt128 GetRingReduceCommunicate(DistributedType distributedType, int[] axes) + { + var ttype = Utilities.DistributedUtility.GetDividedTensorType(distributedType); + var splits = axes.Where(i => distributedType.NdSBP[i] is SBPSplit); + if (!splits.Any()) + { + return 0; + } + + var p = (UInt128)splits.Select(i => distributedType.Placement.Hierarchy[i]).Aggregate(1, (acc, i) => acc * i); + var v = CostUtility.GetMemoryAccess(distributedType.TensorType); + return (p - 1) * (v / p); + } + #if true - private Tensor LayerNormImpl(Tensor input, Tensor scale, Tensor bias, int axis, float epsilon) + private Tensor LayerNormImpl(Tensor input, Tensor scale, Tensor bias, int axis, float epsilon, bool useMean = true) { int outputSize = 1; int innerSize = 1; @@ -96,9 +168,12 @@ private Tensor LayerNormImpl(Tensor input, Tensor scale, Tensor bias, int axis, for (int batch = 0; batch < outputSize; batch++) { float mean1 = 0f; - for (int i = 0; i < innerSize; i++) + if (useMean) { - mean1 += inputArray[(i + (batch * innerSize)) % inputArray.Length] / innerSize; + for (int i = 0; i < innerSize; i++) + { + mean1 += inputArray[(i + (batch * innerSize)) % inputArray.Length] / innerSize; + } } float[] sub = new float[innerSize]; diff --git a/src/Nncase.Evaluator/NN/Normalization.cs b/src/Nncase.Evaluator/NN/Normalization.cs index 1bc2c75bd2..8ad56df403 100644 --- a/src/Nncase.Evaluator/NN/Normalization.cs +++ b/src/Nncase.Evaluator/NN/Normalization.cs @@ -153,15 +153,22 @@ public IValue Visit(IEvaluateContext context, InstanceNormalization i) /// public IRType Visit(ITypeInferenceContext context, InstanceNormalization target) { - var input = context.CheckArgumentType(target, InstanceNormalization.Input); - return Visit(input); + var input = context.CheckArgumentType(target, InstanceNormalization.Input); + var scale = context.CheckArgumentType(target, InstanceNormalization.Scale); + var bias = context.CheckArgumentType(target, InstanceNormalization.Bias); + return (input, scale, bias) switch + { + (DistributedType a, DistributedType b, DistributedType c) => Visit(a, b, c), + (TensorType a, TensorType, TensorType) => Visit(a), + _ => new InvalidType(input.GetType().ToString()), + }; } /// public Cost Visit(ICostEvaluateContext context, InstanceNormalization target) { - var inputType = context.GetArgumentType(target, InstanceNormalization.Input); - var returnType = context.GetReturnType(); + var inputType = context.GetArgumentType(target, InstanceNormalization.Input); + var returnType = context.GetReturnType(); return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), @@ -183,6 +190,40 @@ private IRType Visit(TensorType input) { return input; } + + private IRType Visit(DistributedType input, DistributedType scale, DistributedType bias) + { + var invalid = new InvalidType($"{input}, {scale}, {bias} not support"); + if (input.Placement != scale.Placement || scale.Placement != bias.Placement) + { + return invalid; + } + + var ndsbp = new SBP[input.Placement.Rank]; + + // scale & bias always on Channel + const int rAxis = 1; + + for (int i = 0; i < input.Placement.Rank; i++) + { + switch (input.NdSBP[i], scale.NdSBP[i], bias.NdSBP[i]) + { + case (SBPSplit { Axis: int ix }, SBPSplit { Axis: int sx }, SBPSplit { Axis: int bx }) when ix == rAxis && sx == (ix - rAxis) && bx == sx: + ndsbp[i] = SBP.S(ix); + break; + case (SBPSplit { Axis: int ix }, SBPBroadCast, SBPBroadCast) when ix != rAxis: + ndsbp[i] = SBP.S(ix); + break; + case (SBPBroadCast, SBPBroadCast, SBPBroadCast): + ndsbp[i] = SBP.B; + break; + default: + return invalid; + } + } + + return new DistributedType(input.TensorType, ndsbp, input.Placement); + } } /// diff --git a/src/Nncase.Evaluator/NN/Softmax.cs b/src/Nncase.Evaluator/NN/Softmax.cs index ef5afafbe9..c7640de82f 100644 --- a/src/Nncase.Evaluator/NN/Softmax.cs +++ b/src/Nncase.Evaluator/NN/Softmax.cs @@ -1,6 +1,7 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. +using System.Linq; using Nncase.CostModel; using Nncase.IR; using Nncase.IR.NN; @@ -81,14 +82,20 @@ public IValue Visit(IEvaluateContext context, Softmax softMax) /// public IRType Visit(ITypeInferenceContext context, Softmax target) { - var input = context.CheckArgumentType(target, Softmax.Input); - return Visit(input); + var input = context.CheckArgumentType(target, Softmax.Input); + var axis = context.GetArgument(target, Softmax.Axis); + return input switch + { + TensorType t => Visit(t), + DistributedType d => Visit(d, axis), + _ => new InvalidType(input.GetType().Name), + }; } /// public Cost Visit(ICostEvaluateContext context, Softmax target) { - var ret = context.GetReturnType(); + var ret = context.GetReturnType(); return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), @@ -118,6 +125,17 @@ private IRType Visit(TensorType input) { return input; } + + private IRType Visit(DistributedType input, Expr axisExpr) + { + var axis = ((TensorConst)axisExpr).Value.ToScalar(); + if (input.NdSBP.Any(sbp => sbp is SBPSplit s && s.Axis == axis)) + { + return new InvalidType("Not support split on Axis for Softmax now."); + } + + return input; + } } /// diff --git a/src/Nncase.Evaluator/RNN/LSTM.cs b/src/Nncase.Evaluator/RNN/LSTM.cs index 9abd5964e4..fc1d1f3daa 100644 --- a/src/Nncase.Evaluator/RNN/LSTM.cs +++ b/src/Nncase.Evaluator/RNN/LSTM.cs @@ -6,12 +6,11 @@ using Nncase.CostModel; using Nncase.IR; -// using Nncase.IR.NN; -using Nncase.IR.Tensors; +using Nncase.IR.RNN; using OrtKISharp; using static Nncase.LSTMHelper; -namespace Nncase.Evaluator.NN; +namespace Nncase.Evaluator.RNN; /// /// Evaluator for . diff --git a/src/Nncase.Evaluator/TIR/Load.cs b/src/Nncase.Evaluator/TIR/Load.cs index 6ea6faddff..5898e3d353 100644 --- a/src/Nncase.Evaluator/TIR/Load.cs +++ b/src/Nncase.Evaluator/TIR/Load.cs @@ -30,12 +30,11 @@ public string Visit(IIRPrinterContext context, Load target, bool iLmode) private IRType Visit(Load target, TensorType handle, TensorType index) { - if (!handle.IsScalar && handle.DType is not PointerType) + if (handle is not TensorType { DType: PointerType { } p }) { - throw new NotSupportedException(handle.DType.ToString()); + return new InvalidType("handle must be pointer type!"); } - _ = index.IsScalar ? 1 : index.Shape[0].FixedValue; - return TensorType.Scalar(((PointerType)handle.DType).ElemType); + return TensorType.Scalar(p.ElemType); } } diff --git a/src/Nncase.Evaluator/TIR/Store.cs b/src/Nncase.Evaluator/TIR/Store.cs index b29459bfe2..b46bf57f52 100644 --- a/src/Nncase.Evaluator/TIR/Store.cs +++ b/src/Nncase.Evaluator/TIR/Store.cs @@ -24,21 +24,21 @@ public IRType Visit(ITypeInferenceContext context, Store target) public string Visit(IIRPrinterContext context, Store target, bool iLmode) { var handle = context.GetArgument(target, Store.Handle); - _ = context.GetArgument(target, Store.Value); + var value = context.GetArgument(target, Store.Value); var index = context.GetArgument(target, Store.Index); - return $"{handle}[{index}] = {index}"; - - throw new System.NotImplementedException(); + return $"{handle}[{index}] = {value}"; } private IRType Visit(Store target, TensorType handle, TensorType index, TensorType value) { - _ = index.IsScalar ? 1 : index.Shape[0].FixedValue; + if (handle.DType is not PointerType { ElemType: DataType elemType } || elemType != value.DType) + { + return new InvalidType($"You Can't Load The {value.DType} To {handle.DType}"); + } - var elemType = ((PointerType)handle.DType).ElemType; - if (elemType != value.DType) + if (index.DType != DataTypes.Int32) { - return new InvalidType($"You Can't Load The {value.DType} To {elemType}"); + return new InvalidType($"store value type {index.DType} not supported"); } return TupleType.Void; diff --git a/src/Nncase.Evaluator/Tensors/Cast.cs b/src/Nncase.Evaluator/Tensors/Cast.cs index c7e285d060..6f32939c3c 100644 --- a/src/Nncase.Evaluator/Tensors/Cast.cs +++ b/src/Nncase.Evaluator/Tensors/Cast.cs @@ -24,8 +24,13 @@ public IValue Visit(IEvaluateContext context, Cast cast) /// public IRType Visit(ITypeInferenceContext context, Cast target) { - var input = context.CheckArgumentType(target, Cast.Input); - return Visit(target, input); + var input = context.CheckArgumentType(target, Cast.Input); + return input switch + { + TensorType t => Visit(target, t), + DistributedType d => Visit(target, d), + _ => new InvalidType(input.GetType().ToString()), + }; } /// @@ -37,10 +42,10 @@ public string Visit(IIRPrinterContext context, Cast target, bool iLmode) /// public Cost Visit(ICostEvaluateContext context, Cast target) { - var input = context.GetArgumentType(target, Cast.Input); + var input = context.GetArgumentType(target, Cast.Input); return new() { - [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(input.DType), + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(input), [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(target.NewType), [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(target.NewType, 1), }; @@ -61,4 +66,21 @@ private IRType Visit(Cast target, TensorType input) { return new TensorType(target.NewType, input.Shape); } + + private IRType Visit(Cast target, DistributedType inType) + { + var invalid = new InvalidType(inType.ToString()); + var ndsbp = new SBP[inType.Placement.Rank]; + for (int i = 0; i < inType.Placement.Rank; i++) + { + if (inType.NdSBP[i] is SBPPartialSum) + { + return invalid; + } + + ndsbp[i] = inType.NdSBP[i]; + } + + return new DistributedType(new TensorType(target.NewType, inType.TensorType.Shape), ndsbp, inType.Placement); + } } diff --git a/src/Nncase.Evaluator/Tensors/Concat.cs b/src/Nncase.Evaluator/Tensors/Concat.cs index d1098ccf7b..6cce7d2b66 100644 --- a/src/Nncase.Evaluator/Tensors/Concat.cs +++ b/src/Nncase.Evaluator/Tensors/Concat.cs @@ -2,6 +2,7 @@ // Licensed under the Apache license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.Linq; using NetFabric.Hyperlinq; using Nncase.CostModel; @@ -25,7 +26,7 @@ public class ConcatEvaluator : IEvaluator, ITypeInferencer, ICos public IValue Visit(IEvaluateContext context, Concat cat) { var inputs = context.GetArgumentValueAsTensors(cat, Concat.Input); - var axis = context.GetArgumentValueAsScalar(cat, Concat.Axis); + var axis = cat.Axis; return OrtKI.Concat(inputs.Select(t => t.ToOrtTensor()).ToArray(), axis).ToValue(); } @@ -33,14 +34,13 @@ public IValue Visit(IEvaluateContext context, Concat cat) public IRType Visit(ITypeInferenceContext context, Concat target) { var inputs = context.CheckArgumentType(target, Concat.Input); - var axis = context.CheckArgumentType(target, Concat.Axis); - return Visit(context, target, inputs, axis); + return Visit(inputs, target.Axis); } /// public Cost Visit(ICostEvaluateContext context, Concat target) { - var ret = context.GetReturnType(); + var ret = context.GetReturnType(); return new() { [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), @@ -52,8 +52,7 @@ public Cost Visit(ICostEvaluateContext context, Concat target) public Expr Visit(IShapeEvaluateContext context, Concat target) { var inShape = context.GetArgumentShape(target, Concat.Input); - var axis = context.GetArgument(target, Concat.Axis); - var axisV = ShapeExprUtility.Positive(axis, inShape[0]); + var axisV = ShapeExprUtility.Positive(target.Axis, inShape[0]); var inShapes = ((IR.Tuple)inShape).Fields.ToArray().Select(x => Cast(x, DataTypes.Int64)).ToArray(); var dim = inShapes.ToArray().Aggregate((Expr)0L, (sum, shape) => sum + shape[axisV]); var outShape = ShapeExprUtility.Replace(inShapes[0], axisV, dim); @@ -68,17 +67,18 @@ public Expr Visit(IShapeEvaluateContext context, Concat target) DataType? allDType = null; foreach (var (i, input) in Enumerable.Range(0, inputs.Count).Select(i => (i, inputs[i]))) { - var type = input as TensorType; - if (type is null) + TensorType type; + if (input is TensorType a) { - if (input is InvalidType) - { - return input; - } - else - { - return new InvalidType($"The ConCat Item[{i}] Must Be TensorType But Get {input.GetType().Name}"); - } + type = a; + } + else if (input is DistributedType { TensorType: TensorType b }) + { + type = b; + } + else + { + return new InvalidType($"The ConCat Item[{i}] Must Have TensorType But Get {input}"); } if (type.Shape.IsUnranked) @@ -103,7 +103,14 @@ public Expr Visit(IShapeEvaluateContext context, Concat target) return null; } - private IRType Visit(ITypeInferenceContext context, Concat target, TupleType inputs, TensorType axis) + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + private IRType Visit(TupleType inputs, int axis) { var result = CheckType(inputs); if (result != null) @@ -111,15 +118,15 @@ private IRType Visit(ITypeInferenceContext context, Concat target, TupleType inp return result; } - var sameRank = inputs.All(input => ((TensorType)input).Shape.Rank == ((TensorType)inputs[0]).Shape.Rank); + var sameRank = inputs.All(input => GetTensorType(input).Shape.Rank == GetTensorType(inputs[0]).Shape.Rank); if (!sameRank) { return new InvalidType("Inputs of concat should be same rank"); } - var input0 = (TensorType)inputs[0]; + var input0 = GetTensorType(inputs[0]); InvalidType? invalidType = null; - var axisV = ((TensorConst)context.GetArgument(target, Concat.Axis)).Value.ToScalar(); + var axisV = axis; var axisValue = Util.PositiveIndex(axisV, input0.Shape.Rank); var shapeValue = Enumerable.Range(0, input0.Shape.Rank).Select(i => { @@ -134,18 +141,18 @@ private IRType Visit(ITypeInferenceContext context, Concat target, TupleType inp var allAxisDimIsSame = true; foreach (var inType in inputs.Fields) { - if (((TensorType)inType).Shape.IsUnranked) + if (GetTensorType(inType).Shape.IsUnranked) { continue; } - var d = ((TensorType)inType).Shape[i]; + var d = GetTensorType(inType).Shape[i]; if (d.IsUnknown) { return Dimension.Unknown; } - if (d.FixedValue != ((TensorType)inputs[0]).Shape[i]) + if (d.FixedValue != GetTensorType(inputs[0]).Shape[i]) { allAxisDimIsSame = false; } @@ -153,7 +160,7 @@ private IRType Visit(ITypeInferenceContext context, Concat target, TupleType inp if (allAxisDimIsSame) { - return ((TensorType)inputs[0]).Shape[i]; + return GetTensorType(inputs[0]).Shape[i]; } else { @@ -163,7 +170,56 @@ private IRType Visit(ITypeInferenceContext context, Concat target, TupleType inp } }); var shape = new Shape(shapeValue); - return (invalidType as IRType) ?? new TensorType(input0.DType, shape); + if (invalidType is InvalidType invalid) + { + return invalid; + } + + var tensorType = new TensorType(input0.DType, shape); + + if (inputs[0] is not DistributedType distributedType) + { + return tensorType; + } + + if (inputs.OfType().Select(d => d.Placement).ToHashSet().Count != 1) + { + return new InvalidType("the inputs have different placement"); + } + + var ndsbp = new SBP[distributedType.Placement.Rank]; + + for (int i = 0; i < distributedType.Placement.Rank; i++) + { + var sbps = inputs.OfType().Select(d => d.NdSBP[i]).ToArray(); + if (sbps.Any(sbp => sbp is SBPSplit { Axis: int x } && x == axis)) + { + return new InvalidType("not support distribute on concat axis"); + } + + if (sbps.Any(sbp => sbp is SBPPartialSum)) + { + return new InvalidType("not support distribute with partialsum"); + } + + if (sbps.OfType().ToHashSet() is HashSet setSplit && + sbps.OfType().ToHashSet() is HashSet setBroadcast) + { + switch (setSplit.Count) + { + case 0: + ndsbp[i] = SBP.B; + break; + case 1 when setBroadcast.Count == 0: + ndsbp[i] = setSplit.First(); + break; + default: + return new InvalidType("not support distribute with different axis"); + } + } + } + + return new DistributedType(tensorType, ndsbp, distributedType.Placement); } // axis: if one of inputs shape[axis] is unknown @@ -173,12 +229,12 @@ private Dimension AxisDim(TupleType inputs, int axisValue) { var allAxisDimIsFixed = inputs.Fields.Aggregate( true, - (prod, next) => prod && ((TensorType)next).Shape[axisValue].IsFixed); + (prod, next) => prod && (next switch { TensorType t => t, DistributedType d => d.TensorType, _ => throw new NotSupportedException() }).Shape[axisValue].IsFixed); if (allAxisDimIsFixed) { return inputs.Fields.Aggregate( 0, - (prod, next) => prod + ((TensorType)next).Shape[axisValue].FixedValue); + (prod, next) => prod + (next switch { TensorType t => t, DistributedType d => d.TensorType, _ => throw new NotSupportedException() }).Shape[axisValue].FixedValue); } else { diff --git a/src/Nncase.Evaluator/Tensors/Expand.cs b/src/Nncase.Evaluator/Tensors/Expand.cs index cea0603bfd..e06241f90c 100644 --- a/src/Nncase.Evaluator/Tensors/Expand.cs +++ b/src/Nncase.Evaluator/Tensors/Expand.cs @@ -30,8 +30,8 @@ public IValue Visit(IEvaluateContext context, Expand expand) public Cost Visit(ICostEvaluateContext context, Expand target) { - var input = context.GetArgumentType(target, Expand.Input); - var ret = context.GetReturnType(); + var input = context.GetArgumentType(target, Expand.Input); + var ret = context.GetReturnType(); return CostUtility.GetBroadcastCost(input, ret); } @@ -53,6 +53,18 @@ public Metric Visit(IMetricEvaluateContext context, Expand target) }; } + public IRType Visit(ITypeInferenceContext context, Expand target) + { + var input = context.CheckArgumentType(target, Expand.Input); + var shape = context.CheckArgumentType(target, Expand.Shape); + return input switch + { + TensorType t => Visit(context, target, t, shape), + DistributedType d => Visit(context, target, d, shape), + _ => new InvalidType(input.GetType().ToString()), + }; + } + private IRType Visit(ITypeInferenceContext context, Expand target, TensorType input, TensorType shape) { var shape_expr = context.GetArgument(target, Expand.Shape); @@ -65,4 +77,28 @@ private IRType Visit(ITypeInferenceContext context, Expand target, TensorType in return input with { Shape = TypeInference.ReshapeTo(shape) }; } } + + private IRType Visit(ITypeInferenceContext context, Expand target, DistributedType input, TensorType shape) + { + var invalid = new InvalidType(input.ToString()); + var shape_expr = context.GetArgument(target, Expand.Shape); + if (shape_expr is TensorConst constShape) + { + var newShape = constShape.Value.ToArray(); + var ndsbp = new SBP[input.Placement.Rank]; + for (int i = 0; i < input.Placement.Rank; i++) + { + if (input.NdSBP[i] is SBPSplit sbp && newShape[sbp.Axis] != input.TensorType.Shape[sbp.Axis]) + { + return invalid; + } + + ndsbp[i] = input.NdSBP[i]; + } + + return new DistributedType(new TensorType(input.TensorType.DType, new Shape(newShape)), ndsbp, input.Placement); + } + + return invalid; + } } diff --git a/src/Nncase.Evaluator/Tensors/Gather.cs b/src/Nncase.Evaluator/Tensors/Gather.cs index acfd7ec9b5..2f4bd25d46 100644 --- a/src/Nncase.Evaluator/Tensors/Gather.cs +++ b/src/Nncase.Evaluator/Tensors/Gather.cs @@ -23,7 +23,7 @@ public class GatherEvaluator : IEvaluator, ITypeInferencer, ICos public IValue Visit(IEvaluateContext context, Gather gather) { var input = context.GetOrtArgumentValue(gather, Gather.Input); - var axis = context.GetArgumentValueAsScalar(gather, Gather.Axis); + var axis = gather.Axis; var index = context.GetOrtArgumentValue(gather, Gather.Index); return OrtKI.Gather(input, index, axis).ToValue(); } @@ -31,29 +31,35 @@ public IValue Visit(IEvaluateContext context, Gather gather) /// public IRType Visit(ITypeInferenceContext context, Gather target) { - var input = context.CheckArgumentType(target, Gather.Input); - var axis = context.CheckArgumentType(target, Gather.Axis); - var index = context.CheckArgumentType(target, Gather.Index); - return Visit(context, target, input, axis, index); + var input = context.CheckArgumentType(target, Gather.Input); + var index = context.CheckArgumentType(target, Gather.Index); + + return (input, index) switch + { + (TensorType a, TensorType b) => Visit(a, target.Axis, b), + (DistributedType a, DistributedType b) => Visit(a, target.Axis, b), + _ => new InvalidType($"{input}, {index}"), + }; } /// public Cost Visit(ICostEvaluateContext context, Gather target) { - var ret_type = context.GetReturnType(); + var inputType = context.GetArgumentType(target, Gather.Input); + var indexType = context.GetArgumentType(target, Gather.Index); + var retType = context.GetReturnType(); return new() { - [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret_type.DType), - [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret_type.DType), - [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(ret_type.DType, 1), + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(indexType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(retType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(retType), }; } public Expr Visit(IShapeEvaluateContext context, Gather target) { - var axis = context.GetArgument(target, Gather.Axis); var inShape = context.GetArgumentShape(target, Gather.Input); - axis = ShapeExprUtility.Positive(Cast(axis, DataTypes.Int32), inShape); + var axis = ShapeExprUtility.Positive(target.Axis, inShape); var indexShape = context.GetArgumentShape(target, Gather.Index); var outShape = ShapeExprUtility.ReplaceList(inShape, axis, indexShape); return outShape; @@ -68,26 +74,56 @@ public Metric Visit(IMetricEvaluateContext context, Gather target) }; } - private IRType Visit(ITypeInferenceContext context, Gather target, TensorType input, TensorType axis, TensorType index) + private IRType Visit(TensorType input, int axis, TensorType index) { if (input.Shape.IsUnranked) { return input; } - if (context.GetArgument(target, Gather.Axis) is TensorConst axisValue) + axis = axis < 0 ? axis + input.Shape.Rank : axis; + + // input_shape[:axis] + index_shape + input_shape[axis + 1:] + var inShape = input.Shape.ToArray(); + var newShape = inShape[..axis].Concat(index.Shape).Concat(inShape[(axis + 1)..]).ToArray(); + return new TensorType(input.DType, newShape); + } + + private IRType Visit(DistributedType input, int axis, DistributedType index) + { + var invalid = new InvalidType(input.ToString() + " " + index.ToString()); + if (Visit(input.TensorType, axis, index.TensorType) is not TensorType tensorType) { - var axisV = axisValue.Value.ToScalar(); - axisV = axisV < 0 ? axisV + input.Shape.Rank : axisV; + return invalid; + } - // input_shape[:axis] + index_shape + input_shape[axis + 1:] - var inShape = input.Shape.ToArray(); - var newShape = inShape[..axisV].Concat(index.Shape).Concat(inShape[(axisV + 1)..]).ToArray(); - return new TensorType(input.DType, newShape); + if (input.Placement != index.Placement) + { + return invalid; } - else + + var ndsbp = new SBP[input.Placement.Rank]; + + for (int i = 0; i < input.Placement.Rank; i++) { - return new InvalidType("Gather axis must be constant"); + switch (input.NdSBP[i], index.NdSBP[i]) + { + case (SBPSplit { Axis: int ix }, _) when ix == axis: + return new InvalidType($"the input can't split on {axis}"); + case (SBPBroadCast, SBPSplit { Axis: int ix }): + ndsbp[i] = SBP.S(ix); + break; + case (SBPSplit { Axis: int ix }, SBPBroadCast): + ndsbp[i] = SBP.S(ix - axis + index.TensorType.Shape.Rank - 1); + break; + case (SBPBroadCast, SBPBroadCast): + ndsbp[i] = SBP.B; + break; + default: + return invalid; + } } + + return new DistributedType(tensorType, ndsbp, input.Placement); } } diff --git a/src/Nncase.Evaluator/Tensors/Reshape.cs b/src/Nncase.Evaluator/Tensors/Reshape.cs index 38c4d150ee..7488739f1e 100644 --- a/src/Nncase.Evaluator/Tensors/Reshape.cs +++ b/src/Nncase.Evaluator/Tensors/Reshape.cs @@ -2,8 +2,8 @@ // Licensed under the Apache license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.Linq; -using DryIoc.ImTools; using NetFabric.Hyperlinq; using Nncase.CostModel; using Nncase.IR; @@ -34,8 +34,14 @@ public IValue Visit(IEvaluateContext context, Reshape reshape) /// public IRType Visit(ITypeInferenceContext context, Reshape target) { - var input = context.CheckArgumentType(target, Reshape.Input); - return Visit(context, target, input); + var input = context.CheckArgumentType(target, Reshape.Input); + return input switch + { + TensorType tensorType => Visit(context, target, tensorType), + DistributedType distributedType => Visit(context, target, distributedType), + AnyType => AnyType.Default, + _ => throw new NotImplementedException(), + }; } public Cost Visit(ICostEvaluateContext context, Reshape target) @@ -121,4 +127,85 @@ private IRType Visit(ITypeInferenceContext context, Reshape target, TensorType i var outShape = ReshapeTo(targetType); return input with { Shape = outShape }; } + + private IRType Visit(ITypeInferenceContext context, Reshape target, DistributedType inputType) + { + var outType = Visit(context, target, inputType.TensorType); + if (outType is not TensorType outTensorType) + { + return outType; + } + + var invalid = new InvalidType(inputType.ToString()); + if (outTensorType.Shape.IsUnranked) + { + return invalid; + } + + var newShape = outTensorType.Shape.ToValueArray(); + var oldShape = inputType.TensorType.Shape.ToValueArray(); + + // check is unsequeeze/sequeeze + if (Enumerable.SequenceEqual(oldShape.Where(i => i != 1).ToArray(), newShape.Where(i => i != 1).ToArray())) + { + if (oldShape.Length < newShape.Length) + { + var axis = 0; + var axisMap = new Dictionary(); + for (var n = 0; n < newShape.Length; n++) + { + if (newShape[n] == oldShape[axis]) + { + axisMap.Add(axis++, n); + if (axis >= oldShape.Length) + { + break; + } + } + } + + var ndsbp = new SBP[inputType.Placement.Rank]; + for (int i = 0; i < inputType.Placement.Rank; i++) + { + ndsbp[i] = inputType.NdSBP[i] switch + { + SBPSplit { Axis: int sx } => SBPSplit.S(axisMap[sx]), + SBP sbp => sbp, + }; + } + + return inputType with { TensorType = outTensorType, NdSBP = new(ndsbp) }; + } + else if (oldShape.Length > newShape.Length) + { + var axis = 0; + var axisMap = new Dictionary(); + for (var o = 0; o < oldShape.Length; o++) + { + if (oldShape[o] == newShape[axis]) + { + axisMap.Add(o, axis++); + if (axis >= newShape.Length) + { + break; + } + } + } + + var ndsbp = new SBP[inputType.Placement.Rank]; + for (int i = 0; i < inputType.Placement.Rank; i++) + { + ndsbp[i] = inputType.NdSBP[i] switch + { + SBPSplit { Axis: int sx } => SBPSplit.S(axisMap[sx]), + SBP sbp => sbp, + }; + } + + return inputType with { TensorType = outTensorType, NdSBP = new(ndsbp) }; + } + } + + return invalid; + } } diff --git a/src/Nncase.Evaluator/Tensors/Slice.cs b/src/Nncase.Evaluator/Tensors/Slice.cs index eada657d01..08de735500 100644 --- a/src/Nncase.Evaluator/Tensors/Slice.cs +++ b/src/Nncase.Evaluator/Tensors/Slice.cs @@ -41,18 +41,24 @@ public IValue Visit(IEvaluateContext context, Slice sl) /// public IRType Visit(ITypeInferenceContext context, Slice target) { - var input = context.CheckArgumentType(target, Slice.Input); + var input = context.CheckArgumentType(target, Slice.Input); context.CheckArgumentType(target, Slice.Begins); context.CheckArgumentType(target, Slice.Ends); context.CheckArgumentType(target, Slice.Axes); context.CheckArgumentType(target, Slice.Strides); - return Visit(context, target, input); + return input switch + { + TensorType t => Visit(context, target, t), + DistributedType d => Visit(context, target, d), + AnyType => AnyType.Default, + _ => new InvalidType(input.GetType().Name), + }; } /// public Cost Visit(ICostEvaluateContext context, Slice target) { - var outputType = context.GetReturnType(); + var outputType = context.GetReturnType(); return new() { @@ -227,4 +233,21 @@ end is TensorConst ends_con && return input with { Shape = outShape }; } + + private IRType Visit(ITypeInferenceContext context, Slice target, DistributedType input) + { + var outType = Visit(context, target, input.TensorType); + if (outType is not TensorType tensorType) + { + return new InvalidType("not support input tensor type infer"); + } + + var axes = ((TensorConst)context.GetArgument(target, Slice.Axes)).Value.ToArray(); + if (input.NdSBP.Any(sbp => sbp is SBPSplit s && axes.Contains(s.Axis))) + { + return new InvalidType("not support input tensor type infer"); + } + + return new DistributedType((TensorType)outType, input.NdSBP, input.Placement); + } } diff --git a/src/Nncase.Evaluator/Tensors/Transpose.cs b/src/Nncase.Evaluator/Tensors/Transpose.cs index 77e74d07f1..d643370aa9 100644 --- a/src/Nncase.Evaluator/Tensors/Transpose.cs +++ b/src/Nncase.Evaluator/Tensors/Transpose.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using DryIoc.ImTools; using Nncase.CostModel; using Nncase.IR; using Nncase.IR.Tensors; @@ -64,15 +65,22 @@ public IValue Visit(IEvaluateContext context, Transpose tr) /// public IRType Visit(ITypeInferenceContext context, Transpose target) { - var input = context.CheckArgumentType(target, Transpose.Input); - return Visit(context, target, input); + var input = context.CheckArgumentType(target, Transpose.Input); + + return input switch + { + DistributedType d => Visit(context, target, d), + TensorType t => Visit(context, target, t), + AnyType => AnyType.Default, + _ => new InvalidType(input.GetType().ToString()), + }; } /// public Cost Visit(ICostEvaluateContext context, Transpose target) { - var inputType = context.GetArgumentType(target, Transpose.Input); - var outputType = context.GetReturnType(); + var inputType = context.GetArgumentType(target, Transpose.Input); + var outputType = context.GetReturnType(); return new() { @@ -102,4 +110,36 @@ private IRType Visit(ITypeInferenceContext context, Transpose target, TensorType var permExpr = context.GetArgument(target, Transpose.Perm); return TypeInference.TransposeType(input, permExpr); } + + private IRType Visit(ITypeInferenceContext context, Transpose target, DistributedType input) + { + if (Visit(context, target, input.TensorType) is not TensorType tensorType) + { + throw new InvalidOperationException(); + } + + var permExpr = context.GetArgument(target, Transpose.Perm); + if (permExpr is TensorConst permValue) + { + var perm = permValue.Value.ToArray(); + var ndsbp = new SBP[input.Placement.Rank]; + + for (int i = 0; i < input.Placement.Rank; i++) + { + switch (input.NdSBP[i]) + { + case SBPSplit { Axis: int ix }: + ndsbp[i] = SBP.S(perm.IndexOf(ix)); + break; + default: + ndsbp[i] = input.NdSBP[i]; + break; + } + } + + return new DistributedType(tensorType, ndsbp, input.Placement); + } + + return new InvalidType(input.ToString()); + } } diff --git a/src/Nncase.Evaluator/Tensors/UnSqueeze.cs b/src/Nncase.Evaluator/Tensors/UnSqueeze.cs index bd86940fee..23bed23403 100644 --- a/src/Nncase.Evaluator/Tensors/UnSqueeze.cs +++ b/src/Nncase.Evaluator/Tensors/UnSqueeze.cs @@ -27,9 +27,18 @@ public IValue Visit(IEvaluateContext context, Unsqueeze unSqueeze) /// public IRType Visit(ITypeInferenceContext context, Unsqueeze target) { - var input = context.CheckArgumentType(target, Unsqueeze.Input); + var input = context.CheckArgumentType(target, Unsqueeze.Input); _ = context.CheckArgumentType(target, Unsqueeze.Dim); - return Visit(context, target, input); + if (input is TensorType tensorType) + { + return Visit(context, target, tensorType); + } + else if (input is DistributedType distributedType) + { + return Visit(context, target, distributedType); + } + + return new InvalidType(input.GetType().Name); } /// @@ -81,4 +90,26 @@ private IRType Visit(ITypeInferenceContext context, Unsqueeze target, TensorType return input with { Shape = new Shape(Enumerable.Repeat(Dimension.Unknown, input.Shape.Rank + 1)) }; } + + private IRType Visit(ITypeInferenceContext context, Unsqueeze target, DistributedType input) + { + var tensorType = (TensorType)Visit(context, target, input.TensorType); + + var ndsbp = new SBP[input.NdSBP.Count]; + + if (context.GetArgument(target, Unsqueeze.Dim) is TensorConst tdims) + { + var dimsValue = tdims.Value.Cast(); + for (int i = 0; i < input.NdSBP.Count; i++) + { + ndsbp[i] = input.NdSBP[i] switch + { + SBPSplit { Axis: int axis } => SBP.S(axis + dimsValue.Select(i => i <= axis).Count(b => b)), + SBP sbp => sbp, + }; + } + } + + return new DistributedType(tensorType, ndsbp, input.Placement); + } } diff --git a/src/Nncase.Evaluator/TypeInference.cs b/src/Nncase.Evaluator/TypeInference.cs index a0749e8fdd..bbe74d1739 100644 --- a/src/Nncase.Evaluator/TypeInference.cs +++ b/src/Nncase.Evaluator/TypeInference.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection; using NetFabric.Hyperlinq; using Nncase.IR; +using Nncase.TIR; using static Nncase.IR.TypePatternUtility; namespace Nncase.Evaluator; diff --git a/src/Nncase.Evaluator/TypeInferenceVisitor.cs b/src/Nncase.Evaluator/TypeInferenceVisitor.cs index 8ffd28f715..36528fd045 100644 --- a/src/Nncase.Evaluator/TypeInferenceVisitor.cs +++ b/src/Nncase.Evaluator/TypeInferenceVisitor.cs @@ -52,28 +52,6 @@ protected override IRType VisitLeafBlock(Block expr) return TupleType.Void; } - /// - protected override IRType VisitLeafBufferLoad(BufferLoad expr) - { - IRType type; - VerifySubField(expr, expr.Buffer, TypePatternUtility.IsPointer()); - for (int i = 0; i < expr.Indices.Length; i++) - { - VerifySubField(expr, expr.Indices[i], TypePatternUtility.IsIntegralScalar(), $"BufferLoad.Indices[{i}]"); - } - - if (expr.Buffer.CheckedType is TensorType { IsScalar: true, DType: PointerType { ElemType: PrimType pointedType } }) - { - type = TensorType.Scalar(pointedType); - } - else - { - type = new InvalidType($"Can't load from {expr.Buffer.CheckedType}"); - } - - return type; - } - /// protected override IRType VisitLeafBufferRegion(BufferRegion expr) { @@ -91,27 +69,24 @@ protected override IRType VisitLeafBufferRegion(BufferRegion expr) } /// - protected override IRType VisitLeafBufferStore(BufferStore expr) + protected override IRType VisitLeafBuffer(Nncase.TIR.Buffer expr) { - VerifySubField(expr, expr.Buffer, TypePatternUtility.IsPointer()); - for (int i = 0; i < expr.Indices.Length; i++) + VerifySubField(expr, expr.MemSpan, TypePatternUtility.IsPointer() | TypePatternUtility.IsNoneType()); + foreach (var r in expr.Dimensions) { - VerifySubField(expr, expr.Indices[i], TypePatternUtility.IsIntegralScalar(), $"BufferStore.Indices[{i}]"); + VerifySubField(expr, r, TypePatternUtility.IsIntegralScalar()); } - VerifySubField(expr, expr.Value, TypePatternUtility.IsScalar()); - - IRType type; - if (expr.Value.CheckedType is TensorType { IsScalar: true, DType: PrimType valueType } && - expr.Buffer.CheckedType is TensorType { IsScalar: true, DType: PointerType { ElemType: PrimType pointedType } } - && valueType == pointedType) + foreach (var r in expr.Strides) { - type = TupleType.Void; + VerifySubField(expr, r, TypePatternUtility.IsIntegralScalar()); } - else + + var type = new TensorType(expr.ElemType, expr.Dimensions.AsValueEnumerable().Select(e => e switch { - type = new InvalidType($"Can't store {expr.Value.CheckedType} to {expr.Buffer.CheckedType}"); - } + TensorConst { Value: { Shape: { IsScalar: true } } t } => new Dimension(t.ToScalar()), + _ => Dimension.Unknown, + }).ToArray()); return type; } @@ -222,13 +197,6 @@ protected override IRType VisitLeafLet(Let expr) return type; } - /// - protected override IRType VisitLeafLogicalBuffer(LogicalBuffer expr) - { - var type = new TensorType(expr.ElemType, Shape.Unknown(expr.Rank)); - return type; - } - /// protected override IRType VisitLeafMarker(Marker expr) { @@ -251,13 +219,6 @@ protected override IRType VisitLeafOp(Op expr) return type; } - /// - protected override IRType VisitLeafPhysicalBuffer(PhysicalBuffer expr) - { - var type = new TensorType(expr.ElemType, new(expr.FixedDimensions)); - return type; - } - /// protected override IRType VisitLeafPrimFunction(PrimFunction expr) { @@ -318,6 +279,13 @@ protected override IRType VisitLeafVar(Var expr) return type; } + protected override IRType VisitLeafMemSpan(MemSpan expr) + { + VerifySubField(expr, expr.Start, TypePatternUtility.IsNoneType() | TypePatternUtility.IsIntegralScalar() | TypePatternUtility.IsPointer()); + VerifySubField(expr, expr.Size, TypePatternUtility.IsIntegralScalar()); + return expr.Start.CheckedType; + } + /// protected override IRType VisitLet(Let expr) { diff --git a/src/Nncase.Evaluator/packages.lock.json b/src/Nncase.Evaluator/packages.lock.json index 78fc35c9da..cf9c399201 100644 --- a/src/Nncase.Evaluator/packages.lock.json +++ b/src/Nncase.Evaluator/packages.lock.json @@ -13,11 +13,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "libortki": { @@ -87,8 +87,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -110,6 +110,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -169,6 +170,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Graph/packages.lock.json b/src/Nncase.Graph/packages.lock.json index a439ce21fd..ab3e724693 100644 --- a/src/Nncase.Graph/packages.lock.json +++ b/src/Nncase.Graph/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "libortki": { @@ -78,8 +78,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -101,6 +101,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -176,6 +177,12 @@ "libortki": "0.0.2" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.IO/packages.lock.json b/src/Nncase.IO/packages.lock.json index eb0c4a8b7c..ef24cbccbb 100644 --- a/src/Nncase.IO/packages.lock.json +++ b/src/Nncase.IO/packages.lock.json @@ -4,17 +4,17 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" } } } diff --git a/src/Nncase.Importer/Ncnn/NcnnModel.cs b/src/Nncase.Importer/Ncnn/NcnnModel.cs index 488e191ad4..ec67287e97 100644 --- a/src/Nncase.Importer/Ncnn/NcnnModel.cs +++ b/src/Nncase.Importer/Ncnn/NcnnModel.cs @@ -65,7 +65,7 @@ public static NcnnModel ParseFromStream(Stream stream) throw new InvalidDataException("parse magic failed"); } - if (reader.ReadLine()?.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) is not [var layerCountStr, var blobCountStr]) + if (reader.ReadLine()?.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) is not[var layerCountStr, var blobCountStr]) { throw new InvalidDataException("parse layer_count or blob_count failed"); } diff --git a/src/Nncase.Importer/Ncnn/ParamDict.cs b/src/Nncase.Importer/Ncnn/ParamDict.cs index 954525ea48..bc5c77e6d7 100644 --- a/src/Nncase.Importer/Ncnn/ParamDict.cs +++ b/src/Nncase.Importer/Ncnn/ParamDict.cs @@ -44,7 +44,7 @@ public void LoadFrom(ReadOnlySpan fields) { foreach (var field in fields) { - if (field.Split('=', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) is not [var idStr, var valueStr]) + if (field.Split('=', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) is not[var idStr, var valueStr]) { break; } diff --git a/src/Nncase.Importer/Onnx/Concat.cs b/src/Nncase.Importer/Onnx/Concat.cs index 5f0e471652..433036c7be 100644 --- a/src/Nncase.Importer/Onnx/Concat.cs +++ b/src/Nncase.Importer/Onnx/Concat.cs @@ -14,7 +14,7 @@ private Expr VisitConcat(NodeProto op) { var inputs = Enumerable.Range(0, op.Input.Count).Select(x => GetInputExpr(op, x)).ToArray(); var axis = GetIntAttribute(op, "axis"); - return F.Tensors.Concat(new Tuple(inputs), axis); + return F.Tensors.Concat(new Tuple(inputs), (int)axis); } } } diff --git a/src/Nncase.Importer/Onnx/DataGatter.cs b/src/Nncase.Importer/Onnx/DataGatter.cs index 8001655a67..0cd5da981c 100644 --- a/src/Nncase.Importer/Onnx/DataGatter.cs +++ b/src/Nncase.Importer/Onnx/DataGatter.cs @@ -105,7 +105,7 @@ private Tensor GetTensor(TensorProto tensor) var externalDataCount = tensor.ExternalData.Count; if (externalDataCount != 0) { - if (externalDataCount < 3 && externalDataCount > 5) + if (externalDataCount < 1 || externalDataCount > 5) { throw new NotSupportedException("NotSupport ExternalData format, only support location, offset, length, checksum"); } @@ -113,9 +113,9 @@ private Tensor GetTensor(TensorProto tensor) var parent = Directory.GetParent(CompileSession.CompileOptions.InputFile)?.FullName; var externalData = tensor.ExternalData; var location = Path.Join(parent, externalData[0].Value); - var offset = long.Parse(externalData[1].Value); - var length = int.Parse(externalData[2].Value); + var offset = externalDataCount > 1L ? long.Parse(externalData[1].Value) : 0; using var br = new BinaryReader(new FileStream(location, FileMode.Open)); + var length = externalDataCount > 1 ? int.Parse(externalData[2].Value) : (int)br.BaseStream.Length; br.BaseStream.Seek(offset, SeekOrigin.Begin); var buffer = br.ReadBytes(length); return Tensor.FromBytes(type, buffer, shape); diff --git a/src/Nncase.Importer/Onnx/Gather.cs b/src/Nncase.Importer/Onnx/Gather.cs index eb03f835f6..ae47bb396d 100644 --- a/src/Nncase.Importer/Onnx/Gather.cs +++ b/src/Nncase.Importer/Onnx/Gather.cs @@ -14,7 +14,7 @@ private Expr VisitGather(in NodeProto op) { var (input, indices) = GetInputExprs(op, 0, 1); var axis = GetIntAttribute(op, "axis", 0); - return F.Tensors.Gather(input, axis, indices); + return F.Tensors.Gather(input, (int)axis, indices); } } } diff --git a/src/Nncase.Importer/Onnx/OnnxImporter.cs b/src/Nncase.Importer/Onnx/OnnxImporter.cs index 3004d86822..b87178604a 100644 --- a/src/Nncase.Importer/Onnx/OnnxImporter.cs +++ b/src/Nncase.Importer/Onnx/OnnxImporter.cs @@ -52,7 +52,6 @@ protected override (IEnumerable Inputs, Dictionary VarMap) Cre { var bucketOptions = CompileSession.CompileOptions.ShapeBucketOptions; _fixVarMap = bucketOptions.FixVarMap; - _constTensors = _graph.Initializer .ToDictionary(tensor => tensor.Name, tensor => tensor); diff --git a/src/Nncase.Importer/Onnx/Softmax.cs b/src/Nncase.Importer/Onnx/Softmax.cs index cde4fbb3e6..acd65b9606 100644 --- a/src/Nncase.Importer/Onnx/Softmax.cs +++ b/src/Nncase.Importer/Onnx/Softmax.cs @@ -43,7 +43,7 @@ private Expr SoftmaxV13Process(in NodeProto op, Func f) { var input = GetSingleInputExpr(op); var axis = GetIntAttribute(op, "axis", -1); - return f(input, axis); + return f(input, IR.F.Math.Select(axis < 0, (Rank(input) + axis)[0], axis)); } private Expr SoftmaxV1(in NodeProto op) diff --git a/src/Nncase.Importer/packages.lock.json b/src/Nncase.Importer/packages.lock.json index 845d535c0e..3a6d65fc28 100644 --- a/src/Nncase.Importer/packages.lock.json +++ b/src/Nncase.Importer/packages.lock.json @@ -22,11 +22,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.Bcl.AsyncInterfaces": { @@ -85,8 +85,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -371,6 +371,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -454,6 +455,12 @@ "resolved": "2.0.0", "contentHash": "ir3uek0+7Y8SwOUGUR8y94sgpVDWLAjKGBm9z7cLe/38GyPxWbIYHPnHZHksNTExTsx3Ie9GtwagkgR/jm64hA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Passes/BufferSchedule/BufferScheduleExtensions.cs b/src/Nncase.Passes/BufferSchedule/BufferScheduleExtensions.cs new file mode 100644 index 0000000000..4a07e97a8b --- /dev/null +++ b/src/Nncase.Passes/BufferSchedule/BufferScheduleExtensions.cs @@ -0,0 +1,25 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Nncase.IR; + +namespace Nncase.Passes.BufferSchedule; + +public static class BufferScheduleExtensions +{ + public static IEnumerable GetArguments(this Call call) + { + var hs = new HashSet(ReferenceEqualityComparer.Instance); + hs.UnionWith(call.Arguments.ToArray().Where(e => e is not (BaseFunction or Const)).ToArray().Select(e => e switch { IR.Tuple tp => tp.Fields.ToArray(), _ => new[] { e } }).SelectMany(i => i)); + return hs; + } + + public static IEnumerable GetUsers(this Call call) + { + var hs = new HashSet(ReferenceEqualityComparer.Instance); + hs.UnionWith(call.Users.Where(e => e is not BaseFunction).ToArray().Select(e => e switch { IR.Tuple tp => tp.Users.ToArray(), _ => new[] { e } }).SelectMany(i => i)); + return hs; + } +} diff --git a/src/Nncase.Passes/BufferSchedule/BufferScheduleTypes.cs b/src/Nncase.Passes/BufferSchedule/BufferScheduleTypes.cs new file mode 100644 index 0000000000..13e8ab86f0 --- /dev/null +++ b/src/Nncase.Passes/BufferSchedule/BufferScheduleTypes.cs @@ -0,0 +1,77 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +namespace Nncase.Passes.BufferSchedule; + +internal sealed class TimeInterval +{ + public TimeInterval(int start, int end) + { + Brith = start; + Death = end; + } + + public int Brith { get; set; } + + public int Death { get; set; } + + public int Size => Death - Brith; + + public override string ToString() + { + return $"TimeInterval({Brith}, {Death})"; + } +} + +internal sealed class MemSpan +{ + public MemSpan(int start, int end) + { + Start = start; + End = end; + } + + public int Start { get; set; } + + public int End { get; set; } + + public int Size => End - Start; + + public override string ToString() + { + return $"MemSpan({Start}, {End})"; + } +} + +internal class ScheduleBuffer +{ + public ScheduleBuffer(string name, int number, TimeInterval interval, MemSpan span, int[] shape, int[] strides, bool inplace) + { + Name = name; + Number = number; + Interval = interval; + Span = span; + Shape = shape; + Strides = strides; + Inplace = inplace; + } + + public string Name { get; } + + public int Number { get; } + + public TimeInterval Interval { get; } + + public MemSpan Span { get; } + + public int[] Shape { get; } + + public int[] Strides { get; } + + public bool Inplace { get; } + + public override string ToString() + { + return $"ScheduledBuffer('{Name}', {Number}, {Interval}, {Span}, ConstraintsMode.No, [{string.Join(",", Shape)}], [{string.Join(",", Strides)}], {Inplace})"; + } +} diff --git a/src/Nncase.Passes/BufferSchedule/BufferScheduler.cs b/src/Nncase.Passes/BufferSchedule/BufferScheduler.cs new file mode 100644 index 0000000000..25f7d6f5b8 --- /dev/null +++ b/src/Nncase.Passes/BufferSchedule/BufferScheduler.cs @@ -0,0 +1,215 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reactive; +using Google.OrTools.Sat; +using NetFabric.Hyperlinq; +using Nncase; +using Nncase.IR; + +namespace Nncase.Passes.BufferSchedule; + +internal sealed class BufferScheduler +{ + public IReadOnlyDictionary CollectLifeTime(Function func) + { + var c = new LifeTimeCollector(); + return c.Collect(func); + } + + public void Schedule(IReadOnlyDictionary bufferMap) + { + var model = new CpModel(); + var noOverlap = model.AddNoOverlap2D(); + var boxs = new Dictionary(ReferenceEqualityComparer.Instance); + var timeMap = new Dictionary>(); + var yStarts = new List(); + foreach (var (expr, item) in bufferMap) + { + var xInterval = model.NewIntervalVar(model.NewConstant(item.Interval.Brith), model.NewConstant(item.Interval.Size), model.NewConstant(item.Interval.Death), item.Name + $"{item.Number}_x"); + + var upbound = 2147483648 - item.Span.End; + if (upbound <= 0) + { + throw new System.NotSupportedException(); + } + + var memStartVar = model.NewIntVar(0, upbound, $"{item.Name}_{item.Number}_y_start"); + var yInterval = model.NewFixedSizeIntervalVar(memStartVar, item.Span.End, $"{item.Name}_{item.Number}_y"); + noOverlap.AddRectangle(xInterval, yInterval); + yStarts.Add(memStartVar); + boxs.Add(expr, (xInterval, yInterval)); + + for (int time = item.Interval.Brith; time < item.Interval.Death; time++) + { + if (!timeMap.TryGetValue(time, out var timelist)) + { + timelist = new(); + timeMap.Add(time, timelist); + } + + timelist.Add(expr); + } + } + + foreach (var (expr, item) in bufferMap) + { + if (expr is Call { Target: IR.Tensors.Concat } concatCall && concatCall.Arguments[0] is IR.Tuple tuple) + { + // the concat inputs must contiguous + int offset = 0; + for (int i = 0; i < tuple.Fields.Length; i++) + { + model.Add((boxs[concatCall].Y.StartExpr() + offset) == boxs[tuple.Fields[i]].Y.StartExpr()); + offset += bufferMap[tuple.Fields[i]].Span.Size; + } + } + else if (expr is Call { Target: IR.Tensors.Split } splitCall) + { + // the split must equal with input. + model.Add(boxs[splitCall].Y.StartExpr() == boxs[splitCall.Arguments[0]].Y.StartExpr()); + + // the split outputs must contiguous + var users = splitCall.GetUsers(); + int offset = 0; + foreach (var user in users.OrderBy(e => ((Call)e).Arguments[1].Evaluate().AsTensor().ToScalar())) + { + model.Add((boxs[splitCall].Y.StartExpr() + offset) == boxs[user].Y.StartExpr()); + offset += bufferMap[user].Span.Size; + } + } + else if (expr is Call { Target: IR.Tensors.Reshape } reshapCall) + { + // the reshape must equal with it's input. + model.Add(boxs[reshapCall].Y.StartExpr() == boxs[reshapCall.Arguments[0]].Y.StartExpr()); + } + } + + model.Minimize(LinearExpr.Sum(yStarts)); + + var solver = new CpSolver(); + solver.StringParameters = $"max_time_in_seconds:{60},num_workers:{8}"; + CpSolverStatus solve_status = solver.Solve(model); + if (solve_status != CpSolverStatus.Optimal && solve_status != CpSolverStatus.Feasible) + { + throw new System.NotSupportedException(); + } + + foreach (var (k, v) in bufferMap) + { + bufferMap[k].Span.Start = checked((int)solver.Value(boxs[k].Y.StartExpr())); + bufferMap[k].Span.End = checked((int)solver.Value(boxs[k].Y.EndExpr())); + } + } + + public void Dump(Stream fs, IReadOnlyDictionary buffers) + { + using (var wr = new StreamWriter(fs)) + { + wr.Write(@"from bokeh.models import ColumnDataSource, HoverTool, FuncTickFormatter, SingleIntervalTicker, SaveTool, WheelZoomTool, WheelPanTool, ResetTool +from bokeh.palettes import Category20_20 as palette +from bokeh.plotting import figure, show, save +import itertools +from dataclasses import dataclass +from enum import Enum +from typing import List + +@dataclass +class TimeInterval(): + start: int + end: int + def __str__(self) -> str: + return f'(start: {self.start}, end {self.end})' + +@dataclass +class MemSpan(): + depth_start: int + depth_end: int + def __str__(self) -> str: + return f'(start: {self.depth_start}, size {self.depth_end - self.depth_start})' + +class ConstraintsMode(Enum): + No = 0 + Channel = 1 + +@dataclass +class ScheduledBuffer(): + name: str + number: int + interval: TimeInterval + location: MemSpan + constraints: ConstraintsMode + shape: List[int] + stride: List[int] + inplace: bool + +colors = itertools.cycle(palette) + +buffers = [ +"); + foreach (var (_, v) in buffers) + { + wr.WriteLine(v.ToString() + ","); + } + + wr.Write(@"] + +source = { + 'name': [], + 'x': [], + 'y': [], + 'width': [], + 'height': [], + 'alpha': [], + 'color': [], + 'location': [], + 'interval': [], + 'shape': [], + 'stride': [], +} + +y_range_max = 0 +x_range_max = 0 +color_dict = {} +for buffer in buffers: + source['name'].append(buffer.name) + width = buffer.interval.end - buffer.interval.start + x = buffer.interval.start + (width / 2) + height = buffer.location.depth_end - buffer.location.depth_start + y = buffer.location.depth_start + (height / 2) + y_range_max = max(y_range_max, y) + x_range_max = max(x_range_max, buffer.interval.end) + source['x'].append(x) + source['y'].append(y) + source['width'].append(width) + source['height'].append(height) + color = color_dict.get(buffer.name) + if color == None: + color = next(colors) + color_dict[buffer.name] = color + source['color'].append(color) + source['alpha'].append(0.2 if buffer.inplace else 1.0) + source['interval'].append(str(buffer.interval)) + source['location'].append(str(buffer.location)) + source['shape'].append(','.join([str(s) for s in buffer.shape])) + source['stride'].append(','.join([str(s) for s in buffer.stride])) + +source = ColumnDataSource(source) +hover = HoverTool(tooltips=[('name', '@name'), ('interval', '@interval'), ('location', '@location'), + ('shape', '@shape'), ('stride', '@stride')]) + +p = figure(tools=[hover, WheelPanTool(), SaveTool(), WheelZoomTool(), ResetTool()], width=1280, height=720, + y_range=(0, y_range_max * 1.2), x_range=(-1, x_range_max + 1), + title='Local Buffer LifeTime (by Steps)') +p.rect(x='x', y='y', width='width', height='height', fill_color='color', legend_field='name', fill_alpha='alpha', source=source) +p.xaxis.axis_label = 'Time (steps)' +p.outline_line_color = None + +show(p)"); + } + } +} diff --git a/src/Nncase.Passes/BufferSchedule/LifeTimeCollector.cs b/src/Nncase.Passes/BufferSchedule/LifeTimeCollector.cs new file mode 100644 index 0000000000..1edcc263dd --- /dev/null +++ b/src/Nncase.Passes/BufferSchedule/LifeTimeCollector.cs @@ -0,0 +1,168 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive; +using Nncase; +using Nncase.IR; + +namespace Nncase.Passes.BufferSchedule; + +internal sealed class LifeTimeCollector : ExprVisitor +{ + public int TimeStamp { get; private set; } + + public Dictionary LifenessMap { get; } = new(ReferenceEqualityComparer.Instance); + + public IReadOnlyDictionary Collect(Function entry) + { + Visit(entry.Body); + Update(entry.Body); // avoid final call time interval size == 1. + Alias(); + + var d = new Dictionary(ReferenceEqualityComparer.Instance); + int count = 0; + foreach (var (k, v) in LifenessMap) + { + var name = k switch + { + Call c => c.Target.GetType().Name, + Var va => va.Name, + _ => k.GetType().Name, + }; + var size = GetSize(k.CheckedType, out var shape, out var stride); + + d.Add(k, new(name, count++, v, new(0, size), shape, stride, false)); + } + + return d; + } + + protected override Unit DefaultVisitLeaf(Expr expr) => Unit.Default; + + protected override Unit VisitLeafCall(Call expr) + { + foreach (var arg in expr.Arguments) + { + Update(arg); + } + + Update(expr); + + TimeStamp += 2; + + // note we will update tuple field on the next call. + foreach (var item in expr.Users.Where(e => e is not (BaseFunction or IR.Tuple))) + { + Update(item); + } + + return Unit.Default; + } + + private void Update(Expr expr) + { + if (expr is Const or None) + { + return; + } + + if (expr is IR.Tuple t) + { + foreach (var item in t.Fields) + { + Update(item); + } + + return; + } + + if (!LifenessMap.TryGetValue(expr, out var interval)) + { + interval = new(TimeStamp, TimeStamp + 1); + } + else + { + interval.Death = TimeStamp + 1; + } + + LifenessMap[expr] = interval; + } + + private void Alias() + { + bool changed; + do + { + changed = false; + foreach (var (expr, interval) in LifenessMap) + { + if (expr is Call { Target: IR.Tensors.Reshape } callReshape) + { + changed = AliasTime(callReshape, interval); + } + } + + foreach (var (expr, interval) in LifenessMap) + { + if (expr is Call { Target: IR.Tensors.Concat } concatCall) + { + changed = AliasTime(concatCall, interval); + } + } + + foreach (var (expr, interval) in LifenessMap) + { + if (expr is Call { Target: IR.Tensors.Split } splitCall) + { + changed = AliasTime(splitCall, interval); + } + } + } while (changed); + } + + private bool AliasTime(Call call, TimeInterval interval) + { + var brith = call.GetArguments().Select(arg => LifenessMap[arg].Death).Concat(new[] { interval.Brith }).Max(); + var death = call.GetUsers().Select(usr => LifenessMap[usr].Brith).Concat(new[] { interval.Death }).Min(); + + if (brith == interval.Brith && death == interval.Death) + { + return false; + } + + if (brith >= death) + { + throw new InvalidOperationException(); + } + + interval.Brith = brith; + interval.Death = death; + return true; + } + + private int GetSize(IRType type, out int[] shape, out int[] stride) + { + shape = Array.Empty(); + stride = Array.Empty(); + var size = 0; + if (type is TensorType tensorType) + { + shape = tensorType.Shape.ToValueArray(); + stride = TensorUtilities.GetStrides(shape); + size = TensorUtilities.GetSize(shape, stride, tensorType.DType.SizeInBytes); + } + else if (type is TupleType tupleType) + { + size = 0; + foreach (var item in tupleType) + { + size += GetSize(item, out _, out _); + } + } + + return size; + } +} diff --git a/src/Nncase.Passes/DDrBufferSchdeulePass.cs b/src/Nncase.Passes/DDrBufferSchdeulePass.cs index 8afdb3c5e0..80aebda267 100644 --- a/src/Nncase.Passes/DDrBufferSchdeulePass.cs +++ b/src/Nncase.Passes/DDrBufferSchdeulePass.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Reactive; using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -23,9 +24,9 @@ namespace Nncase.Passes; /// public sealed class DDrBufferSchdeulePass : ModulePass { - private readonly Dictionary> _module_usage = new(); + private readonly Dictionary> _moduleUsage = new(); - private readonly Dictionary> _module_hashset = new(); + private readonly Dictionary>> _moduleRdataMaps = new(); private readonly bool _enbaleMergeCall; @@ -42,41 +43,16 @@ protected override async Task RunCoreAsync(IRModule module, RunPassCon // 1. merge the all call prim func if (_enbaleMergeCall) { - HashSet mergedFuncs = new(ReferenceEqualityComparer.Instance); - HashSet stackvmFuncs = new(ReferenceEqualityComparer.Instance); - for (int i = 0; i < module.Functions.Count; i++) + if (module.Entry is Function { ModuleKind: Callable.StackVMModuleKind, Body: Expr body } func && IsFixedType(body.CheckedType)) { - if (module.Functions[i] is Function { ModuleKind: "stackvm" } func) + var sch = new BufferSchedule.BufferScheduler(); + var buffers = sch.CollectLifeTime(func); + sch.Schedule(buffers); + using (var fs = Diagnostics.DumpScope.Current.OpenFile("draw_buffers.py")) { - var analysis = new Dictionary - { - [typeof(IExprUserAnalysisResult)] = AnalyzerManager.GetAnaylsis(func), - }; - _ = new HashSet(ReferenceEqualityComparer.Instance); - var mergePass = new DataflowPass(); - mergePass.Add(mergedFuncs); - var post = await mergePass.RunAsync(func, new() { AnalysisResults = analysis, RewriteOnce = true }); - module.Replace(i, post); - stackvmFuncs.Add(post); - } - } - - // 2. add the ext func into module. - foreach (var func in stackvmFuncs) - { - var collector = new ExternalFuncCollector(); - collector.Visit(func); - foreach (var ext_func in collector.GetExternalFuncs()) - { - module.Add(ext_func); + sch.Dump(fs, buffers); } } - - // 3. remove the all merged funcs - foreach (var item in mergedFuncs) - { - module.Remove(item); - } } // 4. schedule the prim funcs. @@ -86,149 +62,121 @@ protected override async Task RunCoreAsync(IRModule module, RunPassCon { if (!prim_func.SchedResult.IsScheduled) { - var ddr_allocator = new DDrBufferAllocator(_module_usage, _module_hashset); - ddr_allocator.Visit(prim_func); // changed ddr buffer. - prim_func.SchedResult.DataUsage = ddr_allocator.DataUsage; - prim_func.SchedResult.IsScheduled = ddr_allocator.Changed; + var rewriter = new DDrBufferRewriter(_moduleUsage, _moduleRdataMaps); + var post = (TIR.PrimFunction)rewriter.Rewrite(prim_func); // changed ddr buffer. + if (rewriter.IsMutated) + { + post.SchedResult.DataUsage = rewriter.DataUsage; + post.SchedResult.IsScheduled = true; + } + + module.Replace(i, prim_func); } } } - _module_hashset.Clear(); - _module_usage.Clear(); + _moduleRdataMaps.Clear(); + _moduleUsage.Clear(); return await Task.FromResult(module); } + + private bool IsFixedType(IRType type) => type switch + { + TensorType tensorType => tensorType.Shape.IsFixed, + TupleType tupleType => tupleType.Fields.All(IsFixedType), + _ => false, + }; } -/// -/// collect and assgin the PhysicalBuffer. -/// -internal sealed class DDrBufferAllocator : ExprVisitor +internal sealed class DDrBufferRewriter : ExprRewriter { - private readonly Dictionary _functionUsage; - private readonly HashSet _functionHashset; + private readonly Dictionary _functionUsage; + private readonly Dictionary> _functionRdatas; - private PrimFunction? _entry; - - public DDrBufferAllocator(Dictionary> module_usage, Dictionary> module_hashset) + public DDrBufferRewriter(Dictionary> moduleUsage, Dictionary>> moduleRdataMaps) { - ModuleUsage = module_usage; - ModuleHashSet = module_hashset; + ModuleUsage = moduleUsage; + ModuleRdataMaps = moduleRdataMaps; _functionUsage = new(); - _functionHashset = new(ReferenceEqualityComparer.Instance); + _functionRdatas = new(); Changed = false; } - public Dictionary> ModuleUsage { get; } + public Dictionary> ModuleUsage { get; } - public Dictionary> ModuleHashSet { get; } + public Dictionary>> ModuleRdataMaps { get; } public bool Changed { get; private set; } - public int DataUsage => _functionUsage.GetValueOrDefault(Schedule.MemoryLocation.Data, 0); + public long DataUsage => _functionUsage.GetValueOrDefault(MemoryLocation.Data, 0); + + public PrimFunction Entry => (PrimFunction)VisitRoot!; - /// - /// only visit one prim func. - /// - protected override bool VisitPrimFunction(PrimFunction primFunction) + protected override Expr RewriteLeafBuffer(TIR.Buffer expr) { - _entry ??= primFunction; - if (object.ReferenceEquals(_entry, primFunction)) + if (expr.MemSpan is { Location: TIR.MemoryLocation.Input or TIR.MemoryLocation.Output, Start: None, Size: TensorConst size } memSpan) { - foreach (var physical in primFunction.Parameters) + // input/output write into the FunctionUsage + if (!_functionUsage.TryGetValue(memSpan.Location, out var start)) { - if (physical.MemLocation is Schedule.MemoryLocation.Input or Schedule.MemoryLocation.Output) - { - // avoid visit same buffer - if (!_functionHashset.Contains(physical)) - { - // input/output write into the FunctionUsage - if (!_functionUsage.TryGetValue(physical.MemLocation, out var start)) - { - start = 0; - } - - physical.Start = start; - _functionUsage[physical.MemLocation] = start + physical.Size; - _functionHashset.Add(physical); - Changed = true; - } - } - else - { - throw new NotSupportedException($"The prim function parameters mem location must be input/output but get {physical.MemLocation}!"); - } + start = 0; } - return base.VisitPrimFunction(_entry); + _functionUsage[memSpan.Location] = start + size.Value.ToScalar(); + Changed = true; + + return expr.With(memSpan: memSpan.With(start: Tensor.FromPointer((ulong)start, expr.ElemType))); } - return true; + return expr; } - protected override bool VisitLeafBuffer(TIR.Buffer buffer) + protected override TIR.MemSpan RewriteLeafMemSpan(TIR.MemSpan memSpan) { - if (buffer is not TIR.PhysicalBuffer physical) - { - return true; - } - - // rdata write into the moduleUsage - if (physical.MemLocation is Schedule.MemoryLocation.Rdata) + if (memSpan is { Location: MemoryLocation.Rdata, Start: Call { Target: IR.Buffers.DDrOf, Arguments: var arg } } && arg[0] is Const { ValueType: TensorType constType } @const) { - if (!ModuleHashSet.TryGetValue(_entry!.ModuleKind, out var module_hashset)) + if (!ModuleRdataMaps.TryGetValue(Entry.ModuleKind, out var moduleRdataMap)) { - module_hashset = new(ReferenceEqualityComparer.Instance); - ModuleHashSet.Add(_entry!.ModuleKind, module_hashset); + moduleRdataMap = new(); + ModuleRdataMaps.Add(Entry.ModuleKind, moduleRdataMap); } - if (!ModuleUsage.TryGetValue(_entry!.ModuleKind, out var module_usage)) + if (!ModuleUsage.TryGetValue(Entry.ModuleKind, out var moduleUsage)) { - module_usage = new(); - ModuleUsage.Add(_entry!.ModuleKind, module_usage); + moduleUsage = new(); + ModuleUsage.Add(Entry.ModuleKind, moduleUsage); } - if (!module_hashset.Contains(physical)) + if (!moduleRdataMap.TryGetValue(@const, out var memRange)) { - if (!module_usage.TryGetValue(physical.MemLocation, out var start)) + if (!moduleUsage.TryGetValue(memSpan.Location, out var start)) { start = 0; } - physical.Start = start; - module_usage[physical.MemLocation] = start + physical.Size; - module_hashset.Add(physical); - _entry.SchedResult.Rdatas.Add(physical); - + _ = ComputeSize(@const); + moduleUsage[memSpan.Location] = start + ComputeSize(@const); + memRange = new(start, start + ComputeSize(@const)); + moduleRdataMap.Add(@const, memRange); + Entry.SchedResult.Rdatas.Add(@const, memRange); Changed = true; } - } - else if (physical.MemLocation is Schedule.MemoryLocation.Data) - { - // data write into the FunctionUsage - if (!_functionHashset.Contains(physical)) - { - if (!_functionUsage.TryGetValue(physical.MemLocation, out var start)) - { - start = 0; - } - physical.Start = start; - _functionUsage[physical.MemLocation] = start + physical.Size; - _functionHashset.Add(physical); - Changed = true; - } - } - else if (physical.MemLocation is Schedule.MemoryLocation.SharedData) - { - throw new NotSupportedException("Current Not Support!"); + return memSpan.With(new TensorConst(Tensor.FromPointer((ulong)memRange.Min, constType.DType)), memRange.Max - memRange.Min); } - return true; + return memSpan; } - protected override bool DefaultVisitLeaf(Expr expr) => true; + private long ComputeSize(IValue v) => v.AsTensors().Select(t => t.BytesBuffer.Length).Sum(); + + private long ComputeSize(Const @const) => @const switch + { + TensorConst { Value: Tensor tc } => tc.BytesBuffer.Length, + TupleConst tc => ComputeSize(tc.Value), + _ => throw new NotSupportedException(), + }; } internal sealed class ExternalFuncCollector : ExprWalker diff --git a/src/Nncase.Passes/EGraphExtractPass.cs b/src/Nncase.Passes/EGraphExtractPass.cs index d4ebe21ebc..2c2baa12b8 100644 --- a/src/Nncase.Passes/EGraphExtractPass.cs +++ b/src/Nncase.Passes/EGraphExtractPass.cs @@ -24,7 +24,7 @@ public EGraphExtractPass(IBaseFuncCostEvaluator? costEvaluator = null) protected override Task RunCoreAsync(IEGraph input, RunPassContext context) { - var post = (BaseFunction)input.Extract(input.Root!, _costEvaluator); + var post = (BaseFunction)input.Extract(input.Root!, _costEvaluator, out _); IRHelpers.DCE(post); return Task.FromResult(post); } diff --git a/src/Nncase.Passes/Mutators/IFusionMergeRule.cs b/src/Nncase.Passes/Mutators/IFusionMergeRule.cs index 15878bcc71..9f72015746 100644 --- a/src/Nncase.Passes/Mutators/IFusionMergeRule.cs +++ b/src/Nncase.Passes/Mutators/IFusionMergeRule.cs @@ -668,14 +668,8 @@ private bool ProcessFusionMerge(Func mergedFusionRewriteCallBack, Fu { if (caller_inputs[i] is Call { Target: Fusion }) { - Fusion callee_fusion; - try + if (result.GetValueOrDefault($"callee_fusion_{i}") is not Fusion callee_fusion) { - callee_fusion = (Fusion)result[$"callee_fusion_{i}"]; - } - catch (KeyNotFoundException) - { - // when matched fusion(fusion(x,y)), the input => fusion(x,y) return false; } diff --git a/src/Nncase.Passes/Rules/Neutral/AddRangeOfAndMarker.cs b/src/Nncase.Passes/Rules/Neutral/AddRangeOfAndMarker.cs index ac7cb9feb6..e75489c638 100644 --- a/src/Nncase.Passes/Rules/Neutral/AddRangeOfAndMarker.cs +++ b/src/Nncase.Passes/Rules/Neutral/AddRangeOfAndMarker.cs @@ -10,6 +10,7 @@ using Nncase.IR.Imaging; using Nncase.IR.Math; using Nncase.IR.NN; +using Nncase.IR.RNN; using Nncase.IR.Tensors; using Nncase.PatternMatch; using static Nncase.IR.TypePatternUtility; diff --git a/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs b/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs index e046301045..9519b011c7 100644 --- a/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs +++ b/src/Nncase.Passes/Rules/Neutral/CombineQuantize.cs @@ -34,11 +34,12 @@ public sealed partial class CombineQuantizeConcat : RewriteRule "quantize", _ => true, IsConcat( - IsTuple(IsVArgsRepeat("tupleInputs", () => IsWildcard())), - IsWildcard("axis")), + "concat", + _ => true, + IsTuple(IsVArgsRepeat("tupleInputs", () => IsWildcard()))), IsWildcard("quantParam")); - private Expr? GetReplace(Quantize quantize, IReadOnlyList tupleInputs, Expr axis, Expr quantParam, RunPassContext options) + private Expr? GetReplace(Quantize quantize, IReadOnlyList tupleInputs, IR.Tensors.Concat concat, Expr quantParam, RunPassContext options) { if (options.Driver is DataflowPass) { @@ -54,7 +55,7 @@ public sealed partial class CombineQuantizeConcat : RewriteRule } } - return Concat(new IR.Tuple(tupleInputs.Select(e => IR.F.Math.Quantize(e, quantParam, quantize.TargetType)).ToArray()), axis); + return Concat(new IR.Tuple(tupleInputs.Select(e => IR.F.Math.Quantize(e, quantParam, quantize.TargetType)).ToArray()), concat.Axis); } } diff --git a/src/Nncase.Passes/Rules/Neutral/CombineReshape.cs b/src/Nncase.Passes/Rules/Neutral/CombineReshape.cs index 3f0c2be75d..8c36b0b537 100644 --- a/src/Nncase.Passes/Rules/Neutral/CombineReshape.cs +++ b/src/Nncase.Passes/Rules/Neutral/CombineReshape.cs @@ -201,3 +201,61 @@ public sealed partial class CombineReshapePad : IRewriteRule return null; } } + +/// +/// combine reshape transpose +/// e.g. : +/// %5 // f32[1,77,768] +/// %6 = Reshape(%5, const(i64[4] : {1L,77L,12L,64L})): // f32[1,77,12,64] +/// %7 = Transpose(%6, const(i64[4] : {0L,2L,1L,3L})): // f32[1,12,77,64] +/// %8 = Reshape(%7, const(i32[3] : {12,77,64})): // f32[12,77,64]. +/// after combine : +/// %5 // f32[1,77,768] +/// %6 = Reshape(%5, const(i64[4] : {1L,77L,12L,64L})): // f32[1,77,12,64] +/// %7 = Reshape(%6, const(i64[3] : {77L,12L,64L})): // f32[77L,12L,64L]. +/// %8 = Transpose(%7, const(i64[4] : {1L,0L,2L})): // f32[12,77,64]. +/// then use foldreshape. +/// +[RuleGenerator] +public sealed partial class CombineReshapeTranspose : IRewriteRule +{ + /// + public IPattern Pattern { get; } = IsReshape( + IsTranspose( + null, + "trans", + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("perm")) with + { TypePattern = HasFixedShape() }, + IsTensorConst("newShape")); + + private Expr? GetReplace(Expr input, Call trans, int[] newShape, int[] perm) + { + var transShape = trans.CheckedShape.ToValueArray(); + + if (transShape.Length == newShape.Length + 1) + { + // check reshape is sequeeze + var viewAxis = RulesUtility.FindSqueezeAxis(transShape, newShape); + if (viewAxis == -1) + { + return null; + } + + var inv = perm.Select((p, i) => (p, i)).OrderBy(tp => tp.p).ToArray(); + var invViewAxis = inv.Where(tp => tp.i == viewAxis).First().p; + var invPerm = perm.ToList(); + var invNewShape = input.CheckedShape.ToValueList(); + invNewShape.RemoveAt(invViewAxis); + invPerm.Remove(invViewAxis); + return IR.F.Tensors.Transpose(IR.F.Tensors.Reshape(input, invNewShape.ToArray()), invPerm.Select(i => i > invViewAxis ? i - 1 : i).ToArray()); + } + else if (transShape.Length == newShape.Length - 1) + { + // check rehsape is unsequeeze + return null; + } + + return null; + } +} diff --git a/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs b/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs index dbe323c1ee..4d979c8a56 100644 --- a/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs +++ b/src/Nncase.Passes/Rules/Neutral/CombineTranspose.cs @@ -194,6 +194,7 @@ public sealed partial class CombineTransposeConcat : IRewriteRule public IPattern Pattern { get; } = IsConcat( "concat", "concatCall", + _ => true, PatternMatch.Utility.IsTuple(null, IsVArgsRepeat("tupleInputs", exprs => { var patterns = new Pattern[exprs.Length]; @@ -203,11 +204,11 @@ public sealed partial class CombineTransposeConcat : IRewriteRule } return patterns; - })), - IsTensorConst("axis")); + }))); - private Expr? GetReplace(Expr concat, Call concatCall, IReadOnlyList tupleInputs, int axis, IMatchResult matchResult) + private Expr? GetReplace(IR.Tensors.Concat concat, Call concatCall, IReadOnlyList tupleInputs, IMatchResult matchResult) { + int axis = concat.Axis; var inputs = Enumerable.Range(0, tupleInputs.Count).Select(i => (Expr)matchResult[$"input_{i}"]); var perms = new HashSet>(Enumerable.Range(0, tupleInputs.Count).Select(i => ((TensorConst)matchResult[$"perm_{i}"]).Value.Cast(CastMode.KDefault))); @@ -343,6 +344,50 @@ public sealed partial class CombineTransposeReduce : IRewriteRule } } +/// +/// x // [12, 77, 64] +/// transpose(reshape(x, [1, 12, 77, 64]), [0, 2, 1, 3]) => reshape(transpose(x, [1, 0, 2]), [1, 77, 12, 64]). +/// +[RuleGenerator] +public sealed partial class CombineTransposeReshape : IRewriteRule +{ + /// + public IPattern Pattern { get; } = IsTranspose( + null, + "trans", + IsReshape( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("newShape")) with + { TypePattern = HasFixedShape() }, + IsTensorConst("perm")); + + private Expr? GetReplace(Call trans, Expr input, int[] newShape, int[] perm) + { + var inShape = input.CheckedShape.ToValueArray(); + var outShape = trans.CheckedShape.ToValueArray(); + if (!(newShape.Length == inShape.Length + 1)) + { + return null; + } + + // check reshape is sequeeze + var axis = RulesUtility.FindSqueezeAxis(newShape, inShape); + if (axis == -1) + { + return null; + } + + var newPerm = perm.ToList(); + newPerm.Remove(axis); + newPerm = newPerm.Select(i => i > axis ? i - 1 : i).ToList(); + + var inv = perm.Select((p, i) => (p, i)).OrderBy(tp => tp.p).ToArray(); + var invNewShape = newPerm.Select(i => inShape[i]).ToList(); + invNewShape.Insert(perm.ToList().IndexOf(axis), 1); + return Reshape(Transpose(input, newPerm.ToArray()), invNewShape.ToArray()); + } +} + /// /// Combine Transpose with Unary /// reduce(transpose(x,p), a) => transpose(reduce(x, invtranspose(a, p)), p). diff --git a/src/Nncase.Passes/Rules/Neutral/FocusFull.cs b/src/Nncase.Passes/Rules/Neutral/FocusFull.cs index 3aa0e6b227..5c8b749140 100644 --- a/src/Nncase.Passes/Rules/Neutral/FocusFull.cs +++ b/src/Nncase.Passes/Rules/Neutral/FocusFull.cs @@ -20,18 +20,19 @@ public sealed partial class FocusFull : RewriteRule /// public override Pattern Pattern { get; } = IsConcat( - null, + "concat", "concatCall", + _ => true, PatternMatch.Utility.IsTuple("tp", new[] { IsSlice(Input, IsTensorConst("begin0"), IsTensorConst("end0"), IsTensorConst("axes0"), IsTensorConst("stride0")), IsSlice(Input, IsTensorConst("begin1"), IsTensorConst("end1"), IsTensorConst("axes1"), IsTensorConst("stride1")), IsSlice(Input, IsTensorConst("begin2"), IsTensorConst("end2"), IsTensorConst("axes2"), IsTensorConst("stride2")), IsSlice(Input, IsTensorConst("begin3"), IsTensorConst("end3"), IsTensorConst("axes3"), IsTensorConst("stride3")), - }), - IsTensorConst("axis")); + })); - private Expr? GetReplace(Call concatCall, Expr input, int[] begin0, long[] end0, int[] axes0, int[] stride0, int[] begin1, long[] end1, int[] axes1, int[] stride1, int[] begin2, long[] end2, int[] axes2, int[] stride2, int[] begin3, long[] end3, int[] axes3, int[] stride3, int axis) + private Expr? GetReplace(IR.Tensors.Concat concat, Call concatCall, Expr input, int[] begin0, long[] end0, int[] axes0, int[] stride0, int[] begin1, long[] end1, int[] axes1, int[] stride1, int[] begin2, long[] end2, int[] axes2, int[] stride2, int[] begin3, long[] end3, int[] axes3, int[] stride3) { + int axis = concat.Axis; var inputShape = input.CheckedShape.ToValueArray(); if (inputShape[0] != 1) { diff --git a/src/Nncase.Passes/Rules/Neutral/FoldGatherReshape.cs b/src/Nncase.Passes/Rules/Neutral/FoldGatherReshape.cs index f0b23530f0..4500a41674 100644 --- a/src/Nncase.Passes/Rules/Neutral/FoldGatherReshape.cs +++ b/src/Nncase.Passes/Rules/Neutral/FoldGatherReshape.cs @@ -15,10 +15,14 @@ public sealed partial class FoldGatherReshape : RewriteRule { // Reshape(Gather(Shape, 0, 0), new[] { 0 }) -> GetItem(Shape, 0) public override Pattern Pattern => IsGather( - IsReshape(IsWildcard("input"), IsTensorConst("newShape")), IsTensorConst("axis"), IsTensorConst("index")); + "gather", + _ => true, + IsReshape(IsWildcard("input"), IsTensorConst("newShape")), + IsTensorConst("index")); - private Expr? GetReplace(Expr input, int[] newShape, int axis, int index) + private Expr? GetReplace(Expr input, int[] newShape, IR.Tensors.Gather gather, int index) { + int axis = gather.Axis; if (newShape.SequenceEqual(new[] { 1 }) && axis == 1) { return input[index]; diff --git a/src/Nncase.Passes/Rules/Neutral/FoldLayerNorm.cs b/src/Nncase.Passes/Rules/Neutral/FoldLayerNorm.cs index 41b3a34979..c8df9e9e62 100644 --- a/src/Nncase.Passes/Rules/Neutral/FoldLayerNorm.cs +++ b/src/Nncase.Passes/Rules/Neutral/FoldLayerNorm.cs @@ -279,3 +279,57 @@ public sealed partial class FoldLayerNormPattern4 : RewriteRule return null; } } + +// pattern from llama without mean and beta +[RuleGenerator] +public sealed partial class FoldLayerNormPattern5 : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = + IsBinary( + "mulGamma", + "mulGammaCall", + BinaryOp.Mul, + IsTensorConst("gamma"), + IsBinary( + "mulX", + "mulXCall", + BinaryOp.Mul, + IsWildcard("input"), + IsBinary( + "rsqrt", + "rsqrtCall", + BinaryOp.Div, + IsTensorConst("one"), + IsUnary( + "sqrt", + "sqrtCall", + UnaryOp.Sqrt, + IsBinary( + "addEps", + "addEpsCall", + BinaryOp.Add, + IsReduce( + "rdVar", + "rdVarCall", + ReduceOp.Mean, + IsBinary( + "pow2", + "pow2Call", + BinaryOp.Pow, + IsWildcard(), + IsTensorConst("two"))), + IsTensorConst("eps")))))); + + private Expr? GetReplace(Call pow2Call, TensorConst eps, TensorConst gamma, Expr input, TensorConst one, TensorConst two) + { + if (input == pow2Call[Binary.Lhs] && one.Value.Cast()[0] == 1f && two.Value.Cast()[0] == 2f) + { + var axis = pow2Call.CheckedShape.Count - gamma.CheckedShape.Count; + var beta = Tensor.FromScalar(0f, gamma.CheckedShape); + return LayerNorm(axis, eps.Value.Cast()[0], input, gamma, beta, hasMean: false); + } + + return null; + } +} diff --git a/src/Nncase.Passes/Rules/Neutral/FoldPrePostReshapeSoftmax.cs b/src/Nncase.Passes/Rules/Neutral/FoldPrePostReshapeSoftmax.cs new file mode 100644 index 0000000000..83edfe5e5e --- /dev/null +++ b/src/Nncase.Passes/Rules/Neutral/FoldPrePostReshapeSoftmax.cs @@ -0,0 +1,38 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Nncase.IR; +using Nncase.IR.NN; +using Nncase.PatternMatch; +using static Nncase.IR.F.NN; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Neutral; + +/// +/// Fold nop . +/// +[RuleGenerator] +public sealed partial class FoldPrePostReshapeSoftmax : IRewriteRule +{ + /// + public IPattern Pattern { get; } = IsReshape( + "reshape", + "reshapeCall", + _ => true, + IsSoftmax("softmax", IsReshape("rehsape2", "reshapeCall2", _ => true, IsWildcard("input"), IsTensorConst("shape2"))), + IsTensorConst("shape1")); + + private Expr? GetReplace(Expr input) + { + return Softmax(input, 3); + } +} diff --git a/src/Nncase.Passes/Rules/Neutral/FoldReshape.cs b/src/Nncase.Passes/Rules/Neutral/FoldReshape.cs index 2d12883101..013d76f86e 100644 --- a/src/Nncase.Passes/Rules/Neutral/FoldReshape.cs +++ b/src/Nncase.Passes/Rules/Neutral/FoldReshape.cs @@ -62,6 +62,29 @@ public sealed partial class FoldTwoReshapes : IRewriteRule } } +/// +/// Fold sequeeze reshape(binary(unsequeeze reshape(x), const)). +/// +[RuleGenerator] +public sealed partial class FoldReshapeBinaryConstReshape : IRewriteRule +{ + /// + public IPattern Pattern { get; } = + IsReshape(IsSwappableBinary("binary", null, b => b.BinaryOp is BinaryOp.Add or BinaryOp.Mul, IsReshape(IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("unsqShape")), IsTensorConst("binaryConst")), IsTensorConst("sqShape")); + + private Expr? GetReplace(Expr input, Binary binary, int[] unsqShape, TensorConst binaryConst, int[] sqShape) + { + var inShape = input.CheckedShape.ToValueArray(); + if (!(sqShape.SequenceEqual(inShape) && RulesUtility.FindSqueezeAxis(unsqShape, sqShape) is int axis && axis != -1 && ( + (binaryConst.Value.Shape.Rank == unsqShape.Length && binaryConst.Value.Shape[axis].Value == 1) || (Evaluator.TypeInference.BroadcastType((TensorType)input.CheckedType, (TensorType)binaryConst.CheckedType) is TensorType outType && outType.Shape.ToValueArray().SequenceEqual(inShape))))) + { + return null; + } + + return IR.F.Math.Binary(binary.BinaryOp, input, (binaryConst.Value.Shape.Rank == unsqShape.Length && binaryConst.Value.Shape[axis].Value == 1) ? IR.F.Tensors.Squeeze(binaryConst, new[] { axis }) : binaryConst); + } +} + /// /// Fold nop . /// diff --git a/src/Nncase.Passes/Rules/Neutral/FoldSwish.cs b/src/Nncase.Passes/Rules/Neutral/FoldSwish.cs index 0d53cca146..b8ff58e72e 100644 --- a/src/Nncase.Passes/Rules/Neutral/FoldSwish.cs +++ b/src/Nncase.Passes/Rules/Neutral/FoldSwish.cs @@ -4,6 +4,7 @@ using Nncase.IR; using Nncase.IR.Math; using Nncase.PatternMatch; +using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.Math; using static Nncase.PatternMatch.F.NN; using static Nncase.PatternMatch.Utility; @@ -11,37 +12,37 @@ namespace Nncase.Passes.Rules.Neutral; [RuleGenerator] -public sealed partial class FoldSwishPattern1 : RewriteRule +public sealed partial class FoldSwishPattern1 : RewriteRule { + public FoldSwishPattern1() + { + var input = IsWildcard("input"); + Pattern = IsSwappableBinary(null!, null, b => b.BinaryOp == BinaryOp.Mul, IsSigmoid(input), input); + } + /// - public override CallPattern Pattern { get; } = - IsBinary(null, "binaryCall", BinaryOp.Mul, IsSigmoid(null, "sigmoidCall", IsWildcard("input"))); + public override Pattern Pattern { get; } - private Expr? GetReplace(Call binaryCall, Call sigmoidCall, Expr input) + private Expr? GetReplace(Expr input) { - if (binaryCall[Binary.Rhs] == input) - { - return IR.F.NN.Swish(input); - } - - return null; + return IR.F.NN.Swish(input); } } [RuleGenerator] -public sealed partial class FoldSwishPattern2 : RewriteRule +public sealed partial class FoldSwishPattern2 : RewriteRule { + public FoldSwishPattern2() + { + var input = IsWildcard("input"); + Pattern = IsSwappableBinary(null!, null, b => b.BinaryOp == BinaryOp.Mul, IsSigmoid(IsSwappableBinary(null!, null, b => b.BinaryOp == BinaryOp.Mul, input, IsTensorConst("beta", IsFloatScalar()))), input); + } + /// - public override CallPattern Pattern { get; } = - IsBinary(null, "binaryCall", BinaryOp.Mul, IsWildcard(), IsSigmoid(null, "sigmoidCall", IsWildcard("input"))); + public override Pattern Pattern { get; } - private Expr? GetReplace(Call binaryCall, Call sigmoidCall, Expr input) + private Expr? GetReplace(Expr input, TensorConst beta) { - if (binaryCall[Binary.Lhs] == input) - { - return IR.F.NN.Swish(input); - } - - return null; + return IR.F.NN.Swish(input, beta); } } diff --git a/src/Nncase.Passes/Rules/Neutral/FusionMaker.cs b/src/Nncase.Passes/Rules/Neutral/FusionMaker.cs index 8e371dded0..9b77082ecf 100644 --- a/src/Nncase.Passes/Rules/Neutral/FusionMaker.cs +++ b/src/Nncase.Passes/Rules/Neutral/FusionMaker.cs @@ -22,13 +22,13 @@ namespace Nncase.Passes.Rules.Neutral; public abstract class FusionMaker : RewriteRule { - private int _count; + public int Count { get; set; } public virtual string Name { get; } = "FusionMaker"; public virtual string ModuleKind { get; } = "StackVM"; - public string FullName => $"{Name}_{_count++}"; + public string FullName => $"{Name}_{Count}"; } /// diff --git a/src/Nncase.Passes/Rules/Neutral/NormAxis.cs b/src/Nncase.Passes/Rules/Neutral/NormAxis.cs new file mode 100644 index 0000000000..13e3c0d30b --- /dev/null +++ b/src/Nncase.Passes/Rules/Neutral/NormAxis.cs @@ -0,0 +1,115 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Neutral; + +[RuleGenerator] +public sealed partial class NormAxisGather : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = IsGather("gather", g => g.Axis < 0, IsWildcard("input") with { TypePattern = HasRank() }, IsWildcard("index") with { TypePattern = HasRank() }); + + private Expr? GetReplace(IR.Tensors.Gather gather, Expr input, Expr index) + { + return IR.F.Tensors.Gather(input, gather.Axis + input.CheckedShape.Rank, index); + } +} + +[RuleGenerator] +public sealed partial class NormAxisConcat : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = IsConcat("concat", op => op.Axis < 0, IsTuple(IsVArgsRepeat("inputs", inputs => + { + var ps = new Pattern[inputs.Length]; + for (int i = 0; i < inputs.Length; i++) + { + ps[i] = IsWildcard(i.ToString()) with { TypePattern = HasRank() }; + } + + return ps; + }))); + + private Expr? GetReplace(IR.Tensors.Concat concat, IReadOnlyList inputs) + { + return IR.F.Tensors.Concat(new IR.Tuple(inputs.ToArray()), concat.Axis + inputs[0].CheckedShape.Rank); + } +} + +[RuleGenerator] +public sealed partial class NormAxisReduce : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = IsReduce("reduce", "call", _ => true, IsWildcard("input") with { TypePattern = HasRank() }, IsTensorConst("axes"), IsWildcard("initValue"), IsWildcard("keepDims")); + + private Expr? GetReplace(IR.Math.Reduce reduce, Call call, Expr input, int[] axes, Expr initValue, Expr keepDims) + { + if (axes.Any(axis => axis < 0)) + { + return IR.F.Tensors.Reduce(reduce.ReduceOp, input, axes.Select(axis => axis < 0 ? axis + input.CheckedShape.Rank : axis).ToArray(), initValue, keepDims); + } + + return call; + } +} + +[RuleGenerator] +public sealed partial class NormAxisReduceArg : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = IsReduceArg("reduce", "call", _ => true, IsWildcard("input") with { TypePattern = HasRank() }, IsTensorConst("axis"), IsWildcard("keepDims"), IsWildcard("selectLastIndex")); + + private Expr? GetReplace(IR.Math.ReduceArg reduce, Call call, Expr input, int axis, Expr keepDims, Expr selectLastIndex) + { + if (axis < 0) + { + return IR.F.Tensors.ReduceArg(reduce.ReduceArgOp, reduce.DestType, input, axis + input.CheckedShape.Rank, keepDims, selectLastIndex); + } + + return call; + } +} + +[RuleGenerator] +public sealed partial class NormAxisReshape : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = IsReshape("reshape", "call", IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("newshape")) with { TypePattern = HasFixedShape() }; + + private Expr? GetReplace(Call call, Expr input, int[] newshape) + { + if (newshape.Any(dim => dim < 0)) + { + return IR.F.Tensors.Reshape(input, call.CheckedShape.ToValueArray()); + } + + return null; + } +} + +[RuleGenerator] +public sealed partial class NormAxisSlice : RewriteRule +{ + /// + public override CallPattern Pattern { get; } = IsSlice("slice", "call", IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("begins"), IsTensorConst("ends"), IsTensorConst("axes"), IsTensorConst("strides")) with { TypePattern = HasFixedShape() }; + + private Expr? GetReplace(Call call, Expr input, Expr begins, Expr ends, int[] axes, Expr strides) + { + if (axes.Any(dim => dim < 0)) + { + return IR.F.Tensors.Slice(input, begins, ends, axes.Select(dim => dim < 0 ? dim + input.CheckedShape.Rank : dim).ToArray(), strides); + } + + return null; + } +} diff --git a/src/Nncase.Passes/Rules/Neutral/PrimFuncMergeRule.cs b/src/Nncase.Passes/Rules/Neutral/PrimFuncMergeRule.cs index 30cd890f12..16e9b6727c 100644 --- a/src/Nncase.Passes/Rules/Neutral/PrimFuncMergeRule.cs +++ b/src/Nncase.Passes/Rules/Neutral/PrimFuncMergeRule.cs @@ -21,6 +21,7 @@ namespace Nncase.Passes.Rules.Neutral; +#if false [RuleGenerator] public sealed partial class PrimFuncMergeRule : RewriteRule { @@ -98,7 +99,7 @@ public PrimFuncMergeRule(HashSet mergedFuncs) } // 2. chack and create the data buffer - if (calleeFunc.Parameters.ToArray().Count(b => b.MemLocation == Schedule.MemoryLocation.Output) != 1) + if (calleeFunc.Parameters.ToArray().Count(b => b.MemLocation == MemoryLocation.Output) != 1) { // the direct call mean the callee function only have one output. return null; @@ -128,7 +129,7 @@ public PrimFuncMergeRule(HashSet mergedFuncs) // 5. build the new call. var nameWrapper = callerWrapper.Name; // + '_' + calleeWrapper.Name; - var newWrapper = new PrimFunctionWrapper(nameWrapper, newFunc, newFuncParams.Count(b => b.MemLocation == Schedule.MemoryLocation.Input)); + var newWrapper = new PrimFunctionWrapper(nameWrapper, newFunc, newFuncParams.Count(b => b.MemLocation == MemoryLocation.Input)); var newCallParams = new List(); newCallParams.AddRange(callerParams.Take(calleeBufferIndexs[0])); @@ -151,10 +152,10 @@ private bool BufferCanMerge(TIR.PhysicalBuffer retBuffer, TIR.PhysicalBuffer inB retBuffer.FixedStrides.SequenceEqual(inBuffer.FixedStrides) && retBuffer.ElemType == inBuffer.ElemType && retBuffer.Size == inBuffer.Size && - retBuffer.MemLocation == Schedule.MemoryLocation.Output && - inBuffer.MemLocation == Schedule.MemoryLocation.Input) + retBuffer.MemLocation == MemoryLocation.Output && + inBuffer.MemLocation == MemoryLocation.Input) { - dataBuffer = new TIR.PhysicalBuffer(inBuffer.Name, inBuffer.ElemType, Schedule.MemoryLocation.Data, inBuffer.FixedDimensions, inBuffer.FixedStrides, inBuffer.Start, inBuffer.Size); + dataBuffer = new TIR.PhysicalBuffer(inBuffer.Name, inBuffer.ElemType, MemoryLocation.Data, inBuffer.FixedDimensions, inBuffer.FixedStrides, inBuffer.Start, inBuffer.Size); return true; } @@ -191,3 +192,4 @@ protected override Expr VisitVar(Var var, Unit context) } } } +#endif diff --git a/src/Nncase.Passes/Rules/ShapeExpr/GatherToGetItem.cs b/src/Nncase.Passes/Rules/ShapeExpr/GatherToGetItem.cs index 2a1453f3c9..ec379be4a4 100644 --- a/src/Nncase.Passes/Rules/ShapeExpr/GatherToGetItem.cs +++ b/src/Nncase.Passes/Rules/ShapeExpr/GatherToGetItem.cs @@ -14,10 +14,9 @@ namespace Nncase.Passes.Rules.ShapeExpr; public sealed partial class GatherToGetItem : RewriteRule { // (Gather(input, 0, 0) -> GetItem(input) - public override Pattern Pattern => IsGather( - IsWildcard("input"), IsTensorConst("axis"), IsTensorConst("index") with { TypePattern = IsScalar() }); + public override Pattern Pattern => IsGather("gather", 0, IsWildcard("input"), IsTensorConst("index") with { TypePattern = IsScalar() }); - private Expr? GetReplace(Expr input, int axis, int index) + private Expr? GetReplace(Expr input, int index) { return input[index]; } diff --git a/src/Nncase.Passes/RulesUtility.cs b/src/Nncase.Passes/RulesUtility.cs new file mode 100644 index 0000000000..29afea1468 --- /dev/null +++ b/src/Nncase.Passes/RulesUtility.cs @@ -0,0 +1,38 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System.Linq; + +namespace Nncase.Passes; + +public static class RulesUtility +{ + /// + /// find sequeezed axis index. + /// + /// old shape. + /// new shape. + /// axis, if not found return -1. + public static int FindSqueezeAxis(int[] oldShape, int[] newShape) + { + if (oldShape.Length <= newShape.Length) + { + return -1; + } + + var indices = Enumerable.Range(0, oldShape.Length).ToList(); + foreach (var dim in newShape) + { + for (int i = 0; i < oldShape.Length; i++) + { + if (oldShape[i] == dim && indices.IndexOf(i) != -1) + { + indices.Remove(i); + } + } + } + + var oneindex = (indices.Count == 1) ? indices[0] : -1; + return oneindex; + } +} diff --git a/src/Nncase.Passes/packages.lock.json b/src/Nncase.Passes/packages.lock.json index a910d30fa5..1c39f25003 100644 --- a/src/Nncase.Passes/packages.lock.json +++ b/src/Nncase.Passes/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Google.OrTools.runtime.linux-arm64": { @@ -103,8 +103,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -126,6 +126,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -245,6 +246,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Quantization/packages.lock.json b/src/Nncase.Quantization/packages.lock.json index ccc991e111..59323bcaea 100644 --- a/src/Nncase.Quantization/packages.lock.json +++ b/src/Nncase.Quantization/packages.lock.json @@ -19,11 +19,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "System.Linq.Async": { @@ -132,8 +132,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -155,6 +155,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -274,6 +275,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Schedule/packages.lock.json b/src/Nncase.Schedule/packages.lock.json index 93fabe1e48..1b9a6c1dd5 100644 --- a/src/Nncase.Schedule/packages.lock.json +++ b/src/Nncase.Schedule/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.Extensions.Configuration.Abstractions": { @@ -47,8 +47,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -70,6 +70,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -129,6 +130,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Simulator/packages.lock.json b/src/Nncase.Simulator/packages.lock.json index 93fabe1e48..1b9a6c1dd5 100644 --- a/src/Nncase.Simulator/packages.lock.json +++ b/src/Nncase.Simulator/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.Extensions.Configuration.Abstractions": { @@ -47,8 +47,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -70,6 +70,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -129,6 +130,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Targets/packages.lock.json b/src/Nncase.Targets/packages.lock.json index 4a17a6364e..a434ae4251 100644 --- a/src/Nncase.Targets/packages.lock.json +++ b/src/Nncase.Targets/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.Extensions.Configuration.Abstractions": { @@ -47,8 +47,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -78,6 +78,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -152,6 +153,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Tests.TestFixture/packages.lock.json b/src/Nncase.Tests.TestFixture/packages.lock.json index a7d3cc2c05..b70a73730e 100644 --- a/src/Nncase.Tests.TestFixture/packages.lock.json +++ b/src/Nncase.Tests.TestFixture/packages.lock.json @@ -28,11 +28,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "System.Linq.Async": { @@ -376,8 +376,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -750,6 +750,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -1006,6 +1007,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Tests/CodeGen/CSourceHostCases.cs b/src/Nncase.Tests/CodeGen/CSourceHostCases.cs index d6be29ed3b..97101d3c15 100644 --- a/src/Nncase.Tests/CodeGen/CSourceHostCases.cs +++ b/src/Nncase.Tests/CodeGen/CSourceHostCases.cs @@ -27,8 +27,8 @@ public class SubCase : ICodeGenCase public override PrimFunction GetEntry() { var func = T.PrimFunc("sub", - T.Buffer(TensorType.Scalar(DataTypes.Float32), Schedule.MemoryLocation.Input, out var x), - T.Buffer(TensorType.Scalar(DataTypes.Float32), Schedule.MemoryLocation.Input, out var y)).Body( + T.Buffer(TensorType.Scalar(DataTypes.Float32), MemoryLocation.Input, out var x), + T.Buffer(TensorType.Scalar(DataTypes.Float32), MemoryLocation.Input, out var y)).Body( x - y ); return func; @@ -71,8 +71,8 @@ public override void CompareEqual(IRTModel rtmod) public override PrimFunction GetEntry() { return T.PrimFunc("for_loop", - T.Buffer(new(DataTypes.Int32, new[] { 100 }), Schedule.MemoryLocation.Input, out var A), - T.Buffer(TensorType.Scalar(DataTypes.Int32), Schedule.MemoryLocation.Input, out var n) + T.Buffer(new(DataTypes.Int32, new[] { 100 }), MemoryLocation.Input, out var A), + T.Buffer(TensorType.Scalar(DataTypes.Int32), MemoryLocation.Input, out var n) ).Body( T.Serial(out var i, n).Body( T.Store(A[i], A[i] + 1), diff --git a/src/Nncase.Tests/CodeGen/UnitTestStackVMEmitter.cs b/src/Nncase.Tests/CodeGen/UnitTestStackVMEmitter.cs index 2e1027f3c2..b21d10d10f 100644 --- a/src/Nncase.Tests/CodeGen/UnitTestStackVMEmitter.cs +++ b/src/Nncase.Tests/CodeGen/UnitTestStackVMEmitter.cs @@ -1002,9 +1002,9 @@ public void TestStackVMEmitterGConcat() var memoryStream = new MemoryStream(); var stackVmEmitter = new StackVMEmitter(new BinaryWriter(memoryStream, Encoding.UTF8, true)); var tensorEmitter = new StackVMEmitter.TensorEmitter(stackVmEmitter); - tensorEmitter.Concat(); + tensorEmitter.Concat(0); var actual = memoryStream.ToArray(); - Assert.Equal(new byte[] { 100, actual[1], 0 }, actual); + Assert.Equal(new byte[] { 100, actual[1], 0, 0, 0, 0, 0 }, actual); } [Fact] @@ -1156,9 +1156,9 @@ public void TestStackVMEmitterGGather() var memoryStream = new MemoryStream(); var stackVmEmitter = new StackVMEmitter(new BinaryWriter(memoryStream, Encoding.UTF8, true)); var tensorEmitter = new StackVMEmitter.TensorEmitter(stackVmEmitter); - tensorEmitter.Gather(); + tensorEmitter.Gather(0); var actual = memoryStream.ToArray(); - Assert.Equal(new byte[] { 100, actual[1], 0 }, actual); + Assert.Equal(new byte[] { 100, actual[1], 0, 0, 0, 0, 0 }, actual); } [Fact] @@ -1244,9 +1244,9 @@ public void TestStackVMEmitterGLayerNorm() var memoryStream = new MemoryStream(); var stackVmEmitter = new StackVMEmitter(new BinaryWriter(memoryStream, Encoding.UTF8, true)); var tensorEmitter = new StackVMEmitter.TensorEmitter(stackVmEmitter); - tensorEmitter.LayerNorm(-1, 0f); + tensorEmitter.LayerNorm(-1, 0f, false); var actual = memoryStream.ToArray(); - Assert.Equal(new byte[] { 100, actual[1], 0, 255, 255, 255, 255, 0, 0, 0, 0 }, actual); + Assert.Equal(new byte[] { 100, actual[1], 0, 255, 255, 255, 255, 0, 0, 0, 0, 0 }, actual); } [Fact] diff --git a/src/Nncase.Tests/Core/UnitTestDataTypes.cs b/src/Nncase.Tests/Core/UnitTestDataTypes.cs index 322b989921..183f65a140 100644 --- a/src/Nncase.Tests/Core/UnitTestDataTypes.cs +++ b/src/Nncase.Tests/Core/UnitTestDataTypes.cs @@ -61,7 +61,7 @@ public void TestGetDisplayName() { var a = new QuantParamType(); Assert.Equal(a.ToString(), DataTypes.GetDisplayName(a)); - Assert.Equal("(f32*)", DataTypes.GetDisplayName(new PointerType(DataTypes.Float32))); + Assert.Equal("(f32 *)", DataTypes.GetDisplayName(new PointerType(DataTypes.Float32))); Assert.Equal(DataTypes.Boolean.ShortName, DataTypes.GetDisplayName(DataTypes.Boolean)); Assert.Equal(DataTypes.Utf8Char.ShortName, DataTypes.GetDisplayName(DataTypes.Utf8Char)); Assert.Equal(DataTypes.Int8.ShortName, DataTypes.GetDisplayName(DataTypes.Int8)); @@ -103,7 +103,6 @@ public void TestCSharpName() public void TestBuiltInName() { Assert.Equal("QuantParam", DataTypes.GetBuiltInName(new QuantParamType())); - Assert.Throws(() => DataTypes.GetBuiltInName(new PointerType(DataTypes.Float32))); Assert.Equal("bool", DataTypes.GetBuiltInName(DataTypes.Boolean)); Assert.Equal("Utf8Char", DataTypes.GetBuiltInName(DataTypes.Utf8Char)); Assert.Equal("sbyte", DataTypes.GetBuiltInName(DataTypes.Int8)); diff --git a/src/Nncase.Tests/Core/UnitTestExpression.cs b/src/Nncase.Tests/Core/UnitTestExpression.cs index 8f29fbdab5..dc6ceeb702 100644 --- a/src/Nncase.Tests/Core/UnitTestExpression.cs +++ b/src/Nncase.Tests/Core/UnitTestExpression.cs @@ -261,8 +261,8 @@ public void TestDenseTensorLength() public void TestConstBufferNotEqual() { var c = IR.F.Random.Normal(DataTypes.Float32, 1, 0, 0, new[] { 1, 16, 64, 400 }).Evaluate().AsTensor(); - var ddr_ld_input = new TIR.BufferRegion(Nncase.TIR.T.ConstBuffer(Const.FromTensor(c), out _, "ddr_ld_input"), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); - var ddr_ld_output = new TIR.BufferRegion(new TIR.PhysicalBuffer("ddr_ld_input", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); + var ddr_ld_input = new TIR.BufferRegion(TIR.T.AttachBuffer(Const.FromTensor(c), out _, "ddr_ld_input"), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); + var ddr_ld_output = new TIR.BufferRegion(new TIR.Buffer("ddr_ld_input", DataTypes.Float32, new MemSpan(0, 0, MemoryLocation.Input), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 })), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); Assert.NotEqual(ddr_ld_input.Buffer, ddr_ld_output.Buffer); Assert.NotEqual(ddr_ld_input, ddr_ld_output); } @@ -270,8 +270,8 @@ public void TestConstBufferNotEqual() [Fact] public void TestBufferEqual() { - var ddr_ld_input = new TIR.BufferRegion(new TIR.PhysicalBuffer("ddr_ld_input", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); - var ddr_ld_output = new TIR.BufferRegion(new TIR.PhysicalBuffer("ddr_ld_input", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); + var ddr_ld_input = new TIR.BufferRegion(new TIR.Buffer("ddr_ld_input", DataTypes.Float32, new MemSpan(0, 0, MemoryLocation.Input), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 })), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); + var ddr_ld_output = new TIR.BufferRegion(new TIR.Buffer("ddr_ld_input", DataTypes.Float32, new MemSpan(0, 0, MemoryLocation.Input), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 })), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); Assert.Equal(ddr_ld_input.Buffer, ddr_ld_output.Buffer); Assert.Equal(ddr_ld_input, ddr_ld_output); } @@ -279,8 +279,8 @@ public void TestBufferEqual() [Fact] public void TestBufferNotEqual() { - var ddr_ld_input = new TIR.BufferRegion(new TIR.PhysicalBuffer("ddr_ld_input", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); - var glb_ld_output = new TIR.BufferRegion(new TIR.PhysicalBuffer("glb_ld_output", DataTypes.BFloat16, Schedule.MemoryLocation.Data, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); + var ddr_ld_input = new TIR.BufferRegion(new TIR.Buffer("ddr_ld_input", DataTypes.Float32, new MemSpan(0, 0, MemoryLocation.Input), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 })), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); + var glb_ld_output = new TIR.BufferRegion(new TIR.Buffer("glb_ld_output", DataTypes.BFloat16, new MemSpan(0, 0, MemoryLocation.Data), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 })), new(new TIR.Range[] { 0..1, 0..16, 0..31, 0..400 })); Assert.False(ddr_ld_input.Buffer.Equals(glb_ld_output.Buffer)); Assert.False(ddr_ld_input.Equals(glb_ld_output)); } @@ -468,7 +468,7 @@ public void TestPrintExpr() CompilerServices.InferenceType(x); Assert.Equal("const(i32[4] : {1,2,3,4})", CompilerServices.Print(x)); Assert.Equal("None", CompilerServices.Print(None.Default)); - Assert.Equal("Add", CompilerServices.Print(new Nncase.IR.Math.Binary(BinaryOp.Add))); + Assert.Equal("Binary", CompilerServices.Print(new Nncase.IR.Math.Binary(BinaryOp.Add))); var y = new Var("y"); CompilerServices.InferenceType(y); Assert.Equal("%y: any", CompilerServices.Print(y)); diff --git a/src/Nncase.Tests/Core/UnitTestMutator.cs b/src/Nncase.Tests/Core/UnitTestMutator.cs index b08f089911..83d99fbb60 100644 --- a/src/Nncase.Tests/Core/UnitTestMutator.cs +++ b/src/Nncase.Tests/Core/UnitTestMutator.cs @@ -28,8 +28,5 @@ public void TestMutator() var removeNop = Mutator.RemoveNop().Invoke(); Assert.Equal(new Passes.Mutators.RemoveNop().IsMutated, removeNop.IsMutated); - - var foldMathCall = Mutator.FoldMathCall().Invoke(); - Assert.Equal(new Passes.Mutators.FoldMathCall().IsMutated, foldMathCall.IsMutated); } } diff --git a/src/Nncase.Tests/Core/UnitTestStringUtility.cs b/src/Nncase.Tests/Core/UnitTestStringUtility.cs index 0b01ae0fd6..1efe33a6c8 100644 --- a/src/Nncase.Tests/Core/UnitTestStringUtility.cs +++ b/src/Nncase.Tests/Core/UnitTestStringUtility.cs @@ -16,22 +16,22 @@ namespace Nncase.Tests.CoreTest; public static class TestExtensions { - public static ArrayExtensions.SpanWhereEnumerable> InputOf(this ReadOnlySpan arr) => arr.AsValueEnumerable().Where(b => b.MemLocation == Schedule.MemoryLocation.Input); + public static ArrayExtensions.SpanWhereEnumerable> InputOf(this ReadOnlySpan arr) => arr.AsValueEnumerable().Where(b => b.MemSpan.Location == MemoryLocation.Input); - public static ArrayExtensions.SpanWhereEnumerable> OutputOf(this ReadOnlySpan arr) => arr.AsValueEnumerable().Where(b => b.MemLocation == Schedule.MemoryLocation.Output); + public static ArrayExtensions.SpanWhereEnumerable> OutputOf(this ReadOnlySpan arr) => arr.AsValueEnumerable().Where(b => b.MemSpan.Location == MemoryLocation.Output); } public sealed class UnitTestStringUtility { - private readonly TIR.PrimFunction _entry = new("test_module", new Sequential(1), new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0)); + private readonly TIR.PrimFunction _entry = new("test_module", new Sequential(1), new TIR.Buffer("testInput", DataTypes.Float32, new MemSpan(0, 123, MemoryLocation.Input), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 })), new TIR.Buffer("testInput", DataTypes.Float32, new MemSpan(0, 123, MemoryLocation.Output), new Expr[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new Expr[] { 1, 16, 64, 400 }))); [Fact] public void TestJoin() { var result = StringUtility.Join(",", _entry.Parameters.InputOf().Select(b => b)); - Assert.Equal("PhysicalBuffer(testInput, f32, MemLocation),PhysicalBuffer(testInput, f32, MemLocation)", result); + Assert.Equal("Nncase.TIR.Buffer", result); var result1 = StringUtility.Join(",", _entry.Parameters.OutputOf().Select(b => b)); - Assert.Equal(string.Empty, result1); + Assert.Equal("Nncase.TIR.Buffer", result1); } } diff --git a/src/Nncase.Tests/Core/UnitTestTIR.cs b/src/Nncase.Tests/Core/UnitTestTIR.cs index 54d504f58c..f0c40be178 100644 --- a/src/Nncase.Tests/Core/UnitTestTIR.cs +++ b/src/Nncase.Tests/Core/UnitTestTIR.cs @@ -21,15 +21,6 @@ namespace Nncase.Tests.CoreTest; public sealed class UnitTestTIR { - [Fact] - public void TestLogicalBuffer() - { - var logicalBuffer1 = new LogicalBuffer("logicalBuffer", default, new TensorConst(new Tensor(new[] { 1 }))); - var logicalBuffer2 = new LogicalBuffer("logicalBuffer", DataTypes.Int32, default, new[] { (Expr)new Tensor(new[] { 1 }) }); - Assert.Equal(logicalBuffer2.Length.ToString(), logicalBuffer1.Length.ToString()); - Assert.Equal("LogicalBuffer(logicalBuffer, i32, MemLocation)", logicalBuffer1.ToString()); - } - [Fact] public void TestScheduler() { @@ -47,21 +38,10 @@ public void TestScheduler() [Fact] public void TestBufferStore() { - Assert.Throws(() => T.Store(null!, null!)); - - var variable = new Var("x", DataTypes.Int32); - int index = 0; - Expr loadOp = T.Load(variable, index); Expr value = 42; - _ = T.Store(loadOp, value); - - var physicalBuffer = new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0); - var indices = new Expr[] { 0, 1 }; - Expr storeOp = T.Store(new BufferLoad(physicalBuffer, indices), value); - var store = (BufferStore)storeOp; - Assert.Equal(physicalBuffer, store.Buffer); - Assert.Equal(value, store.Value); - Assert.Equal(new Expr[] { 0 }, store.Indices.ToArray()); + TIR.T.CreateBuffer(new TensorType(DataTypes.Float32, new[] { 1, 16, 64, 400 }), MemoryLocation.Input, out var testInput); + _ = new Expr[] { 0, 1 }; + _ = T.Store(testInput, 0, value); } [Fact] @@ -106,15 +86,6 @@ public void TestSequential() Assert.Equal(expect2, actual2); } - [Fact] - public void TestBuffer() - { - var buffer = T.Buffer(DataTypes.Float32, MemoryLocation.Input, new Expr[] { 1, 16, 64, 400 }, out _); - Assert.Equal(DataTypes.Float32, buffer.ElemType); - var expect = new LogicalBuffer("_", DataTypes.Float32, MemoryLocation.Input, new Expr[] { 1, 16, 64, 400 }); - Assert.Equal(expect, buffer); - } - [Fact] public void TestForSegment() { @@ -143,7 +114,7 @@ public void TestEmit() [Fact] public void TestBufferRegion() { - var buffer = T.Buffer(DataTypes.Float32, MemoryLocation.Input, new Expr[] { 1, 16, 64, 400 }, out _); + var buffer = T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 16, 64, 400 }), MemoryLocation.Input, out _); var region = new Range[] { new Range(1, 2, 2), new Range(-1, 3, 2) }; var bufferRegion = new BufferRegion(buffer, region); @@ -165,8 +136,8 @@ public void TestPrimFunction() { var primFunc = new PrimFunction("test_module", new Sequential(new Expr[] { 1 }), new[] { - new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), - new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), + TIR.T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 16, 64, 400 }), MemoryLocation.Input, out var _), + TIR.T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 16, 64, 400 }), MemoryLocation.Input, out var _), }); var primFuncParameters = primFunc.Parameters; @@ -178,8 +149,8 @@ public void TestPrimFunction() var newBody = new Sequential(new Expr[] { 3 }); var newParams = new[] { - new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), - new TIR.PhysicalBuffer("testInput", DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 16, 64, 400 }, TensorUtilities.GetStrides(new[] { 1, 16, 64, 400 }), 0, 0), + TIR.T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 16, 64, 400 }), MemoryLocation.Input, out var _), + TIR.T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 16, 64, 400 }), MemoryLocation.Input, out var _), }; var newPrimFunc = primFunc.With(moduleKind: newModuleKind, body: newBody, parameters: newParams); @@ -190,7 +161,7 @@ public void TestPrimFunction() Assert.Equal(newParams, newPrimFunc.Parameters.ToArray()); Assert.Equal(primFunc.Name, newPrimFunc.Name); // should not change the name - Assert.NotNull(new PrimFunction("test_module", new Sequential(new Expr[] { 1 }), default(ReadOnlySpan))); + Assert.NotNull(new PrimFunction("test_module", new Sequential(new Expr[] { 1 }), default(ReadOnlySpan))); } [Fact] diff --git a/src/Nncase.Tests/Core/UnitTestTensorUtilities.cs b/src/Nncase.Tests/Core/UnitTestTensorUtilities.cs index d07f20bd78..7e8a70c2bb 100644 --- a/src/Nncase.Tests/Core/UnitTestTensorUtilities.cs +++ b/src/Nncase.Tests/Core/UnitTestTensorUtilities.cs @@ -54,48 +54,68 @@ public sealed class UnitTestTensorUtilities public void TestIsContiguousSlice() { var dim1 = new[] { 1, 512, 14, 14 }; - + int start; Assert.True(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..512, 0..14, 0..14 })); + new[] { 0..1, 0..512, 0..14, 0..14 }, + out start)); + Assert.Equal(0, start); Assert.True(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..1, 0..1, 0..14 })); + new[] { 0..1, 0..1, 0..1, 0..14 }, + out start)); + Assert.Equal(0, start); Assert.True(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..1, 0..1, 7..14 })); + new[] { 0..1, 0..1, 0..1, 7..14 }, + out start)); + Assert.Equal(0, start); Assert.True(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..1, 7..14, 0..14 })); + new[] { 0..1, 0..1, 7..14, 0..14 }, + out start)); + Assert.Equal(0, start); Assert.False(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..512, 0..7, 0..14 })); + new[] { 0..1, 0..512, 0..7, 0..14 }, + out start)); + Assert.Equal(2, start); Assert.False(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..512, 0..7, 0..14, 0..1 })); + new[] { 0..1, 0..512, 0..7, 0..14, 0..1 }, + out start)); + Assert.Equal(4, start); Assert.False(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 10..512, 0..1, 0..1 })); + new[] { 0..1, 10..512, 0..1, 0..1 }, + out start)); + Assert.Equal(2, start); Assert.False(TensorUtilities.IsContiguousSlice( dim1, - new[] { 0..1, 0..512, 0..7, 0..1 })); + new[] { 0..1, 0..512, 0..7, 0..1 }, + out start)); + Assert.Equal(3, start); var dim2 = new[] { 1, 512, 1, 196 }; Assert.True(TensorUtilities.IsContiguousSlice( dim2, - new[] { 0..1, 0..128, 0..1, 0..196 })); + new[] { 0..1, 0..128, 0..1, 0..196 }, + out start)); + Assert.Equal(0, start); Assert.True(TensorUtilities.IsContiguousSlice( dim2, - new[] { 0..1, 0..1, 0..1, 10..15 })); + new[] { 0..1, 0..1, 0..1, 10..15 }, + out start)); + Assert.Equal(0, start); } // long GetProduct(ReadOnlySpan dimensions, int startIndex = 0) diff --git a/src/Nncase.Tests/Diagnostics/UnitTestDumpper.cs b/src/Nncase.Tests/Diagnostics/UnitTestDumpper.cs index 31c6d763ae..dcd94f1303 100644 --- a/src/Nncase.Tests/Diagnostics/UnitTestDumpper.cs +++ b/src/Nncase.Tests/Diagnostics/UnitTestDumpper.cs @@ -65,7 +65,7 @@ public void TestDumpFusion() [Fact] public void TestDumpScript() { - var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 2, 3, 4 }, out _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 2, 3, 4 }, out _)).Body(T.Nop()).Build(); + var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Output, out _)).Body(T.Nop()).Build(); Assert.True(CompilerServices.InferenceType(prim_func_1)); @@ -194,6 +194,17 @@ public void TestDumperCSharpIRFunction() CompilerServices.DumpCSharpIR(main, string.Empty, Dumpper.Directory); } + [Fact] + public void TestDumperPatternIRFunction() + { + var x = IR.F.Math.Quantize(IR.F.Random.Normal(DataTypes.Float32, 0, 1, 0, new[] { 1, 2, 2, 2 }), new QuantParam(1, 2.0f), DataTypes.UInt8); + var y = new Var("y", new TensorType(DataTypes.UInt8, new int[] { 1, 2, 2, 2 })); + var z = IR.F.Random.Normal(DataTypes.UInt8, 0, 1, 0, new[] { 1, 2, 2, 2 }); + var m = IR.F.Random.Normal(DataTypes.UInt8, 0, 1, 0, new[] { 1, 20, 2, 2 }); + var main = new Function("main", IR.F.Tensors.Concat(new IR.Tuple(new Expr[] { x, y, z, m }), 1), new[] { y }); + CompilerServices.DumpPatternIR(main, string.Empty, Dumpper.Directory); + } + [Fact] public void TestDumperCSharpIRFusion() { @@ -214,9 +225,9 @@ public void TestDumperCSharpIRFusion() [Fact] public void TestDumpTIRFusion() { - var lhs = new Var("lhs"); - var main = T.PrimFunc("main", Callable.StackVMModuleKind).Body( - new Call(new TIRTest.MeshNet(), new Fusion("MeshFunc", lhs + 100, lhs), IR.F.Random.Normal(DataTypes.Float32, 0, 1, 123, new[] { 100 }))).Build(); + var lhs = new Var("lhs", TensorType.Scalar(DataTypes.Float32)); + var main = T.PrimFunc("main", DefaultTargetName).Body( + new Call(new TIRTest.MeshNet(), new Fusion("MeshFunc", lhs + 100.0f, lhs), IR.F.Random.Normal(DataTypes.Float32, 0, 1, 123, new[] { 100 }))).Build(); Assert.True(CompilerServices.InferenceType(main)); CompilerServices.DumpIR(main, string.Empty, Dumpper.Directory); } diff --git a/src/Nncase.Tests/EGraph/UnitTestVrp.cs b/src/Nncase.Tests/EGraph/UnitTestVrp.cs index 5a9f4b564e..f421022a7c 100644 --- a/src/Nncase.Tests/EGraph/UnitTestVrp.cs +++ b/src/Nncase.Tests/EGraph/UnitTestVrp.cs @@ -174,6 +174,35 @@ public void TestSimpleEgraphSat() } } + [Fact] + public void TestOverLap() + { + // note ortools no overlap not support 0 size. + var model = new CpModel(); + + var x0 = model.NewIntervalVar(model.NewConstant(0), model.NewConstant(2), model.NewConstant(2), "x0"); + var y0 = model.NewFixedSizeIntervalVar(model.NewIntVar(0, 10, "y0_start"), 7, "y0"); + + var x1 = model.NewIntervalVar(model.NewConstant(2), model.NewConstant(0), model.NewConstant(2), "x1"); + var y1 = model.NewFixedSizeIntervalVar(model.NewIntVar(0, 10, "y1_start"), 7, "y1"); + + var x2 = model.NewIntervalVar(model.NewConstant(2), model.NewConstant(1), model.NewConstant(3), "x2"); + var y2 = model.NewFixedSizeIntervalVar(model.NewIntVar(0, 10, "y2_start"), 7, "y2"); + + model.Add(y0.StartExpr() == y1.StartExpr()); + model.Add(y1.StartExpr() == y2.StartExpr()); + var nooverlap = model.AddNoOverlap2D(); + nooverlap.AddRectangle(x0, y0); + nooverlap.AddRectangle(x1, y1); + nooverlap.AddRectangle(x2, y2); + model.Minimize(y0.StartExpr() + y1.StartExpr() + y2.StartExpr()); + + var solver = new CpSolver(); + var status = solver.Solve(model); + + Assert.Equal(CpSolverStatus.Infeasible, status); + } + private static void PrintSolution(in IDataModel data, in RoutingModel routing, in RoutingIndexManager manager, in Assignment solution) { Console.WriteLine($"Objective {solution.ObjectiveValue()}:"); diff --git a/src/Nncase.Tests/Evaluator/UnitTestEvaluator.cs b/src/Nncase.Tests/Evaluator/UnitTestEvaluator.cs index 3f230b49c5..037e40793b 100755 --- a/src/Nncase.Tests/Evaluator/UnitTestEvaluator.cs +++ b/src/Nncase.Tests/Evaluator/UnitTestEvaluator.cs @@ -97,10 +97,10 @@ public void TestOnnxResizeImage() public void TestLoadStore() { var loop_i = new Var(TensorType.Scalar(DataTypes.Int32)); - var load = T.Load(T.Handle("hd", DataTypes.Float32), loop_i); + T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3 }), MemoryLocation.Input, out var bf); + var load = T.Load(bf, loop_i); CompilerServices.InferenceType(load); - - var store = T.Store((Var)load[TIR.Load.Handle], load[TIR.Load.Index], loop_i); + var store = T.Store(bf, loop_i, IR.F.Tensors.Cast(loop_i, DataTypes.Float32)); CompilerServices.InferenceType(store); } diff --git a/src/Nncase.Tests/Evaluator/UnitTestEvaluatorBuffers.cs b/src/Nncase.Tests/Evaluator/UnitTestEvaluatorBuffers.cs index 04c7a43c44..bed1177d0d 100644 --- a/src/Nncase.Tests/Evaluator/UnitTestEvaluatorBuffers.cs +++ b/src/Nncase.Tests/Evaluator/UnitTestEvaluatorBuffers.cs @@ -29,7 +29,7 @@ public class UnitTestEvaluatorBuffers : TestClassBase public void TestUninitialized() { var shape = new[] { 1 }; - var expr = IR.F.Buffer.Uninitialized(DataTypes.Float32, MemoryLocation.Input, shape); + var expr = IR.F.Buffer.Uninitialized(DataTypes.Float32, TIR.MemoryLocation.Input, shape); CompilerServices.InferenceType(expr); Assert.Equal(Value.None, expr.Evaluate()); } diff --git a/src/Nncase.Tests/Evaluator/UnitTestEvaluatorTensors.cs b/src/Nncase.Tests/Evaluator/UnitTestEvaluatorTensors.cs index 63cb7507a4..4d56cde01b 100644 --- a/src/Nncase.Tests/Evaluator/UnitTestEvaluatorTensors.cs +++ b/src/Nncase.Tests/Evaluator/UnitTestEvaluatorTensors.cs @@ -112,7 +112,7 @@ public void TestConcat3() for (long i = 0; i < shape.Length; i++) { var expect = OrtKI.Concat(new OrtKISharp.Tensor[] { inputA, inputB }, i); - var expr = IR.F.Tensors.Concat(new Tuple(inputA.ToTensor(), inputB.ToTensor()), i); + var expr = IR.F.Tensors.Concat(new Tuple(inputA.ToTensor(), inputB.ToTensor()), (int)i); CompilerServices.InferenceType(expr); Assert.Equal(expect, expr.Evaluate().AsTensor().ToOrtTensor()); } @@ -624,7 +624,7 @@ public void TestGather() long batchDims = 0L; var expect = OrtKI.Gather(input.ToOrtTensor(), indices.ToOrtTensor(), batchDims); - var expr = IR.F.Tensors.Gather(input, batchDims, indices); + var expr = IR.F.Tensors.Gather(input, (int)batchDims, indices); CompilerServices.InferenceType(expr); Assert.Equal(expect, expr.Evaluate().AsTensor().ToOrtTensor()); } diff --git a/src/Nncase.Tests/Match/UnitTestEGraphMatch.cs b/src/Nncase.Tests/Match/UnitTestEGraphMatch.cs index af32681a6d..f9e1691387 100644 --- a/src/Nncase.Tests/Match/UnitTestEGraphMatch.cs +++ b/src/Nncase.Tests/Match/UnitTestEGraphMatch.cs @@ -113,7 +113,7 @@ public void TestMatchVArgs() Expr expr = Concat(tuple, 0); CompilerServices.InferenceType(expr); - var vpat = IsConcat(IsTuple("tp"), IsConst(0)); + var vpat = IsConcat(0, IsTuple("tp")); Assert.True(CompilerServices.TryEMatchRoot(expr, vpat, out var eMatches)); Assert.Single(eMatches); @@ -122,13 +122,11 @@ public void TestMatchVArgs() [Fact] public void TestMatchVArgsTwice() { - ConstPattern wcaxis = IsConst(); - var tuple_lhs = new IR.Tuple(1, new Var(), 3); var tuple_rhs = new IR.Tuple(4, 5, 6); Expr expr = Concat(tuple_lhs, 0) + Concat(tuple_rhs, 1); - var vpat = IsConcat(IsTuple("tp"), wcaxis); + var vpat = IsConcat(_ => true, IsTuple("tp")); Assert.True(CompilerServices.TryEMatchRoot(expr, vpat, out var eMatches)); Assert.Equal(2, eMatches.Count); @@ -151,9 +149,8 @@ public void TestMatchVArgsRecursion() var wc = IsWildcard("wc"); var wcperm = IsWildcard("perm"); - var wcaxis = IsWildcard("axis"); - var pattern = IsConcat(IsTuple(IsVArgsRepeat("wcvargs", () => IsTranspose(IsWildcard(), wcperm))), wcaxis); + var pattern = IsConcat(_ => true, IsTuple(IsVArgsRepeat("wcvargs", () => IsTranspose(IsWildcard(), wcperm)))); Assert.True(CompilerServices.TryEMatchRoot(expr, pattern, out var results)); Assert.Single(results); @@ -163,7 +160,6 @@ public void TestMatchVArgsRecursion() Assert.Equal(((Call)wcvargs[1]).Arguments[0], y); Assert.Equal(((Call)wcvargs[2]).Arguments[0], z); Assert.Equal(result[wcperm], perm); - Assert.Equal(result[wcaxis], (Const)0); } [Fact] diff --git a/src/Nncase.Tests/Properties/launchSettings.json b/src/Nncase.Tests/Properties/launchSettings.json index d081379b07..3109588c45 100644 --- a/src/Nncase.Tests/Properties/launchSettings.json +++ b/src/Nncase.Tests/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Nncase.Tests": { "commandName": "Project", - "nativeDebugging": true + "nativeDebugging": false } } } \ No newline at end of file diff --git a/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs b/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs index 6d2f29ce71..65e694e129 100644 --- a/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs +++ b/src/Nncase.Tests/Quant/UnitTestPytestCalibrationDatasetProvider.cs @@ -33,89 +33,37 @@ public async Task TestPytestCalibrationDatasetProvider1() { var vars = Setup(); var dataset = "./public/test1"; - foreach (var t in vars) + var actuals = DumpTensors(dataset, vars, 2); + var provider = new PytestCalibrationDatasetProvider(vars, dataset); + Assert.Equal(2, provider.Count); + var samples = provider.Samples; + var count = 0; + await foreach (var sample in samples) { - var actual = IR.F.Random.Uniform(t.CheckedDataType, 1.0f, -1.0f, 0, t.CheckedShape).Evaluate().AsTensor(); - DumpTensors(new[] { actual }, dataset, 2); - var provider = new PytestCalibrationDatasetProvider(new[] { t }, dataset); - Assert.Equal(2, provider.Count); - var samples = provider.Samples; - await foreach (var sample in samples) - { - Assert.Equal(sample[t].AsTensor(), actual); - } - } - } - - [Fact] - public async Task TestPytestCalibrationDatasetProvider2() - { - var vars = Setup(); - var dataset = "./public/test2"; - foreach (var t in vars) - { - var actual = IR.F.Random.Uniform(t.CheckedDataType, 1.0f, -1.0f, 0, t.CheckedShape).Evaluate().AsTensor(); - DumpTensors(new[] { actual }, dataset); - var provider = new PytestCalibrationDatasetProvider(new[] { t }, dataset); - Assert.Equal(1, provider.Count); - var samples = provider.Samples; - await foreach (var sample in samples) - { - Assert.Equal(sample[t].AsTensor(), actual); - } - } - } - - [Fact] - public async Task TestPytestCalibrationDatasetProvider3() - { - var vars1 = Setup(); - var dataset = "./public/test3"; - var actual1 = IR.F.Random.Uniform(vars1[0].CheckedDataType, 1.0f, -1.0f, 0, vars1[0].CheckedShape).Evaluate().AsTensor(); - var actual2 = IR.F.Random.Uniform(vars1[1].CheckedDataType, 1.0f, -1.0f, 0, vars1[1].CheckedShape).Evaluate().AsTensor(); - DumpTensors(new[] { actual1, actual2 }, dataset); - var provider1 = new PytestCalibrationDatasetProvider(vars1, dataset); - Assert.Equal(1, provider1.Count); - var samples1 = provider1.Samples; - await foreach (var sample in samples1) - { - Assert.Equal(sample[vars1[0]].AsTensor(), actual1); - Assert.Equal(sample[vars1[1]].AsTensor(), actual2); - } - } - - [Fact] - public async Task TestPytestCalibrationDatasetProvider4() - { - var vars1 = Setup(); - var dataset = "./public/test4"; - var actual1 = IR.F.Random.Uniform(vars1[0].CheckedDataType, 1.0f, -1.0f, 0, vars1[0].CheckedShape).Evaluate().AsTensor(); - var actual2 = IR.F.Random.Uniform(vars1[1].CheckedDataType, 1.0f, -1.0f, 0, vars1[1].CheckedShape).Evaluate().AsTensor(); - DumpTensors(new[] { actual1, actual2 }, dataset, 2); - var provider1 = new PytestCalibrationDatasetProvider(vars1, dataset); - Assert.Equal(2, provider1.Count); - var samples1 = provider1.Samples; - await foreach (var sample in samples1) - { - Assert.Equal(sample[vars1[0]].AsTensor(), actual1); - Assert.Equal(sample[vars1[1]].AsTensor(), actual2); + Assert.Equal(sample[vars[0]].AsTensor(), actuals[count, 0]); + Assert.Equal(sample[vars[1]].AsTensor(), actuals[count, 1]); + count++; } } - private static void DumpTensors(Tensor[] tensorValue, string dir, int sample = 1) + private static Tensor[,] DumpTensors(string dir, Var[] inputs, int sample) { Directory.CreateDirectory(dir); + var outputs = new Tensor[sample, inputs.Length]; for (var s = 0; s < sample; s++) { - for (var t = 0; t < tensorValue.Length; t++) + for (var t = 0; t < inputs.Length; t++) { - var value = tensorValue[t]; + var value = IR.F.Random.Uniform(inputs[t].CheckedDataType, 1.0f, -1.0f, s + t, inputs[t].CheckedShape).Evaluate().AsTensor(); var sr1 = new StreamWriter(Path.Join(dir, $"input_{t}_{s}.txt")); DumpTxt(value, sr1); var sr2 = Path.Join(dir, $"input_{t}_{s}.bin"); DumpBin(value, sr2); + outputs[s, t] = value; } } + + return outputs; } private static void DumpTxt(Tensor tensorValue, StreamWriter writer) diff --git a/src/Nncase.Tests/Rewrite/Fusion/UnitTestFusionMaker.cs b/src/Nncase.Tests/Rewrite/Fusion/UnitTestFusionMaker.cs index 70259ef6eb..c3e4bac4c7 100644 --- a/src/Nncase.Tests/Rewrite/Fusion/UnitTestFusionMaker.cs +++ b/src/Nncase.Tests/Rewrite/Fusion/UnitTestFusionMaker.cs @@ -9,7 +9,7 @@ using NetFabric.Hyperlinq; using Nncase.IR; using Nncase.IR.Math; -using Nncase.IR.Tensors; +using Nncase.IR.RNN; using Nncase.Passes; using Nncase.Passes.Analysis; using Nncase.Passes.Mutators; @@ -20,9 +20,8 @@ using Xunit; using Xunit.Abstractions; using static Nncase.IR.F.Math; +using static Nncase.IR.F.RNN; using static Nncase.IR.F.Tensors; -using static Nncase.IR.TypePatternUtility; -using static Nncase.PatternMatch.F.Math; using static Nncase.PatternMatch.Utility; using Transpose = Nncase.IR.Tensors.Transpose; using Tuple = Nncase.IR.Tuple; @@ -330,9 +329,9 @@ IR.Tuple WrapOutput(Call call) var newVar2 = newVars[2]; var pairs = new[] { - (LSTM.X, (Expr)WrapInput(newVar0)), - (LSTM.InitialC, WrapInput(newVar1)), - (LSTM.InitialH, WrapInput(newVar2)), + (IR.RNN.LSTM.X, (Expr)WrapInput(newVar0)), + (IR.RNN.LSTM.InitialC, WrapInput(newVar1)), + (IR.RNN.LSTM.InitialH, WrapInput(newVar2)), }; var expectLSTM = ReplaceUtility.ReplaceCallParams(lstm.Target, lstm.Arguments.ToArray(), pairs); var expectBody = WrapOutput(expectLSTM); @@ -363,7 +362,7 @@ internal sealed class TestTransposeComplexFusion : ComplexFusion { public override (ParameterInfo, CallPattern)[] InputPatterns { get; } = - GenerateInputPatterns(LSTM.X, LSTM.InitialC, LSTM.InitialH); + GenerateInputPatterns(IR.RNN.LSTM.X, IR.RNN.LSTM.InitialC, IR.RNN.LSTM.InitialH); } } diff --git a/src/Nncase.Tests/Rewrite/RewriteBase.cs b/src/Nncase.Tests/Rewrite/RewriteBase.cs index 33dc381532..c68a1eba45 100644 --- a/src/Nncase.Tests/Rewrite/RewriteBase.cs +++ b/src/Nncase.Tests/Rewrite/RewriteBase.cs @@ -2072,7 +2072,7 @@ public Function PreExpr var input = new Tensor(new[] { 0, 1, 2, 3 }, shape); var indices = new Tensor(new[] { 0L, 0L, 1L, 1L }, shape); long batchDims = 0L; - var expr = IR.F.Tensors.Gather(input, batchDims, indices); + var expr = IR.F.Tensors.Gather(input, (int)batchDims, indices); return new Function(expr, new Var[] { _input }); } } @@ -2906,3 +2906,53 @@ public FoldReshapeWithBranch() public Dictionary FeedDict { get; } } + +public sealed class ReshapeTransposeReshapeCase : IRewriteCase +{ + public ReshapeTransposeReshapeCase() + { + var input = new Var("input", new TensorType(DataTypes.Float32, new[] { 1, 77, 768 })); + { + var v0 = Reshape(input, new[] { 1, 77, 12, 64 }); + var v2 = Transpose(v0, new[] { 0, 2, 1, 3 }); + var v3 = Reshape(v2, new[] { 12, 77, 64 }); + PreExpr = new Function(v3, new[] { input }); + } + + FeedDict = new() { { input, IR.F.Random.Normal(new[] { 1, 77, 768 }).Evaluate() } }; + } + + public Function PreExpr { get; } + + public IEnumerable Rules => new[] { + typeof(CombineReshapeTranspose), + typeof(FoldTwoReshapes), + }; + + public Dictionary FeedDict { get; } +} + +public sealed class ReshapeBinaryConstReshapeCase : IRewriteCase +{ + public ReshapeBinaryConstReshapeCase() + { + var v9 = new Var("v9", new TensorType(DataTypes.Float32, new[] { 12, 77, 77 })); + { + var v10 = Reshape(v9, new[] { 1, 12, 77, 77 }); // f32[1,12,77,77] + var v11 = IR.F.Math.Add(v10, IR.F.Random.Normal(new[] { 1, 1, 77, 77 }).Evaluate().AsTensor()); // f32[1,12,77,77] + var v12 = Reshape(v11, new[] { 12, 77, 77 }); // f32[12,77,77] + + PreExpr = new Function(v12, new[] { v9 }); + } + + FeedDict = new() { { v9, IR.F.Random.Normal(new[] { 12, 77, 77 }).Evaluate() } }; + } + + public Function PreExpr { get; } + + public IEnumerable Rules => new[] { + typeof(FoldReshapeBinaryConstReshape), + }; + + public Dictionary FeedDict { get; } +} diff --git a/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewriteFactory.cs b/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewriteFactory.cs index f9080652a7..06dcc60c9b 100644 --- a/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewriteFactory.cs +++ b/src/Nncase.Tests/Rewrite/UnitTestDataFlowRewriteFactory.cs @@ -15,7 +15,7 @@ public class UnitTestDataFlowRewriteFactory : TestClassBase { public static TheoryData DataOne => new() { - new CombineClampAddMul(), + new ReshapeBinaryConstReshapeCase(), }; public static TheoryData DataAll => new() @@ -31,6 +31,7 @@ public class UnitTestDataFlowRewriteFactory : TestClassBase new Conv2DPadsCase(), new ReduceWindow2DPadsCase(), new MobileNetV1TransposeCase(), + new CombineClampAddMul(), }; [Theory] diff --git a/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs b/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs index 9a1d8faee7..6ec5e4721b 100644 --- a/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs +++ b/src/Nncase.Tests/Rewrite/UnitTestEGraphRewriteFactory.cs @@ -28,7 +28,7 @@ public UnitTestEGraphRewriteFactory() public static TheoryData DataOne => new() { - new PReluTransposeCase(), + new ReshapeTransposeReshapeCase(), }; public static TheoryData DataAll => new() diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestAddMarker.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestAddMarker.cs index 74f6188c93..c705e5dac2 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestAddMarker.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestAddMarker.cs @@ -96,7 +96,7 @@ public async Task TestAddMarkerWithLstm() var module = new IRModule(main); await TestAddMarkerPasses(module); Assert.True(((Function)module.Entry!).Body is Tuple t - && CompilerServices.TryMatchRoot(t, IsWrappedLSTM(PatternMatch.F.Tensors.IsLSTM("lstm", "lstmCall", _ => true), (x, _) => IsRangeOfMarker(x, IsWildcard())), out var result) + && CompilerServices.TryMatchRoot(t, IsWrappedLSTM(PatternMatch.F.RNN.IsLSTM("lstm", "lstmCall", _ => true), (x, _) => IsRangeOfMarker(x, IsWildcard())), out var result) && result["lstmCall"] is Call call && new[] { 0, 1, 2, 5, 6 }.All(i => call.Arguments[i] is Marker)); } @@ -126,7 +126,7 @@ public async Task TestAddMarkerWithLstmInitHEqualsInitC() var module = new IRModule(main); await TestAddMarkerPasses(module); Assert.True(((Function)module.Entry!).Body is Tuple t - && CompilerServices.TryMatchRoot(t, IsWrappedLSTM(PatternMatch.F.Tensors.IsLSTM("lstm", "lstmCall", _ => true), (x, _) => IsRangeOfMarker(x, IsWildcard())), out var result) + && CompilerServices.TryMatchRoot(t, IsWrappedLSTM(PatternMatch.F.RNN.IsLSTM("lstm", "lstmCall", _ => true), (x, _) => IsRangeOfMarker(x, IsWildcard())), out var result) && result["lstmCall"] is Call call && new[] { 0, 1, 2, 5, 6 }.All(i => call.Arguments[i] is Marker)); } diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineReshape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineReshape.cs index 9295d4e159..918b685d23 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineReshape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineReshape.cs @@ -35,6 +35,13 @@ public class UnitTestCombineReshape : TransformTestBase { BinaryOp.Sub, new[] { 1 }, new[] { 1, 32, 32, 64, }, new[] { 1, 1024, 64, 1 }, true }, }; + public static readonly TheoryData TestCombineReshapeTransposeNegativeData = + new() + { + { new[] { 1, 77, 1, 64 }, new[] { 2, 1, 3, 0 }, new[] { 77, 64, 1 } }, + { new[] { 1, 77, 12, 64 }, new[] { 1, 0, 2, 3 }, new[] { 1, 77, 768 } }, + }; + public static IEnumerable CombineBinaryReshapePositiveData => new[] { @@ -197,4 +204,48 @@ public void TestCombineReshapePadNegative(int[] inShape, int[] shape, int[] pads var rootPre = Tensors.Reshape(NN.Pad(a, Tensor.From(pads, new[] { pads.Length / 2, 2 }), PadMode.Constant, 0f), shape); TestNotMatch(rootPre); } + + [Theory] + [ClassData(typeof(CombineReshapeTransposePostiveData))] + public void TestCombineReshapeTransposePostive(int[] inShape, int[] perm, int[] newshape) + { + var input = new Var("input", new TensorType(DataTypes.Float32, inShape)); + var feed_dict = new Dictionary + { + { input, Random.Normal(DataTypes.Float32, 0, 1, 0, inShape).Evaluate() }, + }; + var rootPre = Tensors.Reshape(Tensors.Transpose(input, perm), newshape); + TestMatched(rootPre, feed_dict); + } + + [Theory] + [MemberData(nameof(TestCombineReshapeTransposeNegativeData))] + public void TestCombineReshapeTransposeNegative(int[] inShape, int[] perm, int[] newshape) + { + var input = new Var("input", new TensorType(DataTypes.Float32, inShape)); + var rootPre = Tensors.Reshape(Tensors.Transpose(input, perm), newshape); + TestNotMatch(rootPre); + } + + private sealed class CombineReshapeTransposePostiveData : TheoryData + { + public CombineReshapeTransposePostiveData() + { + var inshapes = new[] { + new[] { 1, 77, 12, 64 }, + new[] { 77, 1, 12, 64 }, + new[] { 77, 12, 1, 64 }, + new[] { 77, 12, 64, 1 }, + }; + + var perms = new[] { 0, 1, 2, 3 }.Permutate().ToArray(); + + foreach (var (inshape, perm) in new[] { inshapes, perms }.CartesianProduct().Select(i => i.ToArray()).Select(i => (i[0], i[1]))) + { + var newshape = perm.Select(i => inshape[i]).ToList(); + newshape.Remove(1); + Add(inshape, perm, newshape.ToArray()); + } + } + } } diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs index 440a53e551..dcde48a8b7 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestCombineTranspose.cs @@ -39,14 +39,14 @@ public class UnitTestCombineTranspose : TransformTestBase }; public static IEnumerable CombineBinaryTransposePositiveData => - new[] - { + new[] + { new object[] { new[] { 5, 4 }, new[] { 5, 4 }, new[] { 1, 0 } }, new object[] { new[] { 4, 4 }, new[] { 4, 4 }, new[] { 1, 0 } }, new object[] { new[] { 4 }, new[] { 4 }, new[] { 0 } }, new object[] { new[] { 1, 3, 4 }, new[] { 1, 3, 4 }, new[] { 0, 2, 1 } }, new object[] { new[] { 1, 3, 2, 4 }, new[] { 1, 3, 2, 4 }, new[] { 0, 2, 3, 1 } }, - }; + }; public static IEnumerable CombineConstBinaryTransposeNotMatchData => new[] @@ -359,4 +359,39 @@ public void TestCombineTransposeUnaryPositive(UnaryOp opType, int[] inShape, int var rootPre = IR.F.Math.Unary(opType, Tensors.Transpose(a, perm)); TestMatched(rootPre, normal); } + + [Theory] + [ClassData(typeof(CombineTransposeReshapePostiveData))] + public void TestCombineTransposeReshapePostive(int[] inShape, int[] newShape, int[] perm) + { + var a = new Var(new TensorType(DataTypes.Float32, inShape)); + var feed_dict = new Dictionary + { + { a, Random.Normal(DataTypes.Float32, 0, 1, 0, inShape).Evaluate() }, + }; + var rootPre = Tensors.Transpose(Tensors.Reshape(a, newShape), perm); + TestMatched(rootPre, feed_dict); + } + + private sealed class CombineTransposeReshapePostiveData : TheoryData + { + public CombineTransposeReshapePostiveData() + { + var inshapes = new[] { new[] { 12, 77, 64 } }; + + var newShapes = new[] { + new[] { 1, 12, 77, 64 }, + new[] { 12, 1, 77, 64 }, + new[] { 12, 77, 1, 64 }, + new[] { 12, 77, 64, 1 }, + }; + + var perms = new[] { 0, 1, 2, 3 }.Permutate().ToArray(); + + foreach (var (a, b, c) in new[] { inshapes, newShapes, perms }.CartesianProduct().Select(i => i.ToArray()).Select(i => (i[0], i[1], i[2]))) + { + Add(a, b, c); + } + } + } } diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs index b4af452d8e..ac8bbef641 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldReshape.cs @@ -21,6 +21,12 @@ namespace Nncase.Tests.Rules.NeutralTest; [AutoSetupTestMethod(InitSession = true)] public class UnitTestFoldReshape : TransformTestBase { + public static TheoryData TestReshapeBinaryConstReshapePositiveData => new() + { + { new[] { 12, 77, 77 }, new[] { 1, 12, 77, 77 }, new[] { 1, 1, 77, 77 }, new[] { 12, 77, 77 } }, + { new[] { 12, 77, 77 }, new[] { 1, 12, 77, 77 }, new[] { 77 }, new[] { 12, 77, 77 } }, + }; + public static IEnumerable TestFoldNopReshapePositiveData => new[] { @@ -101,4 +107,16 @@ public void TestReshapeToTransposeNegative(int[] shape, int[] newShape) var rootPre = Tensors.Reshape(a, newShape); TestNotMatch(rootPre); } + + [Theory] + [MemberData(nameof(TestReshapeBinaryConstReshapePositiveData))] + public void TestReshapeBinaryConstReshapePositive(int[] inShape, int[] unsqShape, int[] constShape, int[] sqShape) + { + var a = Random.Normal(DataTypes.Float32, 0, 1, 0, inShape); + var v0 = Tensors.Reshape(a, unsqShape); + var v1 = Math.Binary(BinaryOp.Add, v0, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 0, constShape).Evaluate().AsTensor()); + var v2 = Tensors.Reshape(v1, sqShape); + var rootPre = v2; + TestMatched(rootPre); + } } diff --git a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldSwish.cs b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldSwish.cs index 897d1b04cb..95c2c010a4 100644 --- a/src/Nncase.Tests/Rules/Neutral/UnitTestFoldSwish.cs +++ b/src/Nncase.Tests/Rules/Neutral/UnitTestFoldSwish.cs @@ -74,7 +74,8 @@ public void TestFoldSwishPattern2Positive2(int[] shape) Expr rootPre; { var v0 = input; - var v1 = IR.F.NN.Sigmoid(v0); + var v0_2 = IR.F.Math.Binary(BinaryOp.Mul, v0, 2.0f); + var v1 = IR.F.NN.Sigmoid(v0_2); var v2 = IR.F.Math.Binary(BinaryOp.Mul, v0, v1); rootPre = v2; } diff --git a/src/Nncase.Tests/Rules/ShapeBucket/ShapeBucketTest.cs b/src/Nncase.Tests/Rules/ShapeBucket/ShapeBucketTest.cs index a65c520bf1..1e0ae120fd 100644 --- a/src/Nncase.Tests/Rules/ShapeBucket/ShapeBucketTest.cs +++ b/src/Nncase.Tests/Rules/ShapeBucket/ShapeBucketTest.cs @@ -368,7 +368,7 @@ public void TestMatMulReshape() var lhs = MakeVar(input); var add = Add(lhs, new[] { 1f }); var rhs = Reshape(add, Concat( - new IR.Tuple(Reshape(Gather(ShapeOf(add), 0L, 0L), new[] { 1L }), new[] { 3L }, new[] { 24L }, new[] { 24L }), 0)); + new IR.Tuple(Reshape(Gather(ShapeOf(add), 0, 0L), new[] { 1L }), new[] { 3L }, new[] { 24L }, new[] { 24L }), 0)); var lhsVar = new Var("lhs", new TensorType(input.ElementType, input.Shape)); var rhsVar = new Var("rhs", new TensorType(input.ElementType, input.Shape)); diff --git a/src/Nncase.Tests/TIR/PrimFunc/IDataFlowPrimFuncCase.cs b/src/Nncase.Tests/TIR/PrimFunc/IDataFlowPrimFuncCase.cs index 9b791049d0..97dbd1915f 100644 --- a/src/Nncase.Tests/TIR/PrimFunc/IDataFlowPrimFuncCase.cs +++ b/src/Nncase.Tests/TIR/PrimFunc/IDataFlowPrimFuncCase.cs @@ -33,11 +33,11 @@ internal static class PrimFuncBuilder public static PrimFunctionWrapper MakeLoadStoreFunc(bool mask) { var allocator = new Allocator(); - var fusion_input = allocator.Allocate($"fusion_{_count}_input", Schedule.MemoryLocation.Input); + var fusion_input = allocator.Allocate($"fusion_{_count}_input", TIR.MemoryLocation.Input); - var glb = allocator.Allocate($"fusion_{_count}_glb", Schedule.MemoryLocation.L2Data); + var glb = allocator.Allocate($"fusion_{_count}_glb", TIR.MemoryLocation.L2Data); - var fusion_output = allocator.Allocate($"fusion_{_count}_output", Schedule.MemoryLocation.Output); + var fusion_output = allocator.Allocate($"fusion_{_count}_output", TIR.MemoryLocation.Output); var fusion_1 = TIR.T.PrimFunc($"fusion_{_count}_{mask}", Callable.StackVMModuleKind, fusion_input, fusion_output).Body( new Call(new TIRTest.LoadT(), fusion_input, glb), @@ -50,12 +50,12 @@ public static PrimFunctionWrapper MakeLoadStoreFunc(bool mask) public static PrimFunctionWrapper MakeBinaryFunc(BinaryOp binaryOp, bool mask) { var allocator = new Allocator(); - var fusion_input_lhs = allocator.Allocate($"fusion_{_count}_input_lhs", Schedule.MemoryLocation.Input); - var fusion_input_rhs = allocator.Allocate($"fusion_{_count}_input_rhs", Schedule.MemoryLocation.Input); - var glb_lhs = allocator.Allocate($"fusion_{_count}_glb_lhs", Schedule.MemoryLocation.L2Data); - var glb_rhs = allocator.Allocate($"fusion_{_count}_glb_rhs", Schedule.MemoryLocation.L2Data); - var glb_output = allocator.Allocate($"fusion_{_count}_glb_output", Schedule.MemoryLocation.L2Data); - var fusion_output = allocator.Allocate($"fusion_{_count}_output", Schedule.MemoryLocation.Output); + var fusion_input_lhs = allocator.Allocate($"fusion_{_count}_input_lhs", TIR.MemoryLocation.Input); + var fusion_input_rhs = allocator.Allocate($"fusion_{_count}_input_rhs", TIR.MemoryLocation.Input); + var glb_lhs = allocator.Allocate($"fusion_{_count}_glb_lhs", TIR.MemoryLocation.L2Data); + var glb_rhs = allocator.Allocate($"fusion_{_count}_glb_rhs", TIR.MemoryLocation.L2Data); + var glb_output = allocator.Allocate($"fusion_{_count}_glb_output", TIR.MemoryLocation.L2Data); + var fusion_output = allocator.Allocate($"fusion_{_count}_output", TIR.MemoryLocation.Output); var fusion = TIR.T.PrimFunc($"fusion_{_count}_{mask}", Callable.StackVMModuleKind, fusion_input_lhs, fusion_input_rhs, fusion_output).Body( new Call(new TIRTest.LoadT(), fusion_input_lhs, glb_lhs), @@ -71,16 +71,16 @@ public static PrimFunctionWrapper MakeBinaryFunc(BinaryOp binaryOp, bool mask) public static PrimFunctionWrapper MakeMultiInputFunc(int length, bool mask) { var allocator = new Allocator(); - var fusion_inputs = new List(); + var fusion_inputs = new List(); for (int i = 0; i < length; i++) { - var fusion_input_i = allocator.Allocate($"fusion_{_count}_input_{i}", Schedule.MemoryLocation.Input); + var fusion_input_i = allocator.Allocate($"fusion_{_count}_input_{i}", TIR.MemoryLocation.Input); fusion_inputs.Add(fusion_input_i); } - var glb1 = allocator.Allocate($"fusion_{_count}_glb1", Schedule.MemoryLocation.L2Data); - var glb2 = allocator.Allocate($"fusion_{_count}_glb2", Schedule.MemoryLocation.L2Data); - var fusion_output = allocator.Allocate($"fusion_{_count}_output", Schedule.MemoryLocation.Output); + var glb1 = allocator.Allocate($"fusion_{_count}_glb1", TIR.MemoryLocation.L2Data); + var glb2 = allocator.Allocate($"fusion_{_count}_glb2", TIR.MemoryLocation.L2Data); + var fusion_output = allocator.Allocate($"fusion_{_count}_output", TIR.MemoryLocation.Output); var fusion = TIR.T.PrimFunc($"multi_fusion_{_count}_{mask}", Callable.StackVMModuleKind, fusion_inputs.Concat(new[] { fusion_output }).ToArray()); @@ -124,18 +124,20 @@ private static IEnumerable GetBinaryOp(int length) private sealed class Allocator { - private readonly Dictionary _useage = new() { - { Schedule.MemoryLocation.Input, 0 }, - { Schedule.MemoryLocation.Output, 0 }, - { Schedule.MemoryLocation.L2Data, 0 }, + private readonly Dictionary _usage = new() { + { TIR.MemoryLocation.Input, 0 }, + { TIR.MemoryLocation.Output, 0 }, + { TIR.MemoryLocation.L2Data, 0 }, }; - public TIR.PhysicalBuffer Allocate(string name, Schedule.MemoryLocation location) + public TIR.Buffer Allocate(string name, TIR.MemoryLocation location) { - var strides = TensorUtilities.GetStrides(Dimensions); - var size = TensorUtilities.GetSize(Dimensions, strides, DataTypes.Float32.SizeInBytes); - var buffer = new TIR.PhysicalBuffer(name, DataTypes.Float32, location, Dimensions, strides, _useage[location], size); - _useage[location] += size; + var dims = Dimensions.Select(d => (Expr)d).ToArray(); + var strides = TensorUtilities.GetStrides(Dimensions).Select(s => (Expr)s).ToArray(); + var size = TensorUtilities.GetSize(Dimensions, TensorUtilities.GetStrides(Dimensions), DataTypes.Float32.SizeInBytes); + + var buffer = new TIR.Buffer(name, DataTypes.Float32, new TIR.MemSpan(Tensor.FromPointer(_usage[location]), size, location), dims, strides); + _usage[location] += (ulong)size; return buffer; } } diff --git a/src/Nncase.Tests/TIR/PrimFunc/UnitTestPrimFuncMerge.cs b/src/Nncase.Tests/TIR/PrimFunc/UnitTestPrimFuncMerge.cs index 9cd1eb7139..96c6bb415d 100644 --- a/src/Nncase.Tests/TIR/PrimFunc/UnitTestPrimFuncMerge.cs +++ b/src/Nncase.Tests/TIR/PrimFunc/UnitTestPrimFuncMerge.cs @@ -44,16 +44,17 @@ public class UnitTestPrimFuncMerge : TestClassBase public IAnalyzerManager AnalyzerMananger => CompileSession.GetRequiredService(); - [Theory] + [Theory(Skip = "Disable")] [MemberData(nameof(Datas))] private async void RunCore(IDataFlowPrimFuncCase fusionCase, int count) { + var dumper = Diagnostics.DumpScope.Current.CreateSubDummper($"case_{count}"); var inputVar = new Var("input", new TensorType(DataTypes.Float32, PrimFuncBuilder.Dimensions)); var main = new Function(fusionCase.BuildBody(inputVar), inputVar); CompilerServices.InferenceType(main); #if DEBUG - Dumpper.DumpDotIR(main, $"{count}_pre"); + Diagnostics.DumpScope.Current.DumpDotIR(main, $"{count}_pre"); #endif var feedDict = new Dictionary(ReferenceEqualityComparer.Instance) { { inputVar, IR.F.Random.Normal(DataTypes.Float32, 0, 1, 12, PrimFuncBuilder.Dimensions).Evaluate() }, @@ -69,7 +70,7 @@ private async void RunCore(IDataFlowPrimFuncCase fusionCase, int count) var post = (Function)module.Entry!; #if DEBUG - Dumpper.DumpDotIR(post, $"{count}_post"); + Diagnostics.DumpScope.Current.DumpDotIR(post, $"{count}_post"); #endif var visitor = new TestVisitor(); @@ -121,11 +122,11 @@ internal sealed class PrimFuncEvaluateVisitor private static readonly int _pool_size = 1 * 4 * 8 * 9 * 4 * 30; private readonly PrimFunctionWrapper _wrapper; private readonly IValue[] _args; - private readonly Dictionary _poolMap = new() { - { Schedule.MemoryLocation.Input, new byte[_pool_size] }, - { Schedule.MemoryLocation.L2Data, new byte[_pool_size] }, - { Schedule.MemoryLocation.Data, new byte[_pool_size] }, - { Schedule.MemoryLocation.Output, new byte[_pool_size] }, + private readonly Dictionary _poolMap = new() { + { TIR.MemoryLocation.Input, new byte[_pool_size] }, + { TIR.MemoryLocation.L2Data, new byte[_pool_size] }, + { TIR.MemoryLocation.Data, new byte[_pool_size] }, + { TIR.MemoryLocation.Output, new byte[_pool_size] }, }; public PrimFuncEvaluateVisitor(PrimFunctionWrapper wrapper, params IValue[] args) @@ -139,8 +140,8 @@ public IValue Evaluate() // 1. copy input into input pool foreach (var (arg, param) in _args.Zip(_wrapper.Target.Parameters[.._wrapper.ParametersCount].ToArray())) { - Assert.Equal(param.Size, arg.AsTensor().BytesBuffer.Length); - arg.AsTensor().BytesBuffer.CopyTo(_poolMap[param.MemLocation].AsSpan(param.Start)); + Assert.Equal(param.MemSpan.Size.Evaluate().AsTensor().ToScalar(), arg.AsTensor().BytesBuffer.Length); + arg.AsTensor().BytesBuffer.CopyTo(_poolMap[param.MemSpan.Location].AsSpan(param.MemSpan.Start.Evaluate().AsTensor().ToScalar())); } // 2. start l2 computing @@ -153,7 +154,7 @@ public IValue Evaluate() var tensors = new List(); foreach (var outputParam in _wrapper.Target.Parameters[_wrapper.ParametersCount..]) { - tensors.Add(Tensor.FromBytes(outputParam.ElemType, GetBufferSpan(outputParam).ToArray(), outputParam.FixedDimensions.ToArray())); + tensors.Add(Tensor.FromBytes(outputParam.ElemType, GetBufferSpan(outputParam).ToArray(), outputParam.Dimensions.AsValueEnumerable().Select(e => e.Evaluate().AsTensor().ToScalar()).ToArray())); } return tensors.Count == 1 ? Value.FromTensor(tensors[0]) : Value.FromTensors(tensors.ToArray()); @@ -208,7 +209,7 @@ private void EvaluateStatement(Expr statement) private Span GetBufferSpan(Expr expr) { - var buffer = Assert.IsType(expr); - return _poolMap[buffer.MemLocation].AsSpan(buffer.Start, buffer.Size); + var buffer = Assert.IsType(expr); + return _poolMap[buffer.MemSpan.Location].AsSpan(buffer.MemSpan.Start.Evaluate().AsTensor().ToScalar(), buffer.MemSpan.Size.Evaluate().AsTensor().ToScalar()); } } diff --git a/src/Nncase.Tests/TIR/UnitTestMutators.cs b/src/Nncase.Tests/TIR/UnitTestMutators.cs index a90571e8a0..e115f9eeb6 100644 --- a/src/Nncase.Tests/TIR/UnitTestMutators.cs +++ b/src/Nncase.Tests/TIR/UnitTestMutators.cs @@ -30,9 +30,9 @@ public UnitTestMutators() [Fact] public async Task TestFoldConstCallWithTuple() { - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Input, new[] { 48 }, out var ddr_if); - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Data, new[] { 9 }, out var glb_if_ping); - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Data, new[] { 9 }, out var glb_if_pong); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 48 }), MemoryLocation.Input, out var ddr_if); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 9 }), MemoryLocation.Data, out var glb_if_ping); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 9 }), MemoryLocation.Data, out var glb_if_pong); PrimFunction main; { main = T.PrimFunc("main", Callable.StackVMModuleKind, ddr_if).Body( @@ -76,7 +76,7 @@ public async Task TestFoldConstCallWithTuple() int count = 0; for (int w = 0; w < 48; w += 9) { - Assert.True(object.ReferenceEquals(getBuffer(count, LoadT.DdrPp), post.Parameters[0])); + // Assert.True(object.ReferenceEquals(getBuffer(count, LoadT.DdrPp), post.Parameters[0])); var name = getBuffer(count++, LoadT.GlbPp).Name[^4..]; // System.Console.WriteLine($"{w} {name}"); @@ -118,8 +118,8 @@ public async Task TestUnRollLoopSequential() [Fact] public async Task TestUnRollLoopSequential2() { - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Input, new[] { 3, 16, 24, 24 }, out var ddr_if); - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Data, new[] { 3, 10, 5, 9 }, out var glb_if); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 3, 16, 24, 24 }), MemoryLocation.Input, out var ddr_if); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 3, 10, 5, 9 }), MemoryLocation.Data, out var glb_if); PrimFunction main; { @@ -201,8 +201,8 @@ public async Task TestUnRollLoopSequential2() [Fact] public async Task TestUnRollLoopSequential3() { - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Input, new[] { 3, 16, 24, 24 }, out var ddr_if); - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Data, new[] { 3, 10, 5, 9 }, out var glb_if); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 3, 16, 24, 24 }), MemoryLocation.Input, out var ddr_if); + T.CreateBuffer(new TensorType(DataTypes.BFloat16, new[] { 3, 10, 5, 9 }), MemoryLocation.Data, out var glb_if); PrimFunction main; { @@ -362,10 +362,10 @@ public async Task TestFoldLet2() [Fact] public async Task TestFoldBufferIndex() { - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Input, new[] { 3, 16, 24, 24 }, out var ddr_if); - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Output, new[] { 3, 16, 24, 24 }, out var ddr_of); - T.PhysicalBuffer(DataTypes.BFloat16, Schedule.MemoryLocation.Data, new[] { 3, 10, 5, 9 }, out var glb_if); - var bufferIndexMap = new Dictionary() { + T.CreateBuffer(new(DataTypes.BFloat16, new[] { 3, 16, 24, 24 }), MemoryLocation.Input, out var ddr_if); + T.CreateBuffer(new(DataTypes.BFloat16, new[] { 3, 16, 24, 24 }), MemoryLocation.Output, out var ddr_of); + T.CreateBuffer(new(DataTypes.BFloat16, new[] { 3, 10, 5, 9 }), MemoryLocation.Data, out var glb_if); + var bufferIndexMap = new Dictionary() { { ddr_if, 2 }, { ddr_of, 4 }, }; @@ -386,7 +386,7 @@ public async Task TestFoldBufferIndex() pass.Add(); pass.Add(Expr? (Expr e) => { - if (e is Call { } call && call.Arguments[0] is PhysicalBuffer physicalBuffer && bufferIndexMap.TryGetValue(physicalBuffer, out var index)) + if (e is Call { } call && call.Arguments[0] is Buffer physicalBuffer && bufferIndexMap.TryGetValue(physicalBuffer, out var index)) { return index; } diff --git a/src/Nncase.Tests/Transform/UnitTestPassManager.cs b/src/Nncase.Tests/Transform/UnitTestPassManager.cs index bc7cb98896..cfc76bb4eb 100644 --- a/src/Nncase.Tests/Transform/UnitTestPassManager.cs +++ b/src/Nncase.Tests/Transform/UnitTestPassManager.cs @@ -22,7 +22,7 @@ public sealed class UnitTestPassManager : TestClassBase [Fact] public void TestPassMangerUpdateDependence() { - var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 2, 3, 4 }, out _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 2, 3, 4 }, out _)).Body(T.Nop()).Build(); + var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Output, out _)).Body(T.Nop()).Build(); var prim_wrapper = new PrimFunctionWrapper(prim_func_1, 1); @@ -30,7 +30,7 @@ public void TestPassMangerUpdateDependence() var main_func = new Function("main", new Call(prim_wrapper, input), input); // prim_func_2 for update - var prim_func_2 = T.PrimFunc("prim_func_2", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 2, 3, 4 }, out _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 2, 3, 4 }, out _)).Body( + var prim_func_2 = T.PrimFunc("prim_func_2", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Output, out _)).Body( T.Nop(), T.Nop()).Build(); @@ -54,15 +54,15 @@ public void TestPassMangerUpdateDependence2() %3 = %func_3(%2): // f16[1,23,30,16] */ - var prim_func_0 = T.PrimFunc("prim_func_0", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 24, 32, 3 }, out var _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 3, 24, 32 }, out var _)).Body( + var prim_func_0 = T.PrimFunc("prim_func_0", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 24, 32, 3 }), MemoryLocation.Input, out var _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 3, 24, 32 }), MemoryLocation.Output, out var _)).Body( T.Nop()).Build(); var func_0 = new PrimFunctionWrapper(prim_func_0, 1); - var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 3, 24, 32 }, out var _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 3, 24, 32 }, out var _)).Body( + var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 3, 24, 32 }), MemoryLocation.Input, out var _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 3, 24, 32 }), MemoryLocation.Output, out var _)).Body( T.Nop()).Build(); var func_1 = new PrimFunctionWrapper(prim_func_1, 1); - var prim_func_2 = T.PrimFunc("prim_func_2", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 3, 24, 32 }, out var _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 23, 30, 16 }, out var _)).Body( + var prim_func_2 = T.PrimFunc("prim_func_2", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 3, 24, 32 }), MemoryLocation.Input, out var _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 23, 30, 16 }), MemoryLocation.Output, out var _)).Body( T.Nop()).Build(); var func_2 = new PrimFunctionWrapper(prim_func_2, 1); @@ -74,7 +74,7 @@ public void TestPassMangerUpdateDependence2() Assert.True(CompilerServices.InferenceType(main_func)); // prim_func_2 for update - var prim_func_1_update = T.PrimFunc("prim_func_1_update", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 3, 24, 32 }, out var _), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 3, 24, 32 }, out var _)).Body( + var prim_func_1_update = T.PrimFunc("prim_func_1_update", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 3, 24, 32 }), MemoryLocation.Input, out var _), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 3, 24, 32 }), MemoryLocation.Output, out var _)).Body( T.Nop(), T.Nop()).Build(); diff --git a/src/Nncase.Tests/Transform/UnitTestSubstitutor.cs b/src/Nncase.Tests/Transform/UnitTestSubstitutor.cs index 9313303252..0864bca6de 100644 --- a/src/Nncase.Tests/Transform/UnitTestSubstitutor.cs +++ b/src/Nncase.Tests/Transform/UnitTestSubstitutor.cs @@ -24,8 +24,9 @@ public sealed class UnitTestSubstitutor : TestClassBase public void TestSubstitutorFailed() { var loop_i = new Var("loop_i", TensorType.Scalar(DataTypes.Int32)); - var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 2, 3, 4 }, out var input_a), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 2, 3, 4 }, out var input_b)).Body( - T.Load(T.Handle("hd", DataTypes.Float32), loop_i)).Build(); + T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out var hd); + var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out var input_a), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Output, out var input_b)).Body( + T.Load(hd, loop_i)).Build(); var prim_wrapper = new PrimFunctionWrapper(prim_func_1, 1); @@ -48,8 +49,9 @@ public void TestSubstitutorFailed() public void TestSubstitutorTrue() { var loop_i = new Var("loop_i", TensorType.Scalar(DataTypes.Int32)); - var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 2, 3, 4 }, out var input_a), T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Output, new[] { 1, 2, 3, 4 }, out var input_b)).Body( - T.Load(T.Handle("hd", DataTypes.Float32), loop_i)).Build(); + T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out var hd); + var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out var input_a), T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Output, out var input_b)).Body( + T.Load(hd, loop_i)).Build(); Dictionary vmap = new() { { loop_i, 1 } }; var substitutor = Mutator.Substitute(e => vmap.TryGetValue(e, out var res) ? res : null)(); @@ -65,8 +67,9 @@ public void TestSubstitutorTrue() public void TestSubstitutorTrue2() { var loop_i = new Var("loop_i", TensorType.Scalar(DataTypes.Int32)); - var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.PhysicalBuffer(DataTypes.Float32, Schedule.MemoryLocation.Input, new[] { 1, 2, 3, 4 }, out var input_a), T.PhysicalBuffer(DataTypes.Int32, Schedule.MemoryLocation.Output, new[] { 1, 2, 3, 4 }, out var input_b)).Body( - T.Load(T.Handle("hd", DataTypes.Float32), loop_i)).Build(); + T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out var hd); + var prim_func_1 = T.PrimFunc("prim_func_1", "k?", T.CreateBuffer(new(DataTypes.Float32, new[] { 1, 2, 3, 4 }), MemoryLocation.Input, out var input_a), T.CreateBuffer(new(DataTypes.Int32, new[] { 1, 2, 3, 4 }), MemoryLocation.Output, out var input_b)).Body( + T.Load(hd, loop_i)).Build(); var prim_wrapper = new PrimFunctionWrapper(prim_func_1, 1); diff --git a/src/Nncase.Tests/packages.lock.json b/src/Nncase.Tests/packages.lock.json index c722181308..beff5e02b5 100644 --- a/src/Nncase.Tests/packages.lock.json +++ b/src/Nncase.Tests/packages.lock.json @@ -80,11 +80,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "System.Linq.Async": { @@ -490,8 +490,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -871,6 +871,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -1099,6 +1100,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/targets/Nncase.Targets.CSource/CSourceTarget.cs b/targets/Nncase.Targets.CSource/CSourceTarget.cs deleted file mode 100644 index e6f3e19a8f..0000000000 --- a/targets/Nncase.Targets.CSource/CSourceTarget.cs +++ /dev/null @@ -1,34 +0,0 @@ - -namespace Nncase.Targets; - -public class CSourceTarget : ITarget -{ - /// - public string Kind { get => "CSource"; set { } } - /// - public Dictionary Options { get; set; } = new(); - /// - public Dictionary Attrs { get; set; } = new(); - /// - public void ConfigOptions() { } - /// - public void ConfigAttrs() { } - - /// - public Schedule.IScheduler CreateScheduler(IR.IRModule main_module) - { - return new Schedule.CSourceScheduler(main_module, this); - } - - /// - public CodeGen.IRTModel CreateRTModel(IR.IRModel model) - { - return new CodeGen.CSourceRTModel(model, this); - } - - /// - public CodeGen.IRTModule CreateRTModule(IR.IRModel model, IR.IRModule module) - { - throw new NotImplementedException("The CSource Target Only Have Runtime Model!"); - } -} diff --git a/targets/Nncase.Targets.CSource/CodeGen/CSource.cs b/targets/Nncase.Targets.CSource/CodeGen/CSource.cs deleted file mode 100644 index afc0de3b68..0000000000 --- a/targets/Nncase.Targets.CSource/CodeGen/CSource.cs +++ /dev/null @@ -1,278 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using Nncase.IR; -using Nncase.Schedule; -using Nncase.TIR; - -namespace Nncase.CodeGen; - -/// -/// the c source runtime function. -/// -/// -/// -public record CSourceRTFunction(string name, Delegate handle) : IRTFunction -{ - public string Name { get => name; set { } } - public Delegate Handle { get => handle; set { } } -} - -public class CSourceSerializeResult : ISerializeResult -{ - -} - -/// -/// c runtime module impl -/// -public class CSourceRTModel : IRTModule, IRTModel -{ - /// - public ModuleType ModuleType { get => CodeGen.ModuleType.Create("CSource"); set { } } - - /// - public ITarget Target { get; set; } - - /// - public IReadOnlyList Modules => throw new NotImplementedException(); - - /// - public string SourcePath { get; private set; } - - public IRModel Model { get; set; } - IRTFunction? _entry = null; - - /// - public bool IsSerialized { get; private set; } - - readonly List _functions = new(); - - /// - /// - /// - public CSourceRTModel(IRModel model, ITarget target) - { - SourcePath = CodeGenUtil.GetTempFileName("c"); - Model = model; - Target = target; - } - - /// - public byte[] Source { get => File.ReadAllBytes(SourcePath); set { } } - - /// - public string SourceExt { get => "c"; set { } } - - /// - public IRTFunction? Entry => _entry; - - /// - public IReadOnlyList Functions => _functions; - - /// - string _dllPath = ""; - - /// - /// write the c source code into source path. - /// - /// - void BuildCode() - { - if (File.Exists(SourcePath)) - File.Delete(SourcePath); - using (var writer = new StreamWriter(SourcePath, false, Encoding.UTF8)) - { - var visior = new CSourceHostBuildVisior(writer); - if (Model.Entry is null) { throw new InvalidProgramException("The Model Entry Is Null!"); } - if (Model.Entry.CheckedType is null && Model.Entry.InferenceType() == false) { throw new InvalidProgramException("The Model Entry Can't Inference Type!"); } - visior.Visit(Model.Entry); - } - } - - public void CompileCode() - { - if (!File.Exists(SourcePath)) - throw new InvalidProgramException("The Source Code Path Is Invalid!"); - var compiler = new CSourceCompiler(); - _dllPath = compiler.Compile(SourcePath); - } - - /// - /// bind each IR.Funtion with C function - /// - /// - public void ExportCode() - { - if (!File.Exists(_dllPath)) - throw new InvalidProgramException("The DLL Path Is Invalid!"); - var dllPtr = NativeLibrary.Load(_dllPath); - foreach (var module in Model.Modules) - { - foreach (var f in module.Callables) - { - var funcType = f.ToDelegateType(Path.GetFileName(_dllPath)); - var funPtr = NativeLibrary.GetExport(dllPtr, f.Name); - _functions.Add(new CSourceRTFunction(f.Name, funPtr.BindDelegate(funcType))); - if (f == Model.Entry) { _entry = _functions.Last(); } - } - } - } - - /// - public ISerializeResult Serialize() - { - if (IsSerialized) { return new CSourceSerializeResult(); } - BuildCode(); - CompileCode(); - ExportCode(); - return new CSourceSerializeResult(); - } - - /// - /// invoke the module entry - /// - /// input args - /// results - /// - public object? Invoke(params object?[]? args) - { - if (Entry is null) - throw new InvalidOperationException("This RTModule Have No Entry Function!"); - return Entry.Handle.DynamicInvoke(args); - } - - public string Dump(string name, string DumpDirPath) - { - var dump_path = $"{DumpDirPath}/{name}.{SourceExt}"; - using var file = File.Open(dump_path, FileMode.OpenOrCreate, FileAccess.Write); - using var writer = new StreamWriter(file); - writer.Write(Source); - return dump_path; - } - -} - -/// -/// the csource code compiler. -/// -public class CSourceCompiler -{ - /// - /// compiler exe name - /// - string _exe = "", _arch = "", _ext = ""; - - /// - /// select current pattern's exe - /// - /// - void PlatformSpecific() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - _exe = "gcc"; - _ext = "so"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - _exe = "clang"; - _ext = "dylib"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - _exe = "cmd"; - _ext = "dll"; - } - } - - void ArchSpecific() - { - _arch = RuntimeInformation.OSArchitecture switch - { - Architecture.X64 => RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "x86-64" : "x86_64", - Architecture.Arm64 => "arm64", - _ => throw new NotSupportedException(RuntimeInformation.OSArchitecture.ToString()), - }; - } - - string ArgumentsSpecific(string sourcePath, string outPath) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - return $"{sourcePath} -fPIC -shared -march={Arch} -o {outPath}"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - return $"{sourcePath} -fPIC -shared -arch {Arch} -o {outPath}"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var vsdir = Environment.GetEnvironmentVariable("VSAPPIDDIR") ?? throw new InvalidOperationException("Cannot find vs"); - var vcvardir = Path.Combine(vsdir, "..\\..\\VC\\Auxiliary\\Build\\vcvarsall.bat"); - return $"/C (\"{vcvardir}\" x64) && (cl /D_USRDLL /D_WINDLL \"{sourcePath}\" /MT /link /DLL /OUT:\"{outPath}\")"; - } - throw new System.ArgumentOutOfRangeException("Only Support Linux/Osx/Windows"); - } - - protected string Exe - { - get => _exe; - } - - protected string Arch - { - get => _arch; - } - - protected string Ext - { - get => _ext; - } - - public CSourceCompiler() - { - PlatformSpecific(); - ArchSpecific(); - } - - /// - /// compile the source txt, write to the out_path - /// - /// c source code - /// out .so path - /// outPath - public string Compile(string sourcePath, string outPath) - { - var errMsg = new StringBuilder(); - using (var errWriter = new StringWriter(errMsg)) - { - using (var proc = new Process()) - { - proc.StartInfo.FileName = Exe; - proc.StartInfo.Arguments = ArgumentsSpecific(sourcePath, outPath); - proc.StartInfo.RedirectStandardError = true; - proc.ErrorDataReceived += (sender, e) => errWriter.WriteLine(e.Data); - proc.Start(); - proc.BeginErrorReadLine(); - proc.WaitForExit(); - if (proc.ExitCode != 0) - { - throw new InvalidOperationException(errMsg.ToString()); - } - } - } - return outPath; - } - - /// - /// create the temp dll file and compile source - /// - /// - public string Compile(string sourcePath) => Compile(sourcePath, CodeGenUtil.GetTempFileName(Ext)); -} \ No newline at end of file diff --git a/targets/Nncase.Targets.CSource/CodeGen/CSourceVisitor.cs b/targets/Nncase.Targets.CSource/CodeGen/CSourceVisitor.cs deleted file mode 100644 index 352b9dc6ea..0000000000 --- a/targets/Nncase.Targets.CSource/CodeGen/CSourceVisitor.cs +++ /dev/null @@ -1,317 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using Nncase.IR; -using Nncase.Runtime; -using Nncase.TIR; - -namespace Nncase.CodeGen; - -/// -/// convert the type/op to c name -/// -internal static class NameConverter -{ - private static readonly Dictionary _primTypeToC = new() - { - { DataTypes.Boolean, "bool" }, - { DataTypes.Int8, "int8_t" }, - { DataTypes.Int16, "int16_t" }, - { DataTypes.Int32, "int32_t" }, - { DataTypes.Int64, "int64_t" }, - { DataTypes.UInt8, "uint8_t" }, - { DataTypes.UInt16, "uint16_t" }, - { DataTypes.UInt32, "uint32_t" }, - { DataTypes.UInt64, "uint64_t" }, - { DataTypes.Float32, "float" }, - { DataTypes.Float64, "double" }, - }; - - public static string toC(this PrimType primType) => - _primTypeToC[primType]; - - public static string toC(this DataType dataType) => dataType switch - { - PrimType ptype => ptype.toC(), - PointerType { ElemType: PrimType etype } => etype.toC() + "*", - _ => throw new NotSupportedException(dataType.ToString()) - }; -} - -/// -/// the c symbol define -/// -internal struct CSymbol -{ - public string Type; - public StringBuilder Doc; - public CSymbol(string type, StringBuilder doc) - { - Type = type; - Doc = doc; - } - public override string ToString() => $"{Type} {Doc}"; -} - -/// -/// collect the csymbol's parameter -/// -internal class CSymbolParamList : IParameterList, IEnumerable -{ - CSymbol[] Symbols; - public CSymbolParamList(CSymbol[] symbols) - { - Symbols = symbols; - } - - public CSymbol this[ParameterInfo parameter] => Symbols[parameter.Index]; - public CSymbol this[int index] => Symbols[index]; - - public IEnumerator GetEnumerator() - { - return ((IEnumerable)Symbols).GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return Symbols.GetEnumerator(); - } -} - - -/// -/// visitor for the build c source code, the expr vistor return (type string , name string) -/// -internal class CSourceHostBuildVisior : ExprFunctor -{ - - /// - /// source writer . - /// TODO we need the decl writer - /// - readonly ScopeWriter Scope; - - /// - /// symbols name memo - /// - readonly Dictionary Symbols = new(ReferenceEqualityComparer.Instance); - - /// - /// - /// - /// - public CSourceHostBuildVisior(TextWriter textWriter) - { - Scope = new ScopeWriter(textWriter); - // insert some declare - Scope.IndWriteLine(@" -#ifdef _WIN32 -#define EXPORT_API __declspec(dllexport) -#else -#define EXPORT_API -#endif"); - Scope.IndWriteLine("#include "); - } - - /// - public override CSymbol Visit(Call expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - var target = Visit(expr.Target); - var args = new CSymbolParamList(expr.Parameters.Select(Visit).ToArray()); - var type = VisitType(expr.CheckedType!); - Scope.Push(); - switch (expr.Target) - { - case IR.Math.Binary: - Scope.Append($"({args[0].Doc} {target.Doc} {args[1].Doc})"); - break; - case Store: - Scope.Append($"{args[Store.Handle].Doc}[{args[Store.Index].Doc}] = {args[Store.Value].Doc}"); - break; - case Load: - Scope.Append($"{args[Store.Handle].Doc}[{args[Store.Index].Doc}]"); - break; - case IR.Tensors.Cast: - Scope.Append($"(({type}){args[IR.Tensors.Cast.Input].Doc})"); - break; - default: - Scope.Append($"{target.Doc}({string.Join(", ", args.Select(x => x.Doc))})"); - break; - } - symbol = new(type, Scope.Pop()); - Symbols.Add(expr, symbol); - return symbol; - } - - /// - public override CSymbol Visit(Const expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - if (expr.CheckedType is TensorType ttype && ttype.IsScalar) - { - var literal = $"{expr}" switch - { - "True" => "1", - "False" => "0", - var x => x - }; - symbol = new(VisitType(ttype), new(literal)); - } - else - { - throw new NotSupportedException($"Not Support {expr.CheckedType} Const"); - } - Symbols.Add(expr, symbol); - return symbol; - } - - /// - public override CSymbol Visit(Function expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - var retType = VisitType(((CallableType)expr.CheckedType!).ReturnType); - Scope.Push(); - // 1. Function signature - Scope.IndWrite($"EXPORT_API {retType} {expr.Name}({string.Join(", ", expr.Parameters.Select(Visit))}) {{"); - // 2. Function body - using (Scope.IndentUp()) - { - Scope.Append(Visit(expr.Body).Doc); - } - // 3. Function closing - Scope.IndWrite("}"); - symbol = new(CallableTypeToPtr((CallableType)expr.CheckedType!, expr.Name), Scope.Pop()); - // 4. write whole code - Scope.IndWrite(symbol.Doc); - return symbol; - } - - /// - public override CSymbol Visit(Op expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - symbol = new("Invalid Op", new(expr switch - { - IR.Math.Binary op => op.BinaryOp switch - { - BinaryOp.Add => "+", - BinaryOp.Sub => "-", - BinaryOp.Mul => "*", - BinaryOp.Div => "/", - BinaryOp.Mod => "%", - _ => throw new ArgumentOutOfRangeException(op.BinaryOp.ToString()) - }, - TIR.Store op => "Store", - TIR.Load op => "Load", - IR.Tensors.Cast op => op.NewType.toC(), - _ => throw new NotSupportedException($"{expr.GetType().Name}") - })); - Symbols.Add(expr, symbol); - return symbol; - } - - /// - public override CSymbol Visit(Var expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - var isymbol = Scope.GetUniqueVarSymbol(expr); - symbol = new(VisitType(expr.CheckedType!), isymbol.Span); - Symbols.Add(expr, symbol); - return symbol; - } - - /// - public override CSymbol Visit(For expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - Scope.Push(); - // 1. For Loop signature - var loopVar = Visit(expr.LoopVar); - Scope.Append($"for ({loopVar} = {Visit(expr.Dom.Start).Doc}; {loopVar.Doc} < {Visit(expr.Dom.Stop).Doc}; {loopVar.Doc}+={expr.Dom.Step}) {{"); - // 2. For Body - Scope.Append(Visit(expr.Body).Doc); - // 3. For closing - Scope.IndWrite("}"); - symbol = new(VisitType(expr.CheckedType!), Scope.Pop()); - Symbols.Add(expr, symbol); - return symbol; - } - - /// - public override CSymbol Visit(Sequential expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - Scope.Push(); - Scope.AppendLine(""); - using (Scope.IndentUp()) - { - foreach (var i in Enumerable.Range(0, expr.Fields.Count)) - { - if (i == expr.Fields.Count - 1 && - expr.Fields[i].CheckedType is TensorType) - { - Scope.IndWrite("return "); - } - else - { - Scope.IndWrite(string.Empty); - } - Scope.Append(Visit(expr.Fields[i]).Doc); - if (expr.Fields[i] is Call) - { - Scope.AppendLine(";"); - } - else - { - Scope.AppendLine(string.Empty); - } - } - } - symbol = new(VisitType(expr.CheckedType!), Scope.Pop()); - Symbols.Add(expr, symbol); - return symbol; - } - - /// - public override CSymbol Visit(IfThenElse expr) - { - if (Symbols.TryGetValue(expr, out var symbol)) { return symbol; } - Scope.Push(); - Scope.Append($"if({Visit(expr.Condition).Doc}) {{"); - Scope.Append(Visit(expr.Then).Doc); - Scope.IndWrite("} else {"); - Scope.Append(Visit(expr.Else).Doc); - Scope.IndWrite("}"); - symbol = new(VisitType(expr.CheckedType!), Scope.Pop()); - Symbols.Add(expr, symbol); - return symbol; - } - - /// - /// - /// void (*fun_ptr)(int) - /// - public string CallableTypeToPtr(CallableType type, string name) => $"{VisitType(type.ReturnType)} (*{name}_ptr)({string.Join(",", type.Parameters.Select(VisitType))})"; - - - /// - public override string VisitType(TensorType type) - { - if (!type.IsScalar) - { - throw new NotSupportedException($"{type}"); - } - return type.DType.toC(); - } - - /// - public override string VisitType(TupleType type) => type == TupleType.Void ? - "void" : - throw new InvalidProgramException($"The C Source Must Not Have TupleType {type}!"); -} \ No newline at end of file diff --git a/targets/Nncase.Targets.CSource/CodeGen/Interop.cs b/targets/Nncase.Targets.CSource/CodeGen/Interop.cs deleted file mode 100644 index 33d31af269..0000000000 --- a/targets/Nncase.Targets.CSource/CodeGen/Interop.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Reflection.Emit; -using System.Runtime.InteropServices; -using Nncase.IR; - -namespace Nncase.CodeGen; - -/// -/// -/// -internal class DynamicAssemble -{ - /// - /// name - /// - AssemblyName assemblyName; - /// - /// asm builder for whole module - /// - AssemblyBuilder asmBuilder; - /// - /// module buidler - /// - ModuleBuilder modBuilder; - /// - /// save the func name <=> func delegate type - /// - readonly Dictionary delegateTypes = new(); - - /// - /// a DynamicAssemble instance, it's contains one rtmodule's all functions defination. - /// - /// asmble name - public DynamicAssemble(string Name) - { - assemblyName = new AssemblyName(Name); - asmBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.RunAndCollect); - modBuilder = asmBuilder.DefineDynamicModule(assemblyName.Name!); - - } - - /// - /// - /// - /// - /// func delegate type - public Type BuildDelegateType(Callable function) - { - Type deleType; - if (function.CheckedType is CallableType ctype) - { - deleType = CreateDelegateType(function.Name, ctype.ReturnType.ToType(), ctype.Parameters.Select(Interop.ToType).ToArray()); - } - else { throw new NotSupportedException(function.CheckedType?.ToString()); } - return deleType; - } - - /// - /// dynamic create delegate type for function. - /// - /// - /// - /// - /// - /// - public Type CreateDelegateType(string funcName, Type returnType, params Type[]? ParamTypes) - { - if (!delegateTypes.TryGetValue(funcName, out var ret)) - { - ParamTypes ??= new Type[] { }; - TypeBuilder tb = modBuilder.DefineType(funcName, TypeAttributes.Public | TypeAttributes.Sealed, typeof(MulticastDelegate)); - tb.DefineConstructor(MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.SpecialName | MethodAttributes.RTSpecialName, CallingConventions.Standard | CallingConventions.HasThis, new[] { typeof(object), typeof(IntPtr) }).SetImplementationFlags(MethodImplAttributes.Runtime | MethodImplAttributes.Managed); - tb.DefineMethod("Invoke", MethodAttributes.Public | MethodAttributes.Virtual | MethodAttributes.HideBySig | MethodAttributes.NewSlot, CallingConventions.Standard | CallingConventions.HasThis, returnType, ParamTypes).SetImplementationFlags(MethodImplAttributes.Runtime | MethodImplAttributes.Managed); - tb.DefineMethod("BeginInvoke", MethodAttributes.Public | MethodAttributes.Virtual | MethodAttributes.HideBySig | MethodAttributes.NewSlot, CallingConventions.Standard | CallingConventions.HasThis, typeof(IAsyncResult), ParamTypes.Concat(new[] { typeof(IAsyncResult), typeof(object) }).ToArray()).SetImplementationFlags(MethodImplAttributes.Runtime | MethodImplAttributes.Managed); - tb.DefineMethod("EndInvoke", MethodAttributes.Public | MethodAttributes.Virtual | MethodAttributes.HideBySig | MethodAttributes.NewSlot, CallingConventions.Standard | CallingConventions.HasThis, returnType, new[] { typeof(IAsyncResult) }).SetImplementationFlags(MethodImplAttributes.Runtime | MethodImplAttributes.Managed); - ret = tb.CreateType(); - if (ret is null) { throw new InvalidProgramException($"Can't Create The Func {funcName}'s delegate Type!"); } - delegateTypes.Add(funcName, ret); - } - return ret; - } -} - -/// -/// Interop helper -/// -public static class Interop -{ - /// - /// collect the all dynamic asmbs - /// - private static readonly Dictionary _definedAsms = new(); - - /// - /// convert the ir type to the system type - /// - /// - /// - /// - public static Type ToType(this IRType iRType) => iRType switch - { - TensorType { IsScalar: true, DType: PrimType { } primType } => primType.CLRType, - TensorType { IsScalar: true, DType: PointerType { ElemType: PrimType primType } } => primType.CLRType.MakeArrayType(), - TupleType ttype => (ttype == TupleType.Void) switch - { - true => typeof(void), - false => throw new NotSupportedException($"Can't Support the {ttype}!") - }, - _ => throw new NotSupportedException($"IRType is {iRType}!") - }; - - - /// - /// convrt function to delegate type - /// - /// input function - /// the dynamic lib name - /// - /// - public static Type ToDelegateType(this Callable function, string libName) - { - if (!_definedAsms.TryGetValue(libName, out var dyasm)) - { - dyasm = new DynamicAssemble(libName); - _definedAsms.Add(libName, dyasm); - } - return dyasm.BuildDelegateType(function); ; - } - - /// - /// bind the delegate to funcptr. - /// - /// - /// - /// - public static Delegate BindDelegate(this IntPtr funcPtr, Type funcType) => Marshal.GetDelegateForFunctionPointer(funcPtr, funcType); -} diff --git a/targets/Nncase.Targets.CSource/Nncase.Targets.CSource.csproj b/targets/Nncase.Targets.CSource/Nncase.Targets.CSource.csproj deleted file mode 100644 index 79226f6c03..0000000000 --- a/targets/Nncase.Targets.CSource/Nncase.Targets.CSource.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net6.0 - enable - enable - $(SolutionDir)/tools/StyleCopAnalyzers.ruleset - - - - - - - - - diff --git a/targets/Nncase.Targets.CSource/Schedule/CSourceScheduler.cs b/targets/Nncase.Targets.CSource/Schedule/CSourceScheduler.cs deleted file mode 100644 index b57cf2419d..0000000000 --- a/targets/Nncase.Targets.CSource/Schedule/CSourceScheduler.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Nncase.IR; - -namespace Nncase.Schedule; - -public class CSourceScheduler : IScheduler -{ - - public CSourceScheduler(IR.IRModule main_module, ITarget target) - { - Module = main_module; - Target = target; - } - - public ITarget Target { get; set; } - public IRModule Module { get; set; } - - - IRModel IScheduler.Schedule(bool skip_buffer_alias) - { - return new IRModel(new[] { Module }); - } -} \ No newline at end of file diff --git a/tests/importer/onnx_/model/test_llama.py b/tests/importer/onnx_/model/test_llama.py new file mode 100644 index 0000000000..c6beee1ff6 --- /dev/null +++ b/tests/importer/onnx_/model/test_llama.py @@ -0,0 +1,127 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""System test: test demo""" +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +# from lzma import MODE_FAST +# from xml.parsers.expat import model +import pytest +from onnx_test_runner import OnnxTestRunner + + +def test_demo(request): + runner = OnnxTestRunner("demo1", "/root/Workspace/config/llama_config.toml") + # model_file = r'/data/huochenghai/onnx_model/shufflenet-9.onnx' + # model_file = '/compiler/huochenghai/GNNE/nncase_demo/examples/release_isp_object_detect_nncase/data/yolov5sFocus_320x3.onnx' + # model_file = '/data/huochenghai/GNNE/nncase_demo/examples/release_isp_retinaface_mb_320_nncase/data/retinaface_mobile0.25_320.onnx' + # model_file = '/data/huochenghai/GNNE/nncase_demo/examples/release_isp_face_landmarks106_nncase/data/retinaface_mobile0.25_320.onnx' + # model_file = '/data/huochenghai/GNNE/nncase_demo/examples/release_isp_face_landmarks106_nncase/data/v3.onnx' + # model_file = '/data/huochenghai/fixed_input.onnx' + # model_file = '/data/huochenghai/GNNE/nncase_demo/examples/release_isp_face_alignment_from_box_nncase/data/mb1_120x120.onnx' + # model_file = '/data/huochenghai/GNNE/nncase_demo/examples/release_isp_face_recog_mbface_nncase/data/mbface.onnx' + # model_file = '/data/huochenghai/GNNE/k510-gnne-compiler-tests/zhoumeng-model/resnet50v1/model_f32.onnx' + # model_file = '/data/huochenghai/deploy_modify.onnx' + # model_file = '/data/huochenghai/nanodet_mobilenetv2_416.onnx' + # model_file = '/data/huochenghai/yolov5_face_n0.5_256x256.onnx' + # model_file = '/data/huochenghai/yolov5s_0.5_640_dropact.onnx' + # model_file = '/data/huochenghai/GNNE/nncase_demo/examples/release_isp_object_detect_nncase/data/yolov5sFocus_320x3.onnx' + # model_file = '/data/huochenghai/nanodet_yolov5s_0.5_head_nospp_640.onnx' + # model_file = '/data/huochenghai/dw_21x21_model.onnx' + # model_file = '/compiler/huochenghai/GNNE/nncase/tests_output/test_decoder_part/simplified.onnx' + # model_file = '/data/huochenghai/onnx_model/yolop_self.onnx' + # model_file = '/data/huochenghai/yolov5s_640x640_sigmoid_weights.onnx' + # model_file = '/data/huochenghai/models/yolov5s_640_sigmoid.onnx' + # model_file = '/data/huochenghai/best_batchsize16_300' + # model_file = '/data/huochenghai/candy-9.onnx' + # model_file = '/data/huochenghai/glint360k_cosface_r18_fp16_0.1.onnx' + # model_file = '/data/huochenghai/cls_fixed2.onnx' + # model_file = '/data/huochenghai/stereo_ranpara.onnx' + # model_file = '/data/huochenghai/stereoNet.onnx' + # model_file = '/data/huochenghai/deploy_modify.onnx' + # model_file = '/data/huochenghai/model.onnx' + # model_file = "/data/huochenghai/onnx_model/lite-transformer-encoder.onnx" + # model_file = '/data/huochenghai/onnx_model/lite-transformer-decoder.onnx' + # model_file = '/data/huochenghai/pose_vgg_half_030.onnx' + # model_file = '/data/huochenghai/pose1040.onnx' + # model_file = '/data/huochenghai/net.onnx' + # model_file = '/data/huochenghai/face_expression.onnx' + # model_file = '/data/huochenghai/model_fixed_input_size.onnx' + # model_file = '/data/huochenghai/model_none_lstm.onnx' + # model_file = '/data/huochenghai/squeezenet1_1.onnx' + # model_file = '/data/huochenghai/resnet_tom.onnx' + # model_file = '/data/huochenghai/Ultralight-Nano-SimplePose.onnx' + # model_file = "/data/huochenghai/yolov5sface_640x640_6output.onnx" + # model_file = "/data/huochenghai/model-y1.onnx" + # model_file = "/data/huochenghai/sim_5.onnx" + # model_file = "/data/huochenghai/person_yolov5s_0.5_nospp_640_nncase.onnx" + # model_file = "/data/huochenghai/rec_2_layer_lstm.onnx" + # model_file = "/compiler/huochenghai/east_128_640.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/CRNN/ocr_rec_model_32-608.onnx" + # model_file = "/data/huochenghai/scrfd_person_2.5g_fixed_input_size_simplify.onnx" + # model_file = "/data/huochenghai/models/model_128-640-11.onnx" + # model_file = "/data/huochenghai/GNNE/nncase/tests_output/simplified.onnx" + # model_file = "/data/huochenghai/dw_deconv.onnx" + # model_file = "/data/huochenghai/GNNE/nncase/tests_output/test_exchannel_rhs_shape0-lhs_shape0_/simplified.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/lite-transformer/lite_transformer_encoder_L10.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/lite-transformer/lite_transformer_decoder_L10.onnx" + # model_file = "/data/huochenghai/lite_transformer_decoder_L10.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/yolov5s/yolov5s_640_sigmoid.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/efficientnet/efficientnet_b0_224x224.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/mobile-facenet/mbface_sim_224.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/mobile-retinaface/retinaface_mobile0.25_320_simplified.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/mobilenet-v1-ssd/ssd_mobilenetv1_300x300.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/mobilenetv2-yolov3/yolov3_mobilenetv2_no_postprocess.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/mobilenet-v2-ssd/ssd_mobilenetv2_300x300.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/yolov5m/yolov5_m_320x320_with_sigmoid.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/yolov5s_face/yolov5sface_640x640_6output.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/yolox/yolox_s.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/Ultralight-SimplePose/Ultralight-SimplePose.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/reid/osnet_x1_0.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/yolov7/0_yolov7-tiny-silu_320x320.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/reid/osnet_ain_x1_0.onnx" + # model_file = "/data/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/reid/osnet_ibn_x1_0.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/wzm/wzm_stereo6g.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/wzm/wzm_stereo.onnx" + # model_file = "/data/huochenghai/GNNE/nncase/tests_output/test_matmul_constant-in_shape0_/simplified.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/lite-transformer/youdaonmt/encoder_model.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/lite-transformer/youdaonmt/decoder_model.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/resnetv1_50/onnx/resnet50v1.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-ccompiler-tests/benchmark-test/lite-transformer/lite_transformer_encoder_L10.onnx" + # model_file = "/compiler/huochenghai/can3_10.0s_20221011084724.onnx" + # model_file = "/compiler/huochenghai/lstm_256.onnx" + # model_file = "/compiler/huochenghai/weilai/simplified_det.onnx" + # model_file = "/compiler/huochenghai/models/daniu_nmt_enc.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/centersnap/CenterSnap.onnx" + # model_file = "/compiler/huochenghai/GNNE/nncase/tests_output/daniu_enc.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/yolop/yolop_self.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/daniu/e2z/dec.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/daniu/z2e/enc.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/daniu/TTS/zho/fix.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/CRNN/ocr_rec_model_32-608.onnx" + # model_file = "/compiler/huochenghai/GNNE/nncase/tests_output/crnn_part.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/benchmark-test/mobile-facenet/mbface_sim_224.onnx" + # model_file = "/compiler/huochenghai/GNNE/k230-gnne-compiler-tests/FasterTransformer/LongFormer/longformer-base-4096.onnx" + # model_file = '/data/huochenghai/GNNE/k230-gnne-compiler-tests/StableDiffusion/onnx-stable-diffusion-v1-5/vae_decoder/model.onnx' + # model_file = "/data/huochenghai/llama_scrach/65B/decoder-merge-0.onnx" + # model_file = "/root/Downloads/decoder-merge-0.onnx" + model_file = "/root/Downloads/64B-4-layers/decoder-merge-all.onnx" + + # runner.set_shape_var({"batch_size": 1, "num_channels_latent": 4, "height_latent": 64, "width_latent": 64}) + runner.set_shape_var({"N": 384}) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main( + ['-vvs', __file__]) diff --git a/tests/importer/onnx_/model/test_text_encoder.py b/tests/importer/onnx_/model/test_text_encoder.py new file mode 100644 index 0000000000..8642743bcc --- /dev/null +++ b/tests/importer/onnx_/model/test_text_encoder.py @@ -0,0 +1,39 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""System test: test demo""" +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +# from lzma import MODE_FAST +# from xml.parsers.expat import model +import pytest +from onnx_test_runner import OnnxTestRunner + + +def test_demo(request): + # runner = OnnxTestRunner(request.node.name, "/root/Workspace/nncase/tests/importer/onnx_/model/llama_config.toml") + runner = OnnxTestRunner("text_encoder", "/root/Workspace/config/text_config.toml") + # runner = OnnxTestRunner("text_encoder") + # + model_file = "/root/Downloads/Models/text_encoder_model.onnx" + + # runner.set_shape_var({"batch_size": 1, "num_channels_latent": 4, "height_latent": 64, "width_latent": 64}) + # runner.set_shape_var({"N": 384}) + runner.set_shape_var({"batch_size": 1, "sequence_length": 77}) + # runner.set_shape_var({"batch_size:1", "sequence_length:77"}) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main( + ['-vvs', __file__]) diff --git a/tests/importer/onnx_/model/test_unet.py b/tests/importer/onnx_/model/test_unet.py new file mode 100644 index 0000000000..b504723bef --- /dev/null +++ b/tests/importer/onnx_/model/test_unet.py @@ -0,0 +1,40 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""System test: test demo""" +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +# from lzma import MODE_FAST +# from xml.parsers.expat import model +import pytest +from onnx_test_runner import OnnxTestRunner + + +def test_demo(request): + # runner = OnnxTestRunner(request.node.name, "/root/Workspace/nncase/tests/importer/onnx_/model/llama_config.toml") + runner = OnnxTestRunner("unet", "/root/Workspace/config/unet_config.toml") + # runner = OnnxTestRunner("unet") + # + model_file = "/root/Downloads/Models/unet/model.onnx" + + # runner.set_shape_var({"batch_size": 1, "num_channels_latent": 4, "height_latent": 64, "width_latent": 64}) + # runner.set_shape_var({"N": 384}) + runner.set_shape_var({"batch_size": 2, "num_channels": 4, "height": 64, + "width": 64, "steps": 2, "sequence_length": 77}) + # runner.set_shape_var({"batch_size:1", "sequence_length:77"}) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main( + ['-vvs', __file__]) diff --git a/tests/importer/onnx_/model/test_vae_decoder.py b/tests/importer/onnx_/model/test_vae_decoder.py new file mode 100644 index 0000000000..0c3eee5818 --- /dev/null +++ b/tests/importer/onnx_/model/test_vae_decoder.py @@ -0,0 +1,40 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""System test: test demo""" +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +# from lzma import MODE_FAST +# from xml.parsers.expat import model +import pytest +from onnx_test_runner import OnnxTestRunner + + +def test_demo(request): + runner = OnnxTestRunner("test_vae_decoder", + "/root/Workspace/config/vae_config.toml") + # runner = OnnxTestRunner("test_vae_decoder") + model_file = "/root/Downloads/Models/vae_decoder.onnx" + # model_file = "/root/Downloads/Models/modified_modified_vae_decoder.onnx" + # model_file = "/root/Downloads/Models/modified_vae_decoder.onnx" + # model_file = "/root/Downloads/Models/model_sim_huo.onnx" + + runner.set_shape_var({"batch_size": 1, "num_channels_latent": 4, + "height_latent": 64, "width_latent": 64}) + # runner.set_shape_var({"N": 384}) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main( + ['-vvs', __file__]) diff --git a/tests/kernels/test_concat.cpp b/tests/kernels/test_concat.cpp index c7da18b3bf..9aedb892f8 100644 --- a/tests/kernels/test_concat.cpp +++ b/tests/kernels/test_concat.cpp @@ -86,14 +86,7 @@ TEST_P(ConcatTest, Concat) { fields.push_back(field2); auto output_tuple = tuple(std::in_place, std::move(fields)); - int64_t axis_ptr[] = {axis_value}; - auto axis = - hrt::create(dt_int64, {1}, - {reinterpret_cast(axis_ptr), sizeof(axis_ptr)}, - true, host_runtime_tensor::pool_cpu_only) - .expect("create tensor failed"); - - auto output = kernels::stackvm::concat(output_tuple, axis.impl()) + auto output = kernels::stackvm::concat((int)axis_value, output_tuple) .expect("concat failed"); runtime_tensor actual(output.as().expect("as tensor failed")); diff --git a/tests/kernels/test_gather.cpp b/tests/kernels/test_gather.cpp index 5910d17cc1..65d5ca45c8 100644 --- a/tests/kernels/test_gather.cpp +++ b/tests/kernels/test_gather.cpp @@ -37,7 +37,7 @@ class GatherTest : public KernelTest, auto shape = GetShapeArray("lhs_shape"); auto indices_shape = GetShapeArray("indices_shape"); auto indices_value = GetDataArray("indices_value"); - auto value = GetNumber("axis"); + auto axis = GetNumber("axis"); auto typecode = GetDataType("lhs_type"); input = hrt::create(typecode, shape, host_runtime_tensor::pool_cpu_only) @@ -61,17 +61,9 @@ class GatherTest : public KernelTest, true, host_runtime_tensor::pool_cpu_only) .expect("create tensor failed"); - batchDims_value = value >= 0 - ? (size_t)value >= shape.size() ? -1 : value - : -(size_t)value > shape.size() ? -1 - : value; - - int64_t batchDims_array[1] = {batchDims_value}; - batchDims = hrt::create(dt_int64, dims_t{1}, - {reinterpret_cast(batchDims_array), - sizeof(batchDims_array)}, - true, host_runtime_tensor::pool_cpu_only) - .expect("create tensor failed"); + batchDims_value = axis >= 0 ? (size_t)axis >= shape.size() ? -1 : axis + : -(size_t)axis > shape.size() ? -1 + : axis; } void TearDown() override { CLEAR_SUBCASE() } @@ -79,7 +71,6 @@ class GatherTest : public KernelTest, protected: runtime_tensor input; runtime_tensor indices; - runtime_tensor batchDims; int64_t batchDims_value; }; @@ -103,7 +94,7 @@ TEST_P(GatherTest, gather) { // actual auto output = - kernels::stackvm::gather(input.impl(), batchDims.impl(), indices.impl()) + kernels::stackvm::gather(batchDims_value, input.impl(), indices.impl()) .expect("gather failed"); runtime_tensor actual(output.as().expect("as tensor failed")); diff --git a/tests/kernels/test_layer_norm.cpp b/tests/kernels/test_layer_norm.cpp index cc8a2696bd..5591bfe57d 100644 --- a/tests/kernels/test_layer_norm.cpp +++ b/tests/kernels/test_layer_norm.cpp @@ -106,8 +106,8 @@ TEST_P(LayerNormTest, layer_norm) { // actual auto output = - kernels::stackvm::layer_norm((int32_t)axis_value, eps, input.impl(), - scale.impl(), b.impl()) + kernels::stackvm::layer_norm((int32_t)axis_value, eps, false, + input.impl(), scale.impl(), b.impl()) .expect("layer_norm failed"); runtime_tensor actual(output.as().expect("as tensor failed")); diff --git a/tests/onnx_test_runner.py b/tests/onnx_test_runner.py index d1299bb234..dbde85f780 100644 --- a/tests/onnx_test_runner.py +++ b/tests/onnx_test_runner.py @@ -13,7 +13,7 @@ # limitations under the License. # pylint: disable=invalid-name, unused-argument, import-outside-toplevel -from onnx import version_converter, helper +from onnx import version_converter, helper, external_data_helper import onnxsim import onnxruntime as ort import onnx @@ -61,8 +61,16 @@ def run(self, model_file): if self.case_dir != os.path.dirname(model_file): new_file = os.path.join(self.case_dir, 'test.onnx') shutil.copy(model_file, new_file) - if os.path.exists(model_file + "_data"): - shutil.copy(model_file + "_data", self.case_dir) + for tensor in external_data_helper._get_all_tensors(onnx.load(model_file, load_external_data=False)): + if external_data_helper.uses_external_data(tensor): + info = external_data_helper.ExternalDataInfo(tensor) + file_location = external_data_helper._sanitize_path(info.location) + external_data_src_path = os.path.join( + os.path.dirname(model_file), file_location) + external_data_dst_path = os.path.join( + self.case_dir, file_location) + if not os.path.exists(external_data_dst_path): + os.symlink(external_data_src_path, external_data_dst_path) model_file = new_file if not self.inputs: @@ -176,7 +184,7 @@ def is_dynamic(output): outputs = onnx_model.graph.output self.dynamic = any(is_dynamic(output) for output in outputs) # make a static model for infer output - if self.dynamic: + if self.dynamic and onnx_model.ByteSize() < 2147483648: input_shapes = list(map(lambda input: {input['name']: input['shape']}, self.inputs)) input_shapes = dict(ChainMap(*input_shapes)) (onnx_model, _) = onnxsim.simplify(onnx_model, input_shapes=input_shapes) diff --git a/third_party/onnx/packages.lock.json b/third_party/onnx/packages.lock.json index 0bdbf312bd..207b93b556 100644 --- a/third_party/onnx/packages.lock.json +++ b/third_party/onnx/packages.lock.json @@ -16,17 +16,17 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" } } } diff --git a/third_party/tflite/packages.lock.json b/third_party/tflite/packages.lock.json index 73d7544eab..325adfa712 100644 --- a/third_party/tflite/packages.lock.json +++ b/third_party/tflite/packages.lock.json @@ -10,17 +10,17 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" } } } diff --git a/toolchains/k230_cpu.linux.toolchain.cmake b/toolchains/k230_cpu.linux.toolchain.cmake new file mode 100644 index 0000000000..4bd9aba679 --- /dev/null +++ b/toolchains/k230_cpu.linux.toolchain.cmake @@ -0,0 +1,33 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR riscv64) + +if(DEFINED ENV{RISCV_ROOT_PATH}) + file(TO_CMAKE_PATH $ENV{RISCV_ROOT_PATH} RISCV_ROOT_PATH) +endif() + +if(NOT RISCV_ROOT_PATH) + message(FATAL_ERROR "RISCV_ROOT_PATH env must be defined") +endif() + +set(RISCV_ROOT_PATH ${RISCV_ROOT_PATH} CACHE STRING "root path to riscv toolchain") +set(CMAKE_C_COMPILER "${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-musl-gcc") +set(CMAKE_CXX_COMPILER "${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-musl-g++") +set(CMAKE_FIND_ROOT_PATH "${RISCV_ROOT_PATH}/riscv64-unknown-linux-musl") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(ENABLE_VULKAN_RUNTIME OFF) +set(ENABLE_OPENMP OFF) +set(ENABLE_HALIDE OFF) +set(DEFAULT_BUILTIN_RUNTIMES OFF) +set(DEFAULT_SHARED_RUNTIME_TENSOR_PLATFORM_IMPL ON) +set(BUILD_BENCHMARK OFF) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=rv64imafdcv -mabi=lp64d -mcmodel=medany") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=rv64imafdcv -mabi=lp64d -mcmodel=medany") +set(CMAKE_EXE_LINKER_FLAGS "-T ${K230_LINUX_SDK_DIR}/src/big/rt-smart/userapps/linker_scripts/riscv64/link.lds --static") + +set(BUILDING_RUNTIME ON) +set(ENABLE_CPU_RUNTIME ON) +set(BUILD_SHARED_LIBS OFF) \ No newline at end of file diff --git a/tools/Nncase.SourceGenerator/Pattern/PatternGenerator.cs b/tools/Nncase.SourceGenerator/Pattern/PatternGenerator.cs index 7495791550..8e2aa7cfa2 100644 --- a/tools/Nncase.SourceGenerator/Pattern/PatternGenerator.cs +++ b/tools/Nncase.SourceGenerator/Pattern/PatternGenerator.cs @@ -85,7 +85,7 @@ select Parameter(Identifier(f.Name.ToLower())) // var x = name_params[0]; statements.Add(ParseStatement(@$"return new( new OpPattern<{cand.Op.ToDisplayString()}>(x => {condition}, {(name_params[0] != null ? "target_name" : "null")}), -new VArgsPattern (new[]{{ {inputs} }}, null), +new VArgsPattern (new Pattern[]{{ {inputs} }}, null), {(name_params[1] != null ? "call_name" : "null")});"). WithLeadingTrivia(ElasticTab). WithTrailingTrivia(ElasticLineFeed)); @@ -125,7 +125,7 @@ select Parameter(Identifier(f.Name.ToLower())) // 1.3 build method return statements.Add(ParseStatement(@$"return new( new OpPattern<{cand.Op.ToDisplayString()}>(condition, {(name_params[0] != null ? "target_name" : "null")}), -new VArgsPattern( new [] {{ {inputs} }}, null ), +new VArgsPattern( new Pattern[] {{ {inputs} }}, null ), {(name_params[1] != null ? "call_name" : "null")});"). WithLeadingTrivia(ElasticTab). WithTrailingTrivia(ElasticLineFeed)); diff --git a/tools/Nncase.SourceGenerator/Rule/RuleGenerator.cs b/tools/Nncase.SourceGenerator/Rule/RuleGenerator.cs index 57cf983aba..f2af3c5a2d 100644 --- a/tools/Nncase.SourceGenerator/Rule/RuleGenerator.cs +++ b/tools/Nncase.SourceGenerator/Rule/RuleGenerator.cs @@ -213,6 +213,7 @@ private void Execute(SourceProductionContext context, ImmutableArray(method)) .WithAttributeLists(new SyntaxList() { }) diff --git a/tools/Nncase.SourceGenerator/packages.lock.json b/tools/Nncase.SourceGenerator/packages.lock.json index 50bdccb9fa..0430a3e081 100644 --- a/tools/Nncase.SourceGenerator/packages.lock.json +++ b/tools/Nncase.SourceGenerator/packages.lock.json @@ -34,11 +34,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.CodeAnalysis.Common": { @@ -62,8 +62,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", diff --git a/tools/stackvm_gen/IsaGen/packages.lock.json b/tools/stackvm_gen/IsaGen/packages.lock.json index 6440c55973..fd04d9883e 100644 --- a/tools/stackvm_gen/IsaGen/packages.lock.json +++ b/tools/stackvm_gen/IsaGen/packages.lock.json @@ -27,11 +27,11 @@ }, "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Microsoft.AspNetCore.Mvc.Razor.Extensions": { @@ -169,8 +169,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -238,6 +238,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -312,6 +313,12 @@ "System.Runtime.CompilerServices.Unsafe": "5.0.0" } }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", From 381707215f7ec18734fb178e92b4df8d269dedb5 Mon Sep 17 00:00:00 2001 From: uranus0515 <110005227+uranus0515@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:55:53 +0800 Subject: [PATCH 12/13] shutdown FoldPrePostReshapeSoftmax transform (#1121) Co-authored-by: guodongliang --- src/Nncase.Compiler/Compiler.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Nncase.Compiler/Compiler.cs b/src/Nncase.Compiler/Compiler.cs index afd0607878..a25eb2bf48 100644 --- a/src/Nncase.Compiler/Compiler.cs +++ b/src/Nncase.Compiler/Compiler.cs @@ -127,7 +127,6 @@ public void TargetIndependentPass(IPassManager passManager) p.Add(); p.Add(); p.Add(); - p.Add(); p.Add(); p.Add(); p.Add(); From e39a9d23a86ea12d55ab13d6c02876c99c2fc10b Mon Sep 17 00:00:00 2001 From: uranus0515 <110005227+uranus0515@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:13:03 +0800 Subject: [PATCH 13/13] GNNE-1714:Fix/fix transform about quant (#1123) * support dim5 slice * add metadata for some transform --------- Co-authored-by: guodongliang --- src/Native/src/kernels/stackvm/optimized/slice.cpp | 3 +++ src/Nncase.Importer/Onnx/Binary.cs | 4 ++-- src/Nncase.Importer/Onnx/Split.cs | 4 ++-- src/Nncase.Importer/Onnx/Transpose.cs | 2 +- src/Nncase.Passes/Rules/Neutral/SqueezeShape.cs | 14 +++++++++----- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Native/src/kernels/stackvm/optimized/slice.cpp b/src/Native/src/kernels/stackvm/optimized/slice.cpp index b56a16fad6..7899c426b3 100644 --- a/src/Native/src/kernels/stackvm/optimized/slice.cpp +++ b/src/Native/src/kernels/stackvm/optimized/slice.cpp @@ -85,6 +85,9 @@ result slice_contiguous_impl( } else if (dims == 3) { _slice_contiguous_dim_copy<3>(begins, ends, line_copy, in_index, std::true_type{}); + } else if (dims == 4) { + _slice_contiguous_dim_copy<4>(begins, ends, line_copy, in_index, + std::true_type{}); } else { assert(false); } diff --git a/src/Nncase.Importer/Onnx/Binary.cs b/src/Nncase.Importer/Onnx/Binary.cs index 71f48c67f3..9426f18840 100644 --- a/src/Nncase.Importer/Onnx/Binary.cs +++ b/src/Nncase.Importer/Onnx/Binary.cs @@ -15,10 +15,10 @@ private Expr VisitBinary(NodeProto op, BinaryOp binaryOp) var (lhs, rhs) = GetInputExprs(op, 0, 1); if (binaryOp == BinaryOp.Pow && lhs.CheckedDataType != rhs.CheckedDataType) { - return F.Math.Binary(binaryOp, lhs, IR.F.Tensors.Cast(rhs, lhs.CheckedDataType)); + return F.Math.Binary(binaryOp, lhs, IR.F.Tensors.Cast(rhs, lhs.CheckedDataType)).With(metadata: new IRMetadata() { OutputNames = op.Output }); } - return F.Math.Binary(binaryOp, lhs, rhs); + return F.Math.Binary(binaryOp, lhs, rhs).With(metadata: new IRMetadata() { OutputNames = op.Output }); } } } diff --git a/src/Nncase.Importer/Onnx/Split.cs b/src/Nncase.Importer/Onnx/Split.cs index 5dee4c2ddf..d6526f969c 100644 --- a/src/Nncase.Importer/Onnx/Split.cs +++ b/src/Nncase.Importer/Onnx/Split.cs @@ -28,7 +28,7 @@ private Expr SplitV11(in NodeProto op) var split = GetOptionIntsAttribute(op, "split") .Map(x => (Expr)Tensor.From(x)) .Or(ComputeSplit(input, op.Output.Count, axis)); - return F.Tensors.Split(input, axis, split); + return F.Tensors.Split(input, axis, split).With(metadata: new IRMetadata() { OutputNames = op.Output, }); } private Expr SplitV13(in NodeProto op) @@ -37,7 +37,7 @@ private Expr SplitV13(in NodeProto op) var axis = GetIntAttribute(op, "axis", 0); var split = GetOptionInputExpr(op, 1) .Or(ComputeSplit(input, op.Output.Count, axis)); - return F.Tensors.Split(input, axis, split); + return F.Tensors.Split(input, axis, split).With(metadata: new IRMetadata() { OutputNames = op.Output, }); } } } diff --git a/src/Nncase.Importer/Onnx/Transpose.cs b/src/Nncase.Importer/Onnx/Transpose.cs index bf86c4b9cd..8bbf23e5be 100644 --- a/src/Nncase.Importer/Onnx/Transpose.cs +++ b/src/Nncase.Importer/Onnx/Transpose.cs @@ -17,7 +17,7 @@ private Expr VisitTranspose(NodeProto op) { var input = GetSingleInputExpr(op); var perm = Tensor.From(GetIntsAttribute(op, "perm")); - return F.Tensors.Transpose(input, perm); + return F.Tensors.Transpose(input, perm).With(metadata: new IRMetadata() { OutputNames = op.Output, }); } } } diff --git a/src/Nncase.Passes/Rules/Neutral/SqueezeShape.cs b/src/Nncase.Passes/Rules/Neutral/SqueezeShape.cs index b4885bcc8b..289e5baacb 100644 --- a/src/Nncase.Passes/Rules/Neutral/SqueezeShape.cs +++ b/src/Nncase.Passes/Rules/Neutral/SqueezeShape.cs @@ -167,7 +167,7 @@ public sealed partial class Squeeze5DTranspose : IRewriteRule throw new NotSupportedException("Not Supported perm!"); } - return Reshape(Transpose(Reshape(tp1, shape3), perm2), call.CheckedShape); + return Reshape(Transpose(Reshape(tp1, shape3), perm2).With(metadata: call.Metadata), call.CheckedShape); } } @@ -175,7 +175,11 @@ public sealed partial class Squeeze5DTranspose : IRewriteRule public sealed partial class SqueezeTransposeShape : IRewriteRule { /// - public IPattern Pattern { get; } = IsTranspose(IsWildcard("input") with { TypePattern = HasFixedShape() & HasRank(x => x > 4, "more than 4D need to squeeze") }, IsWildcard("perm")); + public IPattern Pattern { get; } = IsTranspose( + "transpose", + "call", + IsWildcard("input") with { TypePattern = HasFixedShape() & HasRank(x => x > 4, "more than 4D need to squeeze") }, + IsWildcard("perm")); private Tuple, List> SqueezeTranspose(List oldShape, List oldAxis) { @@ -228,7 +232,7 @@ private Tuple, List> SqueezeTranspose(List oldShape, L return new Tuple, List>(true, newAxis, newShape); } - private Expr? GetReplace(Expr input, int[] perm) + private Expr? GetReplace(Expr input, int[] perm, Expr call) { var inputShape = input.CheckedShape; var (result, new_perm, new_shape) = SqueezeTranspose(inputShape.ToValueList(), perm.ToList()); @@ -243,7 +247,7 @@ private Tuple, List> SqueezeTranspose(List oldShape, L newOutputShape[i] = inputShape[perm[i]].FixedValue; } - return Reshape(Transpose(Reshape(input, new_shape.ToArray()), new_perm.ToArray()), newOutputShape); + return Reshape(Transpose(Reshape(input, new_shape.ToArray()), new_perm.ToArray()).With(metadata: call.Metadata), newOutputShape); } } @@ -398,6 +402,6 @@ private static List GetOutputShape(List a, List b) var outputShape = GetOutputShape(lShape.ToValueList(), rShape.ToValueList()); - return Reshape(Binary(binary.BinaryOp, Reshape(lhs, newLShape.ToArray()), Reshape(rhs, newRShape.ToArray())), outputShape.ToArray()); + return Reshape(Binary(binary.BinaryOp, Reshape(lhs, newLShape.ToArray()), Reshape(rhs, newRShape.ToArray())).With(metadata: binaryCall.Metadata), outputShape.ToArray()); } }