From fb4a0adb1934b1e48b35fc71a11b9fd1dfabc6ce Mon Sep 17 00:00:00 2001 From: Hyeongseok Oh Date: Tue, 30 Jul 2024 17:15:54 +0900 Subject: [PATCH] [onert] Remove layout in graph This commit removes layout field in Graph and TrainableGraph. ONE-DCO-1.0-Signed-off-by: Hyeongseok Oh --- .../backend/basic/BackendContextHelpers.h | 2 -- .../train/TrainableBackendContextHelpers.h | 2 -- runtime/onert/core/include/ir/Graph.h | 3 --- .../core/include/ir/train/TrainableGraph.h | 1 - .../onert/core/include/util/ShapeInference.h | 10 +++----- .../core/src/compiler/ExecutorFactory.cc | 1 - .../onert/core/src/compiler/LoweredGraph.cc | 2 +- .../onert/core/src/compiler/ShapeValidator.cc | 25 ++++++++----------- .../core/src/compiler/StaticShapeInferer.cc | 4 +-- .../compiler/pass/PermutationOperationPass.cc | 4 +-- .../compiler/train/LoweredTrainableGraph.cc | 2 +- runtime/onert/core/src/util/ShapeInference.cc | 20 ++++++--------- 12 files changed, 26 insertions(+), 50 deletions(-) diff --git a/runtime/onert/core/include/backend/basic/BackendContextHelpers.h b/runtime/onert/core/include/backend/basic/BackendContextHelpers.h index cfbcdf520a0..590ae0b9919 100644 --- a/runtime/onert/core/include/backend/basic/BackendContextHelpers.h +++ b/runtime/onert/core/include/backend/basic/BackendContextHelpers.h @@ -190,8 +190,6 @@ template ITensorRegistry *genTensors(T_BackendContex graph.operands().iterate([&](const ir::OperandIndex &ind, const ir::Operand &obj) { if (ctx.external_operands().contains(ind)) return; - // NOTE Assuming there is no layout changes (Always assume NHWC or UNKNOWN) - assert(graph.layout() != ir::Layout::NCHW); tensor_builder->registerTensorInfo(ind, obj.info()); }); diff --git a/runtime/onert/core/include/backend/basic/train/TrainableBackendContextHelpers.h b/runtime/onert/core/include/backend/basic/train/TrainableBackendContextHelpers.h index 27eddce5687..b4f2f7a31de 100644 --- a/runtime/onert/core/include/backend/basic/train/TrainableBackendContextHelpers.h +++ b/runtime/onert/core/include/backend/basic/train/TrainableBackendContextHelpers.h @@ -39,8 +39,6 @@ ITensorRegistry *genTensors(backend::train::TrainableBackendContext &ctx, tgraph.operands().iterate([&](const ir::OperandIndex &ind, const ir::Operand &obj) { if (ctx.external_operands().contains(ind)) return; - // NOTE Assuming there is no layout changes (Always assume NHWC or UNKNOWN) - assert(tgraph.layout() != ir::Layout::NCHW); tensor_builder->registerTensorInfo(ind, obj.info()); }); diff --git a/runtime/onert/core/include/ir/Graph.h b/runtime/onert/core/include/ir/Graph.h index 7eb5d87d7e4..27b85093230 100644 --- a/runtime/onert/core/include/ir/Graph.h +++ b/runtime/onert/core/include/ir/Graph.h @@ -113,7 +113,6 @@ class Graph : public IGraph Operands &operands() { return _operands; } // TODO Remove this non-const accessor const Operations &operations() const override { return _operations; } Operations &operations() { return _operations; } - Layout layout() const { return _layout; } // Topological sort public: @@ -126,8 +125,6 @@ class Graph : public IGraph OperandIndexSequence _outputs; std::unordered_map _name_to_input; std::unordered_map _name_to_output; - // TFLite and circle's default layout is NHWC; - Layout _layout{Layout::NHWC}; }; } // namespace ir diff --git a/runtime/onert/core/include/ir/train/TrainableGraph.h b/runtime/onert/core/include/ir/train/TrainableGraph.h index 66c76fa853d..4952a6b513f 100644 --- a/runtime/onert/core/include/ir/train/TrainableGraph.h +++ b/runtime/onert/core/include/ir/train/TrainableGraph.h @@ -125,7 +125,6 @@ class TrainableGraph : public IGraph const Operations &operations() const override { return _graph.operations(); } const Operands &backward_operands() const { return _backward_operands; } OperandIndex getLossIndex(const IOIndex &pred_io_ind) const; - Layout layout() const { return _graph.layout(); } const Graph &graph() const { return _graph; } public: diff --git a/runtime/onert/core/include/util/ShapeInference.h b/runtime/onert/core/include/util/ShapeInference.h index 8f1aa98a243..5179f8eb4d1 100644 --- a/runtime/onert/core/include/util/ShapeInference.h +++ b/runtime/onert/core/include/util/ShapeInference.h @@ -27,7 +27,6 @@ #include "ir/operation/StridedSlice.h" #include "compiler/LoweredGraph.h" #include "ir/Index.h" -#include "ir/Layout.h" #include "ir/OperationVisitor.h" #include "backend/ITensor.h" #include "backend/ITensorRegistry.h" @@ -58,12 +57,10 @@ ir::Shape inferBroadcastToShape(const ir::Shape shp_shape, const int32_t *shp_bu ir::Shape inferConcatShape(const Shapes &in_shapes, const ir::operation::Concat::Param ¶m); ir::Shape inferConv2DShape(const ir::Shape &in_shape, const ir::Shape &ker_shape, - const ir::operation::Conv2D::Param ¶m, - ir::Layout layout = ir::Layout::NHWC); + const ir::operation::Conv2D::Param ¶m); ir::Shape inferDepthwiseConv2DShape(const ir::Shape &in_shape, const ir::Shape &ker_shape, - const ir::operation::DepthwiseConv2D::Param ¶m, - ir::Layout layout = ir::Layout::NHWC); + const ir::operation::DepthwiseConv2D::Param ¶m); ir::Shape inferEltwiseShape(const ir::Shape &lhs_shape, const ir::Shape &rhs_shape); @@ -82,8 +79,7 @@ ir::Shape inferPackShape(const ir::Shape &input_shape, int axis, int rank, int n ir::Shape inferPadShape(const ir::Shape &in_shape, const int32_t *pad_buf, const size_t num_pads); -ir::Shape inferPoolShape(const ir::Shape &in_shape, const ir::operation::Pool2D::Param ¶m, - ir::Layout layout = ir::Layout::NHWC); +ir::Shape inferPoolShape(const ir::Shape &in_shape, const ir::operation::Pool2D::Param ¶m); template ir::Shape inferRangeShape(T start_val, T limit_val, T delta_val); diff --git a/runtime/onert/core/src/compiler/ExecutorFactory.cc b/runtime/onert/core/src/compiler/ExecutorFactory.cc index 7ddc61ac4ed..82ef1054c56 100644 --- a/runtime/onert/core/src/compiler/ExecutorFactory.cc +++ b/runtime/onert/core/src/compiler/ExecutorFactory.cc @@ -111,7 +111,6 @@ void initializeSubgraphIOTensors(compiler::ILoweredGraph &lowered_graph, for (auto &&ind : indices) { const auto &operand = lowered_graph.graph().operands().at(ind); - assert(lowered_graph.graph().layout() == ir::Layout::NHWC); auto tensor = std::make_unique(operand.info()); // Add tensor to builtin TensorRegistry. diff --git a/runtime/onert/core/src/compiler/LoweredGraph.cc b/runtime/onert/core/src/compiler/LoweredGraph.cc index 8789d6e9a57..70dae2dd508 100644 --- a/runtime/onert/core/src/compiler/LoweredGraph.cc +++ b/runtime/onert/core/src/compiler/LoweredGraph.cc @@ -148,7 +148,7 @@ void LoweredGraph::makeLowerInfo(const compiler::BackendResolver &backend_resolv // Handle graph inputs and outputs const auto builtin_backend = BackendManager::get().getBuiltin(); - auto factor = PermuteFactor{builtin_backend, _graph.layout()}; + auto factor = PermuteFactor{builtin_backend, ir::Layout::NHWC}; for (auto &&index : _graph.getInputs() | ir::Remove::UNDEFINED) { auto &operand_li = lower_info().operand.at(index); diff --git a/runtime/onert/core/src/compiler/ShapeValidator.cc b/runtime/onert/core/src/compiler/ShapeValidator.cc index 0cd14c18607..876480e27f6 100644 --- a/runtime/onert/core/src/compiler/ShapeValidator.cc +++ b/runtime/onert/core/src/compiler/ShapeValidator.cc @@ -82,9 +82,8 @@ void ShapeValidator::visit(const ir::operation::BatchToSpaceND &node) const auto block_size_index{ node.getInputs().at(ir::operation::BatchToSpaceND::Input::BLOCK_SIZE)}; - const auto frontend_layout = _graph.layout(); - const auto input_shape = operands.at(ifm_index).shape().asFeature(frontend_layout); - const auto output_shape = operands.at(ofm_index).shape().asFeature(frontend_layout); + const auto input_shape = operands.at(ifm_index).shape().asFeature(ir::Layout::NHWC); + const auto output_shape = operands.at(ofm_index).shape().asFeature(ir::Layout::NHWC); // All requirement as per NNAPI specification. OP_REQUIRES(operands.at(ifm_index).shape().rank() == 4); @@ -392,9 +391,8 @@ void ShapeValidator::visit(const ir::operation::SpaceToBatchND &node) node.getInputs().at(ir::operation::SpaceToBatchND::Input::BLOCK_SIZE)}; const auto paddings_index{node.getInputs().at(ir::operation::SpaceToBatchND::Input::PADDINGS)}; - const auto frontend_layout = _graph.layout(); - const auto input_shape = operands.at(ifm_index).shape().asFeature(frontend_layout); - const auto output_shape = operands.at(ofm_index).shape().asFeature(frontend_layout); + const auto input_shape = operands.at(ifm_index).shape().asFeature(ir::Layout::NHWC); + const auto output_shape = operands.at(ofm_index).shape().asFeature(ir::Layout::NHWC); // All requirement as per NNAPI specification. OP_REQUIRES(operands.at(ifm_index).shape().rank() == 4); @@ -418,9 +416,8 @@ void ShapeValidator::visit(const ir::operation::SpaceToDepth &node) const auto ifm_index{node.getInputs().at(ir::operation::SpaceToDepth::Input::INPUT)}; - const auto frontend_layout = _graph.layout(); - const auto input_shape = operands.at(ifm_index).shape().asFeature(frontend_layout); - const auto output_shape = operands.at(ofm_index).shape().asFeature(frontend_layout); + const auto input_shape = operands.at(ifm_index).shape().asFeature(ir::Layout::NHWC); + const auto output_shape = operands.at(ofm_index).shape().asFeature(ir::Layout::NHWC); const auto block_size = node.param().block_size; // All assertions as per NNAPI specification. @@ -540,9 +537,8 @@ void ShapeValidator::visit(const ir::operation::TransposeConv &node) OP_REQUIRES(operands.at(ofm_index).shape().rank() == operands.at(ifm_index).shape().rank()); OP_REQUIRES(operands.at(ofm_index).shape().rank() == operands.at(ker_index).shape().rank()); - const auto frontend_layout = _graph.layout(); - const auto ofm_shape = operands.at(ofm_index).shape().asFeature(frontend_layout); - const auto ifm_shape = operands.at(ifm_index).shape().asFeature(frontend_layout); + const auto ofm_shape = operands.at(ofm_index).shape().asFeature(ir::Layout::NHWC); + const auto ifm_shape = operands.at(ifm_index).shape().asFeature(ir::Layout::NHWC); // The kernel has only IHWO layout on frontend // So ker_shape is treated here below // I -> N @@ -587,9 +583,8 @@ void ShapeValidator::visit(const ir::operation::DepthToSpace &node) const auto input_index{node.getInputs().at(ir::operation::DepthToSpace::Input::INPUT)}; - const auto frontend_layout = _graph.layout(); - const auto output_shape = operands.at(output_index).shape().asFeature(frontend_layout); - const auto input_shape = operands.at(input_index).shape().asFeature(frontend_layout); + const auto output_shape = operands.at(output_index).shape().asFeature(ir::Layout::NHWC); + const auto input_shape = operands.at(input_index).shape().asFeature(ir::Layout::NHWC); OP_REQUIRES(operands.at(input_index).shape().rank() == 4); OP_REQUIRES(operands.at(output_index).shape().rank() == 4); diff --git a/runtime/onert/core/src/compiler/StaticShapeInferer.cc b/runtime/onert/core/src/compiler/StaticShapeInferer.cc index 97d4dea6d52..f63cdfb7c11 100644 --- a/runtime/onert/core/src/compiler/StaticShapeInferer.cc +++ b/runtime/onert/core/src/compiler/StaticShapeInferer.cc @@ -851,8 +851,6 @@ void StaticShapeInferer::visit(const ir::operation::Pool2D &op) { auto &operands = _lowered_subg->graph().operands(); - const auto layout = _lowered_subg->graph().layout(); - const auto input_idx{op.getInputs().at(ir::operation::Pool2D::Input::INPUT)}; const auto &input = operands.at(input_idx); if (input.info().shape().rank() != 4) @@ -863,7 +861,7 @@ void StaticShapeInferer::visit(const ir::operation::Pool2D &op) const auto output_idx = op.getOutputs().at(0); ir::Operand &output = operands.at(output_idx); - ir::Shape new_shape = shape_inference::inferPoolShape(input.info().shape(), op.param(), layout); + ir::Shape new_shape = shape_inference::inferPoolShape(input.info().shape(), op.param()); output.info().shape(new_shape); } diff --git a/runtime/onert/core/src/compiler/pass/PermutationOperationPass.cc b/runtime/onert/core/src/compiler/pass/PermutationOperationPass.cc index f014d29d398..ac2151795c0 100644 --- a/runtime/onert/core/src/compiler/pass/PermutationOperationPass.cc +++ b/runtime/onert/core/src/compiler/pass/PermutationOperationPass.cc @@ -43,7 +43,7 @@ void PermutationOperationPass::applyExpandRanks(const Operation &node) assert(output.getDef().valid()); const auto node_index = output.getDef(); - const auto frontend_layout = _graph.layout(); + const auto frontend_layout = ir::Layout::NHWC; const auto backend_layout = _lowered_graph.lower_info().operation.getRawPtr(node_index)->layout(); if (frontend_layout == backend_layout) @@ -86,7 +86,7 @@ void PermutationOperationPass::changeToKeepLayout(const Operation &node) auto &operation_li_map = _lowered_graph.lower_info().operation; auto &operand_li_map = _lowered_graph.lower_info().operand; - const auto frontend_layout = _graph.layout(); + const auto frontend_layout = ir::Layout::NHWC; const auto backend_layout = operation_li_map.getRawPtr(node_index)->layout(); if (frontend_layout == backend_layout) diff --git a/runtime/onert/core/src/compiler/train/LoweredTrainableGraph.cc b/runtime/onert/core/src/compiler/train/LoweredTrainableGraph.cc index 458bd0ecf9f..d745627c6d2 100644 --- a/runtime/onert/core/src/compiler/train/LoweredTrainableGraph.cc +++ b/runtime/onert/core/src/compiler/train/LoweredTrainableGraph.cc @@ -178,7 +178,7 @@ void LoweredTrainableGraph::makeLowerInfo(const compiler::BackendResolver &backe // Handle graph inputs and outputs const auto builtin_backend = BackendManager::get().getBuiltin(); - auto factor = PermuteFactor{builtin_backend, _trainable_graph.layout()}; + auto factor = PermuteFactor{builtin_backend, ir::Layout::NHWC}; for (auto &&index : _trainable_graph.getInputs() | ir::Remove::UNDEFINED) { auto &operand_li = lower_info().operand.at(index); diff --git a/runtime/onert/core/src/util/ShapeInference.cc b/runtime/onert/core/src/util/ShapeInference.cc index 8b6d7597bcb..9259458c78e 100644 --- a/runtime/onert/core/src/util/ShapeInference.cc +++ b/runtime/onert/core/src/util/ShapeInference.cc @@ -323,15 +323,15 @@ ir::Shape inferConcatShape(const Shapes &in_shapes, const ir::operation::Concat: } ir::Shape inferConv2DShape(const ir::Shape &in_shape, const ir::Shape &ker_shape, - const ir::operation::Conv2D::Param ¶m, ir::Layout layout) + const ir::operation::Conv2D::Param ¶m) { if (param.stride.horizontal == 0 || param.stride.vertical == 0) throw std::runtime_error{"Conv2D: stride values must be positive"}; - auto ifm_shape = in_shape.asFeature(layout); + auto ifm_shape = in_shape.asFeature(ir::Layout::NHWC); // Kernel format is [depth_out, kernel_height, kernel_width, depth_in] - auto kf_shape = ker_shape.asFeature(layout); + auto kf_shape = ker_shape.asFeature(ir::Layout::NHWC); assert(ifm_shape.C == kf_shape.C); const auto [out_h, out_w] = calcConvLikeHeightAndWidth( @@ -341,17 +341,15 @@ ir::Shape inferConv2DShape(const ir::Shape &in_shape, const ir::Shape &ker_shape } ir::Shape inferDepthwiseConv2DShape(const ir::Shape &in_shape, const ir::Shape &ker_shape, - const ir::operation::DepthwiseConv2D::Param ¶m, - ir::Layout layout) + const ir::operation::DepthwiseConv2D::Param ¶m) { if (param.stride.horizontal == 0 || param.stride.vertical == 0) throw std::runtime_error{"DepthwiseConv2D: stride values must be positive"}; - assert(layout == ir::Layout::NHWC); - auto ifm_shape = in_shape.asFeature(layout); + auto ifm_shape = in_shape.asFeature(ir::Layout::NHWC); // Kernel format is [1, kernel_height, kernel_width, depth_out] - auto kf_shape = ker_shape.asFeature(layout); + auto kf_shape = ker_shape.asFeature(ir::Layout::NHWC); assert(kf_shape.C == static_cast(ifm_shape.C * param.multiplier)); assert(kf_shape.N == 1); @@ -546,14 +544,12 @@ ir::Shape inferPadShape(const ir::Shape &in_shape, const int32_t *pad_buf, const return ret; } -ir::Shape inferPoolShape(const ir::Shape &in_shape, const ir::operation::Pool2D::Param ¶m, - const ir::Layout layout) +ir::Shape inferPoolShape(const ir::Shape &in_shape, const ir::operation::Pool2D::Param ¶m) { if (param.stride.horizontal == 0 || param.stride.vertical == 0) throw std::runtime_error{"Pool2D: stride values must be positive"}; - assert(layout == ir::Layout::NHWC); - auto ifm_shape = in_shape.asFeature(layout); + auto ifm_shape = in_shape.asFeature(ir::Layout::NHWC); const auto [out_h, out_w] = calcConvLikeHeightAndWidth(ifm_shape.H, ifm_shape.W, param.kh, param.kw, param.padding, param.stride); // Pooling don't change number of channels and batch size