DRAFT experiment with LayerNorm to InstanceNorm

on-going draft to experiment with LayerNorm to InstanceNorm.

Signed-off-by: SaeHie Park <[email protected]>

compiler/circle2circle-dredd-recipe-test/test.lst

@@ -63,6 +63,7 @@ Add(Net_InstanceNorm_005 PASS fuse_instnorm)
 Add(Net_InstanceNorm_006 PASS fuse_instnorm)
 Add(Net_InstanceNorm_007 PASS fuse_instnorm)
 Add(Net_InstanceNorm_008 PASS fuse_instnorm)
+Add(Net_LayerNorm_000 PASS fuse_layernorm_to_instnorm)
 Add(Net_Maximum_Minimum_000 PASS transform_min_max_to_relu6)
 Add(Net_Mul_Add_000 PASS remove_unnecessary_add)
 Add(Net_Mul_Add_001 PASS remove_unnecessary_add)

compiler/circle2circle/src/Circle2Circle.cpp

@@ -109,6 +109,8 @@ int entry(int argc, char **argv)
              "This will fuse BatchNorm operators to Transposed Convolution operator");
   add_switch(arser, "--fuse_bcq", "This will fuse operators and apply Binary Coded Quantization");
   add_switch(arser, "--fuse_instnorm", "This will fuse operators to InstanceNorm operator");
+  add_switch(arser, "--fuse_layernorm_to_instnorm",
+             "This will fuse decomposed LayerNorm operators to InstanceNorm operator");
   add_switch(arser, "--fuse_mean_with_mean",
              "This will fuse two Mean operations when they follow one by one. This will fold them "
              "into one operation and merge reduction indices.");
@@ -297,6 +299,7 @@ int entry(int argc, char **argv)
   option_str_to_enum["fuse_slice_with_tconv"] = Algorithms::FuseSliceWithTConv;
   option_str_to_enum["fuse_bcq"] = Algorithms::FuseBCQ;
   option_str_to_enum["fuse_instnorm"] = Algorithms::FuseInstanceNorm;
+  option_str_to_enum["fuse_layernorm_to_instnorm"] = Algorithms::FuseLayerNormToInstNorm;
   option_str_to_enum["fuse_mean_with_mean"] = Algorithms::FuseMeanWithMean;
   option_str_to_enum["fuse_mul_with_conv"] = Algorithms::FuseMulWithConv;
   option_str_to_enum["fuse_mul_with_div"] = Algorithms::FuseMulWithDiv;

compiler/luci-pass-value-py-test/test.lst

@@ -42,6 +42,7 @@ eval(Net_InstanceNorm_001 fuse_instnorm)
 eval(Net_InstanceNorm_002 fuse_instnorm)
 eval(Net_InstanceNorm_003 fuse_instnorm)
 eval(Net_InstanceNorm_008 fuse_instnorm)
+eval(Net_LayerNorm_000 fuse_layernorm_to_instnorm)
 eval(Net_Mul_Add_000 remove_unnecessary_add)
 eval(Net_Mul_Add_001 remove_unnecessary_add)
 eval(Net_Mul_Add_002 remove_unnecessary_add)

compiler/luci/lang/include/luci/IR/CircleNodes.h

@@ -146,6 +146,7 @@
 // Virtual nodes
 #include "Nodes/CircleConst.h"
 #include "Nodes/CircleInput.h"
+#include "Nodes/CircleLayerNorm.h"
 #include "Nodes/CircleOutput.h"
 #include "Nodes/CircleVariable.h"
 // Multi-output virtual nodes

compiler/luci/lang/include/luci/IR/CircleNodes.lst

@@ -144,6 +144,7 @@ CIRCLE_NODE(ROPE, CircleRoPE)
 // Virtual node(s)
 CIRCLE_VNODE(CIRCLECONST, CircleConst)
 CIRCLE_VNODE(CIRCLEINPUT, CircleInput)
+CIRCLE_VNODE(CIRCLELAYERNORM, CircleLayerNorm)
 CIRCLE_VNODE(CIRCLEOUTPUT, CircleOutput)
 CIRCLE_VNODE(CIRCLEOUTPUTDUMMY, CircleOutputDummy)
 CIRCLE_VNODE(CIRCLEOUTPUTEXCLUDE, CircleOutputExclude)

compiler/luci/lang/include/luci/IR/Nodes/CircleLayerNorm.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LUCI_IR_CIRCLE_LAYER_NORM_H__
+#define __LUCI_IR_CIRCLE_LAYER_NORM_H__
+
+#include "luci/IR/CircleNodeDecl.h"
+#include "luci/IR/CircleOpcode.h"
+
+#include "luci/IR/AttrFusedActFunc.h"
+#include "luci/IR/CircleNodeMixins.h"
+
+namespace luci
+{
+
+/**
+ * @brief Virtual LayerNorm
+ */
+class CircleLayerNorm final
+  : public FixedArityNode<3, CircleNodeImpl<CircleOpcode::CIRCLELAYERNORM>>
+{
+public:
+  loco::Node *input(void) const { return at(0)->node(); }
+  void input(loco::Node *node) { at(0)->node(node); }
+
+  loco::Node *gamma(void) const { return at(1)->node(); }
+  void gamma(loco::Node *node) { at(1)->node(node); }
+
+  loco::Node *beta(void) const { return at(2)->node(); }
+  void beta(loco::Node *node) { at(2)->node(node); }
+
+public:
+  float epsilon() const { return _epsilon; }
+  void epsilon(float epsilon) { _epsilon = epsilon; }
+
+private:
+  float _epsilon{1e-05};
+};
+
+} // namespace luci
+
+#endif // __LUCI_IR_CIRCLE_LAYER_NORM_H__

compiler/luci/logex/include/luci/CircleNodeSummaryBuilders.h

@@ -870,6 +870,13 @@ class CircleWhileOutSummaryBuilder final : public CircleNodeSummaryBuilder
   std::vector<std::string> get_input_names(const luci::CircleNode *);
 };
 
+class CircleLayerNormSummaryBuilder final : public CircleNodeSummaryBuilder
+{
+private:
+  std::vector<std::string> get_input_names(const luci::CircleNode *);
+  void build_attributes(const luci::CircleNode *node, locop::NodeSummary &s);
+};
+
 } // namespace luci
 
 #endif // __LUCI_LOGEX_CIRCLE_NODE_SUMMARY_BUILDERS__

compiler/luci/logex/src/CircleNodeSummaryBuilder.cpp

@@ -265,6 +265,7 @@ CircleNodeSummaryBuilder::create_builder(const luci::CircleNode *node)
     CIRCLE_NODE(CIRCLECUSTOMOUT, CircleCustomOutSummaryBuilder)
     CIRCLE_NODE(CIRCLEIFOUT, CircleIfOutSummaryBuilder)
     CIRCLE_NODE(CIRCLEINPUT, CircleInputSummaryBuilder)
+    CIRCLE_NODE(CIRCLELAYERNORM, CircleLayerNormSummaryBuilder)
     CIRCLE_NODE(CIRCLENONMAXSUPPRESSIONV4OUT, CircleNonMaxSuppressionV4OutSummaryBuilder)
     CIRCLE_NODE(CIRCLENONMAXSUPPRESSIONV5OUT, CircleNonMaxSuppressionV5OutSummaryBuilder)
     CIRCLE_NODE(CIRCLEOUTPUT, CircleOutputSummaryBuilder)

compiler/luci/logex/src/CircleNodeSummaryBuilders.cpp

@@ -1220,6 +1220,18 @@ void CircleWhileSummaryBuilder::build_attributes(const luci::CircleNode *node,
     s.args().append("else_branch", std::to_string(circle_while->body_branch()));
 }
 
+std::vector<std::string> CircleLayerNormSummaryBuilder::get_input_names(const luci::CircleNode *)
+{
+  return {"input", "gamma", "beta"};
+}
+
+void CircleLayerNormSummaryBuilder::build_attributes(const luci::CircleNode *node,
+                                                     locop::NodeSummary &s)
+{
+  auto layernorm = loco::must_cast<const luci::CircleLayerNorm *>(node);
+  s.args().append("epsilon", std::to_string(layernorm->epsilon()));
+}
+
 std::vector<std::string> CircleOutputSummaryBuilder::get_input_names(const luci::CircleNode *)
 {
   return {"from"};

compiler/luci/pass/include/luci/CircleOptimizer.h

@@ -46,6 +46,7 @@ class CircleOptimizer final
       FuseBCQ,
       FuseHorizontalFullyConnected,
       FuseInstanceNorm,
+      FuseLayerNormToInstNorm,
       FuseMeanWithMean,
       FuseMulWithConv,
       FuseMulWithDiv,

compiler/luci/pass/include/luci/Pass/FuseLayerNormToInstNormPass.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2025 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LUCI_FUSE_LAYER_NORM_TO_INST_NORM_PASS_H__
+#define __LUCI_FUSE_LAYER_NORM_TO_INST_NORM_PASS_H__
+
+#include <logo/Pass.h>
+
+namespace luci
+{
+
+/**
+ * @brief  Class to fuse certain pattern of subgraph into CircleLayerNorm
+ *         and then to CircleInstanceNorm with auxiliary nodes
+ *
+ * For detailed subgraph pattern to be fused, please check its implementation.
+ */
+struct FuseLayerNormToInstNormPass final : public logo::Pass
+{
+  const char *name(void) const final { return "luci::FuseLayerNormToInstNormPass"; }
+
+  bool run(loco::Graph *g) final;
+};
+
+} // namespace luci
+
+#endif // __LUCI_FUSE_LAYER_NORM_TO_INST_NORM_PASS_H__

compiler/luci/pass/src/CircleOptimizer.cpp

@@ -45,6 +45,7 @@
 #include "luci/Pass/FuseBCQPass.h"
 #include "luci/Pass/FuseMulToFullyConnectedWeightsPass.h"
 #include "luci/Pass/FuseInstanceNormPass.h"
+#include "luci/Pass/FuseLayerNormToInstNormPass.h"
 #include "luci/Pass/FuseMeanWithMeanPass.h"
 #include "luci/Pass/FuseMulWithConvPass.h"
 #include "luci/Pass/FuseMulWithDivPass.h"
@@ -336,6 +337,7 @@ void CircleOptimizer::optimize(loco::Graph *g) const
   option_to_pass[Options::Algorithm::ResolveCustomOpMaxPoolWithArgmax] = &createPassInstance<luci::ResolveCustomOpMaxPoolWithArgmaxPass>;
   option_to_pass[Options::Algorithm::ResolveCustomOpSplitV] = &createPassInstance<luci::ResolveCustomOpSplitVPass>;
   option_to_pass[Options::Algorithm::FuseInstanceNorm] = &createPassInstance<luci::FuseInstanceNormPass>;
+  option_to_pass[Options::Algorithm::FuseLayerNormToInstNorm] = &createPassInstance<luci::FuseLayerNormToInstNormPass>;
   option_to_pass[Options::Algorithm::FuseBatchNormWithConv] = &createPassInstance<luci::FuseBatchNormWithConvPass>;
   option_to_pass[Options::Algorithm::FuseBatchNormWithDwConv] = &createPassInstance<luci::FuseBatchNormWithDwConvPass>;
   option_to_pass[Options::Algorithm::FuseBatchNormWithTConv] = &createPassInstance<luci::FuseBatchNormWithTConvPass>;