From 90b5f5e0b76e03ce3db36a423ba201e7494cddc7 Mon Sep 17 00:00:00 2001
From: SaeHie Park <saehie.park@gmail.com>
Date: Tue, 16 Jul 2024 05:35:51 +0000
Subject: [PATCH] [one-cmds] Add more fuse/fold options

This will add more fuse/fold options
- fuse Add to following FC bias
- fuse Add to following FC weights
- fold Mul

ONE-DCO-1.0-Signed-off-by: SaeHie Park <saehie.park@gmail.com>
---
 compiler/one-cmds/how-to-use-one-commands.txt | 3 +++
 compiler/one-cmds/onelib/constant.py          | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/compiler/one-cmds/how-to-use-one-commands.txt b/compiler/one-cmds/how-to-use-one-commands.txt
index 08e8a557375..fefbabf9a17 100644
--- a/compiler/one-cmds/how-to-use-one-commands.txt
+++ b/compiler/one-cmds/how-to-use-one-commands.txt
@@ -160,15 +160,18 @@ Current transformation options are
 - fold_dequantize : This removes Dequantize operation which can be folded
 - fold_dwconv : This folds Depthwise Convolution operation which can be folded
 - fold_gather : This removes Gather operation which can be folded
+- fold_mul : This removes Mul operation which can be folded
 - fold_shape : This removes Shape operation which can be folded
 - fold_sparse_to_dense : This removes SparseToDense operation which can be folded
 - forward_reshape_to_unaryop: This will move Reshape after UnaryOp for centain condition
+- fuse_add_to_fullyconnected_bias: This fuses Add operator to following FullyConnected operator bias
 - fuse_add_with_conv: This fuses Add operator with the preceding Convolution operator if possible
 - fuse_add_with_fully_connected: This fuses Add operator with the preceding FullyConnected operator if possible
 - fuse_add_with_tconv: This fuses Add operator with the preceding TConv operator if possible
 - fuse_batchnorm_with_conv : This fuses BatchNorm operator to convolution operator
 - fuse_batchnorm_with_dwconv : This fuses BatchNorm operator to depthwise convolution operator
 - fuse_batchnorm_with_tconv : This fuses BatchNorm operator to transpose convolution operator
+- fuse_mul_to_fullyconnected_weights : This fuses Mul operator to following FullyConnected operator weights
 - fuse_mul_with_conv: This fuses Mul with a preceding Convolution op if possible.
 - fuse_mul_with_div: This fuses Mul and Div op as Div.
 - fuse_slice_with_tconv: This fuses Slice with a preceding TConv if possible.
diff --git a/compiler/one-cmds/onelib/constant.py b/compiler/one-cmds/onelib/constant.py
index 42676d78159..8c5de1b646d 100644
--- a/compiler/one-cmds/onelib/constant.py
+++ b/compiler/one-cmds/onelib/constant.py
@@ -29,12 +29,14 @@ class CONSTANT:
         'fold_dwconv',
         'fold_fully_connected',
         'fold_gather',
+        'fold_mul',
         'fold_reshape',
         'fold_shape',
         'fold_sparse_to_dense',
         'fold_squeeze',
 
         # Operator fusion
+        'fuse_add_to_fullyconnected_bias',
         'fuse_add_with_conv',
         'fuse_add_with_tconv',
         'fuse_add_with_fully_connected',
@@ -42,6 +44,7 @@ class CONSTANT:
         'fuse_batchnorm_with_dwconv',
         'fuse_batchnorm_with_tconv',
         'fuse_activation_function',
+        'fuse_mul_to_fullyconnected_weights',
         'fuse_instnorm',
         'fuse_prelu',
         'fuse_gelu',
@@ -104,18 +107,23 @@ class CONSTANT:
         ('fold_dwconv', 'fold Depthwise Convolution op with constant inputs'),
         ('fold_fully_connected', 'fold FullyConnected op with constant inputs'),
         ('fold_gather', 'fold Gather op'),
+        ('fold_mul', 'fold Mul Op'),
         ('fold_reshape', 'fold Reshape op'),
         ('fold_shape', 'fold Shape op'),
         ('fold_sparse_to_dense', 'fold SparseToDense op'),
         ('fold_squeeze', 'fold Squeeze op'),
         ('forward_reshape_to_unaryop', 'Forward Reshape op'),
         ('forward_transpose_op', 'Forward Transpose op'),
+        ('fuse_add_to_fullyconnected_bias',
+         'Fuse Add op to following FullyConnected op bias'),
         ('fuse_add_with_conv', 'fuse Add op to Convolution op'),
         ('fuse_add_with_tconv', 'fuse Add op to Transposed'),
         ('fuse_add_with_fully_connected', 'fuse Add op to FullyConnected op'),
         ('fuse_batchnorm_with_conv', 'fuse BatchNorm op to Convolution op'),
         ('fuse_batchnorm_with_dwconv', 'fuse BatchNorm op to Depthwise Convolution op'),
         ('fuse_batchnorm_with_tconv', 'fuse BatchNorm op to Transposed Convolution op'),
+        ('fuse_mul_to_fullyconnected_weights',
+         'fuse Mul op to following FullyConnected op weights'),
         ('fuse_slice_with_tconv', 'fuse Slice op to Transposed Convolution op'),
         ('fuse_bcq', 'apply Binary Coded Quantization'),
         ('fuse_preactivation_batchnorm',