[ graph ] Divide addLossLayer process when compiling inference mode g…

…raph - Inference model does not require label and loss layer. - Therefore, we can just simply ignore addLossLayer during the inference mode. - Add unittest to verify such. - Here, reason why I set a default parameter for nn::inference is to remove redundant parameter input for better user interface. **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: skykongkong8 <[email protected]>
nnstreamer · Jan 15, 2025 · c5db575 · c5db575
1 parent 97add51
commit c5db575
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 32 deletions.
diff --git a/api/ccapi/include/model.h b/api/ccapi/include/model.h
@@ -296,9 +296,9 @@ class Model {
    * @retval list of output as float *
    * @note The output memory must not be freed by the caller
    */
-  virtual std::vector<float *> inference(unsigned int batch,
-                                         const std::vector<float *> &input,
-                                         const std::vector<float *> &label) = 0;
+  virtual std::vector<float *>
+  inference(unsigned int batch, const std::vector<float *> &input,
+            const std::vector<float *> &label = std::vector<float *>()) = 0;
 
   /**
    * @brief     Run the incremental inference of the model
@@ -308,16 +308,16 @@ class Model {
    * @param[in] init_seq_len initial sequence length
    * @param[in] from current working step index
    * @param[in] to next working step index
-   * @param[in] output_hidden_state return last hidden state if true else return all hidden state
+   * @param[in] output_hidden_state return last hidden state if true else return
+   * all hidden state
    * @retval list of output as float *
    * @note The output memory must not be freed by the caller
    */
   virtual std::vector<float *>
   incremental_inference(unsigned int batch, const std::vector<float *> &input,
                         const std::vector<float *> &label,
                         unsigned int init_seq_len, unsigned int from,
-                        unsigned int to,
-                        bool output_hidden_state = false) = 0;
+                        unsigned int to, bool output_hidden_state = false) = 0;
 
   /**
    * @brief     Summarize the model

diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp
@@ -65,14 +65,21 @@ int NetworkGraph::compile(const std::string &loss_type) {
 
   graph.realizeInputOutputNode();
 
-  try {
-    /// @todo realize loss beforehand
-    status = addLossLayer(loss_type);
-    NN_RETURN_STATUS();
-  } catch (const std::exception &e) {
-    ml_loge("%s", e.what());
-    status = ML_ERROR_INVALID_PARAMETER;
-    NN_RETURN_STATUS();
+  if (exec_mode != ExecutionMode::INFERENCE) {
+    try {
+      /// @todo realize loss beforehand
+      status = addLossLayer(loss_type);
+      NN_RETURN_STATUS();
+    } catch (const std::exception &e) {
+      ml_loge("%s", e.what());
+      status = ML_ERROR_INVALID_PARAMETER;
+      NN_RETURN_STATUS();
+    }
+  } else {
+    if (!loss_type.empty()) {
+      ml_loge(
+        "Warning : Loss type is given in inference mode. Ignoring loss type.");
+    }
   }
 
   graph.topologicalSort();

diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp
@@ -221,7 +221,7 @@ int NeuralNetwork::initialize(ExecutionMode mode) {
     } else {
       NNTR_THROW_IF(exec_mode == ExecutionMode::TRAIN, std::invalid_argument)
         << "Execution mode mismatch : trying to train with compiled for "
-           "infence";
+           "inference";
     }
   }
 

diff --git a/nntrainer/models/neuralnet.h b/nntrainer/models/neuralnet.h
@@ -365,9 +365,9 @@ class NeuralNetwork : public ml::train::Model {
    * @retval list of output as float *
    * @note The output memory must not be freed by the caller
    */
-  std::vector<float *> inference(unsigned int batch,
-                                 const std::vector<float *> &input,
-                                 const std::vector<float *> &label) override;
+  std::vector<float *> inference(
+    unsigned int batch, const std::vector<float *> &input,
+    const std::vector<float *> &label = std::vector<float *>()) override;
 
   /**
    * @brief     Run NeuralNetwork incremental inference
@@ -403,13 +403,12 @@ s   * @retval shared_ptr<const Tensor>
    * @retval list of output as float *
    * @note The output memory must not be freed by the caller
    */
-  std::vector<float *> incremental_inference(unsigned int batch,
-                                             const std::vector<float *> &input,
-                                             const std::vector<float *> &label,
-                                             unsigned int init_seq_len,
-                                             unsigned int from,
-                                             unsigned int to,
-                                             bool output_hidden_state = false) override;
+  std::vector<float *>
+  incremental_inference(unsigned int batch, const std::vector<float *> &input,
+                        const std::vector<float *> &label,
+                        unsigned int init_seq_len, unsigned int from,
+                        unsigned int to,
+                        bool output_hidden_state = false) override;
 
   /**
    * @brief     Run NeuralNetwork train with callback function by user
@@ -625,11 +624,12 @@ s   * @retval shared_ptr<const Tensor>
                const std::string file_path) override;
 
 private:
-  using FlexiblePropTypes = std::tuple<
-    props::Epochs, props::TrainingBatchSize, props::SavePath,
-    props::ContinueTrain, props::SaveBestPath, props::MemoryOptimization,
-    props::MemorySwap, props::MemorySwapPath, props::MemorySwapLookahead,
-    props::TensorFormat, props::ModelTensorDataType>;
+  using FlexiblePropTypes =
+    std::tuple<props::Epochs, props::TrainingBatchSize, props::SavePath,
+               props::ContinueTrain, props::SaveBestPath,
+               props::MemoryOptimization, props::MemorySwap,
+               props::MemorySwapPath, props::MemorySwapLookahead,
+               props::TensorFormat, props::ModelTensorDataType>;
   using RigidPropTypes =
     std::tuple<props::LossType, std::vector<props::InputConnection>,
                std::vector<props::LabelLayer>, props::ClipGradByGlobalNorm,

diff --git a/test/unittest/integration_tests/integration_test_fsu.cpp b/test/unittest/integration_tests/integration_test_fsu.cpp
@@ -91,7 +91,6 @@ TEST(fsu, simple_fc) {
   uint feature_size = 320;
 
   float input[320];
-  float label[1];
 
   for (uint j = 0; j < feature_size; ++j)
     input[j] = j;
@@ -101,7 +100,6 @@ TEST(fsu, simple_fc) {
   std::vector<float *> answer;
 
   in.push_back(input);
-  l.push_back(label);
 
   answer = model->inference(1, in, l);
 

diff --git a/test/unittest/unittest_nntrainer_graph.cpp b/test/unittest/unittest_nntrainer_graph.cpp
@@ -368,6 +368,51 @@ TEST(nntrainerGraphUnitTest, call_functions) {
   }
 }
 
+TEST(nntrainerGraphUnitTest, NoLossLayerWhenInferenceMode) {
+  std::unique_ptr<ml::train::Model> model =
+    ml::train::createModel(ml::train::ModelType::NEURAL_NET);
+
+  model->addLayer(ml::train::createLayer(
+    "input", {withKey("name", "input0"), withKey("input_shape", "1:1:256")}));
+
+  for (int i = 0; i < 3; ++i) {
+    model->addLayer(ml::train::createLayer(
+      "fully_connected",
+      {withKey("unit", 1024), withKey("weight_initializer", "xavier_uniform"),
+       withKey("bias_initializer", "zeros")}));
+  }
+  model->addLayer(ml::train::createLayer(
+    "fully_connected",
+    {withKey("unit", 100), withKey("weight_initializer", "xavier_uniform"),
+     withKey("bias_initializer", "zeros")}));
+
+  model->setProperty({withKey("batch_size", 1), withKey("epochs", 1),
+                      withKey("memory_swap", "false"),
+                      withKey("model_tensor_type", "FP32-FP32")});
+
+  int status = model->compile(ml::train::ExecutionMode::INFERENCE);
+  EXPECT_EQ(status, ML_ERROR_NONE);
+
+  status = model->initialize(ml::train::ExecutionMode::INFERENCE);
+  EXPECT_EQ(status, ML_ERROR_NONE);
+
+  float input[256];
+
+  for (unsigned int i = 0; i < 256; ++i) {
+    input[i] = i;
+  }
+
+  std::vector<float *> in;
+  std::vector<float *> ans;
+
+  in.push_back(input);
+
+  ans = model->inference(1, in);
+
+  in.clear();
+  ans.clear();
+}
+
 int main(int argc, char **argv) {
   int result = -1;