Skip to content

Commit

Permalink
[onert/nnfw_api] Specify loss per epoch in GenModelTrain tests (#13271)
Browse files Browse the repository at this point in the history 
This commit verifies the loss per epoch by updating the GenModelTrain
code.

ONE-DCO-1.0-Signed-off-by: Jiyoung Yun <[email protected]>
Co-authored-by: Jang Jiseob <[email protected]>
  • Loading branch information
@jyoungyun @ragmani
jyoungyun and ragmani authored Jun 25, 2024
1 parent 2ceb27c commit 2043a78
Show file tree
Hide file tree
Showing 12 changed files with 50 additions and 47 deletions.
35 changes: 18 additions & 17 deletions tests/nnfw_api/lib/GenModelTrain.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ struct TrainCaseData
DataSet dataset;

/**
* @brief A vector of loss buffer
* @brief A vector of loss buffers
*/
std::vector<float> losses;
std::vector<std::vector<float>> losses;

/**
* @brief Append vector data list of inputs that are used in one step
Expand Down Expand Up @@ -84,10 +84,9 @@ struct TrainCaseData
* @tparam T Data type
* @param data vector data array
*/
TrainCaseData &setLosses(std::vector<float> losses)
TrainCaseData &addLosses(const std::vector<float> &loss)
{
this->losses = losses;

losses.emplace_back(loss);
return *this;
}

Expand Down Expand Up @@ -159,19 +158,17 @@ inline void TrainCaseData::addData<bool>(OneStepData &dest, const std::vector<bo
template <typename T>
static TrainCaseData uniformTCD(const std::vector<std::vector<std::vector<T>>> &inputs_dataset,
const std::vector<std::vector<std::vector<T>>> &expects_dataset,
const std::vector<float> &losses)
const std::vector<std::vector<float>> &losses)
{
assert(inputs_dataset.size() == expects_dataset.size());

TrainCaseData ret;
for (const auto &data : inputs_dataset)
ret.addInputs<T>(data);
for (const auto &data : expects_dataset)
{
assert(data.size() == losses.size());
ret.addExpects<T>(data);
}
ret.setLosses(losses);
for (const auto &loss : losses)
ret.addLosses(loss);
return ret;
}

Expand Down Expand Up @@ -360,6 +357,7 @@ class GenModelTrain : public ::testing::Test

// Prepare expected losses
const auto &ref_losses = train_case.losses;
ASSERT_EQ(ref_losses.size(), num_epoch);
std::vector<float> actual_losses(num_expecteds, 0.f);
for (uint32_t epoch = 0; epoch < num_epoch; ++epoch)
{
Expand Down Expand Up @@ -410,14 +408,17 @@ class GenModelTrain : public ::testing::Test
{
actual_losses[i] /= num_step;
}
}

// TODO better way for handling FP error?
for (uint32_t i = 0; i < actual_losses.size(); i++)
{
const float actual = actual_losses[i];
const float expected = ref_losses[i];
EXPECT_NEAR(expected, actual, 0.001) << "Loss #" << i;
ASSERT_EQ(ref_losses[epoch].size(), actual_losses.size());

// TODO better way for handling FP error?
for (uint32_t i = 0; i < actual_losses.size(); i++)
{
const float actual = actual_losses[i];
const float expected = ref_losses[epoch][i];
EXPECT_NEAR(expected, actual, 0.001)
<< "Loss " << epoch + 1 << "/" << num_epoch << " #" << i;
}
}
}

Expand Down
6 changes: 3 additions & 3 deletions tests/nnfw_api/src/GenModelTests/BranchModelTrain.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ TEST_F(GenModelTrain, BranchOps_FC_Add)
_context->addTrainCase(uniformTCD<float>(
{{{1, 3}, {0, 1, 2, 3, 4, 5, 6, 7}}, {{2, 1}, {7, 6, 5, 4, 3, 2, 1, 0}}}, // inputs
{{{2, 1, 5, 5, 2, 1, 5, 5}}, {{2, 1, 5, 5, 2, 1, 5, 6}}}, // expected
{8.4678f} // loss
{{9.2218f}, {8.9554f}, {8.7044f}, {8.4678f}} // loss
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -90,7 +90,7 @@ TEST_F(GenModelTrain, BranchOps_FC_Sub)
_context->addTrainCase(uniformTCD<float>(
{{{0, 1, 2, 3, 4, 5, 1, 3}, {6, 7}}, {{5, 4, 3, 2, 1, 0, 2, 1}, {7, 6}}}, // inputs
{{{2, 1, 5, 5, 2, 1, 5, 5}}, {{2, 1, 5, 5, 2, 1, 5, 6}}}, // expected
{3.2863f} // loss
{{7.3265f}, {4.6811f}, {3.6735f}, {3.2863f}} // loss
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -145,7 +145,7 @@ TEST_F(GenModelTrain, BranchOps_FC_Mul)
_context->addTrainCase(
uniformTCD<float>({{{0, 3}, {6, 7}}, {{5, 4}, {7, 6}}}, // inputs
{{{3, 2, 1, 2, 5, 6, 1, 0}}, {{2, 1, 5, 5, 2, 1, 5, 6}}}, // expected
{12.2822f} // loss
{{12.5488f}, {12.4590f}, {12.3701f}, {12.2822f}} // loss
));

_context->setBackends({"train"});
Expand Down
8 changes: 4 additions & 4 deletions tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/MaxPool2D.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ TEST_F(GenModelTrain, NonTrainableOps_Conv2D_MaxPool2D)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // inputs
{{{1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8, 7, 6, 5, 4, 3, 2, 1}}}, // expected
{24.0089f} // loss
{{31.6667f}, {28.6837f}, {26.1765f}, {24.0089f}} // loss
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -110,7 +110,7 @@ TEST_F(GenModelTrain, NonTrainableOps_Conv2D_MaxPool2D_Depth1_Filter2)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // inputs
{{{1, 2, 3, 4, 5, 6, 7, 8, 9}}}, // expected
{8.4666} // loss
{{31.6667f}, {25.9453f}, {15.4067f}, {8.4666f}} // loss
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -158,7 +158,7 @@ TEST_F(GenModelTrain, NonTrainableOps_Conv2D_MaxPool2D_Depth2_Filter2)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // inputs
{{{1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8, 7, 6, 5, 4, 3, 2, 1}}}, // expected
{9.3556f} // loss
{{31.6667f}, {27.8823f}, {16.9743f}, {9.3556f}} // loss
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -208,7 +208,7 @@ TEST_F(GenModelTrain, NonTrainableOps_Conv2D_MaxPool2D_Stride2Filter2)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // inputs
{{{1, 2, 3, 4, 5, 6, 7, 8}}}, // expected
{9.0784f} // loss
{{25.5000f}, {19.2126f}, {12.9202f}, {9.0784f}} // loss
));

_context->setBackends({"train"});
Expand Down
3 changes: 2 additions & 1 deletion tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/Mean.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ TEST_F(GenModelTrain, NonTrainableOps_FC_Mean)
_context = std::make_unique<GenModelTrainContext>(cgen.finish());
_context->addTrainCase(uniformTCD<float>({{{1, 3}}, {{2, 1}}}, // inputs
{{{5}}, {{3}}}, // expected
{13.3691f} // loss
{{16.7778f}, {15.5544f}, {14.4203f}, {13.3691f}}
// loss
));

_context->setBackends({"train"});
Expand Down
2 changes: 1 addition & 1 deletion tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/Pad.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ TEST_F(GenModelTrain, NonTrainableOps_FC_Pad)
_context->addTrainCase(
uniformTCD<float>({{{0, 1, 2, 3, 4, 5, 6, 7}}, {{7, 6, 5, 4, 3, 2, 1, 0}}}, // inputs
{{{0, 13, 52, 0}}, {{0, 31, 24, 0}}}, // expected
{1.3900f} // loss
{{462.7862f}, {32.8115f}, {5.8401f}, {1.3900f}} // loss
));

_context->setBackends({"train"});
Expand Down
2 changes: 1 addition & 1 deletion tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/Relu.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ TEST_F(GenModelTrain, NonTrainableOps_FC_Relu_FC)
_context->addTrainCase(
uniformTCD<float>({{{1, 3}}, {{2, 1}}}, // inputs
{{{0, 1, 5, 5, 2, 1, 5, 5}}, {{2, 1, 5, 5, 0, 1, 5, 6}}}, // expected
{13.5010f} // loss
{{13.9041f}, {13.7684f}, {13.6340f}, {13.5010f}} // loss
));

_context->setBackends({"train"});
Expand Down
2 changes: 1 addition & 1 deletion tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/Relu6.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ TEST_F(GenModelTrain, NonTrainableOps_FC_Relu6_FC)
_context->addTrainCase(
uniformTCD<float>({{{1, 3}}, {{2, 1}}}, // inputs
{{{2, 0, 6, 5, 2, 1, 6, 5}}, {{2, 1, 6, 5, 0, 1, 6, 6}}}, // expected
{16.3412f} // loss
{{16.8338f}, {16.6680f}, {16.5038f}, {16.3412f}} // loss
));

_context->setBackends({"train"});
Expand Down
4 changes: 2 additions & 2 deletions tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/Reshape.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ TEST_F(GenModelTrain, NonTrainableOps_FC_Reshape)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // input dataset
{{{47, -4, -25, 9, 10, 10, -13, 11, -14, -26, -12, 26, 20, 40, 1, 3, 11,
4}}}, // expected dataset
{226.5260f} // last losses
4}}}, // expected dataset
{{403.3333f}, {324.0978f}, {267.7882f}, {226.5260f}} // losses
));

_context->setBackends({"train"});
Expand Down
2 changes: 1 addition & 1 deletion tests/nnfw_api/src/GenModelTests/nontrainable_op_trains/Softmax.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ TEST_F(GenModelTrain, NonTrainableOps_FC_Softmax)
_context->addTrainCase(
uniformTCD<float>({{{1, 3}}, {{2, 1}}}, // inputs
{{{0, 1, 0, 0, 0, 0, 0, 0}}, {{0, 0, 0, 0, 0, 1, 0, 0}}}, // expected
{0.1092f} // loss
{{0.1094f}, {0.1093f}, {0.1092f}, {0.1092f}} // loss
));

_context->setBackends({"train"});
Expand Down
13 changes: 7 additions & 6 deletions tests/nnfw_api/src/GenModelTests/one_op_trains/Conv2D.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ TEST_F(GenModelTrain, OneOp_Conv2D_training_enabled)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // input dataset
{{{47, -4, -25, 9, 10, 10, -13, 11, -14, -26, -12, 26, 20, 40, 1, 3, 11,
4}}}, // expected dataset
{324.0978f} // last losses
4}}}, // expected dataset
{{403.3333f}, {324.0978f}, {267.7882f}, {226.5260f}} // losses
));

_context->setBackends({"train"});
// To apply backward to loss, epoch should be >= 2
_context->setEpoch(2);
_context->setEpoch(4);

SUCCEED();
}
Expand All @@ -70,13 +70,14 @@ TEST_F(GenModelTrain, OneOp_Conv2D_training_disabled)
uniformTCD<float>({{{4, 0, -5, 1, 0, 4, -1, 1, -1, -3, 3, -2, -4,
1, -2, 2, 4, -4, 2, 2, 0, 4, -1, -2, 4}}}, // input dataset
{{{47, -4, -25, 9, 10, 10, -13, 11, -14, -26, -12, 26, 20, 40, 1, 3, 11,
4}}}, // expected dataset
{403.333f} // gain of loss after each epoch is const (equal 403.333)
4}}}, // expected dataset
{{403.333f}, {403.333f}, {403.333f}, {403.333f}}
// gain of loss after each epoch is const (equal 403.333)
));

_context->setBackends({"train"});
// To apply backward to loss, epoch should be >= 2
_context->setEpoch(10);
_context->setEpoch(4);

SUCCEED();
}
12 changes: 6 additions & 6 deletions tests/nnfw_api/src/GenModelTests/one_op_trains/DepthwiseConvolution.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ TEST_F(GenModelTrain, OneOp_DepthwiseConv2D)

_context = std::make_unique<GenModelTrainContext>(cgen.finish());
_context->addTrainCase(
uniformTCD<float>({{{1, 2, 7, 8, 3, 4, 9, 1, 5, 6, 11, 2}}}, // input dataset
{{{1, -4, 1, -3, 2, -2, 2, -4}}}, // expected dataset
{1.1701f} // last losses
uniformTCD<float>({{{1, 2, 7, 8, 3, 4, 9, 1, 5, 6, 11, 2}}}, // input dataset
{{{1, -4, 1, -3, 2, -2, 2, -4}}}, // expected dataset
{{6.8750f}, {2.5275f}, {1.6320f}, {1.1701f}} // losses
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -78,7 +78,7 @@ TEST_F(GenModelTrain, OneOp_DepthwiseConv2D_No_Multiplier)
_context->addTrainCase(
uniformTCD<float>({{{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}}}, // input dataset
{{{6.5f, 7.5f, 8.5f, 3.5f, 8.5f, 5.5f, 2.5f, 3.5f}}}, // expected dataset
{15.5431f} // last losses
{{38.0000f}, {26.6868f}, {19.8101f}, {15.5431f}} // losses
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -113,7 +113,7 @@ TEST_F(GenModelTrain, OneOp_DepthwiseConv2D_No_Multiplier_RELU6)
_context->addTrainCase(
uniformTCD<float>({{{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}}}, // input dataset
{{{6.0f, 6.0f, 6.0f, 6.0f, 6.0f, 6.0f, 6.0f, 6.0f}}}, // expected dataset
{36.0000f} // last losses
{{36.0000f}, {36.0000f}, {36.0000f}, {36.0000f}} // losses
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -148,7 +148,7 @@ TEST_F(GenModelTrain, OneOp_DepthwiseConv2D_3x3)
_context->addTrainCase(uniformTCD<float>(
{{{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}}}, // input dataset
{{{6.0f, 16.0f, 8.0f, 16.0f, 10.0f, 16.0f, 12.0f, 16.0f}}}, // expected dataset
{13.7338f} // last losses
{{171.0000f}, {69.5150f}, {29.9159f}, {13.7338f}} // losses
));

_context->setBackends({"train"});
Expand Down
8 changes: 4 additions & 4 deletions tests/nnfw_api/src/GenModelTests/one_op_trains/FullyConnected.test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ TEST_F(GenModelTrain, OneOp_FullyConnected)
_context->addTrainCase(
uniformTCD<float>({{{1, 3}}, {{2, 1}}}, // inputs
{{{2, 1, 5, 5, 2, 1, 5, 5}}, {{2, 1, 5, 5, 2, 1, 5, 6}}}, // expected
{11.4484f} // loss
{{14.2234f}, {13.2278f}, {12.3045f}, {11.4484f}} // loss
));

_context->setBackends({"train"});
Expand Down Expand Up @@ -72,9 +72,9 @@ TEST_F(GenModelTrain, OneOp_FullyConnected_OptionalBias)

_context = std::make_unique<GenModelTrainContext>(cgen.finish());
_context->addTrainCase(
uniformTCD<float>({{{1, 3, 2, 1}}}, // inputs
{{{2, 1, 5, 5, 2, 1, 5, 5, 2, 1, 5, 5, 2, 1, 5, 6}}}, // expected
{{12.7512f}} // loss
uniformTCD<float>({{{1, 3, 2, 1}}}, // inputs
{{{2, 1, 5, 5, 2, 1, 5, 5, 2, 1, 5, 5, 2, 1, 5, 6}}}, // expected
{{14.4375f}, {13.9950f}, {13.5668f}, {13.1523f}, {12.7512f}} // loss
));

_context->setBackends({"train"});
Expand Down

0 comments on commit 2043a78

Please sign in to comment.