Add modeler non-regression tests (#2617)

Signed-off-by: Peter Mitri <[email protected]>

.github/workflows/ubuntu.yml

@@ -247,7 +247,7 @@ jobs:
       - name: Run cucumber on short-tests
         uses: ./.github/workflows/cucumber-tests
         with:
-          feature: "features/short_tests.feature"
+          feature: "features/solver-features/short_tests.feature"
 
       - name: Run mps tests
         if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }}
@@ -286,7 +286,7 @@ jobs:
       - name: Run cucumber on medium-tests
         uses: ./.github/workflows/cucumber-tests
         with:
-          feature: "features/medium_tests.feature"
+          feature: "features/solver-features/medium_tests.feature"
 
       - name: Run long-tests-1
         if: ${{ env.RUN_EXTENDED_TESTS == 'true' && !cancelled() }}
@@ -312,6 +312,11 @@ jobs:
           batch-name: long-tests-3
           os: ${{ env.os }}
 
+      - name: Run cucumber on modeler
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/modeler-features"
+
       - name: Barrier
         if: ${{ !success() }}
         run: exit 1

.github/workflows/windows-vcpkg.yml

@@ -248,7 +248,7 @@ jobs:
       - name: Run cucumber on short-tests
         uses: ./.github/workflows/cucumber-tests
         with:
-          feature: "features/short_tests.feature"
+          feature: "features/solver-features/short_tests.feature"
 
       - name: Run mps tests
         if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }}
@@ -287,7 +287,7 @@ jobs:
       - name: Run cucumber on medium-tests
         uses: ./.github/workflows/cucumber-tests
         with:
-          feature: "features/medium_tests.feature"
+          feature: "features/solver-features/medium_tests.feature"
 
       - name: Run long-tests-1
         if: ${{ env.RUN_EXTENDED_TESTS == 'true' && !cancelled() }}
@@ -313,6 +313,11 @@ jobs:
           batch-name: long-tests-3
           os: ${{ env.os }}
 
+      - name: Run cucumber on modeler
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/modeler-features"
+
       - name: Barrier
         if: ${{ !success() }}
         run: exit 1

src/solver/modeler/main.cpp

@@ -39,7 +39,7 @@ using namespace Antares::Solver::Modeler::Api;
 class SystemLinearProblem
 {
 public:
-    explicit SystemLinearProblem(const Antares::Study::SystemModel::System& system):
+    explicit SystemLinearProblem(const Study::SystemModel::System& system):
         system_(system)
     {
     }
@@ -48,11 +48,11 @@ class SystemLinearProblem
 
     void Provide(ILinearProblem& pb, const ModelerParameters& parameters)
     {
-        std::vector<std::unique_ptr<Antares::Optimization::ComponentFiller>> fillers;
-        std::vector<Antares::Solver::Modeler::Api::LinearProblemFiller*> fillers_ptr;
+        std::vector<std::unique_ptr<Optimization::ComponentFiller>> fillers;
+        std::vector<LinearProblemFiller*> fillers_ptr;
         for (const auto& [_, component]: system_.Components())
         {
-            auto cf = std::make_unique<Antares::Optimization::ComponentFiller>(component);
+            auto cf = std::make_unique<Optimization::ComponentFiller>(component);
             fillers.push_back(std::move(cf));
         }
         for (auto& component_filler: fillers)
@@ -67,7 +67,7 @@ class SystemLinearProblem
     }
 
 private:
-    const Antares::Study::SystemModel::System& system_;
+    const Study::SystemModel::System& system_;
 };
 
 static void usage()
@@ -105,6 +105,19 @@ int main(int argc, const char** argv)
         const auto system = LoadFiles::loadSystem(studyPath, libraries);
         logs.info() << "System loaded";
         SystemLinearProblem system_linear_problem(system);
+
+        auto outputPath = studyPath / "output";
+        if (!parameters.noOutput)
+        {
+            logs.info() << "Output folder : " << outputPath;
+            if (!std::filesystem::is_directory(outputPath)
+                && !std::filesystem::create_directory(outputPath))
+            {
+                logs.error() << "Failed to create output directory. Exiting simulation.";
+                return EXIT_FAILURE;
+            }
+        }
+
         logs.info() << "linear problem of System loaded";
         OrtoolsLinearProblem ortools_linear_problem(true, parameters.solver);
 
@@ -118,20 +131,20 @@ int main(int argc, const char** argv)
         if (!parameters.noOutput)
         {
             logs.info() << "Writing problem.lp...";
-            auto mps_path = std::filesystem::current_path() / "problem.lp";
-            ortools_linear_problem.WriteLP(mps_path.string());
+            auto lp_path = outputPath / "problem.lp";
+            ortools_linear_problem.WriteLP(lp_path.string());
         }
 
         logs.info() << "Launching resolution...";
         auto* solution = ortools_linear_problem.solve(parameters.solverLogs);
         switch (solution->getStatus())
         {
-        case Antares::Solver::Modeler::Api::MipStatus::OPTIMAL:
-        case Antares::Solver::Modeler::Api::MipStatus::FEASIBLE:
+        case MipStatus::OPTIMAL:
+        case MipStatus::FEASIBLE:
             if (!parameters.noOutput)
             {
                 logs.info() << "Writing variables...";
-                std::ofstream sol_out(std::filesystem::current_path() / "solution.csv");
+                std::ofstream sol_out(outputPath / "solution.csv");
                 for (const auto& [name, value]: solution->getOptimalValues())
                 {
                     sol_out << name << " " << value << std::endl;

src/tests/cucumber/CMakeLists.txt

@@ -3,5 +3,7 @@ file(GENERATE OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/behave.ini CONTENT
 \n\
 antares-solver = $<TARGET_FILE:antares-solver>\
 \n\
-resources-path = ${CMAKE_SOURCE_DIR}/tests/resources/Antares_Simulator_Tests_NR"
+antares-modeler = $<TARGET_FILE:antares-modeler>\
+\n\
+resources-path = ${CMAKE_SOURCE_DIR}/tests/resources"
         CONDITION $<STREQUAL:$<CONFIG>,${CMAKE_BUILD_TYPE}>)

src/tests/cucumber/features/modeler-features/epic2/us2.5.feature

@@ -0,0 +1,36 @@
+Feature: 2.5 - Pure modeler simple studies, with no ports and no timeseries
+
+  Scenario: 2.5.1: One model with one load and two generators, one timestamp
+    Given the study path is "modeler/epic2/us2.5/study_2.5.1"
+    When I run antares modeler
+    Then the simulation succeeds
+    And the optimal value of variable node1.gen1_p_0 is 80
+    And the optimal value of variable node1.gen2_p_0 is 20
+
+  Scenario: 2.5.2: One model with one load and two generators (minP), three timestamps
+    Given the study path is "modeler/epic2/us2.5/study_2.5.2"
+    When I run antares modeler
+    Then the simulation succeeds
+    And the optimal value of variable node1.gen1_up_0 is 1
+    And the optimal value of variable node1.gen1_up_1 is 1
+    And the optimal value of variable node1.gen1_up_2 is 1
+    And the optimal value of variable node1.gen1_p_0 is 60
+    And the optimal value of variable node1.gen1_p_1 is 60
+    And the optimal value of variable node1.gen1_p_2 is 60
+    And the optimal value of variable node1.gen2_up_0 is 1
+    And the optimal value of variable node1.gen2_up_1 is 1
+    And the optimal value of variable node1.gen2_up_2 is 1
+    And the optimal value of variable node1.gen2_p_0 is 40
+    And the optimal value of variable node1.gen2_p_1 is 40
+    And the optimal value of variable node1.gen2_p_2 is 40
+
+  Scenario: 2.5.3: Two libs, one timestamp
+    Given the study path is "modeler/epic2/us2.5/study_2.5.3"
+    When I run antares modeler
+    Then the simulation succeeds
+    And the optimal value of variable node1.gen1_p_0 is 0
+    And the optimal value of variable node1.gen2_p_0 is 100
+    And the optimal value of variable node2.gen1_p_0 is 500
+    And the optimal value of variable node2.gen1_up_0 is 1
+    And the optimal value of variable node2.gen2_p_0 is 500
+    And the optimal value of variable node2.gen2_up_0 is 1

src/tests/cucumber/features/medium_tests.feature → src/tests/cucumber/features/solver-features/medium_tests.feature

@@ -2,7 +2,7 @@ Feature: medium tests
 
   @fast @medium @incomplete
   Scenario: 035 Mixed Expansion - Smart grid model 2
-    Given the study path is "medium-tests/035 Mixed Expansion - Smart grid model 2"
+    Given the study path is "Antares_Simulator_Tests_NR/medium-tests/035 Mixed Expansion - Smart grid model 2"
     When I run antares simulator
     Then the simulation takes less than 30 seconds
     And the simulation succeeds

src/tests/cucumber/features/short_tests.feature → src/tests/cucumber/features/solver-features/short_tests.feature

@@ -2,15 +2,15 @@ Feature: short tests
 
   @fast @short
   Scenario: 001 One node - passive
-    Given the study path is "short-tests/001 One node - passive"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/001 One node - passive"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
     And the annual system cost is 0
 
   @fast @short
   Scenario: 002 Thermal fleet - Base
-    Given the study path is "short-tests/002 Thermal fleet - Base"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/002 Thermal fleet - Base"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -20,7 +20,7 @@ Feature: short tests
 
   @fast @short
   Scenario: 003 Thermal fleet - Must-run
-    Given the study path is "short-tests/003 Thermal fleet - Must-run"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/003 Thermal fleet - Must-run"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -31,7 +31,7 @@ Feature: short tests
 
   @fast @short
   Scenario: 004 Thermal fleet - Partial must-run
-    Given the study path is "short-tests/004 Thermal fleet - Partial must-run"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/004 Thermal fleet - Partial must-run"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -42,7 +42,7 @@ Feature: short tests
 
   @fast @short
   Scenario: 005 Thermal fleet - Minimum stable power and min up down times
-    Given the study path is "short-tests/005 Thermal fleet - Minimum stable power and min up down times"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/005 Thermal fleet - Minimum stable power and min up down times"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -61,7 +61,7 @@ Feature: short tests
   Scenario: 006 Thermal fleet - Extra costs
     # Like previous test, but with extra non-proportional (NP) costs
     # NP costs = 1756400 ; OP costs = 2.75816e+07 (like test 5) => Total cost = 2.9338e+07
-    Given the study path is "short-tests/006 Thermal fleet - Extra costs"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/006 Thermal fleet - Extra costs"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -82,7 +82,7 @@ Feature: short tests
     # This example is the first of a set of two that are comparing the two unit-commitment modes of Antares.
     # Fast mode
     # => overall cost is not great, there are a lot of startups, and min up & down time are considered equal
-    Given the study path is "short-tests/007 Thermal fleet - Fast unit commitment"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/007 Thermal fleet - Fast unit commitment"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -104,7 +104,7 @@ Feature: short tests
   Scenario: 008 Thermal fleet - Accurate unit commitment
     # Like previous test, but with unit commitment
     # => overall cost is better, there are less startups, and min up & down time are not equal
-    Given the study path is "short-tests/008 Thermal fleet - Accurate unit commitment"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/008 Thermal fleet - Accurate unit commitment"
     When I run antares simulator
     Then the simulation takes less than 5 seconds
     And the simulation succeeds
@@ -124,7 +124,7 @@ Feature: short tests
 
   @fast @short
   Scenario: 021 Four areas - DC law
-    Given the study path is "short-tests/021 Four areas - DC law"
+    Given the study path is "Antares_Simulator_Tests_NR/short-tests/021 Four areas - DC law"
     When I run antares simulator
     Then the simulation takes less than 20 seconds
     And the simulation succeeds

src/tests/cucumber/features/steps/common_steps/modeler_output_handler.py

@@ -0,0 +1,24 @@
+# Antares modeler outputs parsing
+
+import os
+import pandas as pd
+import configparser
+from enum import Enum
+
+
+class modeler_output_handler:
+
+    def __init__(self, study_output_path):
+        self.study_output_path = study_output_path
+        self.results = self.__read_csv("solution.csv")
+
+
+    def __read_csv(self, file_name) -> pd.DataFrame:
+        absolute_path = os.path.join(self.study_output_path, file_name.replace("/", os.sep))
+        return pd.read_csv(absolute_path, header=None, sep=' ', low_memory=False)
+
+    def get_optimal_value(self, var : str) -> float:
+        for row in self.results.iterrows():
+            if row[1][0] == var:
+                return row[1][1]
+        raise ValueError("Variable not found")

src/tests/cucumber/features/steps/common_steps/modeler_utils.py

@@ -0,0 +1,28 @@
+# Methods to run Antares modeler
+
+import subprocess
+import os
+
+from common_steps.modeler_output_handler import modeler_output_handler
+
+
+def run_modeler(context):
+    command = build_antares_modeler_command(context)
+    print(f"Running command: {command}")
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    out, err = process.communicate()
+    context.output_path = os.path.join( context.study_path , "output") # TODO : fixme parse_output_folder_from_logs(out)
+    context.return_code = process.returncode
+    context.moh = modeler_output_handler(context.output_path)
+
+
+def build_antares_modeler_command(context):
+    command = [context.config.userdata["antares-modeler"], str(context.study_path)]
+    return command
+
+
+def parse_output_folder_from_logs(logs: bytes) -> str:
+    for line in logs.splitlines():
+        if b'Output folder : ' in line:
+            return line.split(b'Output folder : ')[1].decode('ascii')
+    raise LookupError("Could not parse output folder in output logs")

src/tests/cucumber/features/steps/common_steps/steps.py

@@ -6,7 +6,10 @@
 from behave import *
 
 from common_steps.assertions import *
-from common_steps.simulator_utils import *
+from common_steps.simulator_utils import run_simulation
+from common_steps.modeler_utils import run_modeler
+
+from features.steps.common_steps.assertions import assert_double_close
 
 
 @given('the study path is "{string}"')
@@ -121,3 +124,13 @@ def check_pmin_pmax(context, area, prod_name, min_p, max_p):
             lambda n: n * max_p)).all(), f"max_p constraint not respected during year {year}"
         assert (actual_hourly_prod >= actual_n_dispatched_units.apply(
             lambda n: n * min_p)).all(), f"min_p constraint not respected during year {year}"
+
+
+@when("I run antares modeler")
+def run_antares_modeler(context):
+    run_modeler(context)
+
+
+@step('the optimal value of variable {var} is {value:g}')
+def modeler_var_optimal_value(context, var, value):
+    assert_double_close(value, context.moh.get_optimal_value(var), 1e-6)

src/tests/resources/modeler/epic2/us2.5/study_2.5.1/input/model-libraries/lib_example1.yml

@@ -0,0 +1,36 @@
+library:
+  id: lib_example1
+  description: test model library
+
+  models:
+    - id: node_2gen_1load
+      description: A simple node with two generators and one load
+      parameters:
+        - id: load
+          time-dependent: false
+          scenario-dependent: false
+        - id: gen1_max_p
+          time-dependent: false
+          scenario-dependent: false
+        - id: gen1_prop_cost
+          time-dependent: false
+          scenario-dependent: false
+        - id: gen2_max_p
+          time-dependent: false
+          scenario-dependent: false
+        - id: gen2_prop_cost
+          time-dependent: false
+          scenario-dependent: false
+      variables:
+        - id: gen1_p
+          lower-bound: 0
+          upper-bound: gen1_max_p
+          variable-type: continuous
+        - id: gen2_p
+          lower-bound: 0
+          upper-bound: gen2_max_p
+          variable-type: continuous
+      constraints:
+        - id: balance
+          expression: gen1_p + gen2_p - load = 0
+      objective: gen1_p * gen1_prop_cost + gen2_p * gen2_prop_cost