From ca6be320104fe1c5b66dcf811657357de7cf58d9 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 2 Nov 2022 15:06:36 +0100 Subject: [PATCH 01/19] ACL --- src/cpp/benders/benders_core/BendersBase.cpp | 2 +- src/cpp/benders/merge_mps/MergeMPS.cpp | 2 +- src/cpp/exe/lpnamer/CMakeLists.txt | 7 +- .../input_reader/VariableFileReader.cpp | 4 +- .../lpnamer/input_reader/VariableFileReader.h | 3 +- .../input_reader/WeightsFileWriter.cpp | 2 +- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 150 ++++++++++++++-- ...aresProblemToXpansionProblemTranslator.cpp | 61 +++++++ ...ntaresProblemToXpansionProblemTranslator.h | 16 ++ .../problem_modifier/ArchiveProblemWriter.cpp | 21 +++ .../problem_modifier/ArchiveProblemWriter.h | 18 ++ .../lpnamer/problem_modifier/CMakeLists.txt | 44 +++-- .../problem_modifier/IProblemProviderPort.h | 14 ++ .../IProblemVariablesProviderPort.h | 23 +++ .../lpnamer/problem_modifier/IProblemWriter.h | 12 ++ .../IXpansionProblemsProvider.h | 15 ++ .../LinkProblemsGenerator.cpp | 169 +++++++++--------- .../problem_modifier/LinkProblemsGenerator.h | 43 +++-- .../lpnamer/problem_modifier/LpsFromAntares.h | 159 ++++++++++++++++ .../MPSFileProblemProviderAdapter.cpp | 21 +++ .../MPSFileProblemProviderAdapter.h | 16 ++ .../problem_modifier/MasterGeneration.cpp | 2 +- .../ProblemVariablesFileAdapter.cpp | 47 +++++ .../ProblemVariablesFileAdapter.h | 26 +++ .../ProblemVariablesZipAdapter.cpp | 53 ++++++ .../ProblemVariablesZipAdapter.h | 28 +++ .../XpansionProblemsFromAntaresProvider.cpp | 31 ++++ .../XpansionProblemsFromAntaresProvider.h | 17 ++ .../ZipProblemProviderAdapter.cpp | 35 ++++ .../ZipProblemProviderAdapter.h | 22 +++ .../ZipProblemsProviderAdapter.cpp | 34 ++++ .../ZipProblemsProviderAdapter.h | 19 ++ 32 files changed, 983 insertions(+), 133 deletions(-) create mode 100644 src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.h create mode 100644 src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.h create mode 100644 src/cpp/lpnamer/problem_modifier/IProblemProviderPort.h create mode 100644 src/cpp/lpnamer/problem_modifier/IProblemVariablesProviderPort.h create mode 100644 src/cpp/lpnamer/problem_modifier/IProblemWriter.h create mode 100644 src/cpp/lpnamer/problem_modifier/IXpansionProblemsProvider.h create mode 100644 src/cpp/lpnamer/problem_modifier/LpsFromAntares.h create mode 100644 src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.h create mode 100644 src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.h create mode 100644 src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.h create mode 100644 src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.h create mode 100644 src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.h create mode 100644 src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.cpp create mode 100644 src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.h diff --git a/src/cpp/benders/benders_core/BendersBase.cpp b/src/cpp/benders/benders_core/BendersBase.cpp index d94a02093..9bc52722d 100644 --- a/src/cpp/benders/benders_core/BendersBase.cpp +++ b/src/cpp/benders/benders_core/BendersBase.cpp @@ -584,7 +584,7 @@ std::string BendersBase::status_from_criterion() const { */ std::filesystem::path BendersBase::GetSubproblemPath( std::string const &slave_name) const { - return std::filesystem::path(_options.INPUTROOT) / (slave_name + MPS_SUFFIX); + return std::filesystem::path(_options.INPUTROOT) / slave_name; } /*! diff --git a/src/cpp/benders/merge_mps/MergeMPS.cpp b/src/cpp/benders/merge_mps/MergeMPS.cpp index 198a71699..3bb6b6b5c 100644 --- a/src/cpp/benders/merge_mps/MergeMPS.cpp +++ b/src/cpp/benders/merge_mps/MergeMPS.cpp @@ -29,7 +29,7 @@ void MergeMPS::launch() { auto reader = ArchiveReader(inputRootDir / _options.MPS_ZIP_FILE); reader.Open(); for (auto const &kvp : input) { - auto problem_name(inputRootDir / (kvp.first + MPS_SUFFIX)); + auto problem_name(inputRootDir / (kvp.first)); SolverAbstract::Ptr solver_l = factory.create_solver(solver_to_use); solver_l->init(); solver_l->set_output_log_level(_options.LOG_LEVEL); diff --git a/src/cpp/exe/lpnamer/CMakeLists.txt b/src/cpp/exe/lpnamer/CMakeLists.txt index fb9252841..df7594402 100644 --- a/src/cpp/exe/lpnamer/CMakeLists.txt +++ b/src/cpp/exe/lpnamer/CMakeLists.txt @@ -14,9 +14,10 @@ add_executable (lp_namer ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ) -target_link_libraries (lp_namer - PRIVATE +target_link_libraries (lp_namer + PRIVATE ${PROJECT_NAME}::problem_generation_main - ${PROJECT_NAME}::lp_namer_helper) + ${PROJECT_NAME}::lp_namer_helper + Boost::serialization) install(TARGETS lp_namer DESTINATION bin) diff --git a/src/cpp/lpnamer/input_reader/VariableFileReader.cpp b/src/cpp/lpnamer/input_reader/VariableFileReader.cpp index 99526b571..91962734c 100644 --- a/src/cpp/lpnamer/input_reader/VariableFileReader.cpp +++ b/src/cpp/lpnamer/input_reader/VariableFileReader.cpp @@ -18,14 +18,14 @@ void updateMapColumn(const std::vector& links, int link_id, } VariableFileReader::VariableFileReader( - const std::string& fileName, const std::vector& links, + const std::filesystem::path& fileName, const std::vector& links, const VariableFileReadNameConfiguration& variable_name_config, ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) : logger_(logger) { std::string line; std::ifstream file(fileName.c_str()); if (!file.good()) { - auto errMsg = std::string("Unable to open '") + fileName + "'"; + auto errMsg = std::string("Unable to open '") + fileName.string() + "'"; (*logger_)(ProblemGenerationLog::LOGLEVEL::FATAL) << errMsg; throw std::runtime_error(errMsg); } diff --git a/src/cpp/lpnamer/input_reader/VariableFileReader.h b/src/cpp/lpnamer/input_reader/VariableFileReader.h index 17f162e93..7be51cf55 100644 --- a/src/cpp/lpnamer/input_reader/VariableFileReader.h +++ b/src/cpp/lpnamer/input_reader/VariableFileReader.h @@ -20,7 +20,8 @@ struct VariableFileReadNameConfiguration { class VariableFileReader { public: VariableFileReader( - const std::string& fileName, const std::vector& links, + const std::filesystem::path& fileName, + const std::vector& links, const VariableFileReadNameConfiguration& variable_name_config, ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); VariableFileReader( diff --git a/src/cpp/lpnamer/input_reader/WeightsFileWriter.cpp b/src/cpp/lpnamer/input_reader/WeightsFileWriter.cpp index 614257b59..b695793b6 100644 --- a/src/cpp/lpnamer/input_reader/WeightsFileWriter.cpp +++ b/src/cpp/lpnamer/input_reader/WeightsFileWriter.cpp @@ -39,7 +39,7 @@ void YearlyWeightsWriter::FillMpsWeightsMap() { auto year_index = std::find(active_years_.begin(), active_years_.end(), year) - active_years_.begin(); - mps_weights_[mps_file.stem()] = weights_vector_[year_index]; + mps_weights_[mps_file.filename()] = weights_vector_[year_index]; } zip_reader.Close(); zip_reader.Delete(); diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 04e38861b..45d5a999b 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -1,29 +1,29 @@ #include "RunProblemGeneration.h" -#include +#include +#include #include -#include #include "ActiveLinks.h" #include "AdditionalConstraints.h" -#include "Candidate.h" -#include "CandidatesINIReader.h" +#include "ArchiveProblemWriter.h" #include "Clock.h" #include "GeneralDataReader.h" #include "LauncherHelpers.h" #include "LinkProblemsGenerator.h" -#include "LinkProfileReader.h" #include "LpFilesExtractor.h" #include "MasterGeneration.h" #include "MasterProblemBuilder.h" #include "MpsTxtWriter.h" #include "ProblemGenerationExeOptions.h" +#include "ProblemVariablesZipAdapter.h" #include "Timer.h" #include "WeightsFileReader.h" #include "WeightsFileWriter.h" +#include "XpansionProblemsFromAntaresProvider.h" +#include "ZipProblemsProviderAdapter.h" #include "common_lpnamer.h" -#include "solver_utils.h" void ProcessWeights( const std::filesystem::path& xpansion_output_dir, @@ -92,15 +92,141 @@ void RunProblemGeneration( Couplings couplings; std::string solver_name = "CBC"; std::vector links = linkBuilder.getLinks(); + + auto const mps_file_name = xpansion_output_dir / common_lpnamer::MPS_TXT; + LinkProblemsGenerator linkProblemsGenerator(links, solver_name, logger, log_file_path, zip_mps); - linkProblemsGenerator.treatloop(xpansion_output_dir, antares_archive_path, - couplings); + auto mpsList = linkProblemsGenerator.readMPSList(mps_file_name); + + bool use_zip_implementation = true; + bool use_file_implementation = false; + if (use_zip_implementation) { + /* Instantiate Zip reader */ + auto reader = std::make_shared(antares_archive_path); + reader->Open(); + + /*Instantiate zip writer */ + auto lpDir_ = xpansion_output_dir / "lp"; + const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; + const auto tmpArchivePath = lpDir_ / tmpArchiveName; + auto writer = std::make_shared(tmpArchivePath); + writer->Open(); + + /* Zip problem writer */ + auto problem_writer = + std::make_shared(xpansion_output_dir, writer); + + /* Main stuff */ + std::vector problem_names; + std::transform(mpsList.begin(), mpsList.end(), + std::back_inserter(problem_names), + [](ProblemData const& data) { return data._problem_mps; }); + auto adapter = std::make_shared(lpDir_, reader, + problem_names); + std::vector> xpansion_problems = + adapter->provideProblems(solver_name); + + std::vector, ProblemData>> + problems_and_data; + for (int i = 0; i < xpansion_problems.size(); ++i) { + xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps; + problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i)); + } + std::for_each(std::execution::par, problems_and_data.begin(), + problems_and_data.end(), [&](const auto& problem_and_data) { + const auto& [problem, data] = problem_and_data; + + auto problem_variables_from_zip_adapter = + std::make_shared( + reader, data, links, logger); + linkProblemsGenerator.treat( + data._problem_mps, couplings, problem_writer, problem, + problem_variables_from_zip_adapter); + }); + + /* Clean up */ + reader->Close(); + reader->Delete(); + + std::filesystem::remove(antares_archive_path); + std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); + writer->Close(); + writer->Delete(); + } else if (use_file_implementation) { + /*Instantiate zip writer */ + auto lpDir_ = xpansion_output_dir / "lp"; + const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; + const auto tmpArchivePath = lpDir_ / tmpArchiveName; + std::filesystem::remove(tmpArchivePath); + auto writer = std::make_shared(tmpArchivePath); + writer->Open(); + + /* Zip problem writer */ + auto problem_writer = + std::make_shared(xpansion_output_dir, writer); + + /* Main stuff */ + linkProblemsGenerator.treatloop_files(xpansion_output_dir, couplings, + mpsList, problem_writer); + + std::filesystem::remove(antares_archive_path); + std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); + writer->Close(); + writer->Delete(); + } else { + std::filesystem::path path = + xpansion_output_dir.parent_path().parent_path() / + "fichierDeSerialisation"; + std::ifstream ifs(xpansion_output_dir.parent_path().parent_path() / + "fichierDeSerialisation"); + boost::archive::text_iarchive ia(ifs); + + LpsFromAntares lps; + ia >> lps; + lps._constant->Mdeb.push_back(lps._constant->NombreDeCoefficients); + + XpansionProblemsFromAntaresProvider adapter(lps); + auto xpansion_problems = adapter.provideProblems(solver_name); + std::vector, ProblemData>> + problems_and_data; + for (int i = 0; i < xpansion_problems.size(); ++i) { + xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps; + problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i)); + } + + auto reader = std::make_shared(antares_archive_path); + reader->Open(); + auto lpDir_ = xpansion_output_dir / "lp"; + const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; + const auto tmpArchivePath = lpDir_ / tmpArchiveName; + auto writer = std::make_shared(tmpArchivePath); + writer->Open(); + auto problem_writer = + std::make_shared(lpDir_, writer); + + std::for_each(std::execution::par, problems_and_data.begin(), + problems_and_data.end(), [&](const auto& problem_and_data) { + const auto& [problem, data] = problem_and_data; + + auto problem_variables_from_zip_adapter = + std::make_shared( + reader, data, links, logger); + linkProblemsGenerator.treat( + data._problem_mps, couplings, problem_writer, problem, + problem_variables_from_zip_adapter); + }); + /* Clean up */ + reader->Close(); + reader->Delete(); + + std::filesystem::remove(antares_archive_path); + std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); + writer->Close(); + writer->Delete(); + } MasterGeneration master_generation( xpansion_output_dir, links, additionalConstraints, couplings, master_formulation, solver_name, logger, log_file_path); - (*logger)(ProblemGenerationLog::LOGLEVEL::INFO) - << "Problem Generation ran in: " - << format_time_str(problem_generation_timer.elapsed()) << std::endl; -} +} \ No newline at end of file diff --git a/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp b/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp new file mode 100644 index 000000000..ad67600db --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp @@ -0,0 +1,61 @@ +// +// Created by marechaljas on 22/11/22. +// + +#include "AntaresProblemToXpansionProblemTranslator.h" + +#include "multisolver_interface/SolverFactory.h" +#include "solver_utils.h" + +std::shared_ptr +AntaresProblemToXpansionProblemTranslator::translateToXpansionProblem( + const LpsFromAntares& lps, unsigned int year, unsigned int week, + const std::string& solver_name) { + SolverFactory factory; + auto problem = std::make_shared(factory.create_solver(solver_name)); + problem->init(); + const auto& constant = lps._constant; + const auto& hebdo = lps._hebdo.at({year, week}); + + std::vector tmp(constant->NombreDeVariables, 0); + std::vector coltypes(constant->NombreDeVariables, 'C'); + // solver_addcols(problem, hebdo->CoutLineaire, tmp, {}, {}, hebdo->Xmin, + // hebdo->Xmax, coltypes, {}); + problem->add_cols(constant->NombreDeVariables, 0, hebdo->CoutLineaire.data(), + tmp.data(), {}, {}, hebdo->Xmin.data(), hebdo->Xmax.data()); + problem->add_rows( + constant->NombreDeContraintes, constant->NombreDeCoefficients, + convertSignToLEG(hebdo->Sens.data()).data(), hebdo->SecondMembre.data(), + {}, constant->Mdeb.data(), constant->IndicesColonnes.data(), + constant->CoefficientsDeLaMatriceDesContraintes.data()); + auto rows = problem->get_nrows(); + auto cols = problem->get_ncols(); + auto elem = problem->get_nelems(); + // On peut ajouter la partie qui renomme les variables ici si on stocke les + // données du type de variables dans ConstantDataFromAntares, i.e. en + // définissant une autre implémentation de IProblemVariablesProviderPort + return problem; +} +std::vector AntaresProblemToXpansionProblemTranslator::convertSignToLEG( + char* data) { + std::vector LEG_vector; + char c = *data; + while (c != '\0') { + if ('=' == c) { + LEG_vector.push_back('E'); + c = *++data; + continue; + } else if ('<' == c) { + LEG_vector.push_back('L'); + c = *++data; + continue; + } else if ('>' == c) { + LEG_vector.push_back('G'); + c = *++data; + continue; + } else { + throw std::runtime_error(&"Bad character parsing "[c]); + } + } + return LEG_vector; +} diff --git a/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.h b/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.h new file mode 100644 index 000000000..47d9fa98c --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.h @@ -0,0 +1,16 @@ +// +// Created by marechaljas on 22/11/22. +// + +#pragma once + +#include "../model/Problem.h" +#include "LpsFromAntares.h" + +class AntaresProblemToXpansionProblemTranslator { + public: + [[nodiscard]] static std::shared_ptr translateToXpansionProblem( + const LpsFromAntares& lps, unsigned int year, unsigned int week, + const std::string& solver_name); + static std::vector convertSignToLEG(char* data); +}; diff --git a/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp b/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp new file mode 100644 index 000000000..02feb442a --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp @@ -0,0 +1,21 @@ +// +// Created by marechaljas on 08/11/22. +// + +#include "ArchiveProblemWriter.h" + +#include + +#include "IProblemWriter.h" +#include "LinkProblemsGenerator.h" + +void ArchiveProblemWriter::Write_problem(std::shared_ptr &in_prblm) { + auto const lp_mps_name = lp_dir_ / in_prblm->_name; + in_prblm->write_prob_mps(lp_mps_name); + writer_->AddFileInArchive(lp_mps_name); + std::filesystem::remove(lp_mps_name); +} + +ArchiveProblemWriter::ArchiveProblemWriter( + std::filesystem::path lp_dir, std::shared_ptr writer) + : lp_dir_(std::move(lp_dir)), writer_(std::move(writer)) {} diff --git a/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.h b/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.h new file mode 100644 index 000000000..13dccc693 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.h @@ -0,0 +1,18 @@ +// +// Created by marechaljas on 08/11/22. +// + +#pragma once + +#include "IProblemWriter.h" +#include "LinkProblemsGenerator.h" + +class ArchiveProblemWriter : public IProblemWriter { + void Write_problem(std::shared_ptr& in_prblm) override; + + public: + ArchiveProblemWriter(std::filesystem::path lp_dir, + std::shared_ptr writer); + std::filesystem::path lp_dir_; + std::shared_ptr writer_; +}; diff --git a/src/cpp/lpnamer/problem_modifier/CMakeLists.txt b/src/cpp/lpnamer/problem_modifier/CMakeLists.txt index c25f22a5d..48abd0628 100644 --- a/src/cpp/lpnamer/problem_modifier/CMakeLists.txt +++ b/src/cpp/lpnamer/problem_modifier/CMakeLists.txt @@ -18,17 +18,39 @@ add_library(lp_namer_problem_modifier STATIC ${CMAKE_CURRENT_SOURCE_DIR}/LinkdataRecord.cpp ${CMAKE_CURRENT_SOURCE_DIR}/LinkProblemsGenerator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/LinkProblemsGenerator.h - ${CMAKE_CURRENT_SOURCE_DIR}/AdditionalConstraints.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/AdditionalConstraints.h - ${CMAKE_CURRENT_SOURCE_DIR}/ProblemModifier.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ProblemModifier.h - ${CMAKE_CURRENT_SOURCE_DIR}/MasterProblemBuilder.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/MasterProblemBuilder.h - ${CMAKE_CURRENT_SOURCE_DIR}/LauncherHelpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/LauncherHelpers.h - ${CMAKE_CURRENT_SOURCE_DIR}/MasterGeneration.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/MasterGeneration.h - ${CMAKE_CURRENT_SOURCE_DIR}/LinkdataRecord.h) + ${CMAKE_CURRENT_SOURCE_DIR}/AdditionalConstraints.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/AdditionalConstraints.h + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemModifier.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemModifier.h + ${CMAKE_CURRENT_SOURCE_DIR}/MasterProblemBuilder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/MasterProblemBuilder.h + ${CMAKE_CURRENT_SOURCE_DIR}/LauncherHelpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/LauncherHelpers.h + ${CMAKE_CURRENT_SOURCE_DIR}/MasterGeneration.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/MasterGeneration.h + ${CMAKE_CURRENT_SOURCE_DIR}/LinkdataRecord.h + ${CMAKE_CURRENT_SOURCE_DIR}/ZipProblemProviderAdapter.h + ${CMAKE_CURRENT_SOURCE_DIR}/ZipProblemProviderAdapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/IProblemProviderPort.h + ${CMAKE_CURRENT_SOURCE_DIR}/IProblemWriter.h + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveProblemWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveProblemWriter.h + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemVariablesZipAdapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemVariablesZipAdapter.h + ${CMAKE_CURRENT_SOURCE_DIR}/IProblemVariablesProviderPort.h + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemVariablesFileAdapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ProblemVariablesFileAdapter.h + ${CMAKE_CURRENT_SOURCE_DIR}/MPSFileProblemProviderAdapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/MPSFileProblemProviderAdapter.h + ${CMAKE_CURRENT_SOURCE_DIR}/LpsFromAntares.h + ${CMAKE_CURRENT_SOURCE_DIR}/XpansionProblemsFromAntaresProvider.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/XpansionProblemsFromAntaresProvider.h + ${CMAKE_CURRENT_SOURCE_DIR}/AntaresProblemToXpansionProblemTranslator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/AntaresProblemToXpansionProblemTranslator.h + ${CMAKE_CURRENT_SOURCE_DIR}/IXpansionProblemsProvider.h + ${CMAKE_CURRENT_SOURCE_DIR}/ZipProblemsProviderAdapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ZipProblemsProviderAdapter.h + ) target_include_directories (lp_namer_problem_modifier PUBLIC diff --git a/src/cpp/lpnamer/problem_modifier/IProblemProviderPort.h b/src/cpp/lpnamer/problem_modifier/IProblemProviderPort.h new file mode 100644 index 000000000..3b02f0e0f --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/IProblemProviderPort.h @@ -0,0 +1,14 @@ +// +// Created by marechaljas on 03/11/22. +// + +#pragma once + +#include "../../lpnamer/model/Problem.h" + +class IProblemProviderPort { + public: + virtual ~IProblemProviderPort() = default; + [[nodiscard]] virtual std::shared_ptr provide_problem( + const std::string& solver_name) const = 0; +}; diff --git a/src/cpp/lpnamer/problem_modifier/IProblemVariablesProviderPort.h b/src/cpp/lpnamer/problem_modifier/IProblemVariablesProviderPort.h new file mode 100644 index 000000000..dac7f5b0a --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/IProblemVariablesProviderPort.h @@ -0,0 +1,23 @@ +// +// Created by marechaljas on 08/11/22. +// + +#pragma once +#include +#include +#include + +#include "ColumnToChange.h" + +struct ProblemVariables { + std::vector variable_names; + std::map ntc_columns; + std::map direct_cost_columns; + std::map indirect_cost_columns; +}; + +class IProblemVariablesProviderPort { + public: + virtual ~IProblemVariablesProviderPort() = default; + virtual ProblemVariables Provide() = 0; +}; \ No newline at end of file diff --git a/src/cpp/lpnamer/problem_modifier/IProblemWriter.h b/src/cpp/lpnamer/problem_modifier/IProblemWriter.h new file mode 100644 index 000000000..bc60cab79 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/IProblemWriter.h @@ -0,0 +1,12 @@ +// +// Created by marechaljas on 08/11/22. +// + +#pragma once + +#include "../../lpnamer/model/Problem.h" +class IProblemWriter { + public: + virtual ~IProblemWriter() = default; + virtual void Write_problem(std::shared_ptr& in_prblm) = 0; +}; diff --git a/src/cpp/lpnamer/problem_modifier/IXpansionProblemsProvider.h b/src/cpp/lpnamer/problem_modifier/IXpansionProblemsProvider.h new file mode 100644 index 000000000..312e15296 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/IXpansionProblemsProvider.h @@ -0,0 +1,15 @@ +// +// Created by marechaljas on 25/11/22. +// + +#pragma once + +#include "../model/Problem.h" +#include "LpsFromAntares.h" + +class IXpansionProblemsProvider { + public: + virtual ~IXpansionProblemsProvider() = default; + [[nodiscard]] virtual std::vector> provideProblems( + const std::string &solver_name) const = 0; +}; diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp index b5eb9be27..41ea68a7f 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp @@ -1,9 +1,15 @@ #include "LinkProblemsGenerator.h" #include -#include #include +#include "IProblemProviderPort.h" +#include "IProblemVariablesProviderPort.h" +#include "IProblemWriter.h" +#include "MPSFileProblemProviderAdapter.h" +#include "ProblemVariablesFileAdapter.h" +#include "ProblemVariablesZipAdapter.h" +#include "ZipProblemProviderAdapter.h" #include "MpsTxtWriter.h" #include "VariableFileReader.h" #include "common_lpnamer.h" @@ -19,77 +25,69 @@ * correspondence between optimizer variables and interconnection candidates * \return void */ -void LinkProblemsGenerator::treat(const std::filesystem::path &root, - ProblemData const &problemData, - Couplings &couplings, - ArchiveReader &reader) const { - // get path of file problem***.mps, variable***.txt and constraints***.txt - auto const lp_mps_name = lpDir_ / problemData._problem_mps; - // List of variables - VariableFileReadNameConfiguration variable_name_config; - variable_name_config.ntc_variable_name = "ValeurDeNTCOrigineVersExtremite"; - variable_name_config.cost_origin_variable_name = - "CoutOrigineVersExtremiteDeLInterconnexion"; - variable_name_config.cost_extremite_variable_name = - "CoutExtremiteVersOrigineDeLInterconnexion"; - auto variableFileContent = - reader.ExtractFileInStringStream(problemData._variables_txt); - auto variableReader = VariableFileReader(variableFileContent, _links, - variable_name_config, logger_); +void LinkProblemsGenerator::treat( + const std::string &problem_name, Couplings &couplings, + std::shared_ptr writer, + std::shared_ptr problem_provider, + std::shared_ptr variable_provider) const { + std::shared_ptr in_prblm = + problem_provider->provide_problem(_solver_name); - std::vector var_names = variableReader.getVariables(); - std::map p_ntc_columns = - variableReader.getNtcVarColumns(); - std::map p_direct_cost_columns = - variableReader.getDirectCostVarColumns(); - std::map p_indirect_cost_columns = - variableReader.getIndirectCostVarColumns(); + ProblemVariables problem_variables = variable_provider->Provide(); + auto problem_modifier = ProblemModifier(logger_); + // auto path = + // "/tmp/pytest-of-marechaljas/zip-1/test_full_study_short_sequenti0/link-profile-with-empty-week/output/out/lp/zip/pre_change" + // + in_prblm->_name; in_prblm->write_prob_mps(path); + solver_rename_vars(in_prblm, problem_variables.variable_names); + + in_prblm = problem_modifier.changeProblem( + in_prblm, _links, problem_variables.ntc_columns, + problem_variables.direct_cost_columns, + problem_variables.indirect_cost_columns); + + // couplings creation + for (const ActiveLink &link : _links) { + for (const Candidate &candidate : link.getCandidates()) { + if (problem_modifier.has_candidate_col_id(candidate.get_name())) { + std::lock_guard guard(coupling_mutex_); + couplings[{candidate.get_name(), problem_name}] = + problem_modifier.get_candidate_col_id(candidate.get_name()); + } + } + } + writer->Write_problem(in_prblm); +} + +void LinkProblemsGenerator::treat( + const std::string &problem_name, Couplings &couplings, + std::shared_ptr writer, std::shared_ptr problem, + std::shared_ptr variable_provider) const { + ProblemVariables problem_variables = variable_provider->Provide(); - SolverFactory factory; - auto in_prblm = std::make_shared( - factory.create_solver(_solver_name, log_file_path_)); - reader.ExtractFile(problemData._problem_mps, lpDir_); - in_prblm->read_prob_mps(lp_mps_name); + // auto path = + // "/tmp/pytest-of-marechaljas/zip-1/test_full_study_short_sequenti0/link-profile-with-empty-week/output/out/lp/memory/pre_change" + // + problem->_name; problem->write_prob_mps(path); - solver_rename_vars(in_prblm, var_names); + solver_rename_vars(problem, problem_variables.variable_names); auto problem_modifier = ProblemModifier(logger_); - in_prblm = problem_modifier.changeProblem( - std::move(in_prblm), _links, p_ntc_columns, p_direct_cost_columns, - p_indirect_cost_columns); + auto in_prblm = problem_modifier.changeProblem( + problem, _links, problem_variables.ntc_columns, + problem_variables.direct_cost_columns, + problem_variables.indirect_cost_columns); // couplings creation for (const ActiveLink &link : _links) { for (const Candidate &candidate : link.getCandidates()) { if (problem_modifier.has_candidate_col_id(candidate.get_name())) { std::lock_guard guard(coupling_mutex_); - couplings[{candidate.get_name(), lp_mps_name}] = + couplings[{candidate.get_name(), problem_name}] = problem_modifier.get_candidate_col_id(candidate.get_name()); } } } - - in_prblm->write_prob_mps(lp_mps_name); -} -/** - * \brief That function create new optimization problems with new candidates - * - * \param root String corresponding to the path where are located input data - * \param mps Strings vector with the list of mps files - * \param couplings map of pair of strings associated to an int. Determine the - * correspondence between optimizer variables and interconnection candidates - * \return void - */ -void LinkProblemsGenerator::treat(const std::filesystem::path &root, - ProblemData const &problemData, - Couplings &couplings, ArchiveReader &reader, - ArchiveWriter &writer) const { - // get path of file problem***.mps, variable***.txt and constraints***.txt - auto const lp_mps_name = lpDir_ / problemData._problem_mps; - treat(root, problemData, couplings, reader); - writer.AddFileInArchive(lp_mps_name); - std::filesystem::remove(lp_mps_name); + writer->Write_problem(in_prblm); } /** @@ -101,34 +99,35 @@ void LinkProblemsGenerator::treat(const std::filesystem::path &root, * \return void */ void LinkProblemsGenerator::treatloop(const std::filesystem::path &root, - const std::filesystem::path &archivePath, - Couplings &couplings) { - auto const mps_file_name = root / common_lpnamer::MPS_TXT; - - auto files_mapper = FilesMapper(archivePath); - auto mpsList = files_mapper.MpsAndVariablesFilesVect(); + Couplings &couplings, + const std::vector &mps_list, + std::shared_ptr writer, + std::shared_ptr reader) { + std::for_each(std::execution::par, mps_list.begin(), mps_list.end(), + [&](const auto &mps) { + auto adapter = std::make_shared( + root, mps._problem_mps, reader); + auto problem_variables_from_zip_adapter = + std::make_shared( + reader, mps, _links, logger_); + treat(mps._problem_mps, couplings, writer, adapter, + problem_variables_from_zip_adapter); + }); +} - lpDir_ = root / "lp"; - auto reader = ArchiveReader(archivePath); - reader.Open(); - if (zip_mps_) { - const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; - const auto tmpArchivePath = lpDir_ / tmpArchiveName; - auto writer = ArchiveWriter(tmpArchivePath); - writer.Open(); - std::for_each( - std::execution::par, mpsList.begin(), mpsList.end(), - [&](const auto &mps) { treat(root, mps, couplings, reader, writer); }); - writer.Close(); - writer.Delete(); - reader.Close(); - reader.Delete(); - std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); - } else { - std::for_each( - std::execution::par, mpsList.begin(), mpsList.end(), - [&](const auto &mps) { treat(root, mps, couplings, reader); }); - reader.Close(); - reader.Delete(); - } +void LinkProblemsGenerator::treatloop_files( + const std::filesystem::path &root, Couplings &couplings, + const std::vector &mps_list, + std::shared_ptr writer) { + std::for_each(std::execution::par, mps_list.begin(), mps_list.end(), + [&](const auto &mps) { + auto adapter = + std::make_shared( + root, mps._problem_mps); + auto variables_file_adapter = + std::make_shared( + mps, _links, logger_, root); + treat(mps._problem_mps, couplings, writer, adapter, + variables_file_adapter); + }); } diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h index 47f286f45..d6d450381 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h @@ -6,14 +6,16 @@ #include #include -#include "ActiveLinks.h" #include "ArchiveReader.h" #include "ArchiveWriter.h" -#include "Candidate.h" #include "FileInBuffer.h" +#include "IProblemProviderPort.h" +#include "IProblemVariablesProviderPort.h" +#include "IProblemWriter.h" #include "MpsTxtWriter.h" #include "ProblemGenerationLogger.h" #include "ProblemModifier.h" +#include "VariableFileReader.h" #include "common_lpnamer.h" const std::string CANDIDATES_INI{"candidates.ini"}; @@ -21,10 +23,9 @@ const std::string STRUCTURE_FILE{"structure.txt"}; const std::string MPS_ZIP_FILE{"MPS_ZIP_FILE"}; const std::string ZIP_EXT{".zip"}; const std::string STUDY_FILE{"study.antares"}; -typedef std::pair - CandidateNameAndMpsFilePath; -typedef unsigned int ColId; -typedef std::map Couplings; +using CandidateNameAndProblemName = std::pair; +using ColId = unsigned int; +using Couplings = std::map; class LinkProblemsGenerator { public: @@ -38,17 +39,20 @@ class LinkProblemsGenerator { log_file_path_(log_file_path), zip_mps_(zip_mps) {} - void treatloop(const std::filesystem::path& root, - const std::filesystem::path& archivePath, - Couplings& couplings); + void treatloop(const std::filesystem::path& root, Couplings& couplings, + const std::vector& mps_list, + std::shared_ptr writer, + std::shared_ptr reader); + std::vector readMPSList( + const std::filesystem::path& mps_filePath_p) const; - private: - void treat(const std::filesystem::path& root, ProblemData const&, - Couplings& couplings, ArchiveReader& reader) const; - void treat(const std::filesystem::path& root, ProblemData const&, - Couplings& couplings, ArchiveReader& reader, - ArchiveWriter& writer) const; + void treat( + const std::string& problem_name, Couplings& couplings, + std::shared_ptr writer, + std::shared_ptr problem_provider, + std::shared_ptr variable_provider) const; + private: const std::vector& _links; std::string _solver_name; std::filesystem::path lpDir_ = ""; @@ -56,4 +60,13 @@ class LinkProblemsGenerator { mutable std::mutex coupling_mutex_; std::filesystem::path log_file_path_; bool zip_mps_ = false; + + public: + void treatloop_files(const std::filesystem::path& root, Couplings& couplings, + const std::vector& mps_list, + std::shared_ptr writer); + void treat( + const std::string& problem_name, Couplings& couplings, + std::shared_ptr writer, std::shared_ptr problem, + std::shared_ptr variable_provider) const; }; diff --git a/src/cpp/lpnamer/problem_modifier/LpsFromAntares.h b/src/cpp/lpnamer/problem_modifier/LpsFromAntares.h new file mode 100644 index 000000000..c46a7b298 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/LpsFromAntares.h @@ -0,0 +1,159 @@ +// Je modifie la structure de Manuel en fonction des remarques que je vous ai +// faites vendredi + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ProblemHebdoId { + unsigned int year; + unsigned int week; + + bool operator<(const ProblemHebdoId& other) const { + if (year < other.year) return true; + if (year == other.year) return week < other.week; + return false; + } + + friend class boost::serialization::access; + template + void serialize(Archive& ar, [[maybe_unused]] const unsigned int version) { + ar& year; + ar& week; + } +}; + +class ConstantDataFromAntares; +class HebdoDataFromAntares; + +using ConstantDataFromAntaresPtr = std::shared_ptr; +using HebdoDataFromAntaresPtr = std::shared_ptr; + +// Type de données inutile car les matrices de tous les pbs hebdo sont +// identiques. Cela pourra changer à l'avenir si des coefficients de contraintes +// couplantes peuvent varier au cours du temps (ex: rendement d'une pompe à +// chaleur qui varie selon la température, FlowBased ?, etc) +using YearWeekHebdoDataFromAntares = + std::map; + +class ConstantDataFromAntares { + public: + int NombreDeVariables; // Mathématiquement : Nb colonnes de la matrice, + // Informatiquement = TypeDeVariable.size() + int NombreDeContraintes; // Mathématiqument : Nb lignes de la matrice, + // Informatiquement = Mdeb.size() + int NombreDeCoefficients; // Mathématiquement : Nb coeffs non nuls de la + // matrice, Informatiquement = Nbterm.size() = + // IndicesColonnes.size()= + // CoefficientsDeLaMatriceDesContraintes.size() + + std::vector TypeDeVariable; // Variables entières ou biniaires + std::vector + Mdeb; // Indique dans les indices dans le vecteur IndicesColonnes qui + // correspondent au début de chaque ligne. Ex : Mdeb[3] = 8 et + // Mdeb[4] = 13 -> Les termes IndicesColonnes[8] à + // IndicesColonnes[12] correspondent à des Id de colonnes de la + // ligne 3 de la matrice (en supposant que les lignes sont indexées + // à partir de 0) + std::vector Nbterm; // Nombre de termes non nuls sur chaque ligne. + // Inutile car NbTerm[i] = Mdeb[i+1] - Mdeb[i] + std::vector IndicesColonnes; // Id des colonnes des termes de + // CoefficientsDeLaMatriceDesContraintes : Ex + // IndicesColonnes[3] = 8 -> + // CoefficientsDeLaMatriceDesContraintes[8] donne la + // valeur du terme de la colonne 8, et de la ligne i où + // i est tel que Mdeb[i] <= 3 < Mdeb[i+1] + std::vector + CoefficientsDeLaMatriceDesContraintes; // Coefficients de la matrice + + std::vector + SignificationMetierDesVariables; // Nouveau par rapport à ce qu'à fait + // Manuel, contient les infos + // actuellement dans variables.txt. -> + // Utile pour l'étape + // problem_generation. Comme ce fichier + // contient des infos de type + // {signification metier, id zone, pas + // horaire} ou {signification metier, id + // lien, pas horaire}, peut-être qu'un + // vecteur de string n'est pas le + // meilleur choix (et il faudra aller + // vérifier précisément les infos dont + // on a besoin qui sont un peu + // différentes selon les variables) + std::vector + SignificationMetierDesContraintes; // Nouveau mais pas utile dans + // l'immédiat (y penser lorsqu'on + // essaiera d'investir directement sur + // les noeuds ou tester des choses + // plus complexes sur la gestion des + // stocks), idem que précédemment mais + // pour les contraintes, nécessite de + // définir précisément les infos dont + // on a besoin -> à faire lorsqu'on + // fera le nommage des contraintes + // dans Antarès ? + + friend class boost::serialization::access; + template + void serialize(Archive& ar, [[maybe_unused]] const unsigned int version) { + ar& NombreDeVariables; + ar& NombreDeContraintes; + ar& NombreDeCoefficients; + ar& TypeDeVariable; + ar& Mdeb; + ar& Nbterm; + ar& IndicesColonnes; + ar& CoefficientsDeLaMatriceDesContraintes; + ar& SignificationMetierDesVariables; + ar& SignificationMetierDesContraintes; + } +}; + +class HebdoDataFromAntares { + public: + std::vector Sens; // Sens de la contrainte : < ou > ou =, taille = + // NombreDeContraintes + std::vector Xmax; // Borne max des variables de la semaine + // considérée, taille = NombreDeVariables + std::vector Xmin; // Borne min des variables de la semaine + // considérée, taille = NombreDeVariables + std::vector + CoutLineaire; // Coefficients du vecteur de coût de la fonction objectif, + // taille = NombreDeVariables + std::vector + SecondMembre; // Vecteur des second membre des contraintes, taille = + // NombreDeContraintes + + friend class boost::serialization::access; + template + void serialize(Archive& ar, [[maybe_unused]] const unsigned int version) { + ar& Sens; + ar& Xmax; + ar& Xmin; + ar& CoutLineaire; + ar& SecondMembre; + } +}; + +class LpsFromAntares { + public: + ConstantDataFromAntaresPtr _constant; + YearWeekHebdoDataFromAntares _hebdo; + + friend class boost::serialization::access; + template + void serialize(Archive& ar, [[maybe_unused]] const unsigned int version) { + ar& _constant; + ar& _hebdo; + } +}; diff --git a/src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.cpp b/src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.cpp new file mode 100644 index 000000000..ba1d170fd --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.cpp @@ -0,0 +1,21 @@ +// +// Created by marechaljas on 09/11/22. +// + +#include "MPSFileProblemProviderAdapter.h" + +#include + +#include "multisolver_interface/SolverFactory.h" +std::shared_ptr MPSFileProblemProviderAdapter::provide_problem( + const std::string& solver_name) const { + SolverFactory factory; + auto const lp_mps_name = lp_dir_ / problem_name_; + auto in_prblm = std::make_shared(factory.create_solver(solver_name)); + + in_prblm->read_prob_mps(lp_mps_name); + return in_prblm; +} +MPSFileProblemProviderAdapter::MPSFileProblemProviderAdapter( + std::filesystem::path root, const std::string& problem_name) + : lp_dir_(std::move(root)), problem_name_(problem_name) {} diff --git a/src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.h b/src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.h new file mode 100644 index 000000000..5c49ef421 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/MPSFileProblemProviderAdapter.h @@ -0,0 +1,16 @@ +// +// Created by marechaljas on 09/11/22. +// + +#pragma once + +#include "IProblemProviderPort.h" +class MPSFileProblemProviderAdapter : public IProblemProviderPort { + public: + MPSFileProblemProviderAdapter(std::filesystem::path root, + const std::string& problem_name); + [[nodiscard]] std::shared_ptr provide_problem( + const std::string& solver_name) const override; + const std::filesystem::path lp_dir_; + const std::string& problem_name_; +}; diff --git a/src/cpp/lpnamer/problem_modifier/MasterGeneration.cpp b/src/cpp/lpnamer/problem_modifier/MasterGeneration.cpp index 78e8ada93..59001d5df 100644 --- a/src/cpp/lpnamer/problem_modifier/MasterGeneration.cpp +++ b/src/cpp/lpnamer/problem_modifier/MasterGeneration.cpp @@ -50,7 +50,7 @@ void MasterGeneration::write_structure_file( const std::filesystem::path &rootPath, const Couplings &couplings) const { std::map> output; for (const auto &[candidate_name_and_mps_filePath, colId] : couplings) { - output[candidate_name_and_mps_filePath.second.stem().string()] + output[candidate_name_and_mps_filePath.second] [candidate_name_and_mps_filePath.first] = colId; } unsigned int i = 0; diff --git a/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp b/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp new file mode 100644 index 000000000..5ca2be7f9 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.cpp @@ -0,0 +1,47 @@ +// +// Created by marechaljas on 09/11/22. +// + +#include "ProblemVariablesFileAdapter.h" + +#include + +#include "VariableFileReader.h" + +void ProblemVariablesFileAdapter::extract_variables( + const std::filesystem::path& file, std::vector& var_names, + std::map& p_ntc_columns, + std::map& p_direct_cost_columns, + std::map& p_indirect_cost_columns) const { + // List of variables + VariableFileReadNameConfiguration variable_name_config; + variable_name_config.ntc_variable_name = "ValeurDeNTCOrigineVersExtremite"; + variable_name_config.cost_origin_variable_name = + "CoutOrigineVersExtremiteDeLInterconnexion"; + variable_name_config.cost_extremite_variable_name = + "CoutExtremiteVersOrigineDeLInterconnexion"; + + VariableFileReader variableReader(file, active_links_, variable_name_config, + logger_); + var_names = variableReader.getVariables(); + p_ntc_columns = variableReader.getNtcVarColumns(); + p_direct_cost_columns = variableReader.getDirectCostVarColumns(); + p_indirect_cost_columns = variableReader.getIndirectCostVarColumns(); +} + +ProblemVariables ProblemVariablesFileAdapter::Provide() { + ProblemVariables result; + extract_variables(lpdir_ / problem_data_._variables_txt, + result.variable_names, result.ntc_columns, + result.direct_cost_columns, result.indirect_cost_columns); + return result; +} + +ProblemVariablesFileAdapter::ProblemVariablesFileAdapter( + ProblemData data, std::vector vector_1, + std::shared_ptr shared_ptr_1, + std::filesystem::path path) + : problem_data_(std::move(data)), + active_links_(std::move(vector_1)), + logger_(std::move(shared_ptr_1)), + lpdir_(std::move(path)) {} diff --git a/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.h b/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.h new file mode 100644 index 000000000..295fbd870 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ProblemVariablesFileAdapter.h @@ -0,0 +1,26 @@ +// +// Created by marechaljas on 09/11/22. +// + +#pragma once + +#include "IProblemVariablesProviderPort.h" +#include "LinkProblemsGenerator.h" +class ProblemVariablesFileAdapter : public IProblemVariablesProviderPort { + public: + ProblemVariablesFileAdapter( + ProblemData data, std::vector vector_1, + std::shared_ptr + shared_ptr_1, + std::filesystem::path path); + ProblemVariables Provide() override; + const ProblemData problem_data_; + const std::vector active_links_; + std::shared_ptr logger_; + void extract_variables( + const std::filesystem::path& file, std::vector& var_names, + std::map& p_ntc_columns, + std::map& p_direct_cost_columns, + std::map& p_indirect_cost_columns) const; + const std::filesystem::path lpdir_; +}; diff --git a/src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.cpp b/src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.cpp new file mode 100644 index 000000000..491707c32 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.cpp @@ -0,0 +1,53 @@ +// +// Created by marechaljas on 08/11/22. +// + +#include "ProblemVariablesZipAdapter.h" + +#include + +#include "ArchiveReader.h" +#include "LinkProblemsGenerator.h" +#include "VariableFileReader.h" + +void ProblemVariablesZipAdapter::extract_variables( + std::istringstream& variableFileContent, + std::vector& var_names, + std::map& p_ntc_columns, + std::map& p_direct_cost_columns, + std::map& p_indirect_cost_columns) const { + // List of variables + VariableFileReadNameConfiguration variable_name_config; + variable_name_config.ntc_variable_name = "ValeurDeNTCOrigineVersExtremite"; + variable_name_config.cost_origin_variable_name = + "CoutOrigineVersExtremiteDeLInterconnexion"; + variable_name_config.cost_extremite_variable_name = + "CoutExtremiteVersOrigineDeLInterconnexion"; + + auto variableReader = VariableFileReader(variableFileContent, links_, + variable_name_config, logger_); + var_names = variableReader.getVariables(); + p_ntc_columns = variableReader.getNtcVarColumns(); + p_direct_cost_columns = variableReader.getDirectCostVarColumns(); + p_indirect_cost_columns = variableReader.getIndirectCostVarColumns(); +} + +ProblemVariables ProblemVariablesZipAdapter::Provide() { + auto variableFileContent = + archive_reader_->ExtractFileInStringStream(problem_data_._variables_txt); + + ProblemVariables result; + extract_variables(variableFileContent, result.variable_names, + result.ntc_columns, result.direct_cost_columns, + result.indirect_cost_columns); + + return result; +} +ProblemVariablesZipAdapter::ProblemVariablesZipAdapter( + std::shared_ptr reader, ProblemData data, + const std::vector& vector, + std::shared_ptr ptr) + : archive_reader_(std::move(reader)), + problem_data_(std::move(data)), + links_(vector), + logger_(std::move(ptr)) {} diff --git a/src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.h b/src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.h new file mode 100644 index 000000000..55b57a160 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ProblemVariablesZipAdapter.h @@ -0,0 +1,28 @@ +// +// Created by marechaljas on 08/11/22. +// + +#pragma once + +#include "IProblemVariablesProviderPort.h" +#include "LinkProblemsGenerator.h" + +class ProblemVariablesZipAdapter : public IProblemVariablesProviderPort { + public: + ProblemVariablesZipAdapter( + std::shared_ptr reader, ProblemData data, + const std::vector& vector, + std::shared_ptr ptr); + + ProblemVariables Provide() override; + std::shared_ptr archive_reader_; + const ProblemData problem_data_; + const std::vector& links_; + const std::shared_ptr logger_; + void extract_variables( + std::istringstream& variableFileContent, + std::vector& var_names, + std::map& p_ntc_columns, + std::map& p_direct_cost_columns, + std::map& p_indirect_cost_columns) const; +}; diff --git a/src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.cpp b/src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.cpp new file mode 100644 index 000000000..8bc9c5b06 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.cpp @@ -0,0 +1,31 @@ +// +// Created by marechaljas on 18/11/22. +// + +#include "XpansionProblemsFromAntaresProvider.h" + +#include + +#include "../model/Problem.h" +#include "AntaresProblemToXpansionProblemTranslator.h" + +XpansionProblemsFromAntaresProvider::XpansionProblemsFromAntaresProvider( + LpsFromAntares lps) + : antares_hebdo_problems(std::move(lps)) {} + +std::vector> +XpansionProblemsFromAntaresProvider::provideProblems( + const std::string& solver_name) const { + std::vector> xpansion_problems; + xpansion_problems.reserve( + XpansionProblemsFromAntaresProvider::antares_hebdo_problems._hebdo + .size()); + for (const auto& [problem_id, hebdo_data] : + XpansionProblemsFromAntaresProvider::antares_hebdo_problems._hebdo) { + xpansion_problems.push_back( + AntaresProblemToXpansionProblemTranslator::translateToXpansionProblem( + XpansionProblemsFromAntaresProvider::antares_hebdo_problems, + problem_id.year, problem_id.week, solver_name)); + } + return xpansion_problems; +} \ No newline at end of file diff --git a/src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.h b/src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.h new file mode 100644 index 000000000..c277b8c53 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/XpansionProblemsFromAntaresProvider.h @@ -0,0 +1,17 @@ +// +// Created by marechaljas on 18/11/22. +// + +#pragma once + +#include "../model/Problem.h" +#include "IXpansionProblemsProvider.h" +#include "LpsFromAntares.h" + +class XpansionProblemsFromAntaresProvider : public IXpansionProblemsProvider { + public: + explicit XpansionProblemsFromAntaresProvider(LpsFromAntares antares); + [[nodiscard]] std::vector> provideProblems( + const std::string& solver_name) const override; + LpsFromAntares antares_hebdo_problems; +}; diff --git a/src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.cpp b/src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.cpp new file mode 100644 index 000000000..1a9fc3c9c --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.cpp @@ -0,0 +1,35 @@ +// +// Created by marechaljas on 02/11/22. +// + +#include "ZipProblemProviderAdapter.h" + +#include + +#include "LinkProblemsGenerator.h" +#include "helpers/StringUtils.h" +#include "solver_utils.h" +void ZipProblemProviderAdapter::reader_extract_file( + const std::string& problem_name, ArchiveReader& reader, + const std::filesystem::path& lpDir) const { + reader.ExtractFile(problem_name, lpDir); +} + +std::shared_ptr ZipProblemProviderAdapter::provide_problem( + const std::string& solver_name) const { + reader_extract_file(problem_name_, *archive_reader_, lp_dir_); + SolverFactory factory; + auto const lp_mps_name = ZipProblemProviderAdapter::lp_dir_ / + ZipProblemProviderAdapter::problem_name_; + auto in_prblm = std::make_shared(factory.create_solver(solver_name)); + + in_prblm->read_prob_mps(lp_mps_name); + return in_prblm; +} + +ZipProblemProviderAdapter::ZipProblemProviderAdapter( + std::filesystem::path lp_dir, std::string problem_name, + std::shared_ptr ptr) + : lp_dir_(std::move(lp_dir)), + problem_name_(std::move(problem_name)), + archive_reader_(std::move(ptr)) {} diff --git a/src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.h b/src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.h new file mode 100644 index 000000000..ae2237ffc --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ZipProblemProviderAdapter.h @@ -0,0 +1,22 @@ +// +// Created by marechaljas on 02/11/22. +// + +#pragma once + +#include "IProblemProviderPort.h" +#include "LinkProblemsGenerator.h" +class ZipProblemProviderAdapter : public IProblemProviderPort { + public: + explicit ZipProblemProviderAdapter(std::filesystem::path lp_dir, + std::string problem_name, + std::shared_ptr ptr); + void reader_extract_file(const std::string& problem_name, + ArchiveReader& reader, + const std::filesystem::path& lpDir) const; + const std::filesystem::path lp_dir_; + const std::string problem_name_; + [[nodiscard]] std::shared_ptr provide_problem( + const std::string& solver_name) const override; + std::shared_ptr archive_reader_; +}; diff --git a/src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.cpp b/src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.cpp new file mode 100644 index 000000000..a9f2af29a --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.cpp @@ -0,0 +1,34 @@ +// +// Created by marechaljas on 25/11/22. +// + +#include "ZipProblemsProviderAdapter.h" + +#include +#include + +#include "ArchiveReader.h" +#include "ZipProblemProviderAdapter.h" +std::vector> +ZipProblemsProviderAdapter::provideProblems( + const std::string& solver_name) const { + std::vector> problems; + // Order is important. Problems need to be in the same order as names + std::for_each( + std::execution::seq /*keep it seq to avoid problems not being in the same + order as names*/ + , + problem_names_.begin(), problem_names_.end(), [&](auto name) { + ZipProblemProviderAdapter problem_provider(lp_dir_, name, + archive_reader_); + problems.push_back(problem_provider.provide_problem(solver_name)); + }); + + return problems; +} +ZipProblemsProviderAdapter::ZipProblemsProviderAdapter( + std::filesystem::path lp_dir, std::shared_ptr archive_reader, + std::vector problem_names) + : archive_reader_(std::move(archive_reader)), + lp_dir_(std::move(lp_dir)), + problem_names_(std::move(problem_names)) {} diff --git a/src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.h b/src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.h new file mode 100644 index 000000000..a2b231624 --- /dev/null +++ b/src/cpp/lpnamer/problem_modifier/ZipProblemsProviderAdapter.h @@ -0,0 +1,19 @@ +// +// Created by marechaljas on 25/11/22. +// + +#pragma once + +#include "ArchiveReader.h" +#include "IXpansionProblemsProvider.h" +class ZipProblemsProviderAdapter : public IXpansionProblemsProvider { + public: + ZipProblemsProviderAdapter(std::filesystem::path lp_dir, + std::shared_ptr archive_reader, + std::vector problem_names); + [[nodiscard]] std::vector> provideProblems( + const std::string& solver_name) const override; + std::shared_ptr archive_reader_; + std::filesystem::path lp_dir_; + std::vector problem_names_; +}; From 6442e20d1fe6d2d25ad4715a53d9b320b6ef4b19 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 1 Feb 2023 16:32:56 +0100 Subject: [PATCH 02/19] Update tests --- data_test/mini_instance_LP/structure.txt | 4 +- .../structure_default_restart.txt | 4 +- data_test/mini_instance_MIP/structure.txt | 4 +- .../structure_default_restart.txt | 4 +- .../mini_instance_UNBOUNDED/structure.txt | 4 +- data_test/mini_network/slave_weights.txt | 4 +- data_test/mini_network/structure.txt | 8 ++-- .../structure_default_restart.txt | 8 ++-- .../structure_weights_restart.txt | 8 ++-- .../test_lpnamer_01/output/economy/mps.txt | 2 +- .../output/economy/reference_lp/structure.txt | 10 ++-- .../output/economy/mps.txt | 2 +- .../output/economy/reference_lp/structure.txt | 2 +- .../output/economy/mps.txt | 2 +- .../output/economy/reference_lp/structure.txt | 4 +- .../output/economy/mps.txt | 2 +- .../output/economy/reference_lp/structure.txt | 4 +- .../output/economy/mps.txt | 2 +- .../output/economy/reference_lp/structure.txt | 4 +- .../output/economy/mps.txt | 2 +- .../output/economy/reference_lp/structure.txt | 2 +- .../output/economy/mps.txt | 8 ++-- .../output/economy/reference_lp/structure.txt | 48 +++++++++---------- .../output/economy/mps.txt | 8 ++-- .../output/economy/reference_lp/structure.txt | 8 ++-- 25 files changed, 79 insertions(+), 79 deletions(-) diff --git a/data_test/mini_instance_LP/structure.txt b/data_test/mini_instance_LP/structure.txt index b3bac76f3..307002714 100644 --- a/data_test/mini_instance_LP/structure.txt +++ b/data_test/mini_instance_LP/structure.txt @@ -1,3 +1,3 @@ master x 0 -SP1 x 0 -SP2 x 0 \ No newline at end of file +SP1.mps x 0 +SP2.mps x 0 \ No newline at end of file diff --git a/data_test/mini_instance_LP/structure_default_restart.txt b/data_test/mini_instance_LP/structure_default_restart.txt index f72a0b1b6..e9c4d72d8 100644 --- a/data_test/mini_instance_LP/structure_default_restart.txt +++ b/data_test/mini_instance_LP/structure_default_restart.txt @@ -1,3 +1,3 @@ master_default_restart x 0 -SP1 x 0 -SP2 x 0 \ No newline at end of file +SP1.mps x 0 +SP2.mps x 0 \ No newline at end of file diff --git a/data_test/mini_instance_MIP/structure.txt b/data_test/mini_instance_MIP/structure.txt index b3bac76f3..307002714 100644 --- a/data_test/mini_instance_MIP/structure.txt +++ b/data_test/mini_instance_MIP/structure.txt @@ -1,3 +1,3 @@ master x 0 -SP1 x 0 -SP2 x 0 \ No newline at end of file +SP1.mps x 0 +SP2.mps x 0 \ No newline at end of file diff --git a/data_test/mini_instance_MIP/structure_default_restart.txt b/data_test/mini_instance_MIP/structure_default_restart.txt index f72a0b1b6..e9c4d72d8 100644 --- a/data_test/mini_instance_MIP/structure_default_restart.txt +++ b/data_test/mini_instance_MIP/structure_default_restart.txt @@ -1,3 +1,3 @@ master_default_restart x 0 -SP1 x 0 -SP2 x 0 \ No newline at end of file +SP1.mps x 0 +SP2.mps x 0 \ No newline at end of file diff --git a/data_test/mini_instance_UNBOUNDED/structure.txt b/data_test/mini_instance_UNBOUNDED/structure.txt index b3bac76f3..307002714 100644 --- a/data_test/mini_instance_UNBOUNDED/structure.txt +++ b/data_test/mini_instance_UNBOUNDED/structure.txt @@ -1,3 +1,3 @@ master x 0 -SP1 x 0 -SP2 x 0 \ No newline at end of file +SP1.mps x 0 +SP2.mps x 0 \ No newline at end of file diff --git a/data_test/mini_network/slave_weights.txt b/data_test/mini_network/slave_weights.txt index 3f76ec39b..09c04c863 100644 --- a/data_test/mini_network/slave_weights.txt +++ b/data_test/mini_network/slave_weights.txt @@ -1,3 +1,3 @@ -SP1 0.99 -SP2 0.01 +SP1.mps 0.99 +SP2.mps 0.01 WEIGHT_SUM 1.0 \ No newline at end of file diff --git a/data_test/mini_network/structure.txt b/data_test/mini_network/structure.txt index eb0bb983c..e87c7fe18 100644 --- a/data_test/mini_network/structure.txt +++ b/data_test/mini_network/structure.txt @@ -1,6 +1,6 @@ master t 0 master p 1 -SP1 t 0 -SP1 p 1 -SP2 t 0 -SP2 p 1 \ No newline at end of file +SP1.mps t 0 +SP1.mps p 1 +SP2.mps t 0 +SP2.mps p 1 \ No newline at end of file diff --git a/data_test/mini_network/structure_default_restart.txt b/data_test/mini_network/structure_default_restart.txt index 108a8ec70..11dda046b 100644 --- a/data_test/mini_network/structure_default_restart.txt +++ b/data_test/mini_network/structure_default_restart.txt @@ -1,6 +1,6 @@ master_default_restart t 0 master_default_restart p 1 -SP1 t 0 -SP1 p 1 -SP2 t 0 -SP2 p 1 \ No newline at end of file +SP1.mps t 0 +SP1.mps p 1 +SP2.mps t 0 +SP2.mps p 1 \ No newline at end of file diff --git a/data_test/mini_network/structure_weights_restart.txt b/data_test/mini_network/structure_weights_restart.txt index 7edcc56e4..8cd290543 100644 --- a/data_test/mini_network/structure_weights_restart.txt +++ b/data_test/mini_network/structure_weights_restart.txt @@ -1,6 +1,6 @@ master_weights_restart t 0 master_weights_restart p 1 -SP1 t 0 -SP1 p 1 -SP2 t 0 -SP2 p 1 \ No newline at end of file +SP1.mps t 0 +SP1.mps p 1 +SP2.mps t 0 +SP2.mps p 1 \ No newline at end of file diff --git a/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/mps.txt b/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/mps.txt index b9b8f6447..9ee49933e 100644 --- a/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/mps.txt @@ -1 +1 @@ -problem-1-1-20210713-163528.mps variables-1-1-20210713-163528.txt constraints-1-1-20210713-163528.txt +problem-1-1-20210713-163528.mps variables-1-1-20210713-163528.txt diff --git a/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/reference_lp/structure.txt index e2513d95e..c8142d560 100644 --- a/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_integer/test_lpnamer_01/output/economy/reference_lp/structure.txt @@ -3,8 +3,8 @@ master pv 2 master semibase 3 master transmission_line 4 - problem-1-1-20210713-163528 battery 5547 - problem-1-1-20210713-163528 peak 5545 - problem-1-1-20210713-163528 pv 5548 - problem-1-1-20210713-163528 semibase 5546 - problem-1-1-20210713-163528 transmission_line 5544 + problem-1-1-20210713-163528.mps battery 5547 + problem-1-1-20210713-163528.mps peak 5545 + problem-1-1-20210713-163528.mps pv 5548 + problem-1-1-20210713-163528.mps semibase 5546 + problem-1-1-20210713-163528.mps transmission_line 5544 diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/mps.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/mps.txt index 780c279ec..74324f355 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/mps.txt @@ -1 +1 @@ -problem-1-1-20210720-102450.mps variables-1-1-20210720-102450.txt constraints-1-1-20210720-102450.txt +problem-1-1-20210720-102450.mps variables-1-1-20210720-102450.txt diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/reference_lp/structure.txt index 4ec7db07d..9d4f491ed 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_one_candidate_1week/output/economy/reference_lp/structure.txt @@ -1,2 +1,2 @@ master transmission_line 0 - problem-1-1-20210720-102450 transmission_line 1848 + problem-1-1-20210720-102450.mps transmission_line 1848 diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/mps.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/mps.txt index de0ea81f4..24f58d19a 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/mps.txt @@ -1 +1 @@ -problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt constraints-1-1-20210721-145117.txt +problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/reference_lp/structure.txt index 3892ca497..0d559d3ed 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_1week/output/economy/reference_lp/structure.txt @@ -1,4 +1,4 @@ master transmission_line 0 master transmission_line_2 1 - problem-1-1-20210721-145117 transmission_line 1848 - problem-1-1-20210721-145117 transmission_line_2 1849 + problem-1-1-20210721-145117.mps transmission_line 1848 + problem-1-1-20210721-145117.mps transmission_line_2 1849 diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/mps.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/mps.txt index de0ea81f4..24f58d19a 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/mps.txt @@ -1 +1 @@ -problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt constraints-1-1-20210721-145117.txt +problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/reference_lp/structure.txt index 30992159c..3e093bdf4 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob/output/economy/reference_lp/structure.txt @@ -1,4 +1,4 @@ master transmission_line_1 0 master transmission_line_2 1 - problem-1-1-20210721-145117 transmission_line_1 1 - problem-1-1-20210721-145117 transmission_line_2 2 + problem-1-1-20210721-145117.mps transmission_line_1 1 + problem-1-1-20210721-145117.mps transmission_line_2 2 diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/mps.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/mps.txt index de0ea81f4..24f58d19a 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/mps.txt @@ -1 +1 @@ -problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt constraints-1-1-20210721-145117.txt +problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/reference_lp/structure.txt index 483768790..349538bbd 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_hurdle_cost/output/economy/reference_lp/structure.txt @@ -1,4 +1,4 @@ master transmission_line_1 0 master transmission_line_2 1 - problem-1-1-20210721-145117 transmission_line_1 3 - problem-1-1-20210721-145117 transmission_line_2 4 + problem-1-1-20210721-145117.mps transmission_line_1 3 + problem-1-1-20210721-145117.mps transmission_line_2 4 diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/mps.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/mps.txt index de0ea81f4..24f58d19a 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/mps.txt @@ -1 +1 @@ -problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt constraints-1-1-20210721-145117.txt +problem-1-1-20210721-145117.mps variables-1-1-20210721-145117.txt diff --git a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/reference_lp/structure.txt index 7daae32a4..c7f8168ed 100644 --- a/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_integer/test_one_link_two_candidates_simple_prob_null_profile/output/economy/reference_lp/structure.txt @@ -1,3 +1,3 @@ master transmission_line_1 0 master transmission_line_2 1 - problem-1-1-20210721-145117 transmission_line_1 1 + problem-1-1-20210721-145117.mps transmission_line_1 1 diff --git a/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/mps.txt b/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/mps.txt index 1b4a9043a..7668966a1 100644 --- a/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/mps.txt @@ -1,4 +1,4 @@ -problem-1-1-20210720-143331.mps variables-1-1-20210720-143331.txt constraints-1-1-20210720-143331.txt -problem-1-2-20210720-143331.mps variables-1-2-20210720-143331.txt constraints-1-2-20210720-143331.txt -problem-1-3-20210720-143331.mps variables-1-3-20210720-143331.txt constraints-1-3-20210720-143331.txt -problem-1-4-20210720-143331.mps variables-1-4-20210720-143331.txt constraints-1-4-20210720-143331.txt +problem-1-1-20210720-143331.mps variables-1-1-20210720-143331.txt +problem-1-2-20210720-143331.mps variables-1-2-20210720-143331.txt +problem-1-3-20210720-143331.mps variables-1-3-20210720-143331.txt +problem-1-4-20210720-143331.mps variables-1-4-20210720-143331.txt diff --git a/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/reference_lp/structure.txt index 0377ae6f8..9e95e3a33 100644 --- a/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_relaxed/SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed/output/economy/reference_lp/structure.txt @@ -4,27 +4,27 @@ master pv 3 master semibase1 4 master semibase2 5 - problem-1-1-20210720-143331 battery 6891 - problem-1-1-20210720-143331 peak1 6888 - problem-1-1-20210720-143331 peak2 6892 - problem-1-1-20210720-143331 pv 6890 - problem-1-1-20210720-143331 semibase1 6889 - problem-1-1-20210720-143331 semibase2 6893 - problem-1-2-20210720-143331 battery 6891 - problem-1-2-20210720-143331 peak1 6888 - problem-1-2-20210720-143331 peak2 6892 - problem-1-2-20210720-143331 pv 6890 - problem-1-2-20210720-143331 semibase1 6889 - problem-1-2-20210720-143331 semibase2 6893 - problem-1-3-20210720-143331 battery 6891 - problem-1-3-20210720-143331 peak1 6888 - problem-1-3-20210720-143331 peak2 6892 - problem-1-3-20210720-143331 pv 6890 - problem-1-3-20210720-143331 semibase1 6889 - problem-1-3-20210720-143331 semibase2 6893 - problem-1-4-20210720-143331 battery 6891 - problem-1-4-20210720-143331 peak1 6888 - problem-1-4-20210720-143331 peak2 6892 - problem-1-4-20210720-143331 pv 6890 - problem-1-4-20210720-143331 semibase1 6889 - problem-1-4-20210720-143331 semibase2 6893 + problem-1-1-20210720-143331.mps battery 6891 + problem-1-1-20210720-143331.mps peak1 6888 + problem-1-1-20210720-143331.mps peak2 6892 + problem-1-1-20210720-143331.mps pv 6890 + problem-1-1-20210720-143331.mps semibase1 6889 + problem-1-1-20210720-143331.mps semibase2 6893 + problem-1-2-20210720-143331.mps battery 6891 + problem-1-2-20210720-143331.mps peak1 6888 + problem-1-2-20210720-143331.mps peak2 6892 + problem-1-2-20210720-143331.mps pv 6890 + problem-1-2-20210720-143331.mps semibase1 6889 + problem-1-2-20210720-143331.mps semibase2 6893 + problem-1-3-20210720-143331.mps battery 6891 + problem-1-3-20210720-143331.mps peak1 6888 + problem-1-3-20210720-143331.mps peak2 6892 + problem-1-3-20210720-143331.mps pv 6890 + problem-1-3-20210720-143331.mps semibase1 6889 + problem-1-3-20210720-143331.mps semibase2 6893 + problem-1-4-20210720-143331.mps battery 6891 + problem-1-4-20210720-143331.mps peak1 6888 + problem-1-4-20210720-143331.mps peak2 6892 + problem-1-4-20210720-143331.mps pv 6890 + problem-1-4-20210720-143331.mps semibase1 6889 + problem-1-4-20210720-143331.mps semibase2 6893 diff --git a/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/mps.txt b/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/mps.txt index 61282b41e..016f3f69f 100644 --- a/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/mps.txt +++ b/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/mps.txt @@ -1,4 +1,4 @@ -problem-1-1-20210720-114739.mps variables-1-1-20210720-114739.txt constraints-1-1-20210720-114739.txt -problem-1-2-20210720-114739.mps variables-1-2-20210720-114739.txt constraints-1-2-20210720-114739.txt -problem-1-3-20210720-114739.mps variables-1-3-20210720-114739.txt constraints-1-3-20210720-114739.txt -problem-1-4-20210720-114739.mps variables-1-4-20210720-114739.txt constraints-1-4-20210720-114739.txt +problem-1-1-20210720-114739.mps variables-1-1-20210720-114739.txt +problem-1-2-20210720-114739.mps variables-1-2-20210720-114739.txt +problem-1-3-20210720-114739.mps variables-1-3-20210720-114739.txt +problem-1-4-20210720-114739.mps variables-1-4-20210720-114739.txt diff --git a/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/reference_lp/structure.txt b/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/reference_lp/structure.txt index da18758c4..55fa44895 100644 --- a/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/reference_lp/structure.txt +++ b/data_test/tests_lpnamer/tests_relaxed/test_one_link_one_candidate-relaxed/output/economy/reference_lp/structure.txt @@ -1,5 +1,5 @@ master transmission_line 0 - problem-1-1-20210720-114739 transmission_line 1848 - problem-1-2-20210720-114739 transmission_line 1848 - problem-1-3-20210720-114739 transmission_line 1848 - problem-1-4-20210720-114739 transmission_line 1848 + problem-1-1-20210720-114739.mps transmission_line 1848 + problem-1-2-20210720-114739.mps transmission_line 1848 + problem-1-3-20210720-114739.mps transmission_line 1848 + problem-1-4-20210720-114739.mps transmission_line 1848 From 8e729d7179be74bec2c59c4fc02977cea1f053ef Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 1 Feb 2023 16:40:38 +0100 Subject: [PATCH 03/19] Properly pass lp directory as problem directory --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 45d5a999b..fc957f691 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -115,7 +115,7 @@ void RunProblemGeneration( /* Zip problem writer */ auto problem_writer = - std::make_shared(xpansion_output_dir, writer); + std::make_shared(lpDir_, writer); /* Main stuff */ std::vector problem_names; From 0823e34ea21f669d40d246076e6b6e54adf42e0d Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 8 Feb 2023 16:24:58 +0100 Subject: [PATCH 04/19] Don't delete mps file between lp and benders --- src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp b/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp index 02feb442a..3b4cbaf18 100644 --- a/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp +++ b/src/cpp/lpnamer/problem_modifier/ArchiveProblemWriter.cpp @@ -13,7 +13,8 @@ void ArchiveProblemWriter::Write_problem(std::shared_ptr &in_prblm) { auto const lp_mps_name = lp_dir_ / in_prblm->_name; in_prblm->write_prob_mps(lp_mps_name); writer_->AddFileInArchive(lp_mps_name); - std::filesystem::remove(lp_mps_name); + // If we remove mps file we can't load them later + // std::filesystem::remove(lp_mps_name); } ArchiveProblemWriter::ArchiveProblemWriter( From 0fe58932c3cb0f5d60c00b3408a942d03df290d3 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 9 Feb 2023 10:32:09 +0100 Subject: [PATCH 05/19] Improve destruction of ArchiveWriter --- src/cpp/helpers/ArchiveIO.h | 2 ++ src/cpp/helpers/ArchiveWriter.cpp | 23 +++++++++++++++++------ src/cpp/helpers/ArchiveWriter.h | 8 ++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/cpp/helpers/ArchiveIO.h b/src/cpp/helpers/ArchiveIO.h index 5f0368999..e4d3af5e2 100644 --- a/src/cpp/helpers/ArchiveIO.h +++ b/src/cpp/helpers/ArchiveIO.h @@ -43,6 +43,8 @@ class ArchiveIO { explicit ArchiveIO(const std::filesystem::path& archivePath) : archivePath_(archivePath) {} ArchiveIO() = default; + virtual ~ArchiveIO() = default; + std::filesystem::path ArchivePath() const { return archivePath_; } void SetArchivePath(const std::filesystem::path& archivePath) { archivePath_ = archivePath; diff --git a/src/cpp/helpers/ArchiveWriter.cpp b/src/cpp/helpers/ArchiveWriter.cpp index 8633501cc..7d0feabc0 100644 --- a/src/cpp/helpers/ArchiveWriter.cpp +++ b/src/cpp/helpers/ArchiveWriter.cpp @@ -35,12 +35,8 @@ int32_t ArchiveWriter::Open() { } return err; } -int32_t ArchiveWriter::Close() { - return mz_zip_writer_close(pmz_zip_writer_instance_); -} -void ArchiveWriter::Delete() { - mz_zip_writer_delete(&pmz_zip_writer_instance_); -} +int32_t ArchiveWriter::Close() { return CloseInternal(); } +void ArchiveWriter::Delete() { return DeleteInternal(); } int32_t ArchiveWriter::AddFileInArchive(const FileBuffer& FileBufferToAdd) { std::unique_lock lock(mutex_); @@ -113,3 +109,18 @@ int32_t ArchiveWriter::AddPathInArchive( return MZ_OK; } +ArchiveWriter::~ArchiveWriter() { + CloseInternal(); + DeleteInternal(); +} + +int32_t ArchiveWriter::CloseInternal() { + if (pmz_zip_writer_instance_) { + return mz_zip_writer_close(pmz_zip_writer_instance_); + } + return MZ_OK; +} + +void ArchiveWriter::DeleteInternal() { + return mz_zip_writer_delete(&pmz_zip_writer_instance_); +} diff --git a/src/cpp/helpers/ArchiveWriter.h b/src/cpp/helpers/ArchiveWriter.h index f97231ab5..c51aeeeaa 100644 --- a/src/cpp/helpers/ArchiveWriter.h +++ b/src/cpp/helpers/ArchiveWriter.h @@ -8,14 +8,18 @@ class ArchiveWriter : public ArchiveIO { private: - void* pmz_zip_writer_instance_ = NULL; - void* pzip_handle_ = NULL; + void* pmz_zip_writer_instance_ = nullptr; + void* pzip_handle_ = nullptr; void Create() override; mz_zip_file fileInfo_ = {0}; + int32_t CloseInternal(); + void DeleteInternal(); + public: explicit ArchiveWriter(const std::filesystem::path& archivePath); ArchiveWriter(); + ~ArchiveWriter() override; int32_t Close() override; void Delete() override; From 53faf6ae2e5fae4c884c6e1aaefea7550ab38af8 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 9 Feb 2023 10:57:18 +0100 Subject: [PATCH 06/19] Reduce duplication --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 47 +++++++++++-------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index fc957f691..4eee2f426 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -25,6 +25,10 @@ #include "ZipProblemsProviderAdapter.h" #include "common_lpnamer.h" +std::shared_ptr InstantiateZipReader( + const std::filesystem::path& antares_archive_path); +std::filesystem::path OpenTmpArchive(const std::filesystem::path& lpDir_, + std::shared_ptr& writer); void ProcessWeights( const std::filesystem::path& xpansion_output_dir, const std::filesystem::path& antares_archive_path, @@ -99,19 +103,15 @@ void RunProblemGeneration( log_file_path, zip_mps); auto mpsList = linkProblemsGenerator.readMPSList(mps_file_name); + auto lpDir_ = xpansion_output_dir / "lp"; + bool use_zip_implementation = true; bool use_file_implementation = false; if (use_zip_implementation) { - /* Instantiate Zip reader */ - auto reader = std::make_shared(antares_archive_path); - reader->Open(); - - /*Instantiate zip writer */ - auto lpDir_ = xpansion_output_dir / "lp"; - const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; - const auto tmpArchivePath = lpDir_ / tmpArchiveName; - auto writer = std::make_shared(tmpArchivePath); - writer->Open(); + std::shared_ptr reader = + InstantiateZipReader(antares_archive_path); + std::shared_ptr writer; + const std::filesystem::path tmpArchivePath = OpenTmpArchive(lpDir_, writer); /* Zip problem writer */ auto problem_writer = @@ -154,13 +154,8 @@ void RunProblemGeneration( writer->Close(); writer->Delete(); } else if (use_file_implementation) { - /*Instantiate zip writer */ - auto lpDir_ = xpansion_output_dir / "lp"; - const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; - const auto tmpArchivePath = lpDir_ / tmpArchiveName; - std::filesystem::remove(tmpArchivePath); - auto writer = std::make_shared(tmpArchivePath); - writer->Open(); + std::shared_ptr writer; + const std::filesystem::path tmpArchivePath = OpenTmpArchive(lpDir_, writer); /* Zip problem writer */ auto problem_writer = @@ -197,7 +192,6 @@ void RunProblemGeneration( auto reader = std::make_shared(antares_archive_path); reader->Open(); - auto lpDir_ = xpansion_output_dir / "lp"; const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; const auto tmpArchivePath = lpDir_ / tmpArchiveName; auto writer = std::make_shared(tmpArchivePath); @@ -229,4 +223,19 @@ void RunProblemGeneration( MasterGeneration master_generation( xpansion_output_dir, links, additionalConstraints, couplings, master_formulation, solver_name, logger, log_file_path); -} \ No newline at end of file +} +std::filesystem::path OpenTmpArchive(const std::filesystem::path& lpDir_, + std::shared_ptr& writer) { + const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; + auto tmpArchivePath = lpDir_ / tmpArchiveName; + writer = std::make_shared( + tmpArchivePath); /*Instantiate zip writer */ + writer->Open(); + return tmpArchivePath; +} +std::shared_ptr InstantiateZipReader( + const std::filesystem::path& antares_archive_path) { + auto reader = std::make_shared(antares_archive_path); + reader->Open(); + return reader; +} From cc2e28b73580d612e44c134f217d98af4f91d1eb Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 9 Feb 2023 11:09:50 +0100 Subject: [PATCH 07/19] Reduce code duplication --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 4eee2f426..089b5ef8d 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -29,6 +29,10 @@ std::shared_ptr InstantiateZipReader( const std::filesystem::path& antares_archive_path); std::filesystem::path OpenTmpArchive(const std::filesystem::path& lpDir_, std::shared_ptr& writer); +void CleanUpArchives(const std::filesystem::path& antares_archive_path, + const std::filesystem::path& lpDir_, + std::shared_ptr& writer, + const std::filesystem::path& tmpArchivePath); void ProcessWeights( const std::filesystem::path& xpansion_output_dir, const std::filesystem::path& antares_archive_path, @@ -149,10 +153,7 @@ void RunProblemGeneration( reader->Close(); reader->Delete(); - std::filesystem::remove(antares_archive_path); - std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); - writer->Close(); - writer->Delete(); + CleanUpArchives(antares_archive_path, lpDir_, writer, tmpArchivePath); } else if (use_file_implementation) { std::shared_ptr writer; const std::filesystem::path tmpArchivePath = OpenTmpArchive(lpDir_, writer); @@ -165,10 +166,7 @@ void RunProblemGeneration( linkProblemsGenerator.treatloop_files(xpansion_output_dir, couplings, mpsList, problem_writer); - std::filesystem::remove(antares_archive_path); - std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); - writer->Close(); - writer->Delete(); + CleanUpArchives(antares_archive_path, lpDir_, writer, tmpArchivePath); } else { std::filesystem::path path = xpansion_output_dir.parent_path().parent_path() / @@ -214,16 +212,22 @@ void RunProblemGeneration( reader->Close(); reader->Delete(); - std::filesystem::remove(antares_archive_path); - std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); - writer->Close(); - writer->Delete(); + CleanUpArchives(antares_archive_path, lpDir_, writer, tmpArchivePath); } MasterGeneration master_generation( xpansion_output_dir, links, additionalConstraints, couplings, master_formulation, solver_name, logger, log_file_path); } +void CleanUpArchives(const std::filesystem::path& antares_archive_path, + const std::filesystem::path& lpDir_, + std::shared_ptr& writer, + const std::filesystem::path& tmpArchivePath) { + std::filesystem::remove(antares_archive_path); + std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); + writer->Close(); + writer->Delete(); +} std::filesystem::path OpenTmpArchive(const std::filesystem::path& lpDir_, std::shared_ptr& writer) { const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; From 1c077ec0e8e7d01cb8d1f6d226ab002a5263a974 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 17 Feb 2023 11:13:40 +0100 Subject: [PATCH 08/19] Add timeout to test on CentOS. Prevent CI getting stuck far longer than necessary --- .github/workflows/build_centos7.yml | 1 + .github/workflows/centos7-system.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build_centos7.yml b/.github/workflows/build_centos7.yml index c142ec511..3d1022be8 100644 --- a/.github/workflows/build_centos7.yml +++ b/.github/workflows/build_centos7.yml @@ -136,6 +136,7 @@ jobs: export PATH=/usr/lib64/openmpi/bin:$PATH cmake3 --build _build --config Release -j8 --target install - name: Running unit tests + timeout-minutes: 45 shell: bash run: | source /etc/profile.d/modules.sh diff --git a/.github/workflows/centos7-system.yml b/.github/workflows/centos7-system.yml index 47103726c..d9b5fc11e 100644 --- a/.github/workflows/centos7-system.yml +++ b/.github/workflows/centos7-system.yml @@ -237,6 +237,7 @@ jobs: path: ${{ steps.create-single-file.outputs.archive-path }} - name: Running unit tests + timeout-minutes: 45 shell: bash run: | source /etc/profile.d/modules.sh From 4518712b40d74244313dbb4f4e346094a3385541 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 17 Feb 2023 11:15:54 +0100 Subject: [PATCH 09/19] Install git from third party package Make git 2.x available globally --- docker/centos7-bare | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/centos7-bare b/docker/centos7-bare index 62e478644..6fc258710 100644 --- a/docker/centos7-bare +++ b/docker/centos7-bare @@ -8,8 +8,11 @@ RUN yum install -y epel-release # Install requirements RUN yum install -y git redhat-lsb-core gcc gcc-c++ make wget centos-release-scl scl-utils rpm-build &&\ - yum install -y cmake3 devtoolset-9 &&\ - yum install -y rh-git227-git + yum install -y cmake3 devtoolset-9 + +RUN yum -y remove git && \ + yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && \ + yum -y install git # Install xpansion requirements RUN yum install -y jsoncpp-devel gtest-devel openmpi-devel boost-openmpi-devel doxygen graphviz boost-program-options environment-modules rpm-build zlib-devel From 927f584999b8f8b45b6eb878f98f78882f8f1244 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 17 Feb 2023 11:17:21 +0100 Subject: [PATCH 10/19] Compile and use boost 181 --- .../action.yml | 24 +++++++++++++------ docker/centos7-bare | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/download-extract-precompiled-libraries-tgz/action.yml b/.github/workflows/download-extract-precompiled-libraries-tgz/action.yml index 6df347534..469b018b3 100644 --- a/.github/workflows/download-extract-precompiled-libraries-tgz/action.yml +++ b/.github/workflows/download-extract-precompiled-libraries-tgz/action.yml @@ -27,10 +27,20 @@ runs: - id: download-extract shell: bash run: | - wget https://github.com/AntaresSimulatorTeam/antares-deps/releases/download/v${{inputs.antares-deps-version}}/rte-antares-deps-${{inputs.os}}-${{inputs.buildtype}}.tar.gz - tar -xvf rte-antares-deps-${{inputs.os}}-${{inputs.buildtype}}.tar.gz - rm -rf rte-antares-deps-${{inputs.os}}-${{inputs.buildtype}}.tar.gz - - wget https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v${{inputs.antares-version}}/antares-${{inputs.antares-version}}-${{inputs.os-full-name}}${{inputs.variant}}.tar.gz - tar -xvf antares-${{inputs.antares-version}}-${{inputs.os-full-name}}${{inputs.variant}}.tar.gz -C rte-antares-deps-${{inputs.buildtype}} --strip-components=1 - rm -rf antares-${{inputs.antares-version}}-${{inputs.os-full-name}}${{inputs.variant}}.tar.gz \ No newline at end of file + export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH + export PATH=/usr/lib64/openmpi/bin:$PATH + + wget https://github.com/AntaresSimulatorTeam/antares-deps/releases/download/v${{inputs.antares-deps-version}}/rte-antares-deps-${{inputs.os}}-${{inputs.buildtype}}.tar.gz + tar -xvf rte-antares-deps-${{inputs.os}}-${{inputs.buildtype}}.tar.gz + rm -rf rte-antares-deps-${{inputs.os}}-${{inputs.buildtype}}.tar.gz + + wget https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v${{inputs.antares-version}}/antares-${{inputs.antares-version}}-${{inputs.os-full-name}}${{inputs.variant}}.tar.gz + tar -xvf antares-${{inputs.antares-version}}-${{inputs.os-full-name}}${{inputs.variant}}.tar.gz -C rte-antares-deps-${{inputs.buildtype}} --strip-components=1 + rm -rf antares-${{inputs.antares-version}}-${{inputs.os-full-name}}${{inputs.variant}}.tar.gz + + wget https://github.com/boostorg/boost/releases/download/boost-1.81.0/boost-1.81.0.tar.gz + tar xvf boost-1.81.0.tar.gz + cd boost-1.81.0 + ./bootstrap.sh --prefix=../rte-antares-deps-${{inputs.buildtype}}/ --with-libraries=serialization,program_options,mpi + echo "using mpi ;" >> project-config.jam + ./b2 install \ No newline at end of file diff --git a/docker/centos7-bare b/docker/centos7-bare index 6fc258710..98deaea83 100644 --- a/docker/centos7-bare +++ b/docker/centos7-bare @@ -15,7 +15,7 @@ RUN yum -y remove git && \ yum -y install git # Install xpansion requirements -RUN yum install -y jsoncpp-devel gtest-devel openmpi-devel boost-openmpi-devel doxygen graphviz boost-program-options environment-modules rpm-build zlib-devel +RUN yum install -y jsoncpp-devel gtest-devel openmpi-devel doxygen graphviz environment-modules rpm-build zlib-devel # Install tbb. tbb is required for execution policy (parallelization c++) but the system version is too old. RUN git clone https://github.com/wjakob/tbb.git && \ From a12d3290028b18cf76e64f5b6513bd03d6cdfc84 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 17 Feb 2023 11:17:45 +0100 Subject: [PATCH 11/19] Fix post merge --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index 089b5ef8d..e9c9a8613 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -105,7 +105,8 @@ void RunProblemGeneration( LinkProblemsGenerator linkProblemsGenerator(links, solver_name, logger, log_file_path, zip_mps); - auto mpsList = linkProblemsGenerator.readMPSList(mps_file_name); + auto files_mapper = FilesMapper(antares_archive_path); + auto mpsList = files_mapper.MpsAndVariablesFilesVect(); auto lpDir_ = xpansion_output_dir / "lp"; From fb19049fb60a4579027f8332decc2264887dfeba Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 22 Feb 2023 11:36:52 +0100 Subject: [PATCH 12/19] Reformat file --- tests/CMakeLists.txt | 86 ++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 50013c3f6..860c3e258 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,38 +1,38 @@ if (MSVC) set(XPANSION_INSTALL_DIR "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}") -else() +else () set(XPANSION_INSTALL_DIR "${CMAKE_BINARY_DIR}") -endif() +endif () #configure file to define xpansion installDir -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build_config.yaml.in ${CMAKE_CURRENT_SOURCE_DIR}/build_config.yaml ) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build_config.yaml.in ${CMAKE_CURRENT_SOURCE_DIR}/build_config.yaml) #check if pytest and numpy are installed find_python_module(pytest) find_python_module(numpy) if (PYTHON_MODULE_pytest_FOUND AND PYTHON_MODULE_numpy_FOUND) - # Python unit test - add_test( - NAME unit_launcher - COMMAND Python3::Interpreter -m pytest --cov=antares_xpansion --cov-report xml:coverage-reports/coverage-antares_xpansion.xml - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python" + # Python unit test + add_test( + NAME unit_launcher + COMMAND Python3::Interpreter -m pytest --cov=antares_xpansion --cov-report xml:coverage-reports/coverage-antares_xpansion.xml + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/python" ) - set_property(TEST unit_launcher PROPERTY LABELS unit) - + set_property(TEST unit_launcher PROPERTY LABELS unit) + # Examples tests add_test( - NAME examples_short_sequential - COMMAND Python3::Interpreter -m pytest -m short_sequential --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples + NAME examples_short_sequential + COMMAND Python3::Interpreter -m pytest -m short_sequential --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples ) set_property(TEST examples_short_sequential PROPERTY LABELS short short_sequential end_to_end) add_test( - NAME examples_short_mpi - COMMAND Python3::Interpreter -m pytest -m short_mpi --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples + NAME examples_short_mpi + COMMAND Python3::Interpreter -m pytest -m short_mpi --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples ) set_property(TEST examples_short_mpi PROPERTY LABELS short short_mpi end_to_end) @@ -51,60 +51,60 @@ if (PYTHON_MODULE_pytest_FOUND AND PYTHON_MODULE_numpy_FOUND) set_property(TEST examples_medium_mpi PROPERTY LABELS medium medium_mpi end_to_end) add_test( - NAME examples_long_sequential - COMMAND Python3::Interpreter -m pytest -m long_sequential --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples + NAME examples_long_sequential + COMMAND Python3::Interpreter -m pytest -m long_sequential --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples ) set_property(TEST examples_long_sequential PROPERTY LABELS long long_sequential end_to_end) add_test( - NAME examples_long_mpi - COMMAND Python3::Interpreter -m pytest -m long_mpi --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples + NAME examples_long_mpi + COMMAND Python3::Interpreter -m pytest -m long_mpi --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} example_test.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/examples ) set_property(TEST examples_long_mpi PROPERTY LABELS long long_mpi end_to_end) - - # benders end to end tests + + # benders end to end tests add_test( - NAME sequential - COMMAND Python3::Interpreter -m pytest --installDir=${XPANSION_INSTALL_DIR} test_bendersSequentialEndToEnd.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/benders + NAME sequential + COMMAND Python3::Interpreter -m pytest --installDir=${XPANSION_INSTALL_DIR} test_bendersSequentialEndToEnd.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/benders ) set_property(TEST sequential PROPERTY LABELS benders benders-sequential end_to_end) add_test( - NAME sequential_restart - COMMAND Python3::Interpreter -m pytest --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} test_restartBendersSequentialEndToEnd.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/restart + NAME sequential_restart + COMMAND Python3::Interpreter -m pytest --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} test_restartBendersSequentialEndToEnd.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/restart ) set_property(TEST sequential_restart PROPERTY LABELS benders benders-sequential end_to_end restart) add_test( - NAME mpibenders - COMMAND Python3::Interpreter -m pytest --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} test_bendersmpibendersEndToEnd.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/benders + NAME mpibenders + COMMAND Python3::Interpreter -m pytest --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} test_bendersmpibendersEndToEnd.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/benders ) set_property(TEST mpibenders PROPERTY LABELS benders benders-mpi end_to_end) add_test( - NAME mpibenders_restart - COMMAND Python3::Interpreter -m pytest --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} test_restartBendersmpibendersEndToEnd.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/restart + NAME mpibenders_restart + COMMAND Python3::Interpreter -m pytest --allow_run_as_root=${ALLOW_RUN_AS_ROOT} --installDir=${XPANSION_INSTALL_DIR} test_restartBendersmpibendersEndToEnd.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/restart ) set_property(TEST mpibenders_restart PROPERTY LABELS benders benders-mpi end_to_end restart) add_test( - NAME merge_mps - COMMAND Python3::Interpreter -m pytest --installDir=${XPANSION_INSTALL_DIR} test_bendersMergeMpsEndToEnd.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/benders + NAME merge_mps + COMMAND Python3::Interpreter -m pytest --installDir=${XPANSION_INSTALL_DIR} test_bendersMergeMpsEndToEnd.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/benders ) set_property(TEST merge_mps PROPERTY LABELS benders merge-mps end_to_end) add_test( NAME lpnamer_end_to_end - COMMAND Python3::Interpreter -m pytest --installDir=${XPANSION_INSTALL_DIR} test_lpnamerEndToEnd.py + COMMAND Python3::Interpreter -m pytest --installDir=${XPANSION_INSTALL_DIR} test_lpnamerEndToEnd.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/end_to_end/lpnamer ) set_property(TEST lpnamer_end_to_end PROPERTY LABELS lpnamer end_to_end) -else() - message(FATAL_ERROR "Module pytest or numpy not installed : can't run python scripts for end to end tests" ) -endif() +else () + message(FATAL_ERROR "Module pytest or numpy not installed : can't run python scripts for end to end tests") +endif () add_subdirectory(cpp) From 9a14fd78868010a28bfee5c66db4cb39f4ea541d Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 22 Feb 2023 15:26:31 +0100 Subject: [PATCH 13/19] Move Close/Delete into ArchiveReader destructor --- src/cpp/helpers/ArchiveReader.cpp | 6 +++++- src/cpp/helpers/ArchiveReader.h | 1 + src/cpp/lpnamer/main/RunProblemGeneration.cpp | 7 ------- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cpp/helpers/ArchiveReader.cpp b/src/cpp/helpers/ArchiveReader.cpp index 40ffadf62..2ead60d9a 100644 --- a/src/cpp/helpers/ArchiveReader.cpp +++ b/src/cpp/helpers/ArchiveReader.cpp @@ -216,4 +216,8 @@ std::vector ArchiveReader::ExtractPattern( } while (mz_zip_reader_goto_next_entry(pmz_zip_reader_instance_) == MZ_OK); } return result; -} \ No newline at end of file +} +ArchiveReader::~ArchiveReader() { + mz_zip_reader_close(pmz_zip_reader_instance_); + mz_zip_reader_delete(&pmz_zip_reader_instance_); +} diff --git a/src/cpp/helpers/ArchiveReader.h b/src/cpp/helpers/ArchiveReader.h index 293f0db77..a206676ab 100644 --- a/src/cpp/helpers/ArchiveReader.h +++ b/src/cpp/helpers/ArchiveReader.h @@ -18,6 +18,7 @@ class ArchiveReader : public ArchiveIO { public: explicit ArchiveReader(const std::filesystem::path& archivePath); ArchiveReader(); + ~ArchiveReader() override; int32_t Close() override; void Delete() override; diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index e9c9a8613..e4d790226 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -150,10 +150,6 @@ void RunProblemGeneration( problem_variables_from_zip_adapter); }); - /* Clean up */ - reader->Close(); - reader->Delete(); - CleanUpArchives(antares_archive_path, lpDir_, writer, tmpArchivePath); } else if (use_file_implementation) { std::shared_ptr writer; @@ -209,9 +205,6 @@ void RunProblemGeneration( data._problem_mps, couplings, problem_writer, problem, problem_variables_from_zip_adapter); }); - /* Clean up */ - reader->Close(); - reader->Delete(); CleanUpArchives(antares_archive_path, lpDir_, writer, tmpArchivePath); } From ef793dde6c0b3a28e61e41acf1b144e94756f21f Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Wed, 22 Feb 2023 15:33:10 +0100 Subject: [PATCH 14/19] Reduce code duplication --- src/cpp/helpers/ArchiveReader.cpp | 6 ++++-- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/cpp/helpers/ArchiveReader.cpp b/src/cpp/helpers/ArchiveReader.cpp index 2ead60d9a..5cf600f21 100644 --- a/src/cpp/helpers/ArchiveReader.cpp +++ b/src/cpp/helpers/ArchiveReader.cpp @@ -218,6 +218,8 @@ std::vector ArchiveReader::ExtractPattern( return result; } ArchiveReader::~ArchiveReader() { - mz_zip_reader_close(pmz_zip_reader_instance_); - mz_zip_reader_delete(&pmz_zip_reader_instance_); + if (pmz_zip_reader_instance_) { + mz_zip_reader_close(pmz_zip_reader_instance_); + mz_zip_reader_delete(&pmz_zip_reader_instance_); + } } diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index e4d790226..faea95e8f 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -185,12 +185,9 @@ void RunProblemGeneration( problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i)); } - auto reader = std::make_shared(antares_archive_path); - reader->Open(); - const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; - const auto tmpArchivePath = lpDir_ / tmpArchiveName; - auto writer = std::make_shared(tmpArchivePath); - writer->Open(); + auto reader = InstantiateZipReader(antares_archive_path); + std::shared_ptr writer; + const std::filesystem::path tmpArchivePath = OpenTmpArchive(lpDir_, writer); auto problem_writer = std::make_shared(lpDir_, writer); From 0b214e5a2b247c526197174dc2f1c64f21e82de8 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 23 Feb 2023 15:40:22 +0100 Subject: [PATCH 15/19] Add timeout to windows unit tests --- .github/workflows/build_windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml index ba6a7dc2e..bbdc096b9 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/build_windows.yml @@ -130,6 +130,7 @@ jobs: run: | cmake --build _build --config Release -j2 --target install - name: Running unit tests + timeout-minutes: 45 shell: cmd run: | set PATH=%PATH%;C:\Program Files\Microsoft MPI\Bin From ba12fa0fe318248aa60c5b45830c1fc0c3dab863 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Fri, 24 Feb 2023 10:44:23 +0100 Subject: [PATCH 16/19] Add more mutex for archive manipulation. PRevent deadlock at delete --- src/cpp/helpers/ArchiveReader.cpp | 5 +++++ src/cpp/helpers/ArchiveWriter.cpp | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/cpp/helpers/ArchiveReader.cpp b/src/cpp/helpers/ArchiveReader.cpp index 5cf600f21..d21f8aad6 100644 --- a/src/cpp/helpers/ArchiveReader.cpp +++ b/src/cpp/helpers/ArchiveReader.cpp @@ -12,6 +12,7 @@ ArchiveReader::ArchiveReader(const std::filesystem::path& archivePath) } ArchiveReader::ArchiveReader() : ArchiveIO() { Create(); } void ArchiveReader::Create() { + std::unique_lock lock(mutex_); mz_zip_reader_create(&pmz_zip_reader_instance_); } @@ -37,9 +38,11 @@ int32_t ArchiveReader::Open() { return err; } int32_t ArchiveReader::Close() { + std::unique_lock lock(mutex_); return mz_zip_reader_close(pmz_zip_reader_instance_); } void ArchiveReader::Delete() { + std::unique_lock lock(mutex_); mz_zip_reader_delete(&pmz_zip_reader_instance_); } @@ -190,6 +193,7 @@ std::vector ArchiveReader::ExtractPattern( std::vector ArchiveReader::ExtractPattern( const std::string& pattern, const std::string& exclude, const std::filesystem::path& destination) { + std::unique_lock lock(mutex_); if (!std::filesystem::is_directory(destination)) { std::ostringstream msg; msg << "ArchiveReader::ExtractPattern destination must be a directory " @@ -218,6 +222,7 @@ std::vector ArchiveReader::ExtractPattern( return result; } ArchiveReader::~ArchiveReader() { + std::unique_lock lock(mutex_); if (pmz_zip_reader_instance_) { mz_zip_reader_close(pmz_zip_reader_instance_); mz_zip_reader_delete(&pmz_zip_reader_instance_); diff --git a/src/cpp/helpers/ArchiveWriter.cpp b/src/cpp/helpers/ArchiveWriter.cpp index 7d0feabc0..4debe5052 100644 --- a/src/cpp/helpers/ArchiveWriter.cpp +++ b/src/cpp/helpers/ArchiveWriter.cpp @@ -15,6 +15,7 @@ ArchiveWriter::ArchiveWriter() : ArchiveIO() { } void ArchiveWriter::Create() { + std::unique_lock lock(mutex_); mz_zip_writer_create(&pmz_zip_writer_instance_); } void ArchiveWriter::InitFileInfo() { @@ -115,6 +116,7 @@ ArchiveWriter::~ArchiveWriter() { } int32_t ArchiveWriter::CloseInternal() { + std::unique_lock lock(mutex_); if (pmz_zip_writer_instance_) { return mz_zip_writer_close(pmz_zip_writer_instance_); } @@ -122,5 +124,6 @@ int32_t ArchiveWriter::CloseInternal() { } void ArchiveWriter::DeleteInternal() { + std::unique_lock lock(mutex_); return mz_zip_writer_delete(&pmz_zip_writer_instance_); } From 8aec3445af360ee4a23c822f300b11fe57f76a1d Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Tue, 28 Feb 2023 17:25:02 +0100 Subject: [PATCH 17/19] Fix concurrent access to archive files at the end of ProblemGeneration --- src/cpp/lpnamer/main/RunProblemGeneration.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cpp/lpnamer/main/RunProblemGeneration.cpp b/src/cpp/lpnamer/main/RunProblemGeneration.cpp index faea95e8f..6ffe5cba1 100644 --- a/src/cpp/lpnamer/main/RunProblemGeneration.cpp +++ b/src/cpp/lpnamer/main/RunProblemGeneration.cpp @@ -150,6 +150,8 @@ void RunProblemGeneration( problem_variables_from_zip_adapter); }); + reader->Close(); + reader->Delete(); CleanUpArchives(antares_archive_path, lpDir_, writer, tmpArchivePath); } else if (use_file_implementation) { std::shared_ptr writer; @@ -214,10 +216,10 @@ void CleanUpArchives(const std::filesystem::path& antares_archive_path, const std::filesystem::path& lpDir_, std::shared_ptr& writer, const std::filesystem::path& tmpArchivePath) { - std::filesystem::remove(antares_archive_path); - std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); writer->Close(); writer->Delete(); + std::filesystem::remove(antares_archive_path); + std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); } std::filesystem::path OpenTmpArchive(const std::filesystem::path& lpDir_, std::shared_ptr& writer) { From c56645616e2266926a85d763755d1c97a7014865 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 2 Mar 2023 13:21:22 +0100 Subject: [PATCH 18/19] Remove code duplication --- .../LinkProblemsGenerator.cpp | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp index 41ea68a7f..1af0f122a 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp @@ -2,16 +2,17 @@ #include #include +#include #include "IProblemProviderPort.h" #include "IProblemVariablesProviderPort.h" #include "IProblemWriter.h" #include "MPSFileProblemProviderAdapter.h" +#include "MpsTxtWriter.h" #include "ProblemVariablesFileAdapter.h" #include "ProblemVariablesZipAdapter.h" -#include "ZipProblemProviderAdapter.h" -#include "MpsTxtWriter.h" #include "VariableFileReader.h" +#include "ZipProblemProviderAdapter.h" #include "common_lpnamer.h" #include "helpers/StringUtils.h" #include "solver_utils.h" @@ -34,29 +35,8 @@ void LinkProblemsGenerator::treat( std::shared_ptr in_prblm = problem_provider->provide_problem(_solver_name); - ProblemVariables problem_variables = variable_provider->Provide(); - auto problem_modifier = ProblemModifier(logger_); - // auto path = - // "/tmp/pytest-of-marechaljas/zip-1/test_full_study_short_sequenti0/link-profile-with-empty-week/output/out/lp/zip/pre_change" - // + in_prblm->_name; in_prblm->write_prob_mps(path); - solver_rename_vars(in_prblm, problem_variables.variable_names); - - in_prblm = problem_modifier.changeProblem( - in_prblm, _links, problem_variables.ntc_columns, - problem_variables.direct_cost_columns, - problem_variables.indirect_cost_columns); - - // couplings creation - for (const ActiveLink &link : _links) { - for (const Candidate &candidate : link.getCandidates()) { - if (problem_modifier.has_candidate_col_id(candidate.get_name())) { - std::lock_guard guard(coupling_mutex_); - couplings[{candidate.get_name(), problem_name}] = - problem_modifier.get_candidate_col_id(candidate.get_name()); - } - } - } - writer->Write_problem(in_prblm); + treat(problem_name, couplings, std::move(writer), in_prblm, + std::move(variable_provider)); } void LinkProblemsGenerator::treat( From 402edcc4cc05f245c0922ad67729dc96764c8d94 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Thu, 2 Mar 2023 13:21:54 +0100 Subject: [PATCH 19/19] Remove commented code --- .../AntaresProblemToXpansionProblemTranslator.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp b/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp index ad67600db..63392e044 100644 --- a/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp +++ b/src/cpp/lpnamer/problem_modifier/AntaresProblemToXpansionProblemTranslator.cpp @@ -19,8 +19,7 @@ AntaresProblemToXpansionProblemTranslator::translateToXpansionProblem( std::vector tmp(constant->NombreDeVariables, 0); std::vector coltypes(constant->NombreDeVariables, 'C'); - // solver_addcols(problem, hebdo->CoutLineaire, tmp, {}, {}, hebdo->Xmin, - // hebdo->Xmax, coltypes, {}); + problem->add_cols(constant->NombreDeVariables, 0, hebdo->CoutLineaire.data(), tmp.data(), {}, {}, hebdo->Xmin.data(), hebdo->Xmax.data()); problem->add_rows(