Allow fitting procedure on a different cluster and make the fitting database accessible via the MongoDB

docs/user/phonon/flows/fitting/fitting.md

@@ -395,3 +395,126 @@ Additional fit settings can again be passed using `fit_kwargs` or `**{...}`.
 > containing at least one entry for "rattled_dir", "phonon_dir" and "isolated_atom" **VASP** calculations, 
 > otherwise the code will not finish successfully.
 
+
+## Is it possible to run the DFT calculations and the MLIP fitting step on different machines?
+
+Very often, we might have the situation that our GPU does not share a hard drive with the compute cluster where we
+perform the VASP runs. In such situations, it is convenient to split up the computations.
+
+This can be done by e.g. using jobflow-remote and the following settings for VASP and fitting jobs. 
+The `local_worker` is the local machine (e.g., a GPU without slurm queue).
+
+```python
+from autoplex.auto.phonons.flows import CompleteDFTvsMLBenchmarkWorkflow, IterativeCompleteDFTvsMLBenchmarkWorkflow
+from jobflow_remote import submit_flow, set_run_config
+from atomate2.vasp.powerups import update_user_incar_settings
+from atomate2.vasp.powerups import update_vasp_custodian_handlers
+
+from atomate2.vasp.jobs.phonons import PhononDisplacementMaker
+from atomate2.settings import Atomate2Settings
+
+autoplex_flow = IterativeCompleteDFTvsMLBenchmarkWorkflow(max_iterations=3, rms_max=0.2,
+                                                          complete_dft_vs_ml_benchmark_workflow_0=CompleteDFTvsMLBenchmarkWorkflow(
+                                                              symprec=1e-3,
+                                                              run_fits_on_different_cluster=True,
+                                                              add_dft_phonon_struct=False,
+                                                              path_to_hyperparameters="/local_machine/mlip-phonon-defaults.json",
+                                                              apply_data_preprocessing=True,
+                                                              add_dft_rattled_struct=True,
+                                                              volume_custom_scale_factors=[1.0,1.0, 1.0],
+                                                              rattle_type=0, distort_type=0,
+                                                              rattle_std=0.1,  # maybe 0.1
+                                                              benchmark_kwargs={"relax_maker_kwargs": {
+                                                                  "relax_cell": False}},
+                                                              supercell_settings={"min_length": 5,
+                                                                                  "max_length": 15,
+                                                                                  "min_atoms": 10,
+                                                                                  "max_atoms": 300,
+                                                                                  "fallback_min_length": 9},
+                                                              # settings that worked with a GAP
+                                                              split_ratio=0.33,
+                                                              regularization=False,
+                                                              separated=False,
+                                                              num_processes_fit=48,
+                                                              displacement_maker=phonon_displacement_maker,
+                                                              phonon_bulk_relax_maker=phonon_bulk_relax_maker,
+                                                              phonon_static_energy_maker=phonon_static_energy_maker,
+                                                              rattled_bulk_relax_maker=phonon_bulk_relax_maker,
+                                                              isolated_atom_maker=static_isolated_atom_maker),
+                                                          complete_dft_vs_ml_benchmark_workflow_1=CompleteDFTvsMLBenchmarkWorkflow(
+                                                              symprec=1e-3,
+                                                              run_fits_on_different_cluster=True,
+                                                              path_to_hyperparameters="/local_machine/mlip-phonon-defaults.json",
+                                                              apply_data_preprocessing=True,
+                                                              add_dft_phonon_struct=False,
+                                                              add_dft_rattled_struct=True,
+                                                              volume_custom_scale_factors=[1.0],
+                                                              rattle_type=0, distort_type=0,
+                                                              rattle_std=0.1,  
+                                                              benchmark_kwargs={"relax_maker_kwargs": {
+                                                                  "relax_cell": False}},
+                                                              supercell_settings={"min_length": 5,
+                                                                                  "max_length": 15,
+                                                                                  "min_atoms": 10,
+                                                                                  "max_atoms": 300,
+                                                                                  "fallback_min_length": 9},
+                                                              split_ratio=0.33,
+                                                              regularization=False,
+                                                              separated=False,
+                                                              num_processes_fit=48,
+                                                              displacement_maker=phonon_displacement_maker,
+                                                              phonon_bulk_relax_maker=phonon_bulk_relax_maker,
+                                                              phonon_static_energy_maker=phonon_static_energy_maker,
+                                                              rattled_bulk_relax_maker=phonon_bulk_relax_maker,
+                                                              isolated_atom_maker=static_isolated_atom_maker)).make(
+    structure_list=structure_list, mp_ids=mpids, benchmark_structures=benchmark_structure_list,
+    benchmark_mp_ids=mpbenchmark,
+    rattle_seed=0,
+    fit_kwargs_list=[{
+        "soap": {"delta": 1.0, "l_max": 12, "n_max": 10,
+                 "atom_sigma": 0.5, "zeta": 4, "cutoff": 5.0,
+                 "cutoff_transition_width": 1.0,
+                 "central_weight": 1.0, "n_sparse": 6000, "f0": 0.0,
+                 "covariance_type": "dot_product",
+                 "sparse_method": "cur_points"},
+        "general": {"two_body": True, "three_body": False, "soap": True,
+                    "default_sigma": "{0.001 0.05 0.05 0.0}", "sparse_jitter": 1.0e-8, }}]
+)
+
+resources = {"nodes": 1, "partition": "micro", "time": "00:55:00", "ntasks": 48, "qverbatim": "#SBATCH --get-user-env",
+             "mail_user": "your_email@adress", "mail_type": "ALL", "account": "xxxxxx"}
+
+resources_phon = {"nodes": 3, "partition": "micro", "time": "0:55:00", "ntasks": 144,
+                  "qverbatim": "#SBATCH --get-user-env",
+                  "mail_user": "your_email@adress", "mail_type": "ALL", "account": "xxxxxx"}
+
+resources_ratt = {"nodes": 3, "partition": "micro", "time": "0:55:00", "ntasks": 144,
+                  "qverbatim": "#SBATCH --get-user-env",
+                  "mail_user": "your_email@adress", "mail_type": "ALL", "account": "xxxxxx"}
+
+resources_mlip = {}
+autoplex_flow = set_run_config(autoplex_flow, name_filter="dft static", resources=resources, worker="supermuc_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="stat_iso_atom", resources=resources, worker="supermuc_worker")
+
+autoplex_flow = set_run_config(autoplex_flow, name_filter="dft phonon static", resources=resources_phon, worker="supermuc_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="static", resources=resources_phon, worker="supermuc_worker")
+
+autoplex_flow = set_run_config(autoplex_flow, name_filter="dft rattle static", resources=resources_ratt, worker="supermuc_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="tight relax", resources=resources, worker="supermuc_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="dft tight relax", resources=resources, worker="supermuc_worker")
+
+autoplex_flow = set_run_config(autoplex_flow, name_filter="machine_learning_fit", resources=resources_mlip, worker="local_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="gap phonon static", resources=resources_mlip, worker="local_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="Force field", resources=resources_mlip, worker="local_worker")
+autoplex_flow = set_run_config(autoplex_flow, name_filter="data_preprocessing_for_fitting", resources=resources, worker="supermuc_worker")
+
+
+autoplex_flow = update_user_incar_settings(autoplex_flow, {"NPAR": 4})
+
+autoplex_flow = update_vasp_custodian_handlers(autoplex_flow, custom_handlers={})
+
+autoplex_flow.name = "small Sn test, test without phonon2"
+
+# submit the workflow to jobflow-remote
+print(submit_flow(autoplex_flow, worker="local_worker", resources=resources_mlip, project="phonons_qha"))
+```

docs/user/phonon/flows/generation/data.md

@@ -189,8 +189,8 @@ INFO Starting job - reduce_supercell_size_job_mp-22905
 INFO Finished job - reduce_supercell_size_job_mp-22905 
 INFO Starting job - rattled supercells_mp-22905 
 INFO Finished job - rattled supercells_mp-22905 
-INFO Starting job - tight relax_mp-22905 
-INFO Finished job - tight relax_mp-22905 
+INFO Starting job - dft tight relax_mp-22905 
+INFO Finished job - dft tight relax_mp-22905 
 INFO Starting job - generate_randomized_structures_mp-22905 
 INFO Finished job - generate_randomized_structures_mp-22905 
 INFO Starting job - run_phonon_displacements_mp-22905 
@@ -202,10 +202,10 @@ INFO Finished job - dft rattle static 2/12_mp-22905
 ...
 INFO Starting job - single-atom displaced supercells_mp-22905 
 INFO Finished job - single-atom displaced supercells_mp-22905 
-INFO Starting job - tight relax_mp-22905
-INFO Finished job - tight relax_mp-22905 
-INFO Starting job - static_mp-22905 
-INFO Finished job - static_mp-22905 
+INFO Starting job - dft tight relax_mp-22905
+INFO Finished job - dft tight relax_mp-22905 
+INFO Starting job - dft static_mp-22905 
+INFO Finished job - dft static_mp-22905 
 INFO Starting job - generate_phonon_displacements_mp-22905 
 INFO Finished job - generate_phonon_displacements_mp-22905 
 INFO Starting job - run_phonon_displacements_mp-22905 

src/autoplex/auto/phonons/flows.py

@@ -176,6 +176,9 @@ class CompleteDFTvsMLBenchmarkWorkflow(Maker):
         Name of the glue.xml file path.
     use_defaults_fitting: bool
         Use the fit defaults.
+    run_fits_on_different_cluster: bool
+        Allows you to run fits on a different cluster than DFT (will transfer
+        fit database via MongoDB, might be slow).
     """
 
     name: str = "add_data"
@@ -226,6 +229,7 @@ class CompleteDFTvsMLBenchmarkWorkflow(Maker):
     glue_xml: bool = False
     glue_file_path: str = "glue.xml"
     use_defaults_fitting: bool = True
+    run_fits_on_different_cluster: bool = False
 
     def make(
         self,
@@ -400,6 +404,7 @@ def make(
                 separated=self.separated,
                 regularization=self.regularization,
                 distillation=self.distillation,
+                run_fits_on_different_cluster=self.run_fits_on_different_cluster,
             ).make(
                 species_list=isoatoms.output["species"],
                 isolated_atom_energies=isoatoms.output["energies"],
@@ -826,6 +831,7 @@ class CompleteDFTvsMLBenchmarkWorkflowMPSettings(CompleteDFTvsMLBenchmarkWorkflo
     phonon_bulk_relax_maker: BaseVaspMaker = field(
         default_factory=lambda: MPGGADoubleRelaxMaker.from_relax_maker(
             MPGGARelaxMaker(
+                name="dft tight relax",
                 run_vasp_kwargs={"handlers": ()},
                 input_set_generator=MPRelaxSet(
                     force_gamma=True,
@@ -849,6 +855,7 @@ class CompleteDFTvsMLBenchmarkWorkflowMPSettings(CompleteDFTvsMLBenchmarkWorkflo
     rattled_bulk_relax_maker: BaseVaspMaker = field(
         default_factory=lambda: MPGGADoubleRelaxMaker.from_relax_maker(
             MPGGARelaxMaker(
+                name="dft tight relax",
                 run_vasp_kwargs={"handlers": ()},
                 input_set_generator=MPRelaxSet(
                     force_gamma=True,

src/autoplex/auto/phonons/jobs.py

@@ -275,7 +275,6 @@ def complete_benchmark(  # this function was put here to prevent circular import
 
     for path in ml_path:
         suffix = Path(path).name
-        print(suffix)
         if suffix == "without_regularization":
             suffix = "without_reg"
         if suffix not in ["phonon", "rattled"]:
@@ -541,6 +540,7 @@ def dft_phonopy_gen_data(
     if phonon_bulk_relax_maker is None:
         phonon_bulk_relax_maker = DoubleRelaxMaker.from_relax_maker(
             TightRelaxMaker(
+                name="dft tight relax",
                 run_vasp_kwargs={"handlers": {}},
                 input_set_generator=TightRelaxSetGenerator(
                     user_incar_settings={
@@ -567,6 +567,7 @@ def dft_phonopy_gen_data(
 
     if phonon_static_energy_maker is None:
         phonon_static_energy_maker = StaticMaker(
+            name="dft static",
             input_set_generator=StaticSetGenerator(
                 auto_ispin=False,
                 user_incar_settings={
@@ -584,13 +585,17 @@ def dft_phonopy_gen_data(
                     # to be removed
                     "NPAR": 4,
                 },
-            )
+            ),
         )
 
     # always set autoplex default as job name
     phonon_displacement_maker.name = "dft phonon static"
-    phonon_bulk_relax_maker.name = "tight relax"
-    phonon_static_energy_maker.name = "static"
+    phonon_static_energy_maker.name = "dft static"
+    try:
+        phonon_bulk_relax_maker.relax_maker1.name = "dft tight relax"
+        phonon_bulk_relax_maker.relax_maker2.name = "dft tight relax"
+    except AttributeError:
+        phonon_bulk_relax_maker.name = "dft tight relax"
 
     for displacement in displacements:
         dft_phonons = DFTPhononMaker(
@@ -725,7 +730,7 @@ def dft_random_gen_data(
 
     # always set autoplex default as job name
     displacement_maker.name = "dft rattle static"
-    rattled_bulk_relax_maker.name = "tight relax"
+    rattled_bulk_relax_maker.name = "dft tight relax"
 
     # TODO: decide if we should remove the additional response here as well
     # looks like only the output is changing

src/autoplex/data/common/jobs.py

@@ -237,7 +237,6 @@ def generate_randomized_structures(
     """
     if supercell_matrix is None:
         supercell_matrix = [[2, 0, 0], [0, 2, 0], [0, 0, 2]]
-
     supercell = get_supercell(
         unitcell=get_phonopy_structure(structure),
         supercell_matrix=supercell_matrix,

src/autoplex/data/common/utils.py

@@ -658,6 +658,7 @@ def force_plot(
 
     """
     in_atoms = ase.io.read(in_file, ":")
+
     for atoms in in_atoms[:-1]:
         kwargs = {
             "forces": atoms.arrays["REF_forces"],