TRUST_2_PDI.cpp

/****************************************************************************
* Copyright (c) 2026, CEA
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*****************************************************************************/
 
#include <TRUST_2_PDI.h>
#include <Motcle.h>
 
int TRUST_2_PDI::PDI_checkpoint_=0;
int TRUST_2_PDI::PDI_restart_=0;
int TRUST_2_PDI::PDI_initialized_=0;
std::vector<std::string> TRUST_2_PDI::shared_data_;
 
/*! @brief Generic method to share the dimensions of a TRUST DoubleTab with PDI
 *
 * @param (const DoubleTab& tab) the array we want to share with PDI
 * @param (Nom name) the name of the array
 * @param (int write) flag to specify if we want to write the dimensions or read into it
 */
void TRUST_2_PDI::share_TRUSTTab_dimensions(const DoubleTab& tab, const Nom& name, int write)
{
#ifdef HAS_PDI
  int nb_dim = tab.nb_dim();
  ArrOfInt dimensions(nb_dim);
  for(int i=0; i< nb_dim; i++)
    dimensions[i] = tab.dimension_tot(i) ? tab.dimension_tot(i) : 1; // can't share null data
 
  // the first dimension can vary from one process of the node to the other
  // so we take the larger one to fix it in the corresponding dataset of the DoubleTab
  int glob_dim_0;
  PE_Groups::get_node_group().mp_collective_op(&dimensions[0], &glob_dim_0, 1, Comm_Group::COLL_MAX);
 
  std::string name_str = name.getString();
  std::string dim_str = "dim_" + name_str;
  std::string glob_dim_str = "glob_dim_" + name_str;
 
  if(write)
    PDI_multi_expose("dimensions", dim_str.c_str(), dimensions.addr(), PDI_OUT, glob_dim_str.c_str(), &glob_dim_0, PDI_OUT, nullptr);
  else
    PDI_multi_expose("dimensions", dim_str.c_str(), dimensions.addr(), PDI_INOUT, glob_dim_str.c_str(), &glob_dim_0, PDI_INOUT, nullptr);
#endif
}
 
/*! @brief Generic method to share the type of a TRUST object
 *
 * @param (Nom name) the name of the object
 * @param (Nom type) type of the object
 */
void TRUST_2_PDI::share_type(const Nom& name, const Nom& type)
{
  Nom uname = Motcle(name);
  std::string tname = "TYPE_" + uname.getString();
  char* t = const_cast<char*>(type.getChar());
 
  std::string size = "size_TYPE_" + uname.getString();
  int sz = type.longueur();
 
  // here we expose the data, so that PDI can keep a copy of it without having to share it later
  write(size, &sz);
  write(tname, t);
}
 
/*! @brief Generic method to read the type of a TRUST object in the HDF5 file
 *
 * @param (Nom name) the name of the object
 * @param (Nom type) type of the object
 */
void TRUST_2_PDI::get_type(const Nom& name, Nom& type)
{
#ifdef HAS_PDI
  int tmp = -1;
  PDI_share("TYPES", &tmp, PDI_INOUT);
  Nom uname = Motcle(name);
  // getting size of string first
  std::string size = "size_TYPE_" + uname.getString();
  int sz = -1;
  PDI_share(size.c_str(), &sz, PDI_INOUT);
  trigger("get_" + size);
  PDI_reclaim(size.c_str());
  assert(sz>=0);
  type.getString().resize(sz);
 
  // getting string
  std::string tname = "TYPE_" + uname.getString();
  char* t = const_cast<char*>(type.getChar());
  PDI_share(tname.c_str(), t, PDI_INOUT);
  trigger("get_" + tname);
  PDI_reclaim(tname.c_str());
 
  PDI_reclaim("TYPES");
#endif
}
 
/*! @brief Generic method to prepare the restart of a computation
 * @param (OWN_PTR(Comm_Group)& nodeGroup) communicator that will be used to read data from the checkpoint files
 * @param (int& last_iteration) the index of the backup iteration we want to recover from
 * @param (double& tinit) the time from which we want to resume the calculation
 * @param (int resume_last_time) flag to specify if we want to resume from the last time or we want to recover from a specific time
 */
void TRUST_2_PDI::prepareRestart(OWN_PTR(Comm_Group)& nodeGroup, int& last_iteration, double& tinit, int resume_last_time)
{
  // Reading previous parallel configuration
  int prev_nb_proc = -1;
  int prev_nb_nodes = -1;
  if(Process::je_suis_maitre()) // if I'm the master (no need for everyone to read)
    {
#ifdef HAS_PDI
      PDI_multi_expose("ReadConfig", "nb_proc", &prev_nb_proc, PDI_INOUT, "nb_nodes", &prev_nb_nodes, PDI_INOUT, nullptr);
#endif
    }
  envoyer_broadcast(prev_nb_proc,0);
  envoyer_broadcast(prev_nb_nodes,0);
 
  // Check that we have the same number of procs used for checkpoint
  int nb_proc = Process::nproc();
  int nb_nodes = PE_Groups::get_node_group().get_number_of_nodes();
  if(nb_proc != prev_nb_proc)
    {
      Cerr << "TRUST_2_PDI::prepareRestart():: PDI Restart Error !" << finl;
      Cerr << "The backup file has been generated with " << prev_nb_proc << " processors." << finl;
      Cerr << "The current computation is launched with " << nb_proc << " processors." << finl;
      Cerr << "With PDI, you need to restart your computation with the same number of processors used for previous computation." << finl;
      Process::exit();
    }
  bool samePartition = nb_nodes == prev_nb_nodes;
  // If we have the same node partition as the one in the checkpoint files,
  // then we can use the current nodes for reading as well
  // otherwise, we need to read the checkpoint configuration file to recreate the proper communicator
  // and identify which file each MPI comm has to read
  if (!samePartition)
    {
      // array containing the node id of each processor during checkpoint
      ArrOfInt nodeRanks(nb_proc);
      if(Process::je_suis_maitre())
        {
          // trigger reading of metadata file to figure out to which group each processor should belong to
          // (reading by master only as the array should be small)
#ifdef HAS_PDI
          PDI_multi_expose("ReadNodeRanks", "nodeRanks", nodeRanks.data(), PDI_INOUT, nullptr);
#endif
        }
      // broadcasting the array to everyone
      envoyer_broadcast_array(nodeRanks.data(), nb_proc, 0);
      int nodeId = nodeRanks[Process::me()];
      // Recreating the same communicators as those used at the checkpoint
      // (every proc of the same MPI group should have the same pe_list)
      ArrOfInt pe_list;
      for(int p=0; p<nb_proc; p++)
        {
          if(nodeRanks[p] == nodeId)
            pe_list.append_array(p);
        }
      PE_Groups::create_group(pe_list, nodeGroup);
      if (PE_Groups::enter_group(nodeGroup.valeur()))
        {
          share_parallelism(nodeGroup.valeur(), nodeId);
          // we can exit the group as the communicators are shared with PDI
          PE_Groups::exit_group();
        }
    }
 
  // Get time scheme information
  int nb_sauv = -1;
  read("iter", &nb_sauv);
  std::vector<double> temps(nb_sauv+1);
  read("temps_sauvegardes", temps.data());
 
  // Restart from the last time
  if (resume_last_time)
    {
      // Look for the last time saved in checkpoint file to init current computation
      tinit = temps.back();
      last_iteration = nb_sauv;
    }
  else // resume from the requested time
    {
      // looking for tinit in backup file
      auto it = std::find_if(temps.begin(), temps.end(), [&](const double &t) { return std::fabs(t-tinit) < 1.e-8 ; } );
      if(it == temps.end())
        {
          Cerr << "------------------------------------------------------------------------------------" << finl;
          Cerr << "Time " << tinit << " not found in backup file. Please adjust tinit in your datafile "     << finl;
          Cerr << "Available times are:" << finl;
          for(auto t: temps)
            Cerr << t << " ";
          Cerr << finl << "------------------------------------------------------------------------------------" << finl;
          Process::exit();
        }
      last_iteration = (int)std::distance(temps.begin(),it);
    }
 
  // letting PDI know which iteration/time to read during restart
  write("iter", &last_iteration);
  write("temps", &tinit);
}