Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/slurm/config.ini
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ resource = gpu2
runner = autotest_gpu
[case.035]
suite = 05_rtTDDFT
name = 10_NO_HHG_TDDFT
name = 10_NO_MD_boundary_shift
resource = gpu2
runner = autotest_gpu
[case.036]
Expand Down
60 changes: 60 additions & 0 deletions source/source_base/parallel_cell.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#include "source_base/parallel_cell.h"

#include <stdexcept>
#include <vector>

namespace ModuleBase
{
#ifdef __MPI
Expand Down Expand Up @@ -69,6 +72,63 @@ double CommunicationDomain::max(double value) const
return value;
}

std::string CommunicationDomain::gather_text(const std::string& value, int maximum_count) const
{
if (maximum_count <= 0)
{
throw std::invalid_argument("CommunicationDomain text batch limit must be positive.");
}
const std::size_t count_limit = static_cast<std::size_t>(maximum_count);
const int local_overflow = value.size() > count_limit;
if (max(local_overflow) != 0)
{
throw std::length_error("CommunicationDomain text exceeds the batch limit.");
}
#ifdef __MPI
if (communicator_ != MPI_COMM_NULL)
{
const int local_size = static_cast<int>(value.size());
const int process_count = size();
std::vector<int> lengths(process_count, 0);
std::vector<int> offsets(process_count, 0);
MPI_Gather(&local_size, 1, MPI_INT, lengths.data(), 1, MPI_INT, 0, communicator_);
std::size_t total_size = 0;
int total_overflow = 0;
if (rank_ == 0)
{
for (int rank = 0; rank < process_count; ++rank)
{
if (total_size + lengths[rank] > count_limit)
{
total_overflow = 1;
break;
}
offsets[rank] = static_cast<int>(total_size);
total_size += lengths[rank];
}
}
MPI_Bcast(&total_overflow, 1, MPI_INT, 0, communicator_);
if (total_overflow != 0)
{
throw std::length_error("CommunicationDomain gathered text exceeds the batch limit.");
}
std::string gathered;
if (rank_ == 0)
{
gathered.resize(total_size);
}
char* buffer = nullptr;
if (!gathered.empty())
{
buffer = &gathered[0];
}
MPI_Gatherv(value.data(), local_size, MPI_CHAR, buffer, lengths.data(), offsets.data(), MPI_CHAR, 0, communicator_);
return gathered;
}
#endif
return value;
}

CommunicationDomain world_comm_domain()
{
CommunicationDomain comm_domain;
Expand Down
5 changes: 5 additions & 0 deletions source/source_base/parallel_cell.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#ifndef PARALLEL_CELL_H
#define PARALLEL_CELL_H

#include <string>

#ifdef __MPI
#include <mpi.h>
#endif
Expand All @@ -20,6 +22,9 @@ class CommunicationDomain
int max(int value) const;
int sum(int value) const;
double max(double value) const;
/// Gather one bounded text batch on rank zero; other ranks receive an empty string.
/// maximum_count bounds both local payloads and their total in MPI count units.
std::string gather_text(const std::string& value, int maximum_count) const;

private:
#ifdef __MPI
Expand Down
12 changes: 9 additions & 3 deletions source/source_md/md_base.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,15 @@ void MD_base::update_pos()
}
pos = pos * mdcell.GT();
atom.frac += pos;
atom.frac.x -= std::floor(atom.frac.x);
atom.frac.y -= std::floor(atom.frac.y);
atom.frac.z -= std::floor(atom.frac.z);
for (int k = 0; k < 3; ++k)
{
atom.frac[k] -= std::floor(atom.frac[k]);
// A tiny negative value can round to exactly 1.0 after adding one.
if (atom.frac[k] >= 1.0)
{
atom.frac[k] = 0.0;
}
}
atom.cart = atom.frac * mdcell.latvec();
}

Expand Down
204 changes: 151 additions & 53 deletions source/source_md/md_func.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,21 @@
#include "source_cell/domain_decomposition.h"

#include "source_base/global_variable.h"
#include "source_base/parallel_cell.h"
#include "source_base/timer.h"
#include "source_io/module_output/output_log.h"

#include <cerrno>
#include <cstring>
#include <fcntl.h>
#include <unistd.h>
#include <algorithm>
#include <fstream>
#include <limits>
#include <map>
#include <sstream>
#include <stdexcept>
#include <utility>


namespace MD_func
{
#ifdef __MPI
namespace
{
bool write_dump_at(const int file, const std::string& data, const MPI_Offset offset)
{
std::size_t written = 0;
while (written < data.size())
{
const ssize_t count = pwrite(file, data.data() + written, data.size() - written,
static_cast<off_t>(offset + written));
if (count <= 0) return false;
written += static_cast<std::size_t>(count);
}
return true;
}
}
#endif

double gaussrand()
{
Expand Down Expand Up @@ -407,13 +394,66 @@ void print_stress(std::ofstream& ofs, const ModuleBase::matrix& virial, const Mo
return;
}

namespace
{
void write_dump_rows(std::ostream& output,
const std::string& text,
const std::int64_t begin,
const std::int64_t end)
{
// Spatial ownership is unrelated to the stable type-major atom index.
std::map<std::int64_t, std::string> rows;
std::istringstream input(text);
std::string line;
while (std::getline(input, line))
{
std::istringstream record(line);
std::int64_t index = 0;
if (!(record >> index) || index < begin || index >= end)
{
ModuleBase::WARNING_QUIT("MD_func::dump_info", "invalid MD_dump atom index.");
}
const bool inserted = rows.emplace(index, line).second;
if (!inserted)
{
ModuleBase::WARNING_QUIT("MD_func::dump_info", "duplicate MD_dump atom index.");
}
}
const std::size_t expected_count = static_cast<std::size_t>(end - begin);
if (rows.size() != expected_count)
{
ModuleBase::WARNING_QUIT("MD_func::dump_info", "missing MD_dump atoms.");
}
for (const auto& row : rows)
{
output << row.second << "\n";
}
}
} // namespace

void dump_info(const int& step,
const std::string& global_out_dir,
const MDCell& mdcell,
const MD_para& mdp,
const bool cal_stress,
const ModuleBase::matrix& virial)
{
const int max_batch_bytes = 1024 * 1024;
dump_info(step, global_out_dir, mdcell, mdp, cal_stress, virial, max_batch_bytes);
}

void dump_info(const int& step,
const std::string& global_out_dir,
const MDCell& mdcell,
const MD_para& mdp,
const bool cal_stress,
const ModuleBase::matrix& virial,
const int max_batch_bytes)
{
if (max_batch_bytes <= 0)
{
throw std::invalid_argument("MD_dump batch budget must be positive.");
}
std::stringstream file;
file << global_out_dir << "MD_dump";
const double unit_pos = mdcell.lat0() / ModuleBase::ANGSTROM_AU;
Expand Down Expand Up @@ -441,44 +481,102 @@ void dump_info(const int& step,
{
type_offsets[it + 1] = type_offsets[it] + mdcell.type_atom_counts()[it];
}
std::ostringstream local;
local << std::fixed << std::setprecision(12);
for (int i = 0; i < mdcell.owned_atoms().size(); ++i)
const auto format_atom = [&](const LocalAtom& atom) -> std::string
{
const LocalAtom& atom = mdcell.owned_atoms()[static_cast<std::size_t>(i)];
local << " " << type_offsets[static_cast<std::size_t>(atom.type)] + atom.type_index
<< " " << mdcell.type_labels()[static_cast<std::size_t>(atom.type)]
<< " " << atom.cart.x * unit_pos << " " << atom.cart.y * unit_pos << " " << atom.cart.z * unit_pos;
std::ostringstream row;
row << std::fixed << std::setprecision(12);
row << " " << type_offsets[static_cast<std::size_t>(atom.type)] + atom.type_index
<< " " << mdcell.type_labels()[static_cast<std::size_t>(atom.type)]
<< " " << atom.cart.x * unit_pos << " " << atom.cart.y * unit_pos << " " << atom.cart.z * unit_pos;
if (mdp.dump_force)
local << " " << atom.force.x * unit_force << " " << atom.force.y * unit_force << " " << atom.force.z * unit_force;
row << " " << atom.force.x * unit_force << " " << atom.force.y * unit_force << " " << atom.force.z * unit_force;
if (mdp.dump_vel)
local << " " << atom.vel.x * unit_vel << " " << atom.vel.y * unit_vel << " " << atom.vel.z * unit_vel;
local << "\n";
row << " " << atom.vel.x * unit_vel << " " << atom.vel.y * unit_vel << " " << atom.vel.z * unit_vel;
row << "\n";
return row.str();
};
std::vector<std::pair<std::int64_t, const LocalAtom*>> ordered_atoms;
const std::size_t local_atom_count = mdcell.owned_atoms().size();
ordered_atoms.reserve(local_atom_count);
int local_row_bytes = 0;
int local_overflow = 0;
const std::size_t maximum_count = std::numeric_limits<int>::max();
for (const LocalAtom& atom : mdcell.owned_atoms())
{
const std::int64_t index = type_offsets[static_cast<std::size_t>(atom.type)] + atom.type_index;
const LocalAtom* atom_pointer = &atom;
ordered_atoms.emplace_back(index, atom_pointer);
const std::string row = format_atom(atom);
if (row.size() > maximum_count)
{
local_overflow = 1;
}
else
{
const int row_bytes = static_cast<int>(row.size());
local_row_bytes = std::max(local_row_bytes, row_bytes);
}
}
local << "\n\n";
std::sort(ordered_atoms.begin(), ordered_atoms.end(),
[](const std::pair<std::int64_t, const LocalAtom*>& left,
const std::pair<std::int64_t, const LocalAtom*>& right)
{ return left.first < right.first; });
ModuleBase::CommunicationDomain domain;
#ifdef __MPI
const MPI_Comm comm = mdcell.communicator();
int rank = 0; MPI_Comm_rank(comm, &rank);
MPI_Offset base = 0;
if (rank == 0)
{
const int file_descriptor = open(file.str().c_str(), O_CREAT | O_WRONLY | (step == 0 ? O_TRUNC : O_APPEND), 0666);
if (file_descriptor < 0) ModuleBase::WARNING_QUIT("MD_func::dump_info", "cannot open MD_dump.");
base = lseek(file_descriptor, 0, SEEK_END);
if (!write_dump_at(file_descriptor, header.str(), base) || close(file_descriptor) != 0) ModuleBase::WARNING_QUIT("MD_func::dump_info", "cannot write MD_dump header.");
}
MPI_Bcast(&base, 1, MPI_OFFSET, 0, comm);
const MPI_Offset atom_offset = base + static_cast<MPI_Offset>(header.str().size());
const MPI_Offset local_size = static_cast<MPI_Offset>(local.str().size());
MPI_Offset rank_offset = 0;
MPI_Exscan(&local_size, &rank_offset, 1, MPI_OFFSET, MPI_SUM, comm);
if (rank == 0) rank_offset = 0;
const int file_descriptor = open(file.str().c_str(), O_WRONLY);
if (file_descriptor < 0 || !write_dump_at(file_descriptor, local.str(), atom_offset + rank_offset) || close(file_descriptor) != 0) ModuleBase::WARNING_QUIT("MD_func::dump_info", "cannot write MD_dump atoms.");
#else
std::ofstream ofs(file.str().c_str(), step == 0 ? std::ios::trunc : std::ios::app);
ofs << header.str() << local.str();
domain.initialize(mdcell.communicator());
#endif
if (domain.max(local_overflow) != 0)
{
throw std::length_error("A single MD_dump row exceeds the MPI count range.");
}
const int maximum_row_bytes = domain.max(local_row_bytes);
const int row_bytes = std::max(maximum_row_bytes, 1);
const int batch_bytes = std::max(max_batch_bytes, row_bytes);
const std::int64_t batch_atoms = batch_bytes / row_bytes;
std::ofstream ofs;
if (domain.rank() == 0)
{
const std::string filename = file.str();
const std::ios_base::openmode mode = step == 0 ? std::ios::trunc : std::ios::app;
ofs.open(filename.c_str(), mode);
if (!ofs)
{
ModuleBase::WARNING_QUIT("MD_func::dump_info", "cannot open MD_dump.");
}
ofs << header.str();
}
std::size_t local_index = 0;
for (std::int64_t begin = 0; begin < type_offsets.back();)
{
// Each global-index range contains at most batch_atoms rows across all ranks.
// Counts and displacements are bounded per batch, independent of frame size.
const std::int64_t remaining = type_offsets.back() - begin;
const std::int64_t count = std::min(batch_atoms, remaining);
const std::int64_t end = begin + count;
std::ostringstream local;
while (local_index < ordered_atoms.size() && ordered_atoms[local_index].first < end)
{
const LocalAtom& atom = *ordered_atoms[local_index].second;
local << format_atom(atom);
++local_index;
}
const std::string local_text = local.str();
const std::string gathered = domain.gather_text(local_text, batch_bytes);
if (domain.rank() == 0)
{
write_dump_rows(ofs, gathered, begin, end);
}
begin = end;
}
if (domain.rank() == 0)
{
ofs << "\n\n";
ofs.close();
if (!ofs)
{
ModuleBase::WARNING_QUIT("MD_func::dump_info", "cannot write MD_dump.");
}
}
}

void get_mass_mbl(const UnitCell& unit_in,
Expand Down
9 changes: 9 additions & 0 deletions source/source_md/md_func.h
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,15 @@ void dump_info(const int& step,
const bool cal_stress,
const ModuleBase::matrix& virial);

/// Explicit buffer budget for bounded dump collectives; one row is the minimum batch.
void dump_info(const int& step,
const std::string& global_out_dir,
const MDCell& mdcell,
const MD_para& mdp,
const bool cal_stress,
const ModuleBase::matrix& virial,
const int max_batch_bytes);

/**
* @brief obtain the atomic mass and whether the freedom is fixed
*
Expand Down
Loading
Loading