Cloned SEACAS for EXODUS library with extra build files for internal package management.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1549 lines
56 KiB

2 years ago
/*
* Copyright(C) 1999-2023 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
* See packages/seacas/LICENSE for details
*/
#include "copy_string_cpp.h"
#include "exodusII.h" // for ex_close, etc
#include "fmt/chrono.h"
#include "fmt/ostream.h"
#include "nem_spread.h" // for NemSpread, second, etc
#include "pe_common.h" // for PEX_MAX
#include "ps_pario_const.h" // for PIO_Time_Array
#include "rf_allo.h" // for safe_free, array_alloc
#include "rf_io_const.h" // for Debug_Flag
#include "sort_utils.h" // for gds_iqsort
#include <cassert> // for assert
#include <cstddef> // for size_t
#include <cstdio> // for nullptr, etc
#include <cstdlib> // for exit, free, malloc
#include <cstring> // for strlen, memset, etc
#include <ctime> // for asctime, localtime, time, etc
#include <numeric>
#include <vector> // for vector
template <typename INT> struct ELEM_COMM_MAP;
template <typename INT> struct NODE_COMM_MAP;
#if (__cplusplus >= 201703L)
#define FALL_THROUGH [[fallthrough]]
#elif defined(__GNUC__) && __GNUC__ >= 7 && !__INTEL_COMPILER
#define FALL_THROUGH [[gnu::fallthrough]]
#else
#define FALL_THROUGH ((void)0)
#endif
namespace {
template <typename INT>
void reverse_map(INT *global, int p01, size_t gsize, INT *glmap, INT *index, INT *mapout);
template <typename INT>
void reverse_map(const std::vector<INT> &global, int p01, size_t gsize,
const std::vector<INT> &glmap, INT *index, std::vector<INT> &mapout);
} // namespace
/*
* need this variable for the 0 processor to hold on to the correct
* Exodus II database title
*/
extern std::string GeomTitle;
/****************************************************************************/
/* This function writes parallel specific mesh information out to the */
/* parallel disk(s) for each processor in the salsa run. */
/* */
/* Author(s): Gary L. Hennigan (1421) */
/*--------------------------------------------------------------------------*/
/* Revision History */
/* */
/* Gary Hennigan: */
/* 11 November 1993 - Date of first working version on the nCube. */
/****************************************************************************/
template void NemSpread<float, int>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc, std::vector<int> &Num_Nodes_In_NS,
std::vector<int> &Num_Elems_In_SS,
std::vector<int> &Num_Elems_In_EB);
template void NemSpread<double, int>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc,
std::vector<int> &Num_Nodes_In_NS,
std::vector<int> &Num_Elems_In_SS,
std::vector<int> &Num_Elems_In_EB);
template void NemSpread<double, int64_t>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc,
std::vector<int64_t> &Num_Nodes_In_NS,
std::vector<int64_t> &Num_Elems_In_SS,
std::vector<int64_t> &Num_Elems_In_EB);
template void NemSpread<float, int64_t>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc,
std::vector<int64_t> &Num_Nodes_In_NS,
std::vector<int64_t> &Num_Elems_In_SS,
std::vector<int64_t> &Num_Elems_In_EB);
template <typename T, typename INT>
void NemSpread<T, INT>::write_parExo_data(int mesh_exoid, int max_name_length, int iproc,
std::vector<INT> &Num_Nodes_In_NS,
std::vector<INT> &Num_Elems_In_SS,
std::vector<INT> &Num_Elems_In_EB)
{
/* Performance metrics. */
size_t bytes_out = 0;
double total_out_time = 0.0;
double tt1;
int error;
/****************************BEGIN EXECUTION*********************************/
/*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
/* PARALLEL EXODUSII SECTION */
/*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int proc_for = Proc_Ids[iproc];
int num_proc_for = Proc_Info[0];
size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
globals.Num_External_Nodes[iproc];
size_t itotal_elems = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
bytes_out += 5 * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting init global info in file id: {}\n", mesh_exoid);
fmt::print("\tNumber Global Nodes: {}\n", globals.Num_Node);
fmt::print("\tNumber Global Elements: {}\n", globals.Num_Elem);
fmt::print("\tNumber Global Element Blocks: {}\n", globals.Num_Elem_Blk);
fmt::print("\tNumber Global Node Sets: {}\n", globals.Num_Node_Set);
fmt::print("\tNumber Global Side Sets: {}\n", globals.Num_Side_Set);
}
if (ex_put_init_global(mesh_exoid, globals.Num_Node, globals.Num_Elem, globals.Num_Elem_Blk,
globals.Num_Node_Set, globals.Num_Side_Set) < 0) {
fmt::print(stderr,
"[{}]: ERROR, Unable to put global initial "
"information in parallel mesh file!\n",
__func__);
exit(1);
}
PIO_Time_Array[0] = (second() - tt1);
total_out_time += PIO_Time_Array[0];
/*
* Find the number of distinct nodal communication maps that will
* be needed. This assumes that there is only a single input
* nodal communication map that stores information for all
* processors.
*/
int ncomm_cnt = 0;
std::vector<NODE_COMM_MAP<INT>> n_comm_map;
std::vector<INT> n_comm_ids;
std::vector<INT> n_comm_ncnts;
if (globals.Num_N_Comm_Maps[iproc] > 0 && globals.N_Comm_Map[iproc].node_cnt != 0) {
ncomm_cnt = 1;
ex_entity_id itemp = globals.N_Comm_Map[iproc].proc_ids[0];
/* First find the count */
for (size_t i1 = 1; i1 < globals.N_Comm_Map[iproc].node_cnt; i1++) {
if (globals.N_Comm_Map[iproc].proc_ids[i1] != itemp) {
ncomm_cnt++;
itemp = globals.N_Comm_Map[iproc].proc_ids[i1];
}
}
/* Allocate memory for the nodal communication maps */
n_comm_map.resize(ncomm_cnt);
n_comm_ids.resize(ncomm_cnt);
n_comm_ncnts.resize(ncomm_cnt);
/* Find the size of each map */
ncomm_cnt = 0;
itemp = globals.N_Comm_Map[iproc].proc_ids[0];
n_comm_map[0].node_cnt = 1;
for (size_t i1 = 1; i1 < globals.N_Comm_Map[iproc].node_cnt; i1++) {
if (globals.N_Comm_Map[iproc].proc_ids[i1] != itemp) {
itemp = globals.N_Comm_Map[iproc].proc_ids[i1];
ncomm_cnt++;
n_comm_map[ncomm_cnt].node_cnt = 1;
}
else {
(n_comm_map[ncomm_cnt].node_cnt)++;
}
}
ncomm_cnt++;
size_t temp = 0;
/* Allocate memory for the maps */
for (int i1 = 0; i1 < ncomm_cnt; i1++) {
n_comm_map[i1].proc_ids.resize(n_comm_map[i1].node_cnt);
n_comm_map[i1].node_ids.resize(n_comm_map[i1].node_cnt);
for (size_t i2 = 0; i2 < n_comm_map[i1].node_cnt; i2++) {
n_comm_map[i1].proc_ids[i2] = globals.N_Comm_Map[iproc].proc_ids[temp];
n_comm_map[i1].node_ids[i2] = globals.N_Comm_Map[iproc].node_ids[temp++];
}
n_comm_ncnts[i1] = n_comm_map[i1].node_cnt;
n_comm_ids[i1] = n_comm_map[i1].proc_ids[0];
}
}
else {
ncomm_cnt = 0;
}
/*
* Find the number of distinct elemental communication maps that will
* be needed. This assumes that there is only a single input
* elemental communication map that stores information for all
* processors.
*/
INT ecomm_cnt = 0;
std::vector<INT> e_comm_ids;
std::vector<INT> e_comm_ecnts;
std::vector<ELEM_COMM_MAP<INT>> e_comm_map;
if (globals.Num_E_Comm_Maps[iproc] > 0) {
ecomm_cnt = 1;
INT itemp = globals.E_Comm_Map[iproc].proc_ids[0];
/* First find the count */
for (size_t i1 = 1; i1 < globals.E_Comm_Map[iproc].elem_cnt; i1++) {
if (globals.E_Comm_Map[iproc].proc_ids[i1] != itemp) {
ecomm_cnt++;
itemp = globals.E_Comm_Map[iproc].proc_ids[i1];
}
}
/* Allocate memory for the elemental communication maps */
e_comm_map.resize(ecomm_cnt);
e_comm_ids.resize(ecomm_cnt);
e_comm_ecnts.resize(ecomm_cnt);
/* Find the size of each map */
ecomm_cnt = 0;
itemp = globals.E_Comm_Map[iproc].proc_ids[0];
e_comm_map[0].elem_cnt = 1;
for (size_t i1 = 1; i1 < globals.E_Comm_Map[iproc].elem_cnt; i1++) {
if (globals.E_Comm_Map[iproc].proc_ids[i1] != itemp) {
itemp = globals.E_Comm_Map[iproc].proc_ids[i1];
ecomm_cnt++;
e_comm_map[ecomm_cnt].elem_cnt = 1;
}
else {
(e_comm_map[ecomm_cnt].elem_cnt)++;
}
}
ecomm_cnt++;
itemp = 0;
/* Allocate memory for the maps */
for (INT i1 = 0; i1 < ecomm_cnt; i1++) {
e_comm_map[i1].proc_ids.resize(e_comm_map[i1].elem_cnt);
e_comm_map[i1].elem_ids.resize(e_comm_map[i1].elem_cnt);
e_comm_map[i1].side_ids.resize(e_comm_map[i1].elem_cnt);
for (size_t i2 = 0; i2 < e_comm_map[i1].elem_cnt; i2++) {
e_comm_map[i1].proc_ids[i2] = globals.E_Comm_Map[iproc].proc_ids[itemp];
e_comm_map[i1].elem_ids[i2] = globals.E_Comm_Map[iproc].elem_ids[itemp];
e_comm_map[i1].side_ids[i2] = globals.E_Comm_Map[iproc].side_ids[itemp++];
}
e_comm_ecnts[i1] = e_comm_map[i1].elem_cnt;
e_comm_ids[i1] = e_comm_map[i1].proc_ids[0];
}
}
else {
ecomm_cnt = 0;
}
/* Output load balance information */
bytes_out += 9 * sizeof(INT) + 2 * sizeof(char);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting init Nemesis info in file id: {}\n", mesh_exoid);
fmt::print("\tNumber of Processor for: {}\n", num_proc_for);
}
if (ex_put_init_info(mesh_exoid, num_proc_for, 1, (char *)"p") < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output init info!\n", __func__);
exit(1);
}
if (Debug_Flag >= 6) {
fmt::print("Putting init load balance info in file id: {}\n", mesh_exoid);
fmt::print("\tNumber Internal Nodes: {}\n", globals.Num_Internal_Nodes[iproc]);
fmt::print("\tNumber Border Nodes: {}\n", globals.Num_Border_Nodes[iproc]);
fmt::print("\tNumber External Nodes: {}\n", globals.Num_External_Nodes[iproc]);
fmt::print("\tNumber Internal Elements: {}\n", globals.Num_Internal_Elems[iproc]);
fmt::print("\tNumber Border Elements: {}\n", globals.Num_Border_Elems[iproc]);
fmt::print("\tNumber Nodal Cmaps: {}\n", ncomm_cnt);
fmt::print("\tNumber Elemental Cmaps: {}\n", ecomm_cnt);
fmt::print("\tProccesor For: {}\n", proc_for);
}
if (ex_put_loadbal_param(mesh_exoid, globals.Num_Internal_Nodes[iproc],
globals.Num_Border_Nodes[iproc], globals.Num_External_Nodes[iproc],
globals.Num_Internal_Elems[iproc], globals.Num_Border_Elems[iproc],
ncomm_cnt, ecomm_cnt, proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output load balance info\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[1] = (second() - tt1);
total_out_time += PIO_Time_Array[1];
/* Output the communication map */
bytes_out += 4 * sizeof(INT);
tt1 = second();
if (ex_put_cmap_params(mesh_exoid, n_comm_ids.data(), n_comm_ncnts.data(), e_comm_ids.data(),
e_comm_ecnts.data(), proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output comm map params!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[2] = (second() - tt1);
total_out_time += PIO_Time_Array[2];
/* Convert Elem_Map to local element numbering */
reverse_map(globals.Elem_Map[iproc], 0, itotal_elems, globals.GElems[iproc], (INT *)nullptr,
globals.Elem_Map[iproc]);
/* Convert element IDs in the comm map to local numbering */
for (INT i0 = 0; i0 < ecomm_cnt; i0++) {
reverse_map(e_comm_map[i0].elem_ids, 0, e_comm_map[i0].elem_cnt, globals.GElems[iproc],
(INT *)nullptr, e_comm_map[i0].elem_ids);
}
/* Sort the globals.GNodes array using the index array 'loc_index' */
std::vector<INT> loc_index(itotal_nodes);
/* Initialize index array */
for (size_t i2 = 0; i2 < itotal_nodes; i2++) {
loc_index[i2] = i2;
}
/*
* Sort the globals.GNodes[iproc] array via the index array
* 'loc_index'
*/
gds_iqsort(globals.GNodes[iproc].data(), loc_index.data(), itotal_nodes);
/* Convert nodal IDs in the comm map to local numbering */
for (INT i0 = 0; i0 < ncomm_cnt; i0++) {
reverse_map(n_comm_map[i0].node_ids, 0, n_comm_map[i0].node_cnt, globals.GNodes[iproc],
loc_index.data(), n_comm_map[i0].node_ids);
}
PIO_Time_Array[3] = 0.0;
if (globals.Num_N_Comm_Maps[iproc] > 0 && globals.N_Comm_Map[iproc].node_cnt != 0) {
bytes_out += 2 * globals.Num_External_Nodes[iproc] * sizeof(INT);
tt1 = second();
for (INT i1 = 0; i1 < ncomm_cnt; i1++) {
if (ex_put_node_cmap(mesh_exoid, n_comm_ids[i1], n_comm_map[i1].node_ids.data(),
n_comm_map[i1].proc_ids.data(), proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output nodal comm map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
n_comm_map[i1].proc_ids.clear();
n_comm_map[i1].node_ids.clear();
}
PIO_Time_Array[3] += (second() - tt1);
}
if (globals.Num_E_Comm_Maps[iproc] > 0) {
bytes_out += 3 * (globals.E_Comm_Map[iproc].elem_cnt) * sizeof(INT);
tt1 = second();
for (INT i1 = 0; i1 < ecomm_cnt; i1++) {
if (ex_put_elem_cmap(mesh_exoid, e_comm_ids[i1], e_comm_map[i1].elem_ids.data(),
e_comm_map[i1].side_ids.data(), e_comm_map[i1].proc_ids.data(),
proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output elemental comm map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
e_comm_map[i1].proc_ids.clear();
e_comm_map[i1].elem_ids.clear();
e_comm_map[i1].side_ids.clear();
}
PIO_Time_Array[3] += (second() - tt1);
}
total_out_time += PIO_Time_Array[3];
/* Output the global node set parameters */
if (globals.Num_Node_Set > 0) {
std::vector<INT> glob_ns_df_cnts(globals.Num_Node_Set);
bytes_out += 3 * globals.Num_Node_Set * sizeof(INT);
tt1 = second();
if (ex_put_ns_param_global(mesh_exoid, Node_Set_Ids.data(), Num_Nodes_In_NS.data(),
glob_ns_df_cnts.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global node-set params\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[4] = (second() - tt1);
total_out_time += PIO_Time_Array[4];
}
/* Output the global side set parameters */
if (globals.Num_Side_Set > 0) {
std::vector<INT> glob_ss_df_cnts(globals.Num_Side_Set);
bytes_out += 3 * globals.Num_Side_Set * sizeof(INT);
tt1 = second();
if (ex_put_ss_param_global(mesh_exoid, Side_Set_Ids.data(), Num_Elems_In_SS.data(),
glob_ss_df_cnts.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global side-set params\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[5] = (second() - tt1);
total_out_time += PIO_Time_Array[5];
}
bytes_out += globals.Num_Elem_Blk * sizeof(INT);
tt1 = second();
if (ex_put_eb_info_global(mesh_exoid, Elem_Blk_Ids.data(), Num_Elems_In_EB.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global elem blk IDs\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[6] = (second() - tt1);
total_out_time += PIO_Time_Array[6];
/*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
/* Generate a QA record for the utility */
time_t date_time = time(nullptr);
auto *lt = std::localtime(&date_time);
char qa_time[MAX_STR_LENGTH + 1];
char qa_name[MAX_STR_LENGTH + 1];
char qa_vers[MAX_STR_LENGTH + 1];
char qa_date[MAX_STR_LENGTH + 1];
std::string time = fmt::format("{:%H:%M:%S}", *lt);
std::string date = fmt::format("{:%Y/%m/%d}", *lt);
copy_string(qa_date, date);
copy_string(qa_time, time);
copy_string(qa_name, UTIL_NAME);
copy_string(qa_vers, VER_STR);
if (globals.Num_QA_Recs > 0) {
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 0], qa_name,
MAX_STR_LENGTH + 1);
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 1], qa_vers,
MAX_STR_LENGTH + 1);
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 2], qa_date,
MAX_STR_LENGTH + 1);
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 3], qa_time,
MAX_STR_LENGTH + 1);
/* Output QA records to screen */
if (Debug_Flag >= 4) {
fmt::print("Number of QA records: {}\n", globals.Num_QA_Recs);
if (Debug_Flag >= 6) {
fmt::print("QA Records:\n");
for (INT i1 = 0; i1 < 4 * (globals.Num_QA_Recs); i1++) {
fmt::print("\t{}\n", globals.QA_Record[i1]);
}
}
}
}
/* Output the QA and Info records */
for (INT i1 = 0; i1 < 4 * globals.Num_QA_Recs; i1++) {
bytes_out += (MAX_STR_LENGTH + MAX_LINE_LENGTH) * sizeof(char);
}
tt1 = second();
if (ex_put_qa(mesh_exoid, globals.Num_QA_Recs, (char *(*)[4]) & globals.QA_Record[0]) < 0) {
fmt::print(stderr, "[{}]: ERROR Could not put QA records\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (globals.Num_Info_Recs > 0) {
if (Debug_Flag >= 4) {
fmt::print("Number of info records: {}\n", globals.Num_Info_Recs);
}
if (ex_put_info(mesh_exoid, globals.Num_Info_Recs, globals.Info_Record) < 0) {
fmt::print(stderr, "[{}]: ERROR Could not put Info records\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
PIO_Time_Array[8] = (second() - tt1);
total_out_time += PIO_Time_Array[8];
/* Output the assembly information (if any). This puts the
file in/out of define mode, so should be early in the write stage
*/
if (Debug_Flag >= 4) {
fmt::print("Number of Assemblies: {}\n", globals.Num_Assemblies);
}
if (globals.Num_Assemblies > 0) {
ex_put_assemblies(mesh_exoid, globals.Assemblies.size(), globals.Assemblies.data());
}
/* Output the coordinate frame information (if any). This puts the
file in/out of define mode, so should be early in the write stage
*/
if (Debug_Flag >= 4) {
fmt::print("Number of Coordinate Frames: {}\n", globals.Num_Coordinate_Frames);
}
if (globals.Num_Coordinate_Frames > 0) {
T *Coordinate_Frame_Coordinates = globals.Coordinate_Frame_Coordinates;
if (ex_put_coordinate_frames(mesh_exoid, globals.Num_Coordinate_Frames,
globals.Coordinate_Frame_Ids, Coordinate_Frame_Coordinates,
globals.Coordinate_Frame_Tags) < 0) {
fmt::print(stderr, "[{}]: ERROR, Unable to put coordinate frame data in parallel mesh file\n",
__func__);
ex_close(mesh_exoid);
exit(1);
}
}
std::string cTitle;
if (proc_for == 0) {
cTitle = GeomTitle;
}
else {
cTitle = fmt::format("Parallel Mesh File for Processor {}", proc_for);
}
/* Output the initial information to the parallel Exodus file(s) */
bytes_out += cTitle.length() * sizeof(char) + 6 * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting init info in file id: {}\n", mesh_exoid);
fmt::print("\tTitle: {}\n", cTitle);
fmt::print("\tNumber Dimensions: {}\n", globals.Num_Dim);
fmt::print("\tNumber Nodes: {}\n", itotal_nodes);
fmt::print("\tNumber Elements: {}\n",
globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
fmt::print("\tNumber Element Blocks: {}\n", globals.Num_Elem_Blk);
fmt::print("\tNumber Node Sets: {}\n", globals.Num_Node_Set);
fmt::print("\tNumber Side Sets: {}\n", globals.Num_Side_Set);
}
if (ex_put_init(mesh_exoid, cTitle.c_str(), globals.Num_Dim, itotal_nodes,
globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc],
globals.Num_Elem_Blk, globals.Num_Node_Set, globals.Num_Side_Set) < 0) {
fmt::print(stderr, "[{}]: ERROR, Unable to put initial info in parallel mesh file\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
/* Output names... */
if (globals.Num_Elem_Blk > 0) {
error = ex_put_names(mesh_exoid, EX_ELEM_BLOCK, &Elem_Blk_Names[0]);
check_exodus_error(error, "ex_put_names");
}
if (globals.Num_Node_Set > 0) {
error = ex_put_names(mesh_exoid, EX_NODE_SET, &Node_Set_Names[0]);
check_exodus_error(error, "ex_put_names");
}
if (globals.Num_Side_Set > 0) {
error = ex_put_names(mesh_exoid, EX_SIDE_SET, &Side_Set_Names[0]);
check_exodus_error(error, "ex_put_names");
}
PIO_Time_Array[7] = (second() - tt1);
total_out_time += PIO_Time_Array[7];
/* Assign the coordinates to the coord_vector */
T *x_coord = nullptr;
T *y_coord = nullptr;
T *z_coord = nullptr;
if (itotal_nodes > 0) {
switch (globals.Num_Dim) {
case 3: z_coord = globals.Coor[iproc][2]; FALL_THROUGH;
case 2: y_coord = globals.Coor[iproc][1]; FALL_THROUGH;
case 1: x_coord = globals.Coor[iproc][0]; break;
}
}
/* Output the coordinates to the parallel Exodus file */
bytes_out += globals.Num_Dim * itotal_nodes * io_ws;
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting coordinate info in file id: {}\n", mesh_exoid);
}
if (ex_put_coord(mesh_exoid, x_coord, y_coord, z_coord) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write out nodal coordinates\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[9] = (second() - tt1);
total_out_time += PIO_Time_Array[9];
bytes_out += itotal_nodes * sizeof(INT);
tt1 = second();
/*
* The Nemesis node maps are lists of internal, border and external
* FEM node numbers. These are output as local node numbers.
*/
std::vector<INT> nem_node_mapi(globals.Num_Internal_Nodes[iproc], 1);
std::vector<INT> nem_node_mapb(globals.Num_Border_Nodes[iproc], 1);
std::vector<INT> nem_node_mape(globals.Num_External_Nodes[iproc], 1);
if (ex_put_processor_node_maps(mesh_exoid, nem_node_mapi.data(), nem_node_mapb.data(),
nem_node_mape.data(), proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write Nemesis nodal number map!\n", __func__);
check_exodus_error(ex_close(mesh_exoid), "ex_close");
ex_close(mesh_exoid);
exit(1);
}
/* If non-nullptr, output the global node id map which preserves
the global node ids in the original mesh */
if (!globals.Proc_Global_Node_Id_Map[iproc].empty()) {
bytes_out += itotal_nodes * sizeof(INT);
if (ex_put_map_param(mesh_exoid, 1, 0) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global node map parameters!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_num_map(mesh_exoid, EX_NODE_MAP, 1, globals.Proc_Global_Node_Id_Map[iproc].data()) <
0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global node id map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_name(mesh_exoid, EX_NODE_MAP, 1, "original_global_id_map") < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global node map name!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
PIO_Time_Array[11] = (second() - tt1);
total_out_time += PIO_Time_Array[11];
nem_node_mapi.clear();
nem_node_mapb.clear();
nem_node_mape.clear();
PIO_Time_Array[13] = 0.0;
PIO_Time_Array[14] = 0.0;
PIO_Time_Array[15] = 0.0;
/*
* Generate a list of the global element blocks, some of which may be
* nullptr on a given processor.
*/
{
/*
* Start a local block so we can define some variables locally
* here instead of a few thousand lines up from here...
*/
std::vector<INT> EB_Ids(globals.Num_Elem_Blk);
std::vector<INT> EB_Cnts(globals.Num_Elem_Blk);
std::vector<INT> EB_NperE(globals.Num_Elem_Blk);
std::vector<INT> EB_Nattr(globals.Num_Elem_Blk);
char **EB_Types = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk,
MAX_STR_LENGTH + 1, sizeof(char));
if (EB_Types == nullptr) {
fmt::print(stderr, "{}: fatal: insufficient memory\n", __func__);
exit(1);
}
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
memset(EB_Types[i1], '\0', (MAX_STR_LENGTH + 1));
}
INT cnt = 0;
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
bool ifound = false;
for (INT i2 = 0; i2 < globals.Proc_Num_Elem_Blk[iproc]; i2++) {
if (globals.Proc_Elem_Blk_Ids[iproc][i2] == Elem_Blk_Ids[i1]) {
ifound = true;
break;
}
}
/*
* If it's not found then this element block is null on the current
* processor.
*/
if (!ifound) {
INT iblk = globals.Proc_Num_Elem_Blk[iproc] + cnt;
globals.Proc_Elem_Blk_Ids[iproc][iblk] = Elem_Blk_Ids[i1];
globals.Proc_Num_Elem_In_Blk[iproc][iblk] = 0;
globals.Proc_Nodes_Per_Elem[iproc][iblk] = 0;
globals.Proc_Num_Attr[iproc][iblk] = 0;
cnt++;
}
}
/* Output the elemental block(s). */
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
/* Find the local element block index */
INT ilocal;
for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
break;
}
}
/* Error check */
if (ilocal >= globals.Num_Elem_Blk) {
fmt::print(stderr, "[{}]: Error finding local element block ID\n", __func__);
exit(1);
}
/* If it's a non-null block output all information */
EB_Ids[i1] = iglobal_blk;
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
/* Generate the ExodusII element name */
copy_string(EB_Types[i1], Elem_Blk_Types[i1], MAX_STR_LENGTH);
EB_Cnts[i1] = globals.Proc_Num_Elem_In_Blk[iproc][ilocal];
EB_NperE[i1] = globals.Proc_Nodes_Per_Elem[iproc][ilocal];
EB_Nattr[i1] = globals.Proc_Num_Attr[iproc][ilocal];
}
}
if (Debug_Flag >= 4) {
fmt::print("Putting concat_elem_block info in file id: {}\n", mesh_exoid);
}
error = ex_put_concat_elem_block(mesh_exoid, &EB_Ids[0], &EB_Types[0], &EB_Cnts[0],
&EB_NperE[0], &EB_Nattr[0], 1);
check_exodus_error(error, "ex_put_concat_elem_block");
safe_free(reinterpret_cast<void **>(&EB_Types));
/* Output attribute names for each element block */
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
/* Find the local element block index */
INT ilocal;
for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
break;
}
}
/* If it's a non-null block output attribute name information */
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
if (globals.Proc_Num_Attr[iproc][ilocal] > 0) {
if (ex_put_attr_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i1],
Elem_Blk_Attr_Names[i1]) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write Exodus attribute names!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
}
}
/* Reset globals.GNodes to start at 1 instead of 0 */
for (size_t i1 = 0; i1 < itotal_nodes; (globals.GNodes[iproc][i1++])++) {
;
}
/* Output the Exodus node number map */
bytes_out += itotal_nodes * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting node_num_map in file id: {}\n", mesh_exoid);
}
if (ex_put_id_map(mesh_exoid, EX_NODE_MAP, globals.GNodes[iproc].data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write Exodus node number map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[10] = (second() - tt1);
total_out_time += PIO_Time_Array[10];
/*
* Allocate memory for the elemental map. Currently this map is assigned
* as a linear array since it is not really used.
*/
std::vector<INT> iElem_Map(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
for (INT i1 = 0; i1 < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
i1++) {
iElem_Map[i1] = globals.GElems[iproc][i1] + 1;
}
bytes_out +=
2 * globals.Num_Internal_Elems[iproc] * globals.Num_Border_Elems[iproc] * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting elem_num_map info in file id: {}\n", mesh_exoid);
}
if (ex_put_id_map(mesh_exoid, EX_ELEM_MAP, iElem_Map.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output element map\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
/* If non-nullptr, output the global element id map which preserves
the global element ids in the original mesh */
if (!globals.Proc_Global_Elem_Id_Map[iproc].empty()) {
bytes_out +=
globals.Num_Internal_Elems[iproc] * globals.Num_Border_Elems[iproc] * sizeof(INT);
if (ex_put_map_param(mesh_exoid, 0, 1) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global map parameters!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_num_map(mesh_exoid, EX_ELEM_MAP, 1,
globals.Proc_Global_Elem_Id_Map[iproc].data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global id map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_name(mesh_exoid, EX_ELEM_MAP, 1, "original_global_id_map") < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global map name!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
/* Also output the Nemesis element map */
if (ex_put_processor_elem_maps(
mesh_exoid, globals.Elem_Map[iproc].data(),
(globals.Elem_Map[iproc].data()) + globals.Num_Internal_Elems[iproc], proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output nemesis element map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[12] = (second() - tt1);
total_out_time += PIO_Time_Array[12];
iElem_Map.clear();
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
/* Find the local element block index */
INT ilocal;
for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
break;
}
}
/* Error check */
if (ilocal >= globals.Num_Elem_Blk) {
fmt::print(stderr, "[{}]: Error finding local element block ID\n", __func__);
exit(1);
}
/* If it's a non-null block output all information */
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
/* Find the first index into the connectivity for this block */
size_t iIndex0 = 0;
for (INT i2 = 0; i2 < ilocal; i2++) {
iIndex0 +=
globals.Proc_Num_Elem_In_Blk[iproc][i2] * globals.Proc_Nodes_Per_Elem[iproc][i2];
}
PIO_Time_Array[13] += (second() - tt1);
size_t tmp_cnt = globals.Proc_Num_Elem_In_Blk[iproc][ilocal] *
globals.Proc_Nodes_Per_Elem[iproc][ilocal];
/* Generate the connectivity array for local node numbering */
{
std::vector<INT> proc_local_conn(tmp_cnt);
reverse_map(&globals.Proc_Elem_Connect[iproc][iIndex0], 1, tmp_cnt,
globals.GNodes[iproc].data(), loc_index.data(), proc_local_conn.data());
bytes_out += globals.Proc_Nodes_Per_Elem[iproc][ilocal] *
globals.Proc_Num_Elem_In_Blk[iproc][ilocal] * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting element_connectivity info in file id: {}\n", mesh_exoid);
}
if (ex_put_conn(mesh_exoid, EX_ELEM_BLOCK, globals.Proc_Elem_Blk_Ids[iproc][ilocal],
proc_local_conn.data(), nullptr, nullptr) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output connectivity\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[14] += (second() - tt1);
}
if (globals.Proc_Num_Attr[iproc][ilocal] > 0) {
/* Find the first index into the attribute list for this block */
size_t iIndex1 = 0;
for (INT i2 = 0; i2 < ilocal; i2++) {
iIndex1 += globals.Proc_Num_Attr[iproc][i2] * globals.Proc_Num_Elem_In_Blk[iproc][i2];
}
bytes_out += globals.Proc_Num_Elem_In_Blk[iproc][ilocal] * io_ws;
tt1 = second();
T *ptr = &(globals.Proc_Elem_Attr[iproc][iIndex1]);
if (ex_put_attr(mesh_exoid, EX_ELEM_BLOCK, globals.Proc_Elem_Blk_Ids[iproc][ilocal],
ptr) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output element attributes\n", __func__);
exit(1);
}
PIO_Time_Array[15] += (second() - tt1);
total_out_time += PIO_Time_Array[15];
}
} /* End "if(ilocal < globals.Num_Elem_Blk[iproc])" */
} /* End "for(i1=0; i1 < globals.Num_Elem_Block; i1++)" */
}
total_out_time += (PIO_Time_Array[13] + PIO_Time_Array[14] + PIO_Time_Array[15]);
/*
* Write out the node-set information. Note that the value of the
* node-set distribution factor is not currently used so only a
* dummy set is output for this value.
*/
INT iMaxLen = 0;
for (INT i1 = 0; i1 < globals.Proc_Num_Node_Sets[iproc]; i1++) {
iMaxLen = PEX_MAX(globals.Proc_NS_Count[iproc][i1], iMaxLen);
}
/* Renumber Node set node lists to use local node numbers */
std::vector<INT> proc_local_ns;
if (globals.Proc_Num_Node_Sets[iproc] > 0) {
proc_local_ns.resize(globals.Proc_NS_List_Length[iproc]);
reverse_map(&globals.Proc_NS_List[iproc][0], 1, globals.Proc_NS_List_Length[iproc],
globals.GNodes[iproc].data(), loc_index.data(), proc_local_ns.data());
}
loc_index.clear();
PIO_Time_Array[16] = 0.0;
PIO_Time_Array[17] = 0.0;
/* Fill in the information for the nullptr node sets */
size_t cnt = 0;
for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
bool ifound = false;
for (INT i2 = 0; i2 < globals.Proc_Num_Node_Sets[iproc]; i2++) {
if (globals.Proc_NS_Ids[iproc][i2] == Node_Set_Ids[i1]) {
ifound = true;
break;
}
}
if (!ifound) {
globals.Proc_NS_Ids[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = Node_Set_Ids[i1];
globals.Proc_NS_Count[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = 0;
globals.Proc_NS_DF_Count[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = 0;
cnt++;
}
}
tt1 = second();
if (globals.Num_Node_Set > 0) {
size_t dcount = 0;
size_t ncount = 0;
for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
dcount += globals.Proc_NS_DF_Count[iproc][i1];
ncount += globals.Proc_NS_Count[iproc][i1];
}
std::vector<INT> conc_ids(globals.Num_Node_Set);
std::vector<INT> conc_nodes(globals.Num_Node_Set);
std::vector<INT> conc_df(globals.Num_Node_Set);
std::vector<INT> conc_nind(globals.Num_Node_Set);
std::vector<INT> conc_dind(globals.Num_Node_Set);
std::vector<INT> conc_nlist(ncount);
std::vector<T> conc_sdf(dcount);
ncount = 0;
dcount = 0;
for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
/* Find the local ID */
INT i2 = 0;
for (i2 = 0; i2 < globals.Num_Node_Set; i2++) {
if (globals.Proc_NS_Ids[iproc][i2] == Node_Set_Ids[i1]) {
break;
}
}
conc_ids[i1] = globals.Proc_NS_Ids[iproc][i2];
conc_nodes[i1] = globals.Proc_NS_Count[iproc][i2];
conc_df[i1] = globals.Proc_NS_DF_Count[iproc][i2];
conc_nind[i1] = ncount;
for (INT i3 = 0; i3 < globals.Proc_NS_Count[iproc][i2]; i3++) {
conc_nlist[ncount++] = proc_local_ns[globals.Proc_NS_Pointers[iproc][i2] + i3];
}
conc_dind[i1] = dcount;
for (INT i3 = 0; i3 < globals.Proc_NS_DF_Count[iproc][i2]; i3++) {
conc_sdf[dcount++] =
globals.Proc_NS_Dist_Fact[iproc][globals.Proc_NS_Pointers[iproc][i2] + i3];
}
}
ex_set_specs set_specs{};
set_specs.sets_ids = conc_ids.data();
set_specs.num_entries_per_set = conc_nodes.data();
set_specs.num_dist_per_set = conc_df.data();
set_specs.sets_entry_index = conc_nind.data();
set_specs.sets_dist_index = conc_dind.data();
set_specs.sets_entry_list = conc_nlist.data();
set_specs.sets_extra_list = nullptr;
set_specs.sets_dist_fact = conc_sdf.data();
ex_put_concat_sets(mesh_exoid, EX_NODE_SET, &set_specs);
}
total_out_time += second() - tt1;
/* Free local number array */
if (globals.Proc_Num_Node_Sets[iproc] > 0) {
proc_local_ns.clear();
}
/* Renumber element SS to use local element numbers */
std::vector<INT> proc_local_ss;
if (globals.Proc_Num_Side_Sets[iproc] > 0) {
proc_local_ss.resize(globals.Proc_SS_Elem_List_Length[iproc]);
reverse_map(&globals.Proc_SS_Elem_List[iproc][0], 0, globals.Proc_SS_Elem_List_Length[iproc],
globals.GElems[iproc].data(), (INT *)nullptr, proc_local_ss.data());
}
PIO_Time_Array[18] = 0.0;
PIO_Time_Array[19] = 0.0;
/* Set up the null side sets */
cnt = 0;
for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
bool ifound = false;
for (INT i2 = 0; i2 < globals.Proc_Num_Side_Sets[iproc]; i2++) {
if (globals.Proc_SS_Ids[iproc][i2] == Side_Set_Ids[i1]) {
ifound = true;
break;
}
}
if (!ifound) {
globals.Proc_SS_Ids[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = Side_Set_Ids[i1];
globals.Proc_SS_Elem_Count[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = 0;
globals.Proc_SS_DF_Count[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = 0;
cnt++;
}
}
/* Output concatenated sidesets. For each processor need:
* side_set ids (size globals.Num_Side_Set)
* num_side_per_set (size globals.Num_Side_Set)
* num_dist_per_set (size globals.Num_Side_Set)
* side_sets_elem_index (size globals.Num_Side_Set)
* side_sets_dist_index (size globals.Num_Side_Set)
* side_sets_elem_list
* side_sets_side_list
* side_sets_dist_fact
*/
tt1 = second();
if (globals.Num_Side_Set > 0) {
size_t df_count = 0;
size_t el_count = 0;
for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
df_count += globals.Proc_SS_DF_Count[iproc][i1];
el_count += globals.Proc_SS_Elem_Count[iproc][i1];
}
std::vector<INT> conc_ids(globals.Num_Side_Set);
std::vector<INT> conc_sides(globals.Num_Side_Set);
std::vector<INT> conc_dist(globals.Num_Side_Set);
std::vector<INT> conc_eind(globals.Num_Side_Set);
std::vector<INT> conc_dind(globals.Num_Side_Set);
std::vector<INT> conc_elist(el_count);
std::vector<INT> conc_slist(el_count);
std::vector<T> conc_sdflist(df_count);
/* Fill in the arrays ... */
df_count = 0;
el_count = 0;
for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
/* Find the local ID of this side set */
INT i2 = 0;
for (i2 = 0; i2 < globals.Num_Side_Set; i2++) {
if (globals.Proc_SS_Ids[iproc][i2] == Side_Set_Ids[i1]) {
break;
}
}
conc_ids[i1] = globals.Proc_SS_Ids[iproc][i2];
conc_sides[i1] = globals.Proc_SS_Elem_Count[iproc][i2];
conc_dist[i1] = globals.Proc_SS_DF_Count[iproc][i2];
conc_eind[i1] = el_count;
for (INT i3 = 0; i3 < globals.Proc_SS_Elem_Count[iproc][i2]; i3++) {
conc_elist[el_count] = proc_local_ss[globals.Proc_SS_Elem_Pointers[iproc][i2] + i3];
conc_slist[el_count] =
globals.Proc_SS_Side_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i2] + i3];
el_count++;
}
conc_dind[i1] = df_count;
for (INT i3 = 0; i3 < globals.Proc_SS_DF_Count[iproc][i2]; i3++) {
conc_sdflist[df_count++] =
globals.Proc_SS_Dist_Fact[iproc][globals.Proc_SS_DF_Pointers[iproc][i2] + i3];
}
}
ex_set_specs set_specs{};
set_specs.sets_ids = conc_ids.data();
set_specs.num_entries_per_set = conc_sides.data();
set_specs.num_dist_per_set = conc_dist.data();
set_specs.sets_entry_index = conc_eind.data();
set_specs.sets_dist_index = conc_dind.data();
set_specs.sets_entry_list = conc_elist.data();
set_specs.sets_extra_list = conc_slist.data();
set_specs.sets_dist_fact = conc_sdflist.data();
ex_put_concat_sets(mesh_exoid, EX_SIDE_SET, &set_specs);
}
PIO_Time_Array[19] += (second() - tt1);
total_out_time += (PIO_Time_Array[18] + PIO_Time_Array[19]);
/* Free unneeded memory */
if (globals.Proc_Num_Side_Sets[iproc] > 0) {
proc_local_ss.clear();
}
/*
* Write out the name of the coordinate axes to the parallel ExodusII
* files.
*/
bytes_out += globals.Num_Dim * 8 * sizeof(char);
tt1 = second();
if (ex_put_coord_names(mesh_exoid, Coord_Name) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not output coordinate names\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[20] = (second() - tt1);
total_out_time += PIO_Time_Array[20];
if (Restart_Info.Flag > 0) {
tt1 = second();
bytes_out +=
write_var_param(mesh_exoid, max_name_length, Restart_Info.NVar_Glob, Restart_Info.GV_Name,
Restart_Info.NVar_Node, Restart_Info.NV_Name, Restart_Info.NVar_Elem,
Restart_Info.EV_Name, Restart_Info.GElem_TT.data(), Restart_Info.NVar_Nset,
Restart_Info.NSV_Name, Restart_Info.GNset_TT.data(), Restart_Info.NVar_Sset,
Restart_Info.SSV_Name, Restart_Info.GSset_TT.data());
PIO_Time_Array[21] = (second() - tt1);
total_out_time += PIO_Time_Array[21];
}
/* Calculate the overall performance */
tt1 = bytes_out;
if (total_out_time == 0) {
tt1 = 0;
}
else {
tt1 = tt1 / total_out_time;
}
tt1 /= 1024.0;
PIO_Time_Array[25] = tt1;
} /* END write_parExo_data() */
template <typename T, typename INT>
int NemSpread<T, INT>::write_var_param(int mesh_exoid, int max_name_length, int num_glob,
char **gv_names, int num_node, char **nv_names, int num_elem,
char **ev_names, int *local_ebtt, int num_nset,
char **ns_names, int *local_nstt, int num_sset,
char **ss_names, int *local_sstt)
{
size_t bytes_out = 0;
int error;
bytes_out += (5 + globals.Num_Elem_Blk * num_elem + globals.Num_Side_Set * num_sset +
globals.Num_Node_Set * num_nset) *
sizeof(INT);
error = ex_put_all_var_param(mesh_exoid, num_glob, num_node, num_elem, local_ebtt, num_nset,
local_nstt, num_sset, local_sstt);
check_exodus_error(error, "ex_put_all_var_param");
if (gv_names != nullptr) {
bytes_out += Restart_Info.NVar_Glob * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_GLOBAL, num_glob, gv_names);
check_exodus_error(error, "ex_put_var_names");
}
if (nv_names != nullptr) {
bytes_out += num_node * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_NODAL, num_node, nv_names);
check_exodus_error(error, "ex_put_var_names");
}
if (ev_names != nullptr) {
bytes_out += Restart_Info.NVar_Elem * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_ELEM_BLOCK, num_elem, ev_names);
check_exodus_error(error, "ex_put_var_names");
}
if (ns_names != nullptr) {
bytes_out += Restart_Info.NVar_Nset * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_NODE_SET, num_nset, ns_names);
check_exodus_error(error, "ex_put_var_names");
}
if (ss_names != nullptr) {
bytes_out += Restart_Info.NVar_Sset * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_SIDE_SET, num_sset, ss_names);
check_exodus_error(error, "ex_put_var_names");
}
return (bytes_out);
}
template void NemSpread<double, int>::write_var_timestep(int exoid, int proc, int time_step,
int *eb_ids_global, int *ss_ids_global,
int *ns_ids_global);
template void NemSpread<float, int>::write_var_timestep(int exoid, int proc, int time_step,
int *eb_ids_global, int *ss_ids_global,
int *ns_ids_global);
template void NemSpread<double, int64_t>::write_var_timestep(int exoid, int proc, int time_step,
int64_t *eb_ids_global,
int64_t *ss_ids_global,
int64_t *ns_ids_global);
template void NemSpread<float, int64_t>::write_var_timestep(int exoid, int proc, int time_step,
int64_t *eb_ids_global,
int64_t *ss_ids_global,
int64_t *ns_ids_global);
template <typename T, typename INT>
void NemSpread<T, INT>::write_var_timestep(int exoid, int proc, int time_step, INT *eb_ids_global,
INT *ss_ids_global, INT *ns_ids_global)
{
int error;
/* output the time */
{
T *var_ptr = (T *)&(Restart_Info.Time);
error = ex_put_time(exoid, time_step, var_ptr);
check_exodus_error(error, "ex_put_time");
}
/* start by outputting the global variables */
if (Restart_Info.NVar_Glob > 0) {
T *var_ptr = &Restart_Info.Glob_Vals[0];
error = ex_put_var(exoid, time_step, EX_GLOBAL, 1, 0, Restart_Info.NVar_Glob, var_ptr);
check_exodus_error(error, "ex_put_glob_vars");
}
if (Restart_Info.NVar_Node > 0) {
size_t num_nodes = globals.Num_Internal_Nodes[proc] + globals.Num_Border_Nodes[proc] +
globals.Num_External_Nodes[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Node; var_num++) {
size_t var_offset = var_num * num_nodes;
T *var_ptr = &(Restart_Info.Node_Vals[proc][var_offset]);
error = ex_put_var(exoid, time_step, EX_NODAL, (var_num + 1), 1, num_nodes, var_ptr);
check_exodus_error(error, "ex_put_var");
}
}
if (Restart_Info.NVar_Elem > 0) {
size_t num_elem = globals.Num_Internal_Elems[proc] + globals.Num_Border_Elems[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Elem; var_num++) {
int eb_num_g = 0;
size_t var_offset = var_num * num_elem;
T *var_ptr = &(Restart_Info.Elem_Vals[proc][var_offset]);
for (int eb_num = 0; eb_num < globals.Proc_Num_Elem_Blk[proc]; eb_num++) {
/* now I have to find the appropriate entry in the truth table */
/* can always assume this eb num is greater than the last one */
for (int cnt1 = eb_num_g; cnt1 < globals.Num_Elem_Blk; cnt1++) {
if (globals.Proc_Elem_Blk_Ids[proc][eb_num] == eb_ids_global[cnt1]) {
eb_num_g = cnt1;
break;
}
}
if (Restart_Info.GElem_TT[eb_num_g * Restart_Info.NVar_Elem + var_num]) {
error = ex_put_var(exoid, time_step, EX_ELEM_BLOCK, (var_num + 1),
globals.Proc_Elem_Blk_Ids[proc][eb_num],
globals.Proc_Num_Elem_In_Blk[proc][eb_num], var_ptr);
check_exodus_error(error, "ex_put_elem_var");
}
/* and now move the variable pointer */
/* Note that the offsetting here must match the 'var_offset'
* treatment in ps_restart.c function read_elem_vars.
* Currently, the offset is applied even if the variable does
* not exist on a particular block.
*/
var_ptr += globals.Proc_Num_Elem_In_Blk[proc][eb_num];
}
}
}
if (Restart_Info.NVar_Sset > 0) {
int ss_num_g = 0;
size_t num_elem = globals.Proc_SS_Elem_List_Length[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Sset; var_num++) {
size_t var_offset = var_num * num_elem;
T *var_ptr = &(Restart_Info.Sset_Vals[proc][var_offset]);
for (int ss_num = 0; ss_num < globals.Proc_Num_Side_Sets[proc]; ss_num++) {
/* now I have to find the appropriate entry in the truth table */
for (int cnt1 = 0; cnt1 < globals.Num_Side_Set; cnt1++) {
if (globals.Proc_SS_Ids[proc][ss_num] == ss_ids_global[cnt1]) {
ss_num_g = cnt1;
break;
}
}
assert(globals.Proc_SS_Ids[proc][ss_num] == ss_ids_global[ss_num_g]);
if (Restart_Info.GSset_TT[ss_num_g * Restart_Info.NVar_Sset + var_num]) {
error = ex_put_var(exoid, time_step, EX_SIDE_SET, (var_num + 1),
globals.Proc_SS_Ids[proc][ss_num],
globals.Proc_SS_Elem_Count[proc][ss_num], var_ptr);
check_exodus_error(error, "ex_put_sset_var");
}
/* and now move the variable pointer */
/* Note that the offsetting here must match the 'var_offset'
* treatment in ps_restart.c function read_elem_vars.
* Currently, the offset is applied even if the variable does
* not exist on a particular block.
*/
var_ptr += globals.Proc_SS_Elem_Count[proc][ss_num];
}
}
}
if (Restart_Info.NVar_Nset > 0) {
int ns_num_g = 0;
size_t num_elem = globals.Proc_NS_List_Length[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Nset; var_num++) {
size_t var_offset = var_num * num_elem;
T *var_ptr = &(Restart_Info.Nset_Vals[proc][var_offset]);
for (int ns_num = 0; ns_num < globals.Proc_Num_Node_Sets[proc]; ns_num++) {
/* now I have to find the appropriate entry in the truth table */
for (int cnt1 = 0; cnt1 < globals.Num_Node_Set; cnt1++) {
if (globals.Proc_NS_Ids[proc][ns_num] == ns_ids_global[cnt1]) {
ns_num_g = cnt1;
break;
}
}
assert(globals.Proc_NS_Ids[proc][ns_num] == ns_ids_global[ns_num_g]);
if (Restart_Info.GNset_TT[ns_num_g * Restart_Info.NVar_Nset + var_num]) {
error = ex_put_var(exoid, time_step, EX_NODE_SET, (var_num + 1),
globals.Proc_NS_Ids[proc][ns_num], globals.Proc_NS_Count[proc][ns_num],
var_ptr);
check_exodus_error(error, "ex_put_nset_var");
}
/* and now move the variable pointer */
/* Note that the offsetting here must match the 'var_offset'
* treatment in ps_restart.c function read_elem_vars.
* Currently, the offset is applied even if the variable does
* not exist on a particular block.
*/
var_ptr += globals.Proc_NS_Count[proc][ns_num];
}
}
}
}
namespace {
template <typename INT>
void reverse_map(const std::vector<INT> &global, int p01, size_t gsize,
const std::vector<INT> &glmap, INT *index, std::vector<INT> &mapout)
{
/*
* The 'global' array is an array of node or element numbers
* in the global id space. It needs to be converted to local
* numbers via the 'glmap' array. The glmap array is sorted
* by the 'index' array. The map from global to local is
* glmap[local_id] = global_id
*
* The 'p01' is either 0 or 1 and is an offset to the values
* in 'global'
*/
/*
* The algorithm used is to sort the 'global' array via the
* 'tmp_index' array (global[tmp_index[0..gsize]] is sorted)
* Then, progress through the 'global' array in sorted order
* and find the location in 'glmap'. Note that since both are
* sorted, it should be easy to progress sequentially through
* both arrays.
*/
std::vector<INT> tmp_index(gsize);
/* Initialize index array */
for (size_t i2 = 0; i2 < gsize; i2++) {
tmp_index[i2] = (INT)i2;
}
/* Sort the 'global' array via the index array 'tmp_index' */
gds_iqsort(global.data(), tmp_index.data(), gsize);
size_t i3 = 0;
if (index != nullptr) {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[index[i3]] < gval) {
i3++;
}
assert(glmap[index[i3]] == gval);
mapout[tmp_index[i2]] = index[i3] + 1;
}
}
else {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[i3] < gval) {
i3++;
}
assert(glmap[i3] == gval);
mapout[tmp_index[i2]] = i3 + 1;
}
}
}
template <typename INT>
void reverse_map(INT *global, int p01, size_t gsize, INT *glmap, INT *index, INT *mapout)
{
/*
* The 'global' array is an array of node or element numbers
* in the global id space. It needs to be converted to local
* numbers via the 'glmap' array. The glmap array is sorted
* by the 'index' array. The map from global to local is
* glmap[local_id] = global_id
*
* The 'p01' is either 0 or 1 and is an offset to the values
* in 'global'
*/
/*
* The algorithm used is to sort the 'global' array via the
* 'tmp_index' array (global[tmp_index[0..gsize]] is sorted)
* Then, progress through the 'global' array in sorted order
* and find the location in 'glmap'. Note that since both are
* sorted, it should be easy to progress sequentially through
* both arrays.
*/
std::vector<INT> tmp_index(gsize);
std::iota(tmp_index.begin(), tmp_index.end(), 0);
/* Sort the 'global' array via the index array 'tmp_index' */
gds_iqsort(global, tmp_index.data(), gsize);
size_t i3 = 0;
if (index != nullptr) {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[index[i3]] < gval) {
i3++;
}
assert(glmap[index[i3]] == gval);
mapout[tmp_index[i2]] = index[i3] + 1;
}
}
else {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[i3] < gval) {
i3++;
}
assert(glmap[i3] == gval);
mapout[tmp_index[i2]] = i3 + 1;
}
}
}
} // namespace