lib
/
EXODUS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Cloned SEACAS for EXODUS library with extra build files for internal package management.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
6 Commits
1 Branch
0 Tags
19 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 6ce61ef92b
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '6ce61ef92b'
${ noResults }
EXODUS/packages/seacas/applications/nem_spread/pe_write_parExo_info.C

1548 lines
56 KiB
Raw Blame History

/*
* Copyright(C) 1999-2023 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
* See packages/seacas/LICENSE for details
*/
#include "copy_string_cpp.h"
#include "exodusII.h" // for ex_close, etc
#include "fmt/chrono.h"
#include "fmt/ostream.h"
#include "nem_spread.h" // for NemSpread, second, etc
#include "pe_common.h" // for PEX_MAX
#include "ps_pario_const.h" // for PIO_Time_Array
#include "rf_allo.h" // for safe_free, array_alloc
#include "rf_io_const.h" // for Debug_Flag
#include "sort_utils.h" // for gds_iqsort
#include <cassert> // for assert
#include <cstddef> // for size_t
#include <cstdio> // for nullptr, etc
#include <cstdlib> // for exit, free, malloc
#include <cstring> // for strlen, memset, etc
#include <ctime> // for asctime, localtime, time, etc
#include <numeric>
#include <vector> // for vector
template <typename INT> struct ELEM_COMM_MAP;
template <typename INT> struct NODE_COMM_MAP;
#if (__cplusplus >= 201703L)
#define FALL_THROUGH [[fallthrough]]
#elif defined(__GNUC__) && __GNUC__ >= 7 && !__INTEL_COMPILER
#define FALL_THROUGH [[gnu::fallthrough]]
#else
#define FALL_THROUGH ((void)0)
#endif
namespace {
template <typename INT>
void reverse_map(INT *global, int p01, size_t gsize, INT *glmap, INT *index, INT *mapout);
template <typename INT>
void reverse_map(const std::vector<INT> &global, int p01, size_t gsize,
const std::vector<INT> &glmap, INT *index, std::vector<INT> &mapout);
} // namespace
/*
* need this variable for the 0 processor to hold on to the correct
* Exodus II database title
*/
extern std::string GeomTitle;
/****************************************************************************/
/* This function writes parallel specific mesh information out to the */
/* parallel disk(s) for each processor in the salsa run. */
/* */
/* Author(s): Gary L. Hennigan (1421) */
/*--------------------------------------------------------------------------*/
/* Revision History */
/* */
/* Gary Hennigan: */
/* 11 November 1993 - Date of first working version on the nCube. */
/****************************************************************************/
template void NemSpread<float, int>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc, std::vector<int> &Num_Nodes_In_NS,
std::vector<int> &Num_Elems_In_SS,
std::vector<int> &Num_Elems_In_EB);
template void NemSpread<double, int>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc,
std::vector<int> &Num_Nodes_In_NS,
std::vector<int> &Num_Elems_In_SS,
std::vector<int> &Num_Elems_In_EB);
template void NemSpread<double, int64_t>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc,
std::vector<int64_t> &Num_Nodes_In_NS,
std::vector<int64_t> &Num_Elems_In_SS,
std::vector<int64_t> &Num_Elems_In_EB);
template void NemSpread<float, int64_t>::write_parExo_data(int mesh_exoid, int max_name_length,
int iproc,
std::vector<int64_t> &Num_Nodes_In_NS,
std::vector<int64_t> &Num_Elems_In_SS,
std::vector<int64_t> &Num_Elems_In_EB);
template <typename T, typename INT>
void NemSpread<T, INT>::write_parExo_data(int mesh_exoid, int max_name_length, int iproc,
std::vector<INT> &Num_Nodes_In_NS,
std::vector<INT> &Num_Elems_In_SS,
std::vector<INT> &Num_Elems_In_EB)
{
/* Performance metrics. */
size_t bytes_out = 0;
double total_out_time = 0.0;
double tt1;
int error;
/****************************BEGIN EXECUTION*********************************/
/*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
/* PARALLEL EXODUSII SECTION */
/*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
int proc_for = Proc_Ids[iproc];
int num_proc_for = Proc_Info[0];
size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
globals.Num_External_Nodes[iproc];
size_t itotal_elems = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
bytes_out += 5 * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting init global info in file id: {}\n", mesh_exoid);
fmt::print("\tNumber Global Nodes: {}\n", globals.Num_Node);
fmt::print("\tNumber Global Elements: {}\n", globals.Num_Elem);
fmt::print("\tNumber Global Element Blocks: {}\n", globals.Num_Elem_Blk);
fmt::print("\tNumber Global Node Sets: {}\n", globals.Num_Node_Set);
fmt::print("\tNumber Global Side Sets: {}\n", globals.Num_Side_Set);
}
if (ex_put_init_global(mesh_exoid, globals.Num_Node, globals.Num_Elem, globals.Num_Elem_Blk,
globals.Num_Node_Set, globals.Num_Side_Set) < 0) {
fmt::print(stderr,
"[{}]: ERROR, Unable to put global initial "
"information in parallel mesh file!\n",
__func__);
exit(1);
}
PIO_Time_Array[0] = (second() - tt1);
total_out_time += PIO_Time_Array[0];
/*
* Find the number of distinct nodal communication maps that will
* be needed. This assumes that there is only a single input
* nodal communication map that stores information for all
* processors.
*/
int ncomm_cnt = 0;
std::vector<NODE_COMM_MAP<INT>> n_comm_map;
std::vector<INT> n_comm_ids;
std::vector<INT> n_comm_ncnts;
if (globals.Num_N_Comm_Maps[iproc] > 0 && globals.N_Comm_Map[iproc].node_cnt != 0) {
ncomm_cnt = 1;
ex_entity_id itemp = globals.N_Comm_Map[iproc].proc_ids[0];
/* First find the count */
for (size_t i1 = 1; i1 < globals.N_Comm_Map[iproc].node_cnt; i1++) {
if (globals.N_Comm_Map[iproc].proc_ids[i1] != itemp) {
ncomm_cnt++;
itemp = globals.N_Comm_Map[iproc].proc_ids[i1];
}
}
/* Allocate memory for the nodal communication maps */
n_comm_map.resize(ncomm_cnt);
n_comm_ids.resize(ncomm_cnt);
n_comm_ncnts.resize(ncomm_cnt);
/* Find the size of each map */
ncomm_cnt = 0;
itemp = globals.N_Comm_Map[iproc].proc_ids[0];
n_comm_map[0].node_cnt = 1;
for (size_t i1 = 1; i1 < globals.N_Comm_Map[iproc].node_cnt; i1++) {
if (globals.N_Comm_Map[iproc].proc_ids[i1] != itemp) {
itemp = globals.N_Comm_Map[iproc].proc_ids[i1];
ncomm_cnt++;
n_comm_map[ncomm_cnt].node_cnt = 1;
}
else {
(n_comm_map[ncomm_cnt].node_cnt)++;
}
}
ncomm_cnt++;
size_t temp = 0;
/* Allocate memory for the maps */
for (int i1 = 0; i1 < ncomm_cnt; i1++) {
n_comm_map[i1].proc_ids.resize(n_comm_map[i1].node_cnt);
n_comm_map[i1].node_ids.resize(n_comm_map[i1].node_cnt);
for (size_t i2 = 0; i2 < n_comm_map[i1].node_cnt; i2++) {
n_comm_map[i1].proc_ids[i2] = globals.N_Comm_Map[iproc].proc_ids[temp];
n_comm_map[i1].node_ids[i2] = globals.N_Comm_Map[iproc].node_ids[temp++];
}
n_comm_ncnts[i1] = n_comm_map[i1].node_cnt;
n_comm_ids[i1] = n_comm_map[i1].proc_ids[0];
}
}
else {
ncomm_cnt = 0;
}
/*
* Find the number of distinct elemental communication maps that will
* be needed. This assumes that there is only a single input
* elemental communication map that stores information for all
* processors.
*/
INT ecomm_cnt = 0;
std::vector<INT> e_comm_ids;
std::vector<INT> e_comm_ecnts;
std::vector<ELEM_COMM_MAP<INT>> e_comm_map;
if (globals.Num_E_Comm_Maps[iproc] > 0) {
ecomm_cnt = 1;
INT itemp = globals.E_Comm_Map[iproc].proc_ids[0];
/* First find the count */
for (size_t i1 = 1; i1 < globals.E_Comm_Map[iproc].elem_cnt; i1++) {
if (globals.E_Comm_Map[iproc].proc_ids[i1] != itemp) {
ecomm_cnt++;
itemp = globals.E_Comm_Map[iproc].proc_ids[i1];
}
}
/* Allocate memory for the elemental communication maps */
e_comm_map.resize(ecomm_cnt);
e_comm_ids.resize(ecomm_cnt);
e_comm_ecnts.resize(ecomm_cnt);
/* Find the size of each map */
ecomm_cnt = 0;
itemp = globals.E_Comm_Map[iproc].proc_ids[0];
e_comm_map[0].elem_cnt = 1;
for (size_t i1 = 1; i1 < globals.E_Comm_Map[iproc].elem_cnt; i1++) {
if (globals.E_Comm_Map[iproc].proc_ids[i1] != itemp) {
itemp = globals.E_Comm_Map[iproc].proc_ids[i1];
ecomm_cnt++;
e_comm_map[ecomm_cnt].elem_cnt = 1;
}
else {
(e_comm_map[ecomm_cnt].elem_cnt)++;
}
}
ecomm_cnt++;
itemp = 0;
/* Allocate memory for the maps */
for (INT i1 = 0; i1 < ecomm_cnt; i1++) {
e_comm_map[i1].proc_ids.resize(e_comm_map[i1].elem_cnt);
e_comm_map[i1].elem_ids.resize(e_comm_map[i1].elem_cnt);
e_comm_map[i1].side_ids.resize(e_comm_map[i1].elem_cnt);
for (size_t i2 = 0; i2 < e_comm_map[i1].elem_cnt; i2++) {
e_comm_map[i1].proc_ids[i2] = globals.E_Comm_Map[iproc].proc_ids[itemp];
e_comm_map[i1].elem_ids[i2] = globals.E_Comm_Map[iproc].elem_ids[itemp];
e_comm_map[i1].side_ids[i2] = globals.E_Comm_Map[iproc].side_ids[itemp++];
}
e_comm_ecnts[i1] = e_comm_map[i1].elem_cnt;
e_comm_ids[i1] = e_comm_map[i1].proc_ids[0];
}
}
else {
ecomm_cnt = 0;
}
/* Output load balance information */
bytes_out += 9 * sizeof(INT) + 2 * sizeof(char);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting init Nemesis info in file id: {}\n", mesh_exoid);
fmt::print("\tNumber of Processor for: {}\n", num_proc_for);
}
if (ex_put_init_info(mesh_exoid, num_proc_for, 1, (char *)"p") < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output init info!\n", __func__);
exit(1);
}
if (Debug_Flag >= 6) {
fmt::print("Putting init load balance info in file id: {}\n", mesh_exoid);
fmt::print("\tNumber Internal Nodes: {}\n", globals.Num_Internal_Nodes[iproc]);
fmt::print("\tNumber Border Nodes: {}\n", globals.Num_Border_Nodes[iproc]);
fmt::print("\tNumber External Nodes: {}\n", globals.Num_External_Nodes[iproc]);
fmt::print("\tNumber Internal Elements: {}\n", globals.Num_Internal_Elems[iproc]);
fmt::print("\tNumber Border Elements: {}\n", globals.Num_Border_Elems[iproc]);
fmt::print("\tNumber Nodal Cmaps: {}\n", ncomm_cnt);
fmt::print("\tNumber Elemental Cmaps: {}\n", ecomm_cnt);
fmt::print("\tProccesor For: {}\n", proc_for);
}
if (ex_put_loadbal_param(mesh_exoid, globals.Num_Internal_Nodes[iproc],
globals.Num_Border_Nodes[iproc], globals.Num_External_Nodes[iproc],
globals.Num_Internal_Elems[iproc], globals.Num_Border_Elems[iproc],
ncomm_cnt, ecomm_cnt, proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output load balance info\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[1] = (second() - tt1);
total_out_time += PIO_Time_Array[1];
/* Output the communication map */
bytes_out += 4 * sizeof(INT);
tt1 = second();
if (ex_put_cmap_params(mesh_exoid, n_comm_ids.data(), n_comm_ncnts.data(), e_comm_ids.data(),
e_comm_ecnts.data(), proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output comm map params!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[2] = (second() - tt1);
total_out_time += PIO_Time_Array[2];
/* Convert Elem_Map to local element numbering */
reverse_map(globals.Elem_Map[iproc], 0, itotal_elems, globals.GElems[iproc], (INT *)nullptr,
globals.Elem_Map[iproc]);
/* Convert element IDs in the comm map to local numbering */
for (INT i0 = 0; i0 < ecomm_cnt; i0++) {
reverse_map(e_comm_map[i0].elem_ids, 0, e_comm_map[i0].elem_cnt, globals.GElems[iproc],
(INT *)nullptr, e_comm_map[i0].elem_ids);
}
/* Sort the globals.GNodes array using the index array 'loc_index' */
std::vector<INT> loc_index(itotal_nodes);
/* Initialize index array */
for (size_t i2 = 0; i2 < itotal_nodes; i2++) {
loc_index[i2] = i2;
}
/*
* Sort the globals.GNodes[iproc] array via the index array
* 'loc_index'
*/
gds_iqsort(globals.GNodes[iproc].data(), loc_index.data(), itotal_nodes);
/* Convert nodal IDs in the comm map to local numbering */
for (INT i0 = 0; i0 < ncomm_cnt; i0++) {
reverse_map(n_comm_map[i0].node_ids, 0, n_comm_map[i0].node_cnt, globals.GNodes[iproc],
loc_index.data(), n_comm_map[i0].node_ids);
}
PIO_Time_Array[3] = 0.0;
if (globals.Num_N_Comm_Maps[iproc] > 0 && globals.N_Comm_Map[iproc].node_cnt != 0) {
bytes_out += 2 * globals.Num_External_Nodes[iproc] * sizeof(INT);
tt1 = second();
for (INT i1 = 0; i1 < ncomm_cnt; i1++) {
if (ex_put_node_cmap(mesh_exoid, n_comm_ids[i1], n_comm_map[i1].node_ids.data(),
n_comm_map[i1].proc_ids.data(), proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output nodal comm map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
n_comm_map[i1].proc_ids.clear();
n_comm_map[i1].node_ids.clear();
}
PIO_Time_Array[3] += (second() - tt1);
}
if (globals.Num_E_Comm_Maps[iproc] > 0) {
bytes_out += 3 * (globals.E_Comm_Map[iproc].elem_cnt) * sizeof(INT);
tt1 = second();
for (INT i1 = 0; i1 < ecomm_cnt; i1++) {
if (ex_put_elem_cmap(mesh_exoid, e_comm_ids[i1], e_comm_map[i1].elem_ids.data(),
e_comm_map[i1].side_ids.data(), e_comm_map[i1].proc_ids.data(),
proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output elemental comm map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
e_comm_map[i1].proc_ids.clear();
e_comm_map[i1].elem_ids.clear();
e_comm_map[i1].side_ids.clear();
}
PIO_Time_Array[3] += (second() - tt1);
}
total_out_time += PIO_Time_Array[3];
/* Output the global node set parameters */
if (globals.Num_Node_Set > 0) {
std::vector<INT> glob_ns_df_cnts(globals.Num_Node_Set);
bytes_out += 3 * globals.Num_Node_Set * sizeof(INT);
tt1 = second();
if (ex_put_ns_param_global(mesh_exoid, Node_Set_Ids.data(), Num_Nodes_In_NS.data(),
glob_ns_df_cnts.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global node-set params\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[4] = (second() - tt1);
total_out_time += PIO_Time_Array[4];
}
/* Output the global side set parameters */
if (globals.Num_Side_Set > 0) {
std::vector<INT> glob_ss_df_cnts(globals.Num_Side_Set);
bytes_out += 3 * globals.Num_Side_Set * sizeof(INT);
tt1 = second();
if (ex_put_ss_param_global(mesh_exoid, Side_Set_Ids.data(), Num_Elems_In_SS.data(),
glob_ss_df_cnts.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global side-set params\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[5] = (second() - tt1);
total_out_time += PIO_Time_Array[5];
}
bytes_out += globals.Num_Elem_Blk * sizeof(INT);
tt1 = second();
if (ex_put_eb_info_global(mesh_exoid, Elem_Blk_Ids.data(), Num_Elems_In_EB.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global elem blk IDs\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[6] = (second() - tt1);
total_out_time += PIO_Time_Array[6];
/*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
/* Generate a QA record for the utility */
time_t date_time = time(nullptr);
auto *lt = std::localtime(&date_time);
char qa_time[MAX_STR_LENGTH + 1];
char qa_name[MAX_STR_LENGTH + 1];
char qa_vers[MAX_STR_LENGTH + 1];
char qa_date[MAX_STR_LENGTH + 1];
std::string time = fmt::format("{:%H:%M:%S}", *lt);
std::string date = fmt::format("{:%Y/%m/%d}", *lt);
copy_string(qa_date, date);
copy_string(qa_time, time);
copy_string(qa_name, UTIL_NAME);
copy_string(qa_vers, VER_STR);
if (globals.Num_QA_Recs > 0) {
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 0], qa_name,
MAX_STR_LENGTH + 1);
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 1], qa_vers,
MAX_STR_LENGTH + 1);
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 2], qa_date,
MAX_STR_LENGTH + 1);
copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 3], qa_time,
MAX_STR_LENGTH + 1);
/* Output QA records to screen */
if (Debug_Flag >= 4) {
fmt::print("Number of QA records: {}\n", globals.Num_QA_Recs);
if (Debug_Flag >= 6) {
fmt::print("QA Records:\n");
for (INT i1 = 0; i1 < 4 * (globals.Num_QA_Recs); i1++) {
fmt::print("\t{}\n", globals.QA_Record[i1]);
}
}
}
}
/* Output the QA and Info records */
for (INT i1 = 0; i1 < 4 * globals.Num_QA_Recs; i1++) {
bytes_out += (MAX_STR_LENGTH + MAX_LINE_LENGTH) * sizeof(char);
}
tt1 = second();
if (ex_put_qa(mesh_exoid, globals.Num_QA_Recs, (char *(*)[4]) & globals.QA_Record[0]) < 0) {
fmt::print(stderr, "[{}]: ERROR Could not put QA records\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (globals.Num_Info_Recs > 0) {
if (Debug_Flag >= 4) {
fmt::print("Number of info records: {}\n", globals.Num_Info_Recs);
}
if (ex_put_info(mesh_exoid, globals.Num_Info_Recs, globals.Info_Record) < 0) {
fmt::print(stderr, "[{}]: ERROR Could not put Info records\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
PIO_Time_Array[8] = (second() - tt1);
total_out_time += PIO_Time_Array[8];
/* Output the assembly information (if any). This puts the
file in/out of define mode, so should be early in the write stage
*/
if (Debug_Flag >= 4) {
fmt::print("Number of Assemblies: {}\n", globals.Num_Assemblies);
}
if (globals.Num_Assemblies > 0) {
ex_put_assemblies(mesh_exoid, globals.Assemblies.size(), globals.Assemblies.data());
}
/* Output the coordinate frame information (if any). This puts the
file in/out of define mode, so should be early in the write stage
*/
if (Debug_Flag >= 4) {
fmt::print("Number of Coordinate Frames: {}\n", globals.Num_Coordinate_Frames);
}
if (globals.Num_Coordinate_Frames > 0) {
T *Coordinate_Frame_Coordinates = globals.Coordinate_Frame_Coordinates;
if (ex_put_coordinate_frames(mesh_exoid, globals.Num_Coordinate_Frames,
globals.Coordinate_Frame_Ids, Coordinate_Frame_Coordinates,
globals.Coordinate_Frame_Tags) < 0) {
fmt::print(stderr, "[{}]: ERROR, Unable to put coordinate frame data in parallel mesh file\n",
__func__);
ex_close(mesh_exoid);
exit(1);
}
}
std::string cTitle;
if (proc_for == 0) {
cTitle = GeomTitle;
}
else {
cTitle = fmt::format("Parallel Mesh File for Processor {}", proc_for);
}
/* Output the initial information to the parallel Exodus file(s) */
bytes_out += cTitle.length() * sizeof(char) + 6 * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting init info in file id: {}\n", mesh_exoid);
fmt::print("\tTitle: {}\n", cTitle);
fmt::print("\tNumber Dimensions: {}\n", globals.Num_Dim);
fmt::print("\tNumber Nodes: {}\n", itotal_nodes);
fmt::print("\tNumber Elements: {}\n",
globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
fmt::print("\tNumber Element Blocks: {}\n", globals.Num_Elem_Blk);
fmt::print("\tNumber Node Sets: {}\n", globals.Num_Node_Set);
fmt::print("\tNumber Side Sets: {}\n", globals.Num_Side_Set);
}
if (ex_put_init(mesh_exoid, cTitle.c_str(), globals.Num_Dim, itotal_nodes,
globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc],
globals.Num_Elem_Blk, globals.Num_Node_Set, globals.Num_Side_Set) < 0) {
fmt::print(stderr, "[{}]: ERROR, Unable to put initial info in parallel mesh file\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
/* Output names... */
if (globals.Num_Elem_Blk > 0) {
error = ex_put_names(mesh_exoid, EX_ELEM_BLOCK, &Elem_Blk_Names[0]);
check_exodus_error(error, "ex_put_names");
}
if (globals.Num_Node_Set > 0) {
error = ex_put_names(mesh_exoid, EX_NODE_SET, &Node_Set_Names[0]);
check_exodus_error(error, "ex_put_names");
}
if (globals.Num_Side_Set > 0) {
error = ex_put_names(mesh_exoid, EX_SIDE_SET, &Side_Set_Names[0]);
check_exodus_error(error, "ex_put_names");
}
PIO_Time_Array[7] = (second() - tt1);
total_out_time += PIO_Time_Array[7];
/* Assign the coordinates to the coord_vector */
T *x_coord = nullptr;
T *y_coord = nullptr;
T *z_coord = nullptr;
if (itotal_nodes > 0) {
switch (globals.Num_Dim) {
case 3: z_coord = globals.Coor[iproc][2]; FALL_THROUGH;
case 2: y_coord = globals.Coor[iproc][1]; FALL_THROUGH;
case 1: x_coord = globals.Coor[iproc][0]; break;
}
}
/* Output the coordinates to the parallel Exodus file */
bytes_out += globals.Num_Dim * itotal_nodes * io_ws;
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting coordinate info in file id: {}\n", mesh_exoid);
}
if (ex_put_coord(mesh_exoid, x_coord, y_coord, z_coord) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write out nodal coordinates\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[9] = (second() - tt1);
total_out_time += PIO_Time_Array[9];
bytes_out += itotal_nodes * sizeof(INT);
tt1 = second();
/*
* The Nemesis node maps are lists of internal, border and external
* FEM node numbers. These are output as local node numbers.
*/
std::vector<INT> nem_node_mapi(globals.Num_Internal_Nodes[iproc], 1);
std::vector<INT> nem_node_mapb(globals.Num_Border_Nodes[iproc], 1);
std::vector<INT> nem_node_mape(globals.Num_External_Nodes[iproc], 1);
if (ex_put_processor_node_maps(mesh_exoid, nem_node_mapi.data(), nem_node_mapb.data(),
nem_node_mape.data(), proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write Nemesis nodal number map!\n", __func__);
check_exodus_error(ex_close(mesh_exoid), "ex_close");
ex_close(mesh_exoid);
exit(1);
}
/* If non-nullptr, output the global node id map which preserves
the global node ids in the original mesh */
if (!globals.Proc_Global_Node_Id_Map[iproc].empty()) {
bytes_out += itotal_nodes * sizeof(INT);
if (ex_put_map_param(mesh_exoid, 1, 0) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global node map parameters!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_num_map(mesh_exoid, EX_NODE_MAP, 1, globals.Proc_Global_Node_Id_Map[iproc].data()) <
0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global node id map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_name(mesh_exoid, EX_NODE_MAP, 1, "original_global_id_map") < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global node map name!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
PIO_Time_Array[11] = (second() - tt1);
total_out_time += PIO_Time_Array[11];
nem_node_mapi.clear();
nem_node_mapb.clear();
nem_node_mape.clear();
PIO_Time_Array[13] = 0.0;
PIO_Time_Array[14] = 0.0;
PIO_Time_Array[15] = 0.0;
/*
* Generate a list of the global element blocks, some of which may be
* nullptr on a given processor.
*/
{
/*
* Start a local block so we can define some variables locally
* here instead of a few thousand lines up from here...
*/
std::vector<INT> EB_Ids(globals.Num_Elem_Blk);
std::vector<INT> EB_Cnts(globals.Num_Elem_Blk);
std::vector<INT> EB_NperE(globals.Num_Elem_Blk);
std::vector<INT> EB_Nattr(globals.Num_Elem_Blk);
char **EB_Types = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk,
MAX_STR_LENGTH + 1, sizeof(char));
if (EB_Types == nullptr) {
fmt::print(stderr, "{}: fatal: insufficient memory\n", __func__);
exit(1);
}
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
memset(EB_Types[i1], '\0', (MAX_STR_LENGTH + 1));
}
INT cnt = 0;
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
bool ifound = false;
for (INT i2 = 0; i2 < globals.Proc_Num_Elem_Blk[iproc]; i2++) {
if (globals.Proc_Elem_Blk_Ids[iproc][i2] == Elem_Blk_Ids[i1]) {
ifound = true;
break;
}
}
/*
* If it's not found then this element block is null on the current
* processor.
*/
if (!ifound) {
INT iblk = globals.Proc_Num_Elem_Blk[iproc] + cnt;
globals.Proc_Elem_Blk_Ids[iproc][iblk] = Elem_Blk_Ids[i1];
globals.Proc_Num_Elem_In_Blk[iproc][iblk] = 0;
globals.Proc_Nodes_Per_Elem[iproc][iblk] = 0;
globals.Proc_Num_Attr[iproc][iblk] = 0;
cnt++;
}
}
/* Output the elemental block(s). */
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
/* Find the local element block index */
INT ilocal;
for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
break;
}
}
/* Error check */
if (ilocal >= globals.Num_Elem_Blk) {
fmt::print(stderr, "[{}]: Error finding local element block ID\n", __func__);
exit(1);
}
/* If it's a non-null block output all information */
EB_Ids[i1] = iglobal_blk;
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
/* Generate the ExodusII element name */
copy_string(EB_Types[i1], Elem_Blk_Types[i1], MAX_STR_LENGTH);
EB_Cnts[i1] = globals.Proc_Num_Elem_In_Blk[iproc][ilocal];
EB_NperE[i1] = globals.Proc_Nodes_Per_Elem[iproc][ilocal];
EB_Nattr[i1] = globals.Proc_Num_Attr[iproc][ilocal];
}
}
if (Debug_Flag >= 4) {
fmt::print("Putting concat_elem_block info in file id: {}\n", mesh_exoid);
}
error = ex_put_concat_elem_block(mesh_exoid, &EB_Ids[0], &EB_Types[0], &EB_Cnts[0],
&EB_NperE[0], &EB_Nattr[0], 1);
check_exodus_error(error, "ex_put_concat_elem_block");
safe_free(reinterpret_cast<void **>(&EB_Types));
/* Output attribute names for each element block */
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
/* Find the local element block index */
INT ilocal;
for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
break;
}
}
/* If it's a non-null block output attribute name information */
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
if (globals.Proc_Num_Attr[iproc][ilocal] > 0) {
if (ex_put_attr_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i1],
Elem_Blk_Attr_Names[i1]) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write Exodus attribute names!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
}
}
/* Reset globals.GNodes to start at 1 instead of 0 */
for (size_t i1 = 0; i1 < itotal_nodes; (globals.GNodes[iproc][i1++])++) {
;
}
/* Output the Exodus node number map */
bytes_out += itotal_nodes * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting node_num_map in file id: {}\n", mesh_exoid);
}
if (ex_put_id_map(mesh_exoid, EX_NODE_MAP, globals.GNodes[iproc].data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not write Exodus node number map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[10] = (second() - tt1);
total_out_time += PIO_Time_Array[10];
/*
* Allocate memory for the elemental map. Currently this map is assigned
* as a linear array since it is not really used.
*/
std::vector<INT> iElem_Map(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
for (INT i1 = 0; i1 < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
i1++) {
iElem_Map[i1] = globals.GElems[iproc][i1] + 1;
}
bytes_out +=
2 * globals.Num_Internal_Elems[iproc] * globals.Num_Border_Elems[iproc] * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting elem_num_map info in file id: {}\n", mesh_exoid);
}
if (ex_put_id_map(mesh_exoid, EX_ELEM_MAP, iElem_Map.data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output element map\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
/* If non-nullptr, output the global element id map which preserves
the global element ids in the original mesh */
if (!globals.Proc_Global_Elem_Id_Map[iproc].empty()) {
bytes_out +=
globals.Num_Internal_Elems[iproc] * globals.Num_Border_Elems[iproc] * sizeof(INT);
if (ex_put_map_param(mesh_exoid, 0, 1) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global map parameters!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_num_map(mesh_exoid, EX_ELEM_MAP, 1,
globals.Proc_Global_Elem_Id_Map[iproc].data()) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output global id map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
if (ex_put_name(mesh_exoid, EX_ELEM_MAP, 1, "original_global_id_map") < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to define global map name!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
}
/* Also output the Nemesis element map */
if (ex_put_processor_elem_maps(
mesh_exoid, globals.Elem_Map[iproc].data(),
(globals.Elem_Map[iproc].data()) + globals.Num_Internal_Elems[iproc], proc_for) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output nemesis element map!\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[12] = (second() - tt1);
total_out_time += PIO_Time_Array[12];
iElem_Map.clear();
for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
/* Find the local element block index */
INT ilocal;
for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
break;
}
}
/* Error check */
if (ilocal >= globals.Num_Elem_Blk) {
fmt::print(stderr, "[{}]: Error finding local element block ID\n", __func__);
exit(1);
}
/* If it's a non-null block output all information */
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
/* Find the first index into the connectivity for this block */
size_t iIndex0 = 0;
for (INT i2 = 0; i2 < ilocal; i2++) {
iIndex0 +=
globals.Proc_Num_Elem_In_Blk[iproc][i2] * globals.Proc_Nodes_Per_Elem[iproc][i2];
}
PIO_Time_Array[13] += (second() - tt1);
size_t tmp_cnt = globals.Proc_Num_Elem_In_Blk[iproc][ilocal] *
globals.Proc_Nodes_Per_Elem[iproc][ilocal];
/* Generate the connectivity array for local node numbering */
{
std::vector<INT> proc_local_conn(tmp_cnt);
reverse_map(&globals.Proc_Elem_Connect[iproc][iIndex0], 1, tmp_cnt,
globals.GNodes[iproc].data(), loc_index.data(), proc_local_conn.data());
bytes_out += globals.Proc_Nodes_Per_Elem[iproc][ilocal] *
globals.Proc_Num_Elem_In_Blk[iproc][ilocal] * sizeof(INT);
tt1 = second();
if (Debug_Flag >= 4) {
fmt::print("Putting element_connectivity info in file id: {}\n", mesh_exoid);
}
if (ex_put_conn(mesh_exoid, EX_ELEM_BLOCK, globals.Proc_Elem_Blk_Ids[iproc][ilocal],
proc_local_conn.data(), nullptr, nullptr) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output connectivity\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[14] += (second() - tt1);
}
if (globals.Proc_Num_Attr[iproc][ilocal] > 0) {
/* Find the first index into the attribute list for this block */
size_t iIndex1 = 0;
for (INT i2 = 0; i2 < ilocal; i2++) {
iIndex1 += globals.Proc_Num_Attr[iproc][i2] * globals.Proc_Num_Elem_In_Blk[iproc][i2];
}
bytes_out += globals.Proc_Num_Elem_In_Blk[iproc][ilocal] * io_ws;
tt1 = second();
T *ptr = &(globals.Proc_Elem_Attr[iproc][iIndex1]);
if (ex_put_attr(mesh_exoid, EX_ELEM_BLOCK, globals.Proc_Elem_Blk_Ids[iproc][ilocal],
ptr) < 0) {
fmt::print(stderr, "[{}]: ERROR, unable to output element attributes\n", __func__);
exit(1);
}
PIO_Time_Array[15] += (second() - tt1);
total_out_time += PIO_Time_Array[15];
}
} /* End "if(ilocal < globals.Num_Elem_Blk[iproc])" */
} /* End "for(i1=0; i1 < globals.Num_Elem_Block; i1++)" */
}
total_out_time += (PIO_Time_Array[13] + PIO_Time_Array[14] + PIO_Time_Array[15]);
/*
* Write out the node-set information. Note that the value of the
* node-set distribution factor is not currently used so only a
* dummy set is output for this value.
*/
INT iMaxLen = 0;
for (INT i1 = 0; i1 < globals.Proc_Num_Node_Sets[iproc]; i1++) {
iMaxLen = PEX_MAX(globals.Proc_NS_Count[iproc][i1], iMaxLen);
}
/* Renumber Node set node lists to use local node numbers */
std::vector<INT> proc_local_ns;
if (globals.Proc_Num_Node_Sets[iproc] > 0) {
proc_local_ns.resize(globals.Proc_NS_List_Length[iproc]);
reverse_map(&globals.Proc_NS_List[iproc][0], 1, globals.Proc_NS_List_Length[iproc],
globals.GNodes[iproc].data(), loc_index.data(), proc_local_ns.data());
}
loc_index.clear();
PIO_Time_Array[16] = 0.0;
PIO_Time_Array[17] = 0.0;
/* Fill in the information for the nullptr node sets */
size_t cnt = 0;
for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
bool ifound = false;
for (INT i2 = 0; i2 < globals.Proc_Num_Node_Sets[iproc]; i2++) {
if (globals.Proc_NS_Ids[iproc][i2] == Node_Set_Ids[i1]) {
ifound = true;
break;
}
}
if (!ifound) {
globals.Proc_NS_Ids[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = Node_Set_Ids[i1];
globals.Proc_NS_Count[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = 0;
globals.Proc_NS_DF_Count[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = 0;
cnt++;
}
}
tt1 = second();
if (globals.Num_Node_Set > 0) {
size_t dcount = 0;
size_t ncount = 0;
for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
dcount += globals.Proc_NS_DF_Count[iproc][i1];
ncount += globals.Proc_NS_Count[iproc][i1];
}
std::vector<INT> conc_ids(globals.Num_Node_Set);
std::vector<INT> conc_nodes(globals.Num_Node_Set);
std::vector<INT> conc_df(globals.Num_Node_Set);
std::vector<INT> conc_nind(globals.Num_Node_Set);
std::vector<INT> conc_dind(globals.Num_Node_Set);
std::vector<INT> conc_nlist(ncount);
std::vector<T> conc_sdf(dcount);
ncount = 0;
dcount = 0;
for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
/* Find the local ID */
INT i2 = 0;
for (i2 = 0; i2 < globals.Num_Node_Set; i2++) {
if (globals.Proc_NS_Ids[iproc][i2] == Node_Set_Ids[i1]) {
break;
}
}
conc_ids[i1] = globals.Proc_NS_Ids[iproc][i2];
conc_nodes[i1] = globals.Proc_NS_Count[iproc][i2];
conc_df[i1] = globals.Proc_NS_DF_Count[iproc][i2];
conc_nind[i1] = ncount;
for (INT i3 = 0; i3 < globals.Proc_NS_Count[iproc][i2]; i3++) {
conc_nlist[ncount++] = proc_local_ns[globals.Proc_NS_Pointers[iproc][i2] + i3];
}
conc_dind[i1] = dcount;
for (INT i3 = 0; i3 < globals.Proc_NS_DF_Count[iproc][i2]; i3++) {
conc_sdf[dcount++] =
globals.Proc_NS_Dist_Fact[iproc][globals.Proc_NS_Pointers[iproc][i2] + i3];
}
}
ex_set_specs set_specs{};
set_specs.sets_ids = conc_ids.data();
set_specs.num_entries_per_set = conc_nodes.data();
set_specs.num_dist_per_set = conc_df.data();
set_specs.sets_entry_index = conc_nind.data();
set_specs.sets_dist_index = conc_dind.data();
set_specs.sets_entry_list = conc_nlist.data();
set_specs.sets_extra_list = nullptr;
set_specs.sets_dist_fact = conc_sdf.data();
ex_put_concat_sets(mesh_exoid, EX_NODE_SET, &set_specs);
}
total_out_time += second() - tt1;
/* Free local number array */
if (globals.Proc_Num_Node_Sets[iproc] > 0) {
proc_local_ns.clear();
}
/* Renumber element SS to use local element numbers */
std::vector<INT> proc_local_ss;
if (globals.Proc_Num_Side_Sets[iproc] > 0) {
proc_local_ss.resize(globals.Proc_SS_Elem_List_Length[iproc]);
reverse_map(&globals.Proc_SS_Elem_List[iproc][0], 0, globals.Proc_SS_Elem_List_Length[iproc],
globals.GElems[iproc].data(), (INT *)nullptr, proc_local_ss.data());
}
PIO_Time_Array[18] = 0.0;
PIO_Time_Array[19] = 0.0;
/* Set up the null side sets */
cnt = 0;
for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
bool ifound = false;
for (INT i2 = 0; i2 < globals.Proc_Num_Side_Sets[iproc]; i2++) {
if (globals.Proc_SS_Ids[iproc][i2] == Side_Set_Ids[i1]) {
ifound = true;
break;
}
}
if (!ifound) {
globals.Proc_SS_Ids[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = Side_Set_Ids[i1];
globals.Proc_SS_Elem_Count[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = 0;
globals.Proc_SS_DF_Count[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = 0;
cnt++;
}
}
/* Output concatenated sidesets. For each processor need:
* side_set ids (size globals.Num_Side_Set)
* num_side_per_set (size globals.Num_Side_Set)
* num_dist_per_set (size globals.Num_Side_Set)
* side_sets_elem_index (size globals.Num_Side_Set)
* side_sets_dist_index (size globals.Num_Side_Set)
* side_sets_elem_list
* side_sets_side_list
* side_sets_dist_fact
*/
tt1 = second();
if (globals.Num_Side_Set > 0) {
size_t df_count = 0;
size_t el_count = 0;
for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
df_count += globals.Proc_SS_DF_Count[iproc][i1];
el_count += globals.Proc_SS_Elem_Count[iproc][i1];
}
std::vector<INT> conc_ids(globals.Num_Side_Set);
std::vector<INT> conc_sides(globals.Num_Side_Set);
std::vector<INT> conc_dist(globals.Num_Side_Set);
std::vector<INT> conc_eind(globals.Num_Side_Set);
std::vector<INT> conc_dind(globals.Num_Side_Set);
std::vector<INT> conc_elist(el_count);
std::vector<INT> conc_slist(el_count);
std::vector<T> conc_sdflist(df_count);
/* Fill in the arrays ... */
df_count = 0;
el_count = 0;
for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
/* Find the local ID of this side set */
INT i2 = 0;
for (i2 = 0; i2 < globals.Num_Side_Set; i2++) {
if (globals.Proc_SS_Ids[iproc][i2] == Side_Set_Ids[i1]) {
break;
}
}
conc_ids[i1] = globals.Proc_SS_Ids[iproc][i2];
conc_sides[i1] = globals.Proc_SS_Elem_Count[iproc][i2];
conc_dist[i1] = globals.Proc_SS_DF_Count[iproc][i2];
conc_eind[i1] = el_count;
for (INT i3 = 0; i3 < globals.Proc_SS_Elem_Count[iproc][i2]; i3++) {
conc_elist[el_count] = proc_local_ss[globals.Proc_SS_Elem_Pointers[iproc][i2] + i3];
conc_slist[el_count] =
globals.Proc_SS_Side_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i2] + i3];
el_count++;
}
conc_dind[i1] = df_count;
for (INT i3 = 0; i3 < globals.Proc_SS_DF_Count[iproc][i2]; i3++) {
conc_sdflist[df_count++] =
globals.Proc_SS_Dist_Fact[iproc][globals.Proc_SS_DF_Pointers[iproc][i2] + i3];
}
}
ex_set_specs set_specs{};
set_specs.sets_ids = conc_ids.data();
set_specs.num_entries_per_set = conc_sides.data();
set_specs.num_dist_per_set = conc_dist.data();
set_specs.sets_entry_index = conc_eind.data();
set_specs.sets_dist_index = conc_dind.data();
set_specs.sets_entry_list = conc_elist.data();
set_specs.sets_extra_list = conc_slist.data();
set_specs.sets_dist_fact = conc_sdflist.data();
ex_put_concat_sets(mesh_exoid, EX_SIDE_SET, &set_specs);
}
PIO_Time_Array[19] += (second() - tt1);
total_out_time += (PIO_Time_Array[18] + PIO_Time_Array[19]);
/* Free unneeded memory */
if (globals.Proc_Num_Side_Sets[iproc] > 0) {
proc_local_ss.clear();
}
/*
* Write out the name of the coordinate axes to the parallel ExodusII
* files.
*/
bytes_out += globals.Num_Dim * 8 * sizeof(char);
tt1 = second();
if (ex_put_coord_names(mesh_exoid, Coord_Name) < 0) {
fmt::print(stderr, "[{}]: ERROR, could not output coordinate names\n", __func__);
ex_close(mesh_exoid);
exit(1);
}
PIO_Time_Array[20] = (second() - tt1);
total_out_time += PIO_Time_Array[20];
if (Restart_Info.Flag > 0) {
tt1 = second();
bytes_out +=
write_var_param(mesh_exoid, max_name_length, Restart_Info.NVar_Glob, Restart_Info.GV_Name,
Restart_Info.NVar_Node, Restart_Info.NV_Name, Restart_Info.NVar_Elem,
Restart_Info.EV_Name, Restart_Info.GElem_TT.data(), Restart_Info.NVar_Nset,
Restart_Info.NSV_Name, Restart_Info.GNset_TT.data(), Restart_Info.NVar_Sset,
Restart_Info.SSV_Name, Restart_Info.GSset_TT.data());
PIO_Time_Array[21] = (second() - tt1);
total_out_time += PIO_Time_Array[21];
}
/* Calculate the overall performance */
tt1 = bytes_out;
if (total_out_time == 0) {
tt1 = 0;
}
else {
tt1 = tt1 / total_out_time;
}
tt1 /= 1024.0;
PIO_Time_Array[25] = tt1;
} /* END write_parExo_data() */
template <typename T, typename INT>
int NemSpread<T, INT>::write_var_param(int mesh_exoid, int max_name_length, int num_glob,
char **gv_names, int num_node, char **nv_names, int num_elem,
char **ev_names, int *local_ebtt, int num_nset,
char **ns_names, int *local_nstt, int num_sset,
char **ss_names, int *local_sstt)
{
size_t bytes_out = 0;
int error;
bytes_out += (5 + globals.Num_Elem_Blk * num_elem + globals.Num_Side_Set * num_sset +
globals.Num_Node_Set * num_nset) *
sizeof(INT);
error = ex_put_all_var_param(mesh_exoid, num_glob, num_node, num_elem, local_ebtt, num_nset,
local_nstt, num_sset, local_sstt);
check_exodus_error(error, "ex_put_all_var_param");
if (gv_names != nullptr) {
bytes_out += Restart_Info.NVar_Glob * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_GLOBAL, num_glob, gv_names);
check_exodus_error(error, "ex_put_var_names");
}
if (nv_names != nullptr) {
bytes_out += num_node * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_NODAL, num_node, nv_names);
check_exodus_error(error, "ex_put_var_names");
}
if (ev_names != nullptr) {
bytes_out += Restart_Info.NVar_Elem * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_ELEM_BLOCK, num_elem, ev_names);
check_exodus_error(error, "ex_put_var_names");
}
if (ns_names != nullptr) {
bytes_out += Restart_Info.NVar_Nset * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_NODE_SET, num_nset, ns_names);
check_exodus_error(error, "ex_put_var_names");
}
if (ss_names != nullptr) {
bytes_out += Restart_Info.NVar_Sset * max_name_length;
error = ex_put_variable_names(mesh_exoid, EX_SIDE_SET, num_sset, ss_names);
check_exodus_error(error, "ex_put_var_names");
}
return (bytes_out);
}
template void NemSpread<double, int>::write_var_timestep(int exoid, int proc, int time_step,
int *eb_ids_global, int *ss_ids_global,
int *ns_ids_global);
template void NemSpread<float, int>::write_var_timestep(int exoid, int proc, int time_step,
int *eb_ids_global, int *ss_ids_global,
int *ns_ids_global);
template void NemSpread<double, int64_t>::write_var_timestep(int exoid, int proc, int time_step,
int64_t *eb_ids_global,
int64_t *ss_ids_global,
int64_t *ns_ids_global);
template void NemSpread<float, int64_t>::write_var_timestep(int exoid, int proc, int time_step,
int64_t *eb_ids_global,
int64_t *ss_ids_global,
int64_t *ns_ids_global);
template <typename T, typename INT>
void NemSpread<T, INT>::write_var_timestep(int exoid, int proc, int time_step, INT *eb_ids_global,
INT *ss_ids_global, INT *ns_ids_global)
{
int error;
/* output the time */
{
T *var_ptr = (T *)&(Restart_Info.Time);
error = ex_put_time(exoid, time_step, var_ptr);
check_exodus_error(error, "ex_put_time");
}
/* start by outputting the global variables */
if (Restart_Info.NVar_Glob > 0) {
T *var_ptr = &Restart_Info.Glob_Vals[0];
error = ex_put_var(exoid, time_step, EX_GLOBAL, 1, 0, Restart_Info.NVar_Glob, var_ptr);
check_exodus_error(error, "ex_put_glob_vars");
}
if (Restart_Info.NVar_Node > 0) {
size_t num_nodes = globals.Num_Internal_Nodes[proc] + globals.Num_Border_Nodes[proc] +
globals.Num_External_Nodes[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Node; var_num++) {
size_t var_offset = var_num * num_nodes;
T *var_ptr = &(Restart_Info.Node_Vals[proc][var_offset]);
error = ex_put_var(exoid, time_step, EX_NODAL, (var_num + 1), 1, num_nodes, var_ptr);
check_exodus_error(error, "ex_put_var");
}
}
if (Restart_Info.NVar_Elem > 0) {
size_t num_elem = globals.Num_Internal_Elems[proc] + globals.Num_Border_Elems[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Elem; var_num++) {
int eb_num_g = 0;
size_t var_offset = var_num * num_elem;
T *var_ptr = &(Restart_Info.Elem_Vals[proc][var_offset]);
for (int eb_num = 0; eb_num < globals.Proc_Num_Elem_Blk[proc]; eb_num++) {
/* now I have to find the appropriate entry in the truth table */
/* can always assume this eb num is greater than the last one */
for (int cnt1 = eb_num_g; cnt1 < globals.Num_Elem_Blk; cnt1++) {
if (globals.Proc_Elem_Blk_Ids[proc][eb_num] == eb_ids_global[cnt1]) {
eb_num_g = cnt1;
break;
}
}
if (Restart_Info.GElem_TT[eb_num_g * Restart_Info.NVar_Elem + var_num]) {
error = ex_put_var(exoid, time_step, EX_ELEM_BLOCK, (var_num + 1),
globals.Proc_Elem_Blk_Ids[proc][eb_num],
globals.Proc_Num_Elem_In_Blk[proc][eb_num], var_ptr);
check_exodus_error(error, "ex_put_elem_var");
}
/* and now move the variable pointer */
/* Note that the offsetting here must match the 'var_offset'
* treatment in ps_restart.c function read_elem_vars.
* Currently, the offset is applied even if the variable does
* not exist on a particular block.
*/
var_ptr += globals.Proc_Num_Elem_In_Blk[proc][eb_num];
}
}
}
if (Restart_Info.NVar_Sset > 0) {
int ss_num_g = 0;
size_t num_elem = globals.Proc_SS_Elem_List_Length[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Sset; var_num++) {
size_t var_offset = var_num * num_elem;
T *var_ptr = &(Restart_Info.Sset_Vals[proc][var_offset]);
for (int ss_num = 0; ss_num < globals.Proc_Num_Side_Sets[proc]; ss_num++) {
/* now I have to find the appropriate entry in the truth table */
for (int cnt1 = 0; cnt1 < globals.Num_Side_Set; cnt1++) {
if (globals.Proc_SS_Ids[proc][ss_num] == ss_ids_global[cnt1]) {
ss_num_g = cnt1;
break;
}
}
assert(globals.Proc_SS_Ids[proc][ss_num] == ss_ids_global[ss_num_g]);
if (Restart_Info.GSset_TT[ss_num_g * Restart_Info.NVar_Sset + var_num]) {
error = ex_put_var(exoid, time_step, EX_SIDE_SET, (var_num + 1),
globals.Proc_SS_Ids[proc][ss_num],
globals.Proc_SS_Elem_Count[proc][ss_num], var_ptr);
check_exodus_error(error, "ex_put_sset_var");
}
/* and now move the variable pointer */
/* Note that the offsetting here must match the 'var_offset'
* treatment in ps_restart.c function read_elem_vars.
* Currently, the offset is applied even if the variable does
* not exist on a particular block.
*/
var_ptr += globals.Proc_SS_Elem_Count[proc][ss_num];
}
}
}
if (Restart_Info.NVar_Nset > 0) {
int ns_num_g = 0;
size_t num_elem = globals.Proc_NS_List_Length[proc];
for (int var_num = 0; var_num < Restart_Info.NVar_Nset; var_num++) {
size_t var_offset = var_num * num_elem;
T *var_ptr = &(Restart_Info.Nset_Vals[proc][var_offset]);
for (int ns_num = 0; ns_num < globals.Proc_Num_Node_Sets[proc]; ns_num++) {
/* now I have to find the appropriate entry in the truth table */
for (int cnt1 = 0; cnt1 < globals.Num_Node_Set; cnt1++) {
if (globals.Proc_NS_Ids[proc][ns_num] == ns_ids_global[cnt1]) {
ns_num_g = cnt1;
break;
}
}
assert(globals.Proc_NS_Ids[proc][ns_num] == ns_ids_global[ns_num_g]);
if (Restart_Info.GNset_TT[ns_num_g * Restart_Info.NVar_Nset + var_num]) {
error = ex_put_var(exoid, time_step, EX_NODE_SET, (var_num + 1),
globals.Proc_NS_Ids[proc][ns_num], globals.Proc_NS_Count[proc][ns_num],
var_ptr);
check_exodus_error(error, "ex_put_nset_var");
}
/* and now move the variable pointer */
/* Note that the offsetting here must match the 'var_offset'
* treatment in ps_restart.c function read_elem_vars.
* Currently, the offset is applied even if the variable does
* not exist on a particular block.
*/
var_ptr += globals.Proc_NS_Count[proc][ns_num];
}
}
}
}
namespace {
template <typename INT>
void reverse_map(const std::vector<INT> &global, int p01, size_t gsize,
const std::vector<INT> &glmap, INT *index, std::vector<INT> &mapout)
{
/*
* The 'global' array is an array of node or element numbers
* in the global id space. It needs to be converted to local
* numbers via the 'glmap' array. The glmap array is sorted
* by the 'index' array. The map from global to local is
* glmap[local_id] = global_id
*
* The 'p01' is either 0 or 1 and is an offset to the values
* in 'global'
*/
/*
* The algorithm used is to sort the 'global' array via the
* 'tmp_index' array (global[tmp_index[0..gsize]] is sorted)
* Then, progress through the 'global' array in sorted order
* and find the location in 'glmap'. Note that since both are
* sorted, it should be easy to progress sequentially through
* both arrays.
*/
std::vector<INT> tmp_index(gsize);
/* Initialize index array */
for (size_t i2 = 0; i2 < gsize; i2++) {
tmp_index[i2] = (INT)i2;
}
/* Sort the 'global' array via the index array 'tmp_index' */
gds_iqsort(global.data(), tmp_index.data(), gsize);
size_t i3 = 0;
if (index != nullptr) {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[index[i3]] < gval) {
i3++;
}
assert(glmap[index[i3]] == gval);
mapout[tmp_index[i2]] = index[i3] + 1;
}
}
else {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[i3] < gval) {
i3++;
}
assert(glmap[i3] == gval);
mapout[tmp_index[i2]] = i3 + 1;
}
}
}
template <typename INT>
void reverse_map(INT *global, int p01, size_t gsize, INT *glmap, INT *index, INT *mapout)
{
/*
* The 'global' array is an array of node or element numbers
* in the global id space. It needs to be converted to local
* numbers via the 'glmap' array. The glmap array is sorted
* by the 'index' array. The map from global to local is
* glmap[local_id] = global_id
*
* The 'p01' is either 0 or 1 and is an offset to the values
* in 'global'
*/
/*
* The algorithm used is to sort the 'global' array via the
* 'tmp_index' array (global[tmp_index[0..gsize]] is sorted)
* Then, progress through the 'global' array in sorted order
* and find the location in 'glmap'. Note that since both are
* sorted, it should be easy to progress sequentially through
* both arrays.
*/
std::vector<INT> tmp_index(gsize);
std::iota(tmp_index.begin(), tmp_index.end(), 0);
/* Sort the 'global' array via the index array 'tmp_index' */
gds_iqsort(global, tmp_index.data(), gsize);
size_t i3 = 0;
if (index != nullptr) {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[index[i3]] < gval) {
i3++;
}
assert(glmap[index[i3]] == gval);
mapout[tmp_index[i2]] = index[i3] + 1;
}
}
else {
for (size_t i2 = 0; i2 < gsize; i2++) {
INT gval = global[tmp_index[i2]] + p01;
while (glmap[i3] < gval) {
i3++;
}
assert(glmap[i3] == gval);
mapout[tmp_index[i2]] = i3 + 1;
}
}
}
} // namespace