Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EXODUS: Better handling of names in parallel io mode #592

Closed
wants to merge 9 commits into from
12 changes: 7 additions & 5 deletions packages/seacas/libraries/exodus/include/exodusII_int.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*

* Copyright(C) 1999-2020, 2022, 2023, 2024 National Technology & Engineering Solutions
* Copyright(C) 1999-2020, 2022, 2023, 2024, 2025 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
Expand Down Expand Up @@ -706,6 +706,7 @@ struct exi_file_item
gzip, 4..32 and even for szip; -131072..22 for zstd, NetCDF-4 only */
unsigned int assembly_count;
unsigned int blob_count;
unsigned int mpi_rank; /**< Only valid if `is_parallel` is true */

unsigned int persist_define_mode : 10; /**< Stay in define mode until exi_persist_leavedef is
called. Set by exi_persist_redef... */
Expand Down Expand Up @@ -786,15 +787,16 @@ EXODUS_EXPORT char *exi_name_red_var_of_object(ex_entity_type /*obj_type*/, int
EXODUS_EXPORT char *exi_name_of_map(ex_entity_type /*map_type*/, int /*map_index*/);

EXODUS_EXPORT int exi_conv_init(int exoid, int *comp_wordsize, int *io_wordsize, int file_wordsize,
int int64_status, bool is_parallel, bool is_hdf5, bool is_pnetcdf,
bool is_write);
int int64_status, int mpi_rank, bool is_parallel, bool is_hdf5,
bool is_pnetcdf, bool is_write);

EXODUS_EXPORT void exi_conv_exit(int exoid);

EXODUS_EXPORT nc_type nc_flt_code(int exoid);
EXODUS_EXPORT int exi_comp_ws(int exoid);
EXODUS_EXPORT int exi_get_cpu_ws(void);
EXODUS_EXPORT int exi_is_parallel(int exoid);
EXODUS_EXPORT int exi_parallel_rank(int exoid);

EXODUS_EXPORT struct exi_list_item **exi_get_counter_list(ex_entity_type obj_type);
EXODUS_EXPORT int exi_get_file_item(int /*exoid*/, struct exi_list_item **/*list_ptr*/);
Expand Down Expand Up @@ -883,8 +885,8 @@ EXODUS_EXPORT int exi_persist_leavedef(int exoid, /* NemesisI file I

EXODUS_EXPORT int exi_check_version(int run_version);
EXODUS_EXPORT int exi_handle_mode(unsigned int my_mode, int is_parallel, int run_version);
EXODUS_EXPORT int exi_populate_header(int exoid, const char *path, int my_mode, int is_parallel,
int *comp_ws, int *io_ws);
EXODUS_EXPORT int exi_populate_header(int exoid, const char *path, int my_mode, int my_rank,
int is_parallel, int *comp_ws, int *io_ws);

EXODUS_EXPORT int exi_get_block_param(int exoid, ex_entity_id id, int ndim,
struct exi_elem_blk_parm *elem_blk_parm);
Expand Down
30 changes: 28 additions & 2 deletions packages/seacas/libraries/exodus/src/ex_conv.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright(C) 1999-2021, 2024 National Technology & Engineering Solutions
* Copyright(C) 1999-2021, 2024, 2025 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
Expand Down Expand Up @@ -108,7 +108,8 @@ int exi_check_valid_file_id(int exoid, const char *func)
}

int exi_conv_init(int exoid, int *comp_wordsize, int *io_wordsize, int file_wordsize,
int int64_status, bool is_parallel, bool is_hdf5, bool is_pnetcdf, bool is_write)
int int64_status, int mpi_rank, bool is_parallel, bool is_hdf5, bool is_pnetcdf,
bool is_write)
{
char errmsg[MAX_ERR_LENGTH];
struct exi_file_item *new_file = NULL;
Expand Down Expand Up @@ -243,6 +244,7 @@ int exi_conv_init(int exoid, int *comp_wordsize, int *io_wordsize, int file_word
new_file->quantize_nsd = 0;
new_file->shuffle = 0;
new_file->file_type = filetype - 1;
new_file->mpi_rank = mpi_rank;
new_file->is_parallel = is_parallel;
new_file->is_hdf5 = is_hdf5;
new_file->is_pnetcdf = is_pnetcdf;
Expand Down Expand Up @@ -575,6 +577,30 @@ int exi_is_parallel(int exoid)
EX_FUNC_LEAVE(file->is_parallel);
}

/*!
* \ingroup Utilities exi_parallel_rank() returns the mpi rank for the
* current file *IF* the file was opened in parallel; otherwise
* (file-per-rank or serial), it will return 0.
*
* NOTE that in this
* case parallel assumes the output of a single file, not a parallel
* run using file-per-processor. \param exoid integer which uniquely
* identifies the file of interest.
*/
int exi_parallel_rank(int exoid)
{
EX_FUNC_ENTER();
struct exi_file_item *file = exi_find_file_item(exoid);

if (!file) {
char errmsg[MAX_ERR_LENGTH];
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: unknown file id %d", exoid);
ex_err(__func__, errmsg, EX_BADFILEID);
EX_FUNC_LEAVE(EX_FATAL);
}
EX_FUNC_LEAVE(file->is_parallel ? file->mpi_rank : 0);
}

/*!
* \ingroup Utilities
* \note
Expand Down
5 changes: 3 additions & 2 deletions packages/seacas/libraries/exodus/src/ex_create.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright(C) 1999-2021, 2023 National Technology & Engineering Solutions
* Copyright(C) 1999-2021, 2023, 2025 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
Expand Down Expand Up @@ -191,7 +191,8 @@ int ex_create_int(const char *path, int cmode, int *comp_ws, int *io_ws, int run
EX_FUNC_LEAVE(EX_FATAL);
}

status = exi_populate_header(exoid, canon_path, my_mode, is_parallel, comp_ws, io_ws);
int rank = 0;
status = exi_populate_header(exoid, canon_path, my_mode, rank, is_parallel, comp_ws, io_ws);
if (status != EX_NOERR) {
free(canon_path);
EX_FUNC_LEAVE(status);
Expand Down
6 changes: 4 additions & 2 deletions packages/seacas/libraries/exodus/src/ex_create_par.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright(C) 1999-2021, 2023, 2024 National Technology & Engineering Solutions
* Copyright(C) 1999-2021, 2023, 2024, 2025 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
Expand Down Expand Up @@ -227,7 +227,9 @@ int ex_create_par_int(const char *path, int cmode, int *comp_ws, int *io_ws, MPI
EX_FUNC_LEAVE(EX_FATAL);
}

status = exi_populate_header(exoid, canon_path, my_mode, is_parallel, comp_ws, io_ws);
int rank = 0;
MPI_Comm_rank(comm, &rank);
status = exi_populate_header(exoid, canon_path, my_mode, rank, is_parallel, comp_ws, io_ws);
if (status != EX_NOERR) {
free(canon_path);
EX_FUNC_LEAVE(status);
Expand Down
4 changes: 2 additions & 2 deletions packages/seacas/libraries/exodus/src/ex_open.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright(C) 1999-2024 National Technology & Engineering Solutions
* Copyright(C) 1999-2025 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
Expand Down Expand Up @@ -417,7 +417,7 @@ int ex_open_int(const char *path, int mode, int *comp_ws, int *io_ws, float *ver
}

/* initialize floating point and integer size conversion. */
if (exi_conv_init(exoid, comp_ws, io_ws, file_wordsize, int64_status, false, false, false,
if (exi_conv_init(exoid, comp_ws, io_ws, file_wordsize, int64_status, 0, false, false, false,
mode & EX_WRITE) != EX_NOERR) {
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to initialize conversion routines in file id %d named %s", exoid,
Expand Down
8 changes: 5 additions & 3 deletions packages/seacas/libraries/exodus/src/ex_open_par.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright(C) 1999-2024 National Technology & Engineering Solutions
* Copyright(C) 1999-2025 National Technology & Engineering Solutions
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
* NTESS, the U.S. Government retains certain rights in this software.
*
Expand Down Expand Up @@ -480,8 +480,10 @@ int ex_open_par_int(const char *path, int mode, int *comp_ws, int *io_ws, float
}

/* initialize floating point and integer size conversion. */
if (exi_conv_init(exoid, comp_ws, io_ws, file_wordsize, int64_status, 1, is_hdf5, is_pnetcdf,
mode & EX_WRITE) != EX_NOERR) {
int rank = 0;
MPI_Comm_rank(comm, &rank);
if (exi_conv_init(exoid, comp_ws, io_ws, file_wordsize, int64_status, rank, 1, is_hdf5,
is_pnetcdf, mode & EX_WRITE) != EX_NOERR) {
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to initialize conversion routines in file id %d", exoid);
ex_err_fn(exoid, __func__, errmsg, EX_LASTERR);
Expand Down
71 changes: 40 additions & 31 deletions packages/seacas/libraries/exodus/src/ex_utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -331,11 +331,33 @@ int exi_put_names(int exoid, int varid, size_t num_names, char *const *names,
if (exi_check_valid_file_id(exoid, __func__) == EX_FATAL) {
EX_FUNC_LEAVE(EX_FATAL);
}

/* inquire previously defined dimensions */
size_t name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_ALLOWED_NAME_LENGTH) + 1;
size_t name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_ALLOWED_NAME_LENGTH);

size_t max_name_len = 0;
for (size_t i = 0; i < num_names; i++) {
if (names != NULL && *names != NULL && *names[i] != '\0') {
size_t length = strlen(names[i]);
if (length > name_length) {
fprintf(stderr,
"Warning: The %s %s name '%s' is too long.\n\tIt will "
"be truncated from %d to %d characters. [Called from %s]\n",
ex_name_of_object(obj_type), subtype, names[i], (int)length, (int)name_length,
routine);
length = name_length;
}
if (length > max_name_len) {
max_name_len = length;
}
}
}
/* Update the maximum_name_length attribute on the file. */
exi_update_max_name_length(exoid, max_name_len);

max_name_len++;
char *int_names = NULL;
if (!(int_names = calloc(num_names * name_length, 1))) {
if (!(int_names = calloc(num_names * max_name_len, 1))) {
char errmsg[MAX_ERR_LENGTH];
snprintf(errmsg, MAX_ERR_LENGTH,
"ERROR: failed to allocate memory for internal int_names "
Expand All @@ -345,32 +367,24 @@ int exi_put_names(int exoid, int varid, size_t num_names, char *const *names,
EX_FUNC_LEAVE(EX_FATAL);
}

size_t idx = 0;
int max_name_len = 0;
int found_name = 0;
size_t idx = 0;
for (size_t i = 0; i < num_names; i++) {
if (names != NULL && *names != NULL && *names[i] != '\0') {
found_name = 1;
ex_copy_string(&int_names[idx], names[i], name_length);
size_t length = strlen(names[i]) + 1;
if (length > (size_t)name_length) {
fprintf(stderr,
"Warning: The %s %s name '%s' is too long.\n\tIt will "
"be truncated from %d to %d characters. [Called from %s]\n",
ex_name_of_object(obj_type), subtype, names[i], (int)length - 1,
(int)name_length - 1, routine);
length = name_length;
}

if (length > (size_t)max_name_len) {
max_name_len = length;
}
ex_copy_string(&int_names[idx], names[i], max_name_len);
}
idx += name_length;
idx += max_name_len;
}

int my_rank = exi_parallel_rank(exoid);
size_t start[2] = {0, 0};
size_t count[2] = {num_names, max_name_len};
if (my_rank != 0) {
// In parallel, only rank 0 writes...
count[0] = 0;
count[1] = 0;
}
int status;
if ((status = nc_put_var_text(exoid, varid, int_names)) != NC_NOERR) {
if ((status = nc_put_vara_text(exoid, varid, start, count, int_names)) != NC_NOERR) {
free(int_names);
char errmsg[MAX_ERR_LENGTH];
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to store %s names in file id %d",
Expand All @@ -379,11 +393,6 @@ int exi_put_names(int exoid, int varid, size_t num_names, char *const *names,
EX_FUNC_LEAVE(EX_FATAL);
}

if (found_name) {

/* Update the maximum_name_length attribute on the file. */
exi_update_max_name_length(exoid, max_name_len - 1);
}
free(int_names);

EX_FUNC_LEAVE(EX_NOERR);
Expand Down Expand Up @@ -2290,8 +2299,8 @@ int exi_handle_mode(unsigned int my_mode, int is_parallel, int run_version)
\internal
\undoc
*/
int exi_populate_header(int exoid, const char *path, int my_mode, int is_parallel, int *comp_ws,
int *io_ws)
int exi_populate_header(int exoid, const char *path, int my_mode, int my_rank, int is_parallel,
int *comp_ws, int *io_ws)
{
int status;
int old_fill;
Expand Down Expand Up @@ -2349,8 +2358,8 @@ int exi_populate_header(int exoid, const char *path, int my_mode, int is_paralle
is_hdf5 = true;
}

if (exi_conv_init(exoid, comp_ws, io_ws, 0, int64_status, is_parallel, is_hdf5, is_pnetcdf,
my_mode & EX_WRITE) != EX_NOERR) {
if (exi_conv_init(exoid, comp_ws, io_ws, 0, int64_status, my_rank, is_parallel, is_hdf5,
is_pnetcdf, my_mode & EX_WRITE) != EX_NOERR) {
snprintf(errmsg, MAX_ERR_LENGTH, "ERROR: failed to init conversion routines in file id %d",
exoid);
ex_err_fn(exoid, __func__, errmsg, EX_LASTERR);
Expand Down