libsc 2.8.5.210-64a7
The SC library provides support for parallel scientific applications.
|
Routines for parallel I/O with the scda format. More...
#include <sc_containers.h>
Go to the source code of this file.
Data Structures | |
struct | sc_scda_fopen_options |
An options struct for the functions sc_scda_fopen_write and sc_scda_fopen_read. More... | |
Macros | |
#define | SC_SCDA_HEADER_BYTES 128 |
number of file header bytes | |
#define | SC_SCDA_USER_STRING_BYTES 58 |
number of user string bytes | |
Typedefs | |
typedef struct sc_scda_fcontext | sc_scda_fcontext_t |
Opaque context used for writing a libsc data file. | |
typedef uint64_t | sc_scda_ulong |
Type for element counts and sizes. | |
typedef enum sc_scda_ferror | sc_scda_ferror_t |
Error values for scdafile functions. More... | |
typedef struct sc_scda_fopen_options | sc_scda_fopen_options_t |
An options struct for the functions sc_scda_fopen_write and sc_scda_fopen_read. More... | |
Enumerations | |
enum | sc_scda_ferror { SC_SCDA_FERR_SUCCESS = 0 , SC_SCDA_FERR_FILE = sc_MPI_ERR_LASTCODE , SC_SCDA_FERR_NOT_SAME , SC_SCDA_FERR_AMODE , SC_SCDA_FERR_NO_SUCH_FILE , SC_SCDA_FERR_FILE_EXIST , SC_SCDA_FERR_BAD_FILE , SC_SCDA_FERR_ACCESS , SC_SCDA_FERR_NO_SPACE , SC_SCDA_FERR_QUOTA , SC_SCDA_FERR_READ_ONLY , SC_SCDA_FERR_IN_USE , SC_SCDA_FERR_IO , SC_SCDA_FERR_UNKNOWN , SC_SCDA_FERR_FORMAT , SC_SCDA_FERR_USAGE , SC_SCDA_FERR_DECODE , SC_SCDA_FERR_INPUT , SC_SCDA_FERR_COUNT , SC_SCDA_FERR_LASTCODE } |
Error values for scdafile functions. More... | |
Functions | |
sc_scda_fcontext_t * | sc_scda_fopen_write (sc_MPI_Comm mpicomm, const char *filename, const char *user_string, size_t *len, sc_scda_fopen_options_t *opt, int *errcode) |
Open a file for writing and write the file header to the file. More... | |
sc_scda_fcontext_t * | sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *inline_data, int root, int *errcode) |
Write an inline data section. More... | |
sc_scda_fcontext_t * | sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *block_data, size_t block_size, int root, int encode, int *errcode) |
Write a fixed-size block file section. More... | |
sc_scda_fcontext_t * | sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *array_data, sc_array_t *elem_counts, size_t elem_size, int indirect, int encode, int *errcode) |
Write a fixed-size array file section. More... | |
int | sc_scda_proc_sizes (sc_array_t *elem_sizes, sc_array_t *elem_counts, sc_array_t *proc_sizes, int *errcode) |
This is a collective function to determine the processor sizes. More... | |
sc_scda_fcontext_t * | sc_scda_fwrite_varray (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *array_data, sc_array_t *elem_counts, sc_array_t *elem_sizes, sc_array_t *proc_sizes, int indirect, int encode, int *errcode) |
Write a variable-size array file section. More... | |
sc_scda_fcontext_t * | sc_scda_fopen_read (sc_MPI_Comm mpicomm, const char *filename, char *user_string, size_t *len, sc_scda_fopen_options_t *opt, int *errcode) |
Open a file for reading and read the file header from the file. More... | |
sc_scda_fcontext_t * | sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, size_t *len, char *type, size_t *elem_count, size_t *elem_size, int *decode, int *errcode) |
Read the next file section header. More... | |
sc_scda_fcontext_t * | sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, int *errcode) |
Read the data of an inline data section. More... | |
sc_scda_fcontext_t * | sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, size_t block_size, int root, int *errcode) |
Read the data of a block of given size. More... | |
sc_scda_fcontext_t * | sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, sc_array_t *elem_counts, size_t elem_size, int indirect, int *errcode) |
Read the data of a fixed-size array. More... | |
sc_scda_fcontext_t * | sc_scda_fread_varray_sizes (sc_scda_fcontext_t *fc, sc_array_t *elem_sizes, sc_array_t *elem_counts, int *errcode) |
Read the element sizes of a variable-size array. More... | |
sc_scda_fcontext_t * | sc_scda_fread_varray_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, sc_array_t *elem_counts, sc_array_t *elem_sizes, sc_array_t *proc_sizes, int indirect, int *errcode) |
Read the data of a variable-size array. More... | |
int | sc_scda_ferror_string (int errcode, char *str, int *len) |
Translate a sc_scda error code to an error string. More... | |
int | sc_scda_fclose (sc_scda_fcontext_t *fc, int *errcode) |
Close a file opened for parallel write/read and the free the file context. More... | |
Routines for parallel I/O with the scda format.
Functionality to write and read in parallel using a prescribed serial-equivalent file format called scda.
The scda format is as in this preprint.
However, in contrast to the preprint the API in this file provides the two functions sc_scda_fopen_write and sc_scda_fopen_read instead of providing one opening function with a mode parameter to decide and writing and reading.
In addition, we add in this file the options structure sc_scda_fopen_options as parameter for opening files.
The file format includes metadata in ASCII and therefore enables the human eye to parse the file structure using a standard text editor.
The file format scda is in particular suitable for parallel I/O and is accompanied by a convention for element-wise compression.
The format is designed such that the the parallel partition and in particular the process count can differ between writing and reading.
The main purpose of scda is to enable the user to implement parallel I/O for numerical appliations, e.g. simulation checkpoint/restart.
We elaborate further on the workflow in Parallel I/O workflow .
The functions
have user_string and len as an argument.
The user string consisting of the two parameters user_string and len is always a collective parameter.
In the case of writing these arguments have the purpose to pass the user string that is written to the file.
There are two options writing the user string to the file.
For both options it must be respected that the number of maximal user_string bytes is SC_SCDA_USER_STRING_BYTES + 1 including the nul-termination.
In case of reading user_string must be at least SC_SCDA_USER_STRING_BYTES + 1 bytes. On output len is set to the number of bytes actually written to user_string excluding the nul-termination.
If it is desired to write arbitrary data without the nul-termination, which is required for the user string, one can write a block data section using the function sc_scda_fwrite_block.
scda provides optional transparent, element-wise compression of data of file sections. The compression for writing can be enabled by passing true for encode to one of the functions
The compression on writing can be decided by the user for each file section separately. All parameters on the size of the data written to the file refer to the uncompressed data.
On reading it sufficies to pass decode true to the function sc_scda_fread_section_header. Then the file section data is decompressed if it was compressed and otherwise the file section data is read raw. Again all data sizes in reading functions refer to the uncompressed data.
If decode is false, the data is read raw even if it was written according to the compression convention.
typedef enum sc_scda_ferror sc_scda_ferror_t |
Error values for scdafile functions.
The error codes can be examined by sc_scda_ferror_string.
typedef struct sc_scda_fopen_options sc_scda_fopen_options_t |
An options struct for the functions sc_scda_fopen_write and sc_scda_fopen_read.
The struct may be extended in the future. type for sc_scda_fopen_options
enum sc_scda_ferror |
Error values for scdafile functions.
The error codes can be examined by sc_scda_ferror_string.
Enumerator | |
---|---|
SC_SCDA_FERR_SUCCESS | successful function call |
SC_SCDA_FERR_FILE | invalid file handle |
SC_SCDA_FERR_NOT_SAME | collective argument not identical |
SC_SCDA_FERR_AMODE | access mode error |
SC_SCDA_FERR_NO_SUCH_FILE | file does not exist |
SC_SCDA_FERR_FILE_EXIST | file exists already |
SC_SCDA_FERR_BAD_FILE | invalid file name |
SC_SCDA_FERR_ACCESS | permission denied |
SC_SCDA_FERR_NO_SPACE | not enough space |
SC_SCDA_FERR_QUOTA | quota exceeded |
SC_SCDA_FERR_READ_ONLY | read only file (system) |
SC_SCDA_FERR_IN_USE | file currently open by other process |
SC_SCDA_FERR_IO | other I/O error |
SC_SCDA_FERR_UNKNOWN | unknown I/O error |
SC_SCDA_FERR_FORMAT | File not conforming to the scda format. |
SC_SCDA_FERR_USAGE | Incorrect workflow of an scda reading function. For example, the user might have identified a certain file section type using sc_scda_fread_section_header but then calls a function to read the section data for a different type. Another example is to try reading the data of a 'V' section before reading its element sizes. This error also occurs when the user tries to read section data before reading the section header. |
SC_SCDA_FERR_DECODE | The decode parameter to sc_scda_fread_section_header is true but the file section header(s) encountered does not conform to the scda encoding convention. |
SC_SCDA_FERR_INPUT | An argument to a scda file function is invalid. This occurs for example when an essential pointer argument is NULL or a user string for writing is too long. |
SC_SCDA_FERR_COUNT | A byte count error that may occur transiently on writing or the file is short on reading. |
SC_SCDA_FERR_LASTCODE | to define own error codes for a higher level application that is using sc_scda functions |
int sc_scda_fclose | ( | sc_scda_fcontext_t * | fc, |
int * | errcode | ||
) |
Close a file opened for parallel write/read and the free the file context.
This is a collective function. Every call of sc_scda_fopen_write and sc_scda_fopen_read must be matched by a corresponding call of sc_scda_fclose on the created file context.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously created by sc_scda_fopen_write or sc_scda_fopen_read. This file context is freed after a call of this function. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
int sc_scda_ferror_string | ( | int | errcode, |
char * | str, | ||
int * | len | ||
) |
Translate a sc_scda error code to an error string.
This is a non-collective function.
[in] | errcode | An errcode that is output by a sc_scda function. |
[out] | str | At least sc_MPI_MAX_ERROR_STRING bytes. |
[out] | len | On output the length of string on return. |
sc_scda_fcontext_t * sc_scda_fopen_read | ( | sc_MPI_Comm | mpicomm, |
const char * | filename, | ||
char * | user_string, | ||
size_t * | len, | ||
sc_scda_fopen_options_t * | opt, | ||
int * | errcode | ||
) |
Open a file for reading and read the file header from the file.
The file must exist and be at least of the size of the file header, i.e. SC_SCDA_HEADER_BYTES bytes. If the file has a file header that does not satisfy the sc_scda file header format, the function reports the error using SC_LERRORF, collectively close the file and deallocate the file context. In this case the function returns NULL on all ranks. A wrong file header format causes SC_SCDA_FERR_FORMAT as errcode.
This function differs from the one opening function for writing and reading introduced in this scda preprint.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in] | mpicomm | The MPI communicator that is used to open the parallel file. |
[in] | filename | Path to parallel file that is to be created or to be opened. |
[out] | user_string | At least SC_SCDA_USER_STRING_BYTES + 1 bytes. user_string is filled with the read user string from file and is nul-terminated. |
[out] | len | On output len is set to the number of bytes written to user_string excluding the terminating nul. |
[in] | opt | An options structure that provides the possibility to pass further options. See sc_scda_fopen_options for more details. It is valid to pass NULL for opt. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fopen_write | ( | sc_MPI_Comm | mpicomm, |
const char * | filename, | ||
const char * | user_string, | ||
size_t * | len, | ||
sc_scda_fopen_options_t * | opt, | ||
int * | errcode | ||
) |
Open a file for writing and write the file header to the file.
This function creates a new file or overwrites an existing one. It is collective and creates the file on a parallel file system.
It is the user's responsibility to write any further metadata of the file that is required by the application. This can be done by writing file sections. However, the user can use sc_scda_fopen_read to open a not already opened file and then use sc_scda_fread_section_header and skipping the respective data bytes using the respective read functions sc_scda_fread_*_data to parse the structure of a given file and some metadata that is written by sc_scda.
This function differs from the one opening function for writing and reading introduced in this scda preprint.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in] | mpicomm | The MPI communicator that is used to open the parallel file. |
[in] | filename | Path to parallel file that is to be created or to be opened. |
[in] | user_string | At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information. |
[in] | len | The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string. |
[in] | opt | An options structure that provides the possibility to pass further options. See sc_scda_fopen_options for more details. It is valid to pass NULL for opt. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fread_array_data | ( | sc_scda_fcontext_t * | fc, |
sc_array_t * | array_data, | ||
sc_array_t * | elem_counts, | ||
size_t | elem_size, | ||
int | indirect, | ||
int * | errcode | ||
) |
Read the data of a fixed-size array.
This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header. This preceding call gives also the required elem_size and the global number of array elements. The user must pass a parallel partition of the array elements by elem_counts.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_read. |
[out] | array_data | If indirect is false, a sc_array with element count equals to the p-th entry of elem_counts for p being the calling rank. The element size must be equal to elem_size. If indirect is true, a sc_array with the same element count as for indirect false but with sizeof (sc_array_t) as element size. Each array element is then again a sc_array but with element count 1 and element size elem_size. The data can be skipped on each process by locally passing NULL. |
[in] | elem_counts | An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to read the array data in parallel. The sum of all array elements must be equal to elem_count as retrieved from sc_scda_fread_section_header. |
[in] | elem_size | The element size of one array element on number of bytes. Must be the same on all ranks and as retrieved from sc_scda_fread_section_header. |
[in] | indirect | A Boolean to determine whether array_data must be a sc_array of sc_arrays to read indirectly and in particular to potentially non-contigous memory. See the documentation of the parameter array_data for more information. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fread_block_data | ( | sc_scda_fcontext_t * | fc, |
sc_array_t * | block_data, | ||
size_t | block_size, | ||
int | root, | ||
int * | errcode | ||
) |
Read the data of a block of given size.
This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header. This preceding call gives also the required block_size.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_read. |
[out] | block_data | A sc_array with element count 1 and element size block_size. On output the sc_array is filled with the block data of the read block data section. |
[in] | block_size | The number of bytes of the block as retrieved from the preceding call of sc_scda_fread_section_header. |
[in] | root | An integer between 0 and mpisize exclusive of the MPI communicator that was used to create fc. root indicates the MPI rank on that block_data is read from the file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fread_inline_data | ( | sc_scda_fcontext_t * | fc, |
sc_array_t * | data, | ||
int | root, | ||
int * | errcode | ||
) |
Read the data of an inline data section.
This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_read. |
[out] | data | Exactly 32 bytes on the rank root or NULL on root to not read the bytes. The parameter is ignored on all ranks unequal to root. |
[in] | root | An integer between 0 and mpisize exclusive of the MPI communicator that was used to create fc. root indicates the MPI rank on that data is read from the file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fread_section_header | ( | sc_scda_fcontext_t * | fc, |
char * | user_string, | ||
size_t * | len, | ||
char * | type, | ||
size_t * | elem_count, | ||
size_t * | elem_size, | ||
int * | decode, | ||
int * | errcode | ||
) |
Read the next file section header.
This is a collective function. This functions reads the next file section header and provides the user information on the subsequent file section that can be used to read the actual data in a next calling depending on the file section type one (or for a variable-size array two) functions out of sc_scda_fread_block_data, sc_scda_fread_array_data, sc_scda_fread_varray_sizes and sc_scda_fread_varray_data.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_read. |
[out] | user_string | At least SC_SCDA_USER_STRING_BYTES +1 bytes. user_string is filled with the read user string from file and is nul-terminated. |
[out] | len | On output len is set to the number of bytes written to user_string excluding the terminating nul. |
[out] | type | On output this char is set to 'I' (inline data), 'B' (block of given size), 'A' (fixed-size array) or 'V' (variable-size array) depending on the file section type. |
[out] | elem_count | On output set to the global number of array elements if type equals 'A' or 'V'. For 'I' and 'B' as type, elem_count is set 0. |
[out] | elem_size | On output set to the byte count of the array elements if type is 'A' and for the type 'B' the number of bytes. Otherwise set to 0. |
[in,out] | decode | On input a Boolean to decide whether the file section shall possibly be interpreted as a compressed section, i.e. they were written by a sc_scda_fwrite_* function with encode set to true. For decode true as input the file section is interpreted as a compressed file section if the type and user string of the first raw file section satisfiy the compression convention. If the compression convention is not satisfied the data is read raw. For false as input the data will be read raw by the subsequent sc_scda_fread_* calls. The output value is always false if the input was set to false. Otherwise, the output is a Boolean that indicates if the nex file section contains a file section type and user string matching the compression convention. The subsequent sc_scda_fread_* calls do not require any adjustment dependent on decode. See also the 'Encoding' section in the detailed description in this file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fread_varray_data | ( | sc_scda_fcontext_t * | fc, |
sc_array_t * | array_data, | ||
sc_array_t * | elem_counts, | ||
sc_array_t * | elem_sizes, | ||
sc_array_t * | proc_sizes, | ||
int | indirect, | ||
int * | errcode | ||
) |
Read the data of a variable-size array.
This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_varray_sizes. This preceding call gives also the required elem_sizes.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_read. |
[out] | array_data | Let p be the calling rank. If indirect is false, array_data must have element count 1 and as element size the p-th entry of proc_sizes. On output the data of the array is set to the local array elements conforming with elem_counts, proc_sizes and elem_sizes. If indirect is true, array_data must be a sc_array with element count equal to the p-th array entry of elem_counts and element size equal to sizeof (sc_array_t). Each array element is again a sc_array. Now, with element count 1 and element size equals to the actual array element size as passed in elem_sizes. On output these arrays are filled with the actual elements of the read variable-size array. The data can be skipped on each process by locally passing NULL. |
[in] | elem_counts | An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to read the array data in parallel. The sum of all array elements must be equal to elem_count as retrieved from sc_scda_fread_section_header. |
[in] | elem_sizes | The local element sizes conforming to the array element partition elem_counts as retrieved from sc_scda_fread_varray_sizes. This parameter is ignored for ranks to which NULL for array_data was passed. |
[in] | proc_sizes | An sc_array that must be equal on all ranks. The element count and element size must be the same as for elem_counts. The array must contain the overall byte count per rank conforming with the passed array element partition elem_counts and the local array element sizes in elem_sizes. This parameter can be computed using sc_scda_proc_sizes. |
[in] | indirect | A Boolean to determine whether array_data must be a sc_array of sc_arrays to read indirectly and in particular to potentially non-contigous memory. See the documentation of the parameter array_data for more information. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fread_varray_sizes | ( | sc_scda_fcontext_t * | fc, |
sc_array_t * | elem_sizes, | ||
sc_array_t * | elem_counts, | ||
int * | errcode | ||
) |
Read the element sizes of a variable-size array.
This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header. This preceding call gives also the for elem_counts required global number of array elements. The user must pass a parallel partition of the array elements by elem_counts.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_read. |
[out] | elem_sizes | A sc_array with element count equals to p-th entry of elem_counts for p being the calling rank. The element size must be sizeof (sc_scda_ulong). On output the array is filled with the local array element byte counts, where locality is determined by elem_counts. The element sizes can be skipped on each process by locally passing NULL. |
[in] | elem_counts | An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to read the array data in parallel. The sum of all array elements must be equal to elem_count as retrieved from sc_scda_fread_section_header. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fwrite_array | ( | sc_scda_fcontext_t * | fc, |
const char * | user_string, | ||
size_t * | len, | ||
sc_array_t * | array_data, | ||
sc_array_t * | elem_counts, | ||
size_t | elem_size, | ||
int | indirect, | ||
int | encode, | ||
int * | errcode | ||
) |
Write a fixed-size array file section.
This is a collective function. The fixed-size array is the simplest file section that enables the user to write and read data in parallel. This function writes an array of a given element global count and a fixed element size.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_write. |
[in] | user_string | At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information. |
[in] | len | The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string. |
[in] | array_data | On rank p the p-th entry of elem_counts must be the element count of array_data. The element size of the sc_array must be equal to elem_size if indirect is false. Otherwise, array_data must be a sc_array of sc_arrays, i.e. a sc_array with element size sizeof (sc_array_t). Each of the elements of array_data is then a sc_array with element count 1 and element size elem_size. See also the documentation of the parameter indirect. |
[in] | elem_counts | An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts (sc_scda_ulong). That is why it induces the partition that is used to write the array data in parallel. |
[in] | elem_size | The element size of one array element on number of bytes. Must be the same on all ranks. |
[in] | indirect | A Boolean to determine whether array_data must be a sc_array of sc_arrays to write indirectly and in particular from potentially non-contigous memory. In the remaining case of indirect being false array_data must be a sc_array with element size equals to elem_size that contains the actual array elements. |
[in] | encode | A Boolean to decide whether the file section is written compressed. This results in two written file sections that can be read without the encoding interpretation by using sc_scda_fread_section_header with decode set to false followed by the usual sc_scda_fread functions. The data can be read as passed to this function by using decode true in sc_scda_fread_section_header and calling the usual sc_scda_fread function. See also the 'Encoding' section in the detailed description in this file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fwrite_block | ( | sc_scda_fcontext_t * | fc, |
const char * | user_string, | ||
size_t * | len, | ||
sc_array_t * | block_data, | ||
size_t | block_size, | ||
int | root, | ||
int | encode, | ||
int * | errcode | ||
) |
Write a fixed-size block file section.
This is a collective function. This function writes a data block of fixed size to the file. The block_data is written on the MPI rank root. The number of block bytes must be less or equal 10^{26} - 1.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_write. |
[in] | user_string | At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information. |
[in] | len | The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string. |
[in] | block_data | On rank root a sc_array with one element and element size equals to block_size. On all other ranks the parameter is ignored. |
[in] | block_size | The size of the data block in bytes. Must be less or equal than 10^{26} - 1. |
[in] | root | An integer between 0 and mpisize of the MPI communicator that was used to create fc. root indicates the MPI rank on that block_data is written to the file. |
[in] | encode | A Boolean to decide whether the file section is written compressed. This results in two written file sections that can be read without the encoding interpretation by using sc_scda_fread_section_header with decode set to false followed by the usual sc_scda_fread functions. The data can be read as passed to this function by using decode true in sc_scda_fread_section_header and calling the usual sc_scda_fread function. See also the 'Encoding' section in the detailed description in this file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fwrite_inline | ( | sc_scda_fcontext_t * | fc, |
const char * | user_string, | ||
size_t * | len, | ||
sc_array_t * | inline_data, | ||
int | root, | ||
int * | errcode | ||
) |
Write an inline data section.
This is a collective function. This function writes 32 bytes of user-defined data preceded by a file section header containing a user string. In contrast to other file sections the inline data section does not end with padded data bytes and therefore require exactly 32 bytes data from the user. This enables the user to implement custom file structuring or padding.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_write. |
[in] | user_string | At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information. |
[in] | len | The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string. |
[in] | inline_data | On the rank root a sc_array with element count 1 and element size 32. On all other ranks this parameter is ignored. |
[in] | root | An integer between 0 and mpisize exclusive of the MPI communicator that was used to create fc. root indicates the MPI rank on that inline_data is written to the file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
sc_scda_fcontext_t * sc_scda_fwrite_varray | ( | sc_scda_fcontext_t * | fc, |
const char * | user_string, | ||
size_t * | len, | ||
sc_array_t * | array_data, | ||
sc_array_t * | elem_counts, | ||
sc_array_t * | elem_sizes, | ||
sc_array_t * | proc_sizes, | ||
int | indirect, | ||
int | encode, | ||
int * | errcode | ||
) |
Write a variable-size array file section.
This is a collective function. This function can be used instead of sc_scda_fwrite_array if the array elements do not have a constant element size in bytes.
This function returns NULL on MPI I/O errors. Without MPI I/O the function may abort on file system dependent errors.
[in,out] | fc | File context previously opened by sc_scda_fopen_write. |
[in] | user_string | At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information. |
[in] | len | The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string. |
[in] | array_data | Let p be the calling rank. If indirect is false, array_data must have element count 1 and as element size the p-th entry of proc_sizes. The data of the array must be the local array elements conforming with elem_counts, proc_sizes and elem_sizes. If indirect is true, array_data must be a sc_array with element count equal to the p-th array entry of elem_counts and element size equal to sizeof (sc_array_t). Each array element is again a sc_array. Now, with element count 1 and element size equals to the actual array element size as passed in elem_sizes. |
[in] | elem_counts | An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (unint8_t). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to write the array data in parallel. |
[in] | elem_sizes | A sc_array with the element sizes for the local array elements. The sc_array has an element count of p-th entry of elem_counts for p being the calling rank. The element size is sizeof (sc_scda_ulong). |
[in] | proc_sizes | An sc_array that must be equal on all ranks. The element count and element size must be the same as for elem_counts. The array must contain the overall byte count per rank conforming with the passed array element partition elem_counts and the local array element sizes in elem_sizes. This parameter can be computed using sc_scda_proc_sizes. |
[in] | indirect | A Boolean to determine whether array_data must be a sc_array of sc_arrays to write indirectly and in particular from potentially non-contigous memory. In the remaining case of indirect being false array_data must be a sc_array with the actual array elements as data as further explained in the documentation of array_data. |
[in] | encode | A Boolean to decide whether the file section is written compressed. This results in two written file sections that can be read without the encoding interpretation by using sc_scda_fread_section_header with decode set to false followed by the usual sc_scda_fread functions. The data can be read as passed to this function by using decode true in sc_scda_fread_section_header and calling the usual sc_scda_fread function. See also the 'Encoding' section in the detailed description in this file. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |
int sc_scda_proc_sizes | ( | sc_array_t * | elem_sizes, |
sc_array_t * | elem_counts, | ||
sc_array_t * | proc_sizes, | ||
int * | errcode | ||
) |
This is a collective function to determine the processor sizes.
The purpose of this function is determine the proc_sizes argument as required by sc_scda_fread_varray_data given the data that can be retrieved by calling sc_scda_fread_varray_sizes or as required by sc_scda_fwrite_varray given elem_sizes and elem_counts as passed to sc_scda_fwrite_varray.
[in] | elem_sizes | The elem_sizes array as retrieved by sc_scda_fread_varray_sizes or passed to sc_scda_fwrite_varray. |
[in] | elem_counts | The elem_counts array as retrieved by sc_scda_fread_varray_sizes or passed to sc_scda_fwrite_varray. |
[out] | proc_sizes | A sc_array with element size sc_scda_ulong that is resized on output to the length of elem_sizes and elem_counts. The array is filled with the number bytes per process. |
[out] | errcode | An errcode that can be interpreted by sc_scda_ferror_string. |