ga_api.tex

\documentclass[12pt]{article}
\include{preamble/preamble}
\begin{document}


\apih{ABS VALUE}{Converts a global array to contain absolute values of its elements}

\begin{capi}
\begin{ccode}
void GA_Abs_value(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_abs_value(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::absValue() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.abs_value(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Take the element-wise absolute value of the array.
\end{desc}

\apih{ABS VALUE PATCH}{Converts a patch of a global array to have absolute values of its elements}

\begin{capi}
\begin{ccode}
void GA_Abs_value_patch(int g_a, int lo[], int hi[])
\end{ccode}
\begin{funcargs}
\inarg{}{g_a}{array handle}
\inarg{}{lo[], hi[]}{g_a patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_abs_value_patch(g_a, lo, hi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{lo(ndim), hi(ndim)}{g_a patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::absValuePatch(int *lo, int *hi) const
void GlobalArray::absValuePatch(int64_t *lo, int64_t *hi) const
\end{cxxcode}
\begin{funcargs}
\inarg{}{lo}{lower corner patch coordinates}
\inarg{}{hi}{upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.abs_value(int g_a, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Take the element-wise absolute value of the patch.
\end{desc}

\seealso{ABS VALUE}

\apih{ACC}{Accumulates data into a global array}

\begin{capi}
\begin{ccode}
void NGA_Acc(int g_a, int lo[], int hi[], void* buf, int ld[],
             void* alpha)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\inarg{void*}{buf}{pointer to the local buffer array}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{double/double complex/long*}{alpha}{scale factor}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_acc(g_a, ilo, ihi, jlo, jhi, buf, ld, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ilo, ihi, jlo, jhi}{}
\inarg{double precision/complex}{buf}{local buffer containing data}
\inarg{integer}{ld}{}
\inarg{double precision/complex}{alpha}{scale argument for accumulate}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_acc(g_a, lo, hi, buf, ld, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer containing data}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\inarg{type}{alpha}{scale argument for accumulate}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::acc(int lo[], int hi[], void *buf,
                      int ld[], void *alpha) const
void GlobalArray::acc(int64_t lo[], int64_t hi[], void *buf,
                      int64_t ld[], void *alpha) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\inarg{void*}{buf}{pointer to the local buffer array}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{void*}{alpha}{scale factor (double/double complex/long *)}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.acc(int g_a, buffer, lo=None, hi=None, alpha=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{object}{alpha}{scale factor, cast to the appropriate type}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Combines data from local array buffer with data in the global array section.
The local array is assumed to be have the same number of dimensions as the
global array.

global array section (lo[],hi[]) += *alpha * buffer

\end{desc}

\apih{ACCESS}{Accesses data locally allocated for a global array}

\begin{capi}
\begin{ccode}
void NGA_Access(int g_a, int lo[], int hi[], void **ptr, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\outarg{void**}{ptr}{points to location of first element in patch}
\outarg{int*}{ld[ndim-1]}{leading dimensions for the patch elements}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_access(g_a, ilo, ihi, jlo, jhi, index, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ilo, ihi, jlo, jhi}{}
\outarg{integer}{index}{}
\outarg{integer}{ld}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_access(g_a, lo, hi, index, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{lo(ndim),hi(ndim)}{patch specification}
\outarg{integer}{index}{reference to local data}
\outarg{integer}{ld(ndim-1)}{array of leading dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::access(int lo[], int hi[], void **ptr, int ld[]) const
void GlobalArray::access(int64_t lo[], int64_t hi[], void *ptr, int64_t ld[]) cons
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\outarg{void**}{ptr}{points to location of first element in patch}
\outarg{int*}{ld[ndim-1]}{leading dimensions for the patch elements}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.access(int g_a, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\local

\begin{cdesc}

Provides access to the specified patch of a global array. Returns array of
leading dimensions ld and a pointer to the first element in the patch.  This
routine allows to access directly, in place elements in the local section of a
global array. It useful for writing new GA operations. A call to GA_Access
normally follows a previous call to GA_Distribution that returns coordinates of
the patch associated with a processor. You need to make sure that the
coordinates of the patch are valid (test values returned from GA_Distribution).

Each call to GA_Access has to be followed by a call to either GA_Release or
GA_Release_update. You can access in this fashion only local data.  Since the
data is shared with other processes, you need to consider issues of mutual
exclusion.

\end{cdesc}

\begin{cxxdesc}

Provides access to the specified patch of a global array. Returns array of
leading dimensions ld and a pointer to the first element in the patch.  This
routine allows to access directly, in place elements in the local section of a
global array. It useful for writing new GA operations. A call to access
normally follows a previous call to distribution that returns coordinates of
the patch associated with a processor. You need to make sure that the
coordinates of the patch are valid (test values returned from distribution).

Each call to access has to be followed by a call to either release or
releaseUpdate. You can access in this fashion only local data.  Since the
data is shared with other processes, you need to consider issues of mutual
exclusion.

\end{cxxdesc}

\begin{pydesc}
Provides access to the specified patch of a global array. Returns array of
leading dimensions ld and a pointer to the first element in the patch.  This
routine allows to access directly, in place elements in the local section of a
global array. It useful for writing new GA operations. A call to access
normally follows a previous call to distribution that returns coordinates of
the patch associated with a processor. You need to make sure that the
coordinates of the patch are valid (test values returned from distribution).

Each call to access has to be followed by a call to either release or
release_update. You can access in this fashion only local data.  Since the
data is shared with other processes, you need to consider issues of mutual
exclusion.

Note: The entire local data is always accessed, but if a smaller patch is
requested, an appropriately sliced ndarray is returned.
\end{pydesc}

\begin{fdesc}

Provides access to the specified patch of array. Returns leading dimension ld
and and MA-like index for the data. This routine is intended for writing new GA
operations. A call to ga_access should normally follow a call to ga_distribution
that returns coordinates of the patch associated with a processor. You need to
make sure that the coordinates of the patch are valid (test values returned
from ga_distribution).

Your code should include a MA include file, mafdecls.h.
\begin{verbatim}
          dbl_mb(index)  - for double precision data
          int_mb(index)  - for integer data
          dcpl_mb(index) - for double complex data
\end{verbatim}

The addressing convention refers the first element of the patch.
However, you can only pass that reference to another subroutine where it could
be used like a normal array, see the following example.

Example

For a given subroutine:
\begin{verbatim}
          subroutine foo(A,  nrows, ncols lda)
          double precision A(lda,*)
          integer nrows, ncols
             ....
          end
\end{verbatim}
you can reference A(ilo:ihi,jlo:jhi) in the following way:

\begin{verbatim}
          call foo(dbl_mb(index), ihi-ilo+1, jhi-jlo+1, lda)
\end{verbatim}

Note that the size of the integer index must match the platform, 32-bit
platforms need to use 4-byte integers and 64-bit platforms need to use 8-byte
integers, regardless of whether the other integers in the program are compiled
as 4 or 8-byte integers. This can create portability problems when code is being
built on different platforms. Users can use the preprocessors symbol
GA_ACCESS_INDEX_TYPE to correctly size the integers used as indices across
different architectures. The GA_ACCESS_INDEX_TYPE is used instead of an integer
declaration and the the global.fh file needs to be included in the subroutine.
This will result in code that works for both 32 and 64-bit platforms without
having to explicity convert the integer size.

\end{fdesc}

\apih{ACCESS BLOCK}{Accesses a block in a block-cyclic distributed global array}

\begin{capi}
\begin{ccode}
void NGA_Access_block(int g_a, int idx, void **ptr, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int}{idx}{block index}
\outarg{void**}{ptr}{pointer to locally held block}
\outarg{int*}{ld[ndim-1]}{array of leading dimensions}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_access_block(g_a, idx, index, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{idx}{block index}
\outarg{integer}{index}{reference to local data}
\outarg{integer}{ld(ndim-1)}{array of leading dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::accessBlock(int idx, void **ptr, int ld[]) const
void GlobalArray::accessBlock(int64_t idx, void **ptr, int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{idx}{index of block}
\outarg{void**}{ptr}{points to location of first element in patch}
\outarg{int*}{ld[ndim-1]}{leading dimensions for the patch elements}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.access_block(int g_a, int idx)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{idx}{block index}
\outarg{ndarray}{ret}{array representing the block at index idx}
\end{funcargs}
\end{pyapi}

\local

\begin{cdesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a block-cyclic data distribution.  The
index idx is the index of the block in the array assuming that blocks are
numbered sequentially in a column-major order. A quick way of determining
whether a block with index idx is held locally on a processor is to calculate
whether idx\%nproc equals the processor ID, where nproc is the total number
of processors. Once the pointer has been returned, local data can be accessed as
described in the documentation for NGA_Access. Each call to NGA_Access_block
should be followed by a call to either NGA_Release_block or
NGA_Release_update_block.
\end{cdesc}

\begin{cxxdesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a block-cyclic data distribution.  The
index idx is the index of the block in the array assuming that blocks are
numbered sequentially in a column-major order. A quick way of determining
whether a block with index idx is held locally on a processor is to calculate
whether idx\%nproc equals the processor ID, where nproc is the total number
of processors. Once the pointer has been returned, local data can be accessed as
described in the documentation for access. Each call to accessBlock
should be followed by a call to either ReleaseBlock or
releaseUpdateBlock.
\end{cxxdesc}

\begin{pydesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a block-cyclic data distribution.  The
index idx is the index of the block in the array assuming that blocks are
numbered sequentially in a column-major order. A quick way of determining
whether a block with index idx is held locally on a processor is to calculate
whether idx\%nproc equals the processor ID, where nproc is the total number
of processors. Once the array has been returned, local data can be accessed as
described in the documentation for access. Each call to access_block
should be followed by a call to either release_block or
release_update_block.
\end{pydesc}

\begin{fdesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a block-cyclic data distribution.  The
index idx is the index of the block in the array assuming that blocks are
numbered sequentially in a column-major order. A quick way of determining
whether a block with index idx is held locally on a processor is to calculate
whether mod(idx,nproc) equals the processor ID, where nproc is the total number
of processors. Once the index has been returned, local data can be accessed as
described in the documentation for nga_access. Each call to nga_access_block
should be followed by a call to either nga_release_block or
nga_release_update_block.

Please check the documentation for the nga_access function for more information on
how to use the index returned by this subroutine to access locally held data.
\end{fdesc}

\seealso{ACCESS,RELEASE BLOCK,RELEASE UPDATE BLOCK}

\apih{ACCESS BLOCK GRID}{Accesses data block in a block-cyclic distributed global array}

\begin{capi}
\begin{ccode}
void NGA_Access_block_grid(int g_a, int subscript[], void **ptr, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{subscript[ndim]}{subscript of block in array}
\outarg{void**}{ptr}{pointer to locally held bloc}
\outarg{int*}{ld[ndim-1]}{array of leading dimensions}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_access_block_grid(g_a, subscript, index, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{subscript(ndim)}{subscript of block in array}
\outarg{integer}{index}{reference to local data}
\outarg{integer}{ld(ndim-1)}{array of leading dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::accessBlockGrid(int index[], void **ptr, int ld[]) const
void GlobalArray::accessBlockGrid(int64_t index[], void **ptr, int64_t ld[])
                                  const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{index[ndim]}{indices of block in processor grid}
\outarg{void**}{ptr}{points to location of first element in patch}
\outarg{int*}{ld[ndim-1]}{leading dimensions for the pacth elements}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.access_block_grid(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{subscript}{subscript of block in array}
\outarg{void**}{ret}{pointer to locally held bloc}
\end{funcargs}
\end{pyapi}

\local

\begin{cdesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a SCALAPACK block-cyclic data distribution
that is based on an underlying processor grid. The subscript array contains the
subscript of the block in the array of blocks. This subscript is based on the
location of the block in a grid, each of whose dimensions is equal to the
number of blocks that fit along that dimension. Once the index has been
returned, local data can be accessed as described in the documentation for
NGA_Access. Each call to NGA_Access_block_grid should be followed by a call to
either NGA_Release_block_grid or NGA_Release_update_block_grid.
\end{cdesc}

\begin{cxxdesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a SCALAPACK block-cyclic data distribution
that is based on an underlying processor grid. The subscript array contains the
subscript of the block in the array of blocks. This subscript is based on the
location of the block in a grid, each of whose dimensions is equal to the
number of blocks that fit along that dimension. Once the index has been
returned, local data can be accessed as described in the documentation for
access. Each call to accessBlockGrid should be followed by a call to
either releaseBlockGrid or releaseUpdateBlockGrid.
\end{cxxdesc}

\begin{pydesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a SCALAPACK block-cyclic data distribution
that is based on an underlying processor grid. The subscript array contains the
subscript of the block in the array of blocks. This subscript is based on the
location of the block in a grid, each of whose dimensions is equal to the
number of blocks that fit along that dimension. Once the index has been
returned, local data can be accessed as described in the documentation for
access. Each call to access_block_grid should be followed by a call to
either release_block_grid or release_update_block_grid.
\end{pydesc}

\begin{fdesc}
This function can be used to gain direct access to the data represented by a
single block in a global array with a SCALAPACK block-cyclic data distribution
that is based on an underlying processor grid. The subscript array contains the
subscript of the block in the array of blocks. This subscript is based on the
location of the block in a grid, each of whose dimensions is equal to the
number of blocks that fit along that dimension. Once the index has been
returned, local data can be accessed as described in the documentation for
nga_access. Each call to nga_access_block_grid should be followed by a call to
either nga_release_block_grid or nga_release_update_block_grid.
\end{fdesc}

\seealso{ACCESS,RELEASE BLOCK GRID,RELEASE UPDATE BLOCK GRID}

\apih{ACCESS BLOCK SEGMENT}{Accesses local data for a specific global array block}

\begin{capi}
\begin{ccode}
void NGA_Access_block_segment(int g_a, int proc, void **ptr, int *len)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{proc}{processor ID}
\outarg{void**}{ptr}{pointer to locally held data}
\outarg{int*}{len}{length of data on processor}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_access_block_segment(g_a, proc, index, len)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{proc}{processor ID}
\outarg{integer}{index}{reference to local data}
\outarg{integer}{len}{length of data on processor}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::accessBlockSegment(int index, void **ptr,
                                     int *len) const
void GlobalArray::accessBlockSegment(int index, void **ptr,
                                     int64_t *len) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{index}{processor ID}
\outarg{void**}{ptr}{points to location of first element}
\outarg{int*}{len}{length of locally held data}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.access_block(int g_a, int proc)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{proc}{processor ID}
\outarg{ndarray}{ret}{locally held data}
\end{funcargs}
\end{pyapi}

\local

\begin{cdesc}
This function can be used to gain access to the all the locally held data on a
particular processor that is associated with a block-cyclic distributed array.
Once the index has been returned, local data can be accessed as described in
the documentation for NGA_Access. The parameter len is the number of data
elements that are held locally. The data inside this segment has a lot of
additional structure so this function is not generally useful to developers. It
is primarily used inside the GA library to implement other GA routines. Each
call to NGA_Access_block_segment should be followed by a call to either
NGA_Release_block_segment or NGA_Release_update_block_segment.
\end{cdesc}

\begin{cxxdesc}
This function can be used to gain access to the all the locally held data on a
particular processor that is associated with a block-cyclic distributed array.
Once the index has been returned, local data can be accessed as described in
the documentation for access. The parameter len is the number of data
elements that are held locally. The data inside this segment has a lot of
additional structure so this function is not generally useful to developers. It
is primarily used inside the GA library to implement other GA routines. Each
call to accessBlockSegment should be followed by a call to either
releaseBlockSegment or releaseUpdateBlockSegment.
\end{cxxdesc}

\begin{pydesc}
This function can be used to gain access to the all the locally held data on a
particular processor that is associated with a block-cyclic distributed array.
Once the index has been returned, local data can be accessed as described in
the documentation for access. The parameter len is the number of data
elements that are held locally. The data inside this segment has a lot of
additional structure so this function is not generally useful to developers. It
is primarily used inside the GA library to implement other GA routines. Each
call to access_block_segment should be followed by a call to either
release_block_segment or release_update_block_segment.
\end{pydesc}

\begin{fdesc}
This function can be used to gain access to the all the locally held data on a
particular processor that is associated with a block-cyclic distributed array.
Once the index has been returned, local data can be accessed as described in
the documentation for nga_access. The parameter len is the number of data
elements that are held locally. The data inside this segment has a lot of
additional structure so this function is not generally useful to developers. It
is primarily used inside the GA library to implement other GA routines. Each
call to nga_access_block_segment should be followed by a call to either
nga_release_block_segment or nga_release_update_block_segment.
\end{fdesc}

\seealso{ACCESS,RELEASE BLOCK SEGMENT,RELEASE UPDATE BLOCK SEGMENT}

\apih{ACCESS GHOST ELEMENT}{Accesses a specific ghost element locally allocated on a GA}

\begin{capi}
\begin{ccode}
void NGA_Access_ghost_element(int g_a, void **ptr, int subscript[],
                              int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{void**}{ptr}{pointer to location of element indexed by subscript[]}
\inarg{int*}{subscript[ndim]}{array of integers that index desired element}
\outarg{int*}{ld[ndim-1]}{array of strides for local data patch.  These include ghost cell widths.}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_access_ghost_element(g_a, index, subscript, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{integer}{index}{index pointing to location of element indexed by subscript()}
\inarg{integer}{subscript(ndim)}{array of integers that index desired element}
\outarg{integer}{ld(ndim-1)}{array of strides for local data patch. These include ghost cell widths.}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::accessGhostElement(void **ptr, int subscript[],
                                     int ld[]) const
void GlobalArray::accessGhostElement(void **ptr, int64_t subscript[],
                                     int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\outarg{void**}{ptr}{index pointing to location of element indexed by subscript[]}
\inarg{int*}{subscript[ndim]}{array of integers that index desired element}
\outarg{int*}{ld[ndim-1]}{array of strides for local data patch.  These include ghost cell widths.}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.access_ghost_element(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{subscript}{index of desired element}
\outarg{ndarray}{ret}{appropriately shaped, possibly non-contiguous ndarray corresponding to global array patch held locally on the processor}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

This function can be used to return a pointer to any data element in the
locally held portion of the global array and can be used to directly access
ghost cell data. The array subscript refers to the local index of the element
relative to the origin of the local patch (which is assumed to be indexed by
(0,0,...)).

\end{desc}

\begin{fdesc}
To use the index returned by the nga_access_ghost_element subroutine, see the
documentation on nga_access.
\end{fdesc}

\seealso{ACCESS,RELEASE GHOST ELEMENT,RELEASE UPDATE GHOST ELEMENT}

\apih{ACCESS GHOSTS}{Accesses the ghost cells allocated locally on a GA}

\begin{capi}
\begin{ccode}
void NGA_Access_ghosts(int g_a, int dims[], void **ptr, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int*}{dims[ndim]}{array of dimensions of local patch, including ghost cells}
\outarg{void**}{ptr}{returns a pointer corresponding to the origin the global array patch held locally on the processor}
\outarg{int*}{ld[ndim-1]}{physical dimenstions of the local array patch, including ghost cells}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_access_ghosts(g_a, dims, index, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{integer}{dims(ndim)}{array of dimensions of local patch, including ghost cells}
\outarg{integer}{index}{returns an index corresponding to the origin the global array patch held locally on the processor}
\outarg{integer}{ld(ndim)}{physical dimenstions of the local array patch, including ghost cells}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::accessGhosts(int dims[], void **ptr, int ld[]) const
void GlobalArray::accessGhosts(int64_t dims[], void **ptr, int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\outarg{int*}{dims[ndim]}{array of dimensions of local patch, including ghost cells}
\outarg{void**}{ptr}{returns an index corresponding to the origin the global array patch held locally on the processor}
\outarg{int*}{ld[ndim-1]}{physical dimensions of the local array patch, including ghost cells}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.access_ghosts(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\outarg{ndarray}{ret}{appropriately shaped, possibly non-contiguous ndarray corresponding to global array patch held locally on the processor}
\end{funcargs}
\end{pyapi}

\local

\begin{cdesc}
Provides access to the local patch of the global array. Returns leading
dimension ld and and pointer for the data.  This routine will provide access to
the ghost cell data residing on each processor. Calls to NGA_Access_ghosts
should normally follow a call to NGA_Distribution that returns coordinates of
the visible data patch associated with a processor. You need to make sure that
the coordinates of the patch are valid (test values returned from
NGA_Distribution).

You can only access local data.
\end{cdesc}

\begin{cxxdesc}
Provides access to the local patch of the global array. Returns leading
dimension ld and and pointer for the data.  This routine will provide access to
the ghost cell data residing on each processor. Calls to accessGhosts
should normally follow a call to distribution that returns coordinates of
the visible data patch associated with a processor. You need to make sure that
the coordinates of the patch are valid (test values returned from
distribution).

You can only access local data.
\end{cxxdesc}

\begin{pydesc}
Provides access to the local patch of the global array. Returns leading
dimension ld and and pointer for the data.  This routine will provide access to
the ghost cell data residing on each processor. Calls to access_ghosts
should normally follow a call to distribution that returns coordinates of
the visible data patch associated with a processor. You need to make sure that
the coordinates of the patch are valid (test values returned from
distribution).

You can only access local data.
\end{pydesc}

\begin{fdesc}
Provides access to the local patch of the global array. Returns leading
dimension ld and and pointer for the data.  This routine will provide access to
the ghost cell data residing on each processor. Calls to nga_access_ghosts
should normally follow a call to nga_distribution that returns coordinates of
the visible data patch associated with a processor. You need to make sure that
the coordinates of the patch are valid (test values returned from
nga_distribution).

You can only access local data. To see how to use the index returned by this
subroutine, check the documentation on nga_acces.
\end{fdesc}

\seealso{ACCESS,RELEASE GHOSTS,RELEASE UPDATE GHOSTS}

\apih{ADD}{Adds corresponding values in two global arrays}

\begin{capi}
\begin{ccode}
void GA_Add(void *alpha, int g_a, void* beta, int g_b, int g_c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle}
\inarg{int}{g_c}{array handle}
\inarg{double/complex/int*}{alpha}{scale factor}
\inarg{double/complex/int*}{beta}{scale factor}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_add(alpha, g_a, beta, g_b, g_c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{g_b}{array handle}
\inarg{integer}{g_c}{array handle}
\inarg{double precision/complex/integer}{alpha,beta}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::add(void *alpha, const GlobalArray * g_a, void *beta, const GlobalArray * g_b) const
\end{cxxcode}
\begin{funcargs}
\inarg{void*}{alpha}{scale factor}
\inarg{int}{g_a}{array}
\inarg{void*}{beta}{scale factor}
\inarg{int}{g_b}{array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.add(int g_a, int g_b, int g_c, alpha=None, beta=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{object}{alpha}{multiplier (converted to appropriate type)}
\inarg{object}{beta}{multiplier (converted to appropriate type)}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

The arrays (which must be the same shape and identically aligned) are added
together element-wise.

\begin{verbatim}
        c = alpha * a  +  beta * b;
\end{verbatim}

The result (c) may replace one of the input arrays (a/b).

\end{desc}

\apih{ADD CONSTANT}{Adds a constant to all elements in a global array}

\begin{capi}
\begin{ccode}
void GA_Add_constant(int g_a, void *alpha)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{void*}{alpha}{double/complex/int/long/float* added value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_add_constant(g_a,  alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{double/complex/integer/float}{alpha}{added value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::addConstant(void* alpha) const
\end{cxxcode}
\begin{funcargs}
\inarg{void*}{alpha}{double/complex/int/long/float* constant to be added}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.add_constant(int g_a, alpha)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{alpha}{constant to add (converted to appropriate type)}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Add the constant pointed by alpha to each element of the array.
\end{desc}

\apih{ADD CONSTANT PATCH}{Adds a constant to all elements in a global array patch}

\begin{capi}
\begin{ccode}
void GA_Add_constant_patch(int g_a, int lo[], int hi[], void *alpha)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{lo[], hi[]}{patch coordinates}
\inarg{void*}{alpha}{double/complex/int/long/float* added value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_add_constant_patch(g_a, lo, hi, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{lo(ndim), hi(ndim)}{patch coordinates}
\inarg{double/complex/integer/float}{alpha}{added value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::addConstantPatch(int *lo, int *hi, void *alpha) const
void GlobalArray::addConstantPatch(int64_t *lo, int64_t *hi, void *alpha) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo}{lower corner patch coordinates}
\inarg{int*}{hi}{upper corner patch coordinates}
\inarg{void*}{alpha}{double/complex/int/long/float constant to be added}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.add_constant(int g_a, alpha, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{alpha}{constant to add (converted to appropriate type)}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Add the constant pointed by alpha to each element of the patch.
\end{desc}

\seealso{ADD CONSTANT}

\apih{ADD DIAGONAL}{Adds to the diagonal elements of a global array}

\begin{capi}
\begin{ccode}
void GA_Add_diagonal(int g_a, int g_v)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_add_diagonal(g_a, g_v)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_v}{array handles}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::addDiagonal(const GlobalArray * g_v) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_v}{global array containing diagonal elements to be added}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.add_diagonal(int g_a, int g_v)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Adds the elements of the vector g_v to the diagonal of this matrix g_a.
\end{desc}

\apih{ADD PATCH}{Adds patches of global arrays}

\begin{capi}
\begin{ccode}
void NGA_Add_patch(void *alpha, int g_a, int alo[], int ahi[],
                   void *beta,  int g_b, int blo[], int bhi[],
                   int g_c, int clo[], int chi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b, g_c}{array handles}
\inarg{int*}{alo[], ahi[]}{patch of g_a}
\inarg{int*}{blo[], bhi[]}{patch of g_b}
\inarg{int*}{clo[], chi[]}{patch of g_c}
\inarg{void*}{alpha, beta}{scale factors}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_add_patch(alpha, g_a, ailo, aihi, ajlo, ajhi,
                        beta,  g_b, bilo, bihi, bjlo, bjhi,
                               g_c, cilo, cihi, cjlo, cjhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b, g_c}{}
\inarg{double precision/complex/integer}{alpha, beta}{scale factors patches}
\inarg{integer}{ailo, aihi, ajlo, ajhi}{g_a patch coordinates}
\inarg{integer}{bilo, bihi, bjlo, bjhi}{g_b patch coordinates}
\inarg{integer}{cilo, cihi, cjlo, cjhi}{g_c patch coordinates}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_add_patch(alpha, g_a, alo, ahi, beta, g_b, blo, bhi
                         g_c, clo, chi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b, g_c}{}
\inarg{double precision/complex/integer}{alpha,beta}{scale factors for patches}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch coordinates}
\inarg{integer}{blo(ndim), bhi(ndim)}{g_b patch coordinates}
\inarg{integer}{clo(ndim), chi(ndim)}{g_c patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::addPatch(void *alpha, const GlobalArray * g_a, int alo[],
                           int ahi[],void *beta, const GlobalArray * g_b,
                           int blo[], int bhi[], int clo[], int chi[]) const
void GlobalArray::addPatch(void *alpha, const GlobalArray * g_a, int64_t alo[],
                           int64_t ahi[], void *beta, const GlobalArray * g_b,
                           int64_t blo[], int64_t bhi[], int64_t clo[],
                           int64_t chi[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{void*}{alpha}{scale factor}
\inarg{int}{g_a}{global array}
\inarg{int*}{alo}{patch of g_a}
\inarg{int*}{ahi}{patch of g_a}
\inarg{void*}{beta}{scale factor}
\inarg{int}{g_b}{global array}
\inarg{int*}{blo}{patch of g_b}
\inarg{int*}{bhi}{patch of g_b}
\inarg{int*}{clo}{patch of this GlobalArray}
\inarg{int*}{chi}{patch of this GlobalArray}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.add(int g_a, int g_b, int g_c, alpha=None, beta=None,
       alo=None, ahi=None, blo=None, bhi=None, clo=None, chi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{object}{alpha}{multiplier (converted to appropriate type)}
\inarg{object}{beta}{multiplier (converted to appropriate type)}
\inarg{1D array-like of ints}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of ints}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of ints}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of ints}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{1D array-like of ints}{clo}{lower bound patch coordinates of g_c, inclusive}
\inarg{1D array-like of ints}{chi}{higher bound patch coordinates of g_c, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Patches of arrays (which must have the same number of elements) are added
together element-wise.
\begin{verbatim}
         c[ ][ ] = alpha * a[ ][ ] + beta * b[ ][ ]
\end{verbatim}

\end{desc}

\seealso{ADD}

\apih{ALLOC GATSCAT BUF}{Allocates an internal buffer for gather/scatter operations}

\begin{capi}
\begin{ccode}
void NGA_Alloc_gatscat_buf(int nelems)
\end{ccode}
\begin{funcargs}
\inarg{int}{nelems}{maximum number of elements to scatter/gather}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
void nga_alloc_gatscat_buf(nelems)
\end{fcode}
\begin{funcargs}
\inarg{integer}{nelems}{maximum number of elements to scatter/gather}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::allocGatscatBuf(int nelems) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{nelems}{maximum number of elements to scatter/gather}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.alloc_gatscat_buf(int nelems)
\end{pycode}
\begin{funcargs}
\inarg{int}{nelems}{maximum number of elements to scatter/gather}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}
This function can be used to enhance the performance when the gather/scatter
operations are being called multiple times in succession. If the maximum number
of elements being called in any gather/scatter operation is known prior to
executing a code segment, then some internal buffers used in the gather/scatter
operations can be allocated beforehand instead of at every individual call. This
can result in substantial performance boosts in some cases. When the buffers are
no longer needed they can be removed using the corresponding free call.
\end{desc}

\seealso{FREE GATSCAT BUF}

\apih{ALLOCATE}{Allocates the array specified by a GA handle}

\begin{capi}
\begin{ccode}
int GA_Allocate(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int}{}{1 if allocation of g_a was successful}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_allocate(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\outarg{logical}{}{.TRUE. if allocation of g_a was successful}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::allocate() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.allocate(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{bool}{ret}{True if allocation of g_a was successful}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function allocates the memory for the global array handle originally
obtained using the GA_Create_handle function. At a minimum, the GA_Set_data
function must be called before the memory is allocated. Other GA_Set_xxx
functions can also be called before invoking this function.

Returns True if allocation of g_a was successful.

\end{desc}

\seealso{DEALLOCATE}

\apih{BRDCST}{Broadcasts elements among all processes}

\begin{capi}
\begin{ccode}
void GA_Brdcst(void *buf, int lenbuf, int root)
\end{ccode}
\begin{funcargs}
\inoutarg{void*}{buf[lenbuf]}{data}
\inarg{int}{lenbuf}{length of buffer (bytes)}
\inarg{int}{root}{root process}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_brdcst(type, buf, lenbuf, root)
\end{fcode}
\begin{funcargs}
\inarg{integer}{type}{}
\inoutarg{byte}{buf(lenbuf)}{buffer contain broadcasted/received data}
\inarg{integer}{lenbuf}{}
\inarg{integer}{root}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GAServices::brdcst(void *buf, int lenbuf, int root)
\end{cxxcode}
\begin{funcargs}
\inoutarg{void*}{buf[lenbuf]}{data}
\inarg{int}{lenbuf}{length of buffer}
\inarg{int}{root}{root process}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.brdcst(buffer, int root=0)
\end{pycode}
\begin{funcargs}
\inarg{1D array-like of objects}{buffer}{the ndarray message (converted to the appropriate type)}
\inarg{int}{root}{the process which is sending}
\end{funcargs}
\end{pyapi}

\wcoll

\begin{desc}
Broadcast from process root to all other processes a message of length lenbuf.

This is operation is provided only for convenience purposes: it is available
regardless of the message-passing library that GA is running.
\end{desc}

\begin{pydesc}
If the buffer is not contiguous, an error is raised. This operation is provided
only for convenience purposes: it is available regardless of the
message-passing library that GA is running with.

Returns: The buffer in case a temporary was passed in.
\end{pydesc}

\apih{CHECK HANDLE}{Checks whether a GA handle is valid}

\begin{capi}
\begin{ccode}
void GA_Check_handle(int g_a, char* string)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{char*}{string}{message string}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_check_handle(g_a, string)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{character(*)*}{string}{message string}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::checkHandle(char* string) const
\end{cxxcode}
\begin{funcargs}
\inarg{}{string}{message}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.check_handle(int g_a, char *message)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{char*}{string}{message string}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Check that the global array handle g_a is valid ... if not, call ga_error with
the string provided and some more info.

\end{desc}

\apih{CLUSTER NNODES}{Returns total number of cluster (shared memory) nodes}

\begin{capi}
\begin{ccode}
int GA_Cluster_nnodes()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_cluster_nnodes()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::clusterNnodes()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.cluster_nnodes()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

This functions returns the total number of nodes that the program is running
on. On SMP architectures, this will be less than or equal to the total number
of processors.

\end{desc}

\apih{CLUSTER NODEID}{Returns cluster node Rank of the invoking process}

\begin{capi}
\begin{ccode}
int GA_Cluster_nodeid()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_cluster_nodeid()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::clusterNodeid()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.cluster_nodeid()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

This function returns the node ID of the process. On SMP architectures with
more than one processor per node, several processes may return the same node
id.

\end{desc}

\apih{CLUSTER NPROCS}{Returns number of processes in a given cluster node}

\begin{capi}
\begin{ccode}
int GA_Cluster_nprocs(int inode)
\end{ccode}
\begin{funcargs}
\inarg{int}{inode}{node id}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_cluster_nprocs(inode)
\end{fcode}
\begin{funcargs}
\inarg{integer}{inode}{node id}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::clusterNprocs(int inode)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{inode}{node id}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.cluster_nprocs(int inode)
\end{pycode}
\begin{funcargs}
\inarg{int}{inode}{node id}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

This function returns the number of processors available on node inode.

\end{desc}

\apih{CLUSTER PROC NODEID}{Returns cluster node rank of a specified process}

\begin{capi}
\begin{ccode}
int GA_Cluster_proc_nodeid(int proc)
\end{ccode}
\begin{funcargs}
\inarg{int}{proc}{process id}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_cluster_proc_nodeid(proc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{proc}{process id}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::clusterProcNodeid(int iproc)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{iproc}{process id}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.cluster_nodeid(int proc=-1)
\end{pycode}
\begin{funcargs}
\inarg{int}{proc}{process id}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

This function returns the node ID of the specified process proc.  On SMP
architectures with more than one processor per node, several processes may
return the same node id.

\end{desc}

\apih{CLUSTER PROCID}{Returns rank of a process from a cluster node rank and intra-node rank}

\begin{capi}
\begin{ccode}
int GA_Cluster_procid(int inode, int iproc)
\end{ccode}
\begin{funcargs}
\inarg{int}{inode}{node id}
\inarg{int}{iproc}{processor id}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_cluster_procid(inode,iproc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{inode}{node id}
\inarg{integer}{iproc}{processor id}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::clusterProcid(int inode, int iproc)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{inode}{node id}
\inarg{int}{iproc}{processor id}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.cluster_procid(int inode, int iproc)
\end{pycode}
\begin{funcargs}
\inarg{int}{inode}{node id}
\inarg{int}{iproc}{processor id}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

This function returns the processor id associated with node inode and the local
processor ID iproc. If node inode has N processors, then the value of iproc
lies between 0 and N-1.

\end{desc}

\apih{COMPARE DISTR}{Compares distributions of two global arrays}

\begin{capi}
\begin{ccode}
int GA_Compare_distr(int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\outarg{int}{}{0 if distributions are identical}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_compare_distr(g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{logical}{}{.TRUE. if distributions are identical}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::compareDistr(const GlobalArray *g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{GlobalArray to compare against}
\outarg{int}{}{0 if distributions are identical}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.compare_distr(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle}
\outarg{bool}{ret}{True if distributions are identical}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Compares distributions of two global arrays. Returns 0 if distributions are
identical and 1 when they are not.

\end{desc}

\apih{COPY}{Copies a global array to another global array}

\begin{capi}
\begin{ccode}
void GA_Copy(int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_copy(g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{g_b}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::copy(const GlobalArray *g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{GlobalArray to copy}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.copy(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle copying from}
\inarg{int}{g_b}{the array handle copying to}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Copies elements in array represented by g_a into the array represented by g_b.
The arrays must be the same type, shape, and identically aligned.

For patch operations, the patches of arrays may be of different shapes but must
have the same number of elements. Patches must be nonoverlapping (if g_a=g_b).
Transposes are allowed for patch operations.

\end{desc}

\apih{COPY PATCH}{Copies a patch of a global array to another global array patch}

\begin{capi}
\begin{ccode}
void NGA_Copy_patch(char trans, int g_a, int alo[], int ahi[],
                    int g_b, int blo[], int bhi[])
\end{ccode}
\begin{funcargs}
\inarg{char}{trans}{transpose operator}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle}
\inarg{int*}{alo[], ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[], bhi[]}{g_b patch coordinates}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_copy_patch(trans, g_a, ailo, aihi, ajlo, ajhi,
                         g_b, bilo, bihi, bjlo, bjhi)
\end{fcode}
\begin{funcargs}
\inarg{character}{trans}{transpose operator}
\inarg{integer}{g_a, g_b}{}
\inarg{integer}{ailo, aihi, ajlo, ajhi}{g_a patch coordinates}
\inarg{integer}{bilo, bihi, bjlo, bjhi}{g_b patch coordinates}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_copy_patch(trans, g_a, alo, ahi, g_b, blo, bhi)
\end{fcode}
\begin{funcargs}
\inarg{character}{trans}{transpose operator}
\inarg{integer}{g_a, g_b}{}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch coordinates}
\inarg{integer}{blo(ndim), bhi(ndim)}{g_b patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::copyPatch(char trans, const GlobalArray* ga, int alo[],
                            int ahi[], int blo[], int bhi[]) const
void GlobalArray::copyPatch(char trans, const GlobalArray* ga, int64_talo[],
                            int64_t ahi[], int64_t blo[], int64_t bhi[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{char}{trans}{use transpose operator}
\inarg{int}{ga}{global array}
\inarg{int*}{alo}{ga patch coordinates}
\inarg{int*}{ahi}{ga patch coordinates}
\inarg{int*}{blo}{this GlobalArray's patch coordinates}
\inarg{int*}{bhi}{this GlobalArray's patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.copy(int g_a, int g_b, alo=None, ahi=None,
        blo=None, bhi=None, int trans=False)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle copying from}
\inarg{int}{g_b}{the array handle copying to}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{bool}{trans}{whether the transpose operator should be applied True=applied}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Copies elements in a patch of one array into another one. The patches of
arrays may be of different shapes but must have the same number of elements.
Patches must be non-overlapping (if g_a=g_b).
\begin{verbatim}
    trans = `N' or `n' means that the transpose operator should
             not be applied.
    trans = `T' or `t' means that transpose operator should be applied.
\end{verbatim}

\end{desc}

\seealso{COPY}

\apih{CREATE}{Creates a GA}

\begin{capi}
\begin{ccode}
int NGA_Create(int type, int ndim, int dims[], char *array_name, int chunk[])
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\outarg{int}{}{handle for future references}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
logical function ga_create(type, dim1, dim2, array_name, chunk1, chunk2, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{dim1,dim2}{array (dim1,dim2) as in FORTRAN}
\inarg{integer}{chunk1,chunk2}{minimum size that dimensions should be chunked up into}
\outarg{integer}{g_a}{handle for future references}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
logical function nga_create(type, ndim, dims, array_name, chunk, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{chunk(ndim)}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\outarg{integer}{g_a}{integer handle for future references}
\outarg{logical}{}{.TRUE. if array creation was successful}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(int type, int ndim, int dims[],
                         char *arrayname, int chunk[])
GlobalArray * GAServices::createGA(int type, int ndim, int dims[],
                                   char *arrayname, int chunk[])
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{data type(C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{char*}{arrayname}{a unique character string}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\outarg{GlobalArray}{}{instance}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create(int gtype, dims, char *name='', chunk=None)
\end{pycode}
\begin{funcargs}
\inarg{str}{name}{a unique character string}
\inarg{int}{type}{data type e.g. C_DBL, C_INT, C_DCPL, etc.}
\inarg{1D array-like of ints}{dims}{shape of array}
\inarg{1D array-like of ints}{chunk}{each element specifies minimum size that given dimensions should be chunked up into}
\outarg{int}{g_a}{global array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an ndim-dimensional array using the regular distribution model and
returns an integer handle representing the array.

The array can be distributed evenly or not. The control over the distribution
is accomplished by specifying chunk (block) size for all or some of array
dimensions. For example, for a 2-dimensional array, setting chunk[0]=dim[0]
gives distribution by vertical strips (chunk[0]*dims[0]); setting
chunk[1]=dim[1] gives distribution by horizontal strips (chunk[1]*dims[1]).
Actual chunks will be modified so that they are at least the size of the
minimum and each process has either zero or one chunk. Specifying chunk[i] as
less than 1 will cause that dimension to be distributed evenly.

As a convenience, when chunk is specified as NULL, the entire array is
distributed evenly.

Return value: a non-zero array handle means the call was succesful.

\end{desc}

\seealso{CREATE CONFIG,CREATE GHOSTS IRREG,CREATE GHOSTS,CREATE GHOSTS CONFIG,CREATE GHOSTS IRREG CONFIG,CREATE HANDLE,CREATE IRREG,CREATE IRREG CONFIG}

\apih{CREATE CONFIG}{Creates a GA for a specific processor group}

\begin{capi}
\begin{ccode}
int NGA_Create_config(int type, int ndim, int dims[], char *array_name,
                      int chunk[], int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{int}{p_handle}{processor list handle}
\outarg{int}{g_a}{global array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_create_config(type, ndim, dims, array_name, chunk,
                                   p_handle, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{chunk(ndim)}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{integer}{p_handle}{processor group handle}
\outarg{integer}{g_a}{integer handle for future references}
\outarg{logical}{}{.TRUE. if array creation successful}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(int type, int ndim, int dims[], char *arrayname,
                         int chunk[],PGroup* p_handle)
GlobalArray::GlobalArray(int type, int ndim, int64_t dims[], char *arrayname,
                         int64_t chunk[], PGroup* p_handle)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{data type(C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{char*}{arrayname}{a unique character string}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{PGroup*}{p_handle}{processor group handle}
\outarg{GlobalArray}{}{instance}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create(int gtype, dims, char *name='', chunk=None,
                int pgroup=-1)
\end{pycode}
\begin{funcargs}
\inarg{str}{name}{a unique character string}
\inarg{int}{type}{data type e.g. C_DBL, C_INT, C_DCPL, etc.}
\inarg{1D array-like of ints}{dims}{shape of array}
\inarg{1D array-like of ints}{chunk}{each element specifies minimum size that given dimensions should be chunked up into}
\inarg{int}{pgroup}{processor group handle}
\outarg{int}{g_a}{global array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an ndim-dimensional array using the regular distribution model but with
an explicitly specified processor group handle and returns an integer handle
representing the array.
\end{desc}

\begin{fdesc}
This call is essentially the same as the nga_create call, except for the
processor group handle p_handle. It can also be used to create mirrored arrays.
\end{fdesc}

\begin{cdesc}
This call is essentially the same as the NGA_Create call, except for the
processor group handle p_handle. It can also be used to create mirrored arrays.
\end{cdesc}

\begin{cxxdesc}
This call is essentially the same as the base GlobalArray constructor, except
for the processor group handle p_handle. It can also be used to create mirrored
arrays.
\end{cxxdesc}

\begin{pydesc}
This call is essentially the same as the create call, except for the
processor group handle p_handle. It can also be used to create mirrored arrays.
\end{pydesc}

\begin{desc}
Return value: a non-zero array handle means the call was succesful.

\end{desc}

\seealso{CREATE,SET PGROUP}

\apih{CREATE GHOSTS}{Creates a GA with ghost cells}

\begin{capi}
\begin{ccode}
int NGA_Create_ghosts(int type, int ndim, int dims[], int width[],
                      char *array_name, int chunk[])
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\outarg{int}{}{global array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_create_ghosts(type, ndim, dims, width, array_name,
                                   chunk, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{width(ndim)}{array of ghost cell widths}
\inarg{integer}{chunk(ndim)}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\outarg{integer}{g_a}{integer handle for future references}
\outarg{logical}{}{.TRUE. if array creation succesful}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(int type, int ndim, int dims[], int width[],
                         char *arrayname, int chunk[], char ghosts)
GlobalArray::GlobalArray(int type, int ndim, int64_t dims[], int64_t width[],
                         char *arrayname, int64_t chunk[], char ghosts)
GlobalArray * GAServices::createGA_Ghosts(int type, int ndim, int dims[],
                                          int width[], char *array_name,
                                          int chunk[])
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{char*}{array_name}{a unique character string}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{char}{ghosts}{this is a dummy parameter: added to increase the number of arguments, in order to avoid the conflicts among constructors. (ghosts = 'g' or 'G')}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create_ghosts(int gtype, dims, width, char *name='',
                       chunk=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{gtype}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{1D array-like of ints}{dims}{array of dimensions}
\inarg{1D array-like of ints}{width}{array of ghost cell widths}
\inarg{char*}{name}{a unique character string}
\inarg{1D array-like of ints}{chunk}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\outarg{int}{}{global array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an ndim-dimensional array with a layer of ghost cells around the
visible data on each processor using the regular distribution model and returns
an integer handle representing the array.

The array can be distributed evenly or not evenly. The control over the
distribution is accomplished by specifying chunk (block) size for all or some
of the array dimensions. For example, for a 2-dimensional array, setting
chunk(1)=dim(1) gives distribution by vertical strips (chunk(1)*dims(1));
setting chunk(2)=dim(2) gives distribution by horizontal strips
(chunk(2)*dims(2)). Actual chunks will be modified so that they are at least
the size of the minimum and each process has either zero or one chunk.
Specifying chunk(i) as $< 1$ will cause that dimension (i-th) to be distributed
evenly. The width of the ghost cell layer in each dimension is specified using
the array width(). The local data of the global array residing on each
processor will have a layer width[n] ghosts cells wide on either side of the
visible data along the dimension n.

Return value: a non-zero array handle means the call was successful.

\end{desc}

\seealso{CREATE,SET GHOSTS}

\apih{CREATE GHOSTS CONFIG}{Creates a GA with ghost cells and specific processor group}

\begin{capi}
\begin{ccode}
int NGA_Create_ghosts_config(int type, int ndim, int dims[],
                             int width[], char *array_name, int chunk[],
                             int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{int*}{p_handle}{processor list handle}
\outarg{int}{}{global array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_create_ghosts_config(type, ndim, dims, width, array_name,
                                          chunk, p_handle, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{width(ndim)}{array of ghost cell widths}
\inarg{integer}{chunk(ndim)}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{integer}{p_handle}{processor group handle}
\outarg{integer}{g_a}{integer handle for future references}
\outarg{logical}{}{.TRUE. if array creation successful}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(int type, int ndim, int dims[], int width[],
                         char *arrayname, int chunk[], PGroup* p_handle,
                         char ghosts)
GlobalArray::GlobalArray(int type, int ndim, int64_t dims[], int64_t width[],
                         char *arrayname, int64_t chunk[], PGroup* p_handle,
                         char ghosts)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{char*}{array_name}{a unique character string}
\inarg{int*}{chunk[ndim]}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{int}{p_handle}{processor group handle}
\inarg{char}{ghosts}{this is a dummy parameter: added to increase the number of arguments, inorder to avoid the conflicts among constructors. (ghosts = 'g' or 'G')}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create_ghosts(int gtype, dims, width, char *name='',
                       chunk=None, int pgroup=-1)
\end{pycode}
\begin{funcargs}
\inarg{int}{gtype}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{1D array-like of ints}{dims}{array of dimensions}
\inarg{1D array-like of ints}{width}{array of ghost cell widths}
\inarg{char*}{name}{a unique character string}
\inarg{1D array-like of ints}{chunk}{array of chunks, each element specifies minimum size that given dimensions should be chunked up into}
\inarg{int}{pgroup}{processor group handle}
\outarg{int}{}{global array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an ndim-dimensional array with a layer of ghost cells around the
visible data on each processor using the regular distribution model and an
explicitly specified processor list and returns an integer handle representing
the array.

This call is essentially the same as the NGA_Create_ghosts call, except for the
processor list handle p_handle. It can be used to create mirrored arrays.

Return value: a non-zero array handle means the call was successful.

\end{desc}

\seealso{CREATE,SET GHOSTS,SET PGROUP}

\apih{CREATE GHOSTS IRREG}{Creates an irregular-distributed GA with ghost cells}

\begin{capi}
\begin{ccode}
int NGA_Create_ghosts_irreg(int type, int ndim, int dims[], width[],
                           char *array_name, nblock[], map[])
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{int*}{nblock[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{map[s]}{starting index for for each block; the size     s is a sum of all elements of nblock array}
\outarg{int}{}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_create_ghosts_irreg(type, ndim, dims, width, array_name, map, nblock, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{width(ndim)}{array of ghost cell widths}
\inarg{integer}{nblock(ndim)}{no. of blocks each dimension is divided into}
\inarg{integer}{map(s)}{starting index for for each block; the size s is a sum of all elements of nblock array}
\outarg{integer}{g_a}{integer handle for future references}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(int type, int ndim, int dims[], int width[],
                         char *arrayname,
                         int block[], int maps[], char ghosts);
GlobalArray::GlobalArray(int type, int ndim, int64_t dims[],
                         int64_t width[], char *arrayname,
                         int64_t block[], int64_t maps[], char ghosts)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{char*}{arrayname}{a unique character string}
\inarg{int*}{block[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{maps[s]}{starting index for for each block; the size s is a sum of all elements of nblock array}
\inarg{char}{ghosts}{this is a dummy parameter: added to increase the number of arguments, inorder to avoid the conflicts among constructors. (ghosts = 'g' or 'G')}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create_ghosts_irreg(int gtype, dims, width, block, map,
                             char *name='')
\end{pycode}
\begin{funcargs}
\inarg{int}{gtype}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{1D array-like of ints}{dims}{array shape}
\inarg{1D array-like of ints}{width}{ghost cell widths per dimension}
\inarg{1D array-like of ints}{block}{no. of blocks each dimension is divided into}
\inarg{1D array-like of ints}{map}{starting index for for each block; len(map) is a sum all elements of block array}
\inarg{char*}{name}{a unique character string}
\outarg{int}{}{array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an array with ghost cells by following the user-specified distribution
and returns an integer handle representing the array.

The distribution is specified as a Cartesian product of distributions for each
dimension.

Figure \ref{crghostir} demonstrates distribution of a 2-dimensional array 8x10
on 6 (or more) processors.
\end{desc}

\begin{fdesc}
nblock(2)=\{3,2\}, the size of map array is s=5 and the array map contains the
following elements map=\{1,3,7, 1, 6\}. The distribution is nonuniform because,
P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only 10
elements each.
\end{fdesc}

\begin{cdesc}
nblock[2]=\{3,2\}, the size of map array is s=5 and the array map contains the
following elements map=\{0,2,6, 0, 6\}. The distribution is nonuniform because,
P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only 10
elements each.
\end{cdesc}

\begin{cxxdesc}
nblock[2]=\{3,2\}, the size of map array is s=5 and the array map contains the
following elements map=\{0,2,6, 0, 6\}. The distribution is nonuniform because,
P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only 10
elements each.
\end{cxxdesc}

\begin{pydesc}
nblock(2)=\{3,2\}, the size of map array is s=5 and the array map contains the
following elements map=\{1,3,7, 1, 6\}. The distribution is nonuniform because,
P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only 10
elements each.
\end{pydesc}

\begin{desc}
The array width is used to control the width of the ghost cell boundary
around the visible data on each processor. The local data of the Global Array
residing on each processor will have a layer width[n] ghosts cells wide on
either side of the visible data along the dimension n.

Return value: a non-zero array handle means the call was succesful.

\begin{figure}
\includegraphics{create-ghosts-irreg}
\centering
\caption{Creating an Array with Ghost Cells}
\label{crghostir}
\end{figure}

\end{desc}

\seealso{CREATE,SET GHOSTS,SET IRREG DISTR}

\apih{CREATE GHOSTS IRREG CONFIG}{Creates an irregular-distributed GA with ghost cells and a specific processor group}

\begin{capi}
\begin{ccode}
int NGA_Create_ghosts_irreg_config(int type, int ndim, int dims[],
                                  width [], char*array_name,nblock[],
                                  map[], int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{int*}{nblock[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{map[s]}{starting index for for each block; the size s is a sum of all elements of nblock array}
\inarg{int}{p_handle}{processor list handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_create_ghosts_irreg_config(type, ndim,
                                                dims, width, array_name,
                                                map, nblock,
                                                p_handle, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{width(ndim)}{array of ghost cell widths}
\inarg{integer}{nblock(ndim)}{no. of blocks each dimension is divided into}
\inarg{integer}{map(s)}{starting index for for each block; the size s is a sum of all elements of nblock array}
\inarg{integer}{p_handle}{processor group handle}
\outarg{integer}{g_a}{integer handle for future references}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(int type, int ndim, int dims[], int width[],
                         char *arrayname, int block[], int maps[],
                         PGroup* p_handle, char ghosts)
GlobalArray::GlobalArray(int type, int ndim, int64_t dims[],
                         int64_t width[], char *arrayname,
                         int64_t block[], int64_t maps[],
                         PGroup* p_handle,char ghosts)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims[ndim]}{array of dimensions}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\inarg{char*}{arrayname}{a unique character string}
\inarg{int*}{block[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{maps[s]}{starting index for for each block; the size s is a sum of all elements of nblock array}
\inarg{PGroup*}{p_handle}{processor group handle}
\inarg{char}{ghosts}{this is a dummy parameter: added to increase the number of arguments, inorder to avoid the conflicts among constructors. (ghosts = 'g' or 'G')}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create_ghosts_irreg(int gtype, dims, width, block, map,
                             char *name='', int pgroup=-1)
\end{pycode}
\begin{funcargs}
\inarg{int}{gtype}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{1D array-like of ints}{dims}{array shape}
\inarg{1D array-like of ints}{width}{ghost cell widths per dimension}
\inarg{1D array-like of ints}{block}{no. of blocks each dimension is divided into}
\inarg{1D array-like of ints}{map}{starting index for for each block; len(map) is a sum all elements of block array}
\inarg{char*}{name}{a unique character string}
\inarg{int}{pgroup}{processor group handle}
\outarg{int}{}{array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an array with ghost cells by following the user-specified distribution
and returns an integer handle representing the array. The user can specify that
the array is created on a particular processor group.
\end{desc}

\begin{fdesc}
This call is similar to the nga_create_ghosts_irreg call.
\end{fdesc}

\begin{cdesc}
This call is similar to the NGA_Create_ghosts_irreg call.
\end{cdesc}

\begin{cxxdesc}
This call is similar to the constructor for creating irregular distributions
with ghost cells.
\end{cxxdesc}

\begin{pydesc}
This call is similar to the create_ghosts_irreg call.
\end{pydesc}

\begin{desc}
Return value: a non-zero array handle means the call was succesful.

\end{desc}

\seealso{CREATE,SET GHOSTS,SET IRREG DISTR,SET PGROUP}

\apih{CREATE HANDLE}{Creates an inactive handle to a global array}

\begin{capi}
\begin{ccode}
int GA_Create_handle()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_create_handle()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.create_handle()
\end{pycode}
\end{pyapi}

\dcoll

\begin{desc}

This function returns a Global Array handle that can then be used to create a
new Global Array. This is part of a new API for creating Global Arrays that is
designed to replace the old interface built around the NGA_Create_xxx calls.
The sequence of operations is to begin with a call to GA_Greate_handle to get a
new array handle. The attributes of the array, such as dimension, size, type,
etc., can then be set using successive calls to the GA_Set_xxx subroutines.
When all array attributes have been set, the GA_Allocate subroutine is called
and the Global Array is actually created and memory for it is allocated.

\end{desc}

\seealso{ALLOCATE,SET ARRAY NAME,SET BLOCK CYCLIC,SET BLOCK CYCLIC PROC GRID,SET CHUNK,SET DATA,SET GHOSTS,SET IRREG DISTR,SET PGROUP,SET RESTRICTED,SET RESTRICTED RANGE}

\apih{CREATE IRREG}{Creates an irregular-distributed GA}

\begin{capi}
\begin{ccode}
int NGA_Create_irreg(int type, int ndim, int dims[], char *array_name,
                     int block[], int map[])
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{MA data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims}{array of dimension values}
\inarg{int*}{nblock[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{map[s]}{starting index for for each block; the size s is a sum all elements of nblock array}
\outarg{int}{}{array handle}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
logical function ga_create_irreg(type, dim1, dim2, array_name, map1,
                                 nblock1, map2, nblock2, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{dim1,dim2}{array (dim1,dim2) as in FORTRAN}
\inarg{integer}{nblock1}{no. of blocks first dimension is divided into}
\inarg{integer}{nblock2}{no. of blocks second dimension is divided into}
\inarg{integer}{map1(*)}{ilo for each block}
\inarg{integer}{map2(*)}{jlo for each block}
\outarg{integer}{g_a}{integer handle for future references}
\outarg{logical}{}{.TRUE. if array creation successful}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
logical function nga_create_irreg(type, ndim, dims, array_name, map,
                                  nblock, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{nblock(ndim)}{no. of blocks each dimension is divided into}
\inarg{integer}{map(s)}{starting index for for each block; the size s is a sum of all elements of nblock array}
\outarg{integer}{g_a}{integer handle for future references}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray * GAServices::createGA(int type, int ndim, int dims[],
                                   char *arrayname,
                                   int block[], int maps[])
GlobalArray::GlobalArray(int type, int ndim, int dims[], char *arrayname,
                         int block[],int maps[]);
GlobalArray::GlobalArray(int type, int ndim, int64_t dims[],
                         char *arrayname, int64_t block[],
                         int64_t maps[])
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{MA data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims}{array of dimension values}
\inarg{char*}{arrayname}{a unique character string}
\inarg{int*}{block[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{maps[s]}{starting index for for each block; the size s is a sum all elements of nblock array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create_irreg(int gtype, dims, block, map, char *name='')
\end{pycode}
\begin{funcargs}
\inarg{int}{gtype}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{1D array-like of ints}{dims}{array of dimension values}
\inarg{1D array-like of ints}{block}{no. of blocks each dimension is divided into}
\inarg{1D array-like of ints}{map}{starting index for for each block; len(map) is a sum all elements of block array}
\inarg{char*}{name}{a unique character string}
\outarg{int}{}{array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an array by following the user-specified distribution and returns an
integer handle representing the array.

The distribution is specified as a Cartesian product of distributions for each
dimension. The array indices start at 0.  For example, Figure \ref{crirreg}
demonstrates the distribution of a 2-dimensional 8x10 array on 6 (or more)
processors.
\end{desc}

\begin{fdesc}
nblock(2)=\{3,2\}, the size of the map array is s=5 and the array map contains
the following elements map=\{1,3,7,1,6\}. The distribution is nonuniform
because P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only
10 elements each.
\end{fdesc}

\begin{cdesc}
nblock[2]=\{3,2\}, the size of the map array is s=5 and the array map contains
the following elements map=\{0,5,0,2,6\}. The distribution is nonuniform
because P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only
10 elements each.
\end{cdesc}

\begin{cxxdesc}
nblock[2]=\{3,2\}, the size of the map array is s=5 and the array map contains
the following elements map=\{0,5,0,2,6\}. The distribution is nonuniform
because P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only
10 elements each.
\end{cxxdesc}

\begin{pydesc}
nblock(2)=\{3,2\}, the size of the map array is s=5 and the array map contains
the following elements map=\{1,3,7, 1, 6\}. The distribution is nonuniform
because P1 and P4 get 20 elements each and processors P0, P2, P3, and P5 only
10 elements each.
\end{pydesc}

\begin{desc}
\begin{figure}
\includegraphics{create-irreg}
\centering
\caption{Creating an Irregular Array}
\label{crirreg}
\end{figure}

Return value: a non-zero array handle means the call was succesful.

\end{desc}

\seealso{CREATE,SET IRREG DISTR}

\apih{CREATE IRREG CONFIG}{Creates an irregular-distributed GA with a specific processor group}

\begin{capi}
\begin{ccode}
int NGA_Create_irreg_config(int type, int ndim, int dims[],
                            char *array_name, int block[], int map[],
                            int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a unique character string}
\inarg{int}{type}{MA data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims}{array of dimension values}
\inarg{int*}{nblock[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{map[s]}{starting index for for each block; the size s is a sum all elements of nblock array}
\inarg{int}{p_handle}{processor list handle}
\outarg{int}{}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_create_irreg_config(type, ndim, dims, array_name,
                                         map, nblock, p_handle, g_a)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a unique character string}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,MT_DCPL,etc.)}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{dims(ndim)}{array of dimensions}
\inarg{integer}{nblock(ndim)}{no. of blocks each dimension is divided into}
\inarg{integer}{map(s)}{starting index for for each block; the size s is a sum of all elements of nblock array}
\inarg{integer}{p_handle}{processor group handle}
\outarg{integer}{g_a}{integer handle for future references}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::  GlobalArray(int type, int ndim, int dims[],
                           char *arrayname, int block[],
                           int maps[], PGroup* p_handle)
GlobalArray::  GlobalArray(int type, int ndim, int64_t dims[],
                           char *arrayname,
                           int64_t block[], int64_t maps[],
                           PGroup* p_handle)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{type}{MA data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int*}{dims}{array of dimension values}
\inarg{char*}{arrayname}{a unique character string}
\inarg{int*}{block[ndim]}{no. of blocks each dimension is divided into}
\inarg{int*}{maps[s]}{starting index for for each block; the size s is a sum all elements of nblock array}
\inarg{}{p_handle}{processor group handle}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
g_a = ga.create_irreg(int gtype, dims, block, map, char *name='',
                      int pgroup=-1)
\end{pycode}
\begin{funcargs}
\inarg{int}{gtype}{data type (C_DBL,C_INT,C_DCPL,etc.)}
\inarg{1D array-like of ints}{dims}{array of dimension values}
\inarg{1D array-like of ints}{block}{no. of blocks each dimension is divided into}
\inarg{1D array-like of ints}{map}{starting index for for each block; len(map) is a sum all elements of block array}
\inarg{char*}{name}{a unique character string}
\inarg{int}{pgroup}{processor group handle}
\outarg{int}{}{array handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

Creates an array by following the user-specified distribution and an explicitly
specified processor group handle and returns an integer handle representing the
array.
\end{desc}

\begin{fdesc}
This call is essentially the same as the nga_create_irreg call, except for the
processor group handle p_handle. It can also be used to create arrays on mirrored arrays.
\end{fdesc}

\begin{cdesc}
This call is essentially the same as the NGA_Create_irreg call, except for the
processor group handle p_handle. It can also be used to create mirrored arrays.
\end{cdesc}

\begin{cxxdesc}
This call is essentially the same as GlobalArray constructors, except for the
processor group handle p_handle. It can also be used to create mirrored arrays.
\end{cxxdesc}

\begin{pydesc}
This call is essentially the same as the create_irreg call, except for the
processor group handle p_handle. It can also be used to create arrays on mirrored arrays.
\end{pydesc}

\begin{desc}
Return value: a non-zero array handle means the call was succesful.

\end{desc}

\seealso{CREATE,SET IRREG DISTR,SET PGROUP}

\apih{CREATE MUTEXES}{Creates mutexes}

\begin{capi}
\begin{ccode}
int GA_Create_mutexes(int number)
\end{ccode}
\begin{funcargs}
\inarg{int}{number}{number of mutexes in mutex array}
\outarg{int}{}{1 if successful}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_create_mutexes(number)
\end{fcode}
\begin{funcargs}
\inarg{integer}{number}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::createMutexes(int number)
\end{cxxcode}
\begin{funcargs}
\inarg{}{number}{of mutexes in mutex array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.create_mutexes(int number)
\end{pycode}
\begin{funcargs}
\inarg{int}{number}{number of mutexes in mutex array}
\outarg{bool}{ret}{True if successful}
\end{funcargs}
\end{pyapi}

\wcoll

\begin{desc}

Creates a set containing the number of mutexes. Returns 0 if the operation
succeeded or 1 if it has failed. Mutex is a simple synchronization object used
to protect Critical Sections. Only one set of mutexes can exist at a time. An
array of mutexes can be created and destroyed as many times as needed.

Mutexes are numbered: 0, ..., number-1.

Returns: True on success, False on failure

\end{desc}

\seealso{DESTROY MUTEXES}

\apih{DEALLOCATE}{Deallocates memory for a global array without removing the array}

\begin{capi}
\begin{ccode}
void GA_Deallocate(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_deallocate(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::deallocate()
\end{cxxcode}
\end{cxxapi}

\gcoll

\begin{desc}

Deallocates memory allocated by the array but leaves the array in place. The
array can be reconstitued later by calling the allocate function.

\end{desc}

\seealso{ALLOCATE}

\apih{DESTROY}{Destroys a global array}

\begin{capi}
\begin{ccode}
void GA_Destroy(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_destroy(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::~GlobalArray()
void GlobalArray::destroy()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.destroy(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Deallocates the array and frees any associated resources.

\end{desc}

\apih{DESTROY MUTEXES}{Destroys mutexes}

\begin{capi}
\begin{ccode}
int GA_Destroy_mutexes()
\end{ccode}
\begin{funcargs}
\outarg{int}{}{1 if successful}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_destroy_mutexes()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::destroyMutexes()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.destroy_mutexes()
\end{pycode}
\begin{funcargs}
\outarg{int}{ret}{True if successful}
\end{funcargs}
\end{pyapi}

\wcoll

\begin{desc}

Destroys the set of mutexes created with ga_create_mutexes. Returns 0 if the
operation succeeded or 1 when failed.

\end{desc}

\seealso{CREATE MUTEXES}

\apih{DIAG}{Diagonalizes a global array}

\begin{capi}
\begin{ccode}
void GA_Diag(int g_a, int g_s, int g_v, void *eval)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{Matrix to diagonalize}
\inarg{int}{g_s}{Metric}
\inarg{int}{g_v}{Global matrix to return evecs}
\outarg{void*}{eval}{Local array to return evals}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_diag(g_a, g_s, g_v, eval)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{Matrix to diagonalize}
\inarg{integer}{g_s}{Metric}
\outarg{integer}{g_v}{Global matrix to return evecs}
\outarg{double precision}{eval(*)}{Local array to return evals}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::diag(const GlobalArray *g_s,
                       GlobalArray *g_v, void *eval)
const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_s}{Matrix to diagonalize}
\inarg{int}{g_v}{Global matrix to return evecs}
\inarg{void*}{eval}{Local array to return evals}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.diag(int g_a, int g_s, int g_v, evalues=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle of the matrix to diagonalize}
\inarg{int}{g_s}{the array handle of the metric}
\inarg{int}{g_v}{the array handle to return evecs}
\inarg{ndarray}{evalues}{local array of appropriate type to return evals}
\outarg{ndarray}{ret}{evals as a local array of appropriate type}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Solve the generalized eigenvalue problem returning all eigenvectors and values
in ascending order. The input matrices are not overwritten or destroyed.

All eigen-values as a vector in ascending order.
\end{desc}

\apih{DIAG REUSE}{Diagonalizes a global array for repeated diagonalizations}

\begin{capi}
\begin{ccode}
void GA_Diag_reuse(int control, int g_a, int g_s, int g_v, void *eval)
\end{ccode}
\begin{funcargs}
\inarg{int}{control}{Control flag}
\inarg{int}{g_a}{Matrix to diagonalize}
\inarg{int}{g_s}{Metric}
\inarg{int}{g_v}{Global matrix to return evecs}
\outarg{void*}{eval}{Local array to return evals}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_diag_reuse(control, g_a, g_s, g_v, eval)
\end{fcode}
\begin{funcargs}
\inarg{integer}{control}{Control flag}
\inarg{integer}{g_a}{Matrix to diagonalize}
\inarg{integer}{g_s}{Metric}
\outarg{integer}{g_v}{Global matrix to return evecs}
\outarg{double precision}{eval(*)}{Local array to return evals}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::diagReuse(int control, const GlobalArray *g_s,
                            GlobalArray *g_v, void *eval) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{control}{Control flag}
\inarg{int}{g_s}{Matrix to diagonalize}
\inarg{int}{g_v}{Global matrix to return evecs}
\outarg{void*}{eval}{Local array to return evals}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.diag_reuse(int control, int g_a, int g_s, int g_v, evalues=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{control}{0 indicates first call to the eigensolver; >0 consecutive calls (reuses factored g_s); <0 only erases factorized g_s; g_v and eval unchanged (should be called after previous use if another eigenproblem, i.e., different g_a and g_s, is to be solved)}
\inarg{int}{g_a}{the array handle of the matrix to diagonalize}
\inarg{int}{g_s}{the array handle of the metric}
\inarg{int}{g_v}{the array handle to return evecs}
\inarg{ndarray}{evalues}{local array of appropriate type to return evals}
\outarg{ndarray}{ret}{evals as a local array of appropriate type}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Solve the generalized eigenvalue problem returning all eigenvectors and
values in ascending order. Recommended for REPEATED calls if g_s is unchanged.
Values of the control flag:
\begin{verbatim}
          value       action/purpose
            0          indicates first call to the eigensolver
           >0          consecutive calls (reuses factored g_s)
           <0          only erases factorized g_s; g_v and eval unchanged
                       (should be called after previous use if another
                        eigenproblem, i.e., different g_a and g_s, is to
                        be solved)
\end{verbatim}

The input matrices are not destroyed.

Returns: All eigen-values as a vector in ascending order.

\end{desc}

\apih{DIAG STD}{Diagonalizes a global array using a standard algorithm}

\begin{capi}
\begin{ccode}
void GA_Diag_std(int g_a, int g_v, void *eval)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{Matrix to diagonalize}
\inarg{int}{g_v}{Global matrix to return evecs}
\outarg{void*}{eval}{Local array to return evals}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_diag_std(g_a, g_v, eval)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{Matrix to diagonalize}
\outarg{integer}{g_v}{Global matrix to return evecs}
\outarg{double precision}{eval(*)}{Local array to return evals}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::diagStd(GlobalArray *g_v, void *eval) const
\end{cxxcode}
\begin{funcargs}
\outarg{int}{g_v}{Global matrix to return evecs}
\outarg{void*}{eval}{Local array to return evals}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.diag_std(int g_a, int g_v, evalues=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle of the matrix to diagonalize}
\inarg{int}{g_v}{the array handle to return evecs}
\inarg{ndarray}{evalues}{local array of appropriate type to return evals}
\outarg{ndarray}{ret}{evals as a local array of appropriate type}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Solve the standard (non-generalized) eigenvalue problem returning all
eigenvectors and values in the ascending order. The input matrix is neither
overwritten nor destroyed.

Returns:
all eigenvectors via the g_v global array, and eigenvalues as an array in
ascending order

\end{desc}

\apih{DISTRIBUTION}{Reports the data range held on a specified processor}

\begin{capi}
\begin{ccode}
void NGA_Distribution(int g_a, int iproc, int lo[], int hi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{iproc}{process number}
\inarg{int}{ndim}{number of dimensions of the global array}
\outarg{int*}{lo[ndim]}{array of starting indices for array section}
\outarg{int*}{hi[ndim]}{array of ending indices for array section}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_distribution(g_a, iproc, ilo, ihi, jlo, jhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{iproc}{process number}
\outarg{integer}{ilo,ihi}{i-range held by process iproc}
\outarg{integer}{jlo,jhi}{j-range held by process iproc}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_distribution(g_a, iproc, lo, hi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{iproc}{process number}
\inarg{integer}{ndim}{number of dimensions}
\outarg{integer}{lo(ndim),hi(ndim)}{range held by process iproc}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::distribution(int me, int* lo, int* hi) const
void GlobalArray::distribution(int me, int64_t* lo, int64_t* hi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{iproc}{process number}
\outarg{int*}{lo[ndim]}{array of starting indices for array section}
\outarg{int*}{hi[ndim]}{array of ending indices for array section}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
lo,hi = ga.distribution(int g_a, int iproc=-1)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{iproc}{process number}
\outarg{1D ndarray of ints}{lo}{array of starting indices for array section}
\outarg{1D ndarray of ints}{hi}{array of ending indices for array section}
\end{funcargs}
\end{pyapi}

\local

\begin{fdesc}
This function returns the bounding indices of the block owned by the process
iproc. These indices are inclusive.

If no array elements are owned by process iproc, the range is returned as
lo()=0 and hi()= -1 for all dimensions.
\end{fdesc}

\begin{cdesc}
This function returns the bounding indices of the block owned by the process
iproc. These indices are inclusive.

If no array elements are owned by process iproc, the range is returned as
lo[]=-1 and hi[]= -2 for all dimensions.
\end{cdesc}

\begin{cxxdesc}
This function returns the bounding indices of the block owned by the process
iproc. These indices are inclusive.

If no array elements are owned by process iproc, the range is returned as
lo[]=-1 and hi[]= -2 for all dimensions.
\end{cxxdesc}

\begin{pydesc}
This function returns the bounding indices of the block owned by the process
iproc. These indices are inclusive.

Return the distribution given to iproc. If iproc is not specified, then
ga.nodeid() is used. The range is returned as -1 for lo and -2 for hi if no
elements are owned by iproc.
\end{pydesc}

Note that the block indices returned only reflect the block owned by a
processor if the data is using a standard data layout. For any of the block
cyclic data layouts, the variable iproc corresponds to the index of the block
in the array where the blocks are indexed in row-major order. This implies
that the value of iproc can exceed the number of processors for block cyclic
layouts. It also follow that all blocks that are separated from each other
by multiples of nproc, where nproc is the number of processors in the
processor group hosting the global array, are on the same processor.

\apih{DOT}{Dot product of two global arrays}

\begin{capi}
\begin{ccode}
int GA_Idot(int g_a, int g_b)
long GA_Ldot(int g_a, int g_b)
float GA_Fdot(int g_a, int g_b)
double GA_Ddot(int g_a, int g_b)
SingleComplex GA_Cdot(int g_a, int g_b)
DoubleComplex GA_Zdot(int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
double precision function ga_ddot(g_a, g_b)
double complex function ga_zdot(g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{g_b}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::idot(const GlobalArray * g_a) const
long GlobalArray::ldot(const GlobalArray * g_a) const
float GlobalArray::fdot(const GlobalArray * g_a) const
double GlobalArray::ddot(const GlobalArray * g_a) const
double complex GlobalArray::zdot(const GlobalArray * g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{GlobalArray*}{g_a}{the other array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.dot(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Computes the element-wise dot product of the two arrays which must be of the
same types and same number of elements.

Return value = SUM_ij a(i,j)*b(i,j)

\end{desc}

\apih{DOT PATCH}{Dot product of patches of global arrays}

\begin{capi}
\begin{ccode}
double NGA_Ddot_patch       (int g_a, char ta, int alo[], int ahi[],
                             int g_b, char tb, int blo[], int bhi[])
int NGA_Idot_patch          (int g_a, char ta, int alo[], int ahi[],
                             int g_b, char tb, int blo[], int bhi[])
long NGA_Ldot_patch         (int g_a, char ta, int alo[], int ahi[],
                             int g_b, char tb, int blo[], int bhi[])
DoubleComplex NGA_Zdot_patch(int g_a, char ta, int alo[], int ahi[],
                             int g_b, char tb, int blo[], int bhi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int*}{alo[], ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[], bhi[]}{g_b patch coordinates}
\inarg{char}{ta, tb}{transpose flags}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
double precision function ga_ddot_patch (g_a, ta, ailo, aihi, ajlo, ajhi,
                                         g_b, tb, bilo, bihi, bjlo, bjhi)
double complex function ga_zdot_patch (g_a, ta, ailo, aihi, ajlo, ajhi,
                                       g_b, tb, bilo, bihi, bjlo, bjhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{}
\inarg{integer}{ailo, aihi, ajlo, ajhi}{g_a patch coordinates}
\inarg{integer}{bilo, bihi, bjlo, bjhi}{g_b patch coordinates}
\inarg{character*1}{ta, tb}{transpose flags}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
double precision function nga_ddot_patch (g_a, ta, alo, ahi,
                                          g_b, tb, bio, bhi)
double complex function nga_zdot_patch (g_a, ta, alo, ahi,
                                        g_b, tb, blo, bhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch coordinates}
\inarg{integer}{blo(ndim), bhi(ndim)}{g_b patch coordinates}
\inarg{character*1}{ta, tb}{transpose flags}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
double GlobalArray::ddotPatch(char ta, int alo[], int ahi[],
                              const GlobalArray * g_a, char tb, int blo[],
                              int bhi[]) const
double GlobalArray::ddotPatch(char ta, int64_t alo[], int64_t ahi[],
                              const GlobalArray * g_a, char tb,
                              int64_t blo[], int64_t bhi[]) const
float GlobalArray::fdotPatch(char ta, int alo[], int ahi[],
                             const GlobalArray * g_a, char tb, int blo[],
                             int bhi[]) const
float GlobalArray::fdotPatch(char ta, int64_t alo[], int64_t ahi[],
                             const GlobalArray * g_a, char tb, int64_t blo[],
                             int64_t bhi[]) const
double complex GlobalArray::zdotPatch(char ta, int alo[], int ahi[],
                                     const GlobalArray * g_a, char tb,
                                     int blo[], int bhi[]) const
double complex GlobalArray::zdotPatch(char ta, int64_t alo[], int64_t ahi[],
                                     const GlobalArray * g_a, char tb,
                                     int64_t blo[], int64_t bhi[]) const
long GlobalArray::idotPatch(char ta, int alo[], int ahi[],
                           const GlobalArray * g_a, char tb, int blo[],
                           int bhi[]) const
long GlobalArray::idotPatch(char ta, int64_t alo[], int64_t ahi[],
                            const GlobalArray * g_a, char tb, int64_t blo[],
                            int64_t bhi[]) const
long GlobalArray::ldotPatch(char ta, int alo[], int ahi[],
                            const GlobalArray * g_a, char tb, int blo[],
                            int bhi[]) const
long GlobalArray::ldotPatch(char ta, int64_t alo[], int64_t ahi[],
                            const GlobalArray * g_a, char tb, int64_t blo[],
                            int64_t bhi[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{char}{ta}{transpose flags}
\inarg{int*}{alo}{g_a patch coordinates}
\inarg{int*}{ahi}{g_a patch coordinates}
\inarg{int}{g_a}{global array}
\inarg{char}{tb}{transpose flags}
\inarg{int*}{blo}{g_b patch coordinates}
\inarg{int*}{bhi}{g_b patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.dot(int g_a, int g_b,
             alo=None, ahi=None, blo=None, bhi=None,
             int ta=False, int tb=False)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{1D array-like of ints}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of ints}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of ints}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of ints}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{bool}{ta}{whether the transpose operator should be applied to g_a True=applied}
\inarg{bool}{tb}{whether the transpose operator should be applied to g_b True=applied}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Computes the element-wise dot product of the two (possibly transposed) patches
which must be of the same type and have the same number of elements.

\end{desc}

\seealso{DOT}

\apih{DUPLICATE}{Creates a new array with the same properties as the given array}

\begin{capi}
\begin{ccode}
int GA_Duplicate(int g_a, char* array_name)
\end{ccode}
\begin{funcargs}
\inarg{char*}{array_name}{a character string}
\inarg{int}{g_a}{integer handle for reference array}
\outarg{int}{}{integer handle for duplicated array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_duplicate(g_a, g_b, array_name)
\end{fcode}
\begin{funcargs}
\inarg{character*(*)}{array_name}{a character string}
\inarg{integer}{g_a}{Integer handle for reference array}
\outarg{integer}{g_b}{Integer handle for new array}
\outarg{logical}{}{.TRUE. if array creation successful}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GlobalArray::GlobalArray(const GlobalArray &g_a, char *arrayname)
GlobalArray::GlobalArray(const GlobalArray &g_a)
GlobalArray * GAServices::createGA(const GlobalArray *g_b, char *arrayname)
GlobalArray * GAServices::createGA(const GlobalArray &g_b)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_b}{integer handle for reference array}
\inarg{char*}{arrayname}{a character string}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.duplicate(int g_a, char *name='')
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{integer handle for reference array}
\inarg{char*}{array_name}{a character string}
\outarg{int}{ret}{integer handle for duplicated array}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Creates a new array by applying all the properties of another existing array.
It returns an array handle.

Return value: a non-zero array handle means the call was succesful.

\end{desc}

\apih{ELEM DIVIDE}{Element-wise division of global arrays}

\begin{capi}
\begin{ccode}
void GA_Elem_divide(int g_a, int g_b, int g_c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int}{g_c}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_divide(g_a, g_b, g_c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemDivide(const GlobalArray * g_a,
                             const GlobalArray * g_b) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{GlobalArray}
\inarg{const GlobalArray*}{g_b}{GlobalArray}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_divide(int g_a, int g_b, int g_c)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise quotient of the two arrays which must be of the same
types and same number of elements. For two-dimensional arrays,
\begin{verbatim}
        c(i,j) = a(i,j)/b(i,j)
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).  If one of the
elements of array g_b is zero, the quotient for the element of g_c will be set
to GA_NEGATIVE_INFINITY.
\end{desc}

\apih{ELEM DIVIDE PATCH}{Element-wise division of global array patches}

\begin{capi}
\begin{ccode}
void GA_Elem_divide_patch(int g_a, int alo[], int ahi[],
                          int g_b, int blo[], int bhi[],
                          int g_c, int clo[], int chi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int}{g_c}{array handle}
\inarg{int*}{alo[], ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[], bhi[]}{g_b patch coordinates}
\inarg{int*}{clo[], chi[]}{g_c patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_divide_patch(g_a, alo, ahi,
                                g_b, blo, bhi,
                                g_c, clo, chi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch dimensions}
\inarg{integer}{blo(ndim), bhi(ndim)}{g_b patch dimensions}
\inarg{integer}{clo(ndim), chi(ndim)}{g_c patch dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemDividePatch(
        const GlobalArray * g_a, int *alo, int *ahi,
        const GlobalArray * g_b, int *blo, int *bhi,
        int *clo, int *chi) const
void GlobalArray::elemDividePatch(
        const GlobalArray * g_a, int64_t *alo, int64_t *ahi,
        const GlobalArray * g_b, int64_t *blo, int64_t *bhi,
        int64_t *clo, int64_t *chi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{global array}
\inarg{int}{g_b}{global array}
\inarg{int*}{alo}{g_a lower corner patch coordinates}
\inarg{int*}{ahi}{g_a upper corner patch coordinates}
\inarg{int*}{blo}{g_b lower corner patch coordinates}
\inarg{int*}{bhi}{g_b upper corner patch coordinates}
\inarg{int*}{clo}{g_c lower corner patch coordinates}
\inarg{int*}{chi}{g_c upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_divide(int g_a, int g_b, int g_c,
               alo=None, ahi=None,
               blo=None, bhi=None,
               clo=None, chi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{1D array-like of integers}{clo}{lower bound patch coordinates of g_c, inclusive}
\inarg{1D array-like of integers}{chi}{higher bound patch coordinates of g_c, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise quotient of the two patches which must be of the same
types and same number of elements. For two-dimensional arrays,
\begin{verbatim}
        c(i,j)  = a(i,j)/b(i,j)
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).
\end{desc}

\seealso{ELEM DIVIDE}

\apih{ELEM MAXIMUM}{Element-wise maximum of global arrays}

\begin{capi}
\begin{ccode}
void GA_Elem_maximum(int g_a, int g_b, int g_c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int}{g_c}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_maximum(g_a, g_b, g_c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemMaximum(const GlobalArray * g_a,
                              const GlobalArray * g_b) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{global array}
\inarg{const GlobalArray*}{g_b}{global array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_maximum(int g_a, int g_b, int g_c)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise maximum of the two arrays which must be of the same
types and same number of elements. For two dimensional arrays,
\begin{verbatim}
    c(i,j)  = max{a(i,j), b(i,j)}
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).
\end{desc}

\apih{ELEM MAXIMUM PATCH}{Element-wise maximum of global array patches}

\begin{capi}
\begin{ccode}
void GA_Elem_maximum_patch(int g_a, int alo[], int ahi[],
                           int g_b, int blo[], int bhi[],
                           int g_c, int clo[], int chi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int}{g_c}{array handle}
\inarg{int*}{alo[], ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[], bhi[]}{g_b patch coordinates}
\inarg{int*}{clo[], chi[]}{g_c patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_maximum_patch(g_a, alo, ahi,
                                 g_b, blo, bhi,
                                 g_c, clo, chi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch dimensions}
\inarg{integer}{blo(ndim), bhi(ndim)}{g_b patch dimensions}
\inarg{integer}{clo(ndim), chi(ndim)}{g_c patch dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemMaximumPatch(
        const GlobalArray * g_a, int *alo, int *ahi,
        const GlobalArray * g_b, int *blo, int *bhi,
        int *clo, int *chi) const
void GlobalArray::elemMaximumPatch(
        const GlobalArray * g_a, int64_t *alo, int64_t *ahi,
        const GlobalArray * g_b, int64_t *blo, int64_t *bhi,
        int64_t *clo, int64_t *chi) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{global array}
\inarg{const GlobalArray*}{g_b}{global array}
\inarg{int*}{alo}{g_a lower corner patch coordinates}
\inarg{int*}{ahi}{g_a upper corner patch coordinates}
\inarg{int*}{blo}{g_b lower corner patch coordinates}
\inarg{int*}{bhi}{g_b upper corner patch coordinates}
\inarg{int*}{clo}{g_c lower corner patch coordinates}
\inarg{int*}{chi}{g_c upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_maximum(int g_a, int g_b, int g_c,
                alo=None, ahi=None,
                blo=None, bhi=None,
                clo=None, chi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{1D array-like of integers}{clo}{lower bound patch coordinates of g_c, inclusive}
\inarg{1D array-like of integers}{chi}{higher bound patch coordinates of g_c, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise maximum of the two patches which must be of the same
types and same number of elements. For two-dimensional noncomplex arrays,
\begin{verbatim}
        c(i,j)  = max{a(i,j), b(i,j)}
\end{verbatim}
If the data type is complex, then
\begin{verbatim}
        c(i,j).real = max{ |a(i,j)|, |b(i,j)| } while c(i,j).image = 0.
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).
\end{desc}

\apih{ELEM MINIMUM}{Element-wise minimum of global arrays}

\begin{capi}
\begin{ccode}
void GA_Elem_minimum(int g_a, int g_b, int g_c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int}{g_c}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_minimum(g_a, g_b, g_c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemMinimum(const GlobalArray * g_a,
                              const GlobalArray * g_b) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{global array}
\inarg{const GlobalArray*}{g_b}{global array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_minimum(int g_a, int g_b, int g_c)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Computes the element-wise minimum of the two arrays
which must be of the same types and same number of
elements. For two dimensional arrays,
\begin{verbatim}
        c(i,j)  = min{a(i,j), b(i,j)}
\end{verbatim}

The result (c) may replace one of the input arrays (a/b).

\end{desc}

\apih{ELEM MINIMUM PATCH}{Element-wise minimum of global array patches}

\begin{capi}
\begin{ccode}
void GA_Elem_minimum_patch(int g_a, int alo[], int ahi[],
                           int g_b, int blo[], int bhi[],
                           int g_c, int clo[], int chi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{array handles}
\inarg{int}{g_c}{array handle}
\inarg{int*}{alo[], ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[], bhi[]}{g_b patch coordinates}
\inarg{int*}{clo[], chi[]}{g_c patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_minimum_patch(g_a, alo, ahi,
                                 g_b, blo, bhi,
                                 g_c, clo, chi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim),ahi(ndim)}{g_a patch dimensions}
\inarg{integer}{blo(ndim),bhi(ndim)}{g_b patch dimensions}
\inarg{integer}{clo(ndim),chi(ndim)}{g_c patch dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemMinimumPatch(
        const GlobalArray * g_a, int *alo, int *ahi,
        const GlobalArray * g_b, int *blo, int *bhi,
        int *clo, int *chi) const
void GlobalArray::elemMinimumPatch(
        const GlobalArray * g_a, int64_t *alo, int64_t *ahi,
        const GlobalArray * g_b, int64_t *blo, int64_t *bhi,
        int64_t *clo, int64_t *chi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{global array}
\inarg{int}{g_b}{global array}
\inarg{int*}{alo}{g_a lower corner patch coordinates}
\inarg{int*}{ahi}{g_a upper corner patch coordinates}
\inarg{int*}{blo}{g_b lower corner patch coordinates}
\inarg{int*}{bhi}{g_b upper corner patch coordinates}
\inarg{int*}{clo}{g_c lower corner patch coordinates}
\inarg{int*}{chi}{g_c upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_minimum(int g_a, int g_b, int g_c,
                alo=None, ahi=None,
                blo=None, bhi=None,
                clo=None, chi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{1D array-like of integers}{clo}{lower bound patch coordinates of g_c, inclusive}
\inarg{1D array-like of integers}{chi}{higher bound patch coordinates of g_c, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise minimum of the two patches which must be of the same
types and same number of elements. For two-dimensional of noncomplex arrays,
\begin{verbatim}
        c(i,j)  = min{a(i,j), b(i,j)}
\end{verbatim}
If the data type is complex, then
\begin{verbatim}
        c(i,j).real = min{ |a(i,j)|, |b(i,j)| } while c(i,j).image = 0.
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).
\end{desc}

\seealso{ELEM MINIMUM}

\apih{ELEM MULTIPLY}{Element-wise multiplication of global arrays}

\begin{capi}
\begin{ccode}
void GA_Elem_multiply(int g_a, int g_b, int g_c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{input array handles}
\inarg{int}{g_c}{output array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_multiply(g_a, g_b, g_c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{input array handles}
\outarg{integer}{g_c}{output array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemMultiply(const GlobalArray * g_a,
                               const GlobalArray * g_b) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{GlobalArray}
\inarg{const GlobalArray*}{g_b}{GlobalArray}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_multiply(int g_a, int g_b, int g_c)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise product of the two arrays which must be of the same
types and same number of elements. For two-dimensional arrays,
\begin{verbatim}
        c(i, j)  = a(i,j)*b(i,j)
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).

\end{desc}

\apih{ELEM MULTIPLY PATCH}{Element-wise multiplication of global array patches}

\begin{capi}
\begin{ccode}
void GA_Elem_multiply_patch(int g_a, int alo[], int ahi[],
                            int g_b, int blo[], int bhi[],
                            int g_c, int clo[], int chi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b}{input array handles}
\inarg{int}{g_c}{output array handle}
\inarg{int*}{alo[], ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[], bhi[]}{g_b patch coordinates}
\inarg{int*}{clo[], chi[]}{g_c patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_elem_multiply_patch(g_a, alo, ahi,
                                  g_b, blo, bhi,
                                  g_c, clo, chi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{array handles}
\outarg{integer}{g_c}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch dimensions}
\inarg{integer}{blo(ndim), bhi(ndim)}{g_b patch dimensions}
\inarg{integer}{clo(ndim), chi(ndim)}{g_c patch dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::elemMultiplyPatch(
        const GlobalArray * g_a, int *alo, int *ahi,
        const GlobalArray * g_b, int *blo, int *bhi,
        int *clo, int *chi) const
void GlobalArray::elemMultiplyPatch(
        const GlobalArray * g_a, int64_t *alo, int64_t *ahi,
        const GlobalArray * g_b, int64_t *blo, int64_t *bhi,
        int64_t *clo, int64_t *chi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{global array}
\inarg{int}{g_b}{global array}
\inarg{int*}{alo}{g_a lower corner patch coordinates}
\inarg{int*}{ahi}{g_a upper corner patch coordinates}
\inarg{int*}{blo}{g_b lower corner patch coordinates}
\inarg{int*}{bhi}{g_b upper corner patch coordinates}
\inarg{int*}{clo}{g_c lower corner patch coordinates}
\inarg{int*}{chi}{g_c upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.elem_multiply(int g_a, int g_b, int g_c,
                 alo=None, ahi=None,
                 blo=None, bhi=None,
                 clo=None, chi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{1D array-like of integers}{clo}{lower bound patch coordinates of g_c, inclusive}
\inarg{1D array-like of integers}{chi}{higher bound patch coordinates of g_c, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the element-wise product of the two patches which must be of the same
types and same number of elements. For two-dimensional arrays,
\begin{verbatim}
        c(i,j)  = a(i,j)*b(i,j)
\end{verbatim}
The result (c) may replace one of the input arrays (a/b).
\end{desc}

\seealso{ELEM MULTIPLY}

\apih{ERROR}{Aborts with an error}

\begin{capi}
\begin{ccode}
void GA_Error(char *message, int code)
\end{ccode}
\begin{funcargs}
\inarg{char*}{message}{string to print}
\inarg{int}{code}{code to print}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_error(message, code)
\end{fcode}
\begin{funcargs}
\inarg{character*1}{message(*)}{}
\inarg{integer}{code}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::error(const char *message, int code)
\end{cxxcode}
\begin{funcargs}
\inarg{char*}{message}{string to print}
\inarg{int}{code}{code to print}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.error(char *message, int code=1)
\end{pycode}
\begin{funcargs}
\inarg{char*}{message}{string to print}
\inarg{int}{code}{code to print}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

To be called in case of an error. Print an error message and an integer value
that represents an error code as well as releasing some system resources. This is the
required way of aborting the program execution.

\end{desc}

\apih{FENCE}{Fences all GA data movement operations initiated by the calling process}

\begin{capi}
\begin{ccode}
void GA_Fence()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_fence()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::fence()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.fence()
\end{pycode}
\end{pyapi}

\ncoll

\begin{desc}

Blocks the calling process until all the data transfers corresponding to GA
operations called after ga_init_fence complete. For example, since ga_put might
return before the data reaches the final destination, ga_init_fence and
ga_fence allow the process to wait until the data tranfer is fully completed:
\begin{verbatim}
        ga_init_fence();
        ga_put(g_a, ...);
        ga_fence();
\end{verbatim}

ga_fence must be called after ga_init_fence. A barrier, ga_sync, assures the
completion of all data transfers and implicitly cancels all outstanding
ga_init_fence calls. ga_init_fence and ga_fence must be used in pairs, multiple
calls to ga_fence require the same number of corresponding ga_init_fence calls.
ga_init_fence/ga_fence pairs can be nested.

ga_fence works for multiple GA operations. For example:
\begin{verbatim}
        ga_init_fence();
        ga_put(g_a, ...);
        ga_scatter(g_a, ...);
        ga_put(g_b, ...);
        ga_fence();
\end{verbatim}

The calling process will be blocked until data movements initiated by two calls
to ga_put and one ga_scatter complete.

\end{desc}

\apih{FILL}{Fills a global array with a specific value}

\begin{capi}
\begin{ccode}
void GA_Fill(int g_a, void *value)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{void*}{value}{pointer to the value of appropriate type (double/double complex/long) that matches array type}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_fill(g_a, s)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{double precision/complex/integer}{s}{fill value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::fill(void *value) const
\end{cxxcode}
\begin{funcargs}
\inarg{void*}{value}{pointer to the value of appropriate type (double/double complex/long) that matches array type.}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.fill(int g_a, value)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{value}{object of the value of appropriate type (double/double complex/long) that matches array type}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Assign a single value to all elements in the array.

\end{desc}

\apih{FILL PATCH}{Fills a patch of a global array with a specified value}

\begin{capi}
\begin{ccode}
void NGA_Fill_patch(int g_a, int lo[], int hi[], void *val)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handles}
\inarg{int*}{lo[], hi[]}{patch of g_a}
\inarg{void*}{val}{value to fill}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_fill_patch(g_a, ailo, aihi, ajlo, ajhi, s)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{double precision/complex/integer}{s}{fill value}
\inarg{integer}{ailo, aihi, ajlo, ajhi}{g_a patch coordinates}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_fill_patch (g_a, alo, ahi, s)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{double precision/complex/integer}{s}{fill value}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::fillPatch (int lo[], int hi[], void *val) const
void GlobalArray::fillPatch (int64_t lo[], int64_t hi[], void *val) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo}{patch of this GlobalArray}
\inarg{int*}{hi}{patch of this GlobalArray}
\inarg{void*}{val}{value to fill}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.fill(int g_a, value, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{value}{object of the value of appropriate type (double/double complex/long) that matches array type}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Fill the patch of g_a with value of `val'

\end{desc}

\seealso{FILL}

\apih{FREE GATSCAT BUF}{Frees an internal buffer for gather/scatter operations}

\begin{capi}
\begin{ccode}
void NGA_Free_gatscat_buf()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
void nga_free_gatscat_buf()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::freeGatscatBuf()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
void ga.free_gatscat_buf()
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function is used to free up internal buffers that were set with the
corresponding allocation call. The buffers can be used to improve performance if
multiple calls are being made to the gather/scatter operations.
\end{desc}

\seealso{ALLOC GATSCAT BUF}

\apih{GATHER}{Gathers elements from a global array}

\begin{capi}
\begin{ccode}
void NGA_Gather(int g_a, void *v, int* subsArray[], int n)
void NGA_Gather_flat(int g_a, void *v, int* subsArray, int n)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{void*}{v[n]}{array containing values}
\inarg{int**}{subsArray[n][ndim]}{array of subscripts for each element}
\inarg{int*}{subsArray[n*ndim]}{(flat) array of subscripts for each element}
\inarg{int}{n}{number of elements}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_gather(g_a, v, i, j, n)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\outarg{double precision}{v(n)}{}
\inarg{integer}{i(n), j(n), n}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_gather(g_a, v, subsArray, n)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{n}{number of elements}
\outarg{type}{v(n)}{array containing values}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{subsArray(ndim,n)}{array of subscripts for each element}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::gather(void *v, int * subsarray[], int n) const
void GlobalArray::gather(void *v, int64_t * subsarray[], int64_t n) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{n}{number of elements}
\inarg{void*}{v[n]}{array containing values}
\inarg{int**}{subsarray[n][ndim]}{array of subscripts for each element}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.gather(int g_a, subsarray, ndarray values=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D or 2D array-like of ints}{subsarray}{array of subscripts for each element}
\inarg{1D array-like}{values}{buffer to be overwritten}
\outarg{1D array-like}{ret}{array containing values}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Gathers array elements from a global array into a local array. The contents of
the input arrays (v, subsArray) are preserved.

\begin{verbatim}
for (k=0; k<= n; k++)
   {v[k] = a[subsArray[k][0]][subsArray[k][1]][subsArray[k][2]]...;}
\end{verbatim}

\end{desc}

\begin{pydesc}

subsarray will be converted to an ndarray if it is not one already.  A
two-dimensional array is allowed so long as its shape is (n,ndim) where n is
the number of elements to gather and ndim is the number of dimensions of the
target array. Also, subsarray must be contiguous.

For example, if the subsarray were two-dimensional:

\begin{verbatim}
for k in range(n):
    v[k] = g_a[subsarray[k,0],subsarray[k,1],subsarray[k,2]...]
\end{verbatim}

For example, if the subsarray were one-dimensional:

\begin{verbatim}
for k in range(n):
    base = n*ndim
    v[k] = g_a[subsarray[base+0],subsarray[base+1],subsarray[base+2]...]
\end{verbatim}

\end{pydesc}

\seealso{GET}

\apih{GEMM}{Performs matrix multiplication of global arrays}

\begin{capi}
\begin{ccode}
void GA_Dgemm(char ta, char tb, int m, int n, int k, double alpha,
              int g_a, int g_b, double beta, int g_c)
void GA_Sgemm(char ta, char tb, int m, int n, int k, float alpha,
              int g_a, int g_b, float beta, int g_c)
void GA_Zgemm(char ta, char tb, int m, int n, int k, double complex alpha,
              int g_a, int g_b, double complex beta, int g_c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{handle to input arrays}
\inarg{int}{g_b}{handle to input arrays}
\inarg{int}{g_c}{handle to output array}
\inarg{char}{ta}{transpose operator for g_a}
\inarg{char}{tb}{transpose operator for g_b}
\inarg{int}{m}{number of rows of op(A) and of matrix C}
\inarg{int}{n}{number of columns of op(B) and of matrix C}
\inarg{int}{k}{number of columns of op(A) and rows of matrix op(B)}
\inarg{}{alpha, beta}{scale factors}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine GA_Dgemm(ta, tb, m, n, k, alpha, g_a, g_b, beta, g_c)
subroutine GA_Sgemm(ta, tb, m, n, k, alpha, g_a, g_b, beta, g_c)
subroutine GA_Zgemm(ta, tb, m, n, k, alpha, g_a, g_b, beta, g_c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, g_b}{handles to input arrays}
\outarg{integer}{g_c}{handle to output array}
\inarg{character(1)}{ta, tb}{transpose operators}
\inarg{integer}{m}{number of rows of op(A) and of matrix  C}
\inarg{integer}{n}{number of columns of op(B) and of matrix  C}
\inarg{integer}{k}{number of columns of op(A) and rows of matrix op(B)}
\inarg{double precision/double complex/real}{alpha, beta}{scale factors}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::dgemm(char ta, char tb, int m, int n, int k,
                        double alpha, const GlobalArray *g_a, const
                        GlobalArray *g_b, double beta) const
void GlobalArray::dgemm(char ta, char tb, int64_t m, int64_t n, int64_t k,
                        double alpha, const GlobalArray *g_a, const
                        GlobalArray *g_b, double beta) const
\end{cxxcode}
\begin{funcargs}
\inarg{char}{ta}{transpose operators}
\inarg{char}{tb}{transpose operators}
\inarg{int}{m}{number of rows of op(A) and of matrix C}
\inarg{int}{n}{number of columns of op(B) and of matrix C}
\inarg{int}{k}{number of columns of op(A) and rows of matrix op(B)}
\inarg{}{alpha}{scale factor}
\inarg{int}{g_a}{input array}
\inarg{int}{g_b}{input array}
\inarg{}{beta}{scale factor}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.gemm(int ta, int tb, int64_t m, int64_t n, int64_t k,
        alpha, int g_a, int g_b, beta, int g_c)
   ta (bool)       - transpose operator
   tb (bool)       - transpose operator
   m (int)         - number of rows of op(A) and of matrix C
   n (int)         - number of columns of op(B) and of matrix C
   k (int)         - number of columns of op(A) and rows of matrix op(B)
   alpha (object)  - scale factor
   g_a (int)       - handle to input array
   g_b (int)       - handle to input array
   beta (object)   - scale factor
   g_c (int)       - handle to output array
\end{pycode}
\begin{funcargs}
\inarg{char}{ta}{transpose operator for g_a}
\inarg{char}{tb}{transpose operator for g_b}
\inarg{int}{m}{number of rows of op(A) and of matrix C}
\inarg{int}{n}{number of columns of op(B) and of matrix C}
\inarg{int}{k}{number of columns of op(A) and rows of matrix op(B)}
\inarg{object}{alpha}{scale factor}
\inarg{int}{g_a}{handle to input arrays}
\inarg{int}{g_b}{handle to input arrays}
\inarg{object}{beta}{scale factor}
\inarg{int}{g_c}{handle to output array}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Performs one of the matrix-matrix operations:
\begin{verbatim}
      C := alpha*op( A )*op( B ) + beta*C,
\end{verbatim}

where op( X ) is one of
\begin{verbatim}
      op( X ) = X   or   op( X ) = X',
\end{verbatim}

alpha and beta are scalars, and A, B, and C are matrices, with op( A )
an m by k matrix, op( B ) a k by n matrix and C an m by n matrix.

On entry, transa specifies the form of op( A ) to be used in the matrix
multiplication as follows:
\begin{verbatim}
           ta = `N' or `n', op( A ) = A.
           ta = `T' or `t', op( A ) = A'.
\end{verbatim}

\end{desc}

\apih{GET}{Gets data from a global array}

\begin{capi}
\begin{ccode}
void NGA_Get(int g_a, int lo[], int hi[], void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\outarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_get(g_a, ilo, ihi, jlo, jhi, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ilo, ihi, jlo, jhi}{}
\outarg{double precision/complex/integer}{buf}{local buffer where data goes to}
\inarg{integer}{ld}{leading dimension}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_get(g_a, lo, hi, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer array where the data goes to}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\end{funcargs}
\end{fapi}

\begin{pyapi}
\begin{pycode}
ret = ga.get(int g_a, lo=None, hi=None, ndarray buffer=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{ndarray}{buffer}{an ndarray of the appropriate type, large enough to hold lo,hi}
\outarg{ndarray}{ret}{local buffer array where the data goes to}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Copies data from global array section to the local array buffer. The local
array is assumed to be have the same number of dimensions as the global array.
Any detected inconsistencies or errors in the input arguments are fatal.

Example: For the ga_get operation transfering data from the [10:14, 0:4]
section of 2-dimensional 15x10 global array into a local buffer 5x10 array we
have:

\begin{verbatim}
lo={10,0,} hi={14,4}, ld={10}
\end{verbatim}

Figure \ref{get} shows the GET operation.

\begin{figure}
\centering
\includegraphics{get}
\caption{Copying Data from 2-dimensional 15x10 Global Array into Local Buffer 5x10 Array}
\label{get}
\end{figure}

Return: The local array buffer.

\end{desc}

\begin{pydesc}
If buffer is None, then a new ndarray will be allocated. Otherwise, the same
buffer that was passed in will be updated and returned.
\end{pydesc}

\apih{GET BLOCK INFO}{Returns information on block-cyclic distribution for a GA}

\begin{capi}
\begin{ccode}
void GA_Get_block_info(int g_a, int num_blocks[], int block_dims[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int*}{num_blocks[ndim]}{number of blocks along each axis}
\outarg{int*}{block_dims[ndim]}{dimensions of block}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_get_block_info(g_a, num_blocks, block_dims)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{integer}{num_blocks(ndim)}{number of blocks along each axis}
\outarg{integer}{block_dims(ndim)}{dimensions of block}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::getBlockInfo(int num_blocks[], int block_dims[])
\end{cxxcode}
\begin{funcargs}
\outarg{int*}{num_blocks[ndim]}{array containing number of blocks along each coordinate direction}
\outarg{int*}{block_dims[ndim]}{array containing block dimensions}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
num_blocks,block_dims = get_block_info(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{1D ndarray of ints}{num_blocks}{number of blocks along each axis}
\outarg{1D ndarray of ints}{block_dims}{dimensions of block}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

This subroutine returns information about the block-cyclic distribution
associated with global array g_a. The number of blocks along each of the array
axes are returned in the array num_blocks and the dimensions of the individual
blocks, specified in the GA_Set_block_cyclic or GA_Set_block_cyclic_proc_grid
subroutines, are returned in block_dims.

This is a local function.

\end{desc}

\seealso{SET BLOCK CYCLIC,SET BLOCK CYCLIC PROC GRID,SET TILED PROC GRID}

\apih{GET DEBUG}{Returns value of GA debug flag}

\begin{capi}
\begin{ccode}
int GA_Get_debug()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_get_debug()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::getDebug()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.get_debug()
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function returns the value of an internal flag in the GA library whose
value can be set using the GA_Set_debug subroutine.
\end{desc}

\apih{GET DIAG}{Copies diagonal elements of a global array into another global array}

\begin{capi}
\begin{ccode}
void GA_Get_diag(int g_a, int g_v)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_get_diag(g_a, g_v)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{g_v}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::getDiagonal(const GlobalArray * g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{global array containing diagonal elements}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.get_diag(int g_a, int g_v)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Inserts the diagonal elements of this matrix g_a into the vector g_v.
\end{desc}

\apih{GET GHOST BLOCK}{Gets data, including ghost cells, from a global array with ghost cells}

\begin{capi}
\begin{ccode}
void NGA_Get_ghost_block(int g_a, int lo[], int hi[], void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\outarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_get_ghost_block(g_a, lo, hi, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer array where the data goes to}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\end{funcargs}
\end{fapi}

\begin{pyapi}
\begin{pycode}
ret = ga.get_ghost_block(int g_a, lo=None, hi=None, ndarray buffer=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{ndarray}{buffer}{an ndarray of the appropriate type, large enough to hold lo,hi}
\outarg{ndarray}{ret}{local buffer array where the data goes to}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

This operation behaves similarly to the GA Get operation and will default to
a standard Get if the global array has no ghost cells or the request does not
fit into the ghost cell region of the calling processor. It is easier to use
than the NGA_Access_ghosts function but it may be slower and requires more
memory. The operation will copy data from the locally held portions of the global
array, including the ghost cells, if the requested block falls within the region
defined by visible block held by the process plus the ghost cell region. For
example, if the process holds the visible block [2:8, 2:8] with a ghost cell
width that is one element deep, then a request for the block [1:9,1:9] will use
the local ghost cells to fill the local buffer. In this case, the data transfer
is completely local. If the request is for a block such as [1:10,1:10], which
would require data from another process, then the NGA_Get_ghost_block call
reverts to an ordinary NGA_Get operation and ignores the locally held ghost
data.

Return: The local array buffer.
\end{desc}

\seealso{GET,ACCESS GHOSTS,RELEASE GHOSTS,RELEASE UPDATE GHOSTS}

\apih{GOP}{Global commutative vector operations of elements among all processes}

\begin{capi}
\begin{ccode}
void GA_Fgop(float x[], int n, char *op)
void GA_Dgop(double x[], int n, char *op)
void GA_Igop(int x[], int n, char *op)
void GA_Lgop(long x[], int n, char *op)
void GA_Cgop(SingleComplex x[], int n, char *op)
void GA_Zgop(DoubleComplex x[], int n, char *op)
\end{ccode}
\begin{funcargs}
\inoutarg{float*}{x[n]}{array of elements}
\inoutarg{double*}{x[n]}{array of elements}
\inoutarg{int*}{x[n]}{array of elements}
\inoutarg{long*}{x[n]}{array of elements}
\inoutarg{SingleComplex*}{x[n]}{array of elements}
\inoutarg{DoubleComplex*}{x[n]}{array of elements}
\inarg{int}{n}{number of elements}
\inarg{char}{op}{operator}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_igop(type, x, n, op)
subroutine ga_sgop(type, x, n, op)
subroutine ga_dgop(type, x, n, op)
subroutine ga_cgop(type, x, n, op)
subroutine ga_zgop(type, x, n, op)
\end{fcode}
\begin{funcargs}
\inarg{integer}{type}{this argument is deprecated}
\inoutarg{integer}{x(n)}{local array of initial/final values}
\inoutarg{real}{x(n)}{local array of initial/final values}
\inoutarg{double precision}{x(n)}{local array of initial/final values}
\inoutarg{single complex}{x(n)}{local array of initial/final values}
\inoutarg{double complex}{x(n)}{local array of initial/final values}
\inarg{character*(*)}{op}{global operation}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GAServices::dgop(double x[], int n, char *op);
void GAServices::igop(int x[], int n, char *op);
void GAServices::lgop(long x[], int n, char *op);
void GAServices::gop(int x[], int n, char *op);
void GAServices::gop(long x[], int n, char *op);
void GAServices::gop(float x[], int n, char *op);
void GAServices::gop(double x[], int n, char *op);
\end{cxxcode}
\begin{funcargs}
\inoutarg{double}{x[n]}{array of elements}
\inoutarg{int}{x[n]}{array of elements}
\inoutarg{long}{x[n]}{array of elements}
\inoutarg{ing}{x[n]}{array of elements}
\inoutarg{long}{x[n]}{array of elements}
\inoutarg{float}{x[n]}{array of elements}
\inoutarg{double}{x[n]}{array of elements}
\inarg{int}{n}{number of elements}
\inarg{char}{op}{operator}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.gop(X, char *op)
ret = ga.gop_add(X)
ret = ga.gop_multiply(X)
ret = ga.gop_max(X)
ret = ga.gop_min(X)
ret = ga.gop_absmax(X)
ret = ga.gop_absmin(X)
\end{pycode}
\begin{funcargs}
\inarg{1D array-like}{X}{elements}
\inarg{char*}{op}{operator}
\outarg{ndarray}{ret}{result after numpy.asarray(X) followed by the operation}
\end{funcargs}
\end{pyapi}

\wcoll

\begin{desc}

Global OPeration.

$X(1:N)$ is a vector present on each process. GOP `sums' elements of X accross
all nodes using the commutative operator OP. The result is broadcast to all
nodes. Supported operations include `+', `*', `max', `min', `absmax', `absmin'.
The use of lowerecase for operators is necessary.

This operation is provided only for convenience purposes: it is available
regardless of the message-passing library that GA is running with.

\end{desc}

\apih{HAS GHOSTS}{Checks whether a GA has ghost cells}

\begin{capi}
\begin{ccode}
int GA_Has_ghosts(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int}{}{1 if array has ghost cells}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_has_ghosts(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{logical}{}{.TRUE. if array has ghost cells}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::hasGhosts() const
\end{cxxcode}
\begin{funcargs}
\outarg{int}{}{1 if array has ghost cells}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.has_ghosts(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int}{ret}{True if array has ghost cells}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function returns 1 if the global array has some dimensions for which the
ghost cell width is greater than zero, it returns 0 otherwise.

\end{desc}

\apih{INIT FENCE}{Initializes tracing of completion of data movement operations}

\begin{capi}
\begin{ccode}
void GA_Init_fence()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_init_fence()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::initFence()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.init_fence()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

Initializes tracing of the completion status of data movement operations.

\end{desc}

\apih{INITIALIZE}{Initializes GA}

\begin{capi}
\begin{ccode}
void NGA_Initialize()
void GA_Initialize()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_initialize()
subroutine ga_initialize()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GA::Initialize(int argc, char *argv[], size_t limit=0)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{argc}{number of command line arguments}
\inarg{char**}{argv}{command line arguments}
\inarg{size_t}{limit}{amount of memory in bytes per process}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
from ga4py import ga
\end{pycode}
\end{pyapi}

\gcoll

\begin{desc}

Allocate and initialize internal data structures in Global Arrays.

\end{desc}

\seealso{INITIALIZE LTD}

\apih{INITIALIZE COMM}{Initializes GA on an MPI communicator}

\begin{capi}
\begin{ccode}
int NGA_Initialize_comm(MPI_Comm comm)
int GA_Initialize_comm(MPI_Comm comm)
\end{ccode}
\begin{funcargs}
\inarg{MPI_Comm}{comm}{MPI communicator used to initialize GA}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_initialize_comm(comm)
logical function ga_initialize_comm(comm)
\end{fcode}
\begin{funcargs}
\inarg{MPI_Comm}{comm}{MPI communicator used to initialize GA}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}

Allocate and initialize internal data structures in Global Arrays on a
restricted set of processors described by comm. This can be used to initialize
a GA application that is being called as a subroutine by an application that
does not used GA without requiring that the calling application initialize GA.

Note that if you are using the Progress Ranks runtime, you need to take care
that the communicator you pass to the GA application has the correct properties.
Specifically, the communicator must have at least 2 ranks per SMP node.

This initialization returns a logical value that is used to guarantee that the
program does not hang when exiting the subroutine initializing GA. This can
happen when using the progress ranks runtime. A properly constructed function
using this initialization is shown below.
\end{desc}

\begin{fdesc}
\begin{verbatim}
subroutine my_ga_app(...., comm)
  MPI_Comm comm
  if (nga_initialize_comm(comm)) then
!   Main body of code here
    call nga_terminate()
  endif
end
\end{verbatim}
\end{fdesc}

\begin{cdesc}
\begin{verbatim}
void my_ga_app(...., MPI_Comm comm)
{
  if (NGA_Initialize_comm(comm)) {
    /* Main body of code here */
    NGA_Terminate();
  }
}
\end{verbatim}
\end{cdesc}

\begin{desc}
For the progress rank runtime or other runtimes that use a helper process to
manage asynchronous progress, the conditional guarantees that the function or
subroutine exits cleanly. Without the conditional, the progress ranks will
attempt to execute the code after the application has finished, usually with
poor results.
\end{desc}

\seealso{INITIALIZE,INITIALIZE LTD}

\apih{INITIALIZE LTD}{Initializes GA with memory limit}

\begin{capi}
\begin{ccode}
void GA_Initialize_ltd(size_t limit)
\end{ccode}
\begin{funcargs}
\inarg{size_t}{limit}{amount of memory in bytes per process}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_initialize_ltd(limit)
\end{fcode}
\begin{funcargs}
\inarg{integer}{limit}{amount of memory in bytes per process}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GA::Initialize(int argc, char *argv[], unsigned long heapSize,
                    unsigned long stackSize, int type, size_t limit=0)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{argc}{number of command line arguments}
\inarg{char**}{argv}{command line arguments}
\inarg{size_t}{limit}{amount of memory in bytes per process}
\inarg{unsigned long}{heapSize}{all of the dynamically allocated local memory}
\inarg{unsigned long}{stackSize}{all of the dynamically allocated local memory}
\inarg{int}{type}{data type}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.initialize_ltd(size_t limit)
\end{pycode}
\begin{funcargs}
\inarg{size_t}{limit}{amount of memory in bytes per process}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Allocate and initialize internal data structures and set the limit for memory
used in Global Arrays. The limit is per process: it is the amount of memory
that the given processor can contribute to collective allocation of Global
Arrays. It does not include temporary storage that GA might be allocating (and
releasing) during execution of a particular operation.

$*limit < 0$ means ``allow unlimited memory usage'' in which case this operation
is equivalent to GA_initialize.

\end{desc}

\seealso{INITIALIZE,SET MEMORY LIMIT}

\apih{INQUIRE}{Inquires the data type and shape of a global array}

\begin{capi}
\begin{ccode}
void NGA_Inquire(int g_a, int *type, int *ndim, int dims[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int*}{type}{data type}
\outarg{int*}{ndim}{number of dimensions}
\outarg{int*}{dims}{array of dimensions}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_inquire(g_a, type, dim1, dim2)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\outarg{integer}{type}{}
\outarg{integer}{dim1}{}
\outarg{integer}{dim2}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_inquire(g_a, type, ndim, dims)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{integer}{type}{data type id}
\outarg{integer}{ndim}{number of dimensions}
\outarg{integer}{dims(ndim)}{array of dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::inquire(int *type, int *ndim, int dims[]) const
void GlobalArray::inquire(int *type, int *ndim, int64_t dims[]) const
\end{cxxcode}
\begin{funcargs}
\outarg{int*}{type}{data type}
\outarg{int*}{ndim}{number of dimensions}
\outarg{int*}{dims}{array of dimensions}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
type,shape = ga.inquire(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int}{type}{data type}
\outarg{1D ndarray of ints}{shape}{shape of array g_a}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Returns data type and dimensions of the array.

\end{desc}

\apih{INQUIRE MEMORY}{Inquires the memory used by global arrays on the calling processor}

\begin{capi}
\begin{ccode}
size_t GA_Inquire_memory()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_inquire_memory()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
size_t GAServices::inquireMemory()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.inquire_memory()
\end{pycode}
\end{pyapi}

\begin{desc}

Returns the amount of memory (in bytes) used in the allocated global arrays on
the calling processor.

\end{desc}

\apih{INQUIRE NAME}{Inquires a global array's name}

\begin{capi}
\begin{ccode}
char* GA_Inquire_name(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_inquire_name(g_a, array_name)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\outarg{character*(*)}{array_name}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
char* GlobalArray::inquireName() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = inquire_name(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Returns the name of an array represented by the handle g_a.

\end{desc}

\apih{IS MIRRORED}{Checks whether a global array is mirrored}

\begin{capi}
\begin{ccode}
int GA_Is_mirrored(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer ga_is_mirrored(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::isMirrored()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.is_mirrored(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{bool}{ret}{True/False}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}
This subroutine checks if the array is a mirrored array or not. Returns 1 if it
is a mirrored array, else it returns 0.
\end{desc}

\apih{LLT SOLVE}{Cholesky factorization of a global array}

\begin{capi}
\begin{ccode}
int GA_Llt_solve(int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{coefficient matrix}
\inarg{int}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_llt_solve(g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{coefficient matrix}
\inoutarg{integer}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::lltSolve(const GlobalArray * g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{coefficient matrix}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.llt_solve(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{coefficient matrix}
\inarg{int}{g_b}{rhs/solution matrix}
\outarg{int}{ret}{0 if successful}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Solves a system of linear equations
\begin{verbatim}
            A * X = B
\end{verbatim}
using the Cholesky factorization of an NxN double precision symmetric positive
definite matrix A (represented by handle g_a). On successful exit B will
contain the solution X.

It returns:
\begin{verbatim}
         = 0 : successful exit
         > 0 : the leading minor of this order is not positive
               definite and the factorization could
               not be completed.
\end{verbatim}

\end{desc}

\apih{LOCATE}{Locates the processor containing a specified element of a global array}

\begin{capi}
\begin{ccode}
int NGA_Locate(int g_a, int subscript[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{subscript[ndim]}{element subscript}
\outarg{int}{}{process ID owning the element at subscript}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
logical function ga_locate(g_a, i, j, owner)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{i, j}{element subscript}
\outarg{integer}{owner}{process id}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
logical function nga_locate(g_a, subscript, owner)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{subscript}{element subscript}
\outarg{integer}{owner}{process id}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::locate(int subscript[]) const
int GlobalArray::locate(int64_t subscript[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{subscript[ndim]}{element subscript}
\outarg{int}{}{process ID owning the element at subscript}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.locate(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{subscript}{element subscript}
\outarg{int}{ret}{process ID owning the element at subscript}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Return the GA compute process ID that 'owns' the data. If any element
of subscript[] is out of bounds ``-1'' is returned.

\end{desc}

\apih{LOCATE REGION}{Locates a region of a global array}

\begin{capi}
\begin{ccode}
int NGA_Locate_region(int g_a, int lo[], int hi[], int map[], int procs[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\outarg{int*}{map[]}{array with mapping information}
\outarg{int*}{procs[nproc]}{list of processes that own a part of array section}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
logical function ga_locate_region(g_a, ilo, ihi, jlo, jhi, map, np)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ilo, ihi, jlo, jhi}{bounding indices for array section}
\outarg{integer}{map(5,*)}{array containing mapping information}
\outarg{integer}{np}{number of processors containing data}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
logical function nga_locate_region(g_a, lo, hi, map, proclist, np)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{lo(ndim),hi(ndim)}{region(patch) specifications}
\outarg{integer}{map(2*ndim,*)}{patch ownership array}
\outarg{integer}{proclist(np)}{list of processes}
\outarg{integer}{np}{number of processes containing data}
\inarg{}{map(1:ndim,i)}{contains lower bound dimensions for part owned by process proclist(i)}
\inarg{}{map(ndim+1:2*ndim,i)}{contains upper bound dimensions for part owned by process proclist(i)}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::locateRegion(int lo[], int hi[], int map[], int procs[]) const;
int GlobalArray::locateRegion(int64_t lo[], int64_t hi[], int64_t map[], int procs[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\outarg{int*}{map[]}{array with mapping information}
\outarg{int*}{procs[nproc]}{list of processes that own a part of selection}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
map,procs = locate_region(int g_a, lo, hi)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\outarg{2D ndarray of ints}{map}{array with mapping information; map.shape = 2,ndim}
\outarg{1D ndarray of ints}{procs}{list of processes that own a part of array section}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Return a list of the GA processes ID that `own' the data. Parts of the
specified patch might be actually `owned' by several processes. If lo/hi are
out of bounds ``0'' is returned, otherwise the return value is equal to the
number of processes that hold the data.
\end{desc}

\begin{fdesc}
\begin{verbatim}
     map(1:ndim,i)            - lo(1:ndim)
     map(ndim+1:2*ndim,i)     - hi(1:ndim)
     procs(i)                 - processor id that owns data in patch
                                described by lo,hi
\end{verbatim}
\end{fdesc}

\begin{cdesc}
\begin{verbatim}
     map[n*2*ndim+i]         - lo[i] for block n
     map[n*2*ndim+ndim+i]    - hi[i] for block n
     procs[n]                - processor id that owns data in patch
                               described by lo,hi
\end{verbatim}
\end{cdesc}

\begin{cxxdesc}
\begin{verbatim}
     map[n*2*ndim+i]         - lo[i] for block n
     map[n*2*ndim+ndim+i]    - hi[i] for block n
     procs[n]                - processor id that owns data in patch
                               described by lo,hi
\end{verbatim}
\end{cxxdesc}

For block cyclic layouts, the indices returned in the procs array are
actually the block indices of the individual blocks that overlap with
the bounding indices specified by lo and hi. For block cyclic layouts,
the number of blocks n can exceed the number of processors and the
block indices in procs can also be larger than the number of processors.


\seealso{LOCATE}

\apih{LOCK}{Locks a specific mutex}

\begin{capi}
\begin{ccode}
void GA_Lock(int mutex)
\end{ccode}
\begin{funcargs}
\inarg{int}{mutex}{mutex object id}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_lock(mutex)
\end{fcode}
\begin{funcargs}
\inarg{integer}{mutex}{mutex id}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::lock(int mutex)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{mutex}{mutex object id}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.lock(int mutex)
\end{pycode}
\end{pyapi}

\ncoll

\begin{desc}

Locks a mutex object identified by the mutex number. It is a fatal error for a
process to attempt to lock a mutex which was already locked by this process.

\end{desc}

\seealso{CREATE MUTEXES,DESTROY MUTEXES,UNLOCK}

\apih{LU SOLVE}{LU decomposition of a global array}

\begin{capi}
\begin{ccode}
void GA_Lu_solve(char trans, int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{char}{trans}{transpose or not transpose}
\inarg{int}{g_a}{coefficient matrix}
\inarg{int}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_lu_solve(trans, g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{character}{trans}{transpose or not transpose}
\inarg{integer}{g_a}{coefficient matrix}
\inoutarg{integer}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::luSolve(char trans, const GlobalArray * g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{}{trans}{transpose or not transpose}
\inarg{}{g_a}{coefficient matrix}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.lu_solve(int g_a, int g_b, bint trans=False)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle for the coefficient matrix}
\inarg{int}{g_b}{the array handle for the solution matrix}
\inarg{bool}{trans}{transpose (True) or not transpose (False)}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Solve the system of linear equations op(A)X = B based on the LU factorization.

op(A) = A or A' depending on the parameter trans:
\begin{verbatim}
     trans = `N' or `n' means that the transpose operator should not be applied.
     trans = `T' or `t' means that the transpose operator should be applied.
\end{verbatim}

Matrix A is a general real matrix. Matrix B contains possibly multiple rhs
vectors.  The array associated with the handle g_b is overwritten by the
solution matrix X.

\end{desc}

\apih{MASK SYNC}{Masks GA synchronization operations}

\begin{capi}
\begin{ccode}
void GA_Mask_sync(int first, int last)
\end{ccode}
\begin{funcargs}
\inarg{int}{first}{mask (0/1) for prior internal synchronization}
\inarg{int}{last}{mask (0/1) for post internal synchronization}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_mask_sync(first,last)
\end{fcode}
\begin{funcargs}
\inarg{logical}{first}{mask for prior internal synchronization}
\inarg{logical}{last}{mask for post internal synchronization}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GAServices::maskSync(int first, int last)
void GA::maskSync(int first, int last)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{first}{masks the sync at the begining of the collective call}
\inarg{int}{last}{masks the sync at the end of the collective call}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.mask_sync(int first, int last)
\end{pycode}
\end{pyapi}
\begin{funcargs}
\inarg{bool}{first}{mask for prior internal synchronization}
\inarg{bool}{last}{mask for post internal synchronization}
\end{funcargs}

\dcoll

\begin{desc}

This subroutine can be used to remove synchronization calls from around
collective operations. Setting the parameter first = .false. removes the
synchronization prior to the collective operation, setting last = .false.
removes the synchronization call after the collective operation. This call is
applicable to all collective operations.  It most be invoked before each
collective operation.

\end{desc}

\seealso{SYNC}

\apih{MATMUL PATCH}{Performs matrix multiplication of patches of global arrays}

\begin{capi}
\begin{ccode}
void GA_Matmul_patch (char transa, char transb, void* alpha, void *beta,
                      int g_a, int ailo, int aihi, int ajlo, int ajhi,
                      int g_b, int bilo, int bihi, int bjlo, int bjhi,
                      int g_c, int cilo, int cihi, int cjlo, int cjhi)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b, g_c}{array handles}
\inarg{int}{ailo, aihi, ajlo, ajhi}{patch of g_a}
\inarg{int}{bilo, bihi, bjlo, bjhi}{patch of g_b}
\inarg{int}{cilo, cihi, cjlo, cjhi}{patch of g_c}
\inarg{void*}{alpha, beta}{scale factors}
\inarg{char}{transa, transb}{transpose operators}
\end{funcargs}
\end{capi}

\begin{capi}
\begin{ccode}
void NGA_Matmul_patch(char transa, char transb, void* alpha, void *beta,
                      int g_a, int alo[], int ahi[],
                      int g_b, int blo[], int bhi[],
                      int g_c, int clo[], int chi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a, g_b, g_c}{array handles}
\inarg{int*}{alo, ahi}{patch of g_a}
\inarg{int*}{blo, bhi}{patch of g_b}
\inarg{int*}{clo, chi}{patch of g_c}
\inarg{void*}{alpha, beta}{scale factors}
\inarg{char}{transa, transb}{transpose operators}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_matmul_patch (transa, transb, alpha, beta,
                            g_a, ailo, aihi, ajlo, ajhi,
                            g_b, bilo, bihi, bjlo, bjhi,
                            g_c, cilo, cihi, cjlo, cjhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, ailo, aihi, ajlo, ajhi}{patch of g_a}
\inarg{integer}{g_b, bilo, bihi, bjlo, bjhi}{patch of g_b}
\inarg{integer}{g_c, cilo, cihi, cjlo, cjhi}{patch of g_c}
\inarg{double precision/complex}{alpha, beta}{scale factors}
\inarg{character*1}{transa, transb}{transpose operators}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_matmul_patch(transa, transb, alpha, beta,
                            g_a, alo, ahi,
                            g_b, blo, bhi,
                            g_c, clo, chi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a, alo, ahi}{patch of g_a}
\inarg{integer}{g_b, blo, bhi}{patch of g_b}
\inarg{integer}{g_c, clo, chi}{patch of g_c}
\inarg{double precision/complex}{alpha, beta}{scale factors}
\inarg{character*1}{transa, transb}{transpose operators}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::matmulPatch(char transa, char transb,
                              void* alpha, void *beta, const GlobalArray *g_a,
                              int ailo, int aihi, int ajlo, int ajhi,
                              const GlobalArray *g_b, int bilo, int bihi,
                              int bjlo, int bjhi, int cilo, int cihi,
                              int cjlo, int cjhi) const;
void GlobalArray::matmulPatch(char transa, char transb,
                              void* alpha, void *beta, const GlobalArray *g_a,
                              int64_t ailo, int64_t aihi, int64_t ajlo,
                              int64_t ajhi, const GlobalArray *g_b, int64_t
                              bilo, int64_t bihi, int64_t bjlo, int64_t bjhi,
                              int64_t cilo, int64_t cihi, int64_t cjlo,
                              int64_t cjhi) const
\end{cxxcode}
\begin{funcargs}
\inarg{char}{transa}{transpose operators}
\inarg{char}{transb}{transpose operators}
\inarg{int}{g_a}{global array}
\inarg{int}{g_b}{global array}
\inarg{int}{ailo}{patch of g_a}
\inarg{int}{aihi}{patch of g_a}
\inarg{int}{ajlo}{patch of g_a}
\inarg{int}{ajhi}{patch of g_a}
\inarg{int}{bilo}{patch of g_b}
\inarg{int}{bihi}{patch of g_b}
\inarg{int}{bjlo}{patch of g_b}
\inarg{int}{bjhi}{patch of g_b}
\inarg{int}{cilo}{patch of g_c}
\inarg{int}{cihi}{patch of g_c}
\inarg{int}{cjlo}{patch of g_c}
\inarg{int}{cjhi}{patch of g_c}
\inarg{void*}{alpha}{scale factors}
\inarg{void*}{beta}{scale factors}
\end{funcargs}
\end{cxxapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::matmulPatch(char transa, char transb, void* alpha,
                              void *beta,const GlobalArray *g_a,
                              int *alo, int *ahi, const GlobalArray *g_b,
                              int *blo, int *bhi, int *clo, int *chi) const
void GlobalArray::matmulPatch(char transa, char transb, void* alpha,
                              void *beta, const GlobalArray *g_a,
                              int64_t *alo, int64_t *ahi, const GlobalArray
                              *g_b, int64_t *blo, int64_t *bhi,
                              int64_t *clo, int64_t *chi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{global array}
\inarg{int}{g_b}{global array}
\inarg{int*}{alo}{array of patch of g_a}
\inarg{int*}{ahi}{array of patch of g_a}
\inarg{int*}{blo}{array of patch of g_b}
\inarg{int*}{bhi}{array of patch of g_b}
\inarg{int*}{clo}{array of patch of g_c}
\inarg{int*}{chi}{array of patch of g_c}
\inarg{void*}{alpha}{scale factors}
\inarg{void*}{beta}{scale factors}
\inarg{char}{transa}{transpose operators}
\inarg{char}{transb}{transpose operators}
\end{funcargs}
\end{cxxapi}


\begin{pyapi}
\begin{pycode}
matmul_patch(bint transa, bint transb, alpha, beta,
             int g_a, alo, ahi,
             int g_b, blo, bhi,
             int g_c, clo, chi)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a, g_b, g_c}{array handles}
\inarg{1D array-like of ints}{alo, ahi}{patch of g_a}
\inarg{1D array-like of ints}{blo, bhi}{patch of g_b}
\inarg{1D array-like of ints}{clo, chi}{patch of g_c}
\inarg{object}{alpha, beta}{scale factors}
\inarg{bool}{transa, transb}{transpose operators}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

ga_matmul_patch is a patch version of ga_dgemm and comes in 2-D and N-D
versions. The 2-D interface performs the operation:
\begin{verbatim}
         C[cilo:cihi,cjlo:cjhi] := alpha* AA[ailo:aihi,ajlo:ajhi] *
                                   BB[bilo:bihi,bjlo:bjhi] ) +
                                   beta*C[cilo:cihi,cjlo:cjhi],
\end{verbatim}

where AA = op(A), BB = op(B), and op(X) is one of
\begin{verbatim}
      op(X) = X   or   op(X) = X',
\end{verbatim}

Valid values for transpose arguments: 'n', 'N', 't', 'T'. It works for both
double and double complex data tape.

nga_matmul_patch is a N-dimensional patch version of ga_dgemm and is similar to
the 2-D interface:
\begin{verbatim}
      C[clo[]:chi[]] := alpha* AA[alo[]:ahi[]] *
                               BB[blo[]:bhi[]] ) + beta*C[clo[]:chi[]],
\end{verbatim}

\end{desc}

\seealso{GEMM}

\apih{MEDIAN}{Computes the element-wise median of three arrays}

\begin{capi}
\begin{ccode}
void GA_Median(int g_a, int g_b, int g_c, int g_m)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{input array handle}
\inarg{int}{g_b}{input array handle}
\inarg{int}{g_c}{input array handle}
\inarg{int}{g_m}{output array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_median(g_a, g_b, g_c, g_m)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{input array handle}
\inarg{integer}{g_b}{input array handle}
\inarg{integer}{g_c}{input array handle}
\inarg{integer}{g_m}{output array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::median(const GlobalArray * g_a,
                         const GlobalArray * g_b,
                         const GlobalArray * g_c) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{global array}
\inarg{const GlobalArray*}{g_b}{global array}
\inarg{const GlobalArray*}{g_c}{global array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.median(int g_a, int g_b, int g_c, int g_m)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{int}{g_m}{the array handle for the result}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the componentwise Median of three arrays g_a, g_b, and g_c, and stores
the result in this array g_m.  The result (m) may replace one of the input
arrays (a/b/c).
\end{desc}

\apih{MEDIAN PATCH}{Computes the element-wise median of three global array patches}

\begin{capi}
\begin{ccode}
void GA_Median_patch(int g_a, int alo[], int ahi[],
                     int g_b, int blo[], int bhi[],
                     int g_c, int clo[], int chi[],
                     int g_m, int mlo[], int mhi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{input array handle}
\inarg{int}{g_b}{input array handle}
\inarg{int}{g_c}{input array handle}
\inarg{int}{g_m}{output array handle}
\inarg{int*}{alo[],ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[],bhi[]}{g_b patch coordinates}
\inarg{int*}{clo[],chi[]}{g_c patch coordinates}
\inarg{int*}{mlo[],mhi[]}{g_m patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_median_patch(g_a, alo, ahi,
                           g_b, blo, bhi,
                           g_c, clo, chi,
                           g_m, mlo, mhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{input array handle}
\inarg{integer}{g_b}{input array handle}
\inarg{integer}{g_c}{input array handle}
\inarg{integer}{g_m}{output array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim),ahi(ndim)}{g_a patch dimensions}
\inarg{integer}{blo(ndim),bhi(ndim)}{g_b patch dimensions}
\inarg{integer}{clo(ndim),chi(ndim)}{g_c patch dimensions}
\inarg{integer}{mlo(ndim),mhi(ndim)}{g_m patch dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::medianPatch(
        const GlobalArray *g_a, int *alo, int *ahi,
        const GlobalArray *g_b, int *blo, int *bhi,
        const GlobalArray *g_c, int *clo, int *chi,
        int *mlo, int *mhi) const;
void GlobalArray::medianPatch(
        const GlobalArray *g_a, int64_t *alo, int64_t *ahi,
        const GlobalArray *g_b, int64_t *blo, int64_t *bhi,
        const GlobalArray *g_c, int64_t *clo, int64_t *chi,
        int64_t *mlo, int64_t *mhi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{global array}
\inarg{int}{g_b}{global array}
\inarg{int}{g_c}{global array}
\inarg{int*}{alo}{g_a lower corner patch coordinates}
\inarg{int*}{ahi}{g_a upper corner patch coordinates}
\inarg{int*}{blo}{g_b lower corner patch coordinates}
\inarg{int*}{bhi}{g_b upper corner patch coordinates}
\inarg{int*}{clo}{g_c lower corner patch coordinates}
\inarg{int*}{chi}{g_c upper corner patch coordinates}
\inarg{int*}{mlo}{g_m lower corner patch coordinates}
\inarg{int*}{mhi}{g_m upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.median(int g_a, int g_b, int g_c, int g_m,
          alo=None, ahi=None, blo=None, bhi=None,
          clo=None, chi=None, mlo=None, mhi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{int}{g_b}{the array handle}
\inarg{int}{g_c}{the array handle}
\inarg{int}{g_m}{the array handle for the result}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\inarg{1D array-like of integers}{clo}{lower bound patch coordinates of g_c, inclusive}
\inarg{1D array-like of integers}{chi}{higher bound patch coordinates of g_c, exclusive}
\inarg{1D array-like of integers}{mlo}{lower bound patch coordinates of g_m, inclusive}
\inarg{1D array-like of integers}{mhi}{higher bound patch coordinates of g_m, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the componentwise Median of three patches g_a, g_b, and g_c, and
stores the result in this patch g_m.  The result (m) may replace one of the
input patches (a/b/c).
\end{desc}

\seealso{MEDIAN}

\apih{MEMORY AVAIL}{Inquires about memory available on the invoking processor to allocate global arrays}

\begin{capi}
\begin{ccode}
size_t GA_Memory_avail()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_memory_avail()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::memoryAvailable() ;
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.memory_avail()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

Returns amount of memory (in bytes) left for allocation of new global arrays on
the calling processor.

Note: If GA_uses_ma returns true, then GA_Memory_avail returns the lesser of
the amount available under the GA limit and the amount available from MA
(according to ma_inquire_avail operation). If no GA limit has been set, it
returns what MA says is available.

If ( !GA_Uses_ma() \&\& !GA_Memory_limited() ) returns $< 0$, indicating
that the bound on currently available memory cannot be determined.

\end{desc}

\apih{MEMORY LIMITED}{Checks whether memory available to GA's runtime is limited}

\begin{capi}
\begin{ccode}
int GA_Memory_limited()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_memory_limited()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::memoryLimited()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.memory_limited()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

Indicates if limit is set on memory usage in Global Arrays on the calling
processor.  ``1'' means ``yes'', ``0'' means ``no''.

Returns: True for ``yes'', False for ``no''

\end{desc}

\apih{MERGE DISTR PATCH}{Merges a patched of a mirrored global array}

\begin{capi}
\begin{ccode}
int NGA_Merge_distr_patch(int g_a, int alo[], int ahi[],
                          int g_b, int blo[], int bhi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle}
\inarg{int*}{alo[],ahi[]}{g_a patch coordinates}
\inarg{int*}{blo[],bhi[]}{g_b patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer nga_merge_distr_patch(g_a, alo, ahi, g_b, blo, bhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_b}{array handles}
\inarg{integer}{alo(ndim),ahi(ndim)}{g_a patch coordinates}
\inarg{integer}{blo(ndim),bhi(ndim)}{g_b patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::mergeDistrPatch(
        int alo[], int ahi[], GlobalArray *g_a,
        int blo[], int bhi[])
void GlobalArray::mergeDistrPatch(
        int64_t alo[], int64_t ahi[], GlobalArray *g_a,
        int64_t blo[], int64_t bhi[])
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{alo[ndim]}{patch indices of mirrored array}
\inarg{int*}{ahi[ndim]}{patch indices of mirrored array}
\inarg{int*}{blo[ndim]}{patch indices of result array}
\inarg{int*}{bhi[ndim]}{patch indices of result array}
\inarg{GlobalArray*}{g_a}{global array containing result}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.merge_distr_patch(int g_a, alo, ahi, int g_b, blo, bhi)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of integers}{alo}{g_a patch coordinate}
\inarg{1D array-like of integers}{ahi}{g_a patch coordinate}
\inarg{int}{g_b}{array handle}
\inarg{1D array-like of integers}{blo}{g_b patch coordinate}
\inarg{1D array-like of integers}{bhi}{g_b patch coordinate}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
This function merges all copies of a patch of a mirrored array (g_a) into a
patch in a distributed array (g_b).
\end{desc}

\seealso{MERGE MIRRORED}

\apih{MERGE MIRRORED}{Merges a mirrored global array}

\begin{capi}
\begin{ccode}
int GA_Merge_mirrored(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handles}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_merge_mirrored(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::mergeMirrored()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.merge_mirrored(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handles}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
This subroutine merges mirrored arrays by adding the contents of each array
across nodes. The result is that each mirrored copy of the array represented by
g_a is the sum of the individual arrays before the merge operation. After the
merge, all mirrored arrays are equal.
\end{desc}

\apih{NBACC}{Accumulates data into a global array without blocking the calling process}

\begin{capi}
\begin{ccode}
void NGA_NbAcc(int g_a, int lo[], int hi[], void* buf, int ld[],
               void *alpha, ga_nbhdl_t* nbhandle)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{void*}{buf}{pointer to the local buffer array where the data is}
\inarg{int}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{void*}{alpha}{double/double complex/long* scale factor}
\outarg{}{nbhandle}{pointer to the non-blocking request handle}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_nbacc(g_a, ilo, ihi, jlo, jhi, buf, ld, alpha, nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ilo,ihi}{starting indices for global array section}
\inarg{integer}{jlo,jhi}{ending indices for global array section}
\inarg{}{buf}{local buffer array where the data is}
\inarg{integer}{ld}{leading dimension/stride/extent for buffer array}
\inarg{double precision/complex}{alpha}{scale factor}
\outarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_nbacc(g_a, lo, hi, buf, ld, alpha, nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo[ndim]}{array of starting indices for global array section}
\inarg{integer}{hi[ndim]}{array of ending indices for global array section}
\inarg{}{buf}{local buffer array where the data is}
\inarg{integer}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{double precision/complex}{alpha}{scale factor}
\outarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::nbAcc(int lo[], int hi[],
        void *buf, int ld[], void *alpha, GANbhdl *nbhandle)
void GlobalArray::nbAcc(int64_t lo[], int64_t hi[],
        void *buf, int64_t ld[], void *alpha, GANbhdl *nbhandle)
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{patch coordinates of block}
\inarg{int*}{hi[ndim]}{patch coordinates of block}
\inarg{void*}{buf}{local buffer to receive data}
\inarg{int*}{ld[ndim-1]}{array of strides for local data}
\inarg{void*}{alpha}{multiplier for data before adding to existing results}
\outarg{GANbhdl*}{nbhandle}{nonblocking handle}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
nbhandle = ga.acc(int g_a, buffer, lo=None, hi=None, alpha=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{object}{alpha}{scale factor, cast to the appropriate type}
\outarg{ga_nbhdl_t}{nbhandle}{nonblocking handle}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}
A non-blocking version of the blocking accumulate operation. The accumulate
operation can be completed locally by making a call to the wait (e.g.,
NGA_NbWait) routine.

Non-blocking version of ga.acc.

The accumulate operation can be completed locally by making a call to the
ga.nbwait() routine.

Combines data from buffer with data in the global array patch.

The buffer array is assumed to be have the same number of dimensions as the global array. If the buffer is not contiguous, a contiguous copy will be made.

global array section (lo[],hi[]) += alpha * buffer
\end{desc}

\seealso{ACC}

\apih{NBGET}{Gets data from a global array without blocking the calling process}

\begin{capi}
\begin{ccode}
void NGA_NbGet(int g_a, int lo[], int hi[],
               void* buf, int ld[], ga_nbhdl_t* nbhandle)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inoutarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\outarg{ga_nbhdl_t*}{nbhandle}{pointer to the non-blocking request handle}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_nbget(g_a, ilo, ihi, jlo, jhi, buf, ld, nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ilo,ihi}{starting indices for global array section}
\inarg{integer}{jlo,jhi}{ending indices for global array section}
\inoutarg{type}{buf}{local buffer array where the data goes}
\inarg{integer}{ld}{leading dimension/stride/extent for buffer array}
\outarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_nbget(g_a, lo, hi, buf, ld, nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{lo[ndim]}{array of starting indices for global array section}
\inarg{integer}{hi[ndim]}{array of ending indices for global array section}
\inoutarg{type}{buf}{local buffer array where the data goes}
\inarg{integer}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\outarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::nbGet(int lo[], int hi[],
                        void *buf, int ld[], GANbhdl *nbhandle)
void GlobalArray::nbGet(int64_t lo[], int64_t hi[],
                        void *buf, int64_t ld[], GANbhdl *nbhandle)
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{patch coordinates of block}
\inarg{int*}{hi[ndim]}{patch coordinates of block}
\inoutarg{void*}{buf}{local buffer to receive data}
\inarg{int*}{ld[ndim-1]}{array of strides for local data}
\outarg{GANbhdl*}{nbhandle}{nonblocking handle}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret,nbhandle = ga.nbget(int g_a, lo=None, hi=None, ndarray buffer=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{ndarray}{buffer}{an ndarray of the appropriate type, large enough to hold lo,hi}
\inoutarg{ndarray}{ret}{local buffer array where the data goes to}
\outarg{ga_nbhdl_t}{nbhandle}{non-blocking handle}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}
A non-blocking version of the blocking get operation. The get operation can be
completed locally by making a call to the wait (e.g., NGA_NbWait) routine.

Copies data from global array section to the local array buffer.

The local array is assumed to be have the same number of dimensions as the
global array. Any detected inconsitencies/errors in the input arguments are
fatal.

Returns: The local array buffer.
\end{desc}

\seealso{GET}

\apih{NBLOCK}{Inquires the number of blocks along each dimension of a global array}

\begin{capi}
\begin{ccode}
void GA_Nblock(int g_a, int nblock[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int*}{nblock[ndim]}{number of partitions for each dimension}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_nblock(g_a, nblock)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{integer}{nblock[ndim]}{number of partitions for each dimension}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::nblock(int nblock[]) const
\end{cxxcode}
\begin{funcargs}
\outarg{int*}{nblock[ndim]}{number of partitions for each dimension}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.nblock(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{1D ndarray of ints}{ret}{number of partitions for each dimension}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Given a distribution of an array represented by the handle g_a, returns the
number of partitions of each array dimension.

\end{desc}

\apih{NBPUT}{Puts data into a global array without blocking the calling process}

\begin{capi}
\begin{ccode}
void NGA_NbPut(int g_a, int lo[], int hi[],
               void* buf, int ld[], ga_nbhdl_t* nbhandle)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{void*}{buf}{pointer to the local buffer array where the data is}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\outarg{ga_nbhdl_t*}{nbhandle}{pointer to the non-blocking request handle}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_nbput(g_a, ilo, ihi, jlo, jhi, buf, ld, nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ilo,ihi}{starting indices for global array section}
\inarg{integer}{jlo,jhi}{ending indices for global array section}
\inarg{}{typebuf}{local buffer array where the data is}
\inarg{integer}{ld}{leading dimension/stride/extent for buffer array}
\outarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_nbput(g_a, lo, hi, buf, ld, nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{lo[ndim]}{array of starting indices for global array section}
\inarg{integer}{hi[ndim]}{array of ending indices for global array section}
\inarg{}{typebuf}{local buffer array where the data is}
\inarg{integer}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\outarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::nbPut(int lo[], int hi[],
                        void *buf, int ld[], GANbhdl *nbhandle)
void GlobalArray::nbPut(int64_t lo[], int64_t hi[],
                        void *buf, int64_t ld[], GANbhdl *nbhandle)
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{patch coordinates of block}
\inarg{int*}{hi[ndim]}{patch coordinates of block}
\inarg{void*}{buf}{local buffer to receive data}
\inarg{int*}{ld[ndim-1]}{array of strides for local data}
\outarg{GANbhdl*}{nbhandle}{nonblocking handle}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
nbhandle = ga.nbput(int g_a, buffer, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\outarg{ga_nbhdl_t}{nbhandle}{non-blocking handle}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}
A non-blocking version of the blocking put operation. The put operation can be
completed locally by making a call to the wait (e.g., NGA_NbWait) routine.

Copies data from local array buffer to the global array section.

The local array is assumed to be have the same number of dimensions as the
global array. Any detected inconsitencies/errors in input arguments are fatal.
\end{desc}

\seealso{PUT}

\apih{NBTEST}{Tests a non-blocking GA operation for completion}

\begin{capi}
\begin{ccode}
int NGA_NbTest(ga_nbhdl_t* nbhandle)
\end{ccode}
\begin{funcargs}
\inarg{ga_nbhdl_t*}{nbhandle}{pointer to the non-blocking request handle}
\outarg{int}{}{1 if operation has completed}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_nbtest(nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{nbhandle}{non-blocking request handle}
\outarg{logical}{}{.TRUE. if operation has completed}
\end{funcargs}
\end{fapi}

\begin{fapi}
\begin{fcode}
logical function nga_nbtest(nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{nbhandle}{non-blocking request handle}
\outarg{logical}{}{.TRUE. if operation has completed}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::nbTest(GANbhdl *nbhandle)
\end{cxxcode}
\begin{funcargs}
\inarg{GANbhdl*}{nbhandle}{nonblocking handle}
\outarg{int}{}{1 if operation has completed}
\end{funcargs}
\end{cxxapi}

\ncoll

\begin{desc}
This function tests a non-blocking one-sided operation for completion. If true,
the function is completed locally and the buffer is available for either use or
reuse, depending on the operation. Once this operation has returned true, there
is no need to call nbwait on the handle. The test function is properly
implemented only on the Progress Ranks and RMA runtimes. For other runtimes, it
defaults to the non-blocking wait function and always returns true.
\end{desc}

\seealso{NBWAIT,NBGET,NBPUT,NBACC}

\apih{NBWAIT}{Waits for a non-blocking GA operation to complete}

\begin{capi}
\begin{ccode}
void NGA_NbWait(ga_nbhdl_t* nbhandle)
\end{ccode}
\begin{funcargs}
\inarg{ga_nbhdl_t*}{nbhandle}{pointer to the non-blocking request handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_nbwait(nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{fapi}

\begin{fapi}
\begin{fcode}
subroutine nga_nbwait(nbhandle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{nbhandle}{non-blocking request handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GAServices::nbWait(GANbhdl *nbhandle)
\end{cxxcode}
\begin{funcargs}
\inarg{GANbhdl*}{nbhandle}{nonblocking handle}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.nbwait(ga_nbhdl_t nbhandle)
\end{pycode}
\end{pyapi}

\ncoll

\begin{desc}
This function completes a non-blocking one-sided operation locally. Waiting on
a nonblocking put or an accumulate operation assures that data was injected
into the network and the user buffer can now be reused. Completing a get
operation assures data has arrived into the user memory and is ready for use.
The wait operation ensures only local completion. Not all runtimes support true
non-blocking capability. For those that don't all operations are blocking and
the wait function is a no-op.

Unlike their blocking counterparts, the nonblocking operations are not ordered
with respect to the destination. Performance being one reason, the other reason
is that ensuring ordering would incur additional and possibly unnecessary
overhead on applications that do not require their operations to be ordered.
For cases where ordering is necessary, it can be done by calling a fence
operation. The fence operation is provided to the user to confirm remote
completion if needed.
\end{desc}

\seealso{NBTEST,NBGET,NBPUT,NBACC}

\apih{NDIM}{Inquires the number of dimensions in a global array}

\begin{capi}
\begin{ccode}
int GA_Ndim(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_ndim(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::ndim() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.ndim(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Returns the number of dimensions in the array represented by the handle g_a.

\end{desc}

\apih{NNODES}{Returns the total number of GA ranks}

\begin{capi}
\begin{ccode}
int GA_Nnodes()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_nnodes()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::nodes()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.nnodes()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

Returns the number of the GA compute (user) processes.

\end{desc}

\apih{NODEID}{Returns the GA rank of the invoking process}

\begin{capi}
\begin{ccode}
int GA_Nodeid()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_nodeid()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GAServices::nodeid()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.nodeid()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

Returns the GA process id (0, ..., ga_Nnodes()-1) of the requesting compute
process.

\end{desc}

\apih{NORM INFINITY}{Computes a global array's infinite norm}

\begin{capi}
\begin{ccode}
void GA_Norm_infinity(int g_a, double *nm)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{double*}{nm}{matrix/vector infinity-norm value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_norm_infinity(g_a, nm)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{double precision}{nm}{matrix/vector infinity-norm value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::normInfinity(double *nm) const
\end{cxxcode}
\begin{funcargs}
\outarg{double*}{nm}{matrix/vector infinity-norm value}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.norm_infinity(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{double*}{ret}{matrix/vector infinity-norm value}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the infinity-norm of the matrix or vector g_a.
\end{desc}

\apih{NORM1}{Computes a global array's 1-norm}

\begin{capi}
\begin{ccode}
void GA_Norm1(int g_a, double *nm)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{double*}{nm}{matrix/vector 1-norm value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_norm1(g_a, nm)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{double precision}{nm}{matrix/vector 1-norm value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::norm1(double *nm) const
\end{cxxcode}
\begin{funcargs}
\outarg{double*}{nm}{matrix/vector 1-norm value}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.norm1(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{double}{ret}{matrix/vector 1-norm value}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Computes the 1-norm of the matrix or vector g_a.
\end{desc}

\apih{OVERLAY}{Create a global array on top of an existing allocation}

\begin{capi}
\begin{ccode}
int GA_Overlay(int g_a, int g_p)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{handle for overlay array}
\inarg{int}{g_p}{handle for parent array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_overlay(g_a, g_p)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{handle for overlay array}
\outarg{integer}{g_p}{handle for parent array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::overlay(const GlobalArray *g_p)
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_p}{parent array}
\end{funcargs}
\end{cxxapi}

\gcoll

\begin{desc}
The overlay function is designed to allow users to create a global array on top
of an existing global array. This can be used in situations where it is
desirable to create and destroy a large number of global arrays in rapid
succession and where the size of these global arrays can be bounded beforehand.
A large global array is created at the start using conventional create or
allocate calls and then is used as the parent for new global arrays that are
allocated using the overlay call. This approach removes some collectives and
memory allocation and deallocation calls from the process of creating a global
array and should result in improved performance. Note that the parent global
array should not be used while another array is overlayed on top of it. 
\end{desc}

\gcoll

\seealso{ALLOCATE}

\apih{PACK}{Compresses values from a source vector based on a corresponding integer mask}

\begin{capi}
\begin{ccode}
void GA_Pack(int g_src, int g_dest, int g_mask,
             int lo, int hi, int *icount)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source array}
\inarg{int}{g_dest}{handle for destination array}
\inarg{int}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo,hi}{low and high values of range on which operation is performed}
\outarg{int*}{icount}{number of values in compressed array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_pack(g_src, g_dest, g_mask, lo, hi, icount)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_src}{handle for source array}
\outarg{integer}{g_dest}{handle for destination array}
\inarg{integer}{g_mask}{handle for integer array representing a bit mask}
\inarg{integer}{lo,hi}{low and high values of range on which operation is performed}
\outarg{integer}{icount}{number of values in compressed array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::pack(const GlobalArray *g_dest,
        const GlobalArray *g_mask, int lo, int hi,
        int *icount) const
void GlobalArray::pack(const GlobalArray *g_dest,
        const GlobalArray *g_mask, int64_t lo, int64_t hi,
        int64_t *icount) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_dest}{destination array}
\inarg{const GlobalArray*}{g_mask}{mask array}
\inarg{int}{lo}{coordinate interval to pack}
\inarg{int}{hi}{coordinate interval to pack}
\outarg{int*}{icount}{number of packed elements}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.pack(int g_src, int g_dst, int g_msk, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source arrray}
\inarg{int}{g_dst}{handle for destination array}
\inarg{int}{g_msk}{handle for integer array representing mask}
\inarg{1D array-like of ints}{lo}{low value of range on which operation is performed}
\inarg{1D array-like of ints}{hi}{hi value of range on which operation is performed}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
The pack subroutine is designed to compress the values in the source vector
g_src into a smaller destination array g_dest based on the values in an integer
mask array g_mask. The values lo and hi denote the range of elements that
should be compressed and icount is a variable that on output lists the number
of values placed in the compressed array. This operation is the complement of
the GA_Unpack operation. An example is shown below
\begin{verbatim}
GA_Pack(g_src, g_dest, g_mask, 1, n, &icount);

g_mask:   1  0  0  0  0  0  1  0  1  0  0  1  0  0  1  1  0
g_src:    1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17
g_dest:   1  7  9 12 15 16
icount:   6
\end{verbatim}

The current implementation requires that the distribution of the g_mask array
matches the distribution of the g_src array.
\end{desc}

\apih{PATCH ENUM}{Enumerates a global array patch}

\begin{capi}
\begin{ccode}
void GA_Patch_enum(int g_a, int lo, int hi, void *start, void *inc)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{lo,hi}{patch coordinates}
\inarg{void*}{start}{starting value of enumeration}
\inarg{void*}{inc}{increment value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_patch_enum(g_a, lo, hi, start, inc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{lo,hi}{low and high values of array patch}
\inarg{}{start}{integer/double precision/complex starting value of enumeration}
\inarg{}{inc}{integer/double precision/complex increment value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::patchEnum(int lo, int hi, void *istart, void *inc)
void GlobalArray::patchEnum(int64_t lo, int64_t hi, void *start, void *inc)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{lo}{coordinate interval to enumerate}
\inarg{int}{hi}{coordinate interval to enumerate}
\inarg{void*}{istart}{starting value of enumeration}
\inarg{void*}{inc}{increment value}
\end{funcargs}
\end{cxxapi}

\gcoll

\begin{desc}
This subroutine enumerates the values of an array between elements lo and hi
starting with the value start and incrementing each subsequent value by inc.
This operation is only applicable to 1-dimensional arrays. An example of its
use is shown below:
\begin{verbatim}
GA_Patch_enum(g_a, 1, n, 7, 2);

g_a:  7  9 11 13 15 17 19 21 23 ...
\end{verbatim}
\end{desc}

\apih{PERIODIC ACC}{Accumulates data into a global array using periodic boundary conditions}

\begin{capi}
\begin{ccode}
void NGA_Periodic_acc(int g_a, int lo[], int hi[], void* buf, int ld[],
                      void* alpha)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\inarg{void*}{buf}{pointer to the local buffer array}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{double/double complex/long*}{alpha}{scale factor}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_periodic_acc(g_a, lo, hi, buf, ld, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer array where the local data is}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\inarg{type}{alpha}{scale argument for accumulate}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::periodicAcc(int lo[], int hi[], void* buf,
                              int ld[], void* alpha) const
void GlobalArray::periodicAcc(int64_t lo[], int64_t hi[], void* buf,
                              int64_t ld[], void* alpha) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\inarg{void*}{buf}{pointer to the local buffer array}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{}{alpha}{double/double complex/long scale factor}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.periodic_acc(int g_a, buffer, lo=None, hi=None, alpha=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{object}{alpha}{scale factor, cast to the appropriate type}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Same as nga_acc except the indices can extend beyond the array
boundary/dimensions in which case the library wraps them around. For Python,
this is the periodic version of ga.acc.

Combines data from buffer with data in the global array patch.

The buffer array is assumed to be have the same number of dimensions as the
global array. If the buffer is not contiguous, a contiguous copy will be made.

global array section (lo[],hi[]) += alpha * buffer

\end{desc}

\seealso{ACC}

\apih{PERIODIC GET}{Gets data from a global array using periodic boundary conditions}

\begin{capi}
\begin{ccode}
void NGA_Periodic_get(int g_a, int lo[], int hi[], void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\outarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_periodic_get(g_a, lo, hi,  buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer array where the data goes  to}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::periodicGet(int lo[], int hi[], void* buf, int ld[]) const
void GlobalArray::periodicGet(int64_t lo[], int64_t hi[], void* buf,
								  int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\outarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.periodic_get(int g_a, lo, hi, buffer, alpha=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{ndarray}{buffer}{an ndarray of the appropriate type, large enough to hold lo,hi}
\outarg{ndarray}{ret}{local buffer array where the data goes to}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Same as nga_get except the indices can extend beyond the array
boundary/dimensions in which case the library wraps them around.

The local array is assumed to be have the same number of dimensions as the
global array. Any detected inconsitencies/errors in the input arguments are
fatal.

Returns: The local Array buffer.

\end{desc}

\seealso{GET}

\apih{PERIODIC PUT}{Puts data into a global array using periodic boundary conditions}

\begin{capi}
\begin{ccode}
void NGA_Periodic_put(int g_a, int lo[], int hi[], void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{void*}{buf}{pointer to the local buffer array where the data is}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_periodic_put(g_a, lo, hi,  buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer array where the data comes from}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::periodicPut(int lo[], int hi[], void* buf, int ld[]) const
void GlobalArray::periodicPut(int64_t lo[], int64_t hi[], void* buf, int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.periodic_put(int g_a, buffer, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Same as nga_put except the indices can extend beyond the array
boundary/dimensions in which case the library wraps them around.  The indices
can extend beyond the array boundary/dimensions in which case the libray wraps
them around.  Copies data from local array buffer to the global array section.
The local array is assumed to be have the same number of dimensions as the
global array. Any detected inconsitencies/errors in input arguments are fatal.

\end{desc}

\seealso{PUT}

\apih{PGROUP BRDCST}{Broadcasts elements among processes in a processor group}

\begin{capi}
\begin{ccode}
void GA_Pgroup_brdcst(int p_handle, void* buf, int lenbuf, int root)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\inoutarg{void*}{buf}{pointer to buffer containing data}
\inarg{int}{lenbuf}{length of data (in bytes)}
\inarg{int}{root}{processor sending message}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_pgroup_brdcst(p_handle, type, buf, lenbuf, root)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\inarg{integer}{type}{message index}
\inoutarg{byte}{buf(lenbuf)}{local message buffer}
\inarg{integer}{lenbuf}{length of message}
\inarg{integer}{root}{processor sending message}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void PGroup::brdcst(void* buf, int lenbuf, int root)
\end{cxxcode}
\begin{funcargs}
\inoutarg{void*}{buf}{pointer to buffer containing data}
\inarg{int}{lenbuf}{length of data (in bytes)}
\inarg{int}{root}{processor sending message}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_brdcst(int pgroup, buffer, int root=0)
\end{pycode}
\begin{funcargs}
\inarg{int}{pgroup}{processor group handle}
\inarg{1D array-like of objects}{buffer}{the ndarray message (converted to the appropriate type)}
\inarg{int}{root}{the process which is sending}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Broadcast data from processor specified by root to all other processors in the
processor group specified by p_handle. The length of the message in bytes is
specified by lenbuf. The initial and broadcasted data can be found in the
buffer specified by the pointer buf.

If the buffer is not contiguous, an error is raised. This operation is provided
only for convenience purposes: it is available regardless of the
message-passing library that GA is running with.

Returns: The buffer in case a temporary was passed in.
\end{desc}

\seealso{BRDCST}

\apih{PGROUP CREATE}{Creates a GA processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_create(int *list, int size)
\end{ccode}
\begin{funcargs}
\inarg{int*}{list[size]}{list of processor IDs in group}
\inarg{int}{size}{number of processors in group}
\outarg{int}{}{pgroup handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_create(list, size)
\end{fcode}
\begin{funcargs}
\inarg{integer}{size}{number of processors in group}
\inarg{integer}{list(size)}{list of processors in processor group}
\outarg{integer}{}{pgroup handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
PGroup::PGroup(int *plist, int size)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{size}{number of processors in group}
\inarg{int*}{plist[size]}{list of processor IDs in group}
\outarg{PGroup}{}{pgroup object}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_create(list)
\end{pycode}
\begin{funcargs}
\inarg{1D array-like of ints}{list}{list of processor IDs in group}
\outarg{int}{pgroup}{pgroup handle}
\end{funcargs}
\end{pyapi}

\dcoll

\begin{desc}

This command is used to create a processor group. At present, it must be
invoked by all processors in the current default processor group. The list of
processors use the indexing scheme of the default processor group. If the
default processor group is the world group, then these indices are the usual
processor indices. This function returns a process group handle that can be
used to reference this group by other functions.

\end{desc}

\seealso{PGROUP DESTROY,PGROUP SET DEFAULT}

\apih{PGROUP DESTROY}{Destroys a GA processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_destroy(int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\outarg{int}{}{0 if processor group was not previously active}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_pgroup_destroy(p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\outarg{integer}{}{.FALSE. if processor group was not previously active}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
PGroup::~PGroup()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_destroy(int pgroup)
\end{pycode}
\begin{funcargs}
\inarg{int}{pgroup}{processor group handle}
\outarg{bool}{ret}{False if processor group was not previously active}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This command is used to free up a processor group handle. It returns 0 if the
processor group handle was not previously active.

\end{desc}

\seealso{PGROUP CREATE,PGROUP SET DEFAULT}

\apih{PGROUP DUPLICATE}{Duplicate an existing GA processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_duplicate(int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\outarg{int}{}{handle to new processor group}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_duplicate(p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\outarg{integer}{}{handle to new processor group}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
Pgroup PGroup::duplicate()
\end{cxxcode}
\end{cxxapi}

\begin{desc}

Return a copy of an existing processor group.

\end{desc}

\gcoll

\seealso{PGROUP CREATE}

\apih{PGROUP GET DEFAULT}{Sets default GA processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_get_default()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_get_default()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
static PGroup* PGroup::getDefault()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_get_default()
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function will return a handle to the default processor group, which can
then be used to create a global array using one of the NGA_create_*_config or
GA_Set_pgroup calls.
\end{desc}

\apih{PGROUP GET MIRROR}{Gets the mirrored processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_get_mirror()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_get_mirror()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
static PGroup * PGroup::getMirror()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_get_mirror()
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function will return a handle to the mirrored processor group, which can
then be used to create a global array using one of the NGA_create_*_config or
GA_Set_pgroup calls.
\end{desc}

\apih{PGROUP GET WORLD}{Gets the world processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_get_world()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_get_world()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
static PGroup * PGroup::getWorld()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_get_world()
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function will return a handle to the world processor group, which can then
be used to create a global array using one of the NGA_create_*_config or
GA_Set_pgroup calls.
\end{desc}

\apih{PGROUP GOP}{Global operation with a processor group}

\begin{capi}
\begin{ccode}
void GA_Pgroup_fgop(int p_handle, float buf*, int n, char* op)
void GA_Pgroup_dgop(int p_handle, double buf*, int n, char* op)
void GA_Pgroup_igop(int p_handle, int buf*, int n, char* op)
void GA_Pgroup_lgop(int p_handle, long buf*, int n, char* op)
void GA_Pgroup_llgop(int p_handle, long long buf*, int n, char* op)
void GA_Pgroup_cgop(int p_handle, SingleComplex buf*, int n, char* op)
void GA_Pgroup_zgop(int p_handle, DoubleComplex buf*, int n, char* op)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\inoutarg{float*}{buf}{buffer containing data}
\inoutarg{double*}{buf}{buffer containing data}
\inoutarg{int*}{buf}{buffer containing data}
\inoutarg{long*}{buf}{buffer containing data}
\inoutarg{long long*}{buf}{buffer containing data}
\inoutarg{SingleComplex*}{buf}{buffer containing data}
\inoutarg{DoubleComplex*}{buf}{buffer containing data}
\inarg{int}{n}{number of elements in x}
\inarg{char*}{op}{operation to be performed}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_pgroup_igop(p_handle, type, buf, n, op)
subroutine ga_pgroup_sgop(p_handle, type, buf, n, op)
subroutine ga_pgroup_dgop(p_handle, type, buf, n, op)
subroutine ga_pgroup_cgop(p_handle, type, buf, n, op)
subroutine ga_pgroup_zgop(p_handle, type, buf, n, op)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\inarg{integer}{type}{message index (deprecated)}
\inoutarg{integer}{buf(n)}{array}
\inoutarg{real}{buf(n)}{array}
\inoutarg{double precision}{buf(n)}{array}
\inoutarg{single complex}{buf(n)}{array}
\inoutarg{double complex}{buf(n)}{array}
\inarg{integer}{n}{number elements in array}
\inarg{character*(*)}{op}{operation on data}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void PGroup::gop(float *buf, int n, char* op)
void PGroup::gop(double *buf, int n, char* op)
void PGroup::gop(int *buf, int n, char* op)
void PGroup::gop(long *buf, int n, char* op)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{n}{number of elements}
\inoutarg{float*}{x[n]}{array of elements}
\inoutarg{double*}{x[n]}{array of elements}
\inoutarg{int*}{x[n]}{array of elements}
\inoutarg{long*}{x[n]}{array of elements}
\inarg{char*}{op}{operator}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_gop(int pgroup, X, char *op)
ret = ga.pgroup_gop_add(int pgroup, X)
ret = ga.pgroup_gop_multiply(int pgroup, X)
ret = ga.pgroup_gop_max(int pgroup, X)
ret = ga.pgroup_gop_min(int pgroup, X)
ret = ga.pgroup_gop_absmax(int pgroup, X)
ret = ga.pgroup_gop_absmin(int pgroup, X)
\end{pycode}
\begin{funcargs}
\inarg{1D array-like}{X}{elements}
\inarg{char*}{op}{operator}
\outarg{ndarray}{ret}{result after numpy.asarray(X) followed by the operation}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
The buf[n] is an array present on each processor in the processor group
p_handle. The GA_Pgroup_dgop `sums' all elements in buf[n] across all
processors in the group specified by p_handle using the commutative operation
specified by the character string op.  The result is broadcast to all processor
in p_handle. Allowed strings are `+', `*', `max', `min', `absmax', `absmin'.
The use of lowerecase for operators is necessary.
\end{desc}

\seealso{GOP}

\seealso{GOP}

\apih{PGROUP NNODES}{Returns number of GA ranks in a processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_nnodes(int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_nnodes(p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int PGroup::nodes()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_nnodes(int pgroup)
\end{pycode}
\begin{funcargs}
\inarg{int}{pgroup}{processor group handle}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}
This function returns the number of processors contained in the group specified
by p_handle.

Returns the number of processors contained in the group specified by
pgroup.
\end{desc}

\seealso{NNODES}

\apih{PGROUP NODEID}{Returns GA rank of invoking process in a processor group}

\begin{capi}
\begin{ccode}
int GA_Pgroup_nodeid(int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_nodeid(p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int PGroup::nodeid()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.pgroup_nodeid(int pgroup)
\end{pycode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}
This function returns the relative index of the processor in the processor
group specified by p_handle. This index will generally differ from the absolute
processor index returned by GA_Nodeid if the processor group is not the world
group.

Returns the relative index of the processor in the processor group
specified by pgroup.
\end{desc}

\seealso{NODEID}

\apih{PGROUP SELF}{Create a process group with only the calling process in it}

\begin{capi}
\begin{ccode}
int GA_Pgroup_self()
\end{ccode}
\begin{funcargs}
\outarg{int}{}{handle to new processor group}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_self()
\end{fcode}
\begin{funcargs}
\outarg{integer}{}{handle to new processor group}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
Pgroup PGroup::self()
\end{cxxcode}
\end{cxxapi}

\begin{desc}

Return a processor group that only contains the calling process.

\end{desc}

\ncoll

\seealso{PGROUP CREATE}

\apih{PGROUP SET DEFAULT}{Sets a default GA processor group}

\begin{capi}
\begin{ccode}
void GA_Pgroup_set_default(int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_pgroup_set_default(p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
static void PGroup::setDefault(PGroup *p_handle)
\end{cxxcode}
\begin{funcargs}
\inarg{PGroup*}{p_handle}{processor group}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.pgroup_set_default(int pgroup)
\end{pycode}
\begin{funcargs}
\inarg{int}{pgroup}{processor group handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to reset the default processor group on a collection
of processors. All processors in the group referenced by p_handle must make a
call to this function. Any standard global array call that is made after
resetting the default processor group will be restricted to processors in that
group. Global arrays that are created after resetting the default processor
group will only be defined on that group and global operations, such as GA_Sync
or GA_Igop, and will be restricted to processors in that group. The
GA_Pgroup_set_default call can be used to rapidly convert large applications,
written with GA, into routines that run on processor groups.

The default processor group can be overridden by using GA calls that require an
explicit group handle as one of the arguments.

\end{desc}

\apih{PGROUP SPLIT}{Creates a set of new groups by dividing a parent group into equal sized subgroups}

\begin{capi}
\begin{ccode}
int GA_Pgroup_split(int p_handle, int num_group)
int NGA_Pgroup_split(int p_handle, int num_group)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle of parent}
\inarg{int}{num_group}{number of new groups}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_split(p_handle, num_group)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\inarg{integer}{num_group}{number of new groups}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}

This function can be used to divide the parent group, represented by p_handle,
into num_group smaller groups. The function returns the group handle of the subgroup
containing the calling processor. If num_group does not evenly divide the number
of processors in p_handle, then one group will contain fewer processors than the
others. All processors in the group referenced by p_handle must make a
call to this function.

\end{desc}

\seealso{PGROUP CREATE,PGROUP DESTROY}

\apih{PGROUP SPLIT IRREG}{Creates a set of new groups based on a color variable}

\begin{capi}
\begin{ccode}
int GA_Pgroup_split_irreg(int p_handle, int color)
int NGA_Pgroup_split_irreg(int p_handle, int color)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle of parent}
\inarg{int}{color}{color variable of calling process}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_pgroup_split_irreg(p_handle, color)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\inarg{integer}{color}{color variable of calling process}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}

This function can be used to divide the parent group, represented by p_handle,
into subgroups of different sizes based on the color variable. All processes
with the same color variable end up in the same group.  The function returns the
group handle of the subgroup containing the calling processor. Color variables
should be greater than or equal to zero.
All processors in the group referenced by p_handle must make a
call to this function.

\end{desc}

\seealso{PGROUP CREATE,PGROUP DESTROY}

\apih{PGROUP SYNC}{Synchronizes processes in a processor group}

\begin{capi}
\begin{ccode}
void GA_Pgroup_sync(int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_pgroup_sync(p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{p_handle}{processor group handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void PGroup::sync()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.pgroup_sync(int pgroup)
\end{pycode}
\begin{funcargs}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
This operation executes a synchronization group across the processors in the
processor group specified by p_handle. Nodes outside this group are unaffected.
\end{desc}

\seealso{SYNC}

\apih{PRINT}{Prints the contents of a global array}

\begin{capi}
\begin{ccode}
void GA_Print(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_print(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::print() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.print_stdout(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Prints an entire array to the standard output.

\end{desc}

\begin{pydesc}

Since ``print'' is a reserved word in Python, the function is named different
than in the other languages.

\end{pydesc}

\apih{PRINT DISTRIBUTION}{Prints the distribution of a global array}

\begin{capi}
\begin{ccode}
void GA_Print_distribution(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_print_distribution(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::printDistribution() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.print_distribution(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Prints the array distribution.

\end{desc}

\apih{PRINT FILE}{Prints the contents of a global array to a file}

\begin{capi}
\begin{ccode}
void GA_Print_file(FILE *file, int g_a)
\end{ccode}
\begin{funcargs}
\inarg{}{file}{file pointer}
\inarg{}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::printFile(FILE *file) const
\end{cxxcode}
\begin{funcargs}
\inarg{}{file}{file pointer}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.print_file(int g_a, file)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{file-like}{file}{}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Prints an entire array to a file.

\end{desc}

\apih{PRINT PATCH}{Prints a patch of a global array to stdout}

\begin{capi}
\begin{ccode}
void NGA_Print_patch(int g_a, int lo[],int hi[],int pretty)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{lo[],hi[]}{coordinates of the patch}
\inarg{int}{pretty}{formatting flag}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_print_patch(g_a,ilo,ihi,jlo,jhi,pretty)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ilo,ihi,jlo,jhi}{}
\inarg{integer}{pretty}{}
\end{funcargs}
\end{f2dapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::printPatch(int* lo, int* hi, int pretty) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo}{low coordinates of the patch}
\inarg{int*}{hi}{high coordinates of the patch}
\inarg{int}{pretty}{formatting flag}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.print_patch(int g_a, lo=None, hi=None, int pretty=True)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{bool}{pretty}{formatting flag}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Prints a patch of g_a array to the standard output. If the variable pretty has
the value 0 then output is printed in a dense fashion. If pretty has the value
1 then output is formatted and rows/columns are labeled.

\end{desc}

\apih{PRINT STATS}{Prints GA runtime statistics}

\begin{capi}
\begin{ccode}
void GA_Print_stats()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_print_stats()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GAServices::printStats()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.print_stats()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

This non-collective (MIMD) operation prints information about:
\begin{itemize}
    \item Number of calls to the GA create/duplicate, destroy, get, put,
    scatter, gather, and read_and_inc operations
    \item Total amount of data moved in the GA primitive operations
    \item Amount of data moved in GA primitive operations to logicaly remote
    locations
    \item Maximum memory consumption in global arrays, and
    \item Number of requests serviced in the interrupt-driven implementations
    by the calling process.
\end{itemize}

\end{desc}

\apih{PROC TOPOLOGY}{Inquires the linear location of a processor in the processor topology employed by a global array}

\begin{capi}
\begin{ccode}
void NGA_Proc_topology(int g_a, int proc, int coordinates[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{ndim}{number of array dimensions}
\inarg{int}{proc}{process id}
\outarg{int*}{coordinates[ndim]}{coordinates in processor grid}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_proc_topology(g_a, proc, prow, pcol)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{proc}{}
\outarg{integer}{prow, pcol}{}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::procTopology(int proc, int coord[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{proc}{process id}
\outarg{int*}{coord[ndim]}{coordinates in processor grid}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.proc_topology(int g_a, int proc)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{proc}{process id}
\outarg{1D ndarray of ints}{ret}{coordinates in processor grid}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Based on the distribution of an array associated with handle g_a, determines
coordinates of the specified processor in the virtual processor grid
corresponding to the distribution of array g_a. The numbering starts from 0.
The values of -1 means that the processor doesn't ``own'' any section of the
array represented by g_a.

\end{desc}

\seealso{SET BLOCK CYCLIC PROC GRID}

\apih{PUT}{Puts data into a global array}

\begin{capi}
\begin{ccode}
void NGA_Put(int g_a, int lo[], int hi[], void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{void*}{buf}{pointer to the local buffer array where the data is}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_put(g_a, ilo, ihi, jlo, jhi, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ilo, ihi, jlo, jhi}{bounding indices for global array section}
\outarg{double precision/complex/integer}{buf}{local buffer where data comes from}
\inarg{integer}{ld}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_put(g_a, lo, hi, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\outarg{type}{buf}{local buffer array where the data comes from}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::put(int lo[], int hi[], void *buf, int ld[]) const
void GlobalArray::put(int64_t lo[], int64_t hi[], void *buf,
                      int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{void*}{buf}{pointer to the local buffer array where the data is}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.put(int g_a, buffer, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Copies data from the local array buffer to the global array section. The local
array is assumed to have the same number of dimensions as the global array.
Any detected inconsistencies or errors in input arguments are fatal.

\end{desc}

\apih{RAND}{Random number generator}

\begin{capi}
\begin{ccode}
double NGA_Rand(int iseed)
\end{ccode}
\begin{funcargs}
\inarg{int}{iseed}{integer seed for random number generator}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
double function ga_rand(iseed)
\end{fcode}
\begin{funcargs}
\inarg{integer}{iseed}{integer seed for random number generator}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
double function nga_rand(iseed)
\end{fcode}
\begin{funcargs}
\inarg{integer}{iseed}{integer seed for random number generator}
\end{funcargs}
\end{fapi}

\ncoll

\begin{desc}

Convenience function to generate a pseudo-random sequence of numbers based on
the MT19937 Mersenne twister algorithm. The numbers generated by this
implementation are uniformly distributed on the interval [0,1). This code is
adapted from the open source implementation written by E. Sultanik and available
from \href{https://github.com/ESultanik/mtwister}{here}.

The first time this function is called, it will use the value of iseed to
initialize the random number generator. After this, the random number seed is
ignored and can be set to zero. Note that this implementation generates the
same sequence on all processors if started from the same seed. To get different
sequences on different processors, it is necessary to provide different seeds
on each process.

\end{desc}

\apih{READ INC}{Atomically read and increment an element in a global array}

\begin{capi}
\begin{ccode}
long NGA_Read_inc(int g_a, int subscript[], long inc)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{subscript[ndim]}{subscript array for the referenced element}
\inarg{long}{inc}{amount element is incremented after read}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
integer function ga_read_inc(g_a, i, j, inc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{i, j, inc}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
integer function nga_read_inc(g_a, subscript, inc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{subscript}{(ndim)}{subscript array for the referenced element}
\inarg{inc}{}{amount element is incremented after read}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
long GlobalArray::readInc(int subscript[], long inc)
long GlobalArray::readInc(int64_t subscript[], long inc)
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{subscript[ndim]}{subscript array for the referenced element}
\inarg{long}{inc}{amount element is incremented after read}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.read_inc(int g_a, subscript, long inc=1)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{subscript}{index for the referenced element}
\inarg{long}{inc}{the increment}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Atomically read and increment an element in an integer array.

\begin{verbatim}
   *BEGIN CRITICAL SECTION*
   old_value = a(subscript)
   a(subscript) += inc
   *END CRITICAL SECTION*
   return old_value
\end{verbatim}

\end{desc}

\apih{RECIP}{Translates a global array to contain reciprocal of its elements}

\begin{capi}
\begin{ccode}
void GA_Recip(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_recip(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::recip() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.recip(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Take the element-wise reciprocal of the array.
\end{desc}

\apih{RECIP PATCH}{Translates a global array patch to contain reciprocal of its elements}

\begin{capi}
\begin{ccode}
void GA_Recip_patch(int g_a, int lo[], int hi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{lo[], hi[]}{patch coordinates}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_recip_patch(g_a, lo, hi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{lo(ndim), hi(ndim)}{patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::recipPatch(int *lo, int *hi) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo}{lower corner patch coordinates}
\inarg{int*}{hi}{upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.recip(int g_a, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Take element-wise reciprocal of the patch.
\end{desc}

\seealso{RECIP}

\apih{RELEASE}{Releases access to a global array}

\begin{capi}
\begin{ccode}
void NGA_Release(int g_a, int lo[], int hi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_release(g_a, ilo, ihi, jlo, jhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ilo, ihi, jlo, jhi}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_release(g_a, lo, hi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{lo(ndim),hi(ndim)}{patch specification}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::release(int lo[], int hi[]) const
void GlobalArray::release(int64_t lo[], int64_t hi[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release(int g_a, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to a global array when the data was read only.

Your code should look like:
\begin{verbatim}
        NGA_Distribution(g_a, myproc, lo,hi);
        NGA_Access(g_a, lo, hi, \&ptr, ld);

             <operate on the data referenced by ptr>
        GA_Release(g_a, lo, hi);
\end{verbatim}
NOTE: see restrictions specified for ga_access.

\end{desc}

\seealso{ACCESS}

\apih{RELEASE BLOCK}{Releases access to a block of a global array}

\begin{capi}
\begin{ccode}
void NGA_Release_block(int g_a, int index)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{index}{block index}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_block(g_a, index)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{index}{block index}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseBlock(int index) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{index}{block index}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_block(int g_a, int index)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{index}{block index}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the block of data specified by the integer index when data
was accessed as read only. This is only applicable to block-cyclic data
distributions created using the simple block-cyclic distribution.

\end{desc}

\seealso{ACCESS BLOCK}

\apih{RELEASE BLOCK GRID}{Releases access to a block-cyclic distributed global array}

\begin{capi}
\begin{ccode}
void NGA_Release_block_grid(int g_a, int subscript[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{subscript[ndim]}{indices of block in array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_block_grid(g_a, subscript)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{subscript(ndim)}{indices of block in array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseBlockGrid(int index[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{index[ndim]}{indices of block in array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_block_grid(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{subscript}{indices of block in array}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the block of data specified by the subscript array when data
was accessed as read only. This is only applicable to block-cyclic data
distributions created using the SCALAPACK data distribution.

\end{desc}

\seealso{ACCESS BLOCK GRID}

\apih{RELEASE BLOCK SEGMENT}{Releases access to a block in a GA}

\begin{capi}
\begin{ccode}
void NGA_Release_block_segment(int g_a, int iproc)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{iproc}{processor ID}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_block_segment(g_a, iproc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{iproc}{processor ID}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseBlockSegment(int proc) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{proc}{process ID/rank}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_block_segment(int g_a, int iproc)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{iproc}{processor ID}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the block of locally held data for a block-cyclic array,
when data was accessed as read-only.

\end{desc}

\seealso{ACCESS BLOCK SEGMENT}

\apih{RELEASE GHOST ELEMENT}{Releases access to ghost cells in a GA}

\begin{capi}
\begin{ccode}
void NGA_Release_ghost_element(int g_a, int subscript[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{subscript[ndim]}{element subscript}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_ghost_element(g_a, subscript)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{subscript(ndim)}{element subscript}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseGhostElement(int subscript[]) const
void GlobalArray::releaseGhostElement(int64_t subscript[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{subscript[ndim]}{indices of element}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_ghost_element(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{subscript}{element subscript}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the locally held data for an array with ghost elements, when
data was accessed as read-only.

\end{desc}

\seealso{ACCESS GHOST ELEMENT}

\apih{RELEASE GHOSTS}{Releases access to ghost cells}

\begin{capi}
\begin{ccode}
void NGA_Release_ghosts(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_ghosts(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseGhosts() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_ghosts(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the locally held block of data containing ghost elements,
when data was accessed as read-only.

\end{desc}

\seealso{ACCESS GHOSTS}

\apih{RELEASE UPDATE}{Releases access to a global array after an update}

\begin{capi}
\begin{ccode}
void NGA_Release_update(int g_a, int lo[], int hi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_release_update(g_a, ilo, ihi, jlo, jhi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ilo, ihi, jlo, jhi}{}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_update(g_a, lo, hi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{lo(ndim),hi(ndim)}{patch specification}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseUpdate(int lo[], int hi[]) const
void GlobalArray::releaseUpdate(int64_t lo[], int64_t hi[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for array section}
\inarg{int*}{hi[ndim]}{array of ending indices for array section}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_update(int g_a, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the data. It must be used if the data was accessed for
writing.

NOTE: see restrictions specified for ga_access.

\end{desc}

\seealso{ACCESS}

\apih{RELEASE UPDATE BLOCK}{Releases after update access to a block in a global array}

\begin{capi}
\begin{ccode}
void NGA_Release_update_block(int g_a, int index)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{index}{block index}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_update_block(g_a, index)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{index}{block index}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseUpdateBlock(int index) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{index}{block index}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_update_block(int g_a, int index)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{index}{block index}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the block of data specified by the integer index when data
was accessed in read-write mode. This is only applicable to block-cyclic data
distributions created using the simple block-cyclic distribution.

\end{desc}

\seealso{ACCESS BLOCK}

\apih{RELEASE UPDATE BLOCK GRID}{Releases after update access to a block in a block-cyclic distributed global array}

\begin{capi}
\begin{ccode}
void NGA_Release_update_block_grid(int g_a, int subscript[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{subscript[ndim]}{indices of block in array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_update_block_grid(g_a, subscript)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{subscript(ndim)}{indices of block in array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseUpdateBlockGrid(int index[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{index[ndim]}{indices of block in array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_update_block_grid(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{subscript}{indices of block in array}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the block of data specified by the subscript array when data
was accessed in read-write mode. This is only applicable to block-cyclic data
distributions created using the SCALAPACK data distribution.

\end{desc}

\seealso{ACCESS BLOCK GRID}

\apih{RELEASE UPDATE BLOCK SEGMENT}{Releases access to a block of a GA}

\begin{capi}
\begin{ccode}
void NGA_Release_block_segment(int g_a, int iproc)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{iproc}{processor ID}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_update_block_segment(g_a, iproc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{iproc}{processor ID}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseUpdateBlockSegment(int proc) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{proc}{process ID/rank}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_update_block_segment(int g_a, int iproc)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{iproc}{processor ID}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the block of locally held data for a block-cyclic array,
when data was accessed as read-only.

\end{desc}

\seealso{ACCESS BLOCK SEGMENT}

\apih{RELEASE UPDATE GHOST ELEMENT}{Releases after update access to ghost cells in a GA}

\begin{capi}
\begin{ccode}
void NGA_Release_update_ghost_element(int g_a, int subscript[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{subscript[ndim]}{element subscript}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_update_ghost_element(g_a, subscript)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{subscript(ndim)}{element subscript}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseUpdateGhostElement(int subscript[]) const
void GlobalArray::releaseUpdateGhostElement(int64_t subscript[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{subscript[ndim]}{indices of element}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_update_ghost_element(int g_a, subscript)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{subscript}{element subscript}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the locally held data for an array with ghost elements, when
data was accessed in read-write mode.

\end{desc}

\seealso{ACCESS GHOST ELEMENT}

\apih{RELEASE UPDATE GHOSTS}{Releases after access to ghosts}

\begin{capi}
\begin{ccode}
void NGA_Release_update_ghosts(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_release_update_ghosts(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::releaseUpdateGhosts() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.release_update_ghosts(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Releases access to the locally held block of data containing ghost elements,
when data was accessed in read-write mode.

\end{desc}

\seealso{ACCESS GHOSTS}

\apih{SCALE}{Scales a global array by the specified value}

\begin{capi}
\begin{ccode}
void GA_Scale(int g_a, void *value)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{void*}{value}{pointer to the value of appropriate type (double/double complex/long) that matches array type}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_scale(g_a, s)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{double precision/complex/integer}{s}{scale factor}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scale(void *value) const
\end{cxxcode}
\begin{funcargs}
\inarg{void*}{value}{pointer to the value of appropriate type}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scale(int g_a, value)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{value}{object of the value of appropriate type (double/double complex/long) that matches array type}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Scales an array by the constant s. Note that the library is unable to detect
errors when the pointed value is of a different type than the array.

\end{desc}

\apih{SCALE COLS}{Scales columns of a global array with elements in another gobal array}

\begin{capi}
\begin{ccode}
void GA_Scale_cols(int g_a, int g_v)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_scale_cols(g_a, g_v)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_v}{array handles}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scaleCols(const GlobalArray * g_v) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_v}{global array containing scale factors}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scale_cols(int g_a, int g_v)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Scales the columns of this matrix g_a using the vector g_v.
\end{desc}

\apih{SCALE PATCH}{Scales elements in the patch of a global array}

\begin{capi}
\begin{ccode}
void NGA_Scale_patch(int g_a, int lo[], int hi[], void *val)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handles}
\inarg{int*}{lo[], hi[]}{patch of g_a}
\inarg{void*}{val}{scale factor}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_scale_patch(g_a, ailo, aihi, ajlo, ajhi, s)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{double precision/complex/integer}{s}{scale factor}
\inarg{integer}{ailo, aihi, ajlo, ajhi}{g_a patch coordinates}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_scale_patch(g_a, alo, ahi, s)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{double precision/complex/integer}{s}{scale factor}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scalePatch (int lo[], int hi[], void *val) const;
void GlobalArray::scalePatch (int64_t lo[], int64_t hi[], void *val) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo}{patch of this GlobalArray}
\inarg{int*}{hi}{patch of this GlobalArray}
\inarg{void*}{val}{scale factor}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scale(int g_a, value, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{value}{scale value; object of the appropriate type (double/double complex/long) that matches array type}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Scale an array by the factor `val'

\end{desc}

\seealso{SCALE}

\apih{SCALE ROWS}{Scales the rows of a global array with elements in another global array}

\begin{capi}
\begin{ccode}
void GA_Scale_rows(int g_a, int g_v)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_scale_rows(g_a, g_v)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_v}{array handles}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scaleRows(const GlobalArray * g_v) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_v}{global array containing scale factors}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scale_rows(int g_a, int g_v)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Scales the rows of this matrix g_a using the vector g_v.
\end{desc}

\apih{SCAN ADD}{Adds successive elements in a source vector based on a corresponding intenger mask}

\begin{capi}
\begin{ccode}
void GA_Scan_add(int g_src, int g_dest, int g_mask, int lo, int hi, int excl)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source array}
\inarg{int}{g_dest}{handle for destination array}
\inarg{int}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo,hi}{low and high values of range on which operation is performed}
\inarg{int}{excl}{value to signify if masked values are included in add}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_scan_add(g_src, g_dest, g_mask, lo, hi, excl)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_src}{handle for source array}
\outarg{integer}{g_dest}{handle for destination array}
\inarg{integer}{g_mask}{handle for integer array representing a bit mask}
\inarg{integer}{lo,hi}{low and high values of range on which operation is performed}
\inarg{integer}{excl}{value to signify if masked values are included in add}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scanAdd(const GlobalArray *g_dest, const GlobalArray *g_mask,
                          int lo, int hi, int excl) const
void GlobalArray::scanAdd(const GlobalArray *g_dest, const GlobalArray *g_mask,
                          int64_t lo, int64_t hi, int excl) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_dest}{handle for destination array}
\inarg{int}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo}{low and high values of range on which operation is performed}
\inarg{int}{hi}{low and high values of range on which operation is performed}
\inarg{int}{excl}{value to signify if masked values are included in in add}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scan_add(int g_src, int g_dst, int g_msk, lo=None, hi=None, int excl=False)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source arrray}
\inarg{int}{g_dst}{handle for destination array}
\inarg{int}{g_msk}{handle for integer array representing mask}
\inarg{1D array-like of ints}{lo}{low value of range on which operation is performed}
\inarg{1D array-like of ints}{hi}{hi value of range on which operation is performed}
\inarg{bool}{excl}{whether the first value is set to 0 (see above)}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
This operation will add successive elements in a source vector g_src and put
the results in a destination vector g_dest. The addition will restart based on
the values of the integer mask vector g_mask. The scan is performed within the
range specified by the integer values lo and hi. Note that this operation can
only be applied to 1-dimensional arrays. The excl flag determines whether the
sum starts with the value in the source vector corresponding to the location of
a 1 in the mask vector (excl=0) or whether the first value is set equal to 0
(excl=1). Some examples of this operation are given below.
\begin{verbatim}
GA_Scan_add(g_src, g_dest, g_mask, 1, n, 0);

g_mask:   1  0  0  0  0  0  1  0  1  0  0  1  0  0  1  1  0
g_src:    1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17
g_dest:   1  3  6 10 16 21  7 15  9 19 30 12 25 39 15 16 33

GA_Scan_add(g_src, g_dest, g_mask, 1, n, 1);

g_mask:   1  0  0  0  0  0  1  0  1  0  0  1  0  0  1  1  0
g_src:    1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17
g_dest:   0  1  3  6 10 15  0  7  0  9 19  0 12 25  0  0 16
\end{verbatim}
\end{desc}

\apih{SCAN COPY}{Copies successive elements in a source vector based on a corresponding intenger mask}

\begin{capi}
\begin{ccode}
void GA_Scan_copy(int g_src, int g_dest, int g_mask, int lo, int hi)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source array}
\inarg{int}{g_dest}{handle for destination array}
\inarg{int}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo,hi}{low and high values of range on which operation is performed}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_scan_copy(g_src, g_dest, g_mask, lo, hi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_src}{handle for source array}
\outarg{integer}{g_dest}{handle for destination array}
\inarg{integer}{g_mask}{handle for integer array representing a bit mask}
\inarg{integer}{lo,hi}{low and high values of range on which operation is performed}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scanCopy(const GlobalArray *g_dest,
        const GlobalArray *g_mask, int lo, int hi) const
void GlobalArray::scanCopy(const GlobalArray *g_dest,
        const GlobalArray *g_mask, int64_t lo, int64_t hi) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_dest}{handle for destination array}
\inarg{const GlobalArray*}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo}{low values of range on which operation is performed}
\inarg{int}{hi}{high values of range on which operation is performed}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scan_copy(int g_src, int g_dst, int g_msk, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source arrray}
\inarg{int}{g_dst}{handle for destination array}
\inarg{int}{g_msk}{handle for integer array representing mask}
\inarg{1D array-like of ints}{lo}{low value of range on which operation is performed}
\inarg{1D array-like of ints}{hi}{hi value of range on which operation is performed}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
This subroutine does a segmented scan-copy of values in the source array g_src
into a destination array g_dest with segments defined by values in the integer
mask array g_mask. The scan-copy operation is only applied to the range between
the lo and hi indices. This operation is restriced to 1-dimensional arrays. The
resulting destination array will consist of segments of consecutive elements
with the same value. An example is shown below.
\begin{verbatim}
GA_Scan_copy(g_src, g_dest, g_mask, 1, n);

g_mask:   1  0  0  0  0  0  1  0  1  0  0  1  0  0  1  1  0
g_src:    1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17
g_dest:   1  1  1  1  1  1  7  7  9  9  9 12 12 12 15 16 16

\end{verbatim}
\end{desc}

\apih{SCATTER}{Scatters elements into a global array}

\begin{capi}
\begin{ccode}
void NGA_Scatter(int g_a, void *v, int* subsArray[], int n)
void NGA_Scatter_flat(int g_a, void *v, int* subsArray, int n)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{void*}{v[n]}{array containing values}
\inarg{int**}{subsArray[n][ndim]}{array of subscripts for each element}
\inarg{int*}{subsArray[n*ndim]}{(flat) array of subscripts for each element}
\inarg{int}{n}{number of elements}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_scatter(g_a, v, i, j, n)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{double precision}{v(n)}{vector of values to be scattered}
\inarg{integer}{i(n), j(n)}{array of subscripts for each destination element}
\inarg{integer}{n}{number of elements to be scattered}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_scatter(g_a, v, subsArray, n)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{n}{number of elements}
\inarg{type}{v(n)}{array containing values}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{subsArray(ndim,n)}{array of subscripts for each element}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scatter(void *v, int *subsarray[], int n) const
void GlobalArray::scatter(void *v, int64_t *subsarray[], int64_t n) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{n}{number of elements}
\inarg{void*}{v[n]}{array containing values}
\inarg{int**}{subsarray[n][ndim]}{array of subscripts for each element}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scatter(int g_a, values, subsarray)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D array-like}{values}{array containing values}
\inarg{1D or 2D array-like of ints}{subsarray}{array of subscripts for each element}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Scatters array elements into a global array. The contents of the input arrays
(v,subsArray) are preserved.

\begin{verbatim}
for (k=0; k<= n; k++)
   {a[[subsArray[k][0]][subsArray[k][1]][subsArray[k][2]]... = v[k];}
\end{verbatim}

\end{desc}

\begin{pydesc}

subsarray will be converted to an ndarray if it is not one already.  A
two-dimensional array is allowed so long as its shape is (n,ndim) where n is
the number of elements to gather and ndim is the number of dimensions of the
target array. Also, subsarray must be contiguous.

For example, if the subsarray were two-dimensional:

\begin{verbatim}
for k in range(n):
    v[k] = g_a[subsarray[k,0],subsarray[k,1],subsarray[k,2]...]
\end{verbatim}

For example, if the subsarray were one-dimensional:

\begin{verbatim}
for k in range(n):
    base = n*ndim
    v[k] = g_a[subsarray[base+0],subsarray[base+1],subsarray[base+2]...]
\end{verbatim}

\end{pydesc}

\seealso{PUT}

\apih{SCATTER ACC}{Scatters and accumulates elements into a global array}

\begin{capi}
\begin{ccode}
void NGA_Scatter_acc(int g_a, void *v, int* subsArray[], int n, void *alpha)
void NGA_Scatter_acc_flat(int g_a, void *v, int* subsArray, int n, void *alpha)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{void*}{v[n]}{array containing values}
\inarg{int**}{subsArray[n][ndim]}{array of subscripts for each element}
\inarg{int*}{subsArray[n*ndim]}{(flat) array of subscripts for each element}
\inarg{void*}{alpha}{multiplicative factor}
\inarg{int}{n}{number of elements}
\end{funcargs}
\end{capi}

\begin{f2dapi}
\begin{fcode}
subroutine ga_scatter_acc(g_a, v, i, j, n, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{n}{number of elements}
\inarg{type}{v(n)}{array containing value}
\inarg{integer}{i(n),j(n)}{arrays of indices}
\inarg{double precision/complex}{alpha}{multiplicative value}
\end{funcargs}
\end{f2dapi}

\begin{fapi}
\begin{fcode}
subroutine nga_scatter_acc(g_a, v, subsArray, n, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{n}{number of elements}
\inarg{type}{v(n)}{array containing value}
\inarg{integer}{ndim}{number of array dimensions}
\inarg{integer}{subsArray(ndim,n)}{array of subscripts}
\inarg{double precision/complex}{alpha}{multiplicative value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::scatterAcc(void *v, int *subsarray[], int n, void *alpha) const
void GlobalArray::scatterAcc(void *v, int64_t *subsarray[], int64_t n, void *alpha) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{n}{number of elements}
\inarg{void*}{v[n]}{array containing values}
\inarg{int**}{subsarray[n][ndim]}{array of subscripts for each element}
\inarg{void*}{alpha}{multiplicative factor}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.scatter_acc(int g_a, values, subsarray, alpha=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D array-like}{values}{array containing values}
\inarg{1D or 2D array-like of ints}{subsarray}{array of subscripts for each element}
\inarg{object}{alpha}{multiplicative factor, converted to the appropriate type}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Scatters array elements from a local array into a global array. Adds values
from the local array to existing values in the global array after multiplying
by alpha. The contents of the input arrays (v, subsArray) are preserved.

\begin{verbatim}
for (k=0; k<= n; k++)
   {a[subsArray[k][0]][subsArray[k][1]][subsArray[k][2]]... += v[k];}
\end{verbatim}

Like scatter, but adds values to existing values in the global array after
multiplying by alpha.

\end{desc}

\begin{pydesc}

subsarray will be converted to an ndarray if it is not one already. A
two-dimensional array is allowed so long as its shape is (n,ndim) where n is
the number of elements to gather and ndim is the number of dimensions of the
target array. Also, subsarray must be contiguous.

For example, if the subsarray were two-dimensional:

\begin{verbatim}
for k in range(n):
    v[k] = g_a[subsarray[k,0],subsarray[k,1],subsarray[k,2]...]
\end{verbatim}

For example, if the subsarray were one-dimensional:

\begin{verbatim}
for k in range(n):
    base = n*ndim
    v[k] = g_a[subsarray[base+0],subsarray[base+1],subsarray[base+2]...]
\end{verbatim}

\end{pydesc}

\seealso{ACC}

\apih{SELECT ELEM}{Selects an element in a global returned by the chosen operation (eg., min, max, etc.)}

\begin{capi}
\begin{ccode}
void NGA_Select_elem(int g_a, char *op, void* val, int index[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle Control}
\inarg{char*}{op}{operator {`min',`max'}}
\outarg{void*}{val}{address where value should be stored}
\outarg{int*}{index[ndim]}{array index for the selected element}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_select_elem(g_a, op, val, index)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle Control}
\inarg{character(*)*}{op}{operator {`min',`max'}}
\outarg{}{val}{address where selected value should be stored}
\outarg{integer}{index(ndim)}{array index for the selected element}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::selectElem(char *op, void* val, int index[]) const
void GlobalArray::selectElem(char *op, void* val, int64_t index[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{char*}{op}{operator (``min'',``max'')}
\outarg{void*}{val}{address where value should be stored}
\outarg{int*}{index[ndim]}{array index for the selected element}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
value,index = select_elem(int g_a, char *op)
value,index = select_elem_min(int g_a)
value,index = select_elem_max(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle Control}
\inarg{char*}{op}{operator {`min',`max'}}
\outarg{object}{val}{Python scalar value}
\outarg{1D ndarray of ints}{index}{array index for the selected element}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Returns the value and index for an element that is selected by the specified
operator ("min" or "max") in a global array corresponding to g_a handle.

\end{desc}

\apih{SET ARRAY NAME}{Specifies what the name will be for the GA handle}

\begin{capi}
\begin{ccode}
void GA_Set_array_name (int g_a, char *name)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{char*}{name}{array name}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_array_name(g_a, name)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{character*(*)}{name}{a unique character string}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setArrayName(char *name) const
\end{cxxcode}
\begin{funcargs}
\inarg{char*}{name}{array name}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_array_name(int g_a, char *name)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{char*}{name}{array name}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to assign a unique character string name to a Global
Array handle that was obtained using the GA_Create_handle function.

\end{desc}

\seealso{ALLOCATE}

\apih{SET BLOCK CYCLIC}{Specifies GA handle will have a round-robin distribution}

\begin{capi}
\begin{ccode}
void GA_Set_block_cyclic(int g_a, int dims[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{dims[]}{array of block dimensions}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_block_cyclic(g_a, dims)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{dims(ndim)}{array of block dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setBlockCyclic(int dims[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{dims}{array of block dimensions}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_block_cyclic(int g_a, dims)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D array-like of ints}{dims}{block dimensions}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This subroutine is used to create a global array with a simple block-cyclic
data distribution. The array is broken up into blocks of size dims and each
block is numbered sequentially using a column major indexing scheme. The blocks
are then assigned in a simple round-robin fashion to processors.

Figure \ref{stblkcy} illustrates an array containing 25 blocks distributed on 4
processors.

Blocks at the edge of the array may be smaller than the block size specified in
dims. In the example below, blocks 4, 9, 14, 19, 20, 21, 22, 23, and 24 might
be smaller than the remaining blocks. Most global array operations are
insensitive to whether or not a block-cyclic data distribution is used,
although performance may be slower in some cases if the global array is using a
block-cyclic data distribution. Individual data blocks can be accessesed using
the block-cyclic access functions.

\begin{figure}
\centering
\includegraphics{set-block-cyclic}
\caption{Set Block Cyclic Data Distribution}
\label{stblkcy}
\end{figure}

\end{desc}

\apih{SET BLOCK CYCLIC PROC GRID}{Specifies the GA handle will have a block-cyclic processor distribution}

\begin{capi}
\begin{ccode}
void GA_Set_block_cyclic_proc_grid(int g_a, int dims[], int proc_grid[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{dims[]}{array of block dimensions}
\inarg{int*}{proc_grid[]}{processor grid dimensions}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_block_cyclic_proc_grid(g_a, dims, proc_grid)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{dims(ndim)}{array of block dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setBlockCyclicProcGrid(int dims[], int proc_grid[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{dims}{array of block dimensions}
\inarg{int*}{proc_grid}{processor grid dimensions}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_block_cyclic_proc_grid(int g_a, block, proc_grid)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D array-like of ints}{dims}{block dimensions}
\inarg{1D array-like of ints}{proc_grid}{processor grid dimensions}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This subroutine is used to create a global array with a SCALAPACK-type block
cyclic data distribution. The user specifies the dimensions of the processor
grid in the array proc_grid. The product of the processor grid dimensions must
equal the number of total number of processors and the number of dimensions in
the processor grid must be the same as the number of dimensions in the global
array. The data blocks are mapped onto the processor grid in a cyclic manner
along each of the processor grid axes.

Figure \ref{setblkcyprocgrid} illustrates an array consisting of 25 data blocks
distributed on 6 processors.

The 6 processors are configured in a 3 by 2 processor grid. Blocks at the edge
of the array may be smaller than the block size specified in dims.  Most global
array operations  are insensitive to whether or not a block-cyclic data
distribution is used, although performance may be slower in some cases if the
global array is using a block-cyclic data distribution. Individual data blocks
can be accessesed using the block-cyclic access functions.

\begin{figure}
\centering
\includegraphics{set-block-cyclic-proc-grid}
\caption{Creating a SCALAPACK-type Block Cyclic Data Distribution}
\label{setblkcyprocgrid}
\end{figure}

\end{desc}

\apih{SET CHUNK}{Specifies GA handle will have the given chunk size}

\begin{capi}
\begin{ccode}
void GA_Set_chunk(int g_a, int chunk[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{chunk[]}{array of chunk widths}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_chunk(g_a, chunk)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{chunk(ndim)}{array of chunk widths}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setChunk(int chunk[]) const
void GlobalArray::setChunk(int64_t chunk[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{chunk}{array of chunk widths}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_chunk(int g_a, chunk)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{chunk}{chunk widths}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function is used to set the chunk array for a global array handle that was
obtained using the GA_Create_handle function. The chunk array is used to
determine the minimum number of array elements assigned to each processor along
each coordinate direction.

\end{desc}

\apih{SET DATA}{Specifies what the array shape and data type wlll be for a GA handle}

\begin{capi}
\begin{ccode}
void GA_Set_data (int g_a, int ndim, int dims[], int type)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{ndim}{dimension of global array}
\inarg{int*}{dims[]}{dimensions of global array}
\inarg{int}{type}{data type of global array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_data (g_a, ndim, dims, type)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{dimension of array}
\inarg{integer}{dims(ndim)}{array dimensions}
\inarg{integer}{type}{data type (MT_DBL,MT_INT,etc.)}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setData(int ndim, int dims[], int type) const
void GlobalArray::setData(int ndim, int64_t dims[], int type) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{ndim}{dimension of global array}
\inarg{int*}{dims}{dimensions of global array}
\inarg{int}{type}{data type of global array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_data(int g_a, dims, int type)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{dims}{shape of global array}
\inarg{int}{type}{data type of global array}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to set the array dimension, the coordinate
dimensions, and the data type assigned to a Global Array handle obtained using
the GA_Create_handle function.

\end{desc}

\seealso{ALLOCATE}

\apih{SET DEBUG}{Sets GA debug flag}

\begin{capi}
\begin{ccode}
void GA_Set_debug(int dbg)
\end{ccode}
\begin{funcargs}
\inarg{int}{dbg}{value to set internal flag}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_debug(flag)
\end{fcode}
\begin{funcargs}
\inarg{logical}{flag}{value to set internal debug flag}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GAServices::setDebug(int dbg);
\end{cxxcode}
\begin{funcargs}
\inarg{}{dbg}{value to set internal flag}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_debug(int debug)
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function sets an internal flag in the GA library to either true or false.
The value of this flag can be recovered at any time using the GA_Get_debug
function. The flag is set to false when the the GA library is initialized. This
can be useful in a number of debugging situations, especially when examining
the behavior of routines that are called in multiple locations in a code.
\end{desc}

\apih{SET DIAGONAL}{Sets the diagonal elements of a global array}

\begin{capi}
\begin{ccode}
void GA_Set_diagonal(int g_a, int g_v)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_diagonal(g_a, g_v)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_v}{array handles}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setDiagonal(const GlobalArray * g_v) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_v}{global array containing diagonal values}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_diagonal(int g_a, int g_v)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_v}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Sets the diagonal elements of this matrix g_a with the elements of the vector
g_v.
\end{desc}

\apih{SET GHOSTS}{Specifies the GA handle will have ghost cells}

\begin{capi}
\begin{ccode}
void GA_Set_ghosts(int g_a, int width[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{width[ndim]}{array of ghost cell widths}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_ghosts(g_a, width)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{width(ndim)}{array of ghost cell widths}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setGhosts(int width[]) const
void GlobalArray::setGhosts(int64_t width[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{}{width[ndim]}{array of ghost cell widths}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_ghosts(int g_a, width)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{width}{ghost cell widths}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to set the ghost cell widths for a global array
handle that was obtained using the GA_Create_handle function.  The ghosts cells
widths indicate how many ghost cells are used to pad the locally held array
data along each dimension. The padding can be set independently for each
coordinate dimension.

\end{desc}

\apih{SET IRREG DISTR}{Specifies that the GA handle will have an irregular distribution}

\begin{capi}
\begin{ccode}
void GA_Set_irreg_distr(int g_a, int mapc[], int nblock[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int*}{mapc[s]}{starting index for each block; the size s is the sum of all elements of the array nblock}
\inarg{int*}{nblock[ndim]}{number of blocks that each dimension is divided into}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_irreg_distr(g_a, mapc, nblock)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{map(s)}{starting index for for each block; the size s is a sum of all elements of nblock array}
\inarg{integer}{nblock(ndim)}{no. of blocks each dimension is divided into}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setIrregDistr(int mapc[], int nblock[]) const
void GlobalArray::setIrregDistr(int64_t mapc[], int64_t nblock[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{}{mapc[s]}{starting index for each block; the size s is the sum of all elements of the array nblock}
\inarg{}{nblock[ndim]}{number of blocks that each dimension is divided into}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_irreg_distr(int g_a, mapc, nblock)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{mapc}{starting index for for each block; len(map) is a sum all elements of block array}
\inarg{1D array-like of ints}{nblock}{no. of blocks each dimension is divided into}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to partition the array data among the individual
processors for a global array handle obtained using the GA_Create_handle
function.
\end{desc}

\begin{fdesc}
The distribution is specified as a Cartesian product of distributions for each
dimension. For example, the following figure demonstrates the distribution of a
2-dimensional array 8x10 on 6 processors. nblock(2)=\{3,2\}, the size
of mapc array is s=5 and array mapc contains the following elements mapc=\{1,
3, 7, 1, 6\}. The distribution is nonuniform because, P1 and P4 get 20 elements
each and processors P0, P2, P3, and P5 only 10 elements each.
\end{fdesc}

\begin{cdesc}
The distribution is specified as a Cartesian product of distributions for each
dimension. For example, the following figure demonstrates the distribution of a
2-dimensional array 8x10 on 6 processors. nblock(2)=\{3,2\}, the size
of mapc array is s=5 and array mapc contains the following elements
mapc=\{0,2,6,0,5\}. The distribution is nonuniform because, P1 and P4 get 20
elements each and processors P0, P2, P3, and P5 only 10 elements each.
\end{cdesc}

\begin{cxxdesc}
The distribution is specified as a Cartesian product of distributions for each
dimension. For example, the following figure demonstrates the distribution of a
2-dimensional array 8x10 on 6 processors. nblock(2)=\{3,2\}, the size
of mapc array is s=5 and array mapc contains the following elements
mapc=\{0,2,6,0,5\}. The distribution is nonuniform because, P1 and P4 get 20
elements each and processors P0, P2, P3, and P5 only 10 elements each.
\end{cxxdesc}

\begin{pydesc}
The distribution is specified as a Cartesian product of distributions for each
dimension. For example, the following figure demonstrates the distribution of a
2-dimensional array 8x10 on 6 processors. nblock(2)=\{3,2\}, the size
of mapc array is s=5 and array mapc contains the following elements mapc=\{1,
3, 7, 1, 6\}. The distribution is nonuniform because, P1 and P4 get 20 elements
each and processors P0, P2, P3, and P5 only 10 elements each.
\end{pydesc}

\begin{desc}
The array width() is used to control the width of the ghost cell boundary
around the visible data on each processor. The local data of the global array
residing on each processor will have a layer width(n) ghosts cells wide on
either side of the visible data along the dimension n.

An example is shown in the figure below.

\begin{figure}
\centering
\includegraphics{set-irreg-dist}
\caption{Set an Irregular Distribution}
\label{setirregdist}
\end{figure}

\end{desc}

\seealso{ALLOCATE}

\apih{SET MEMORY DEV}{Specify the type of memory used for the global array}

\begin{capi}
\begin{ccode}
void GA_Set_memory_dev(int g_a, char *device)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{char*}{device}{character string specifying the memory type}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_memory_dev(g_a, device)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{character*(*)}{character string specifying the memory type}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setMemoryDev(char *device)
\end{cxxcode}
\begin{funcargs}
\inarg{char*}{property}{character string specifying the memory type}
\end{funcargs}
\end{cxxapi}

\gcoll

\begin{desc}

This function allows users to specify what type of memory is used for allocating
global arrays. This functionality relies on the Simpified Interface for Complex
Memory library, available at
\href{https://github.com/lanl/SICM}{https://github.com/lanl/SICM}. If Global Arrays is
not linked to SICM, then this property is ignored and the array will be
allocated on ordinary memory. This should still produce correct code, but
performance may be lower.

The type of memory available depends on the hardware being used. Currently
supported options are "dram" (this is widely available but only corresponds to
regular memory), "knl_hbm", and "ppc_hbm". The "hbm" options are high-bandwidth
memory on the KNL and Power PC chips, respectively. If Global Arrays has not been
compiled using SICM, the memory device option is ignored and a conventional
global array is created.
\end{desc}

\apih{SET MEMORY LIMIT}{Limits the internal memory used by the GA runtime}

\begin{capi}
\begin{ccode}
void GA_Set_memory_limit(size_t limit)
\end{ccode}
\begin{funcargs}
\inarg{size_t}{limit}{the amount of memory in bytes per process}
\end{funcargs}
\end{capi}
\begin{fapi}
\begin{fcode}
subroutine ga_set_memory_limit(limit)
\end{fcode}
\begin{funcargs}
\inarg{integer}{limit}{the amount of memory in bytes per process}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setMemoryLimit(size_t limit);
\end{cxxcode}
\begin{funcargs}
\inarg{size_t}{limit}{the amount of memory in bytes per process}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_memory_limit(size_t limit)
\end{pycode}
\begin{funcargs}
\inarg{size_t}{limit}{the amount of memory in bytes per process}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

Sets the amount of memory to be used (in bytes) per process

\end{desc}

\seealso{INITIALIZE LTD}

\apih{SET PGROUP}{Specifies what the processor group will be for a GA handle}

\begin{capi}
\begin{ccode}
void GA_Set_pgroup(int g_a, int p_handle)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_pgroup(g_a, p_handle)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{p_handle}{processor group handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setPGroup(PGroup *pHandle) const
\end{cxxcode}
\begin{funcargs}
\inarg{PGroup*}{pHandle}{processor group instance}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_pgroup(int g_a, int pgroup)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{p_handle}{processor group handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to set the processor configuration assigned to a
global array handle that was obtained using the GA_Create_handle function. It
can be used to create mirrored arrays by using the mirrored array processor
configuration in this function call. It can also be used to create an array on
a processor group by using a processor group handle in this call.

\end{desc}

\seealso{ALLOCATE}

\apih{SET PROPERTY}{Assign a property to a global array}

\begin{capi}
\begin{ccode}
void GA_Set_property(int g_a, char *property)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{char*}{property}{character string representing the property}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_property(g_a, property)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{character*(*)}{character string representing the property}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setProperty(char *property)
\end{cxxcode}
\begin{funcargs}
\inarg{char*}{property}{character string representing the property}
\end{funcargs}
\end{cxxapi}

\gcoll

\begin{desc}

This function allows users to assign properties to an array if there is advanced
knowledge that an array will be used in a certain way. The two properties
currently supported are "read_only" and "read_cache". These can both be set
after the global array is allocated.

The read_only property is similar to creating a mirrored array in that the global
array is replicated across an SMP node. This speeds up access to array data
since all transfers can occur by shared memory copies on the same node. However,
memory utilization increases since the global array is replicated across nodes even
though it is distributed within a node. Setting the read_only property results in
significant overhead since data for the array must be copied and redistributed.
However, if the array is only used for read operations after initialization, this
setting may be usefull in applications.

The read_cache property is useful when an application may be repeatedly
accessing the same data. The read_cache property stores the recent blocks of
data obtained via "get" operations. If a subsequent "get" operation requests the same data or
a subset of the data, the data is obtained from the stored request instead of 
accessing it over the network. This can speed up performance if there are many
duplicate requests for data. The total number of requests that are cached is
limited and requests are stored in the cache on a first in, first out basis.

\end{desc}

\seealso{UNSET PROPERTY}

\apih{SET RESTRICTED}{Specifies a GA handle will be allocated on a subset of processors}

\begin{capi}
\begin{ccode}
void GA_Set_restricted(int g_a, int list[], int nproc)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{list[nproc]}{list of processor IDs that contain data}
\inarg{int}{nproc}{number of processors that contain data}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_restricted(g_a, list, nproc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{list(nproc)}{list of processor IDs that contain data}
\inarg{integer}{nproc}{number of processors that contain data}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setRestricted(int list[], int nprocs) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{list}{list of processors that should contain data}
\inarg{int}{nprocs}{number of processors in list}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_restricted(int g_a, list)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{1D array-like of ints}{list}{list of processor IDs that contain data}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function restricts data in the global array g_a to only the nproc
processors listed in the array list. The value of nproc must be less than or
equal to the number of available processors. If this call is used in
conjunction with GA_Set_irreg_distr, then the decomposition in the
GA_Set_irreg_distr call must be done assuming that the number of processors is
nproc. The data that ordinarily would be mapped to process 0 is mapped to the
process in list[0], the data that would be mapped to process 1 will be mapped
to list[1], etc. This can be used to remap the data distribution to different
processors, even if nproc equals the number of available processors.

\end{desc}

\seealso{SET RESTRICTED RANGE}

\apih{SET RESTRICTED RANGE}{Specifies a GA handle will be created on a subset (as a range) of processors}

\begin{capi}
\begin{ccode}
void GA_set_restricted_range(int g_a, int lo_proc, int hi_proc)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{lo_proc}{range of processors (inclusive) that contain data}
\inarg{int}{hi_proc}{range of processors (inclusive) that contain data}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_restricted_range(g_a, lo_proc, hi_proc)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{lo_proc}{range of processors (inclusive) that contain data}
\inarg{integer}{hi_proc}{range of processors (inclusive) that contain data}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setRestrictedRange(int lo_proc, int hi_proc) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{lo_proc}{low end of processor range}
\inarg{int}{hi_proc}{high end of processor range}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.set_restricted_range(int g_a, int lo_proc, int hi_proc)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{lo_proc}{range of processors (inclusive) that contain data}
\inarg{int}{hi_proc}{range of processors (inclusive) that contain data}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function restricts data in the global array to the range of processors
beginning with lo_proc and ending with hi_proc. Both lo_proc and hi_proc must
be less than or equal to the total number of processors minus one (e.g., in the
range [0,N-1], where N is the total number of processors) and lo_proc must be
less than or equal to hi_proc. If lo_proc = 0 and hi_proc = N-1 then this call
has no effect on the data distribution.

\end{desc}

\seealso{SET RESTRICTED}

\apih{SET TILED IRREG PROC GRID}{This function specifies that the GA will use a distribution of irregularly sized blocks that will be assigned to processors using SCALAPACK-type block cyclic data distribution with individual blocks being contiguous in memory}

\begin{capi}
\begin{ccode}
void GA_Set_tiled_irreg_proc_grid(int g_a, int mapc[], int nblocks[], int proc_grid[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{mapc[]}{starting index of each block}
\inarg{int}{nblocks[]}{array containing the number of blocks in each dimension}
\inarg{int}{proc_grid[]}{processor grid dimensions}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_tiled_irreg_proc_grid(g_a, mapc, nblocks, proc_grid)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{mapc(s)}{starting index of each block. s is equal to the sum of indices in nblocks array}
\inarg{integer}{nblocks(ndim)}{array containing the number of blocks in each dimension}
\inarg{integer}{proc_grid(ndim)}{processor grid dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setTiledIrregProcGrid(int mapc[], int nblocks[], int proc_grid[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{mapc[]}{starting index of each block}
\inarg{int}{nblocks[]}{array containing number of blocks in each dimension}
\inarg{int}{proc_grid[]}{processor grid dimensions}
\end{funcargs}
\end{cxxapi}

\gcoll

\begin{desc}
This subroutine is a combination of the function that set an irregular
distribution and the function for creating a tiled block-cyclic distribution.
Like the conventional tiled block-cylic distribution, individual blocks are
contiguous in memory. Unlike the conventional tiled distribution, individual
blocks can vary in size. The mapc array consists of the starting index of each
block along each dimension and the number of blocks along each axis is stored in
the nblocks array. The number of entries in mapc equals the sum of the entries
in nblocks. The proc_grid array describes how blocks are distributed over
processors using a block-cylic distribution. The product of the entries in
proc_grid is equal to the number of processors in the group that the global
array is created on.

Note that the values in the nblocks arrays should be greater than or equal to
the corresponding values in the proc_grid array.
\end{desc}

\seealso{SET BLOCK CYCLIC PROC GRID,SET IRREG DISTR}

\apih{SET TILED PROC GRID}{This function specifies that the GA will use a regular SCALAPACK-type block cyclic data distribution but individual blocks will be contiguous in memory}

\begin{capi}
\begin{ccode}
void GA_Set_tiled_proc_grid(int g_a, int block_dims[], int proc_grid[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int*}{block_dims[]}{array of block dimensions}
\inarg{int*}{proc_grid[]}{processor grid dimensions}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_set_tiled_proc_grid(g_a, block_dims, proc_grid)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{block_dims(ndim)}{array of block dimensions}
\inarg{integer}{proc_grid(ndim)}{array of processor grid dimensions}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::setTiledProcGrid(int block_dims[], int proc_grid[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{block_dims}{array of block dimensions}
\inarg{int*}{proc_grid}{processor grid dimensions}
\end{funcargs}
\end{cxxapi}

\gcoll

\begin{desc}
This subroutine is used to create a global array with a SCALAPACK-type block
cyclic data distribution but with individual blocks that are contiguous in
memory. This layout may be preferable for algorithms that are moving single
blocks of data since the corresponding messages will involve contiguous data
transfers instead of strided calls. This may result in performance gains for
communication bound programs. Otherwise, the data layout is similar to that
created by the block-cyclic processor grid function.

The user specifies the dimensions of the processor
grid in the array proc_grid. The product of the processor grid dimensions must
equal the number of total number of processors and the number of dimensions in
the processor grid must be the same as the number of dimensions in the global
array. The data blocks are mapped onto the processor grid in a cyclic manner
along each of the processor grid axes. The dimensions of individual blocks are
located in the block_dims array.
\end{desc}

\seealso{SET BLOCK CYCLIC PROC GRID}

\apih{SHIFT DIAGONAL}{Adds specified constant to diagonal elements of a global array}

\begin{capi}
\begin{ccode}
void GA_Shift_diagonal(int g_a, void *c)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{void*}{c}{double/complex/int/long/float shift value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_shift_diagonal(g_a, c)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{double/complex/integer/float}{c}{shift value}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::shiftDiagonal(void *c) const
\end{cxxcode}
\begin{funcargs}
\inarg{void*}{c}{double/complex/int/long/float constant to add}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.shift_diagoal(int g_a, value=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{object}{value}{double/complex/int/long/float shift value}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Adds this constant to the diagonal elements of the matrix.
\end{desc}

\apih{SOLVE}{Solves a system of linear equations}

\begin{capi}
\begin{ccode}
int GA_solve(int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{coefficient matrix}
\inarg{int}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_solve(g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{coefficient matrix}
\inarg{integer}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::solve(const GlobalArray * g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_a}{coefficient matrix}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.solve(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{coefficient matrix}
\inarg{int}{g_b}{rhs/solution matrix}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Solves a system of linear equations
\begin{verbatim}
            A * X = B
\end{verbatim}

It first will call the Cholesky factorization routine and, if sucessfully, will
solve the system with the Cholesky solver. If Cholesky will be not be able to
factorize A, then it will call the LU factorization routine and will solve the
system with forward/backward substitution. On exit B will contain the solution
X.

It returns
\begin{verbatim}
         = 0 : Cholesky factoriztion was succesful
         > 0 : the leading minor of this order
               is not positive definite, Cholesky factorization
               could not be completed and LU factoriztion was used
\end{verbatim}

\end{desc}

\apih{SPD INVERT}{Inverts a symmetric positive definite matrix}

\begin{capi}
\begin{ccode}
int GA_Spd_invert(int g_a)
\end{ccode}
\begin{funcargs}
\inoutarg{int}{g_a}{matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_spd_invert(g_a)
\end{fcode}
\begin{funcargs}
\inoutarg{integer}{g_a}{matrix}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::spdInvert() const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{coefficient matrix}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.spd_invert(int g_a)
\end{pycode}
\begin{funcargs}
\inoutarg{int}{g_a}{matrix}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

It computes the inverse of a double precision using the Cholesky factorization
of a NxN double precision symmetric positive definite matrix A stored in the
global array represented by g_a. On successful exit, A will contain the
inverse.

It returns
\begin{verbatim}
         = 0 : successful exit
         > 0 : the leading minor of this order is not positive
               definite and the factorization could not be completed
         < 0 : it returns the index i of the (i,i)
               element of the factor L/U that is zero and
               the inverse could not be computed
\end{verbatim}

\end{desc}

\apih{SPRS ARRAY ACCESS COL BLOCK}{Directly access a locally owned sublock of matrix}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_access_col_block(int s_a, int iproc, int **idx, int **jdx,
void **val)
void NGA_Sprs_array_access_col_block64_t(int s_a, int iproc, int64_t **idx,
int64_t **jdx,
void **val)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{iproc}{process rank associated with column block}
\outarg{int**}{idx}{pointer to offsets for each row}
\outarg{int**}{jdx}{pointer to column indices for each row}
\outarg{void**}{val}{pointer to values for each row}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_access_col_block(s_a, iproc, idx, jdx, val)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{iproc}{process rank associated with column block}
\outarg{AccessIndex}{idx}{pointer to offsets for each row}
\outarg{AccessIndex}{jdx}{pointer to column indices for each row}
\outarg{AccessIndex}{val}{pointer to values for each row}
\end{fapi}

\ncoll

\begin{desc}
This function returns information on locally held values of a sub-block of a
sparse matrix. Each matrix is divided up into row-blocks and each row-block is
further divided up into column blocks. Calling this function provides access to
one of the column blocks. The data for each block is stored in compressed sparse
row format. The idx pointer points to the array of offsets into jdx and val
corresponding to each set of row values, jdx contains the column indices of
each value and val contains the non-zero values.

If the calling process contains N rows from the matrix, the array idx will
contain N+1 values. The last entry consists of the total number of non-zero
elements in the sub-block and is included so that the total number of non-zero
elements in each row i can be found from the expression idx[i+1]-idx[i]. The
index i runs from 0 to N-1. The true index for the row can be found by adding
i to the low index returned from the NGA_Sprs_array_row_distribution function.

The j indices in jdx represent the global location of the element in the matrix.
The local location in the sub-block can be found by subtracting the lower index
found by calling the NGA_Sprs_array_col_distribution function. Note array
elements in each row are not necessarily sorted in increasing order of the
column index. The values of matrix elements are stored in the array val and
are organized in the same way as the values in jdx.

If j is an index that runs between 0 and idx[i+1]-idx[i]-1 for local index i
and ilo and jlo are the lower indices returned by the
NGA_Sprs_array_row_distribution and NGA_Sprs_array_col_distribution functions,
then the corresponding global index of the row is i+ilo and the global column
index for each of the non-zero elements is jdx[idx[i]+j]. The local column
index is jdx[idx[i]+j]-jlo.
\end{desc}

\begin{cdesc}
If iproc  corresponds to a sub-block with no data, the pointers idx, jdx and val
will all return as NULL.

The arrays idx and jdx reflect the size of the indices that where chosen when
the array was created. If the NGA_Sprs_array_create64 call was used, implying
64-bit integers, then idx and jdx should be int64_t* pointers, otherwise they
should be int* pointers.
\end{cdesc}
\begin{fdesc}
If iproc corresponds to a sub-block with no data, the indices idx, jdx and val
will return as 0. See the discussion of the ga_access function for information
on how the reference indices are used.
\end{fdesc}

\seealso{ACCESS,SPRS ARRAY ROW DISTRIBUTION,SPRS ARRAY COLUMN DISTRIBUTION}

\apih{SPRS ARRAY ADD ELEMENT}{Add a single element to a sparse array data structure}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_add_element(int s_a, int idx, int jdx, void *value)
void NGA_Sprs_array_add_element64(int s_a, int64_t idx, int64_t jdx, void *value)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{idx}{row row index of array element}
\inarg{int}{jdx}{column index of array element}
\inarg{void*}{value}{pointer to element value}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_add_element(s_a, idx, jdx, value)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{idx}{row row index of array element}
\inarg{integer}{jdx}{column index of array element}
\inarg{type}{value}{element value}
\end{fapi}

\ncoll

\begin{desc}
Once a sparse array data structure has been created, non-zero elements can be
added to the array using this function. Any processor can add any element, the
only restriction is that an element can only be added to the array once. It is
up to the user to guarantee that an element is not added multiple times to the
same array. It is advisable to add all diagonal elements to the array even if
they are zero, otherwise some array operations may not work. This function can
only be used after an array is created and before the assemble function is
called on the array.
\end{desc}

\begin{cdesc}
If the 64-bit function is used to create the sparse array, then the 64 bit
version of this function should be used to add elements to the array.
\end{cdesc}

\apih{SPRS ARRAY ASSEMBLE}{Organize and distribute elements of sparse array}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_assemble(int s_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_sprs_array_assemble(s_a)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\end{fapi}

\gcoll

\begin{desc}
Assemble all elements that have been added to a sparse array data structure into
a form that is suitable for computation. After calling this function, no more
elements can be added to the array, although is is possible to modify the values
of existing array elements.  This function returns true if the assembly is
successful.

Internally, this function partitions the sparse array into row blocks, with the
number of row blocks equal to the number of processors. Each row block is
further partitioned into column blocks, with the number of column blocks
equal to the number of processors. The size of the partitions along both the row
and column axes are approximately equal. The data inside each column partion is
organized in a compressed sparse row (CSR) form. Column blocks containing no
data are not included in the data structure, leading to a final data layout the
is equivalent to a sparse array of sparse blocks. This is illustrated
schematically in Figure \ref{sparsearray}.

\begin{figure}
\centering
\includegraphics[width=6.0in,keepaspectratio=true]{sparse-array-layout}
\caption{Sparse array data layout.}
\label{sparsearray}
\end{figure}

\end{desc}

\seealso{SPRS ARRAY CREATE}

\apih{SPRS ARRAY COL BLOCK LIST}{Return a list of column blocks with non-zero entries}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_col_block_list(int s_a, int **idx, int *n)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\outarg{int**}{idx}{array of length n containing indices of non-zero blocks}
\outarg{int*}{n}{number of non-zero blocks}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_col_block_list(s_a, idx, n)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\outarg{integer}{idx(n)}{array of length n to hold indices}
\inoutarg{integer}{n}{on input: length of idx, on output: number of non-zero
blocks}
\end{fapi}

\ncoll

\begin{desc}
This function returns the number of column blocks in the row block owned by the
calling process containing non-zero values.  Typically, this will be smaller than
the number of processors. This function can be used for algorithms that loop
over the column blocks in the row block.
\end{desc}

\begin{cdesc}
The pointer idx is passed into the function and returns pointing to an
allocation containing the requested values. It is up to the calling program to
free idx when it is done using the values.
\end{cdesc}

\begin{fdesc}
The array idx is allocated by the calling program and has length n. If the
number of non-zero blocks on the process is greater than n, then the
subroutine throws an error. The number of non-zero blocks is guaranteed to be
less than or equal to the number of processors, but for many applications it may
be possible to guarantee that the number of non-zero blocks is much smaller than
the number of processors.
\end{fdesc}

\apih{SPRS ARRAY COLUMN DISTRIBUTION}{Column block associated with each process}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_column_distribution(int s_a, int iproc, int *lo, int *hi)
void NGA_Sprs_array_column_distribution64(int s_a, int iproc, int64_t *lo, int64_t *hi)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{iproc}{process rank}
\outarg{int*}{jlo}{lowest column index associated with process iproc}
\outarg{int*}{jlo}{highest column index associated with process iproc}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_column_distribution(s_a, iproc, ilo, ihi)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{iproc}{process rank}
\outarg{integer}{jlo}{lowest column index associated with process iproc}
\outarg{integer}{jlo}{highest column index associated with process iproc}
\end{fapi}

\local

\begin{desc}
This function returns the lower and upper column indices associated with
processor iproc. Note
that ownership of matrix elements is solely determined by the row index but
within a row block the data is further partitioned into blocks associated with
each process. Some blocks may have no data. The bounding column indices of this
secondary partition are returned by this function.  This function will return
valid index ranges even if the block contains no data.  Both C and Fortran
interfaces use 0-based indexing for the column indices.
\end{desc}

\seealso{SPRS ARRAY ACCESS COL BLOCK}

\apih{SPRS ARRAY CREATE}{Creates a sparse array handle}
\begin{capi}
\begin{ccode}
int NGA_Sprs_array_create(int idim, int jdim, int type)
int NGA_Sprs_array_create64(int64_t idim, int64_t jdim, int type)
\end{ccode}
\begin{funcargs}
\inarg{int}{idim}{row dimension}
\inarg{int}{jdim}{column dimension}
\inarg{int}{type}{array datatype}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_create(idim, jdim, type)
\end{fcode}
\begin{funcargs}
\inarg{integer}{idim}{row dimension}
\inarg{integer}{jdim}{column dimension}
\inarg{integer}{type}{array datatype}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}
This function returns a handle to a sparse array data structure with row
dimension idim and column dimension jdim. The datatype can be any of the
standard global array supported data types (user datatypes are currently not
supported).
\end{desc}

\begin{cdesc}
If the 64-bit function is used to create the sparse array, then all other
operations on this array should use the 64-bit versions, if available. In
addition, pointers to internal arrays that access CSR formatted blocks of the
array should be of type int64_t. It is also advisable to configure Global
Arrays using 64-bit integers.
\end{cdesc}

\seealso{SPRS ARRAY DESTROY}

\apih{SPRS ARRAY CREATE FROM DENSE}{Creates a sparse array from a dense array}
\begin{capi}
\begin{ccode}
int NGA_Sprs_array_create_from_dense(int g_a)
int NGA_Sprs_array_create_from_dense64(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{handle for dense array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_create_from_dense(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{handle for dense array}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}
This function returns a handle to a sparse array formed from the non-zero data
values in the dense array g_b. All diagonal values in g_b are included in the
sparse array, regardless of whether or not they are zero.
\end{desc}

\begin{cdesc}
If the 64-bit function is used to create the sparse array, then the index
locations will be stored as 64-bit integers and all other operations on this
array should use the 64-bit versions, if available.
\end{cdesc}

\seealso{SPRS ARRAY DESTROY}

\apih{SPRS ARRAY CREATE FROM SPARSE}{Creates a conventional dense array from a sparse array}
\begin{capi}
\begin{ccode}
int NGA_Sprs_array_create_from_sparse(int s_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_create_from_sparse(s_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{s_a}{handle for sparse array}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}
This function returns a handle to a conventional dense array formed from a
sparse array. All values in the sparse array are copied to the dense array and
all other values are set to zero.
\end{desc}

\seealso{SPRS ARRAY DESTROY}

\apih{SPRS ARRAY DESTROY}{Destroy an existing sparse array}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_destroy(int s_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_sprs_destroy(s_a)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\end{fapi}

\gcoll

\begin{desc}
This function destroys and existing sparse array and frees up resources used by
the array. It returns true if the operation was successful.
\end{desc}

\apih{SPRS ARRAY DIAG LEFT MULTIPLY}{Left multiply the sparse matrix by a diagonal matrix}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_diag_left_multiply(int s_a, int g_d)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{g_d}{handle for global array representing diagonal matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_diag_left_multiply(s_a, g_d)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{g_d}{handle for global array representing diagonal matrix}
\end{fapi}

\gcoll

\begin{desc}
This function will left multiply the sparse matrix s_a by the diagonal matrix
g_d, where g_d is a 1-dimensional vector containing the diagoanl values of the
diagonal matrix. The result is stored in s_a
\end{desc}

\apih{SPRS ARRAY DIAG RIGHT MULTIPLY}{Right multiply the sparse matrix by a diagonal matrix}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_diag_right_multiply(int s_a, int g_d)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{g_d}{handle for global array representing diagonal matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_diag_right_multiply(s_a, g_d)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{g_d}{handle for global array representing diagonal matrix}
\end{fapi}

\gcoll

\begin{desc}
This function will right multiply the sparse matrix s_a by the diagonal matrix
g_d, where g_d is a 1-dimensional vector containing the diagonal values of the
diagonal matrix. The result is stored in s_a.
\end{desc}

\apih{SPRS ARRAY DNSSPRS MULTIPLY}{Multiply a dense matrix by a sparse matrix to get a dense matrix}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_dnssprs_multiply(int g_a, int s_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{handle for dense array}
\inarg{int}{s_b}{handle for sparse array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_dnssprs_multiply(g_a, s_b)
\end{fcode}
\inarg{integer}{g_a}{handle for dense array}
\inarg{integer}{s_b}{handle for sparse array}
\end{fapi}

\gcoll

\begin{desc}
This function multiplies dense matrix A times sparse matrix B to return a
handle to product dense matrix C.
\end{desc}

\seealso{SPRS ARRAY SPRSDNS MULTIPLY,SPRS ARRAY MATMAT MULTIPLY}

\apih{SPRS ARRAY DUPLICATE}{Duplicate an existing sparse array}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_duplicate(int s_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_duplicate(s_a)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\end{fapi}

\gcoll

\begin{desc}
This function returns a handle to a new sparse array that is a duplicate of the
sparse array represented by the handle s_a.
\end{desc}

\apih{SPRS ARRAY EXPORT}{Write out non-zero values of a matrix to a file}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_export(int s_a, char *file)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{char*}{file}{name of file to write out matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_export(s_a, file)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{character(*)*}{file}{name of file to write out matrix}
\end{fapi}

\gcoll

\begin{desc}
This function will export the non-zero values of a sparse matrix to a file. The
data is written in a matrix market format
\begin{verbatim}
          %%MatrixMarket matrix coordinate real general
          idim, jdim, nnz
          i(1), j(1), val(1)
          i(2), j(2), val(2)
                :
          i(nnz), j(nnz), val(nnz)
\end{verbatim}
for nnz non-zero values.
\end{desc}

\apih{SPRS ARRAY GET BLOCK}{Retrieve a subblock of the sparse array}
\begin{capi}
\begin{ccode}
void NGA_Sprs_array_get_block{int s_a, int irow, int icol, int **idx, int **jdx,
void **data, int *ilo, int *ihi, int *jlo, int *jhi)
void NGA_Sprs_array_get_block64{int s_a, int64_t irow, int64_t icol,
int64_t **idx, int64_t **jdx, void **data, int64_t *ilo, int64_t *ihi,
int64_t *jlo, int64_t *jhi)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{sparse array handle}
\inarg{int}{irow}{row index of block}
\inarg{int}{icol}{column index of block}
\outarg{int**}{idx}{array of starting indices for rows in block}
\outarg{int**}{jdx}{array containing column indices of values}
\outarg{void**}{data}{array containing values}
\outarg{int*}{ilo}{lower row index of block}
\outarg{int*}{ihi}{upper row index of block}
\outarg{int*}{jlo}{lower column index of block}
\outarg{int*}{jhi}{upper column index of block}
\end{funcargs}
\end{capi}

\ncoll

\begin{desc}
This function returns arrays in compressed sparse row format (CSR) containing a
block from the sparse array. The block is indexed by irow and icol, both of
which are in the range 0 to nprocs-1. The index irow corresponds to the row
partition (it also corresponds to the process that the data is actually located
on) and icol corresponds to the column partition within the row. If the block
contains no data then idx, jdx and data will all return as NULL pointers. The
remaining parameters ilo, ihi, jlo, jhi are the bounding indices of the block in
the sparse array. These indices are 0-based.

Note that this function allocates the arrays idx, jdx, data so these arrays must
subsequently be freed by the calling program. The idx array is of length
ihi-ilo+2. The extra value at the end is included so that the number of non-zero
values in row i can always be calculated using idx[i+1]-idx[i], even when i =
ihi-ilo. The absolute row index of row i in the block can be calculated using
i + ilo. The elements within each row are not sorted so that the column indices
are in ascending order. This does not appear to be an issue for most
applications.

This function is not availabe at present in the Fortran interface.
\end{desc} 

\apih{SPRS ARRAY GET COLUMN}{Extract the diagonal values of a sparse matrix}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_get_column(int s_a, int icol)
int NGA_Sprs_array_get_column64(int s_a, int64_t icol)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{g_d}{index of column to be extracted}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_get_column(s_a, icol)
\end{fcode}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{g_d}{index of column to be extracted}
\end{fapi}

\gcoll

\begin{desc}
This function will extract the values indexed by icol from a sparse array and
return them in a 1D  global array. The index icol is 0-based for both C and
Fortran interfaces. The return vector is allocated by this routine and must be
destroyed by the application.
\end{desc}

\apih{SPRS ARRAY GET DIAG}{Extract the diagonal values of a sparse matrix}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_get_diag(int s_a, int *g_d)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\outarg{int*}{g_d}{handle for global array containing diagonal values}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_get_diag(s_a, g_d)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\outarg{integer}{g_d}{handle for global array containing diagonal values}
\end{fapi}

\gcoll

\begin{desc}
This function will extract the diagonal values from a square matrix and return
them in a vector (1-dimensional global array). The return vector is allocated by
this routine and must be destroyed by the application.
\end{desc}

\apih{SPRS ARRAY MATMAT MULTIPLY}{Multiply two sparse matrices to get a third sparse matrix}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_matmat_multiply(int s_a, int s_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{s_b}{handle for sparse array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_matmat_multiply(s_a, s_b)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{s_b}{handle for sparse array}
\end{fapi}

\gcoll

\begin{desc}
This function multiplies sparse matrix A times sparse matrix B to return a
handle to product sparse matrix C.
\end{desc}

\apih{SPRS ARRAY MATVEC MULTIPLY}{Multiply a sparse matrix by a vector}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_matvec_multiply(int s_a, int g_a, int g_v)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{g_a}{handle for vector}
\outarg{int}{g_v}{handle for vector}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_matvec_multiply(s_a, g_a, g_v)
\end{fcode}
\inarg{integer}{idim}{row dimension of array}
\inarg{integer}{jdim}{column dimension of array}
\outarg{integer}{type}{array data type}
\end{fapi}

\gcoll

\begin{desc}
This function performs a sparse matrix-vector multiplication. The sparse matrix
represented by the handle s_a is multiplied by the vector g_a (a standard
1-dimensional global array) to get the vector g_v.
\end{desc}

\apih{SPRS ARRAY ROW DISTRIBUTION}{Find rows of sparse array owned by process iproc}
\begin{capi}
\begin{ccode}
void NGA_Sprs_array_row_distribution(int s_a, int iproc, int *lo, int *hi)
void NGA_Sprs_array_row_distribution64(int s_a, int iproc, int64_t *lo, int64_t *hi)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{sparse array handle}
\inarg{int}{iproc}{processor rank}
\outarg{int*}{lo}{lower index range of rows held by process}
\outarg{int*}{hi}{upper index range of rows held by process}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_row_distribution(s_a, idx, jdx, val)
\end{fcode}
\begin{funcargs}
\inarg{integer}{s_a}{sparse array handle}
\inarg{integer}{iproc}{processor rank}
\outarg{integer}{lo}{lower index range of rows held by process}
\outarg{integer}{hi}{upper index range of rows held by process}
\end{funcargs}
\end{fapi}

\local

\begin{desc}
Return the upper and lower row indices of the range of rows held by process
iproc. All rows between ilo and ihi, inclusive, are stored on process iproc.
Both C and Fortran interfaces use 0-based indexing for the row indices.
\end{desc}

\seealso{SPRS ARRAY ACCESS COL BLOCK}

\apih{SPRS ARRAY SHIFT DIAG}{Shift the diagonal values of the matrix by a constant value}

\begin{capi}
\begin{ccode}
void NGA_Sprs_array_shift_diag(int s_a, void *val)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{void*}{val}{amount to shift diagonal}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_sprs_array_shift_diag(s_a, val)
\end{fcode}
\inarg{int}{s_a}{handle for sparse array}
\inarg{type}{val}{amount to shift diagonal}
\end{fapi}

\gcoll

\begin{desc}
This function will shift all diagonal values in the sparse matrix s_a by a fixed
amount. In order for this routine to work correctly, all diagonal values in the
matrix must be present. If a diagonal value in the original matrix is zero, it
should be added to the matrix anyway.
\end{desc}

\apih{SPRS ARRAY SPRSDNS MULTIPLY}{Multiply a sparse matrix by a dense matrix to get a dense matrix}

\begin{capi}
\begin{ccode}
int NGA_Sprs_array_sprsdns_multiply(int s_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{s_a}{handle for sparse array}
\inarg{int}{g_b}{handle for dense array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function nga_sprs_array_sprsdns_multiply(s_a, g_b)
\end{fcode}
\inarg{integer}{s_a}{handle for sparse array}
\inarg{integer}{g_b}{handle for dense array}
\end{fapi}

\gcoll

\begin{desc}
This function multiplies sparse matrix A times dense matrix B to return a
handle to product dense matrix C.
\end{desc}

\seealso{SPRS ARRAY DNSSPRS MULTIPLY,SPRS ARRAY MATMAT MULTIPLY}

\apih{STEP MAX}{Computes a global array's step max}

\begin{capi}
\begin{ccode}
void GA_Step_max(int g_a, int g_b, void *step)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle; step direction}
\outarg{void*}{step}{maximum step size}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_step_max(g_a, g_b, step)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_b}{array handles}
\outarg{}{step}{double precision/integer; the maximum step}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::stepMax(const GlobalArray * g_b,
                          void *step) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_b}{the step direction}
\outarg{void*}{step}{the maximum step}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.step_max(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle; step direction}
\outarg{object}{ret}{scalar same type as array; maximum step size}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Calculates the largest multiple of a vector g_b that can be added to this
vector g_a while keeping each element of this vector non-negative.
\end{desc}

\apih{STEP MAX PATCH}{Computes a global array patch's step max}

\begin{capi}
\begin{ccode}
void GA_Step_max_patch(int g_a, int alo[], int ahi[],
                       int g_b, int blo[], int bhi[],
                       void *step)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle; step direction}
\outarg{void*}{step}{the maximum step}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_step_max_patch(g_a, alo, ahi, g_b, blo, bhi, step)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a,g_b}{array handles where g_b is step direction}
\inarg{integer}{alo,ahi,blo,bhi}{patch coordinates of g_a and g_b}
\outarg{double precision}{step}{the maximum step}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::stepMaxPatch(int *alo, int *ahi,
        const GlobalArray *g_b, int *blo, int *bhi,
        double *step) const
void GlobalArray::stepMaxPatch(
        int64_t *alo, int64_t *ahi,
        const GlobalArray *g_b, int64_t *blo, int64_t *bhi,
        double *step) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_b}{global array representing step direction}
\inarg{int*}{alo}{g_a lower corner patch coordinates}
\inarg{int*}{ahi}{g_a upper corner patch coordinates}
\inarg{int*}{blo}{g_b lower corner patch coordinates}
\inarg{int*}{bhi}{g_b upper corner patch coordinates}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.step_max(int g_a, int g_b, alo=None, ahi=None, blo=None, bhi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{g_b}{array handle; step direction}
\inarg{1D array-like of integers}{alo}{lower bound patch coordinates of g_a, inclusive}
\inarg{1D array-like of integers}{ahi}{higher bound patch coordinates of g_a, exclusive}
\inarg{1D array-like of integers}{blo}{lower bound patch coordinates of g_b, inclusive}
\inarg{1D array-like of integers}{bhi}{higher bound patch coordinates of g_b, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Calculates the largest multiple of a vector g_b that can be added to this
vector g_a while keeping each element of this vector non-negative.
\end{desc}

\apih{STRIDED ACC}{Accumulates strided data into a global array}

\begin{capi}
\begin{ccode}
void NGA_Strided_acc(int g_a, int lo[], int hi[], int skip[], void* buf,
                     int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{int*}{skip[ndim]}{array of strides for each dimension}
\inarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{double/DoubleComplex/long*}{alpha}{scale factor}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_strided_acc(g_a, lo, hi, skip, buf, ld, alpha)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\inarg{integer}{skip(ndim)}{array of strides for each dimension}
\inarg{type}{buf}{local buffer array where the data comes from}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\inarg{type}{alpha}{scale argument for accumulate}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::stridedAcc(int lo[], int hi[], int skip[], void *buf,
                             int ld[], void *alpha) const;
void GlobalArray::stridedAcc(int64_t lo[], int64_t hi[], int64_t skip[],
                             void *buf, int64_t ld[], void *alpha) const;
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for glob array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{int*}{skip[ndim]}{array of strides for each dimension}
\inarg{void*}{buf}{pointer to local buffer array where data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\inarg{void*}{alpha}{double/DoubleComplex/long scale factor}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.strided_acc(int g_a, buffer, lo=None, hi=None, skip=None, alpha=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{1D array-like of ints}{skip}{strides for each dimension}
\inarg{object}{alpha}{scale factor, cast to the appropriate type}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

This operation is the same as NGA_Acc, except that the values corresponding to
dimension n in buf are accumulated to every skip[n] values of the global array
g_a.

Combines data from buffer with data in the global array patch.

The buffer array is assumed to be have the same number of dimensions as the
global array.

global array section (lo[],hi[]) += alpha * buffer

\end{desc}

\seealso{ACC}

\apih{STRIDED GET}{Get strided data from a global array}

\begin{capi}
\begin{ccode}
void NGA_Strided_get(int g_a, int lo[], int hi[], int skip[],
                     void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\inarg{int}{ndim}{number of dimensions of the global array}
\inarg{int*}{lo[ndim]}{array of starting indices for global array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{int*}{skip[ndim]}{array of strides for each dimension}
\outarg{void*}{buf}{pointer to the local buffer array where the data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_strided_get(g_a, lo, hi, skip, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global array section}
\inarg{integer}{hi(ndim)}{array of ending indices for global array section}
\inarg{integer}{skip(ndim)}{array of strides for each dimension}
\outarg{type}{buf}{local buffer array where the data comes from}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for buffer array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::stridedGet(int lo[], int hi[], int skip[],
                             void *buf, int ld[]) const
void GlobalArray::stridedGet(int64_t lo[], int64_t hi[], int64_t skip[],
                             void *buf, int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for glob array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{int*}{skip[ndim]}{array of strides for each dimension}
\outarg{void*}{buf}{pointer to local buffer array where data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.strided_get(int g_a, lo=None, hi=None, skip=None, ndarray buffer=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\inarg{1D array-like of ints}{skip}{strides for each dimension}
\inarg{ndarray}{buffer}{an ndarray of the appropriate type, large enough to hold lo,hi}
\outarg{ndarray}{ret}{local buffer array where the data goes to}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

This operation is the same as NGA_Get, except that the values corresponding to
dimension n in buf correspond to every skip[n] values of the global array g_a.

The local array is assumed to be have the same number of dimensions as the
global array. Any detected inconsitencies/errors in the input arguments are
fatal.

Returns: The local array buffer.

\end{desc}

\seealso{GET}

\apih{STRIDED PUT}{Puts strided data into a global array}

\begin{capi}
\begin{ccode}
void NGA_Strided_put(int g_a, int lo[], int hi[], int skip[],
                     void* buf, int ld[])
\end{ccode}
\begin{funcargs}
\inarg{}{g_a}{global array handle}
\inarg{}{ndim}{number of dimensions of the global array}
\inarg{}{lo[ndim]}{array of starting indices for global array section}
\inarg{}{hi[ndim]}{array of ending indices for global array section}
\inarg{}{skip[ndim]}{array of strides for each dimension}
\outarg{}{buf}{pointer to the local buffer array where the data goes}
\inarg{}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_strided_put(g_a, lo, hi, skip, buf, ld)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\inarg{integer}{ndim}{number of dimensions of the global array}
\inarg{integer}{lo(ndim)}{array of starting indices for global}
\inarg{integer}{hi(ndim)}{array of ending indices for global array}
\inarg{integer}{skip(ndim)}{array of strides for each dimension}
\outarg{type}{buf}{local buffer array where the data comes from}
\inarg{integer}{ld(ndim-1)}{array specifying leading dimensions for array section}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::stridedPut(int lo[], int hi[], int skip[],
                             void*buf, int ld[]) const
void GlobalArray::stridedPut(int64_t lo[], int64_t hi[], int64_t skip[],
                             void *buf, int64_t ld[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo[ndim]}{array of starting indices for glob array section}
\inarg{int*}{hi[ndim]}{array of ending indices for global array section}
\inarg{int*}{skip[ndim]}{array of strides for each dimension}
\inarg{void*}{buf}{pointer to local buffer array where data goes}
\inarg{int*}{ld[ndim-1]}{array specifying leading dimensions/strides/extents for buffer array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.strided_put(int g_a, buffer, lo=None, hi=None, skip=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{the array handle}
\inarg{array-like}{buffer}{the data to put}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\ncoll

\begin{desc}

Strided version of put.  This operation is the same as NGA_Put, except that the
values corresponding to dimension n in buf are copied to every skip[n] values
of the global array g_a.

Copies data from local array buffer to the global array section.

The local array is assumed to be have the same number of dimensions as the
global array.

Any detected inconsitencies/errors in input arguments are fatal.

\end{desc}

\seealso{PUT}

\apih{SUMMARIZE}{Prints summary information on a global array}

\begin{capi}
\begin{ccode}
void GA_Summarize(int verbose)
\end{ccode}
\begin{funcargs}
\inarg{int}{verbose}{If true print distribution info}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_summarize(verbose)
\end{fcode}
\begin{funcargs}
\inarg{logical}{verbose}{If true print distribution info}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::summarize(int verbose) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{verbose}{If true print distribution info}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.summarize(bint verbose)
\end{pycode}
\begin{funcargs}
\inarg{bool}{verbose}{if True print distribution info}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}
Prints info about allocated arrays.
\end{desc}

\apih{SYMMETRIZE}{Symmetrizes a global array}

\begin{capi}
\begin{ccode}
void GA_Symmetrize(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_symmetrize(g_a)
\end{fcode}
\begin{funcargs}
\inoutarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::symmetrize() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.symmetrize(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Symmetrizes matrix A represented with handle g_a: A:= .5 * (A+A').
\end{desc}

\apih{SYNC}{Synchronizes all processes in the default processor group}

\begin{capi}
\begin{ccode}
void GA_Sync()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_sync()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::sync()
GA::sync()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.sync()
\end{pycode}
\end{pyapi}

\dcoll

\begin{desc}

Synchronize processes (a barrier) and ensure that all GA operations completed.

\end{desc}

\apih{TERMINATE}{Terminates GA}

\begin{capi}
\begin{ccode}
void GA_Terminate()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_terminate()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GA::Terminate()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.terminate()
\end{pycode}
\end{pyapi}

\wcoll

\begin{desc}

Delete all active arrays and destroy internal data structures.

\end{desc}

\begin{pydesc}

This functions is automatically called during atexit(). There is no need (and
is an error) to call this function explicitly.

\end{pydesc}

\apih{TOTAL BLOCKS}{Returns number of blocks allocated when using block-cyclic distribution}

\begin{capi}
\begin{ccode}
int GA_Total_blocks(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int}{}{total number of blocks in the block-cyclic distribution}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
integer function ga_total_blocks(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\outarg{integer}{}{total number of blocks in the block-cyclic distribution}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::totalBlocks() const
\end{cxxcode}
\begin{funcargs}
\outarg{int}{}{total number of blocks in the block-cyclic distribution}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.total_blocks(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\outarg{int}{ret}{total number of blocks in the block-cyclic distribution}
\end{funcargs}
\end{pyapi}

\local

\begin{desc}

This function returns the total number of blocks contained in a global array
with a block-cyclic data distribution.

\end{desc}

\seealso{SET BLOCK CYCLIC,SET BLOCK CYCLIC PROC GRID}

\apih{TRANSPOSE}{Transposes a global array}

\begin{capi}
\begin{ccode}
void GA_Transpose(int g_a, int g_b)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{original matrix}
\inarg{int}{g_b}{solution matrix}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_transpose(g_a, g_b)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{remains unchanged}
\inarg{integer}{g_b}{solution matrix}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::transpose(const GlobalArray * g_a) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{g_a}{assign transpose to this GlobalArray}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.transpose(int g_a, int g_b)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{original matrix}
\inarg{int}{g_b}{solution matrix}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Transposes a matrix: B = A', where A and B are represented by handles g_a and
g_b.
\end{desc}

\apih{UNLOCK}{Unlocks a mutex}

\begin{capi}
\begin{ccode}
void GA_Unlock(int mutex)
\end{ccode}
\begin{funcargs}
\inarg{int}{mutex}{mutex object id}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_unlock(mutex)
\end{fcode}
\begin{funcargs}
\inarg{integer}{mutex}{mutex id}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
GAServices::unlock(int mutex)
\end{cxxcode}
\begin{funcargs}
\inarg{int}{mutex}{mutex object id}
\end{funcargs}
\end{cxxapi}

\ncoll

\begin{desc}

Unlocks a mutex object identified by the mutex number. It is a fatal error for
a process to attempt to unlock a mutex which has not been locked by this
process.

\end{desc}

\seealso{CREATE MUTEXES,DESTROY MUTEXES,LOCK}

\apih{UNPACK}{Expands values from a source vector based on a corresponding integer mask}

\begin{capi}
\begin{ccode}
void GA_Unpack(int g_src, int g_dest, int g_mask,
               int lo, int hi, int *icount)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source array}
\inarg{int}{g_dest}{handle for destination array}
\inarg{int}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo,hi}{low and high values of range on which operation is performed}
\outarg{int*}{icount}{number of values in uncompressed array}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_unpack(g_src, g_dest, g_mask, lo, hi, icount)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_src}{handle for source array}
\outarg{integer}{g_dest}{handle for destination array}
\inarg{integer}{g_mask}{handle for integer array representing a bit mask}
\inarg{integer}{lo,hi}{low and high values of range on which operation is performed}
\outarg{integer}{icount}{number of values in uncompressed array}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::unpack(GlobalArray *g_dest, GlobalArray *g_mask,
                         int lo, int hi, int *icount) const
void GlobalArray::unpack(GlobalArray *g_dest, GlobalArray *g_mask,
                         int64_t lo, int64_t hi, int64_t *icount) const
\end{cxxcode}
\begin{funcargs}
\inarg{const GlobalArray*}{g_dest}{handle for destination array}
\inarg{const GlobalArray*}{g_mask}{handle for integer array representing mask}
\inarg{int}{lo}{low value of range on which operation is performed}
\inarg{int}{hi}{high value of range on which operation is performed}
\outarg{int*}{icount}{number of values in uncompressed array}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.unpack(int g_src, int g_dst, int g_msk, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_src}{handle for source arrray}
\inarg{int}{g_dst}{handle for destination array}
\inarg{int}{g_msk}{handle for integer array representing mask}
\inarg{1D array-like of ints}{lo}{low value of range on which operation is performed}
\inarg{1D array-like of ints}{hi}{hi value of range on which operation is performed}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
The unpack subroutine is designed to expand the values in the source vector
g_src into a larger destination array g_dest based on the values in an integer
mask array g_mask. The values lo and hi denote the range of elements that
should be compressed and icount is a variable that on output lists the number
of values placed in the uncompressed array. This operation is the complement of
the GA_Pack operation. An example is shown below.
\begin{verbatim}
GA_Unpack(g_src, g_dest, g_mask, 1, n, &icount);

g_src:    1  7  9 12 15 16
g_mask:   1  0  0  0  0  0  1  0  1  0  0  1  0  0  1  1  0
g_dest:   1  0  0  0  0  0  7  0  9  0  0 12  0  0 15 16  0
icount:   6

\end{verbatim}

The current implementation requires that the distribution of the g_mask array
matches the distribution of the g_dest array.
\end{desc}

\apih{UNSET PROPERTY}{Assign a property to a global array}

\begin{capi}
\begin{ccode}
void GA_Unset_property(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{global array handle}
\end{funcargs}
\end{capi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::unsetProperty()
\end{cxxcode}
\end{cxxapi}

\begin{fapi}
\begin{fcode}
subroutine ga_unset_property(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{global array handle}
\end{funcargs}
\end{fapi}

\gcoll

\begin{desc}
This function clears properties from the global array.
\end{desc}

\seealso{SET PROPERTY}

\apih{UPDATE GHOST DIR}{Updates ghost cells along a specific direction}

\begin{capi}
\begin{ccode}
int NGA_Update_ghost_dir(int g_a, int dimension, int idir, int cflag)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{dimension}{array dimension that is to be updated}
\inarg{int}{idir}{direction of update (+/- 1)}
\inarg{int}{cflag}{flag (0/1) to include corners in update}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function nga_update_ghost_dir(g_a,dimension,idir,cflag)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\inarg{integer}{dimension}{array dimension that is to be updated}
\inarg{integer}{idir}{direction of update (+/-1)}
\inarg{logical}{cflag}{flag to include corners in update}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::updateGhostDir(int dimension, int idir, int cflag) const
\end{cxxcode}
\begin{funcargs}
\inarg{int}{dimension}{array dimension that is to be updated}
\inarg{int}{idir}{direction of update (+/- 1)}
\inarg{int}{cflag}{flag (0/1) to include corners in update}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.update_ghost_dir(int g_a, int dimension, int dir, int flag)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{int}{dimension}{array dimension that is to be updated}
\inarg{int}{dir}{direction of update (+/- 1)}
\inarg{int}{flag}{flag (0/1) to include corners in update}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This function can be used to update the ghost cells along individual
directions. It is designed for algorithms that can overlap updates with
computation. The variable dimension indicates which coordinate direction is to
be updated (e.g. dimension = 1 would correspond to the y axis in a two or three
dimensional system), the variable idir can take the values +/-1 and indicates
whether the side that is to be updated lies in the positive or negative
direction, and cflag indicates whether or not the corners on the side being
updated are to be included in the update. The following calls would be
equivalent to a call to GA_Update_ghosts for a 2-dimensional system:

\begin{verbatim}
     status = NGA_Update_ghost_dir(g_a,0,-1,1);
     status = NGA_Update_ghost_dir(g_a,0,1,1);
     status = NGA_Update_ghost_dir(g_a,1,-1,0);
     status = NGA_Update_ghost_dir(g_a,1,1,0);
\end{verbatim}

The variable cflag is set equal to 1 (or non-zero) in the first two calls so
that the corner ghost cells are update, it is set equal to 0 in the second two
calls to avoid redundant updates of the corners. Note that updating the ghosts
cells using several independent calls to the nga_update_ghost_dir functions is
generally not as efficient as using GA_Update_ghosts unless the individual
calls can be effectively overlapped with computation.

\end{desc}

\apih{UPDATE GHOSTS}{Updates ghost cells}

\begin{capi}
\begin{ccode}
void GA_Update_ghosts(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_update_ghosts(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::updateGhosts() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.update_ghosts(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

This call updates the ghost cell regions on each processor with the
corresponding neighbor data from other processors. The operation assumes that
all data is wrapped around using periodic boundary data so that ghost cell data
that goes beyound an array boundary is wrapped around to the other end of the
array. The GA_Update_ghosts call contains two GA_Sync calls before and after
the actual update operation. For some applications these calls may be
unecessary, if so they can be removed using the GA_Mask_sync subroutine.

\end{desc}

\apih{USES MA}{Checks whether GA uses MA}

\begin{capi}
\begin{ccode}
int GA_Uses_ma()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
logical function ga_uses_ma()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
int GlobalArray::usesMA()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.uses_ma()
\end{pycode}
\end{pyapi}

\local

\begin{desc}

Returns ``1'' if memory in global arrays comes from the Memory Allocator (MA).
``0'' means that memory comes from another source, for example System V shared
memory is used.

TODO

\end{desc}

\apih{VERSION}{Returns information on release used to build library}

\begin{capi}
\begin{ccode}
void GA_Version(int *major, int *minor, int *patch)
void NGA_Version(int *major, int *minor, int *patch)
\end{ccode}
\begin{funcargs}
\outarg{int}{major}{major release number}
\outarg{int}{minor}{minor release number}
\outarg{int}{patch}{patch/hotfix release number}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_version(major, minor, patch)
subroutine nga_version(major, minor, patch)
\end{fcode}
\begin{funcargs}
\outarg{integer}{major}{major release number}
\outarg{integer}{minor}{minor release number}
\outarg{integer}{patch}{patch/hotfix release number}
\end{funcargs}
\end{fapi}

\local

\begin{desc}
Function to query version information about the release used to build the
library. This can be used in codes to check for availability of functionality.
\end{desc}

\apih{WTIME}{Returns time in seconds since an arbitrary time in the past}

\begin{capi}
\begin{ccode}
double GA_Wtime()
\end{ccode}
\end{capi}

\begin{fapi}
\begin{fcode}
double precision function ga_wtime()
\end{fcode}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
double GlobalArray::wtime()
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ret = ga.wtime()
\end{pycode}
\end{pyapi}

\local

\begin{desc}
This function returns a wall (or elapsed) time on the calling processor.
Returns time in seconds representing elapsed wall-clock time since an arbitrary
time in the past. Example:
\begin{verbatim}
double starttime, endtime;
starttime = GA_Wtime();
.... code snippet to be timed ....
endtime   = GA_Wtime();
printf("Time taken = %lf seconds\n", endtime-starttime);
\end{verbatim}
This function is only available in release 4.1 or greater.
\end{desc}

\apih{ZERO}{Zeros a global array}

\begin{capi}
\begin{ccode}
void GA_Zero(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_zero(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::zero() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.zero(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Sets value of all elements in the array to zero.

\end{desc}

\apih{ZERO DIAGONAL}{Zeros the diagonal elements of a global array}

\begin{capi}
\begin{ccode}
void GA_Zero_diagonal(int g_a)
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine ga_zero_diagonal(g_a)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{array handle}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::zeroDiagonal() const
\end{cxxcode}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.zero_diagonal(int g_a)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}
Sets the diagonal elements of this matrix g_a with zeros.
\end{desc}

\apih{ZERO PATCH}{Zeros a patch of a global array}

\begin{capi}
\begin{ccode}
void NGA_Zero_patch(int g_a, int lo[], int hi[])
\end{ccode}
\begin{funcargs}
\inarg{int}{g_a}{array handles}
\inarg{int*}{lo[], hi[]}{patch of g_a}
\end{funcargs}
\end{capi}

\begin{fapi}
\begin{fcode}
subroutine nga_zero_patch(g_a, alo, ahi)
\end{fcode}
\begin{funcargs}
\inarg{integer}{g_a}{}
\inarg{integer}{ndim}{number of dimensions}
\inarg{integer}{alo(ndim), ahi(ndim)}{g_a patch coordinates}
\end{funcargs}
\end{fapi}

\begin{cxxapi}
\begin{cxxcode}
void GlobalArray::zeroPatch (int lo[], int hi[]) const
void GlobalArray::zeroPatch (int64_t lo[], int64_t hi[]) const
\end{cxxcode}
\begin{funcargs}
\inarg{int*}{lo}{patch of this GlobalArray}
\inarg{int*}{hi}{patch of this GlobalArray}
\end{funcargs}
\end{cxxapi}

\begin{pyapi}
\begin{pycode}
ga.zero(int g_a, lo=None, hi=None)
\end{pycode}
\begin{funcargs}
\inarg{int}{g_a}{array handle}
\inarg{1D array-like of ints}{lo}{lower bound patch coordinates, inclusive}
\inarg{1D array-like of ints}{hi}{higher bound patch coordinates, exclusive}
\end{funcargs}
\end{pyapi}

\gcoll

\begin{desc}

Set all the elements in the patch to zero.

\end{desc}

\seealso{ZERO}
\end{document}