p86739025
/
PHengLEI-NCCR
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
PHengLEI-NCCR/phenglei/3rdparty/hypre/include/_hypre_utilities.h

3117 lines
121 KiB
C
Raw Blame History

/*** DO NOT EDIT THIS FILE DIRECTLY (use 'headers' to generate) ***/
#ifndef hypre_UTILITIES_HEADER
#define hypre_UTILITIES_HEADER
#include "HYPRE_utilities.h"
#ifdef HYPRE_USING_OPENMP
#include <omp.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* General structures and values
*
*****************************************************************************/
#ifndef hypre_GENERAL_HEADER
#define hypre_GENERAL_HEADER
/* This allows us to consistently avoid 'int' throughout hypre */
typedef int hypre_int;
typedef long int hypre_longint;
typedef unsigned int hypre_uint;
typedef unsigned long int hypre_ulongint;
typedef unsigned long long int hypre_ulonglongint;
/* This allows us to consistently avoid 'double' throughout hypre */
typedef double hypre_double;
/*--------------------------------------------------------------------------
* Define various functions
*--------------------------------------------------------------------------*/
#ifndef hypre_max
#define hypre_max(a,b) (((a)<(b)) ? (b) : (a))
#endif
#ifndef hypre_min
#define hypre_min(a,b) (((a)<(b)) ? (a) : (b))
#endif
#ifndef hypre_abs
#define hypre_abs(a) (((a)>0) ? (a) : -(a))
#endif
#ifndef hypre_round
#define hypre_round(x) ( ((x) < 0.0) ? ((HYPRE_Int)(x - 0.5)) : ((HYPRE_Int)(x + 0.5)) )
#endif
#ifndef hypre_pow2
#define hypre_pow2(i) ( 1 << (i) )
#endif
#endif /* hypre_GENERAL_HEADER */
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef hypre_PRINTF_HEADER
#define hypre_PRINTF_HEADER
#include <stdio.h>
/* printf.c */
// #ifdef HYPRE_BIGINT
HYPRE_Int hypre_ndigits( HYPRE_BigInt number );
HYPRE_Int hypre_printf( const char *format, ... );
HYPRE_Int hypre_fprintf( FILE *stream, const char *format, ... );
HYPRE_Int hypre_sprintf( char *s, const char *format, ... );
HYPRE_Int hypre_scanf( const char *format, ... );
HYPRE_Int hypre_fscanf( FILE *stream, const char *format, ... );
HYPRE_Int hypre_sscanf( char *s, const char *format, ... );
// #else
// #define hypre_printf printf
// #define hypre_fprintf fprintf
// #define hypre_sprintf sprintf
// #define hypre_scanf scanf
// #define hypre_fscanf fscanf
// #define hypre_sscanf sscanf
// #endif
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef hypre_ERROR_HEADER
#define hypre_ERROR_HEADER
#include <assert.h>
/*--------------------------------------------------------------------------
* Global variable used in hypre error checking
*--------------------------------------------------------------------------*/
extern HYPRE_Int hypre__global_error;
#define hypre_error_flag hypre__global_error
/*--------------------------------------------------------------------------
* HYPRE error macros
*--------------------------------------------------------------------------*/
void hypre_error_handler(const char *filename, HYPRE_Int line, HYPRE_Int ierr, const char *msg);
#define hypre_error(IERR) hypre_error_handler(__FILE__, __LINE__, IERR, NULL)
#define hypre_error_w_msg(IERR, msg) hypre_error_handler(__FILE__, __LINE__, IERR, msg)
#define hypre_error_in_arg(IARG) hypre_error(HYPRE_ERROR_ARG | IARG<<3)
#if defined(HYPRE_DEBUG)
/* host assert */
#define hypre_assert(EX) do { if (!(EX)) { fprintf(stderr, "[%s, %d] hypre_assert failed: %s\n", __FILE__, __LINE__, #EX); hypre_error(1); assert(0); } } while (0)
/* device assert */
#if defined(HYPRE_USING_CUDA)
#define hypre_device_assert(EX) assert(EX)
#elif defined(HYPRE_USING_HIP)
/* FIXME: Currently, asserts in device kernels in HIP do not behave well */
#define hypre_device_assert(EX)
#endif
#else /* #ifdef HYPRE_DEBUG */
/* this is to silence compiler's unused variable warnings */
#ifdef __cplusplus
#define hypre_assert(EX) do { if (0) { static_cast<void> (EX); } } while (0)
#else
#define hypre_assert(EX) do { if (0) { (void) (EX); } } while (0)
#endif
#define hypre_device_assert(EX)
#endif
#endif /* hypre_ERROR_HEADER */
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* Fake mpi stubs to generate serial codes without mpi
*
*****************************************************************************/
#ifndef hypre_MPISTUBS
#define hypre_MPISTUBS
#ifdef __cplusplus
extern "C" {
#endif
#ifdef HYPRE_SEQUENTIAL
/******************************************************************************
* MPI stubs to generate serial codes without mpi
*****************************************************************************/
/*--------------------------------------------------------------------------
* Change all MPI names to hypre_MPI names to avoid link conflicts.
*
* NOTE: MPI_Comm is the only MPI symbol in the HYPRE user interface,
* and is defined in `HYPRE_utilities.h'.
*--------------------------------------------------------------------------*/
#define MPI_Comm hypre_MPI_Comm
#define MPI_Group hypre_MPI_Group
#define MPI_Request hypre_MPI_Request
#define MPI_Datatype hypre_MPI_Datatype
#define MPI_Status hypre_MPI_Status
#define MPI_Op hypre_MPI_Op
#define MPI_Aint hypre_MPI_Aint
#define MPI_Info hypre_MPI_Info
#define MPI_COMM_WORLD hypre_MPI_COMM_WORLD
#define MPI_COMM_NULL hypre_MPI_COMM_NULL
#define MPI_COMM_SELF hypre_MPI_COMM_SELF
#define MPI_COMM_TYPE_SHARED hypre_MPI_COMM_TYPE_SHARED
#define MPI_BOTTOM hypre_MPI_BOTTOM
#define MPI_FLOAT hypre_MPI_FLOAT
#define MPI_DOUBLE hypre_MPI_DOUBLE
#define MPI_LONG_DOUBLE hypre_MPI_LONG_DOUBLE
#define MPI_INT hypre_MPI_INT
#define MPI_LONG_LONG_INT hypre_MPI_LONG_LONG_INT
#define MPI_CHAR hypre_MPI_CHAR
#define MPI_LONG hypre_MPI_LONG
#define MPI_BYTE hypre_MPI_BYTE
#define MPI_C_DOUBLE_COMPLEX hypre_MPI_COMPLEX
#define MPI_SUM hypre_MPI_SUM
#define MPI_MIN hypre_MPI_MIN
#define MPI_MAX hypre_MPI_MAX
#define MPI_LOR hypre_MPI_LOR
#define MPI_LAND hypre_MPI_LAND
#define MPI_SUCCESS hypre_MPI_SUCCESS
#define MPI_STATUSES_IGNORE hypre_MPI_STATUSES_IGNORE
#define MPI_UNDEFINED hypre_MPI_UNDEFINED
#define MPI_REQUEST_NULL hypre_MPI_REQUEST_NULL
#define MPI_INFO_NULL hypre_MPI_INFO_NULL
#define MPI_ANY_SOURCE hypre_MPI_ANY_SOURCE
#define MPI_ANY_TAG hypre_MPI_ANY_TAG
#define MPI_SOURCE hypre_MPI_SOURCE
#define MPI_TAG hypre_MPI_TAG
#define MPI_Init hypre_MPI_Init
#define MPI_Finalize hypre_MPI_Finalize
#define MPI_Abort hypre_MPI_Abort
#define MPI_Wtime hypre_MPI_Wtime
#define MPI_Wtick hypre_MPI_Wtick
#define MPI_Barrier hypre_MPI_Barrier
#define MPI_Comm_create hypre_MPI_Comm_create
#define MPI_Comm_dup hypre_MPI_Comm_dup
#define MPI_Comm_f2c hypre_MPI_Comm_f2c
#define MPI_Comm_group hypre_MPI_Comm_group
#define MPI_Comm_size hypre_MPI_Comm_size
#define MPI_Comm_rank hypre_MPI_Comm_rank
#define MPI_Comm_free hypre_MPI_Comm_free
#define MPI_Comm_split hypre_MPI_Comm_split
#define MPI_Comm_split_type hypre_MPI_Comm_split_type
#define MPI_Group_incl hypre_MPI_Group_incl
#define MPI_Group_free hypre_MPI_Group_free
#define MPI_Address hypre_MPI_Address
#define MPI_Get_count hypre_MPI_Get_count
#define MPI_Alltoall hypre_MPI_Alltoall
#define MPI_Allgather hypre_MPI_Allgather
#define MPI_Allgatherv hypre_MPI_Allgatherv
#define MPI_Gather hypre_MPI_Gather
#define MPI_Gatherv hypre_MPI_Gatherv
#define MPI_Scatter hypre_MPI_Scatter
#define MPI_Scatterv hypre_MPI_Scatterv
#define MPI_Bcast hypre_MPI_Bcast
#define MPI_Send hypre_MPI_Send
#define MPI_Recv hypre_MPI_Recv
#define MPI_Isend hypre_MPI_Isend
#define MPI_Irecv hypre_MPI_Irecv
#define MPI_Send_init hypre_MPI_Send_init
#define MPI_Recv_init hypre_MPI_Recv_init
#define MPI_Irsend hypre_MPI_Irsend
#define MPI_Startall hypre_MPI_Startall
#define MPI_Probe hypre_MPI_Probe
#define MPI_Iprobe hypre_MPI_Iprobe
#define MPI_Test hypre_MPI_Test
#define MPI_Testall hypre_MPI_Testall
#define MPI_Wait hypre_MPI_Wait
#define MPI_Waitall hypre_MPI_Waitall
#define MPI_Waitany hypre_MPI_Waitany
#define MPI_Allreduce hypre_MPI_Allreduce
#define MPI_Reduce hypre_MPI_Reduce
#define MPI_Scan hypre_MPI_Scan
#define MPI_Request_free hypre_MPI_Request_free
#define MPI_Type_contiguous hypre_MPI_Type_contiguous
#define MPI_Type_vector hypre_MPI_Type_vector
#define MPI_Type_hvector hypre_MPI_Type_hvector
#define MPI_Type_struct hypre_MPI_Type_struct
#define MPI_Type_commit hypre_MPI_Type_commit
#define MPI_Type_free hypre_MPI_Type_free
#define MPI_Op_free hypre_MPI_Op_free
#define MPI_Op_create hypre_MPI_Op_create
#define MPI_User_function hypre_MPI_User_function
#define MPI_Info_create hypre_MPI_Info_create
/*--------------------------------------------------------------------------
* Types, etc.
*--------------------------------------------------------------------------*/
/* These types have associated creation and destruction routines */
typedef HYPRE_Int hypre_MPI_Comm;
typedef HYPRE_Int hypre_MPI_Group;
typedef HYPRE_Int hypre_MPI_Request;
typedef HYPRE_Int hypre_MPI_Datatype;
typedef void (hypre_MPI_User_function) ();
typedef struct
{
HYPRE_Int hypre_MPI_SOURCE;
HYPRE_Int hypre_MPI_TAG;
} hypre_MPI_Status;
typedef HYPRE_Int hypre_MPI_Op;
typedef HYPRE_Int hypre_MPI_Aint;
typedef HYPRE_Int hypre_MPI_Info;
#define hypre_MPI_COMM_SELF 1
#define hypre_MPI_COMM_WORLD 0
#define hypre_MPI_COMM_NULL -1
#define hypre_MPI_COMM_TYPE_SHARED 0
#define hypre_MPI_BOTTOM 0x0
#define hypre_MPI_FLOAT 0
#define hypre_MPI_DOUBLE 1
#define hypre_MPI_LONG_DOUBLE 2
#define hypre_MPI_INT 3
#define hypre_MPI_CHAR 4
#define hypre_MPI_LONG 5
#define hypre_MPI_BYTE 6
#define hypre_MPI_REAL 7
#define hypre_MPI_COMPLEX 8
#define hypre_MPI_LONG_LONG_INT 9
#define hypre_MPI_SUM 0
#define hypre_MPI_MIN 1
#define hypre_MPI_MAX 2
#define hypre_MPI_LOR 3
#define hypre_MPI_LAND 4
#define hypre_MPI_SUCCESS 0
#define hypre_MPI_STATUSES_IGNORE 0
#define hypre_MPI_UNDEFINED -9999
#define hypre_MPI_REQUEST_NULL 0
#define hypre_MPI_INFO_NULL 0
#define hypre_MPI_ANY_SOURCE 1
#define hypre_MPI_ANY_TAG 1
#else
/******************************************************************************
* MPI stubs to do casting of HYPRE_Int and hypre_int correctly
*****************************************************************************/
typedef MPI_Comm hypre_MPI_Comm;
typedef MPI_Group hypre_MPI_Group;
typedef MPI_Request hypre_MPI_Request;
typedef MPI_Datatype hypre_MPI_Datatype;
typedef MPI_Status hypre_MPI_Status;
typedef MPI_Op hypre_MPI_Op;
typedef MPI_Aint hypre_MPI_Aint;
typedef MPI_Info hypre_MPI_Info;
typedef MPI_User_function hypre_MPI_User_function;
#define hypre_MPI_COMM_WORLD MPI_COMM_WORLD
#define hypre_MPI_COMM_NULL MPI_COMM_NULL
#define hypre_MPI_BOTTOM MPI_BOTTOM
#define hypre_MPI_COMM_SELF MPI_COMM_SELF
#define hypre_MPI_COMM_TYPE_SHARED MPI_COMM_TYPE_SHARED
#define hypre_MPI_FLOAT MPI_FLOAT
#define hypre_MPI_DOUBLE MPI_DOUBLE
#define hypre_MPI_LONG_DOUBLE MPI_LONG_DOUBLE
/* HYPRE_MPI_INT is defined in HYPRE_utilities.h */
#define hypre_MPI_INT HYPRE_MPI_INT
#define hypre_MPI_CHAR MPI_CHAR
#define hypre_MPI_LONG MPI_LONG
#define hypre_MPI_BYTE MPI_BYTE
/* HYPRE_MPI_REAL is defined in HYPRE_utilities.h */
#define hypre_MPI_REAL HYPRE_MPI_REAL
/* HYPRE_MPI_COMPLEX is defined in HYPRE_utilities.h */
#define hypre_MPI_COMPLEX HYPRE_MPI_COMPLEX
#define hypre_MPI_SUM MPI_SUM
#define hypre_MPI_MIN MPI_MIN
#define hypre_MPI_MAX MPI_MAX
#define hypre_MPI_LOR MPI_LOR
#define hypre_MPI_SUCCESS MPI_SUCCESS
#define hypre_MPI_STATUSES_IGNORE MPI_STATUSES_IGNORE
#define hypre_MPI_UNDEFINED MPI_UNDEFINED
#define hypre_MPI_REQUEST_NULL MPI_REQUEST_NULL
#define hypre_MPI_INFO_NULL MPI_INFO_NULL
#define hypre_MPI_ANY_SOURCE MPI_ANY_SOURCE
#define hypre_MPI_ANY_TAG MPI_ANY_TAG
#define hypre_MPI_SOURCE MPI_SOURCE
#define hypre_MPI_TAG MPI_TAG
#define hypre_MPI_LAND MPI_LAND
#endif
/******************************************************************************
* Everything below this applies to both ifdef cases above
*****************************************************************************/
/*--------------------------------------------------------------------------
* Prototypes
*--------------------------------------------------------------------------*/
/* mpistubs.c */
HYPRE_Int hypre_MPI_Init( hypre_int *argc, char ***argv );
HYPRE_Int hypre_MPI_Finalize( void );
HYPRE_Int hypre_MPI_Abort( hypre_MPI_Comm comm, HYPRE_Int errorcode );
HYPRE_Real hypre_MPI_Wtime( void );
HYPRE_Real hypre_MPI_Wtick( void );
HYPRE_Int hypre_MPI_Barrier( hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Comm_create( hypre_MPI_Comm comm, hypre_MPI_Group group,
hypre_MPI_Comm *newcomm );
HYPRE_Int hypre_MPI_Comm_dup( hypre_MPI_Comm comm, hypre_MPI_Comm *newcomm );
hypre_MPI_Comm hypre_MPI_Comm_f2c( hypre_int comm );
HYPRE_Int hypre_MPI_Comm_size( hypre_MPI_Comm comm, HYPRE_Int *size );
HYPRE_Int hypre_MPI_Comm_rank( hypre_MPI_Comm comm, HYPRE_Int *rank );
HYPRE_Int hypre_MPI_Comm_free( hypre_MPI_Comm *comm );
HYPRE_Int hypre_MPI_Comm_group( hypre_MPI_Comm comm, hypre_MPI_Group *group );
HYPRE_Int hypre_MPI_Comm_split( hypre_MPI_Comm comm, HYPRE_Int n, HYPRE_Int m,
hypre_MPI_Comm * comms );
HYPRE_Int hypre_MPI_Group_incl( hypre_MPI_Group group, HYPRE_Int n, HYPRE_Int *ranks,
hypre_MPI_Group *newgroup );
HYPRE_Int hypre_MPI_Group_free( hypre_MPI_Group *group );
HYPRE_Int hypre_MPI_Address( void *location, hypre_MPI_Aint *address );
HYPRE_Int hypre_MPI_Get_count( hypre_MPI_Status *status, hypre_MPI_Datatype datatype,
HYPRE_Int *count );
HYPRE_Int hypre_MPI_Alltoall( void *sendbuf, HYPRE_Int sendcount, hypre_MPI_Datatype sendtype,
void *recvbuf, HYPRE_Int recvcount, hypre_MPI_Datatype recvtype, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Allgather( void *sendbuf, HYPRE_Int sendcount, hypre_MPI_Datatype sendtype,
void *recvbuf, HYPRE_Int recvcount, hypre_MPI_Datatype recvtype, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Allgatherv( void *sendbuf, HYPRE_Int sendcount, hypre_MPI_Datatype sendtype,
void *recvbuf, HYPRE_Int *recvcounts, HYPRE_Int *displs, hypre_MPI_Datatype recvtype,
hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Gather( void *sendbuf, HYPRE_Int sendcount, hypre_MPI_Datatype sendtype,
void *recvbuf, HYPRE_Int recvcount, hypre_MPI_Datatype recvtype, HYPRE_Int root,
hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Gatherv( void *sendbuf, HYPRE_Int sendcount, hypre_MPI_Datatype sendtype,
void *recvbuf, HYPRE_Int *recvcounts, HYPRE_Int *displs, hypre_MPI_Datatype recvtype,
HYPRE_Int root, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Scatter( void *sendbuf, HYPRE_Int sendcount, hypre_MPI_Datatype sendtype,
void *recvbuf, HYPRE_Int recvcount, hypre_MPI_Datatype recvtype, HYPRE_Int root,
hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Scatterv( void *sendbuf, HYPRE_Int *sendcounts, HYPRE_Int *displs,
hypre_MPI_Datatype sendtype, void *recvbuf, HYPRE_Int recvcount, hypre_MPI_Datatype recvtype,
HYPRE_Int root, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Bcast( void *buffer, HYPRE_Int count, hypre_MPI_Datatype datatype,
HYPRE_Int root, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Send( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype, HYPRE_Int dest,
HYPRE_Int tag, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Recv( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype, HYPRE_Int source,
HYPRE_Int tag, hypre_MPI_Comm comm, hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Isend( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype, HYPRE_Int dest,
HYPRE_Int tag, hypre_MPI_Comm comm, hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Irecv( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype,
HYPRE_Int source, HYPRE_Int tag, hypre_MPI_Comm comm, hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Send_init( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype,
HYPRE_Int dest, HYPRE_Int tag, hypre_MPI_Comm comm, hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Recv_init( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype,
HYPRE_Int dest, HYPRE_Int tag, hypre_MPI_Comm comm, hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Irsend( void *buf, HYPRE_Int count, hypre_MPI_Datatype datatype, HYPRE_Int dest,
HYPRE_Int tag, hypre_MPI_Comm comm, hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Startall( HYPRE_Int count, hypre_MPI_Request *array_of_requests );
HYPRE_Int hypre_MPI_Probe( HYPRE_Int source, HYPRE_Int tag, hypre_MPI_Comm comm,
hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Iprobe( HYPRE_Int source, HYPRE_Int tag, hypre_MPI_Comm comm, HYPRE_Int *flag,
hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Test( hypre_MPI_Request *request, HYPRE_Int *flag, hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Testall( HYPRE_Int count, hypre_MPI_Request *array_of_requests, HYPRE_Int *flag,
hypre_MPI_Status *array_of_statuses );
HYPRE_Int hypre_MPI_Wait( hypre_MPI_Request *request, hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Waitall( HYPRE_Int count, hypre_MPI_Request *array_of_requests,
hypre_MPI_Status *array_of_statuses );
HYPRE_Int hypre_MPI_Waitany( HYPRE_Int count, hypre_MPI_Request *array_of_requests,
HYPRE_Int *index, hypre_MPI_Status *status );
HYPRE_Int hypre_MPI_Allreduce( void *sendbuf, void *recvbuf, HYPRE_Int count,
hypre_MPI_Datatype datatype, hypre_MPI_Op op, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Reduce( void *sendbuf, void *recvbuf, HYPRE_Int count,
hypre_MPI_Datatype datatype, hypre_MPI_Op op, HYPRE_Int root, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Scan( void *sendbuf, void *recvbuf, HYPRE_Int count,
hypre_MPI_Datatype datatype, hypre_MPI_Op op, hypre_MPI_Comm comm );
HYPRE_Int hypre_MPI_Request_free( hypre_MPI_Request *request );
HYPRE_Int hypre_MPI_Type_contiguous( HYPRE_Int count, hypre_MPI_Datatype oldtype,
hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_vector( HYPRE_Int count, HYPRE_Int blocklength, HYPRE_Int stride,
hypre_MPI_Datatype oldtype, hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_hvector( HYPRE_Int count, HYPRE_Int blocklength, hypre_MPI_Aint stride,
hypre_MPI_Datatype oldtype, hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_struct( HYPRE_Int count, HYPRE_Int *array_of_blocklengths,
hypre_MPI_Aint *array_of_displacements, hypre_MPI_Datatype *array_of_types,
hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_commit( hypre_MPI_Datatype *datatype );
HYPRE_Int hypre_MPI_Type_free( hypre_MPI_Datatype *datatype );
HYPRE_Int hypre_MPI_Op_free( hypre_MPI_Op *op );
HYPRE_Int hypre_MPI_Op_create( hypre_MPI_User_function *function, hypre_int commute,
hypre_MPI_Op *op );
#if defined(HYPRE_USING_GPU)
HYPRE_Int hypre_MPI_Comm_split_type(hypre_MPI_Comm comm, HYPRE_Int split_type, HYPRE_Int key,
hypre_MPI_Info info, hypre_MPI_Comm *newcomm);
HYPRE_Int hypre_MPI_Info_create(hypre_MPI_Info *info);
HYPRE_Int hypre_MPI_Info_free( hypre_MPI_Info *info );
#endif
#ifdef __cplusplus
}
#endif
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef HYPRE_SMP_HEADER
#define HYPRE_SMP_HEADER
#endif
#define HYPRE_SMP_SCHEDULE schedule(static)
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* Header file for memory management utilities
*
* The abstract memory model has a Host (think CPU) and a Device (think GPU) and
* three basic types of memory management utilities:
*
* 1. Malloc(..., location)
* location=LOCATION_DEVICE - malloc memory on the device
* location=LOCATION_HOST - malloc memory on the host
* 2. MemCopy(..., method)
* method=HOST_TO_DEVICE - copy from host to device
* method=DEVICE_TO_HOST - copy from device to host
* method=DEVICE_TO_DEVICE - copy from device to device
* 3. SetExecutionMode
* location=LOCATION_DEVICE - execute on the device
* location=LOCATION_HOST - execute on the host
*
* Although the abstract model does not explicitly reflect a managed memory
* model (i.e., unified memory), it can support it. Here is a summary of how
* the abstract model would be mapped to specific hardware scenarios:
*
* Not using a device, not using managed memory
* Malloc(..., location)
* location=LOCATION_DEVICE - host malloc e.g., malloc
* location=LOCATION_HOST - host malloc e.g., malloc
* MemoryCopy(..., locTo,locFrom)
* locTo=LOCATION_HOST, locFrom=LOCATION_DEVICE - copy from host to host e.g., memcpy
* locTo=LOCATION_DEVICE, locFrom=LOCATION_HOST - copy from host to host e.g., memcpy
* locTo=LOCATION_DEVICE, locFrom=LOCATION_DEVICE - copy from host to host e.g., memcpy
* SetExecutionMode
* location=LOCATION_DEVICE - execute on the host
* location=LOCATION_HOST - execute on the host
*
* Using a device, not using managed memory
* Malloc(..., location)
* location=LOCATION_DEVICE - device malloc e.g., cudaMalloc
* location=LOCATION_HOST - host malloc e.g., malloc
* MemoryCopy(..., locTo,locFrom)
* locTo=LOCATION_HOST, locFrom=LOCATION_DEVICE - copy from device to host e.g., cudaMemcpy
* locTo=LOCATION_DEVICE, locFrom=LOCATION_HOST - copy from host to device e.g., cudaMemcpy
* locTo=LOCATION_DEVICE, locFrom=LOCATION_DEVICE - copy from device to device e.g., cudaMemcpy
* SetExecutionMode
* location=LOCATION_DEVICE - execute on the device
* location=LOCATION_HOST - execute on the host
*
* Using a device, using managed memory
* Malloc(..., location)
* location=LOCATION_DEVICE - managed malloc e.g., cudaMallocManaged
* location=LOCATION_HOST - host malloc e.g., malloc
* MemoryCopy(..., locTo,locFrom)
* locTo=LOCATION_HOST, locFrom=LOCATION_DEVICE - copy from device to host e.g., cudaMallocManaged
* locTo=LOCATION_DEVICE, locFrom=LOCATION_HOST - copy from host to device e.g., cudaMallocManaged
* locTo=LOCATION_DEVICE, locFrom=LOCATION_DEVICE - copy from device to device e.g., cudaMallocManaged
* SetExecutionMode
* location=LOCATION_DEVICE - execute on the device
* location=LOCATION_HOST - execute on the host
*
*****************************************************************************/
#ifndef hypre_MEMORY_HEADER
#define hypre_MEMORY_HEADER
#include <stdio.h>
#include <stdlib.h>
#if defined(HYPRE_USING_UNIFIED_MEMORY) && defined(HYPRE_USING_DEVICE_OPENMP)
//#pragma omp requires unified_shared_memory
#endif
#if defined(HYPRE_USING_UMPIRE)
#include "umpire/interface/umpire.h"
#define HYPRE_UMPIRE_POOL_NAME_MAX_LEN 1024
#endif
/* stringification:
* _Pragma(string-literal), so we need to cast argument to a string
* The three dots as last argument of the macro tells compiler that this is a variadic macro.
* I.e. this is a macro that receives variable number of arguments.
*/
#define HYPRE_STR(...) #__VA_ARGS__
#define HYPRE_XSTR(...) HYPRE_STR(__VA_ARGS__)
#ifdef __cplusplus
extern "C" {
#endif
typedef enum _hypre_MemoryLocation
{
hypre_MEMORY_UNDEFINED = -1,
hypre_MEMORY_HOST,
hypre_MEMORY_HOST_PINNED,
hypre_MEMORY_DEVICE,
hypre_MEMORY_UNIFIED
} hypre_MemoryLocation;
/*-------------------------------------------------------
* hypre_GetActualMemLocation
* return actual location based on the selected memory model
*-------------------------------------------------------*/
static inline hypre_MemoryLocation
hypre_GetActualMemLocation(HYPRE_MemoryLocation location)
{
if (location == HYPRE_MEMORY_HOST)
{
return hypre_MEMORY_HOST;
}
if (location == HYPRE_MEMORY_DEVICE)
{
#if defined(HYPRE_USING_HOST_MEMORY)
return hypre_MEMORY_HOST;
#elif defined(HYPRE_USING_DEVICE_MEMORY)
return hypre_MEMORY_DEVICE;
#elif defined(HYPRE_USING_UNIFIED_MEMORY)
return hypre_MEMORY_UNIFIED;
#else
#error Wrong HYPRE memory setting.
#endif
}
return hypre_MEMORY_UNDEFINED;
}
#ifdef HYPRE_USING_MEMORY_TRACKER
typedef struct
{
char _action[16];
void *_ptr;
size_t _nbytes;
hypre_MemoryLocation _memory_location;
char _filename[256];
char _function[256];
HYPRE_Int _line;
size_t _pair;
} hypre_MemoryTrackerEntry;
typedef struct
{
size_t actual_size;
size_t alloced_size;
size_t prev_end;
hypre_MemoryTrackerEntry *data;
} hypre_MemoryTracker;
/* These Allocs are with memory tracker, for debug */
#define hypre_TAlloc(type, count, location) \
( \
{ \
void *ptr = hypre_MAlloc((size_t)(sizeof(type) * (count)), location); \
hypre_MemoryTrackerInsert("malloc", ptr, sizeof(type)*(count), hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__);\
(type *) ptr; \
} \
)
#define _hypre_TAlloc(type, count, location) \
( \
{ \
void *ptr = _hypre_MAlloc((size_t)(sizeof(type) * (count)), location); \
hypre_MemoryTrackerInsert("malloc", ptr, sizeof(type)*(count), location, __FILE__, __func__, __LINE__); \
(type *) ptr; \
} \
)
#define hypre_CTAlloc(type, count, location) \
( \
{ \
void *ptr = hypre_CAlloc((size_t)(count), (size_t)sizeof(type), location); \
hypre_MemoryTrackerInsert("calloc", ptr, sizeof(type)*(count), hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__);\
(type *) ptr; \
} \
)
#define hypre_TReAlloc(ptr, type, count, location) \
( \
{ \
hypre_MemoryTrackerInsert("rfree", ptr, (size_t) -1, hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__); \
void *new_ptr = hypre_ReAlloc((char *)ptr, (size_t)(sizeof(type) * (count)), location); \
hypre_MemoryTrackerInsert("rmalloc", new_ptr, sizeof(type)*(count), hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__);\
(type *) new_ptr; \
} \
)
#define hypre_TReAlloc_v2(ptr, old_type, old_count, new_type, new_count, location) \
( \
{ \
hypre_MemoryTrackerInsert("rfree", ptr, sizeof(old_type)*(old_count), hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__); \
void *new_ptr = hypre_ReAlloc_v2((char *)ptr, (size_t)(sizeof(old_type)*(old_count)), (size_t)(sizeof(new_type)*(new_count)), location); \
hypre_MemoryTrackerInsert("rmalloc", new_ptr, sizeof(new_type)*(new_count), hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__);\
(new_type *) new_ptr; \
} \
)
#define hypre_TMemcpy(dst, src, type, count, locdst, locsrc) \
( \
{ \
hypre_Memcpy((void *)(dst), (void *)(src), (size_t)(sizeof(type) * (count)), locdst, locsrc); \
} \
)
#define hypre_TFree(ptr, location) \
( \
{ \
hypre_MemoryTrackerInsert("free", ptr, (size_t) -1, hypre_GetActualMemLocation(location), __FILE__, __func__, __LINE__); \
hypre_Free((void *)ptr, location); \
ptr = NULL; \
} \
)
#define _hypre_TFree(ptr, location) \
( \
{ \
hypre_MemoryTrackerInsert("free", ptr, (size_t) -1, location, __FILE__, __func__, __LINE__); \
_hypre_Free((void *)ptr, location); \
ptr = NULL; \
} \
)
#else /* #ifdef HYPRE_USING_MEMORY_TRACKER */
#define hypre_TAlloc(type, count, location) \
( (type *) hypre_MAlloc((size_t)(sizeof(type) * (count)), location) )
#define _hypre_TAlloc(type, count, location) \
( (type *) _hypre_MAlloc((size_t)(sizeof(type) * (count)), location) )
#define hypre_CTAlloc(type, count, location) \
( (type *) hypre_CAlloc((size_t)(count), (size_t)sizeof(type), location) )
#define hypre_TReAlloc(ptr, type, count, location) \
( (type *) hypre_ReAlloc((char *)ptr, (size_t)(sizeof(type) * (count)), location) )
#define hypre_TReAlloc_v2(ptr, old_type, old_count, new_type, new_count, location) \
( (new_type *) hypre_ReAlloc_v2((char *)ptr, (size_t)(sizeof(old_type)*(old_count)), (size_t)(sizeof(new_type)*(new_count)), location) )
#define hypre_TMemcpy(dst, src, type, count, locdst, locsrc) \
(hypre_Memcpy((void *)(dst), (void *)(src), (size_t)(sizeof(type) * (count)), locdst, locsrc))
#define hypre_TFree(ptr, location) \
( hypre_Free((void *)ptr, location), ptr = NULL )
#define _hypre_TFree(ptr, location) \
( _hypre_Free((void *)ptr, location), ptr = NULL )
#endif /* #ifdef HYPRE_USING_MEMORY_TRACKER */
/*--------------------------------------------------------------------------
* Prototypes
*--------------------------------------------------------------------------*/
/* memory.c */
void * hypre_Memset(void *ptr, HYPRE_Int value, size_t num, HYPRE_MemoryLocation location);
void hypre_MemPrefetch(void *ptr, size_t size, HYPRE_MemoryLocation location);
void * hypre_MAlloc(size_t size, HYPRE_MemoryLocation location);
void * hypre_CAlloc( size_t count, size_t elt_size, HYPRE_MemoryLocation location);
void hypre_Free(void *ptr, HYPRE_MemoryLocation location);
void hypre_Memcpy(void *dst, void *src, size_t size, HYPRE_MemoryLocation loc_dst,
HYPRE_MemoryLocation loc_src);
void * hypre_ReAlloc(void *ptr, size_t size, HYPRE_MemoryLocation location);
void * hypre_ReAlloc_v2(void *ptr, size_t old_size, size_t new_size, HYPRE_MemoryLocation location);
void * _hypre_MAlloc(size_t size, hypre_MemoryLocation location);
void _hypre_Free(void *ptr, hypre_MemoryLocation location);
HYPRE_ExecutionPolicy hypre_GetExecPolicy1(HYPRE_MemoryLocation location);
HYPRE_ExecutionPolicy hypre_GetExecPolicy2(HYPRE_MemoryLocation location1,
HYPRE_MemoryLocation location2);
HYPRE_Int hypre_GetPointerLocation(const void *ptr, hypre_MemoryLocation *memory_location);
HYPRE_Int hypre_PrintMemoryTracker();
HYPRE_Int hypre_SetCubMemPoolSize( hypre_uint bin_growth, hypre_uint min_bin, hypre_uint max_bin,
size_t max_cached_bytes );
HYPRE_Int hypre_umpire_host_pooled_allocate(void **ptr, size_t nbytes);
HYPRE_Int hypre_umpire_host_pooled_free(void *ptr);
void *hypre_umpire_host_pooled_realloc(void *ptr, size_t size);
HYPRE_Int hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes);
HYPRE_Int hypre_umpire_device_pooled_free(void *ptr);
HYPRE_Int hypre_umpire_um_pooled_allocate(void **ptr, size_t nbytes);
HYPRE_Int hypre_umpire_um_pooled_free(void *ptr);
HYPRE_Int hypre_umpire_pinned_pooled_allocate(void **ptr, size_t nbytes);
HYPRE_Int hypre_umpire_pinned_pooled_free(void *ptr);
#ifdef HYPRE_USING_MEMORY_TRACKER
hypre_MemoryTracker * hypre_MemoryTrackerCreate();
void hypre_MemoryTrackerDestroy(hypre_MemoryTracker *tracker);
void hypre_MemoryTrackerInsert(const char *action, void *ptr, size_t nbytes,
hypre_MemoryLocation memory_location, const char *filename, const char *function, HYPRE_Int line);
HYPRE_Int hypre_PrintMemoryTracker();
#endif
/* memory_dmalloc.c */
HYPRE_Int hypre_InitMemoryDebugDML( HYPRE_Int id );
HYPRE_Int hypre_FinalizeMemoryDebugDML( void );
char *hypre_MAllocDML( HYPRE_Int size, char *file, HYPRE_Int line );
char *hypre_CAllocDML( HYPRE_Int count, HYPRE_Int elt_size, char *file, HYPRE_Int line );
char *hypre_ReAllocDML( char *ptr, HYPRE_Int size, char *file, HYPRE_Int line );
void hypre_FreeDML( char *ptr, char *file, HYPRE_Int line );
/* GPU malloc prototype */
typedef void (*GPUMallocFunc)(void **, size_t);
typedef void (*GPUMfreeFunc)(void *);
#ifdef __cplusplus
}
#endif
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef HYPRE_OMP_DEVICE_H
#define HYPRE_OMP_DEVICE_H
#if defined(HYPRE_USING_DEVICE_OPENMP)
#include "omp.h"
/* OpenMP 4.5 device memory management */
extern HYPRE_Int hypre__global_offload;
extern HYPRE_Int hypre__offload_device_num;
extern HYPRE_Int hypre__offload_host_num;
/* stats */
extern size_t hypre__target_allc_count;
extern size_t hypre__target_free_count;
extern size_t hypre__target_allc_bytes;
extern size_t hypre__target_free_bytes;
extern size_t hypre__target_htod_count;
extern size_t hypre__target_dtoh_count;
extern size_t hypre__target_htod_bytes;
extern size_t hypre__target_dtoh_bytes;
/* CHECK MODE: check if offloading has effect (turned on when configured with --enable-debug)
* if we ``enter'' an address, it should not exist in device [o.w NO EFFECT]
* if we ``exit'' or ''update'' an address, it should exist in device [o.w ERROR]
* hypre__offload_flag: 0 == OK; 1 == WRONG
*/
#ifdef HYPRE_DEVICE_OPENMP_CHECK
#define HYPRE_OFFLOAD_FLAG(devnum, hptr, type) HYPRE_Int hypre__offload_flag = (type[1] == 'n') == omp_target_is_present(hptr, devnum);
#else
#define HYPRE_OFFLOAD_FLAG(...) HYPRE_Int hypre__offload_flag = 0; /* non-debug mode, always OK */
#endif
/* OMP 4.5 offloading macro */
#define hypre_omp_device_offload(devnum, hptr, datatype, offset, count, type1, type2) \
{\
/* devnum: device number \
* hptr: host poiter \
* datatype \
* type1: ``e(n)ter'', ''e(x)it'', or ``u(p)date'' \
* type2: ``(a)lloc'', ``(t)o'', ``(d)elete'', ''(f)rom'' \
*/ \
datatype *hypre__offload_hptr = (datatype *) hptr; \
/* if hypre__global_offload == 0, or
* hptr (host pointer) == NULL,
* this offload will be IGNORED */ \
if (hypre__global_offload && hypre__offload_hptr != NULL) { \
/* offloading offset and size (in datatype) */ \
size_t hypre__offload_offset = offset, hypre__offload_size = count; \
/* in the CHECK mode, we test if this offload has effect */ \
HYPRE_OFFLOAD_FLAG(devnum, hypre__offload_hptr, type1) \
if (hypre__offload_flag) { \
printf("[!NO Effect! %s %d] device %d target: %6s %6s, data %p, [%ld:%ld]\n", __FILE__, __LINE__, devnum, type1, type2, (void *)hypre__offload_hptr, hypre__offload_offset, hypre__offload_size); exit(0); \
} else { \
size_t offload_bytes = count * sizeof(datatype); \
/* printf("[ %s %d] device %d target: %6s %6s, data %p, [%d:%d]\n", __FILE__, __LINE__, devnum, type1, type2, (void *)hypre__offload_hptr, hypre__offload_offset, hypre__offload_size); */ \
if (type1[1] == 'n' && type2[0] == 't') { \
/* enter to */\
hypre__target_allc_count ++; \
hypre__target_allc_bytes += offload_bytes; \
hypre__target_htod_count ++; \
hypre__target_htod_bytes += offload_bytes; \
_Pragma (HYPRE_XSTR(omp target enter data map(to:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
} else if (type1[1] == 'n' && type2[0] == 'a') { \
/* enter alloc */ \
hypre__target_allc_count ++; \
hypre__target_allc_bytes += offload_bytes; \
_Pragma (HYPRE_XSTR(omp target enter data map(alloc:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
} else if (type1[1] == 'x' && type2[0] == 'd') { \
/* exit delete */\
hypre__target_free_count ++; \
hypre__target_free_bytes += offload_bytes; \
_Pragma (HYPRE_XSTR(omp target exit data map(delete:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
} else if (type1[1] == 'x' && type2[0] == 'f') {\
/* exit from */ \
hypre__target_free_count ++; \
hypre__target_free_bytes += offload_bytes; \
hypre__target_dtoh_count ++; \
hypre__target_dtoh_bytes += offload_bytes; \
_Pragma (HYPRE_XSTR(omp target exit data map(from:hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
} else if (type1[1] == 'p' && type2[0] == 't') { \
/* update to */ \
hypre__target_htod_count ++; \
hypre__target_htod_bytes += offload_bytes; \
_Pragma (HYPRE_XSTR(omp target update to(hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
} else if (type1[1] == 'p' && type2[0] == 'f') {\
/* update from */ \
hypre__target_dtoh_count ++; \
hypre__target_dtoh_bytes += offload_bytes; \
_Pragma (HYPRE_XSTR(omp target update from(hypre__offload_hptr[hypre__offload_offset:hypre__offload_size]))) \
} else {\
printf("error: unrecognized offloading type combination!\n"); exit(-1); \
} \
} \
} \
}
HYPRE_Int HYPRE_OMPOffload(HYPRE_Int device, void *ptr, size_t num, const char *type1,
const char *type2);
HYPRE_Int HYPRE_OMPPtrIsMapped(void *p, HYPRE_Int device_num);
HYPRE_Int HYPRE_OMPOffloadOn();
HYPRE_Int HYPRE_OMPOffloadOff();
HYPRE_Int HYPRE_OMPOffloadStatPrint();
#endif /* HYPRE_USING_DEVICE_OPENMP */
#endif /* HYPRE_OMP_DEVICE_H */
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef hypre_THREADING_HEADER
#define hypre_THREADING_HEADER
#ifdef HYPRE_USING_OPENMP
HYPRE_Int hypre_NumThreads( void );
HYPRE_Int hypre_NumActiveThreads( void );
HYPRE_Int hypre_GetThreadNum( void );
void hypre_SetNumThreads(HYPRE_Int nt);
#else
#define hypre_NumThreads() 1
#define hypre_NumActiveThreads() 1
#define hypre_GetThreadNum() 0
#define hypre_SetNumThreads(x)
#endif
void hypre_GetSimpleThreadPartition( HYPRE_Int *begin, HYPRE_Int *end, HYPRE_Int n );
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* Header file for doing timing
*
*****************************************************************************/
#ifndef HYPRE_TIMING_HEADER
#define HYPRE_TIMING_HEADER
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef __cplusplus
extern "C" {
#endif
/*--------------------------------------------------------------------------
* Prototypes for low-level timing routines
*--------------------------------------------------------------------------*/
/* timer.c */
HYPRE_Real time_getWallclockSeconds( void );
HYPRE_Real time_getCPUSeconds( void );
HYPRE_Real time_get_wallclock_seconds_( void );
HYPRE_Real time_get_cpu_seconds_( void );
/*--------------------------------------------------------------------------
* With timing off
*--------------------------------------------------------------------------*/
#ifndef HYPRE_TIMING
#define hypre_InitializeTiming(name) 0
#define hypre_FinalizeTiming(index)
#define hypre_IncFLOPCount(inc)
#define hypre_BeginTiming(i)
#define hypre_EndTiming(i)
#define hypre_PrintTiming(heading, comm)
#define hypre_ClearTiming()
/*--------------------------------------------------------------------------
* With timing on
*--------------------------------------------------------------------------*/
#else
/*-------------------------------------------------------
* Global timing structure
*-------------------------------------------------------*/
typedef struct
{
HYPRE_Real *wall_time;
HYPRE_Real *cpu_time;
HYPRE_Real *flops;
char **name;
HYPRE_Int *state; /* boolean flag to allow for recursive timing */
HYPRE_Int *num_regs; /* count of how many times a name is registered */
HYPRE_Int num_names;
HYPRE_Int size;
HYPRE_Real wall_count;
HYPRE_Real CPU_count;
HYPRE_Real FLOP_count;
} hypre_TimingType;
#ifdef HYPRE_TIMING_GLOBALS
hypre_TimingType *hypre_global_timing = NULL;
#else
extern hypre_TimingType *hypre_global_timing;
#endif
/*-------------------------------------------------------
* Accessor functions
*-------------------------------------------------------*/
#define hypre_TimingWallTime(i) (hypre_global_timing -> wall_time[(i)])
#define hypre_TimingCPUTime(i) (hypre_global_timing -> cpu_time[(i)])
#define hypre_TimingFLOPS(i) (hypre_global_timing -> flops[(i)])
#define hypre_TimingName(i) (hypre_global_timing -> name[(i)])
#define hypre_TimingState(i) (hypre_global_timing -> state[(i)])
#define hypre_TimingNumRegs(i) (hypre_global_timing -> num_regs[(i)])
#define hypre_TimingWallCount (hypre_global_timing -> wall_count)
#define hypre_TimingCPUCount (hypre_global_timing -> CPU_count)
#define hypre_TimingFLOPCount (hypre_global_timing -> FLOP_count)
/*-------------------------------------------------------
* Prototypes
*-------------------------------------------------------*/
/* timing.c */
HYPRE_Int hypre_InitializeTiming( const char *name );
HYPRE_Int hypre_FinalizeTiming( HYPRE_Int time_index );
HYPRE_Int hypre_FinalizeAllTimings();
HYPRE_Int hypre_IncFLOPCount( HYPRE_BigInt inc );
HYPRE_Int hypre_BeginTiming( HYPRE_Int time_index );
HYPRE_Int hypre_EndTiming( HYPRE_Int time_index );
HYPRE_Int hypre_ClearTiming( void );
HYPRE_Int hypre_PrintTiming( const char *heading, MPI_Comm comm );
#endif
#ifdef __cplusplus
}
#endif
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* Header file link lists
*
*****************************************************************************/
#ifndef HYPRE_LINKLIST_HEADER
#define HYPRE_LINKLIST_HEADER
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef __cplusplus
extern "C" {
#endif
struct double_linked_list
{
HYPRE_Int data;
struct double_linked_list *next_elt;
struct double_linked_list *prev_elt;
HYPRE_Int head;
HYPRE_Int tail;
};
typedef struct double_linked_list hypre_ListElement;
typedef hypre_ListElement *hypre_LinkList;
#ifdef __cplusplus
}
#endif
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef hypre_EXCHANGE_DATA_HEADER
#define hypre_EXCHANGE_DATA_HEADER
#define hypre_BinaryTreeParentId(tree) (tree->parent_id)
#define hypre_BinaryTreeNumChild(tree) (tree->num_child)
#define hypre_BinaryTreeChildIds(tree) (tree->child_id)
#define hypre_BinaryTreeChildId(tree, i) (tree->child_id[i])
typedef struct
{
HYPRE_Int parent_id;
HYPRE_Int num_child;
HYPRE_Int *child_id;
} hypre_BinaryTree;
/* In the fill_response() function the user needs to set the recv__buf
and the response_message_size. Memory of size send_response_storage has been
alllocated for the send_buf (in exchange_data) - if more is needed, then
realloc and adjust
the send_response_storage. The realloc amount should be storage+overhead.
If the response is an empty "confirmation" message, then set
response_message_size =0 (and do not modify the send_buf) */
typedef struct
{
HYPRE_Int (*fill_response)(void* recv_buf, HYPRE_Int contact_size,
HYPRE_Int contact_proc, void* response_obj,
MPI_Comm comm, void** response_buf,
HYPRE_Int* response_message_size);
HYPRE_Int send_response_overhead; /*set by exchange data */
HYPRE_Int send_response_storage; /*storage allocated for send_response_buf*/
void *data1; /*data fields user may want to access in fill_response */
void *data2;
} hypre_DataExchangeResponse;
HYPRE_Int hypre_CreateBinaryTree(HYPRE_Int, HYPRE_Int, hypre_BinaryTree*);
HYPRE_Int hypre_DestroyBinaryTree(hypre_BinaryTree*);
HYPRE_Int hypre_DataExchangeList(HYPRE_Int num_contacts, HYPRE_Int *contact_proc_list,
void *contact_send_buf, HYPRE_Int *contact_send_buf_starts, HYPRE_Int contact_obj_size,
HYPRE_Int response_obj_size, hypre_DataExchangeResponse *response_obj, HYPRE_Int max_response_size,
HYPRE_Int rnum, MPI_Comm comm, void **p_response_recv_buf, HYPRE_Int **p_response_recv_buf_starts);
#endif /* end of header */
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* Header file for Caliper instrumentation macros
*
*****************************************************************************/
#ifndef CALIPER_INSTRUMENTATION_HEADER
#define CALIPER_INSTRUMENTATION_HEADER
#include "HYPRE_config.h"
#ifdef HYPRE_USING_CALIPER
#ifdef __cplusplus
extern "C++"
{
#endif
#include <caliper/cali.h>
#ifdef __cplusplus
}
#endif
static char hypre__levelname[16];
static char hypre__markname[1024];
#define HYPRE_ANNOTATE_FUNC_BEGIN CALI_MARK_FUNCTION_BEGIN
#define HYPRE_ANNOTATE_FUNC_END CALI_MARK_FUNCTION_END
#define HYPRE_ANNOTATE_LOOP_BEGIN(id, str) CALI_MARK_LOOP_BEGIN(id, str)
#define HYPRE_ANNOTATE_LOOP_END(id) CALI_MARK_LOOP_END(id)
#define HYPRE_ANNOTATE_ITER_BEGIN(id, it) CALI_MARK_ITERATION_BEGIN(id, it)
#define HYPRE_ANNOTATE_ITER_END(id) CALI_MARK_ITERATION_END(id)
#define HYPRE_ANNOTATE_REGION_BEGIN(...)\
{\
hypre_sprintf(hypre__markname, __VA_ARGS__);\
CALI_MARK_BEGIN(hypre__markname);\
}
#define HYPRE_ANNOTATE_REGION_END(...)\
{\
hypre_sprintf(hypre__markname, __VA_ARGS__);\
CALI_MARK_END(hypre__markname);\
}
#define HYPRE_ANNOTATE_MGLEVEL_BEGIN(lvl)\
{\
hypre_sprintf(hypre__levelname, "MG level %d", lvl);\
CALI_MARK_BEGIN(hypre__levelname);\
}
#define HYPRE_ANNOTATE_MGLEVEL_END(lvl)\
{\
hypre_sprintf(hypre__levelname, "MG level %d", lvl);\
CALI_MARK_END(hypre__levelname);\
}
#else
#define HYPRE_ANNOTATE_FUNC_BEGIN
#define HYPRE_ANNOTATE_FUNC_END
#define HYPRE_ANNOTATE_LOOP_BEGIN(id, str)
#define HYPRE_ANNOTATE_LOOP_END(id)
#define HYPRE_ANNOTATE_ITER_BEGIN(id, it)
#define HYPRE_ANNOTATE_ITER_END(id)
#define HYPRE_ANNOTATE_REGION_BEGIN(...)
#define HYPRE_ANNOTATE_REGION_END(...)
#define HYPRE_ANNOTATE_MAX_MGLEVEL(lvl)
#define HYPRE_ANNOTATE_MGLEVEL_BEGIN(lvl)
#define HYPRE_ANNOTATE_MGLEVEL_END(lvl)
#endif
#endif /* CALIPER_INSTRUMENTATION_HEADER */
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* General structures and values
*
*****************************************************************************/
#ifndef HYPRE_HANDLE_H
#define HYPRE_HANDLE_H
struct hypre_DeviceData;
typedef struct hypre_DeviceData hypre_DeviceData;
typedef struct
{
HYPRE_Int hypre_error;
HYPRE_MemoryLocation memory_location;
HYPRE_ExecutionPolicy default_exec_policy;
HYPRE_ExecutionPolicy struct_exec_policy;
#if defined(HYPRE_USING_GPU)
hypre_DeviceData *device_data;
/* device G-S options */
HYPRE_Int device_gs_method;
#endif
#if defined(HYPRE_USING_UMPIRE)
char umpire_device_pool_name[HYPRE_UMPIRE_POOL_NAME_MAX_LEN];
char umpire_um_pool_name[HYPRE_UMPIRE_POOL_NAME_MAX_LEN];
char umpire_host_pool_name[HYPRE_UMPIRE_POOL_NAME_MAX_LEN];
char umpire_pinned_pool_name[HYPRE_UMPIRE_POOL_NAME_MAX_LEN];
size_t umpire_device_pool_size;
size_t umpire_um_pool_size;
size_t umpire_host_pool_size;
size_t umpire_pinned_pool_size;
size_t umpire_block_size;
HYPRE_Int own_umpire_device_pool;
HYPRE_Int own_umpire_um_pool;
HYPRE_Int own_umpire_host_pool;
HYPRE_Int own_umpire_pinned_pool;
umpire_resourcemanager umpire_rm;
#endif
/* user malloc/free function pointers */
GPUMallocFunc user_device_malloc;
GPUMfreeFunc user_device_free;
} hypre_Handle;
/* accessor macros to hypre_Handle */
#define hypre_HandleMemoryLocation(hypre_handle) ((hypre_handle) -> memory_location)
#define hypre_HandleDefaultExecPolicy(hypre_handle) ((hypre_handle) -> default_exec_policy)
#define hypre_HandleStructExecPolicy(hypre_handle) ((hypre_handle) -> struct_exec_policy)
#define hypre_HandleDeviceData(hypre_handle) ((hypre_handle) -> device_data)
#define hypre_HandleDeviceGSMethod(hypre_handle) ((hypre_handle) -> device_gs_method)
#define hypre_HandleCurandGenerator(hypre_handle) hypre_DeviceDataCurandGenerator(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCublasHandle(hypre_handle) hypre_DeviceDataCublasHandle(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCusparseHandle(hypre_handle) hypre_DeviceDataCusparseHandle(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleComputeStream(hypre_handle) hypre_DeviceDataComputeStream(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCubBinGrowth(hypre_handle) hypre_DeviceDataCubBinGrowth(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCubMinBin(hypre_handle) hypre_DeviceDataCubMinBin(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCubMaxBin(hypre_handle) hypre_DeviceDataCubMaxBin(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCubMaxCachedBytes(hypre_handle) hypre_DeviceDataCubMaxCachedBytes(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCubDevAllocator(hypre_handle) hypre_DeviceDataCubDevAllocator(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleCubUvmAllocator(hypre_handle) hypre_DeviceDataCubUvmAllocator(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleDevice(hypre_handle) hypre_DeviceDataDevice(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleDeviceMaxWorkGroupSize(hypre_handle) hypre_DeviceDataDeviceMaxWorkGroupSize(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleComputeStreamNum(hypre_handle) hypre_DeviceDataComputeStreamNum(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleReduceBuffer(hypre_handle) hypre_DeviceDataReduceBuffer(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleStructCommRecvBuffer(hypre_handle) hypre_DeviceDataStructCommRecvBuffer(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleStructCommSendBuffer(hypre_handle) hypre_DeviceDataStructCommSendBuffer(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleStructCommRecvBufferSize(hypre_handle) hypre_DeviceDataStructCommRecvBufferSize(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleStructCommSendBufferSize(hypre_handle) hypre_DeviceDataStructCommSendBufferSize(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleSpgemmUseCusparse(hypre_handle) hypre_DeviceDataSpgemmUseCusparse(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleSpgemmAlgorithm(hypre_handle) hypre_DeviceDataSpgemmAlgorithm(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleSpgemmRownnzEstimateMethod(hypre_handle) hypre_DeviceDataSpgemmRownnzEstimateMethod(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleSpgemmRownnzEstimateNsamples(hypre_handle) hypre_DeviceDataSpgemmRownnzEstimateNsamples(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleSpgemmRownnzEstimateMultFactor(hypre_handle) hypre_DeviceDataSpgemmRownnzEstimateMultFactor(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleSpgemmHashType(hypre_handle) hypre_DeviceDataSpgemmHashType(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleDeviceAllocator(hypre_handle) hypre_DeviceDataDeviceAllocator(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleUseGpuRand(hypre_handle) hypre_DeviceDataUseGpuRand(hypre_HandleDeviceData(hypre_handle))
#define hypre_HandleUserDeviceMalloc(hypre_handle) ((hypre_handle) -> user_device_malloc)
#define hypre_HandleUserDeviceMfree(hypre_handle) ((hypre_handle) -> user_device_free)
#define hypre_HandleUmpireResourceMan(hypre_handle) ((hypre_handle) -> umpire_rm)
#define hypre_HandleUmpireDevicePoolSize(hypre_handle) ((hypre_handle) -> umpire_device_pool_size)
#define hypre_HandleUmpireUMPoolSize(hypre_handle) ((hypre_handle) -> umpire_um_pool_size)
#define hypre_HandleUmpireHostPoolSize(hypre_handle) ((hypre_handle) -> umpire_host_pool_size)
#define hypre_HandleUmpirePinnedPoolSize(hypre_handle) ((hypre_handle) -> umpire_pinned_pool_size)
#define hypre_HandleUmpireBlockSize(hypre_handle) ((hypre_handle) -> umpire_block_size)
#define hypre_HandleUmpireDevicePoolName(hypre_handle) ((hypre_handle) -> umpire_device_pool_name)
#define hypre_HandleUmpireUMPoolName(hypre_handle) ((hypre_handle) -> umpire_um_pool_name)
#define hypre_HandleUmpireHostPoolName(hypre_handle) ((hypre_handle) -> umpire_host_pool_name)
#define hypre_HandleUmpirePinnedPoolName(hypre_handle) ((hypre_handle) -> umpire_pinned_pool_name)
#define hypre_HandleOwnUmpireDevicePool(hypre_handle) ((hypre_handle) -> own_umpire_device_pool)
#define hypre_HandleOwnUmpireUMPool(hypre_handle) ((hypre_handle) -> own_umpire_um_pool)
#define hypre_HandleOwnUmpireHostPool(hypre_handle) ((hypre_handle) -> own_umpire_host_pool)
#define hypre_HandleOwnUmpirePinnedPool(hypre_handle) ((hypre_handle) -> own_umpire_pinned_pool)
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
#ifndef HYPRE_GSELIM_H
#define HYPRE_GSELIM_H
#define hypre_gselim(A,x,n,error) \
{ \
HYPRE_Int j,k,m; \
HYPRE_Real factor; \
HYPRE_Real divA; \
error = 0; \
if (n == 1) /* A is 1x1 */ \
{ \
if (A[0] != 0.0) \
{ \
x[0] = x[0]/A[0]; \
} \
else \
{ \
error++; \
} \
} \
else/* A is nxn. Forward elimination */ \
{ \
for (k = 0; k < n-1; k++) \
{ \
if (A[k*n+k] != 0.0) \
{ \
divA = 1.0/A[k*n+k]; \
for (j = k+1; j < n; j++) \
{ \
if (A[j*n+k] != 0.0) \
{ \
factor = A[j*n+k]*divA; \
for (m = k+1; m < n; m++) \
{ \
A[j*n+m] -= factor * A[k*n+m]; \
} \
x[j] -= factor * x[k]; \
} \
} \
} \
} \
/* Back Substitution */ \
for (k = n-1; k > 0; --k) \
{ \
if (A[k*n+k] != 0.0) \
{ \
x[k] /= A[k*n+k]; \
for (j = 0; j < k; j++) \
{ \
if (A[j*n+k] != 0.0) \
{ \
x[j] -= x[k] * A[j*n+k]; \
} \
} \
} \
} \
if (A[0] != 0.0) x[0] /= A[0]; \
} \
}
#endif /* #ifndef HYPRE_GSELIM_H */
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/******************************************************************************
*
* Header file for hypre_IntArray struct for holding an array of integers
*
*****************************************************************************/
#ifndef hypre_INTARRAY_HEADER
#define hypre_INTARRAY_HEADER
/*--------------------------------------------------------------------------
* hypre_IntArray
*--------------------------------------------------------------------------*/
typedef struct
{
/* pointer to data and size of data */
HYPRE_Int *data;
HYPRE_Int size;
/* memory location of array data */
HYPRE_MemoryLocation memory_location;
} hypre_IntArray;
/*--------------------------------------------------------------------------
* Accessor functions for the IntArray structure
*--------------------------------------------------------------------------*/
#define hypre_IntArrayData(array) ((array) -> data)
#define hypre_IntArraySize(array) ((array) -> size)
#define hypre_IntArrayMemoryLocation(array) ((array) -> memory_location)
#endif
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/* amg_linklist.c */
void hypre_dispose_elt ( hypre_LinkList element_ptr );
void hypre_remove_point ( hypre_LinkList *LoL_head_ptr, hypre_LinkList *LoL_tail_ptr,
HYPRE_Int measure, HYPRE_Int index, HYPRE_Int *lists, HYPRE_Int *where );
hypre_LinkList hypre_create_elt ( HYPRE_Int Item );
void hypre_enter_on_lists ( hypre_LinkList *LoL_head_ptr, hypre_LinkList *LoL_tail_ptr,
HYPRE_Int measure, HYPRE_Int index, HYPRE_Int *lists, HYPRE_Int *where );
/* binsearch.c */
HYPRE_Int hypre_BinarySearch ( HYPRE_Int *list, HYPRE_Int value, HYPRE_Int list_length );
HYPRE_Int hypre_BigBinarySearch ( HYPRE_BigInt *list, HYPRE_BigInt value, HYPRE_Int list_length );
HYPRE_Int hypre_BinarySearch2 ( HYPRE_Int *list, HYPRE_Int value, HYPRE_Int low, HYPRE_Int high,
HYPRE_Int *spot );
HYPRE_Int *hypre_LowerBound( HYPRE_Int *first, HYPRE_Int *last, HYPRE_Int value );
HYPRE_BigInt *hypre_BigLowerBound( HYPRE_BigInt *first, HYPRE_BigInt *last, HYPRE_BigInt value );
/* log.c */
HYPRE_Int hypre_Log2( HYPRE_Int p );
/* complex.c */
#ifdef HYPRE_COMPLEX
HYPRE_Complex hypre_conj( HYPRE_Complex value );
HYPRE_Real hypre_cabs( HYPRE_Complex value );
HYPRE_Real hypre_creal( HYPRE_Complex value );
HYPRE_Real hypre_cimag( HYPRE_Complex value );
#else
#define hypre_conj(value) value
#define hypre_cabs(value) fabs(value)
#define hypre_creal(value) value
#define hypre_cimag(value) 0.0
#endif
/* general.c */
#ifdef HYPRE_USING_MEMORY_TRACKER
hypre_MemoryTracker* hypre_memory_tracker();
#endif
hypre_Handle* hypre_handle();
hypre_Handle* hypre_HandleCreate();
HYPRE_Int hypre_HandleDestroy(hypre_Handle *hypre_handle_);
HYPRE_Int hypre_SetDevice(hypre_int device_id, hypre_Handle *hypre_handle_);
HYPRE_Int hypre_GetDevice(hypre_int *device_id);
HYPRE_Int hypre_GetDeviceCount(hypre_int *device_count);
HYPRE_Int hypre_GetDeviceLastError();
HYPRE_Int hypre_UmpireInit(hypre_Handle *hypre_handle_);
HYPRE_Int hypre_UmpireFinalize(hypre_Handle *hypre_handle_);
/* qsort.c */
void hypre_swap ( HYPRE_Int *v, HYPRE_Int i, HYPRE_Int j );
void hypre_swap_c ( HYPRE_Complex *v, HYPRE_Int i, HYPRE_Int j );
void hypre_swap2 ( HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int i, HYPRE_Int j );
void hypre_BigSwap2 ( HYPRE_BigInt *v, HYPRE_Real *w, HYPRE_Int i, HYPRE_Int j );
void hypre_swap2i ( HYPRE_Int *v, HYPRE_Int *w, HYPRE_Int i, HYPRE_Int j );
void hypre_BigSwap2i ( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int i, HYPRE_Int j );
void hypre_swap3i ( HYPRE_Int *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int i, HYPRE_Int j );
void hypre_swap3_d ( HYPRE_Real *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int i, HYPRE_Int j );
void hypre_swap3_d_perm(HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int *z, HYPRE_Int i, HYPRE_Int j );
void hypre_BigSwap4_d ( HYPRE_Real *v, HYPRE_BigInt *w, HYPRE_Int *z, HYPRE_Int *y, HYPRE_Int i,
HYPRE_Int j );
void hypre_swap_d ( HYPRE_Real *v, HYPRE_Int i, HYPRE_Int j );
void hypre_qsort0 ( HYPRE_Int *v, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort1 ( HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int left, HYPRE_Int right );
void hypre_BigQsort1 ( HYPRE_BigInt *v, HYPRE_Real *w, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort2i ( HYPRE_Int *v, HYPRE_Int *w, HYPRE_Int left, HYPRE_Int right );
void hypre_BigQsort2i( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort2 ( HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort2_abs ( HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort3i ( HYPRE_Int *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort3ir ( HYPRE_Int *v, HYPRE_Real *w, HYPRE_Int *z, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort3( HYPRE_Real *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int left, HYPRE_Int right );
void hypre_qsort3_abs ( HYPRE_Real *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int left,
HYPRE_Int right );
void hypre_BigQsort4_abs ( HYPRE_Real *v, HYPRE_BigInt *w, HYPRE_Int *z, HYPRE_Int *y,
HYPRE_Int left, HYPRE_Int right );
void hypre_qsort_abs ( HYPRE_Real *w, HYPRE_Int left, HYPRE_Int right );
void hypre_BigSwapbi(HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int i, HYPRE_Int j );
void hypre_BigQsortbi( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int left, HYPRE_Int right );
void hypre_BigSwapLoc(HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int i, HYPRE_Int j );
void hypre_BigQsortbLoc( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int left, HYPRE_Int right );
void hypre_BigSwapb2i(HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int i, HYPRE_Int j );
void hypre_BigQsortb2i( HYPRE_BigInt *v, HYPRE_Int *w, HYPRE_Int *z, HYPRE_Int left,
HYPRE_Int right );
void hypre_BigSwap( HYPRE_BigInt *v, HYPRE_Int i, HYPRE_Int j );
void hypre_BigQsort0( HYPRE_BigInt *v, HYPRE_Int left, HYPRE_Int right );
void hypre_topo_sort(const HYPRE_Int *row_ptr, const HYPRE_Int *col_inds, const HYPRE_Complex *data,
HYPRE_Int *ordering, HYPRE_Int n);
void hypre_dense_topo_sort(const HYPRE_Complex *L, HYPRE_Int *ordering, HYPRE_Int n,
HYPRE_Int is_col_major);
/* qsplit.c */
HYPRE_Int hypre_DoubleQuickSplit ( HYPRE_Real *values, HYPRE_Int *indices, HYPRE_Int list_length,
HYPRE_Int NumberKept );
/* random.c */
/* HYPRE_CUDA_GLOBAL */ void hypre_SeedRand ( HYPRE_Int seed );
/* HYPRE_CUDA_GLOBAL */ HYPRE_Int hypre_RandI ( void );
/* HYPRE_CUDA_GLOBAL */ HYPRE_Real hypre_Rand ( void );
/* prefix_sum.c */
/**
* Assumed to be called within an omp region.
* Let x_i be the input of ith thread.
* The output of ith thread y_i = x_0 + x_1 + ... + x_{i-1}
* Additionally, sum = x_0 + x_1 + ... + x_{nthreads - 1}
* Note that always y_0 = 0
*
* @param workspace at least with length (nthreads+1)
* workspace[tid] will contain result for tid
* workspace[nthreads] will contain sum
*/
void hypre_prefix_sum(HYPRE_Int *in_out, HYPRE_Int *sum, HYPRE_Int *workspace);
/**
* This version does prefix sum in pair.
* Useful when we prefix sum of diag and offd in tandem.
*
* @param worksapce at least with length 2*(nthreads+1)
* workspace[2*tid] and workspace[2*tid+1] will contain results for tid
* workspace[3*nthreads] and workspace[3*nthreads + 1] will contain sums
*/
void hypre_prefix_sum_pair(HYPRE_Int *in_out1, HYPRE_Int *sum1, HYPRE_Int *in_out2, HYPRE_Int *sum2,
HYPRE_Int *workspace);
/**
* @param workspace at least with length 3*(nthreads+1)
* workspace[3*tid:3*tid+3) will contain results for tid
*/
void hypre_prefix_sum_triple(HYPRE_Int *in_out1, HYPRE_Int *sum1, HYPRE_Int *in_out2,
HYPRE_Int *sum2, HYPRE_Int *in_out3, HYPRE_Int *sum3, HYPRE_Int *workspace);
/**
* n prefix-sums together.
* workspace[n*tid:n*(tid+1)) will contain results for tid
* workspace[nthreads*tid:nthreads*(tid+1)) will contain sums
*
* @param workspace at least with length n*(nthreads+1)
*/
void hypre_prefix_sum_multiple(HYPRE_Int *in_out, HYPRE_Int *sum, HYPRE_Int n,
HYPRE_Int *workspace);
/* hopscotch_hash.c */
#ifdef HYPRE_USING_OPENMP
/* Check if atomic operations are available to use concurrent hopscotch hash table */
#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
#define HYPRE_USING_ATOMIC
//#elif defined _MSC_VER // JSP: haven't tested, so comment out for now
//#define HYPRE_USING_ATOMIC
//#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
// JSP: not many compilers have implemented this, so comment out for now
//#define HYPRE_USING_ATOMIC
//#include <stdatomic.h>
#endif
#endif // HYPRE_USING_OPENMP
#ifdef HYPRE_HOPSCOTCH
#ifdef HYPRE_USING_ATOMIC
// concurrent hopscotch hashing is possible only with atomic supports
#define HYPRE_CONCURRENT_HOPSCOTCH
#endif
#endif
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
typedef struct
{
HYPRE_Int volatile timestamp;
omp_lock_t lock;
} hypre_HopscotchSegment;
#endif
/**
* The current typical use case of unordered set is putting input sequence
* with lots of duplication (putting all colidx received from other ranks),
* followed by one sweep of enumeration.
* Since the capacity is set to the number of inputs, which is much larger
* than the number of unique elements, we optimize for initialization and
* enumeration whose time is proportional to the capacity.
* For initialization and enumeration, structure of array (SoA) is better
* for vectorization, cache line utilization, and so on.
*/
typedef struct
{
HYPRE_Int volatile segmentMask;
HYPRE_Int volatile bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* volatile segments;
#endif
HYPRE_Int *volatile key;
hypre_uint *volatile hopInfo;
HYPRE_Int *volatile hash;
} hypre_UnorderedIntSet;
typedef struct
{
HYPRE_Int volatile segmentMask;
HYPRE_Int volatile bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* volatile segments;
#endif
HYPRE_BigInt *volatile key;
hypre_uint *volatile hopInfo;
HYPRE_BigInt *volatile hash;
} hypre_UnorderedBigIntSet;
typedef struct
{
hypre_uint volatile hopInfo;
HYPRE_Int volatile hash;
HYPRE_Int volatile key;
HYPRE_Int volatile data;
} hypre_HopscotchBucket;
typedef struct
{
hypre_uint volatile hopInfo;
HYPRE_BigInt volatile hash;
HYPRE_BigInt volatile key;
HYPRE_Int volatile data;
} hypre_BigHopscotchBucket;
/**
* The current typical use case of unoredered map is putting input sequence
* with no duplication (inverse map of a bijective mapping) followed by
* lots of lookups.
* For lookup, array of structure (AoS) gives better cache line utilization.
*/
typedef struct
{
HYPRE_Int volatile segmentMask;
HYPRE_Int volatile bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* volatile segments;
#endif
hypre_HopscotchBucket* volatile table;
} hypre_UnorderedIntMap;
typedef struct
{
HYPRE_Int volatile segmentMask;
HYPRE_Int volatile bucketMask;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* volatile segments;
#endif
hypre_BigHopscotchBucket* volatile table;
} hypre_UnorderedBigIntMap;
/* merge_sort.c */
/**
* Why merge sort?
* 1) Merge sort can take advantage of eliminating duplicates.
* 2) Merge sort is more efficiently parallelizable than qsort
*/
HYPRE_Int hypre_MergeOrderedArrays( HYPRE_Int size1, HYPRE_Int *array1, HYPRE_Int size2,
HYPRE_Int *array2, HYPRE_Int *size3_ptr, HYPRE_Int **array3_ptr);
void hypre_union2(HYPRE_Int n1, HYPRE_BigInt *arr1, HYPRE_Int n2, HYPRE_BigInt *arr2, HYPRE_Int *n3,
HYPRE_BigInt *arr3, HYPRE_Int *map1, HYPRE_Int *map2);
void hypre_merge_sort(HYPRE_Int *in, HYPRE_Int *temp, HYPRE_Int len, HYPRE_Int **sorted);
void hypre_big_merge_sort(HYPRE_BigInt *in, HYPRE_BigInt *temp, HYPRE_Int len,
HYPRE_BigInt **sorted);
void hypre_sort_and_create_inverse_map(HYPRE_Int *in, HYPRE_Int len, HYPRE_Int **out,
hypre_UnorderedIntMap *inverse_map);
void hypre_big_sort_and_create_inverse_map(HYPRE_BigInt *in, HYPRE_Int len, HYPRE_BigInt **out,
hypre_UnorderedBigIntMap *inverse_map);
#if defined(HYPRE_USING_GPU)
HYPRE_Int hypre_SyncComputeStream(hypre_Handle *hypre_handle);
HYPRE_Int hypre_SyncCudaDevice(hypre_Handle *hypre_handle);
HYPRE_Int hypre_ResetCudaDevice(hypre_Handle *hypre_handle);
HYPRE_Int hypreDevice_DiagScaleVector(HYPRE_Int n, HYPRE_Int *A_i, HYPRE_Complex *A_data,
HYPRE_Complex *x, HYPRE_Complex beta, HYPRE_Complex *y);
HYPRE_Int hypreDevice_DiagScaleVector2(HYPRE_Int n, HYPRE_Int *A_i, HYPRE_Complex *A_data,
HYPRE_Complex *x, HYPRE_Complex beta, HYPRE_Complex *y, HYPRE_Complex *z);
HYPRE_Int hypreDevice_IVAXPY(HYPRE_Int n, HYPRE_Complex *a, HYPRE_Complex *x, HYPRE_Complex *y);
HYPRE_Int hypreDevice_IVAXPYMarked(HYPRE_Int n, HYPRE_Complex *a, HYPRE_Complex *x,
HYPRE_Complex *y, HYPRE_Int *marker, HYPRE_Int marker_val);
HYPRE_Int hypreDevice_BigIntFilln(HYPRE_BigInt *d_x, size_t n, HYPRE_BigInt v);
HYPRE_Int hypreDevice_Filln(HYPRE_Complex *d_x, size_t n, HYPRE_Complex v);
HYPRE_Int hypreDevice_Scalen(HYPRE_Complex *d_x, size_t n, HYPRE_Complex v);
HYPRE_Int* hypreDevice_CsrRowPtrsToIndices(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ptr);
HYPRE_Int hypreDevice_CsrRowPtrsToIndices_v2(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ptr,
HYPRE_Int *d_row_ind);
HYPRE_Int* hypreDevice_CsrRowIndicesToPtrs(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ind);
HYPRE_Int hypreDevice_CsrRowIndicesToPtrs_v2(HYPRE_Int nrows, HYPRE_Int nnz, HYPRE_Int *d_row_ind,
HYPRE_Int *d_row_ptr);
#endif
HYPRE_Int hypre_CurandUniform( HYPRE_Int n, HYPRE_Real *urand, HYPRE_Int set_seed,
hypre_ulonglongint seed, HYPRE_Int set_offset, hypre_ulonglongint offset);
HYPRE_Int hypre_CurandUniformSingle( HYPRE_Int n, float *urand, HYPRE_Int set_seed,
hypre_ulonglongint seed, HYPRE_Int set_offset, hypre_ulonglongint offset);
HYPRE_Int hypre_bind_device(HYPRE_Int myid, HYPRE_Int nproc, MPI_Comm comm);
/* nvtx.c */
void hypre_GpuProfilingPushRangeColor(const char *name, HYPRE_Int cid);
void hypre_GpuProfilingPushRange(const char *name);
void hypre_GpuProfilingPopRange();
/* utilities.c */
HYPRE_Int hypre_multmod(HYPRE_Int a, HYPRE_Int b, HYPRE_Int mod);
void hypre_partition1D(HYPRE_Int n, HYPRE_Int p, HYPRE_Int j, HYPRE_Int *s, HYPRE_Int *e);
char *hypre_strcpy(char *destination, const char *source);
HYPRE_Int hypre_SetSyncCudaCompute(HYPRE_Int action);
HYPRE_Int hypre_RestoreSyncCudaCompute();
HYPRE_Int hypre_GetSyncCudaCompute(HYPRE_Int *cuda_compute_stream_sync_ptr);
HYPRE_Int hypre_SyncComputeStream(hypre_Handle *hypre_handle);
/* handle.c */
HYPRE_Int hypre_SetSpGemmUseCusparse( HYPRE_Int use_cusparse );
HYPRE_Int hypre_SetSpGemmAlgorithm( HYPRE_Int value );
HYPRE_Int hypre_SetSpGemmRownnzEstimateMethod( HYPRE_Int value );
HYPRE_Int hypre_SetSpGemmRownnzEstimateNSamples( HYPRE_Int value );
HYPRE_Int hypre_SetSpGemmRownnzEstimateMultFactor( HYPRE_Real value );
HYPRE_Int hypre_SetSpGemmHashType( char value );
HYPRE_Int hypre_SetUseGpuRand( HYPRE_Int use_gpurand );
HYPRE_Int hypre_SetGaussSeidelMethod( HYPRE_Int gs_method );
HYPRE_Int hypre_SetUserDeviceMalloc(GPUMallocFunc func);
HYPRE_Int hypre_SetUserDeviceMfree(GPUMfreeFunc func);
/* int_array.c */
hypre_IntArray* hypre_IntArrayCreate( HYPRE_Int size );
HYPRE_Int hypre_IntArrayDestroy( hypre_IntArray *array );
HYPRE_Int hypre_IntArrayInitialize_v2( hypre_IntArray *array,
HYPRE_MemoryLocation memory_location );
HYPRE_Int hypre_IntArrayInitialize( hypre_IntArray *array );
HYPRE_Int hypre_IntArrayCopy( hypre_IntArray *x, hypre_IntArray *y );
hypre_IntArray* hypre_IntArrayCloneDeep_v2( hypre_IntArray *x,
HYPRE_MemoryLocation memory_location );
hypre_IntArray* hypre_IntArrayCloneDeep( hypre_IntArray *x );
HYPRE_Int hypre_IntArraySetConstantValues( hypre_IntArray *v, HYPRE_Int value );
/******************************************************************************
* Copyright 1998-2019 Lawrence Livermore National Security, LLC and other
* HYPRE Project Developers. See the top-level COPYRIGHT file for details.
*
* SPDX-License-Identifier: (Apache-2.0 OR MIT)
******************************************************************************/
/**
* Hopscotch hash is modified from the code downloaded from
* https://sites.google.com/site/cconcurrencypackage/hopscotch-hashing
* with the following terms of usage
*/
////////////////////////////////////////////////////////////////////////////////
//TERMS OF USAGE
//------------------------------------------------------------------------------
//
// Permission to use, copy, modify and distribute this software and
// its documentation for any purpose is hereby granted without fee,
// provided that due acknowledgments to the authors are provided and
// this permission notice appears in all copies of the software.
// The software is provided "as is". There is no warranty of any kind.
//
//Authors:
// Maurice Herlihy
// Brown University
// and
// Nir Shavit
// Tel-Aviv University
// and
// Moran Tzafrir
// Tel-Aviv University
//
// Date: July 15, 2008.
//
////////////////////////////////////////////////////////////////////////////////
// Programmer : Moran Tzafrir (MoranTza@gmail.com)
// Modified : Jongsoo Park (jongsoo.park@intel.com)
// Oct 1, 2015.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef hypre_HOPSCOTCH_HASH_HEADER
#define hypre_HOPSCOTCH_HASH_HEADER
//#include <strings.h>
#include <string.h>
#include <stdio.h>
#include <limits.h>
//#include <math.h>
#ifdef HYPRE_USING_OPENMP
#include <omp.h>
#endif
//#include "_hypre_utilities.h"
// Potentially architecture specific features used here:
// __sync_val_compare_and_swap
#ifdef __cplusplus
extern "C" {
#endif
/******************************************************************************
* This next section of code is here instead of in _hypre_utilities.h to get
* around some portability issues with Visual Studio. By putting it here, we
* can explicitly include this '.h' file in a few files in hypre and compile
* them with C++ instead of C (VS does not support C99 'inline').
******************************************************************************/
#ifdef HYPRE_USING_ATOMIC
static inline HYPRE_Int
hypre_compare_and_swap( HYPRE_Int *ptr, HYPRE_Int oldval, HYPRE_Int newval )
{
#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
return __sync_val_compare_and_swap(ptr, oldval, newval);
//#elif defind _MSC_VER
//return _InterlockedCompareExchange((long *)ptr, newval, oldval);
//#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
// JSP: not many compilers have implemented this, so comment out for now
//_Atomic HYPRE_Int *atomic_ptr = ptr;
//atomic_compare_exchange_strong(atomic_ptr, &oldval, newval);
//return oldval;
#endif
}
static inline HYPRE_Int
hypre_fetch_and_add( HYPRE_Int *ptr, HYPRE_Int value )
{
#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
return __sync_fetch_and_add(ptr, value);
//#elif defined _MSC_VER
//return _InterlockedExchangeAdd((long *)ptr, value);
//#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
// JSP: not many compilers have implemented this, so comment out for now
//_Atomic HYPRE_Int *atomic_ptr = ptr;
//return atomic_fetch_add(atomic_ptr, value);
#endif
}
#else // !HYPRE_USING_ATOMIC
static inline HYPRE_Int
hypre_compare_and_swap( HYPRE_Int *ptr, HYPRE_Int oldval, HYPRE_Int newval )
{
if (*ptr == oldval)
{
*ptr = newval;
return oldval;
}
else { return *ptr; }
}
static inline HYPRE_Int
hypre_fetch_and_add( HYPRE_Int *ptr, HYPRE_Int value )
{
HYPRE_Int oldval = *ptr;
*ptr += value;
return oldval;
}
#endif // !HYPRE_USING_ATOMIC
/******************************************************************************/
// Constants ................................................................
#define HYPRE_HOPSCOTCH_HASH_HOP_RANGE (32)
#define HYPRE_HOPSCOTCH_HASH_INSERT_RANGE (4*1024)
#define HYPRE_HOPSCOTCH_HASH_EMPTY (0)
#define HYPRE_HOPSCOTCH_HASH_BUSY (1)
// Small Utilities ..........................................................
static inline HYPRE_Int
first_lsb_bit_indx( hypre_uint x )
{
HYPRE_Int pos;
#if defined(_MSC_VER) || defined(__MINGW64__)
if (x == 0)
{
pos = 0;
}
else
{
for (pos = 1; !(x & 1); ++pos)
{
x >>= 1;
}
}
#else
pos = ffs(x);
#endif
return (pos - 1);
}
/**
* hypre_Hash is adapted from xxHash with the following license.
*/
/*
xxHash - Extremely Fast Hash algorithm
Header File
Copyright (C) 2012-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/***************************************
* Constants
***************************************/
#define HYPRE_XXH_PRIME32_1 2654435761U
#define HYPRE_XXH_PRIME32_2 2246822519U
#define HYPRE_XXH_PRIME32_3 3266489917U
#define HYPRE_XXH_PRIME32_4 668265263U
#define HYPRE_XXH_PRIME32_5 374761393U
#define HYPRE_XXH_PRIME64_1 11400714785074694791ULL
#define HYPRE_XXH_PRIME64_2 14029467366897019727ULL
#define HYPRE_XXH_PRIME64_3 1609587929392839161ULL
#define HYPRE_XXH_PRIME64_4 9650029242287828579ULL
#define HYPRE_XXH_PRIME64_5 2870177450012600261ULL
#define HYPRE_XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
#define HYPRE_XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
#if defined(HYPRE_MIXEDINT) || defined(HYPRE_BIGINT)
static inline HYPRE_BigInt
hypre_BigHash( HYPRE_BigInt input )
{
hypre_ulongint h64 = HYPRE_XXH_PRIME64_5 + sizeof(input);
hypre_ulongint k1 = input;
k1 *= HYPRE_XXH_PRIME64_2;
k1 = HYPRE_XXH_rotl64(k1, 31);
k1 *= HYPRE_XXH_PRIME64_1;
h64 ^= k1;
h64 = HYPRE_XXH_rotl64(h64, 27) * HYPRE_XXH_PRIME64_1 + HYPRE_XXH_PRIME64_4;
h64 ^= h64 >> 33;
h64 *= HYPRE_XXH_PRIME64_2;
h64 ^= h64 >> 29;
h64 *= HYPRE_XXH_PRIME64_3;
h64 ^= h64 >> 32;
#ifndef NDEBUG
if (HYPRE_HOPSCOTCH_HASH_EMPTY == h64)
{
hypre_printf("hash(%lld) = %d\n", h64, HYPRE_HOPSCOTCH_HASH_EMPTY);
hypre_assert(HYPRE_HOPSCOTCH_HASH_EMPTY != h64);
}
#endif
return h64;
}
#else
static inline HYPRE_Int
hypre_BigHash(HYPRE_Int input)
{
hypre_uint h32 = HYPRE_XXH_PRIME32_5 + sizeof(input);
// 1665863975 is added to input so that
// only -1073741824 gives HYPRE_HOPSCOTCH_HASH_EMPTY.
// Hence, we're fine as long as key is non-negative.
h32 += (input + 1665863975) * HYPRE_XXH_PRIME32_3;
h32 = HYPRE_XXH_rotl32(h32, 17) * HYPRE_XXH_PRIME32_4;
h32 ^= h32 >> 15;
h32 *= HYPRE_XXH_PRIME32_2;
h32 ^= h32 >> 13;
h32 *= HYPRE_XXH_PRIME32_3;
h32 ^= h32 >> 16;
//hypre_assert(HYPRE_HOPSCOTCH_HASH_EMPTY != h32);
return h32;
}
#endif
#ifdef HYPRE_BIGINT
static inline HYPRE_Int
hypre_Hash(HYPRE_Int input)
{
hypre_ulongint h64 = HYPRE_XXH_PRIME64_5 + sizeof(input);
hypre_ulongint k1 = input;
k1 *= HYPRE_XXH_PRIME64_2;
k1 = HYPRE_XXH_rotl64(k1, 31);
k1 *= HYPRE_XXH_PRIME64_1;
h64 ^= k1;
h64 = HYPRE_XXH_rotl64(h64, 27) * HYPRE_XXH_PRIME64_1 + HYPRE_XXH_PRIME64_4;
h64 ^= h64 >> 33;
h64 *= HYPRE_XXH_PRIME64_2;
h64 ^= h64 >> 29;
h64 *= HYPRE_XXH_PRIME64_3;
h64 ^= h64 >> 32;
#ifndef NDEBUG
if (HYPRE_HOPSCOTCH_HASH_EMPTY == h64)
{
hypre_printf("hash(%lld) = %d\n", h64, HYPRE_HOPSCOTCH_HASH_EMPTY);
hypre_assert(HYPRE_HOPSCOTCH_HASH_EMPTY != h64);
}
#endif
return h64;
}
#else
static inline HYPRE_Int
hypre_Hash(HYPRE_Int input)
{
hypre_uint h32 = HYPRE_XXH_PRIME32_5 + sizeof(input);
// 1665863975 is added to input so that
// only -1073741824 gives HYPRE_HOPSCOTCH_HASH_EMPTY.
// Hence, we're fine as long as key is non-negative.
h32 += (input + 1665863975) * HYPRE_XXH_PRIME32_3;
h32 = HYPRE_XXH_rotl32(h32, 17) * HYPRE_XXH_PRIME32_4;
h32 ^= h32 >> 15;
h32 *= HYPRE_XXH_PRIME32_2;
h32 ^= h32 >> 13;
h32 *= HYPRE_XXH_PRIME32_3;
h32 ^= h32 >> 16;
//hypre_assert(HYPRE_HOPSCOTCH_HASH_EMPTY != h32);
return h32;
}
#endif
static inline void
hypre_UnorderedIntSetFindCloserFreeBucket( hypre_UnorderedIntSet *s,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *start_seg,
#endif
HYPRE_Int *free_bucket,
HYPRE_Int *free_dist )
{
HYPRE_Int move_bucket = *free_bucket - (HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1);
HYPRE_Int move_free_dist;
for (move_free_dist = HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1; move_free_dist > 0; --move_free_dist)
{
hypre_uint start_hop_info = s->hopInfo[move_bucket];
HYPRE_Int move_new_free_dist = -1;
hypre_uint mask = 1;
HYPRE_Int i;
for (i = 0; i < move_free_dist; ++i, mask <<= 1)
{
if (mask & start_hop_info)
{
move_new_free_dist = i;
break;
}
}
if (-1 != move_new_free_dist)
{
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* move_segment = &(s->segments[move_bucket & s->segmentMask]);
if (start_seg != move_segment)
{
omp_set_lock(&move_segment->lock);
}
#endif
if (start_hop_info == s->hopInfo[move_bucket])
{
// new_free_bucket -> free_bucket and empty new_free_bucket
HYPRE_Int new_free_bucket = move_bucket + move_new_free_dist;
s->key[*free_bucket] = s->key[new_free_bucket];
s->hash[*free_bucket] = s->hash[new_free_bucket];
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
++move_segment->timestamp;
#pragma omp flush
#endif
s->hopInfo[move_bucket] |= (1U << move_free_dist);
s->hopInfo[move_bucket] &= ~(1U << move_new_free_dist);
*free_bucket = new_free_bucket;
*free_dist -= move_free_dist - move_new_free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
return;
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
}
++move_bucket;
}
*free_bucket = -1;
*free_dist = 0;
}
static inline void
hypre_UnorderedBigIntSetFindCloserFreeBucket( hypre_UnorderedBigIntSet *s,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *start_seg,
#endif
HYPRE_Int *free_bucket,
HYPRE_Int *free_dist )
{
HYPRE_Int move_bucket = *free_bucket - (HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1);
HYPRE_Int move_free_dist;
for (move_free_dist = HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1; move_free_dist > 0; --move_free_dist)
{
hypre_uint start_hop_info = s->hopInfo[move_bucket];
HYPRE_Int move_new_free_dist = -1;
hypre_uint mask = 1;
HYPRE_Int i;
for (i = 0; i < move_free_dist; ++i, mask <<= 1)
{
if (mask & start_hop_info)
{
move_new_free_dist = i;
break;
}
}
if (-1 != move_new_free_dist)
{
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* move_segment = &(s->segments[move_bucket & s->segmentMask]);
if (start_seg != move_segment)
{
omp_set_lock(&move_segment->lock);
}
#endif
if (start_hop_info == s->hopInfo[move_bucket])
{
// new_free_bucket -> free_bucket and empty new_free_bucket
HYPRE_Int new_free_bucket = move_bucket + move_new_free_dist;
s->key[*free_bucket] = s->key[new_free_bucket];
s->hash[*free_bucket] = s->hash[new_free_bucket];
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
++move_segment->timestamp;
#pragma omp flush
#endif
s->hopInfo[move_bucket] |= (1U << move_free_dist);
s->hopInfo[move_bucket] &= ~(1U << move_new_free_dist);
*free_bucket = new_free_bucket;
*free_dist -= move_free_dist - move_new_free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
return;
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
}
++move_bucket;
}
*free_bucket = -1;
*free_dist = 0;
}
static inline void
hypre_UnorderedIntMapFindCloserFreeBucket( hypre_UnorderedIntMap *m,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *start_seg,
#endif
hypre_HopscotchBucket **free_bucket,
HYPRE_Int *free_dist)
{
hypre_HopscotchBucket* move_bucket = *free_bucket - (HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1);
HYPRE_Int move_free_dist;
for (move_free_dist = HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1; move_free_dist > 0; --move_free_dist)
{
hypre_uint start_hop_info = move_bucket->hopInfo;
HYPRE_Int move_new_free_dist = -1;
hypre_uint mask = 1;
HYPRE_Int i;
for (i = 0; i < move_free_dist; ++i, mask <<= 1)
{
if (mask & start_hop_info)
{
move_new_free_dist = i;
break;
}
}
if (-1 != move_new_free_dist)
{
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* move_segment = &(m->segments[(move_bucket - m->table) & m->segmentMask]);
if (start_seg != move_segment)
{
omp_set_lock(&move_segment->lock);
}
#endif
if (start_hop_info == move_bucket->hopInfo)
{
// new_free_bucket -> free_bucket and empty new_free_bucket
hypre_HopscotchBucket* new_free_bucket = move_bucket + move_new_free_dist;
(*free_bucket)->data = new_free_bucket->data;
(*free_bucket)->key = new_free_bucket->key;
(*free_bucket)->hash = new_free_bucket->hash;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
++move_segment->timestamp;
#pragma omp flush
#endif
move_bucket->hopInfo |= (1U << move_free_dist);
move_bucket->hopInfo &= ~(1U << move_new_free_dist);
*free_bucket = new_free_bucket;
*free_dist -= move_free_dist - move_new_free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
return;
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
}
++move_bucket;
}
*free_bucket = NULL;
*free_dist = 0;
}
static inline void
hypre_UnorderedBigIntMapFindCloserFreeBucket( hypre_UnorderedBigIntMap *m,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *start_seg,
#endif
hypre_BigHopscotchBucket **free_bucket,
HYPRE_Int *free_dist)
{
hypre_BigHopscotchBucket* move_bucket = *free_bucket - (HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1);
HYPRE_Int move_free_dist;
for (move_free_dist = HYPRE_HOPSCOTCH_HASH_HOP_RANGE - 1; move_free_dist > 0; --move_free_dist)
{
hypre_uint start_hop_info = move_bucket->hopInfo;
HYPRE_Int move_new_free_dist = -1;
hypre_uint mask = 1;
HYPRE_Int i;
for (i = 0; i < move_free_dist; ++i, mask <<= 1)
{
if (mask & start_hop_info)
{
move_new_free_dist = i;
break;
}
}
if (-1 != move_new_free_dist)
{
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment* move_segment = &(m->segments[(move_bucket - m->table) & m->segmentMask]);
if (start_seg != move_segment)
{
omp_set_lock(&move_segment->lock);
}
#endif
if (start_hop_info == move_bucket->hopInfo)
{
// new_free_bucket -> free_bucket and empty new_free_bucket
hypre_BigHopscotchBucket* new_free_bucket = move_bucket + move_new_free_dist;
(*free_bucket)->data = new_free_bucket->data;
(*free_bucket)->key = new_free_bucket->key;
(*free_bucket)->hash = new_free_bucket->hash;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
++move_segment->timestamp;
#pragma omp flush
#endif
move_bucket->hopInfo |= (1U << move_free_dist);
move_bucket->hopInfo &= ~(1U << move_new_free_dist);
*free_bucket = new_free_bucket;
*free_dist -= move_free_dist - move_new_free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
return;
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (start_seg != move_segment)
{
omp_unset_lock(&move_segment->lock);
}
#endif
}
++move_bucket;
}
*free_bucket = NULL;
*free_dist = 0;
}
void hypre_UnorderedIntSetCreate( hypre_UnorderedIntSet *s,
HYPRE_Int inCapacity,
HYPRE_Int concurrencyLevel);
void hypre_UnorderedBigIntSetCreate( hypre_UnorderedBigIntSet *s,
HYPRE_Int inCapacity,
HYPRE_Int concurrencyLevel);
void hypre_UnorderedIntMapCreate( hypre_UnorderedIntMap *m,
HYPRE_Int inCapacity,
HYPRE_Int concurrencyLevel);
void hypre_UnorderedBigIntMapCreate( hypre_UnorderedBigIntMap *m,
HYPRE_Int inCapacity,
HYPRE_Int concurrencyLevel);
void hypre_UnorderedIntSetDestroy( hypre_UnorderedIntSet *s );
void hypre_UnorderedBigIntSetDestroy( hypre_UnorderedBigIntSet *s );
void hypre_UnorderedIntMapDestroy( hypre_UnorderedIntMap *m );
void hypre_UnorderedBigIntMapDestroy( hypre_UnorderedBigIntMap *m );
// Query Operations .........................................................
static inline HYPRE_Int
hypre_UnorderedIntSetContains( hypre_UnorderedIntSet *s,
HYPRE_Int key )
{
//CALCULATE HASH ..........................
#ifdef HYPRE_BIGINT
HYPRE_Int hash = hypre_BigHash(key);
#else
HYPRE_Int hash = hypre_Hash(key);
#endif
//CHECK IF ALREADY CONTAIN ................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &s->segments[hash & s->segmentMask];
#endif
HYPRE_Int bucket = hash & s->bucketMask;
hypre_uint hopInfo = s->hopInfo[bucket];
if (0 == hopInfo)
{
return 0;
}
else if (1 == hopInfo )
{
if (hash == s->hash[bucket] && key == s->key[bucket])
{
return 1;
}
else { return 0; }
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
HYPRE_Int startTimestamp = segment->timestamp;
#endif
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
HYPRE_Int currElm = bucket + i;
if (hash == s->hash[currElm] && key == s->key[currElm])
{
return 1;
}
hopInfo &= ~(1U << i);
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (segment->timestamp == startTimestamp)
{
return 0;
}
#endif
HYPRE_Int i;
for (i = 0; i < HYPRE_HOPSCOTCH_HASH_HOP_RANGE; ++i)
{
if (hash == s->hash[bucket + i] && key == s->key[bucket + i])
{
return 1;
}
}
return 0;
}
static inline HYPRE_Int
hypre_UnorderedBigIntSetContains( hypre_UnorderedBigIntSet *s,
HYPRE_BigInt key )
{
//CALCULATE HASH ..........................
#if defined(HYPRE_BIGINT) || defined(HYPRE_MIXEDINT)
HYPRE_BigInt hash = hypre_BigHash(key);
#else
HYPRE_BigInt hash = hypre_Hash(key);
#endif
//CHECK IF ALREADY CONTAIN ................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &s->segments[(HYPRE_Int)(hash & s->segmentMask)];
#endif
HYPRE_Int bucket = (HYPRE_Int)(hash & s->bucketMask);
hypre_uint hopInfo = s->hopInfo[bucket];
if (0 == hopInfo)
{
return 0;
}
else if (1 == hopInfo )
{
if (hash == s->hash[bucket] && key == s->key[bucket])
{
return 1;
}
else { return 0; }
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
HYPRE_Int startTimestamp = segment->timestamp;
#endif
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
HYPRE_Int currElm = bucket + i;
if (hash == s->hash[currElm] && key == s->key[currElm])
{
return 1;
}
hopInfo &= ~(1U << i);
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (segment->timestamp == startTimestamp)
{
return 0;
}
#endif
HYPRE_Int i;
for (i = 0; i < HYPRE_HOPSCOTCH_HASH_HOP_RANGE; ++i)
{
if (hash == s->hash[bucket + i] && key == s->key[bucket + i])
{
return 1;
}
}
return 0;
}
/**
* @ret -1 if key doesn't exist
*/
static inline HYPRE_Int
hypre_UnorderedIntMapGet( hypre_UnorderedIntMap *m,
HYPRE_Int key )
{
//CALCULATE HASH ..........................
#ifdef HYPRE_BIGINT
HYPRE_Int hash = hypre_BigHash(key);
#else
HYPRE_Int hash = hypre_Hash(key);
#endif
//CHECK IF ALREADY CONTAIN ................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &m->segments[hash & m->segmentMask];
#endif
hypre_HopscotchBucket *elmAry = &(m->table[hash & m->bucketMask]);
hypre_uint hopInfo = elmAry->hopInfo;
if (0 == hopInfo)
{
return -1;
}
else if (1 == hopInfo )
{
if (hash == elmAry->hash && key == elmAry->key)
{
return elmAry->data;
}
else { return -1; }
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
HYPRE_Int startTimestamp = segment->timestamp;
#endif
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
hypre_HopscotchBucket* currElm = elmAry + i;
if (hash == currElm->hash && key == currElm->key)
{
return currElm->data;
}
hopInfo &= ~(1U << i);
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (segment->timestamp == startTimestamp)
{
return -1;
}
#endif
hypre_HopscotchBucket *currBucket = &(m->table[hash & m->bucketMask]);
HYPRE_Int i;
for (i = 0; i < HYPRE_HOPSCOTCH_HASH_HOP_RANGE; ++i, ++currBucket)
{
if (hash == currBucket->hash && key == currBucket->key)
{
return currBucket->data;
}
}
return -1;
}
static inline
HYPRE_Int hypre_UnorderedBigIntMapGet( hypre_UnorderedBigIntMap *m,
HYPRE_BigInt key )
{
//CALCULATE HASH ..........................
#if defined(HYPRE_BIGINT) || defined(HYPRE_MIXEDINT)
HYPRE_BigInt hash = hypre_BigHash(key);
#else
HYPRE_BigInt hash = hypre_Hash(key);
#endif
//CHECK IF ALREADY CONTAIN ................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &m->segments[(HYPRE_Int)(hash & m->segmentMask)];
#endif
hypre_BigHopscotchBucket *elmAry = &(m->table[(HYPRE_Int)(hash & m->bucketMask)]);
hypre_uint hopInfo = elmAry->hopInfo;
if (0 == hopInfo)
{
return -1;
}
else if (1 == hopInfo )
{
if (hash == elmAry->hash && key == elmAry->key)
{
return elmAry->data;
}
else { return -1; }
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
HYPRE_Int startTimestamp = segment->timestamp;
#endif
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
hypre_BigHopscotchBucket* currElm = elmAry + i;
if (hash == currElm->hash && key == currElm->key)
{
return currElm->data;
}
hopInfo &= ~(1U << i);
}
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
if (segment->timestamp == startTimestamp)
{
return -1;
}
#endif
hypre_BigHopscotchBucket *currBucket = &(m->table[hash & m->bucketMask]);
HYPRE_Int i;
for (i = 0; i < HYPRE_HOPSCOTCH_HASH_HOP_RANGE; ++i, ++currBucket)
{
if (hash == currBucket->hash && key == currBucket->key)
{
return currBucket->data;
}
}
return -1;
}
//status Operations .........................................................
static inline
HYPRE_Int hypre_UnorderedIntSetSize( hypre_UnorderedIntSet *s )
{
HYPRE_Int counter = 0;
HYPRE_Int n = s->bucketMask + HYPRE_HOPSCOTCH_HASH_INSERT_RANGE;
HYPRE_Int i;
for (i = 0; i < n; ++i)
{
if (HYPRE_HOPSCOTCH_HASH_EMPTY != s->hash[i])
{
++counter;
}
}
return counter;
}
static inline
HYPRE_Int hypre_UnorderedBigIntSetSize( hypre_UnorderedBigIntSet *s )
{
HYPRE_Int counter = 0;
HYPRE_BigInt n = s->bucketMask + HYPRE_HOPSCOTCH_HASH_INSERT_RANGE;
HYPRE_Int i;
for (i = 0; i < n; ++i)
{
if (HYPRE_HOPSCOTCH_HASH_EMPTY != s->hash[i])
{
++counter;
}
}
return counter;
}
static inline HYPRE_Int
hypre_UnorderedIntMapSize( hypre_UnorderedIntMap *m )
{
HYPRE_Int counter = 0;
HYPRE_Int n = m->bucketMask + HYPRE_HOPSCOTCH_HASH_INSERT_RANGE;
HYPRE_Int i;
for (i = 0; i < n; ++i)
{
if ( HYPRE_HOPSCOTCH_HASH_EMPTY != m->table[i].hash )
{
++counter;
}
}
return counter;
}
static inline HYPRE_Int
hypre_UnorderedBigIntMapSize( hypre_UnorderedBigIntMap *m )
{
HYPRE_Int counter = 0;
HYPRE_Int n = m->bucketMask + HYPRE_HOPSCOTCH_HASH_INSERT_RANGE;
HYPRE_Int i;
for (i = 0; i < n; ++i)
{
if ( HYPRE_HOPSCOTCH_HASH_EMPTY != m->table[i].hash )
{
++counter;
}
}
return counter;
}
HYPRE_Int *hypre_UnorderedIntSetCopyToArray( hypre_UnorderedIntSet *s, HYPRE_Int *len );
HYPRE_BigInt *hypre_UnorderedBigIntSetCopyToArray( hypre_UnorderedBigIntSet *s, HYPRE_Int *len );
//modification Operations ...................................................
static inline void
hypre_UnorderedIntSetPut( hypre_UnorderedIntSet *s,
HYPRE_Int key )
{
//CALCULATE HASH ..........................
#ifdef HYPRE_BIGINT
HYPRE_Int hash = hypre_BigHash(key);
#else
HYPRE_Int hash = hypre_Hash(key);
#endif
//LOCK KEY HASH ENTERY ....................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &s->segments[hash & s->segmentMask];
omp_set_lock(&segment->lock);
#endif
HYPRE_Int bucket = hash & s->bucketMask;
//CHECK IF ALREADY CONTAIN ................
hypre_uint hopInfo = s->hopInfo[bucket];
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
HYPRE_Int currElm = bucket + i;
if (hash == s->hash[currElm] && key == s->key[currElm])
{
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return;
}
hopInfo &= ~(1U << i);
}
//LOOK FOR FREE BUCKET ....................
HYPRE_Int free_bucket = bucket;
HYPRE_Int free_dist = 0;
for ( ; free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE; ++free_dist, ++free_bucket)
{
if ( (HYPRE_HOPSCOTCH_HASH_EMPTY == s->hash[free_bucket]) &&
(HYPRE_HOPSCOTCH_HASH_EMPTY ==
hypre_compare_and_swap((HYPRE_Int *)&s->hash[free_bucket],
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_EMPTY,
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_BUSY)) )
{
break;
}
}
//PLACE THE NEW KEY .......................
if (free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE)
{
do
{
if (free_dist < HYPRE_HOPSCOTCH_HASH_HOP_RANGE)
{
s->key[free_bucket] = key;
s->hash[free_bucket] = hash;
s->hopInfo[bucket] |= 1U << free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return;
}
hypre_UnorderedIntSetFindCloserFreeBucket(s,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
segment,
#endif
&free_bucket, &free_dist);
}
while (-1 != free_bucket);
}
//NEED TO RESIZE ..........................
hypre_error_w_msg(HYPRE_ERROR_GENERIC, "ERROR - RESIZE is not implemented\n");
/*fprintf(stderr, "ERROR - RESIZE is not implemented\n");*/
exit(1);
return;
}
static inline void
hypre_UnorderedBigIntSetPut( hypre_UnorderedBigIntSet *s,
HYPRE_BigInt key )
{
//CALCULATE HASH ..........................
#if defined(HYPRE_BIGINT) || defined(HYPRE_MIXEDINT)
HYPRE_BigInt hash = hypre_BigHash(key);
#else
HYPRE_BigInt hash = hypre_Hash(key);
#endif
//LOCK KEY HASH ENTERY ....................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &s->segments[hash & s->segmentMask];
omp_set_lock(&segment->lock);
#endif
HYPRE_Int bucket = (HYPRE_Int)(hash & s->bucketMask);
//CHECK IF ALREADY CONTAIN ................
hypre_uint hopInfo = s->hopInfo[bucket];
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
HYPRE_Int currElm = bucket + i;
if (hash == s->hash[currElm] && key == s->key[currElm])
{
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return;
}
hopInfo &= ~(1U << i);
}
//LOOK FOR FREE BUCKET ....................
HYPRE_Int free_bucket = bucket;
HYPRE_Int free_dist = 0;
for ( ; free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE; ++free_dist, ++free_bucket)
{
if ( (HYPRE_HOPSCOTCH_HASH_EMPTY == s->hash[free_bucket]) &&
(HYPRE_HOPSCOTCH_HASH_EMPTY ==
hypre_compare_and_swap((HYPRE_Int *)&s->hash[free_bucket],
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_EMPTY,
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_BUSY)) )
{
break;
}
}
//PLACE THE NEW KEY .......................
if (free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE)
{
do
{
if (free_dist < HYPRE_HOPSCOTCH_HASH_HOP_RANGE)
{
s->key[free_bucket] = key;
s->hash[free_bucket] = hash;
s->hopInfo[bucket] |= 1U << free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return;
}
hypre_UnorderedBigIntSetFindCloserFreeBucket(s,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
segment,
#endif
&free_bucket, &free_dist);
}
while (-1 != free_bucket);
}
//NEED TO RESIZE ..........................
hypre_error_w_msg(HYPRE_ERROR_GENERIC, "ERROR - RESIZE is not implemented\n");
/*fprintf(stderr, "ERROR - RESIZE is not implemented\n");*/
exit(1);
return;
}
static inline HYPRE_Int
hypre_UnorderedIntMapPutIfAbsent( hypre_UnorderedIntMap *m,
HYPRE_Int key, HYPRE_Int data )
{
//CALCULATE HASH ..........................
#ifdef HYPRE_BIGINT
HYPRE_Int hash = hypre_BigHash(key);
#else
HYPRE_Int hash = hypre_Hash(key);
#endif
//LOCK KEY HASH ENTERY ....................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &m->segments[hash & m->segmentMask];
omp_set_lock(&segment->lock);
#endif
hypre_HopscotchBucket* startBucket = &(m->table[hash & m->bucketMask]);
//CHECK IF ALREADY CONTAIN ................
hypre_uint hopInfo = startBucket->hopInfo;
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
hypre_HopscotchBucket* currElm = startBucket + i;
if (hash == currElm->hash && key == currElm->key)
{
HYPRE_Int rc = currElm->data;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return rc;
}
hopInfo &= ~(1U << i);
}
//LOOK FOR FREE BUCKET ....................
hypre_HopscotchBucket* free_bucket = startBucket;
HYPRE_Int free_dist = 0;
for ( ; free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE; ++free_dist, ++free_bucket)
{
if ( (HYPRE_HOPSCOTCH_HASH_EMPTY == free_bucket->hash) &&
(HYPRE_HOPSCOTCH_HASH_EMPTY ==
hypre_compare_and_swap((HYPRE_Int *)&free_bucket->hash,
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_EMPTY,
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_BUSY)) )
{
break;
}
}
//PLACE THE NEW KEY .......................
if (free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE)
{
do
{
if (free_dist < HYPRE_HOPSCOTCH_HASH_HOP_RANGE)
{
free_bucket->data = data;
free_bucket->key = key;
free_bucket->hash = hash;
startBucket->hopInfo |= 1U << free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return HYPRE_HOPSCOTCH_HASH_EMPTY;
}
hypre_UnorderedIntMapFindCloserFreeBucket(m,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
segment,
#endif
&free_bucket, &free_dist);
}
while (NULL != free_bucket);
}
//NEED TO RESIZE ..........................
hypre_error_w_msg(HYPRE_ERROR_GENERIC, "ERROR - RESIZE is not implemented\n");
/*fprintf(stderr, "ERROR - RESIZE is not implemented\n");*/
exit(1);
return HYPRE_HOPSCOTCH_HASH_EMPTY;
}
static inline HYPRE_Int
hypre_UnorderedBigIntMapPutIfAbsent( hypre_UnorderedBigIntMap *m,
HYPRE_BigInt key, HYPRE_Int data)
{
//CALCULATE HASH ..........................
#if defined(HYPRE_BIGINT) || defined(HYPRE_MIXEDINT)
HYPRE_BigInt hash = hypre_BigHash(key);
#else
HYPRE_BigInt hash = hypre_Hash(key);
#endif
//LOCK KEY HASH ENTERY ....................
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
hypre_HopscotchSegment *segment = &m->segments[hash & m->segmentMask];
omp_set_lock(&segment->lock);
#endif
hypre_BigHopscotchBucket* startBucket = &(m->table[hash & m->bucketMask]);
//CHECK IF ALREADY CONTAIN ................
hypre_uint hopInfo = startBucket->hopInfo;
while (0 != hopInfo)
{
HYPRE_Int i = first_lsb_bit_indx(hopInfo);
hypre_BigHopscotchBucket* currElm = startBucket + i;
if (hash == currElm->hash && key == currElm->key)
{
HYPRE_Int rc = currElm->data;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return rc;
}
hopInfo &= ~(1U << i);
}
//LOOK FOR FREE BUCKET ....................
hypre_BigHopscotchBucket* free_bucket = startBucket;
HYPRE_Int free_dist = 0;
for ( ; free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE; ++free_dist, ++free_bucket)
{
if ( (HYPRE_HOPSCOTCH_HASH_EMPTY == free_bucket->hash) &&
(HYPRE_HOPSCOTCH_HASH_EMPTY ==
hypre_compare_and_swap((HYPRE_Int *)&free_bucket->hash,
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_EMPTY,
(HYPRE_Int)HYPRE_HOPSCOTCH_HASH_BUSY)) )
{
break;
}
}
//PLACE THE NEW KEY .......................
if (free_dist < HYPRE_HOPSCOTCH_HASH_INSERT_RANGE)
{
do
{
if (free_dist < HYPRE_HOPSCOTCH_HASH_HOP_RANGE)
{
free_bucket->data = data;
free_bucket->key = key;
free_bucket->hash = hash;
startBucket->hopInfo |= 1U << free_dist;
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
omp_unset_lock(&segment->lock);
#endif
return HYPRE_HOPSCOTCH_HASH_EMPTY;
}
hypre_UnorderedBigIntMapFindCloserFreeBucket(m,
#ifdef HYPRE_CONCURRENT_HOPSCOTCH
segment,
#endif
&free_bucket, &free_dist);
}
while (NULL != free_bucket);
}
//NEED TO RESIZE ..........................
hypre_error_w_msg(HYPRE_ERROR_GENERIC, "ERROR - RESIZE is not implemented\n");
/*fprintf(stderr, "ERROR - RESIZE is not implemented\n");*/
exit(1);
return HYPRE_HOPSCOTCH_HASH_EMPTY;
}
#ifdef __cplusplus
} // extern "C"
#endif
#endif // hypre_HOPSCOTCH_HASH_HEADER
#ifdef __cplusplus
}
#endif
#endif
Reference in New Issue View Git Blame Copy Permalink