31#ifndef __TASMANIAN_SPARSE_GRID_ACCELERATED_DATA_STRUCTURES_HPP
32#define __TASMANIAN_SPARSE_GRID_ACCELERATED_DATA_STRUCTURES_HPP
34#include "tsgAcceleratedHandles.hpp"
81struct AccelerationContext;
103 GpuVector(
GpuVector<T> &&other) : num_entries(std::exchange(other.num_entries, 0)), gpu_data(std::exchange(other.gpu_data, nullptr))
104 #ifdef Tasmanian_ENABLE_DPCPP
105 , sycl_queue(other.sycl_queue)
111 std::swap(num_entries, temp.num_entries);
112 std::swap(gpu_data, temp.gpu_data);
113 #ifdef Tasmanian_ENABLE_DPCPP
114 std::swap(sycl_queue, temp.sycl_queue);
138 template<
typename IteratorLike>
140 load(acc, std::distance(ibegin, iend), std::addressof(*ibegin));
146 size_t size()
const{
return num_entries; }
150 const T*
data()
const{
return gpu_data; }
157 bool empty()
const{
return (num_entries == 0); }
164 load(acc, cpu_data.size(), cpu_data.data());
177 if constexpr (std::is_same_v<U, T>) {
178 load_internal(acc, count, cpu_data);
181 std::vector<T> converted(count);
182 std::transform(cpu_data, cpu_data + count, converted.begin(), [](U
const &x)->T{ return static_cast<T>(x); });
183 load_internal(acc, count, converted.data());
189 cpu_data.resize(num_entries);
190 unload(acc, cpu_data.data());
205 T* external = gpu_data;
225 size_t num_entries = 0;
226 T *gpu_data =
nullptr;
227 #ifdef Tasmanian_ENABLE_DPCPP
228 void* sycl_queue =
nullptr;
250 #ifdef Tasmanian_ENABLE_CUDA
266 #ifdef Tasmanian_ENABLE_HIP
268 void setRocBlasHandle(
void *handle);
270 void setRocSparseHandle(
void *handle);
273 std::unique_ptr<int, HandleDeleter<AccHandle::Rocblas>> rblas_handle;
275 std::unique_ptr<int, HandleDeleter<AccHandle::Rocsparse>> rsparse_handle;
278 #ifdef Tasmanian_ENABLE_DPCPP
280 void setSyclQueue(
void *queue);
282 std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>> internal_queue;
303 set(acc, transform_a, transform_b);
316 operator bool ()
const {
return (acceleration !=
nullptr); }
325 acceleration =
nullptr;
331 int num_dimensions = 0, padded_size = 0;
356 void dtrans2can(
AccelerationContext const *acc,
bool use01,
int dims,
int num_x,
int pad_size,
357 const double *gpu_trans_a,
const double *gpu_trans_b,
358 const T *gpu_x_transformed, T *gpu_x_canonical);
373 void devalpwpoly(
AccelerationContext const *acc,
int order,
TypeOneDRule rule,
int num_dimensions,
int num_x,
int num_basis,
const T *gpu_x,
const T *gpu_nodes,
const T *gpu_support, T *gpu_y);
405 void devalseq(
AccelerationContext const *acc,
int dims,
int num_x,
const std::vector<int> &max_levels,
const T *gpu_x,
427 void devalglo(
AccelerationContext const *acc,
bool is_nested,
bool is_clenshawcurtis0,
int dims,
int num_x,
int num_p,
int num_basis,
438 void fillDataGPU(
AccelerationContext const *acc,
double value,
long long N,
long long stride,
double data[]);
444 template<
typename T>
void load_n_internal(
AccelerationContext const *acc, T
const *cpu_data,
size_t num_entries, T *gpu_data);
450 template<
typename T,
typename U>
451 void load_n(
AccelerationContext const *acc, U
const *cpu_data,
size_t num_entries, T *gpu_data){
452 if constexpr (std::is_same_v<U, T>) {
453 load_n_internal(acc, cpu_data, num_entries, gpu_data);
455 std::vector<T> converted(num_entries);
456 std::transform(cpu_data, cpu_data + num_entries, converted.begin(), [](U
const &x)->T{ return static_cast<T>(x); });
457 load_n_internal(acc, converted.data(), num_entries, gpu_data);
462 #ifdef __TASMANIAN_COMPILE_FALLBACK_CUDA_KERNELS__
468 void cudaDgemm(
int M,
int N,
int K,
const double *gpu_a,
const double *gpu_b,
double *gpu_c);
473 void cudaSparseMatmul(
int M,
int N,
int num_nz,
const int* gpu_spntr,
const int* gpu_sindx,
const double* gpu_svals,
const double *gpu_B,
double *gpu_C);
478 void cudaSparseVecDenseMat(
int M,
int N,
int num_nz,
const double *A,
const int *indx,
const double *vals,
double *C);
483 void convert_sparse_to_dense(
int num_rows,
int num_columns,
const int *gpu_pntr,
const int *gpu_indx,
const double *gpu_vals,
double *gpu_destination);
491namespace AccelerationMeta{
497 std::map<std::string, TypeAcceleration> getStringToAccelerationMap();
518 #ifdef Tasmanian_ENABLE_MAGMA
521 #ifdef Tasmanian_ENABLE_CUDA
525 #ifdef Tasmanian_ENABLE_HIP
529 #ifdef Tasmanian_ENABLE_DPCPP
533 #ifdef Tasmanian_ENABLE_BLAS
554 int getNumGpuDevices();
561 void setDefaultGpuDevice(
int deviceID);
568 unsigned long long getTotalGPUMemory(
int deviceID);
575 std::string getGpuDeviceName(
int deviceID);
580 template<
typename T>
void recvGpuArray(
AccelerationContext const*,
size_t num_entries,
const T *gpu_data, std::vector<T> &cpu_data);
591 void *createCublasHandle();
596 void deleteCublasHandle(
void *);
660 #ifdef Tasmanian_ENABLE_BLAS
668 #ifdef Tasmanian_ENABLE_DPCPP
697 #ifdef Tasmanian_ENABLE_BLAS
706 #if defined(Tasmanian_ENABLE_CUDA) || defined(Tasmanian_ENABLE_HIP)
715 TypeAcceleration effective_acc = AccelerationMeta::getAvailableFallback(acc);
716 #ifdef Tasmanian_ENABLE_DPCPP
717 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < -1 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
718 throw std::runtime_error(
"Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
720 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < 0 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
721 throw std::runtime_error(
"Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
727 return (AccelerationMeta::isAccTypeGPU(effective_acc)) ? device_change :
change_gpu_device;
729 return (AccelerationMeta::isAccTypeGPU(effective_acc)) ?
change_gpu_enabled : mode_change;
736 TypeAcceleration effective_acc = AccelerationMeta::getAvailableFallback(acc);
738 #ifdef Tasmanian_ENABLE_DPCPP
739 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < -1 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
740 throw std::runtime_error(
"Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
742 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < 0 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
743 throw std::runtime_error(
"Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
745 if (AccelerationMeta::isAccTypeGPU(effective_acc)){
755 mode = effective_acc;
766#ifdef Tasmanian_ENABLE_DPCPP
780struct InternalSyclQueue{
782 InternalSyclQueue() =
default;
784 void init_testing(
int gpuid);
786 operator std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>> (){
787 return std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>>(test_queue.get(),
788 HandleDeleter<AccHandle::Syclqueue>(
false));
791 bool use_testing =
false;
793 std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>> test_queue;
802extern InternalSyclQueue test_queue;
Implements the domain transform algorithms in case the user data is provided on the GPU.
Definition tsgAcceleratedDataStructures.hpp:297
void clear()
resets the transform, used when changing the compute device
Definition tsgAcceleratedDataStructures.hpp:320
AccelerationDomainTransform(AccelerationContext const *acc, std::vector< double > const &transform_a, std::vector< double > const &transform_b)
Constructor, load the transform data to the GPU, the vectors are the same as used in the TasmanianSpa...
Definition tsgAcceleratedDataStructures.hpp:302
void set(AccelerationContext const *, std::vector< double > const &transform_a, std::vector< double > const &transform_b)
set the domain transform
void getCanonicalPoints(bool use01, T const gpu_transformed_x[], int num_x, GpuVector< T > &gpu_canonical_x)
Transform a set of points, used in the calls to evaluateHierarchicalFunctionsGPU() Takes the user pro...
AccelerationDomainTransform()=default
construct an empty acceleration transform
Template class that wraps around a single GPU array, providing functionality that mimics std::vector.
Definition tsgAcceleratedDataStructures.hpp:95
GpuVector()=default
Default constructor, creates an empty (null) array.
~GpuVector()
Destructor, release all allocated memory.
Definition tsgAcceleratedDataStructures.hpp:143
bool empty() const
Return true if the size() is zero.
Definition tsgAcceleratedDataStructures.hpp:157
void load(AccelerationContext const *acc, const std::vector< U > &cpu_data)
Takes a vector with entries of possibly different precision, maybe converts and loads.
Definition tsgAcceleratedDataStructures.hpp:163
void unload(AccelerationContext const *acc, std::vector< T > &cpu_data) const
Copy the data from the GPU array to cpu_data, the cpu_data will be resized and overwritten.
Definition tsgAcceleratedDataStructures.hpp:188
GpuVector(AccelerationContext const *acc, IteratorLike ibegin, IteratorLike iend)
Construct a vector by loading from a given range.
Definition tsgAcceleratedDataStructures.hpp:139
GpuVector(GpuVector< T > const &)=delete
Delete the copy-constructor.
std::vector< T > unload(AccelerationContext const *acc) const
Return a CPU vector holding the data of the GPU.
Definition tsgAcceleratedDataStructures.hpp:193
T * eject()
Move the data to the external array, the vector is set to empty (unlike move command on std::vector).
Definition tsgAcceleratedDataStructures.hpp:204
void unload(AccelerationContext const *acc, size_t num, T *cpu_data) const
Copy the first num entries to the cpu_data buffer, assumes that the buffer is sufficiently large.
GpuVector(AccelerationContext const *acc, int dim1, int dim2)
Same as GpuVector(dim1 * dim2), but guards against overflow.
Definition tsgAcceleratedDataStructures.hpp:132
GpuVector(AccelerationContext const *acc, size_t count)
Construct a vector with count number of entries.
Definition tsgAcceleratedDataStructures.hpp:122
void unload(AccelerationContext const *acc, T *cpu_data) const
Copy the data from the GPU array to the cpu_data buffer, assumes that the buffer is sufficiently larg...
Definition tsgAcceleratedDataStructures.hpp:201
T value_type
The data-type of the vector entries.
Definition tsgAcceleratedDataStructures.hpp:212
void clear()
Delete all allocated memory and reset the array to empty.
GpuVector< T > & operator=(GpuVector< T > const &)=delete
Delete the copy-assignment.
size_t size() const
Return the current size of the GPU array.
Definition tsgAcceleratedDataStructures.hpp:146
const T * data() const
Get a const-reference to the GPU array, which an be used as input to GPU libraries and kernels.
Definition tsgAcceleratedDataStructures.hpp:150
void resize(AccelerationContext const *acc, size_t count)
Clear all data currently stored in the vector and allocate a new array (unlike std::vector this does ...
void load(AccelerationContext const *acc, size_t count, const U *cpu_data)
Takes a vector with entries of possibly different precision, maybe converts and always loads.
Definition tsgAcceleratedDataStructures.hpp:176
GpuVector(AccelerationContext const *acc, const std::vector< T > &cpu_data)
Create a vector with size that matches cpu_data and copy the data to the GPU device.
Definition tsgAcceleratedDataStructures.hpp:134
GpuVector(AccelerationContext const *acc, int dim1, int dim2, T const *cpu_data)
Construct a vector and load with date provided on to the cpu.
Definition tsgAcceleratedDataStructures.hpp:136
GpuVector(GpuVector< T > &&other)
Allow for move-construction.
Definition tsgAcceleratedDataStructures.hpp:103
T * data()
Get a reference to the GPU array, which an be used as input to GPU libraries and kernels.
Definition tsgAcceleratedDataStructures.hpp:148
TypeOneDRule
Used to specify the one dimensional family of rules that induces the sparse grid.
Definition tsgEnumerates.hpp:288
constexpr TypeAcceleration accel_gpu_rocblas
At the front API, the HIP and CUDA options are equivalent, see TasGrid::TypeAcceleration.
Definition tsgEnumerates.hpp:578
constexpr TypeAcceleration accel_gpu_hip
At the front API, the HIP and CUDA options are equivalent, see TasGrid::TypeAcceleration.
Definition tsgEnumerates.hpp:573
TypeAcceleration
Modes of acceleration.
Definition tsgEnumerates.hpp:554
@ accel_cpu_blas
Default (if available), uses both BLAS and LAPACK libraries.
Definition tsgEnumerates.hpp:558
@ accel_none
Usually the slowest mode, uses only OpenMP multi-threading, but optimized for memory and could be the...
Definition tsgEnumerates.hpp:556
@ accel_gpu_magma
Same the CUDA option but uses the UTK MAGMA library for the linear algebra operations.
Definition tsgEnumerates.hpp:566
@ accel_gpu_cublas
Mixed usage of the CPU (OpenMP) and GPU libraries.
Definition tsgEnumerates.hpp:562
@ accel_gpu_cuda
Similar to the cuBLAS option but also uses a set of Tasmanian custom GPU kernels.
Definition tsgEnumerates.hpp:564
size_t size_mult(IntA a, IntB b)
Converts two integer-like variables to size_t and returns the product..
Definition tsgUtils.hpp:82
Encapsulates the Tasmanian Sparse Grid module.
Definition TasmanianSparseGrid.hpp:68
Wrapper class around GPU device ID, acceleration type and GpuEngine.
Definition tsgAcceleratedDataStructures.hpp:609
ChangeType testEnable(TypeAcceleration acc, int new_gpu_id) const
Returns the ChangeType if enable() is called, but does not change the acceleration.
Definition tsgAcceleratedDataStructures.hpp:714
AlgorithmPreference algorithm_select
The preference to use dense or sparse algorithms.
Definition tsgAcceleratedDataStructures.hpp:651
void enable(TypeAcceleration acc, int new_gpu_id)
Accepts parameters directly from TasmanianSparseGrid::enableAcceleration()
Definition tsgAcceleratedDataStructures.hpp:734
AccelerationContext(AccelerationContext &&)=default
Can move the context.
static constexpr int getDefaultAccDevice()
Returns the default acceleration device, CUDA/HIP use GPU 0, SYCL uses -1 which uses sycl::default_se...
Definition tsgAcceleratedDataStructures.hpp:667
ChangeType favorSparse(bool favor)
Sets algorithm affinity in the direction of sparse.
Definition tsgAcceleratedDataStructures.hpp:679
TypeAcceleration mode
The current active acceleration mode.
Definition tsgAcceleratedDataStructures.hpp:649
bool on_gpu() const
Returns true if any of the GPU-based acceleration modes have been enabled.
Definition tsgAcceleratedDataStructures.hpp:763
AlgorithmPreference
Defines the sparse-dense algorithm flavors, whenever applicable.
Definition tsgAcceleratedDataStructures.hpp:620
@ algorithm_dense
Use dense algorithm.
Definition tsgAcceleratedDataStructures.hpp:622
@ algorithm_sparse
Use sparse algorithm.
Definition tsgAcceleratedDataStructures.hpp:624
@ algorithm_autoselect
Use automatically select based on heuristics.
Definition tsgAcceleratedDataStructures.hpp:626
bool blasCompatible() const
Returns true if BLAS is enabled and the current mode is not none.
Definition tsgAcceleratedDataStructures.hpp:696
static constexpr TypeAcceleration getDefaultAccMode()
Returns the default acceleration mode, cpu_blas if BLAS is enabled and none otherwise.
Definition tsgAcceleratedDataStructures.hpp:659
int device
If using a GPU acceleration mode, holds the active device.
Definition tsgAcceleratedDataStructures.hpp:653
std::optional< GpuEngine > engine
Holds the context to the GPU TPL handles, e.g., MAGMA queue.
Definition tsgAcceleratedDataStructures.hpp:656
void setDevice() const
Set default device.
Definition tsgAcceleratedDataStructures.hpp:759
AccelerationContext & operator=(AccelerationContext const &)=delete
Cannot copy the context.
ChangeType
Defines the types of acceleration context updates so they can be linked to acceleration cache updates...
Definition tsgAcceleratedDataStructures.hpp:635
@ change_none
No change, do nothing.
Definition tsgAcceleratedDataStructures.hpp:637
@ change_sparse_dense
Change the sparse-dense AlgorithmPreference.
Definition tsgAcceleratedDataStructures.hpp:645
@ change_gpu_device
Change the associated GPU device.
Definition tsgAcceleratedDataStructures.hpp:639
@ change_cpu_blas
Change BLAS to none or none to BLAS.
Definition tsgAcceleratedDataStructures.hpp:643
@ change_gpu_enabled
Change from BLAS or none to a GPU acceleration mode.
Definition tsgAcceleratedDataStructures.hpp:641
AccelerationContext()
Creates a default context, the device id is set to 0 and acceleration is BLAS (if available) or none.
Definition tsgAcceleratedDataStructures.hpp:676
bool useKernels() const
Returns true if the current mode implies the use of custom GPU kernels.
Definition tsgAcceleratedDataStructures.hpp:705
AccelerationContext(AccelerationContext const &)=delete
Cannot copy the context.
Wrapper class around calls GPU accelerated linear algebra libraries.
Definition tsgAcceleratedDataStructures.hpp:238
GpuEngine(GpuEngine const &)=delete
Do not copy, even if no GPU backend is enabled.
GpuEngine()=default
Default construction, does not use any GPU acceleration, will set handles on-the-fly.
GpuEngine(GpuEngine &&)=default
Moving the engine is allowed, unique pointers will be moved.
void setCuSolverDnHandle(void *handle)
Manually sets the cuSparse handle, handle must be a valid cusolverDnHandle_t associated with this CUD...
GpuEngine & operator=(GpuEngine const &)=delete
Do not copy, even if no GPU backend is enabled.
std::unique_ptr< int, HandleDeleter< AccHandle::Cusolver > > cusolver_handle
Holds the cuSolver handle.
Definition tsgAcceleratedDataStructures.hpp:263
void setCuBlasHandle(void *handle)
Manually sets the cuBlas handle, handle must be a valid cublasHandle_t associated with this CUDA devi...
void setCuSparseHandle(void *handle)
Manually sets the cuSparse handle, handle must be a valid cusparseHandle_t associated with this CUDA ...
bool called_magma_init
Avoids an empty engine when no acceleration is enabled, allows for default constructor/move/copy,...
Definition tsgAcceleratedDataStructures.hpp:286
std::unique_ptr< int, HandleDeleter< AccHandle::Cublas > > cublas_handle
Holds the cuBlas handle.
Definition tsgAcceleratedDataStructures.hpp:259
std::unique_ptr< int, HandleDeleter< AccHandle::Cusparse > > cusparse_handle
Holds the cuSparse handle.
Definition tsgAcceleratedDataStructures.hpp:261