Doxygen 1.9.8
Toolkit for Adaptive Stochastic Modeling and Non-Intrusive ApproximatioN: Tasmanian v8.3
 
Loading...
Searching...
No Matches
tsgAcceleratedDataStructures.hpp
Go to the documentation of this file.
1/*
2 * Copyright (c) 2017, Miroslav Stoyanov
3 *
4 * This file is part of
5 * Toolkit for Adaptive Stochastic Modeling And Non-Intrusive ApproximatioN: TASMANIAN
6 *
7 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions
12 * and the following disclaimer in the documentation and/or other materials provided with the distribution.
13 *
14 * 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse
15 * or promote products derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
20 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
21 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
22 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 * UT-BATTELLE, LLC AND THE UNITED STATES GOVERNMENT MAKE NO REPRESENTATIONS AND DISCLAIM ALL WARRANTIES, BOTH EXPRESSED AND IMPLIED.
25 * THERE ARE NO EXPRESS OR IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY PATENT,
26 * COPYRIGHT, TRADEMARK, OR OTHER PROPRIETARY RIGHTS, OR THAT THE SOFTWARE WILL ACCOMPLISH THE INTENDED RESULTS OR THAT THE SOFTWARE OR ITS USE WILL NOT RESULT IN INJURY OR DAMAGE.
27 * THE USER ASSUMES RESPONSIBILITY FOR ALL LIABILITIES, PENALTIES, FINES, CLAIMS, CAUSES OF ACTION, AND COSTS AND EXPENSES, CAUSED BY, RESULTING FROM OR ARISING OUT OF,
28 * IN WHOLE OR IN PART THE USE, STORAGE OR DISPOSAL OF THE SOFTWARE.
29 */
30
31#ifndef __TASMANIAN_SPARSE_GRID_ACCELERATED_DATA_STRUCTURES_HPP
32#define __TASMANIAN_SPARSE_GRID_ACCELERATED_DATA_STRUCTURES_HPP
33
34#include "tsgAcceleratedHandles.hpp"
35
45
79namespace TasGrid{
80
81struct AccelerationContext; // forward declaration, CUDA and HIP GpuVector do not use the context, but DPC++ needs it
82
94template<typename T>
96public:
98 GpuVector(GpuVector<T> const &) = delete;
101
103 GpuVector(GpuVector<T> &&other) : num_entries(std::exchange(other.num_entries, 0)), gpu_data(std::exchange(other.gpu_data, nullptr))
104 #ifdef Tasmanian_ENABLE_DPCPP
105 , sycl_queue(other.sycl_queue)
106 #endif
107 {}
110 GpuVector<T> temp(std::move(other));
111 std::swap(num_entries, temp.num_entries);
112 std::swap(gpu_data, temp.gpu_data);
113 #ifdef Tasmanian_ENABLE_DPCPP
114 std::swap(sycl_queue, temp.sycl_queue);
115 #endif
116 return *this;
117 }
118
120 GpuVector() = default;
122 GpuVector(AccelerationContext const *acc, size_t count){ resize(acc, count); }
123
132 GpuVector(AccelerationContext const *acc, int dim1, int dim2){ resize(acc, Utils::size_mult(dim1, dim2)); }
134 GpuVector(AccelerationContext const *acc, const std::vector<T> &cpu_data){ load(acc, cpu_data); }
136 GpuVector(AccelerationContext const *acc, int dim1, int dim2, T const *cpu_data){ load(acc, Utils::size_mult(dim1, dim2), cpu_data); }
138 template<typename IteratorLike>
139 GpuVector(AccelerationContext const *acc, IteratorLike ibegin, IteratorLike iend) : GpuVector(){
140 load(acc, std::distance(ibegin, iend), std::addressof(*ibegin));
141 }
144
146 size_t size() const{ return num_entries; }
148 T* data(){ return gpu_data; }
150 const T* data() const{ return gpu_data; }
151
153 void resize(AccelerationContext const *acc, size_t count);
155 void clear();
157 bool empty() const{ return (num_entries == 0); }
158
162 template<typename U>
163 void load(AccelerationContext const *acc, const std::vector<U> &cpu_data){
164 load(acc, cpu_data.size(), cpu_data.data());
165 }
166
175 template<typename U>
176 void load(AccelerationContext const *acc, size_t count, const U* cpu_data) {
177 if constexpr (std::is_same_v<U, T>) {
178 load_internal(acc, count, cpu_data);
179 } else {
180 // need to convert the array
181 std::vector<T> converted(count);
182 std::transform(cpu_data, cpu_data + count, converted.begin(), [](U const &x)->T{ return static_cast<T>(x); });
183 load_internal(acc, count, converted.data());
184 }
185 }
186
188 void unload(AccelerationContext const *acc, std::vector<T> &cpu_data) const{
189 cpu_data.resize(num_entries);
190 unload(acc, cpu_data.data());
191 }
193 std::vector<T> unload(AccelerationContext const *acc) const{
194 std::vector<T> y;
195 unload(acc, y);
196 return y;
197 }
199 void unload(AccelerationContext const *acc, size_t num, T* cpu_data) const;
201 void unload(AccelerationContext const *acc, T* cpu_data) const{ unload(acc, num_entries, cpu_data); }
202
204 T* eject(){
205 T* external = gpu_data;
206 gpu_data = nullptr;
207 num_entries = 0;
208 return external;
209 }
210
212 using value_type = T;
213
214private:
223 void load_internal(AccelerationContext const *acc, size_t count, const T* cpu_data);
224
225 size_t num_entries = 0; // keep track of the size, update on every call that changes the gpu_data
226 T *gpu_data = nullptr; // the GPU array
227 #ifdef Tasmanian_ENABLE_DPCPP
228 void* sycl_queue = nullptr;
229 #endif
230};
231
240 GpuEngine() = default;
242 GpuEngine(GpuEngine const &) = delete;
244 GpuEngine &operator = (GpuEngine const &) = delete;
246 GpuEngine(GpuEngine &&) = default;
249
250 #ifdef Tasmanian_ENABLE_CUDA
252 void setCuBlasHandle(void *handle);
254 void setCuSparseHandle(void *handle);
256 void setCuSolverDnHandle(void *handle);
257
259 std::unique_ptr<int, HandleDeleter<AccHandle::Cublas>> cublas_handle;
261 std::unique_ptr<int, HandleDeleter<AccHandle::Cusparse>> cusparse_handle;
263 std::unique_ptr<int, HandleDeleter<AccHandle::Cusolver>> cusolver_handle;
264 #endif
265
266 #ifdef Tasmanian_ENABLE_HIP
268 void setRocBlasHandle(void *handle);
270 void setRocSparseHandle(void *handle);
271
273 std::unique_ptr<int, HandleDeleter<AccHandle::Rocblas>> rblas_handle;
275 std::unique_ptr<int, HandleDeleter<AccHandle::Rocsparse>> rsparse_handle;
276 #endif
277
278 #ifdef Tasmanian_ENABLE_DPCPP
280 void setSyclQueue(void *queue);
282 std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>> internal_queue;
283 #endif
284
286 bool called_magma_init = false;
287};
288
292
298public:
302 AccelerationDomainTransform(AccelerationContext const *acc, std::vector<double> const &transform_a, std::vector<double> const &transform_b){
303 set(acc, transform_a, transform_b);
304 }
305
313 template<typename T>
314 void getCanonicalPoints(bool use01, T const gpu_transformed_x[], int num_x, GpuVector<T> &gpu_canonical_x);
316 operator bool () const { return (acceleration != nullptr); }
318 void set(AccelerationContext const *, std::vector<double> const &transform_a, std::vector<double> const &transform_b);
320 void clear() {
321 gpu_trans_a.clear();
322 gpu_trans_b.clear();
323 num_dimensions = 0;
324 padded_size = 0;
325 acceleration = nullptr;
326 }
327
328private:
329 // these actually store the rate and shift and not the hard upper/lower limits
330 GpuVector<double> gpu_trans_a, gpu_trans_b;
331 int num_dimensions = 0, padded_size = 0;
332 AccelerationContext const *acceleration = nullptr;
333};
334
338namespace TasGpu{
342
355 template<typename T>
356 void dtrans2can(AccelerationContext const *acc, bool use01, int dims, int num_x, int pad_size,
357 const double *gpu_trans_a, const double *gpu_trans_b,
358 const T *gpu_x_transformed, T *gpu_x_canonical);
359
363
372 template<typename T>
373 void devalpwpoly(AccelerationContext const *acc, int order, TypeOneDRule rule, int num_dimensions, int num_x, int num_basis, const T *gpu_x, const T *gpu_nodes, const T *gpu_support, T *gpu_y);
374
378
386 template<typename T>
387 void devalpwpoly_sparse(AccelerationContext const *acc, int order, TypeOneDRule rule, int dims, int num_x, const T *gpu_x,
388 const GpuVector<T> &gpu_nodes, const GpuVector<T> &gpu_support,
389 const GpuVector<int> &gpu_hpntr, const GpuVector<int> &gpu_hindx, const GpuVector<int> &gpu_hroots,
390 GpuVector<int> &gpu_spntr, GpuVector<int> &gpu_sindx, GpuVector<T> &gpu_svals);
391
395
404 template<typename T>
405 void devalseq(AccelerationContext const *acc, int dims, int num_x, const std::vector<int> &max_levels, const T *gpu_x,
406 const GpuVector<int> &num_nodes,
407 const GpuVector<int> &points, const GpuVector<T> &nodes, const GpuVector<T> &coeffs, T *gpu_result);
408
412
415 template<typename T>
416 void devalfor(AccelerationContext const *acc, int dims, int num_x, const std::vector<int> &max_levels, const T *gpu_x, const GpuVector<int> &num_nodes, const GpuVector<int> &points, T *gpu_wreal, typename GpuVector<T>::value_type *gpu_wimag);
417
426 template<typename T>
427 void devalglo(AccelerationContext const *acc, bool is_nested, bool is_clenshawcurtis0, int dims, int num_x, int num_p, int num_basis,
428 T const *gpu_x, GpuVector<T> const &nodes, GpuVector<T> const &coeff, GpuVector<T> const &tensor_weights,
429 GpuVector<int> const &nodes_per_level, GpuVector<int> const &offset_per_level, GpuVector<int> const &map_dimension, GpuVector<int> const &map_level,
430 GpuVector<int> const &active_tensors, GpuVector<int> const &active_num_points, GpuVector<int> const &dim_offsets,
431 GpuVector<int> const &map_tensor, GpuVector<int> const &map_index, GpuVector<int> const &map_reference, T *gpu_result);
432
433
438 void fillDataGPU(AccelerationContext const *acc, double value, long long N, long long stride, double data[]);
439
444 template<typename T> void load_n_internal(AccelerationContext const *acc, T const *cpu_data, size_t num_entries, T *gpu_data);
445
450 template<typename T, typename U>
451 void load_n(AccelerationContext const *acc, U const *cpu_data, size_t num_entries, T *gpu_data){
452 if constexpr (std::is_same_v<U, T>) {
453 load_n_internal(acc, cpu_data, num_entries, gpu_data);
454 } else {
455 std::vector<T> converted(num_entries);
456 std::transform(cpu_data, cpu_data + num_entries, converted.begin(), [](U const &x)->T{ return static_cast<T>(x); });
457 load_n_internal(acc, converted.data(), num_entries, gpu_data);
458 }
459 }
460
461 // #define __TASMANIAN_COMPILE_FALLBACK_CUDA_KERNELS__ // uncomment to compile a bunch of custom CUDA kernels that provide some functionality similar to cuBlas
462 #ifdef __TASMANIAN_COMPILE_FALLBACK_CUDA_KERNELS__
463 // CUDA kernels that provide essentially the same functionality as cuBlas and MAGMA, but nowhere near as optimal
464 // those functions should not be used in a Release or production builds
465 // the kernels are useful because they are simple and do not depend on potentially poorly documented 3d party library
466 // since the kernels are useful for testing and some debugging, the code should not be deleted (for now), but also don't waste time compiling in most cases
467
468 void cudaDgemm(int M, int N, int K, const double *gpu_a, const double *gpu_b, double *gpu_c);
469 // lazy cuda dgemm, nowhere near as powerful as cuBlas, but does not depend on cuBlas
470 // gpu_a is M by K, gpu_b is K by N, gpu_c is M by N, all in column-major format
471 // on exit gpu_c = gpu_a * gpu_b
472
473 void cudaSparseMatmul(int M, int N, int num_nz, const int* gpu_spntr, const int* gpu_sindx, const double* gpu_svals, const double *gpu_B, double *gpu_C);
474 // lazy cuda sparse dgemm, less efficient (especially for large N), but more memory conservative then cusparse as there is no need for a transpose
475 // C is M x N, B is K x N (K is max(gpu_sindx)), both are given in row-major format, num_nz/spntr/sindx/svals describe row compressed A which is M by K
476 // on exit C = A * B
477
478 void cudaSparseVecDenseMat(int M, int N, int num_nz, const double *A, const int *indx, const double *vals, double *C);
479 // dense matrix A (column major) times a sparse vector defiend by num_nz, indx, and vals
480 // A is M by N, C is M by 1,
481 // on exit C = A * (indx, vals)
482
483 void convert_sparse_to_dense(int num_rows, int num_columns, const int *gpu_pntr, const int *gpu_indx, const double *gpu_vals, double *gpu_destination);
484 // converts a sparse matrix to a dense representation (all data sits on the gpu and is pre-allocated)
485 #endif
486}
487
491namespace AccelerationMeta{
495 TypeAcceleration getIOAccelerationString(const char * name);
497 std::map<std::string, TypeAcceleration> getStringToAccelerationMap();
501 const char* getIOAccelerationString(TypeAcceleration accel);
505 int getIOAccelerationInt(TypeAcceleration accel);
509 TypeAcceleration getIOIntAcceleration(int accel);
513 bool isAccTypeGPU(TypeAcceleration accel);
514
516 inline bool isAvailable(TypeAcceleration accel){
517 switch(accel){
518 #ifdef Tasmanian_ENABLE_MAGMA
519 case accel_gpu_magma: return true;
520 #endif
521 #ifdef Tasmanian_ENABLE_CUDA
522 case accel_gpu_cuda: return true;
523 case accel_gpu_cublas: return true;
524 #endif
525 #ifdef Tasmanian_ENABLE_HIP
526 case accel_gpu_hip: return true;
527 case accel_gpu_rocblas: return true;
528 #endif
529 #ifdef Tasmanian_ENABLE_DPCPP
530 case accel_gpu_cuda: return true;
531 case accel_gpu_cublas: return true;
532 #endif
533 #ifdef Tasmanian_ENABLE_BLAS
534 case accel_cpu_blas: return true;
535 #endif
536 case accel_none:
537 return true;
538 default:
539 return false;
540 }
541 }
542
546
549 TypeAcceleration getAvailableFallback(TypeAcceleration accel);
550
554 int getNumGpuDevices();
555
559
561 void setDefaultGpuDevice(int deviceID);
562
566
568 unsigned long long getTotalGPUMemory(int deviceID);
569
573
575 std::string getGpuDeviceName(int deviceID);
576
580 template<typename T> void recvGpuArray(AccelerationContext const*, size_t num_entries, const T *gpu_data, std::vector<T> &cpu_data);
581
585 template<typename T> void delGpuArray(AccelerationContext const*, T *x);
586
591 void *createCublasHandle();
596 void deleteCublasHandle(void *);
597}
598
618
628
647
654
656 mutable std::optional<GpuEngine> engine;
657
659 inline static constexpr TypeAcceleration getDefaultAccMode() {
660 #ifdef Tasmanian_ENABLE_BLAS
661 return accel_cpu_blas;
662 #else
663 return accel_none;
664 #endif
665 }
667 inline static constexpr int getDefaultAccDevice() {
668 #ifdef Tasmanian_ENABLE_DPCPP
669 return -1;
670 #else
671 return 0;
672 #endif
673 }
674
677
680 AlgorithmPreference new_preference = [as=algorithm_select, favor]()->AlgorithmPreference{
681 if (favor){
683 }else{
685 }
686 }();
687 if (new_preference != algorithm_select){
688 algorithm_select = new_preference;
689 return change_sparse_dense;
690 }else{
691 return change_none;
692 }
693 }
694
696 bool blasCompatible() const{
697 #ifdef Tasmanian_ENABLE_BLAS
698 return (mode != accel_none);
699 #else
700 return false;
701 #endif
702 }
703
705 bool useKernels() const{
706 #if defined(Tasmanian_ENABLE_CUDA) || defined(Tasmanian_ENABLE_HIP)
707 return ((mode == accel_gpu_cuda) or (mode == accel_gpu_magma));
708 #else
709 return false;
710 #endif
711 }
712
714 ChangeType testEnable(TypeAcceleration acc, int new_gpu_id) const{
715 TypeAcceleration effective_acc = AccelerationMeta::getAvailableFallback(acc);
716 #ifdef Tasmanian_ENABLE_DPCPP
717 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < -1 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
718 throw std::runtime_error("Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
719 #else
720 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < 0 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
721 throw std::runtime_error("Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
722 #endif
723 ChangeType mode_change = (effective_acc == mode) ? change_none : change_cpu_blas;
724 ChangeType device_change = (device == new_gpu_id) ? change_none : change_gpu_device;
725
726 if (on_gpu()){
727 return (AccelerationMeta::isAccTypeGPU(effective_acc)) ? device_change : change_gpu_device;
728 }else{
729 return (AccelerationMeta::isAccTypeGPU(effective_acc)) ? change_gpu_enabled : mode_change;
730 }
731 }
732
734 void enable(TypeAcceleration acc, int new_gpu_id){
735 // get the effective new acceleration mode (use the fallback if acc is not enabled)
736 TypeAcceleration effective_acc = AccelerationMeta::getAvailableFallback(acc);
737 // if switching to a GPU mode, check if the device id is valid
738 #ifdef Tasmanian_ENABLE_DPCPP
739 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < -1 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
740 throw std::runtime_error("Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
741 #else
742 if (AccelerationMeta::isAccTypeGPU(effective_acc) and ((new_gpu_id < 0 or new_gpu_id >= AccelerationMeta::getNumGpuDevices())))
743 throw std::runtime_error("Invalid GPU device ID, see ./tasgrid -v for list of detected devices.");
744 #endif
745 if (AccelerationMeta::isAccTypeGPU(effective_acc)){
746 // if the new mode is GPU-based, make an engine or reset the engine if the device has changed
747 // if the engine exists and the device is not changed, then keep the existing engine
748 if (!engine or new_gpu_id != device)
749 engine.emplace();
750 }else{
751 engine.reset();
752 }
753
754 // assign the new values for the mode and device
755 mode = effective_acc;
756 device = new_gpu_id;
757 }
759 void setDevice() const{ AccelerationMeta::setDefaultGpuDevice(device); }
761 operator GpuEngine* () const{ return std::addressof(engine.value()); }
763 bool on_gpu() const{ return !!engine; }
764};
765
766#ifdef Tasmanian_ENABLE_DPCPP
780struct InternalSyclQueue{
782 InternalSyclQueue() = default;
784 void init_testing(int gpuid);
786 operator std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>> (){
787 return std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>>(test_queue.get(),
788 HandleDeleter<AccHandle::Syclqueue>(false));
789 }
791 bool use_testing = false;
793 std::unique_ptr<int, HandleDeleter<AccHandle::Syclqueue>> test_queue;
794};
802extern InternalSyclQueue test_queue;
803#endif
804
805}
806
807#endif // __TASMANIAN_SPARSE_GRID_ACCELERATED_DATA_STRUCTURES_HPP
Implements the domain transform algorithms in case the user data is provided on the GPU.
Definition tsgAcceleratedDataStructures.hpp:297
void clear()
resets the transform, used when changing the compute device
Definition tsgAcceleratedDataStructures.hpp:320
AccelerationDomainTransform(AccelerationContext const *acc, std::vector< double > const &transform_a, std::vector< double > const &transform_b)
Constructor, load the transform data to the GPU, the vectors are the same as used in the TasmanianSpa...
Definition tsgAcceleratedDataStructures.hpp:302
void set(AccelerationContext const *, std::vector< double > const &transform_a, std::vector< double > const &transform_b)
set the domain transform
void getCanonicalPoints(bool use01, T const gpu_transformed_x[], int num_x, GpuVector< T > &gpu_canonical_x)
Transform a set of points, used in the calls to evaluateHierarchicalFunctionsGPU() Takes the user pro...
AccelerationDomainTransform()=default
construct an empty acceleration transform
Template class that wraps around a single GPU array, providing functionality that mimics std::vector.
Definition tsgAcceleratedDataStructures.hpp:95
GpuVector()=default
Default constructor, creates an empty (null) array.
~GpuVector()
Destructor, release all allocated memory.
Definition tsgAcceleratedDataStructures.hpp:143
bool empty() const
Return true if the size() is zero.
Definition tsgAcceleratedDataStructures.hpp:157
void load(AccelerationContext const *acc, const std::vector< U > &cpu_data)
Takes a vector with entries of possibly different precision, maybe converts and loads.
Definition tsgAcceleratedDataStructures.hpp:163
void unload(AccelerationContext const *acc, std::vector< T > &cpu_data) const
Copy the data from the GPU array to cpu_data, the cpu_data will be resized and overwritten.
Definition tsgAcceleratedDataStructures.hpp:188
GpuVector(AccelerationContext const *acc, IteratorLike ibegin, IteratorLike iend)
Construct a vector by loading from a given range.
Definition tsgAcceleratedDataStructures.hpp:139
GpuVector(GpuVector< T > const &)=delete
Delete the copy-constructor.
std::vector< T > unload(AccelerationContext const *acc) const
Return a CPU vector holding the data of the GPU.
Definition tsgAcceleratedDataStructures.hpp:193
T * eject()
Move the data to the external array, the vector is set to empty (unlike move command on std::vector).
Definition tsgAcceleratedDataStructures.hpp:204
void unload(AccelerationContext const *acc, size_t num, T *cpu_data) const
Copy the first num entries to the cpu_data buffer, assumes that the buffer is sufficiently large.
GpuVector(AccelerationContext const *acc, int dim1, int dim2)
Same as GpuVector(dim1 * dim2), but guards against overflow.
Definition tsgAcceleratedDataStructures.hpp:132
GpuVector(AccelerationContext const *acc, size_t count)
Construct a vector with count number of entries.
Definition tsgAcceleratedDataStructures.hpp:122
void unload(AccelerationContext const *acc, T *cpu_data) const
Copy the data from the GPU array to the cpu_data buffer, assumes that the buffer is sufficiently larg...
Definition tsgAcceleratedDataStructures.hpp:201
T value_type
The data-type of the vector entries.
Definition tsgAcceleratedDataStructures.hpp:212
void clear()
Delete all allocated memory and reset the array to empty.
GpuVector< T > & operator=(GpuVector< T > const &)=delete
Delete the copy-assignment.
size_t size() const
Return the current size of the GPU array.
Definition tsgAcceleratedDataStructures.hpp:146
const T * data() const
Get a const-reference to the GPU array, which an be used as input to GPU libraries and kernels.
Definition tsgAcceleratedDataStructures.hpp:150
void resize(AccelerationContext const *acc, size_t count)
Clear all data currently stored in the vector and allocate a new array (unlike std::vector this does ...
void load(AccelerationContext const *acc, size_t count, const U *cpu_data)
Takes a vector with entries of possibly different precision, maybe converts and always loads.
Definition tsgAcceleratedDataStructures.hpp:176
GpuVector(AccelerationContext const *acc, const std::vector< T > &cpu_data)
Create a vector with size that matches cpu_data and copy the data to the GPU device.
Definition tsgAcceleratedDataStructures.hpp:134
GpuVector(AccelerationContext const *acc, int dim1, int dim2, T const *cpu_data)
Construct a vector and load with date provided on to the cpu.
Definition tsgAcceleratedDataStructures.hpp:136
GpuVector(GpuVector< T > &&other)
Allow for move-construction.
Definition tsgAcceleratedDataStructures.hpp:103
T * data()
Get a reference to the GPU array, which an be used as input to GPU libraries and kernels.
Definition tsgAcceleratedDataStructures.hpp:148
TypeOneDRule
Used to specify the one dimensional family of rules that induces the sparse grid.
Definition tsgEnumerates.hpp:288
constexpr TypeAcceleration accel_gpu_rocblas
At the front API, the HIP and CUDA options are equivalent, see TasGrid::TypeAcceleration.
Definition tsgEnumerates.hpp:578
constexpr TypeAcceleration accel_gpu_hip
At the front API, the HIP and CUDA options are equivalent, see TasGrid::TypeAcceleration.
Definition tsgEnumerates.hpp:573
TypeAcceleration
Modes of acceleration.
Definition tsgEnumerates.hpp:554
@ accel_cpu_blas
Default (if available), uses both BLAS and LAPACK libraries.
Definition tsgEnumerates.hpp:558
@ accel_none
Usually the slowest mode, uses only OpenMP multi-threading, but optimized for memory and could be the...
Definition tsgEnumerates.hpp:556
@ accel_gpu_magma
Same the CUDA option but uses the UTK MAGMA library for the linear algebra operations.
Definition tsgEnumerates.hpp:566
@ accel_gpu_cublas
Mixed usage of the CPU (OpenMP) and GPU libraries.
Definition tsgEnumerates.hpp:562
@ accel_gpu_cuda
Similar to the cuBLAS option but also uses a set of Tasmanian custom GPU kernels.
Definition tsgEnumerates.hpp:564
size_t size_mult(IntA a, IntB b)
Converts two integer-like variables to size_t and returns the product..
Definition tsgUtils.hpp:82
Encapsulates the Tasmanian Sparse Grid module.
Definition TasmanianSparseGrid.hpp:68
Wrapper class around GPU device ID, acceleration type and GpuEngine.
Definition tsgAcceleratedDataStructures.hpp:609
ChangeType testEnable(TypeAcceleration acc, int new_gpu_id) const
Returns the ChangeType if enable() is called, but does not change the acceleration.
Definition tsgAcceleratedDataStructures.hpp:714
AlgorithmPreference algorithm_select
The preference to use dense or sparse algorithms.
Definition tsgAcceleratedDataStructures.hpp:651
void enable(TypeAcceleration acc, int new_gpu_id)
Accepts parameters directly from TasmanianSparseGrid::enableAcceleration()
Definition tsgAcceleratedDataStructures.hpp:734
AccelerationContext(AccelerationContext &&)=default
Can move the context.
static constexpr int getDefaultAccDevice()
Returns the default acceleration device, CUDA/HIP use GPU 0, SYCL uses -1 which uses sycl::default_se...
Definition tsgAcceleratedDataStructures.hpp:667
ChangeType favorSparse(bool favor)
Sets algorithm affinity in the direction of sparse.
Definition tsgAcceleratedDataStructures.hpp:679
TypeAcceleration mode
The current active acceleration mode.
Definition tsgAcceleratedDataStructures.hpp:649
bool on_gpu() const
Returns true if any of the GPU-based acceleration modes have been enabled.
Definition tsgAcceleratedDataStructures.hpp:763
AlgorithmPreference
Defines the sparse-dense algorithm flavors, whenever applicable.
Definition tsgAcceleratedDataStructures.hpp:620
@ algorithm_dense
Use dense algorithm.
Definition tsgAcceleratedDataStructures.hpp:622
@ algorithm_sparse
Use sparse algorithm.
Definition tsgAcceleratedDataStructures.hpp:624
@ algorithm_autoselect
Use automatically select based on heuristics.
Definition tsgAcceleratedDataStructures.hpp:626
bool blasCompatible() const
Returns true if BLAS is enabled and the current mode is not none.
Definition tsgAcceleratedDataStructures.hpp:696
static constexpr TypeAcceleration getDefaultAccMode()
Returns the default acceleration mode, cpu_blas if BLAS is enabled and none otherwise.
Definition tsgAcceleratedDataStructures.hpp:659
int device
If using a GPU acceleration mode, holds the active device.
Definition tsgAcceleratedDataStructures.hpp:653
std::optional< GpuEngine > engine
Holds the context to the GPU TPL handles, e.g., MAGMA queue.
Definition tsgAcceleratedDataStructures.hpp:656
void setDevice() const
Set default device.
Definition tsgAcceleratedDataStructures.hpp:759
AccelerationContext & operator=(AccelerationContext const &)=delete
Cannot copy the context.
ChangeType
Defines the types of acceleration context updates so they can be linked to acceleration cache updates...
Definition tsgAcceleratedDataStructures.hpp:635
@ change_none
No change, do nothing.
Definition tsgAcceleratedDataStructures.hpp:637
@ change_sparse_dense
Change the sparse-dense AlgorithmPreference.
Definition tsgAcceleratedDataStructures.hpp:645
@ change_gpu_device
Change the associated GPU device.
Definition tsgAcceleratedDataStructures.hpp:639
@ change_cpu_blas
Change BLAS to none or none to BLAS.
Definition tsgAcceleratedDataStructures.hpp:643
@ change_gpu_enabled
Change from BLAS or none to a GPU acceleration mode.
Definition tsgAcceleratedDataStructures.hpp:641
AccelerationContext()
Creates a default context, the device id is set to 0 and acceleration is BLAS (if available) or none.
Definition tsgAcceleratedDataStructures.hpp:676
bool useKernels() const
Returns true if the current mode implies the use of custom GPU kernels.
Definition tsgAcceleratedDataStructures.hpp:705
AccelerationContext(AccelerationContext const &)=delete
Cannot copy the context.
Wrapper class around calls GPU accelerated linear algebra libraries.
Definition tsgAcceleratedDataStructures.hpp:238
GpuEngine(GpuEngine const &)=delete
Do not copy, even if no GPU backend is enabled.
GpuEngine()=default
Default construction, does not use any GPU acceleration, will set handles on-the-fly.
GpuEngine(GpuEngine &&)=default
Moving the engine is allowed, unique pointers will be moved.
void setCuSolverDnHandle(void *handle)
Manually sets the cuSparse handle, handle must be a valid cusolverDnHandle_t associated with this CUD...
GpuEngine & operator=(GpuEngine const &)=delete
Do not copy, even if no GPU backend is enabled.
std::unique_ptr< int, HandleDeleter< AccHandle::Cusolver > > cusolver_handle
Holds the cuSolver handle.
Definition tsgAcceleratedDataStructures.hpp:263
void setCuBlasHandle(void *handle)
Manually sets the cuBlas handle, handle must be a valid cublasHandle_t associated with this CUDA devi...
void setCuSparseHandle(void *handle)
Manually sets the cuSparse handle, handle must be a valid cusparseHandle_t associated with this CUDA ...
bool called_magma_init
Avoids an empty engine when no acceleration is enabled, allows for default constructor/move/copy,...
Definition tsgAcceleratedDataStructures.hpp:286
std::unique_ptr< int, HandleDeleter< AccHandle::Cublas > > cublas_handle
Holds the cuBlas handle.
Definition tsgAcceleratedDataStructures.hpp:259
std::unique_ptr< int, HandleDeleter< AccHandle::Cusparse > > cusparse_handle
Holds the cuSparse handle.
Definition tsgAcceleratedDataStructures.hpp:261