tsgMPIScatterGrid.hpp

Go to the documentation of this file.

/*
 * Copyright (c) 2017, Miroslav Stoyanov
 *
 * This file is part of
 * Toolkit for Adaptive Stochastic Modeling And Non-Intrusive ApproximatioN: TASMANIAN
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions
 *    and the following disclaimer in the documentation and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse
 *    or promote products derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * UT-BATTELLE, LLC AND THE UNITED STATES GOVERNMENT MAKE NO REPRESENTATIONS AND DISCLAIM ALL WARRANTIES, BOTH EXPRESSED AND IMPLIED.
 * THERE ARE NO EXPRESS OR IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY PATENT,
 * COPYRIGHT, TRADEMARK, OR OTHER PROPRIETARY RIGHTS, OR THAT THE SOFTWARE WILL ACCOMPLISH THE INTENDED RESULTS OR THAT THE SOFTWARE OR ITS USE WILL NOT RESULT IN INJURY OR DAMAGE.
 * THE USER ASSUMES RESPONSIBILITY FOR ALL LIABILITIES, PENALTIES, FINES, CLAIMS, CAUSES OF ACTION, AND COSTS AND EXPENSES, CAUSED BY, RESULTING FROM OR ARISING OUT OF,
 * IN WHOLE OR IN PART THE USE, STORAGE OR DISPOSAL OF THE SOFTWARE.
 */
 
#ifndef __TASMANIAN_ADDONS_MPIGRIDSCATTER_HPP
#define __TASMANIAN_ADDONS_MPIGRIDSCATTER_HPP
 
#include "tsgAddonsCommon.hpp"
 
#ifdef Tasmanian_ENABLE_MPI
 
namespace TasGrid{
 
class VectorToStreamBuffer : public std::basic_streambuf<char, std::char_traits<char>>{
public:
    VectorToStreamBuffer(std::vector<char> &data){
        setg(&*data.begin(), data.data(), &*data.end());
    }
};
 
inline int getMPIRank(MPI_Comm comm){ int rank; MPI_Comm_rank(comm, &rank); return rank; }
 
template<bool binary = TasGrid::mode_binary>
int MPIGridSend(TasmanianSparseGrid const &grid, int destination, int tag, MPI_Comm comm){
    std::stringstream ss;
    grid.write(ss, binary);
    while(ss.str().size() % 16 != 0) ss << " ";
    return MPI_Send(ss.str().c_str(), (int) (ss.str().size() / 16), MPI_LONG_DOUBLE, destination, tag, comm);
}
 
template<bool binary = TasGrid::mode_binary>
int MPIGridRecv(TasmanianSparseGrid &grid, int source, int tag, MPI_Comm comm, MPI_Status *status = MPI_STATUS_IGNORE){
    MPI_Status internal_status;
    if (status == MPI_STATUS_IGNORE) status = &internal_status;
 
    int short_data_size;
    MPI_Probe(source, tag, comm, status);
    MPI_Get_count(status, MPI_LONG_DOUBLE, &short_data_size);
 
    size_t data_size = Utils::size_mult(short_data_size, 16);
 
    std::vector<char> buff(data_size);
    auto result = MPI_Recv(buff.data(), (int) (data_size / 16), MPI_LONG_DOUBLE, source, tag, comm, status);
 
    VectorToStreamBuffer data_buffer(buff); // do not modify buff after this point
    std::istream is(&data_buffer);
    grid.read(is, binary);
    return result;
}
 
template<bool binary = TasGrid::mode_binary>
int MPIGridBcast(TasmanianSparseGrid &grid, int root, MPI_Comm comm){
    int me = getMPIRank(comm); // my rank within the comm
    if (me == root){ // sends the grid
        std::stringstream ss;
        grid.write(ss, binary);
 
        while(ss.str().size() % 16 != 0) ss << " "; // pad with empty chars to align to 16 bytes, i.e., long double
 
        unsigned long long data_size = (unsigned long long) ss.str().size();
        auto result = MPI_Bcast(&data_size, 1, MPI_UNSIGNED_LONG_LONG, me, comm);
        if (result != MPI_SUCCESS) return result;
 
        return MPI_Bcast(const_cast<char*>(ss.str().c_str()), (int) (data_size / 16), MPI_LONG_DOUBLE, me, comm); // Bcast root does not modify the buffer, this is const-correct
    }else{ // receives the grid
        unsigned long long data_size;
 
        auto result = MPI_Bcast(&data_size, 1, MPI_UNSIGNED_LONG_LONG, root, comm);
        if (result != MPI_SUCCESS) return result;
 
        std::vector<char> buff((size_t) data_size);
        result = MPI_Bcast(buff.data(), (int) (buff.size() / 16), MPI_LONG_DOUBLE, root, comm);
 
        VectorToStreamBuffer data_buffer(buff); // do not modify buff after this point
        std::istream is(&data_buffer);
        grid.read(is, binary);
        return result;
    }
}
 
template<bool binary = TasGrid::mode_binary>
int MPIGridScatterOutputs(TasmanianSparseGrid const &source, TasmanianSparseGrid &destination, int root, int tag, MPI_Comm comm){
    int me = getMPIRank(comm); // my rank within the comm
 
    if (me == root){ // splitting and sending the grid
        int num_ranks; MPI_Comm_size(comm, &num_ranks);
        int num_effective_ranks = std::min(num_ranks, source.getNumOutputs());
 
        int stride = source.getNumOutputs() / num_effective_ranks;
        int extras = source.getNumOutputs() % num_effective_ranks;
 
        // return the starting offset for the given rank
        auto offset = [&](int rank)->int{ return rank * stride + std::min(rank, extras); };
 
        for(int rank=0; rank<num_effective_ranks; rank++){
            if (rank == root){ // this is me, take my own copy of the grid
                destination = copyGrid(source, offset(rank), offset(rank+1));
            }else{ // send the grid out
                auto result = MPIGridSend<binary>(copyGrid(source, offset(rank), offset(rank+1)) , rank, tag, comm);
                if (result != MPI_SUCCESS) return result;
            }
        }
        for(int rank=num_effective_ranks; rank<num_ranks; rank++){ // if there are any grid remaining, set those to empty
            if (rank == root){ // this is me, take my own copy of the grid
                destination = makeEmpty();
            }else{ // send the grid out
                auto result = MPIGridSend<binary>(makeEmpty() , rank, tag, comm);
                if (result != MPI_SUCCESS) return result;
            }
        }
        return MPI_SUCCESS; // if we got here, all was successful
    }else{ // receiving a grid
        return MPIGridRecv<binary>(destination, root, tag, comm);
    }
}
 
}
 
#endif // Tasmanian_ENABLE_MPI
 
#endif