pico/pico/DomainCoupler.h

Go to the documentation of this file.

// -*- C++ -*-
//
//  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
//                                   Fehmi Cirak
//                        California Institute of Technology
//                           (C) 2003 All Rights Reserved
//
//  <LicenseText>
//
//  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//

#ifndef _DomainCoupler_h_
#define _DomainCoupler_h_

#include <mpi.h>

#include "PicoNames.h"
#include "SubDomain.h"

#include <map>
#include <cassert>


namespace pico {
    class DomainCoupler;
}


class pico::DomainCoupler{
    //
    // Each rank of the MPI communicator _pComm owns upto two types subdomains (0/1).
    //
    // Communication between subdomains of type 0 and 1 is called inter communication. 
    // Communication between subdomains of only  type 0 or only type 1 is called  
    // intra communication. 
    //
    
public: 
    DomainCoupler(MPI_Comm pComm):_pComm(pComm){assert(_pComm != MPI_COMM_NULL);}
    ~DomainCoupler(){}

    // registration
    void registerSubDomain(const pico::SolvType& type, const double lo[3], 
                           const double up[3], const double& maxElemSize);
    void registerSubDomain(const pico::SolvType& type, const double * const coor, 
                           const int& nodes, const double& maxElemSize);
   
    // communication
    void prepareCommunicationsIntra(const pico::SolvType& type);
    void exchangeSubdomainDataIntra(const pico::SolvType& type);    
    void prepareCommunicationsInter();
    
    // register buffers
    template <typename T>
    void registerSendBuffer(const pico::SolvType& type, T* const start, int size);

    // accessors for receive buffer
    template <typename RT>
    RT* recvBuffer(const pico::SolvType& type, const RT& notUsed);
    template <typename T>
    int recvBufferSize(const pico::SolvType& type);
    
    // helper - for debugging purposes
    template <typename Iterator> 
    void relevantSubDomainNumbers(Iterator domainNumbers, int t, char *tch);
        
// copy and equality constructor
private:
    DomainCoupler(const DomainCoupler &);
    const DomainCoupler & operator=(const DomainCoupler &);

private:
    void gatherSubDomains();
    void relevantSubDomainsInter();
    void relevantSubDomainsIntra(const pico::SolvType& type);
    void exchangeBufferSizesIntra(const pico::SolvType& type);
    
private:
    typedef std::map<int, SubDomain *>       _SubDomainCont;
    typedef _SubDomainCont::iterator         _SubDomainIt;
  
    // std::map<processor number, subdomain pointer>
    _SubDomainCont                        _subDomains[2];
    
    _SubDomainCont                        _relevSubDomainsInter[2];
    _SubDomainCont                        _relevSubDomainsIntra[2];
     
    MPI_Comm                              _pComm;
}; 



namespace pico {
    // helper - for debugging purposes
    template <typename Iterator> 
    void DomainCoupler::relevantSubDomainNumbers(Iterator domainNumbers, int t, char *tch) 
    {
        std::string commT(tch);
        _SubDomainIt it, ite;
        
        if (commT== "Inter") {
            it = _relevSubDomainsInter[t].begin();
            ite = _relevSubDomainsInter[t].end();       
        } else if (commT=="Intra") {
            it = _relevSubDomainsIntra[t].begin();
            ite = _relevSubDomainsIntra[t].end();
        } else 
            return;
        
        for ( ; it!=ite; ++it) *(domainNumbers++) = it->first;

        return;
    }
            

    template <typename T>
    void DomainCoupler::registerSendBuffer(const pico::SolvType& type, 
                                           T* const start, int size) 
    {
        int rank;
        MPI_Comm_rank(_pComm, &rank);
        
        _SubDomainIt it, ite=_subDomains[type].end();
        
        it = _subDomains[type].find(rank);
        if (it==ite) return;
        
        SubDomain* myDomain = it->second;
        
        size_t unit = sizeof(T);
        size *= unit;
        
        char *startCh = reinterpret_cast<char *>(start);

        myDomain->registerSendBuffer(startCh, size);
        
        return;
    }


    template <typename RT>
    RT* DomainCoupler::recvBuffer(const pico::SolvType& type, const RT& notUsed) 
    {
        // notUsed is only "used" by the compiler
        char *tmp=NULL;
        int rank;
        MPI_Comm_rank(_pComm, &rank);
        
        _SubDomainIt it, ite=_subDomains[type].end();
        it = _subDomains[type].find(rank);
        if (it==ite) return NULL;
        SubDomain* myDomain = it->second;
        
        tmp = myDomain->recvBuffer();
        
        RT *tmpRT = reinterpret_cast<RT *>(tmp);

        return tmpRT;
    }
    
    
    template <typename T>
    int DomainCoupler::recvBufferSize(const pico::SolvType& type)  
    {
        int tmp=0, rank;
        MPI_Comm_rank(_pComm, &rank);
        
        _SubDomainIt it, ite=_subDomains[type].end();
        it = _subDomains[type].find(rank);
        if (it==ite) return tmp;
        SubDomain* myDomain = it->second;
        
        tmp = myDomain->recvBufferSize();
        
        size_t unit = sizeof(T);
        tmp /= unit;

        return tmp;
    }
}


#endif

//  End of file

---