shells/parallel/ZoltanIface.h

Go to the documentation of this file.

// -*- C++ -*- 
//
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
//                                   Fehmi Cirak
//                        California Institute of Technology
//                           (C) 2004 All Rights Reserved
//
// <LicenseText>
//
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
#ifndef ZOLTANIFACE_H
#define ZOLTANIFACE_H
#include <set>
#include <vector>
#include <algorithm>

#include "zoltan.h"
#include "mpi.h"



// wrapper struct for passing the stl vectors through zoltan
namespace parallel {
    typedef struct PointSetContS {
        std::vector<double>   _coordinate;
        std::vector<int>      _globalID;
    } PointSetCont;
}    


// function declarations
namespace parallel{
    // initialize zoltan
    struct Zoltan_Struct* zoltanIfaceInit(const MPI_Comm& comm);

    // main interface for zoltan
    template<typename Iterator>
    bool zoltanIfacePartition (struct Zoltan_Struct *lb, 
                               PointSetCont *pointSet, 
                               Iterator insElements);

    // delete zoltan
    void zoltanFinalize(struct Zoltan_Struct *lb);

  extern "C" {    
    // call back functions
    int callBackNumElems(void *data, int *ierr);    
    void callBackElemList(void *data, int num_gid_entries, int num_lid_entries,
                          ZOLTAN_ID_PTR global_id, ZOLTAN_ID_PTR local_id, 
                          int wgt_dim, float *obj_wgts, int *ierr);
    void callBackCoords(void *data, int num_gid_entries, int num_lid_entries, 
                        ZOLTAN_ID_PTR global_id,ZOLTAN_ID_PTR local_id,
                        double *geom_vec, int *ierr);
    int callBackDimension(void *data, int *ierr);
    // callback function pointer type
    typedef void (*zoltan_cft)();
  }
}



// templated functions
template<typename Iterator>
bool parallel::zoltanIfacePartition (struct Zoltan_Struct *lb, 
                                     PointSetCont *pointSet, 
                                     Iterator insElements) 
{       
    // register call back functions
    Zoltan_Set_Fn(lb, ZOLTAN_NUM_GEOM_FN_TYPE, 
                  reinterpret_cast<zoltan_cft>(callBackDimension), pointSet);
    Zoltan_Set_Fn(lb, ZOLTAN_GEOM_FN_TYPE, 
                  reinterpret_cast<zoltan_cft>(callBackCoords), pointSet);  
    Zoltan_Set_Fn(lb, ZOLTAN_NUM_OBJ_FN_TYPE,  
                  reinterpret_cast<zoltan_cft>(callBackNumElems), pointSet); 
    Zoltan_Set_Fn(lb, ZOLTAN_OBJ_LIST_FN_TYPE,                     
                  reinterpret_cast<zoltan_cft>(callBackElemList), pointSet); 
    
    // call the load balancer 
    ZOLTAN_ID_PTR imp_global_ids, exp_global_ids, imp_local_ids, exp_local_ids;
    int changes, num_imp, num_exp, *imp_procs, *exp_procs;
    int num_gid_entries, num_lid_entries;
    Zoltan_LB_Balance(lb, &changes, &num_gid_entries, &num_lid_entries, 
                      &num_imp, &imp_global_ids, &imp_local_ids, &imp_procs,  
                      &num_exp, &exp_global_ids, &exp_local_ids, &exp_procs); 
    
    // check if zoltan has changed something
    if (!changes) {
        std::copy(pointSet->_globalID.begin(), pointSet->_globalID.end(), 
                  insElements);    
        return false;
    }    

    // do some set operations to find the elements on this processor
    // note that set operations work only on sorted ranges
    std::vector<int> current(pointSet->_globalID);
    std::sort(current.begin(), current.end());
    
    // add the global element ID's on this processor to insElements iterator
    std::vector<int> exported(exp_global_ids, exp_global_ids+num_exp);
    std::sort(exported.begin(), exported.end());
    
    std::set<int> tmpSet;
    std::insert_iterator<std::set<int> > insTmpSet(tmpSet, tmpSet.begin());
    std::set_difference(current.begin(), current.end(), 
                        exported.begin(), exported.end(), insTmpSet );
    
    std::vector<int> imported(imp_global_ids, imp_global_ids+num_imp);
    std::sort(imported.begin(), imported.end());
    
    std::set_union(tmpSet.begin(), tmpSet.end(), 
                   imported.begin(), imported.end(), insElements);
    
    return true;
}

#endif

---