amroc/hds/Timing.h

Go to the documentation of this file.

// -*- C++ -*-

// Copyright (C) 2002 Ralf Deiterding
// Brandenburgische Universitaet Cottbus
//
// Copyright (C) 2003-2007 California Institute of Technology
// Ralf Deiterding, ralf@cacr.caltech.edu

#ifndef AMROC_TIMING_H
#define AMROC_TIMING_H

#include <iostream>
#include <cstdio>
#include <vector>

#ifdef DAGH_NO_MPI

#include <sys/time.h>
#include <time.h>
inline static double MPI_Wtime() { 
  struct timeval t; 
  gettimeofday(&t,0); 
  return (double)t.tv_sec + ((double)t.tv_usec)*0.000001; 
}

#endif

#include "CommServer.h"

class Timing;
extern Timing timing;

#ifdef TIMING_AMR

#define START_WATCH                         timing.start();
#define END_WATCH(what)                     timing.stop(Timing::what);

#define END_WATCH_INITIALIZATION            END_WATCH(INITIALIZATION)
#define END_WATCH_FLAGGING                  END_WATCH(FLAGGING)
#define END_WATCH_COMPOSING_WHOLE           END_WATCH(COMPOSING_WHOLE)
#define END_WATCH_RECOMPOSING_WHOLE         END_WATCH(RECOMPOSING_WHOLE)
#define END_WATCH_RECOMPOSING_INTERPOLATION END_WATCH(RECOMPOSING_INTERPOLATION)
#define END_WATCH_BOUNDARIES_SYNC           END_WATCH(BOUNDARIES_SYNC)
#define END_WATCH_BOUNDARIES_INTERPOLATION  END_WATCH(BOUNDARIES_INTERPOLATION)
#define END_WATCH_BOUNDARIES_EXTERNAL       END_WATCH(BOUNDARIES_EXTERNAL)
#define END_WATCH_INTEGRATION(which)        timing.stop((which==1)?Timing::INTEGRATION_ESTIMATE:(which==2)?Timing::INTEGRATION_SHADOW:Timing::INTEGRATION_MAIN);
#define END_WATCH_SOURCE_INTEGRATION        END_WATCH(SOURCE_INTEGRATION)
#define END_WATCH_FIXUP_WHOLE               END_WATCH(FIXUP_WHOLE)
#define END_WATCH_FIXUP_SYNC                END_WATCH(FIXUP_SYNC)
#define END_WATCH_CLUSTERING                END_WATCH(CLUSTERING)
#define END_WATCH_OUPUT                     END_WATCH(OUTPUT)
#define END_WATCH_EXAMINE                   END_WATCH(EXAMINE1)

#define START_WATCH_WHOLE                   timing.start_timing();
#define END_WATCH_WHOLE                     timing.stop(Timing::MISC);

#define COUNT(what)                         timing.count(Timing::what);
#define COUNT_FREQ(what,freq)               timing.count(Timing::what,freq); 
#else

#define START_WATCH_WHOLE            
#define START_WATCH                 
#define END_WATCH(what)

#define END_WATCH_INITIALIZATION     
#define END_WATCH_FLAGGING           
#define END_WATCH_RECOMPOSING_WHOLE       
#define END_WATCH_RECOMPOSING_INTERPOLATION     
#define END_WATCH_BOUNDARIES_SYNC          
#define END_WATCH_BOUNDARIES_INTERPOLATION         
#define END_WATCH_BOUNDARIES_EXTERNAL       
#define END_WATCH_INTEGRATION(which)     
#define END_WATCH_SOURCE_INTEGRATION 
#define END_WATCH_FIXUP_WHOLE              
#define END_WATCH_FIXUP_SYNC               
#define END_WATCH_CLUSTERING        
#define END_WATCH_OUPUT              
#define END_WATCH_EXAMINE                   
#define END_WATCH_WHOLE              
#define COUNT(what)
#define COUNT_FREQ(what,freq)

#endif

class Timing {
public:
  enum TimingAccounts { 
    WHOLE, 
    
    MISC,

    /* user accounts */
    INITIALIZATION, FLAGGING, CLUSTERING, GATHERCFL, OUTPUT,
    EXAMINE1, EXAMINE2, EXAMINE3, EXAMINE4,

    PARTITION_INIT, PARTITION_CALC,

    COMPOSING_WHOLE, COMPOSING_GLOBALLISTS_MERGE, COMPOSING_GDBCREATE, COMPOSING_COMMSERVERS, 

    RECOMPOSING_WHOLE, RECOMPOSING_INTERPOLATION, RECOMPOSING_DATASYNC, 
    RECOMPOSING_BOXARRAYCOMM, RECOMPOSING_GRIDBOXCOMM, RECOMPOSING_COMMSERVERS,  
    RECOMPOSING_GLOBALLISTS_NESTING, RECOMPOSING_GLOBALLISTS_REFINE, RECOMPOSING_GLOBALLISTS_MERGE,
    RECOMPOSING_GLOBALLISTS_GFLISTS, RECOMPOSING_GDBCREATE,

    MEMORY_RESTART_CHECKPOINT, MEMORY_RESTART_WHOLE, MEMORY_RESTART_GDBCREATE, MEMORY_RESTART_COMMSERVERS,

    RESTART_CHECKPOINT, RESTART_WHOLE, RESTART_GDBCREATE, RESTART_COMMSERVERS,
    
    BOUNDARIES_SYNC, BOUNDARIES_INTERPOLATION, BOUNDARIES_EXTERNAL, 
    
    INTEGRATION_MAIN, INTEGRATION_ESTIMATE, 
    INTEGRATION_SHADOW, SOURCE_INTEGRATION, 

    EX_INIT, EX_SSEND, EX_SRECV,
    GFSYNC_GETINFO, GFSYNC_CHECKREADY,
    GFSYNC_IWAIT, GFSYNC_ITEST, GFSYNC_OWAIT, GFSYNC_OTEST,

    GFSYNC_READGHOSTS, GFSYNC_READGHOSTS_SELF, GFSYNC_WRITEGHOSTS,
    GFSYNC_READDATA, GFSYNC_WRITEDATA,

    UPDATE_PATCH,

    FIXUP_WHOLE, FIXUP_CORRECTION, FIXUP_SYNC,

    GFM_FINDING_CELLS, GFM_GEOMETRY, GFM_TRANSFORM, 
    GFM_EXTRAPOLATION, GFM_SETBNDRY,
    GFM_AUXILIARY_VALUES, GFM_SETBNDRY_WHOLE, 

    LS_SET_WHOLE, LS_SYNC, LS_CPT_TRANSFORM, LS_CPT_FLOODFILL, 

    ELC_RECEIVEBOUNDARY, ELC_SENDPRESSURE,
    
    FLUID_CPL_RECEIVE_OVERHEAD, FLUID_CPL_SEND_OVERHEAD, FLUID_CPL_INTERPOLATE,
    FLUID_CPL_VELOCITY_SEARCH, FLUID_CPL_PRESSURE_CALCULATE, FLUID_CPL_ELC_GEOMETRY,

    /* maker for internal use */
    _ACMAX };

  enum CountingAccounts { 
    GF_SYNC, GF_PATCHITERATOR,

    CS_TEST, CS_WAIT,

    /* maker for internal use */
    _CCMAX };

public:
  /***************************************************************************/
  /* counting functions */

  inline void count(enum CountingAccounts account, int i=1)
    { counts[account] += i; }

  /***************************************************************************/
  /* timing functions */

  inline void start_timing() 
  { tos = &timestack[0]; ntos = 0; *tos = 0.0; start_time = MPI_Wtime(); }

  inline void start() 
  { add_time(); tos++; ntos++; assert (ntos<MAXRECURSIONS); *tos = 0.0; }

  inline void stop(enum TimingAccounts account) 
  { add_time(); times[account] += *tos; calls[account]++; tos--; ntos--; assert (ntos>=-1); }

protected:
  inline void add_time() {
    double end_time = MPI_Wtime();
    *tos += end_time - start_time; start_time = end_time;
  }

public:
  /***************************************************************************/
  /* global report generation interface */

  static void collect(MPI_Comm Comm) { 
#ifdef TIMING_AMR
    timing.collect_timing(Comm);
#endif
  }

  static void print(std::ostream& os) { 
#ifdef TIMING_AMR
    timing.print_local_times = false;
    timing.print_timing(os);
#endif
  }

  static void print_local(std::ostream& os) { 
#ifdef TIMING_AMR
    timing.print_local_times = true;
    timing.print_timing(os);
#endif
  }

  void collect_timing(MPI_Comm Comm, enum TimingAccounts ac, 
                      double* stat_times) {
#ifdef DAGH_NO_MPI
    stat_times[0] = times[ac];
    stat_times[1] = times[ac];
    stat_times[2] = times[ac];
#else
    MPI_Reduce(&times[ac], &stat_times[0], 1, MPI_DOUBLE, 
               MPI_MAX, 0, Comm);
    MPI_Reduce(&times[ac], &stat_times[1], 1, MPI_DOUBLE, 
               MPI_MIN, 0, Comm);
    MPI_Reduce(&times[ac], &stat_times[2], 1, MPI_DOUBLE, 
               MPI_SUM, 0, Comm);
 
    int procs;
    MPI_Comm_size(Comm, &procs);
    stat_times[2] /= procs;
#endif
  }

  void collect_timing(MPI_Comm Comm) {
    /* calculate sums */
    times[WHOLE] = calcsum(times, MISC, _ACMAX-1, 0.0);
    calls[WHOLE] = calcsum(calls, MISC, _ACMAX-1, (unsigned)0);

    /* collect values from all nodes */
    stat_times_max.resize(_ACMAX);
    stat_times_min.resize(_ACMAX);
    stat_times_avg.resize(_ACMAX);
#ifdef DAGH_NO_MPI
    stat_times_max = times;
    stat_times_min = times;
    stat_times_avg = times;
#else
    MPI_Reduce(&times[0], &stat_times_max[0], _ACMAX, MPI_DOUBLE, 
               MPI_MAX, 0, Comm);
    MPI_Reduce(&times[0], &stat_times_min[0], _ACMAX, MPI_DOUBLE, 
               MPI_MIN, 0, Comm);
    MPI_Reduce(&times[0], &stat_times_avg[0], _ACMAX, MPI_DOUBLE, 
               MPI_SUM, 0, Comm);
 
    int procs;
    MPI_Comm_size(Comm, &procs);
    for (int i=0; i<_ACMAX; i++)
      stat_times_avg[i] /= procs;
#endif
  }

  void print_timing(std::ostream& os) {
    cl(os,"CommServer - MPI_Waitsome    ", CS_WAIT);
    cl(os,"             MPI_Testsome    ", CS_TEST,true);
    cl(os,"GridFunction - Sync          ", GF_SYNC);
    cl(os,"               PatchIterator ", GF_PATCHITERATOR,true);
    pl(os,"Initialization               ", INITIALIZATION);
    pl(os,"Gather max. CFL in parallel  ", GATHERCFL,true);
    pl(os,"Integration -   Main         ", INTEGRATION_MAIN); 
    pl(os,"                Estimation   ", INTEGRATION_ESTIMATE,true);
    pl(os,"                Coarsen      ", INTEGRATION_SHADOW,true);
    pl(os,"                Source       ", SOURCE_INTEGRATION,true);
    pl(os,"Fixup -         Overhead     ", FIXUP_WHOLE,true);
    pl(os,"                Correction   ", FIXUP_CORRECTION,true);
    pl(os,"                Syncing      ", FIXUP_SYNC,true);
    pl(os,"Boundary Val. - Physical     ", BOUNDARIES_EXTERNAL);
    pl(os,"                Syncing      ", BOUNDARIES_SYNC,true);    
    pl(os,"                Interpolation", BOUNDARIES_INTERPOLATION,true);
    pl(os,"Regridding -    Flagging     ", FLAGGING,true);
    pl(os,"                Clustering   ", CLUSTERING,true);
    pl(os,"Partition -     Init         ", PARTITION_INIT,true);
    pl(os,"                Calculate    ", PARTITION_CALC,true);
    pl(os,"Composition - Overhead       ", COMPOSING_WHOLE,true); 
    pl(os,"      GlobalGridBoxList merge", COMPOSING_GLOBALLISTS_MERGE,true);
    pl(os,"         GridDataBlock create", COMPOSING_GDBCREATE,true);
    pl(os,"       GF CommServer creation", COMPOSING_COMMSERVERS,true);
    pl(os,"Recomposition - Overhead     ", RECOMPOSING_WHOLE,true); 
    pl(os,"                Interpolation", RECOMPOSING_INTERPOLATION,true);
    pl(os,"                Data syncing ", RECOMPOSING_DATASYNC,true);
    pl(os,"         GlobalBBoxList comm.", RECOMPOSING_BOXARRAYCOMM,true);
    pl(os,"      GlobalGridBoxList comm.", RECOMPOSING_GRIDBOXCOMM,true);
    pl(os,"       GlobalBBoxList nesting", RECOMPOSING_GLOBALLISTS_NESTING,true);
    pl(os,"       GlobalBBoxList refine ", RECOMPOSING_GLOBALLISTS_REFINE,true);
    pl(os,"      GlobalGridBoxList merge", RECOMPOSING_GLOBALLISTS_MERGE,true);
    pl(os,"       GF GridBoxLists create", RECOMPOSING_GLOBALLISTS_GFLISTS,true);
    pl(os,"         GridDataBlock create", RECOMPOSING_GDBCREATE,true);
    pl(os,"       GF CommServer creation", RECOMPOSING_COMMSERVERS,true);
    pl(os,"MemoryRestart - Checkpointing", MEMORY_RESTART_CHECKPOINT,true);   
    pl(os,"MemoryRestart - Overhead     ", MEMORY_RESTART_WHOLE,true); 
    pl(os,"         GridDataBlock create", MEMORY_RESTART_GDBCREATE,true);
    pl(os,"       GF CommServer creation", MEMORY_RESTART_COMMSERVERS,true);
    pl(os,"Restart - Checkpointing      ", RESTART_CHECKPOINT,true);   
    pl(os,"Restart - Overhead           ", RESTART_WHOLE,true); 
    pl(os,"         GridDataBlock create", RESTART_GDBCREATE,true);
    pl(os,"       GF CommServer creation", RESTART_COMMSERVERS,true);
    pl(os,"ExchangeServ. - Init         ", EX_INIT,true);
    pl(os,"                start sends  ", EX_SSEND,true);
    pl(os,"                start recvs  ", EX_SRECV,true);
    pl(os,"GFSync - gather depend. info ", GFSYNC_GETINFO,true);
    pl(os,"         check dependencies  ", GFSYNC_CHECKREADY,true);
    pl(os,"      instant wait for msg   ", GFSYNC_IWAIT,true);
    pl(os,"      instant test for msg   ", GFSYNC_ITEST,true);
    pl(os,"   overlapped wait for msg   ", GFSYNC_OWAIT,true);
    pl(os,"   overlapped test for msg   ", GFSYNC_OTEST,true);
    pl(os,"         read ghosts         ", GFSYNC_READGHOSTS,true);
    pl(os,"         read ghosts self    ", GFSYNC_READGHOSTS_SELF,true);
    pl(os,"         write ghosts        ", GFSYNC_WRITEGHOSTS,true);
    pl(os,"         read data           ", GFSYNC_READDATA,true);
    pl(os,"   AMRSolver update patch    ", UPDATE_PATCH,true);
    pl(os,"Ghost-fluid values - Overhead", GFM_SETBNDRY_WHOLE,true);
    pl(os,"    Finding internal cells   ", GFM_FINDING_CELLS,true);
    pl(os,"    Calculating geometry info", GFM_GEOMETRY,true);
    pl(os,"    Calc. extrapolated values", GFM_TRANSFORM,true);
    pl(os,"    Extra- / Interpolation   ", GFM_EXTRAPOLATION,true);
    pl(os,"    Setting auxiliary values ", GFM_AUXILIARY_VALUES,true);
    pl(os,"    Applying bndry values    ", GFM_SETBNDRY,true); 
    pl(os,"Level sets - Overhead        ", LS_SET_WHOLE,true);
    pl(os,"    CPT - point_transform()  ", LS_CPT_TRANSFORM,true);
    pl(os,"    CPT - flood_fill()       ", LS_CPT_FLOODFILL,true);
    pl(os,"    Sync level set functions ", LS_SYNC,true);
    pl(os,"ELC - receive_mesh()         ", ELC_RECEIVEBOUNDARY,true);
    pl(os,"      send_pressure()        ", ELC_SENDPRESSURE,true);
    pl(os,"Coupling Send - Overhead     ", FLUID_CPL_SEND_OVERHEAD,true);
    pl(os,"    pressure calculation()   ", FLUID_CPL_PRESSURE_CALCULATE,true);
    pl(os,"    pressure interpolation() ", FLUID_CPL_INTERPOLATE,true);
    pl(os,"    ELC: face normal&centers ", FLUID_CPL_ELC_GEOMETRY,true);
    pl(os,"Coupling Receive - Overhead  ", FLUID_CPL_RECEIVE_OVERHEAD,true);
    pl(os,"    velocity search&constr.  ", FLUID_CPL_VELOCITY_SEARCH,true);
    pl(os,"Output                       ", OUTPUT,true); 
    pl(os,"Examined in detail 1         ", EXAMINE1,true);
    pl(os,"Examined in detail 2         ", EXAMINE2,true);
    pl(os,"Examined in detail 3         ", EXAMINE3,true);
    pl(os,"Examined in detail 4         ", EXAMINE4,true);
    pl(os,"Misc                         ", MISC);
    pl(os,"Whole time                   ", WHOLE);
  }

protected:
  template<typename T>
  inline T calcsum(typename std::vector<T> &data, 
                   int start, int end, T sum) {
    for (int i=start; i<=end; i++) sum += data[i];
    return sum;
  }

  inline void pl(std::ostream& OS, const char *name,
                 enum TimingAccounts ac, bool opt=false) {
    char str[500];
    if (print_local_times) {
      if (opt && times[ac]==0.) return;
      std::sprintf(str, "   %s : %4.3fs (%3.2f%%)   calls: %4ix", 
                   name,
                   times[ac], (times[WHOLE]>0. ? 100.0*times[ac]/times[WHOLE] : 0.),
                   calls[ac]     
                   );
    } else {
      if (opt && stat_times_avg[ac]==0.) return;
      std::sprintf(str, "   %s : %4.3fs (%3.2f%%)   balance: %2.3f   diff: %4.2fs (%9.2fs - %4.2fs)", 
                   name,
                   stat_times_avg[ac], (stat_times_avg[WHOLE]>0. ? 
                                        100.0*stat_times_avg[ac]/stat_times_avg[WHOLE] : 0.),
                   (stat_times_avg[ac]>0. ? stat_times_max[ac] / stat_times_avg[ac] : 1.),
                   stat_times_max[ac] - stat_times_min[ac],
                   stat_times_max[ac], stat_times_min[ac]
                   );
    }
    OS << str << std::endl;
  }

  inline void cl(std::ostream& OS, const char *name, enum CountingAccounts ac, bool opt=false) {
    if (opt && counts[ac]==0) return;
    OS << "   " << name << " : " << counts[ac] << "x" 
       << std::endl;
  }

public:
  Timing() { 
    times.resize(_ACMAX, 0.0);
    calls.resize(_ACMAX, 0);
    counts.resize(_CCMAX, 0);
    timestack.resize(MAXRECURSIONS);
  }

protected:
  std::vector<double> times; // [_ACMAX]
  std::vector<unsigned int> calls; // [_ACMAX]
  std::vector<int> counts; // [_CCMAX]
  
  enum {MAXRECURSIONS=300};
  std::vector<double> timestack; // [MAXRECURSIONS]
  double *tos;
  int ntos;

  bool print_local_times;
  std::vector<double> stat_times_max; // [_ACMAX]
  std::vector<double> stat_times_min; // [_ACMAX]
  std::vector<double> stat_times_avg; // [_ACMAX]

  double start_time;
};

#endif

---