VTK
vtkPPixelTransfer.h
Go to the documentation of this file.
1 /*=========================================================================
2 
3  Program: Visualization Toolkit
4  Module: vtkPPixelTransfer.h
5 
6  Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
7  All rights reserved.
8  See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
9 
10  This software is distributed WITHOUT ANY WARRANTY; without even
11  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
12  PURPOSE. See the above copyright notice for more information.
13 
14 =========================================================================*/
29 #ifndef vtkPPixelTransfer_h
30 #define vtkPPixelTransfer_h
31 
32 #include "vtkPixelTransfer.h"
33 #include "vtkRenderingParallelLICModule.h" // for export
34 #include "vtkSetGet.h" // for macros
35 #include "vtkPixelExtent.h" // for pixel extent
36 #include "vtkMPI.h" // for mpi
37 #include "vtkMPIPixelTT.h" // for type traits
38 #include "vtkMPIPixelView.h" // for mpi subarrays
39 
40 // included vtkSystemIncludes.h in the base class.
41 #include <iostream> // for ostream
42 #include <vector> // for vector
43 #include <cstring> // for memcpy
44 
45 // #define vtkPPixelTransferDEBUG
46 
47 class VTKRENDERINGPARALLELLIC_EXPORT vtkPPixelTransfer : public vtkPixelTransfer
48 {
49 public:
51  :
52  SrcRank(0),
53  DestRank(0),
54  UseBlockingSend(0),
55  UseBlockingRecv(0)
56  {}
57 
63  int srcRank,
64  const vtkPixelExtent &srcWholeExt,
65  const vtkPixelExtent &srcExt,
66  int destRank,
67  const vtkPixelExtent &destWholeExt,
68  const vtkPixelExtent &destExt,
69  int id=0)
70  :
71  Id(id),
72  SrcRank(srcRank),
73  SrcWholeExt(srcWholeExt),
74  SrcExt(srcExt),
75  DestRank(destRank),
76  DestWholeExt(destWholeExt),
77  DestExt(destExt),
78  UseBlockingSend(0),
79  UseBlockingRecv(0)
80  {}
81 
87  int srcRank,
88  const vtkPixelExtent &srcWholeExt,
89  const vtkPixelExtent &targetExt,
90  int destRank,
91  const vtkPixelExtent &destWholeExt,
92  int id)
93  :
94  Id(id),
95  SrcRank(srcRank),
96  SrcWholeExt(srcWholeExt),
97  SrcExt(targetExt),
98  DestRank(destRank),
99  DestWholeExt(destWholeExt),
100  DestExt(targetExt),
101  UseBlockingSend(0),
102  UseBlockingRecv(0)
103  {}
104 
110  int srcRank,
111  int destRank,
112  const vtkPixelExtent &wholeExt,
113  const vtkPixelExtent &targetExt,
114  int id=0)
115  :
116  Id(id),
117  SrcRank(srcRank),
118  SrcWholeExt(wholeExt),
119  SrcExt(targetExt),
120  DestRank(destRank),
121  DestWholeExt(wholeExt),
122  DestExt(targetExt),
123  UseBlockingSend(0),
124  UseBlockingRecv(0)
125  {}
126 
132  int srcRank,
133  int destRank,
134  const vtkPixelExtent &ext,
135  int id=0)
136  :
137  Id(id),
138  SrcRank(srcRank),
139  SrcWholeExt(ext),
140  SrcExt(ext),
141  DestRank(destRank),
142  DestWholeExt(ext),
143  DestExt(ext),
144  UseBlockingSend(0),
145  UseBlockingRecv(0)
146  {}
147 
153  int srcRank,
154  const vtkPixelExtent &srcWholeExt,
155  int destRank,
156  const vtkPixelExtent &destWholeExt,
157  int id=0)
158  :
159  Id(id),
160  SrcRank(srcRank),
161  SrcWholeExt(srcWholeExt),
162  SrcExt(srcWholeExt),
163  DestRank(destRank),
164  DestWholeExt(destWholeExt),
165  DestExt(destWholeExt),
166  UseBlockingSend(0),
167  UseBlockingRecv(0)
168  {}
169 
176  const vtkPixelExtent &srcWholeExt,
177  const vtkPixelExtent &srcExt,
178  const vtkPixelExtent &destWholeExt,
179  const vtkPixelExtent &destExt)
180  :
181  Id(0),
182  SrcRank(0),
183  SrcWholeExt(srcWholeExt),
184  SrcExt(srcExt),
185  DestRank(0),
186  DestWholeExt(destWholeExt),
187  DestExt(destExt),
188  UseBlockingSend(0),
189  UseBlockingRecv(0)
190  {}
191 
193 
198  void SetSourceRank(int rank)
199  { this->SrcRank=rank; }
200 
201  int GetSourceRank() const
202  { return this->SrcRank; }
203 
204  void SetDestinationRank(int rank)
205  { this->DestRank=rank; }
206 
207  int GetDestinationRank() const
208  { return this->DestRank; }
209 
215  bool Sender(int rank) const { return (this->SrcRank == rank); }
216  bool Receiver(int rank) const { return (this->DestRank == rank); }
217  bool Local(int rank) const
218  { return (this->Sender(rank) && this->Receiver(rank)); }
219 
225  { this->SrcWholeExt=srcExt; }
226 
228  { return this->SrcWholeExt; }
229 
231  { return this->SrcWholeExt; }
232 
238  { this->SrcExt=srcExt; }
239 
241  { return this->SrcExt; }
242 
244  { return this->SrcExt; }
245 
251  { this->DestWholeExt=destExt; }
252 
254  { return this->DestWholeExt; }
255 
257  { return this->DestWholeExt; }
258 
264  { this->DestExt=destExt; }
265 
267  { return this->DestExt; }
268 
270  { return this->DestExt; }
271 
275  void SetTransactionId(int id)
276  { this->Id=id; }
277 
278  int GetTransactionId() const
279  { return this->Id; }
280 
284  void SetUseBlockingSend(int val)
285  { this->UseBlockingSend=val; }
286 
287  int GetUseBlockingSend() const
288  { return this->UseBlockingSend; }
289 
290  void SetUseBlockingRecv(int val)
291  { this->UseBlockingRecv=val; }
292 
293  int GetUseBlockingRecv() const
294  { return this->UseBlockingRecv; }
295 
299  template<typename SOURCE_TYPE, typename DEST_TYPE>
300  int Execute(
301  MPI_Comm comm,
302  int rank,
303  int nComps,
304  SOURCE_TYPE *srcData,
305  DEST_TYPE *destData,
306  std::vector<MPI_Request> &reqs,
307  std::deque<MPI_Datatype> &types,
308  int tag);
309 
314  int Execute(
315  MPI_Comm comm,
316  int rank,
317  int nComps,
318  int srcType,
319  void *srcData,
320  int destType,
321  void *destData,
322  std::vector<MPI_Request> &reqs,
323  std::deque<MPI_Datatype> &types,
324  int tag);
325 
329  int Blit(
330  int nComps,
331  int srcType,
332  void *srcData,
333  int destType,
334  void *destData);
335 
336 private:
337  // distpatch helper for vtk data type enum
338  template<typename SOURCE_TYPE>
339  int Execute(
340  MPI_Comm comm,
341  int rank,
342  int nComps,
343  SOURCE_TYPE *srcData,
344  int destType,
345  void *destData,
346  std::vector<MPI_Request> &reqs,
347  std::deque<MPI_Datatype> &types,
348  int tag);
349 
350 private:
351  int Id; // transaction id
352  int SrcRank; // rank who owns source memory
353  vtkPixelExtent SrcWholeExt; // source extent
354  vtkPixelExtent SrcExt; // source subset to transfer
355  int DestRank; // rank who owns destination memory
356  vtkPixelExtent DestWholeExt; // destination extent
357  vtkPixelExtent DestExt; // destination subset
358  int UseBlockingSend; // controls for non-blocking comm
359  int UseBlockingRecv;
360 };
361 
362 //-----------------------------------------------------------------------------
363 template<typename SOURCE_TYPE>
365  MPI_Comm comm,
366  int rank,
367  int nComps,
368  SOURCE_TYPE *srcData,
369  int destType,
370  void *destData,
371  std::vector<MPI_Request> &reqs,
372  std::deque<MPI_Datatype> &types,
373  int tag)
374 {
375  // second layer of dispatch
376  switch(destType)
377  {
378  vtkTemplateMacro(
379  return this->Execute(
380  comm,
381  rank,
382  nComps,
383  srcData,
384  (VTK_TT*)destData,
385  reqs,
386  types,
387  tag));
388  }
389  return 0;
390 }
391 
392 //-----------------------------------------------------------------------------
393 template<typename SOURCE_TYPE, typename DEST_TYPE>
395  MPI_Comm comm,
396  int rank,
397  int nComps,
398  SOURCE_TYPE *srcData,
399  DEST_TYPE *destData,
400  std::vector<MPI_Request> &reqs,
401  std::deque<MPI_Datatype> &types,
402  int tag)
403 {
404  int iErr = 0;
405  if ((comm == MPI_COMM_NULL) || (this->Local(rank)))
406  {
407  // transaction is local, bypass mpi in favor of memcpy
408  return vtkPixelTransfer::Blit(
409  this->SrcWholeExt,
410  this->SrcExt,
411  this->DestWholeExt,
412  this->DestExt,
413  nComps,
414  srcData,
415  nComps,
416  destData);
417  }
418 
419  if (rank == this->DestRank)
420  {
421  // use mpi to receive the data
422  if (destData == NULL)
423  {
424  return -1;
425  }
426 
427  MPI_Datatype subarray;
428  iErr = vtkMPIPixelViewNew<DEST_TYPE>(
429  this->DestWholeExt,
430  this->DestExt,
431  nComps,
432  subarray);
433  if (iErr)
434  {
435  return -4;
436  }
437 
438  if (this->UseBlockingRecv)
439  {
440  MPI_Status stat;
441  iErr = MPI_Recv(
442  destData,
443  1,
444  subarray,
445  this->SrcRank,
446  tag,
447  comm,
448  &stat);
449  }
450  else
451  {
452  reqs.push_back(MPI_REQUEST_NULL);
453  iErr = MPI_Irecv(
454  destData,
455  1,
456  subarray,
457  this->SrcRank,
458  tag,
459  comm,
460  &reqs.back());
461  }
462 
463  #define HOLD_RECV_TYPES
464  #ifdef HOLD_RECV_YPES
465  types.push_back(subarray);
466  #else
467  MPI_Type_free(&subarray);
468  #endif
469 
470  if (iErr)
471  {
472  return -5;
473  }
474  }
475 
476  if (rank == this->SrcRank)
477  {
478  // use mpi to send the data
479  if (srcData == NULL)
480  {
481  return -1;
482  }
483 
484  MPI_Datatype subarray;
485  iErr = vtkMPIPixelViewNew<SOURCE_TYPE>(
486  this->SrcWholeExt,
487  this->SrcExt,
488  nComps,
489  subarray);
490  if (iErr)
491  {
492  return -2;
493  }
494 
495  if (this->UseBlockingSend)
496  {
497  iErr = MPI_Ssend(
498  srcData,
499  1,
500  subarray,
501  this->DestRank,
502  tag,
503  comm);
504  }
505  else
506  {
507  MPI_Request req;
508  iErr = MPI_Isend(
509  srcData,
510  1,
511  subarray,
512  this->DestRank,
513  tag,
514  comm,
515  &req);
516  #define SAVE_SEND_REQS
517  #ifdef SAVE_SEND_REQS
518  reqs.push_back(req);
519  #else
520  MPI_Request_free(&req);
521  #endif
522  }
523 
524  #define HOLD_SEND_TYPES
525  #ifdef HOLD_SEND_TYPES
526  types.push_back(subarray);
527  #else
528  MPI_Type_free(&subarray);
529  #endif
530 
531  if (iErr)
532  {
533  return -3;
534  }
535  }
536 
537  return iErr;
538 }
539 
540 VTKRENDERINGPARALLELLIC_EXPORT
541 ostream &operator<<(std::ostream &os, const vtkPPixelTransfer &gt);
542 
543 #endif
544 // VTK-HeaderTest-Exclude: vtkPPixelTransfer.h
vtkPixelExtent & GetSourceExtent()
void SetUseBlockingSend(int val)
Enable/diasable non-blocking communication.
vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent &ext, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, both the whole and the subs...
const vtkPixelExtent & GetDestinationExtent() const
void SetDestinationRank(int rank)
int GetSourceRank() const
void SetSourceWholeExtent(vtkPixelExtent &srcExt)
Set/Get the source extent.
vtkPixelExtent & GetDestinationWholeExtent()
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &srcExt, int destRank, const vtkPixelExtent &destWholeExt, const vtkPixelExtent &destExt, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are diffe...
int GetDestinationRank() const
int GetUseBlockingRecv() const
class to handle inter-process communication of pixel data from non-contiguous regions of a shared ind...
vtkPixelExtent & GetSourceWholeExtent()
void SetUseBlockingRecv(int val)
void SetSourceExtent(vtkPixelExtent &srcExt)
Set/Get the source extent.
vtkPPixelTransfer(int srcRank, int destRank, const vtkPixelExtent &wholeExt, const vtkPixelExtent &targetExt, int id=0)
Initialize a transaction from sub extent of source to sub extent of dest, both the whole and the subs...
pixel extents
int GetTransactionId() const
const vtkPixelExtent & GetSourceExtent() const
static int Blit(const vtkPixelExtent &ext, int nComps, int srcType, void *srcData, int destType, void *destData)
for memory to memory transfers.
const vtkPixelExtent & GetSourceWholeExtent() const
VTKRENDERINGPARALLELLIC_EXPORT ostream & operator<<(std::ostream &os, const vtkPPixelTransfer &gt)
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &targetExt, int destRank, const vtkPixelExtent &destWholeExt, int id)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are the s...
vtkPPixelTransfer(int srcRank, const vtkPixelExtent &srcWholeExt, int destRank, const vtkPixelExtent &destWholeExt, int id=0)
Initialize a transaction from whole extent of source to whole extent of dest, where source and destin...
bool Sender(int rank) const
Tests to determine a given rank&#39;s role in this transaction.
int Execute(MPI_Comm comm, int rank, int nComps, SOURCE_TYPE *srcData, DEST_TYPE *destData, std::vector< MPI_Request > &reqs, std::deque< MPI_Datatype > &types, int tag)
Transfer data from source to destination.
void SetDestinationWholeExtent(vtkPixelExtent &destExt)
Set/get the destination extent.
vtkPPixelTransfer(const vtkPixelExtent &srcWholeExt, const vtkPixelExtent &srcExt, const vtkPixelExtent &destWholeExt, const vtkPixelExtent &destExt)
Initialize a transaction from sub extent of source to sub extent of dest, where the subsets are diffe...
Representation of a cartesian pixel plane and common operations on it.
void SetSourceRank(int rank)
Set/Get the MPI rank of source and destination processes.
bool Local(int rank) const
int GetUseBlockingSend() const
void SetTransactionId(int id)
Set/get the transaction id.
bool Receiver(int rank) const
void SetDestinationExtent(vtkPixelExtent &destExt)
Set/get the destination extent.
const vtkPixelExtent & GetDestinationWholeExtent() const
vtkPixelExtent & GetDestinationExtent()