Actual source code: mpicusp.cu

petsc-3.8.3 2017-12-09
Report Typos and Errors

  2: /*
  3:    This file contains routines for Parallel vector operations.
  4:  */
  5: #define PETSC_SKIP_COMPLEX
  6: #define PETSC_SKIP_SPINLOCK

  8: #include <petscconf.h>
  9:  #include <../src/vec/vec/impls/mpi/pvecimpl.h>
 10:  #include <../src/vec/vec/impls/seq/seqcusp/cuspvecimpl.h>

 12: PetscErrorCode VecDestroy_MPICUSP(Vec v)
 13: {

 17:   try {
 18:     if (v->spptr) {
 19:       delete ((Vec_CUSP*)v->spptr)->GPUarray;
 20:       delete (Vec_CUSP*) v->spptr;
 21:     }
 22:   } catch(char * ex) {
 23:     SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CUSP error: %s", ex);
 24:   }
 25:   VecDestroy_MPI(v);
 26:   return(0);
 27: }

 29: PetscErrorCode VecNorm_MPICUSP(Vec xin,NormType type,PetscReal *z)
 30: {
 31:   PetscReal      sum,work = 0.0;

 35:   if (type == NORM_2 || type == NORM_FROBENIUS) {
 36:     VecNorm_SeqCUSP(xin,NORM_2,&work);
 37:     work *= work;
 38:     MPIU_Allreduce(&work,&sum,1,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
 39:     *z    = PetscSqrtReal(sum);
 40:   } else if (type == NORM_1) {
 41:     /* Find the local part */
 42:     VecNorm_SeqCUSP(xin,NORM_1,&work);
 43:     /* Find the global max */
 44:     MPIU_Allreduce(&work,z,1,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
 45:   } else if (type == NORM_INFINITY) {
 46:     /* Find the local max */
 47:     VecNorm_SeqCUSP(xin,NORM_INFINITY,&work);
 48:     /* Find the global max */
 49:     MPIU_Allreduce(&work,z,1,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)xin));
 50:   } else if (type == NORM_1_AND_2) {
 51:     PetscReal temp[2];
 52:     VecNorm_SeqCUSP(xin,NORM_1,temp);
 53:     VecNorm_SeqCUSP(xin,NORM_2,temp+1);
 54:     temp[1] = temp[1]*temp[1];
 55:     MPIU_Allreduce(temp,z,2,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
 56:     z[1] = PetscSqrtReal(z[1]);
 57:   }
 58:   return(0);
 59: }

 61: PetscErrorCode VecDot_MPICUSP(Vec xin,Vec yin,PetscScalar *z)
 62: {
 63:   PetscScalar    sum,work;

 67:   VecDot_SeqCUSP(xin,yin,&work);
 68:   MPIU_Allreduce(&work,&sum,1,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
 69:   *z   = sum;
 70:   return(0);
 71: }

 73: PetscErrorCode VecTDot_MPICUSP(Vec xin,Vec yin,PetscScalar *z)
 74: {
 75:   PetscScalar    sum,work;

 79:   VecTDot_SeqCUSP(xin,yin,&work);
 80:   MPIU_Allreduce(&work,&sum,1,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
 81:   *z   = sum;
 82:   return(0);
 83: }

 85: PetscErrorCode VecMDot_MPICUSP(Vec xin,PetscInt nv,const Vec y[],PetscScalar *z)
 86: {
 87:   PetscScalar    awork[128],*work = awork;

 91:   if (nv > 128) {
 92:     PetscMalloc1(nv,&work);
 93:   }
 94:   VecMDot_SeqCUSP(xin,nv,y,work);
 95:   MPIU_Allreduce(work,z,nv,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
 96:   if (nv > 128) {
 97:     PetscFree(work);
 98:   }
 99:   return(0);
100: }

102: /*MC
103:    VECMPICUSP - VECMPICUSP = "mpicusp" - The basic parallel vector, modified to use CUSP

105:    Options Database Keys:
106: . -vec_type mpicusp - sets the vector type to VECMPICUSP during a call to VecSetFromOptions()

108:   Level: beginner

110: .seealso: VecCreate(), VecSetType(), VecSetFromOptions(), VecCreateMpiWithArray(), VECMPI, VecType, VecCreateMPI(), VecCreateMpi()
111: M*/


114: PetscErrorCode VecDuplicate_MPICUSP(Vec win,Vec *v)
115: {
117:   Vec_MPI        *vw,*w = (Vec_MPI*)win->data;
118:   PetscScalar    *array;

121:   VecCreate(PetscObjectComm((PetscObject)win),v);
122:   PetscLayoutReference(win->map,&(*v)->map);

124:   VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,0);
125:   vw   = (Vec_MPI*)(*v)->data;
126:   PetscMemcpy((*v)->ops,win->ops,sizeof(struct _VecOps));

128:   /* save local representation of the parallel vector (and scatter) if it exists */
129:   if (w->localrep) {
130:     VecGetArray(*v,&array);
131:     VecCreateSeqWithArray(PETSC_COMM_SELF,1,win->map->n+w->nghost,array,&vw->localrep);
132:     PetscMemcpy(vw->localrep->ops,w->localrep->ops,sizeof(struct _VecOps));
133:     VecRestoreArray(*v,&array);
134:     PetscLogObjectParent((PetscObject)*v,(PetscObject)vw->localrep);
135:     vw->localupdate = w->localupdate;
136:     if (vw->localupdate) {
137:       PetscObjectReference((PetscObject)vw->localupdate);
138:     }
139:   }

141:   /* New vector should inherit stashing property of parent */
142:   (*v)->stash.donotstash   = win->stash.donotstash;
143:   (*v)->stash.ignorenegidx = win->stash.ignorenegidx;

145:   /* change type_name appropriately */
146:   PetscObjectChangeTypeName((PetscObject)(*v),VECMPICUSP);

148:   PetscObjectListDuplicate(((PetscObject)win)->olist,&((PetscObject)(*v))->olist);
149:   PetscFunctionListDuplicate(((PetscObject)win)->qlist,&((PetscObject)(*v))->qlist);
150:   (*v)->map->bs   = PetscAbs(win->map->bs);
151:   (*v)->bstash.bs = win->bstash.bs;
152:   return(0);
153: }

155: PetscErrorCode VecDotNorm2_MPICUSP(Vec s,Vec t,PetscScalar *dp,PetscScalar *nm)
156: {
158:   PetscScalar    work[2],sum[2];

161:   VecDotNorm2_SeqCUSP(s,t,work,work+1);
162:   MPIU_Allreduce(&work,&sum,2,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)s));
163:   *dp  = sum[0];
164:   *nm  = sum[1];
165:   return(0);
166: }

168: PETSC_EXTERN PetscErrorCode VecCreate_MPICUSP(Vec vv)
169: {

173:   VecCreate_MPI_Private(vv,PETSC_FALSE,0,0);
174:   PetscObjectChangeTypeName((PetscObject)vv,VECMPICUSP);

176:   vv->ops->dotnorm2               = VecDotNorm2_MPICUSP;
177:   vv->ops->waxpy                  = VecWAXPY_SeqCUSP;
178:   vv->ops->duplicate              = VecDuplicate_MPICUSP;
179:   vv->ops->dot                    = VecDot_MPICUSP;
180:   vv->ops->mdot                   = VecMDot_MPICUSP;
181:   vv->ops->tdot                   = VecTDot_MPICUSP;
182:   vv->ops->norm                   = VecNorm_MPICUSP;
183:   vv->ops->scale                  = VecScale_SeqCUSP;
184:   vv->ops->copy                   = VecCopy_SeqCUSP;
185:   vv->ops->set                    = VecSet_SeqCUSP;
186:   vv->ops->swap                   = VecSwap_SeqCUSP;
187:   vv->ops->axpy                   = VecAXPY_SeqCUSP;
188:   vv->ops->axpby                  = VecAXPBY_SeqCUSP;
189:   vv->ops->maxpy                  = VecMAXPY_SeqCUSP;
190:   vv->ops->aypx                   = VecAYPX_SeqCUSP;
191:   vv->ops->axpbypcz               = VecAXPBYPCZ_SeqCUSP;
192:   vv->ops->pointwisemult          = VecPointwiseMult_SeqCUSP;
193:   vv->ops->setrandom              = VecSetRandom_SeqCUSP;
194:   vv->ops->placearray             = VecPlaceArray_SeqCUSP;
195:   vv->ops->replacearray           = VecReplaceArray_SeqCUSP;
196:   vv->ops->resetarray             = VecResetArray_SeqCUSP;
197:   vv->ops->dot_local              = VecDot_SeqCUSP;
198:   vv->ops->tdot_local             = VecTDot_SeqCUSP;
199:   vv->ops->norm_local             = VecNorm_SeqCUSP;
200:   vv->ops->mdot_local             = VecMDot_SeqCUSP;
201:   vv->ops->destroy                = VecDestroy_MPICUSP;
202:   vv->ops->pointwisedivide        = VecPointwiseDivide_SeqCUSP;
203:   vv->ops->getlocalvector         = VecGetLocalVector_SeqCUSP;
204:   vv->ops->restorelocalvector     = VecRestoreLocalVector_SeqCUSP;
205:   vv->ops->getlocalvectorread     = VecGetLocalVector_SeqCUSP;
206:   vv->ops->restorelocalvectorread = VecRestoreLocalVector_SeqCUSP;
207:   VecCUSPAllocateCheck(vv);CHKERRCUSP(ierr);
208:   vv->valid_GPU_array      = PETSC_CUSP_GPU;
209:   VecSet(vv,0.0);
210:   return(0);
211: }

213: PETSC_EXTERN PetscErrorCode VecCreate_CUSP(Vec v)
214: {
216:   PetscMPIInt    size;

219:   MPI_Comm_size(PetscObjectComm((PetscObject)v),&size);
220:   if (size == 1) {
221:     VecSetType(v,VECSEQCUSP);
222:   } else {
223:     VecSetType(v,VECMPICUSP);
224:   }
225:   return(0);
226: }