Actual source code: mpiviennacl.cxx
petsc-3.7.1 2016-05-15
2: /*
3: This file contains routines for Parallel vector operations.
4: */
5: #include <petscconf.h>
6: #include <../src/vec/vec/impls/mpi/pvecimpl.h> /*I "petscvec.h" I*/
7: #include <../src/vec/vec/impls/seq/seqviennacl/viennaclvecimpl.h>
11: PetscErrorCode VecDestroy_MPIViennaCL(Vec v)
12: {
16: try {
17: if (v->spptr) {
18: delete ((Vec_ViennaCL*)v->spptr)->GPUarray;
19: delete (Vec_ViennaCL*) v->spptr;
20: }
21: } catch(std::exception const & ex) {
22: SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ViennaCL error: %s", ex.what());
23: }
24: VecDestroy_MPI(v);
25: return(0);
26: }
30: PetscErrorCode VecNorm_MPIViennaCL(Vec xin,NormType type,PetscReal *z)
31: {
32: PetscReal sum,work = 0.0;
36: if (type == NORM_2 || type == NORM_FROBENIUS) {
37: VecNorm_SeqViennaCL(xin,NORM_2,&work);
38: work *= work;
39: MPIU_Allreduce(&work,&sum,1,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
40: *z = PetscSqrtReal(sum);
41: } else if (type == NORM_1) {
42: /* Find the local part */
43: VecNorm_SeqViennaCL(xin,NORM_1,&work);
44: /* Find the global max */
45: MPIU_Allreduce(&work,z,1,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
46: } else if (type == NORM_INFINITY) {
47: /* Find the local max */
48: VecNorm_SeqViennaCL(xin,NORM_INFINITY,&work);
49: /* Find the global max */
50: MPIU_Allreduce(&work,z,1,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)xin));
51: } else if (type == NORM_1_AND_2) {
52: PetscReal temp[2];
53: VecNorm_SeqViennaCL(xin,NORM_1,temp);
54: VecNorm_SeqViennaCL(xin,NORM_2,temp+1);
55: temp[1] = temp[1]*temp[1];
56: MPIU_Allreduce(temp,z,2,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)xin));
57: z[1] = PetscSqrtReal(z[1]);
58: }
59: return(0);
60: }
64: PetscErrorCode VecDot_MPIViennaCL(Vec xin,Vec yin,PetscScalar *z)
65: {
66: PetscScalar sum,work;
70: VecDot_SeqViennaCL(xin,yin,&work);
71: MPIU_Allreduce(&work,&sum,1,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
72: *z = sum;
73: return(0);
74: }
78: PetscErrorCode VecTDot_MPIViennaCL(Vec xin,Vec yin,PetscScalar *z)
79: {
80: PetscScalar sum,work;
84: VecTDot_SeqViennaCL(xin,yin,&work);
85: MPIU_Allreduce(&work,&sum,1,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
86: *z = sum;
87: return(0);
88: }
92: PetscErrorCode VecMDot_MPIViennaCL(Vec xin,PetscInt nv,const Vec y[],PetscScalar *z)
93: {
94: PetscScalar awork[128],*work = awork;
98: if (nv > 128) {
99: PetscMalloc1(nv,&work);
100: }
101: VecMDot_SeqViennaCL(xin,nv,y,work);
102: MPIU_Allreduce(work,z,nv,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)xin));
103: if (nv > 128) {
104: PetscFree(work);
105: }
106: return(0);
107: }
109: /*MC
110: VECMPIVIENNACL - VECMPIVIENNACL = "mpiviennacl" - The basic parallel vector, modified to use ViennaCL
112: Options Database Keys:
113: . -vec_type mpiviennacl - sets the vector type to VECMPIVIENNACL during a call to VecSetFromOptions()
115: Level: beginner
117: .seealso: VecCreate(), VecSetType(), VecSetFromOptions(), VecCreateMpiWithArray(), VECMPI, VecType, VecCreateMPI(), VecCreateMpi()
118: M*/
123: PetscErrorCode VecDuplicate_MPIViennaCL(Vec win,Vec *v)
124: {
126: Vec_MPI *vw,*w = (Vec_MPI*)win->data;
127: PetscScalar *array;
130: VecCreate(PetscObjectComm((PetscObject)win),v);
131: PetscLayoutReference(win->map,&(*v)->map);
133: VecCreate_MPI_Private(*v,PETSC_FALSE,w->nghost,0);
134: vw = (Vec_MPI*)(*v)->data;
135: PetscMemcpy((*v)->ops,win->ops,sizeof(struct _VecOps));
137: /* save local representation of the parallel vector (and scatter) if it exists */
138: if (w->localrep) {
139: VecGetArray(*v,&array);
140: VecCreateSeqWithArray(PETSC_COMM_SELF,1,win->map->n+w->nghost,array,&vw->localrep);
141: PetscMemcpy(vw->localrep->ops,w->localrep->ops,sizeof(struct _VecOps));
142: VecRestoreArray(*v,&array);
143: PetscLogObjectParent((PetscObject)*v,(PetscObject)vw->localrep);
144: vw->localupdate = w->localupdate;
145: if (vw->localupdate) {
146: PetscObjectReference((PetscObject)vw->localupdate);
147: }
148: }
150: /* New vector should inherit stashing property of parent */
151: (*v)->stash.donotstash = win->stash.donotstash;
152: (*v)->stash.ignorenegidx = win->stash.ignorenegidx;
154: /* change type_name appropriately */
155: PetscObjectChangeTypeName((PetscObject)(*v),VECMPIVIENNACL);
157: PetscObjectListDuplicate(((PetscObject)win)->olist,&((PetscObject)(*v))->olist);
158: PetscFunctionListDuplicate(((PetscObject)win)->qlist,&((PetscObject)(*v))->qlist);
159: (*v)->map->bs = PetscAbs(win->map->bs);
160: (*v)->bstash.bs = win->bstash.bs;
161: return(0);
162: }
166: PetscErrorCode VecDotNorm2_MPIViennaCL(Vec s,Vec t,PetscScalar *dp,PetscScalar *nm)
167: {
169: PetscScalar work[2],sum[2];
172: VecDotNorm2_SeqViennaCL(s,t,work,work+1);
173: MPIU_Allreduce((void*)&work,(void*)&sum,2,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)s));
174: *dp = sum[0];
175: *nm = sum[1];
176: return(0);
177: }
181: PETSC_EXTERN PetscErrorCode VecCreate_MPIViennaCL(Vec vv)
182: {
186: VecCreate_MPI_Private(vv,PETSC_FALSE,0,0);
187: PetscObjectChangeTypeName((PetscObject)vv,VECMPIVIENNACL);
189: vv->ops->dotnorm2 = VecDotNorm2_MPIViennaCL;
190: vv->ops->waxpy = VecWAXPY_SeqViennaCL;
191: vv->ops->duplicate = VecDuplicate_MPIViennaCL;
192: vv->ops->dot = VecDot_MPIViennaCL;
193: vv->ops->mdot = VecMDot_MPIViennaCL;
194: vv->ops->tdot = VecTDot_MPIViennaCL;
195: vv->ops->norm = VecNorm_MPIViennaCL;
196: vv->ops->scale = VecScale_SeqViennaCL;
197: vv->ops->copy = VecCopy_SeqViennaCL;
198: vv->ops->set = VecSet_SeqViennaCL;
199: vv->ops->swap = VecSwap_SeqViennaCL;
200: vv->ops->axpy = VecAXPY_SeqViennaCL;
201: vv->ops->axpby = VecAXPBY_SeqViennaCL;
202: vv->ops->maxpy = VecMAXPY_SeqViennaCL;
203: vv->ops->aypx = VecAYPX_SeqViennaCL;
204: vv->ops->axpbypcz = VecAXPBYPCZ_SeqViennaCL;
205: vv->ops->pointwisemult = VecPointwiseMult_SeqViennaCL;
206: vv->ops->setrandom = VecSetRandom_SeqViennaCL;
207: vv->ops->replacearray = VecReplaceArray_SeqViennaCL;
208: vv->ops->dot_local = VecDot_SeqViennaCL;
209: vv->ops->tdot_local = VecTDot_SeqViennaCL;
210: vv->ops->norm_local = VecNorm_SeqViennaCL;
211: vv->ops->mdot_local = VecMDot_SeqViennaCL;
212: vv->ops->destroy = VecDestroy_MPIViennaCL;
213: vv->ops->pointwisedivide = VecPointwiseDivide_SeqViennaCL;
214: /* place array?
215: reset array?
216: get values?
217: */
218: VecViennaCLAllocateCheck(vv);
219: vv->valid_GPU_array = PETSC_VIENNACL_GPU;
220: VecSet(vv,0.0);
221: return(0);
222: }
227: PETSC_EXTERN PetscErrorCode VecCreate_ViennaCL(Vec v)
228: {
230: PetscMPIInt size;
233: MPI_Comm_size(PetscObjectComm((PetscObject)v),&size);
234: if (size == 1) {
235: VecSetType(v,VECSEQVIENNACL);
236: } else {
237: VecSetType(v,VECMPIVIENNACL);
238: }
239: return(0);
240: }