Actual source code: plexfem.c

petsc-3.11.0 2019-03-29
Report Typos and Errors
  1:  #include <petsc/private/dmpleximpl.h>
  2:  #include <petscsf.h>
  3:  #include <petscsnes.h>

  5:  #include <petsc/private/hashsetij.h>
  6:  #include <petsc/private/petscfeimpl.h>
  7:  #include <petsc/private/petscfvimpl.h>

  9: static PetscErrorCode PetscContainerUserDestroy_PetscFEGeom (void *ctx)
 10: {
 11:   PetscFEGeom *geom = (PetscFEGeom *) ctx;

 15:   PetscFEGeomDestroy(&geom);
 16:   return(0);
 17: }

 19: static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
 20: {
 21:   char            composeStr[33] = {0};
 22:   PetscObjectId   id;
 23:   PetscContainer  container;
 24:   PetscErrorCode  ierr;

 27:   PetscObjectGetId((PetscObject)quad,&id);
 28:   PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%x\n", id);
 29:   PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container);
 30:   if (container) {
 31:     PetscContainerGetPointer(container, (void **) geom);
 32:   } else {
 33:     DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom);
 34:     PetscContainerCreate(PETSC_COMM_SELF,&container);
 35:     PetscContainerSetPointer(container, (void *) *geom);
 36:     PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom);
 37:     PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container);
 38:     PetscContainerDestroy(&container);
 39:   }
 40:   return(0);
 41: }

 43: static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
 44: {
 46:   *geom = NULL;
 47:   return(0);
 48: }

 50: /*@
 51:   DMPlexGetScale - Get the scale for the specified fundamental unit

 53:   Not collective

 55:   Input Arguments:
 56: + dm   - the DM
 57: - unit - The SI unit

 59:   Output Argument:
 60: . scale - The value used to scale all quantities with this unit

 62:   Level: advanced

 64: .seealso: DMPlexSetScale(), PetscUnit
 65: @*/
 66: PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale)
 67: {
 68:   DM_Plex *mesh = (DM_Plex*) dm->data;

 73:   *scale = mesh->scale[unit];
 74:   return(0);
 75: }

 77: /*@
 78:   DMPlexSetScale - Set the scale for the specified fundamental unit

 80:   Not collective

 82:   Input Arguments:
 83: + dm   - the DM
 84: . unit - The SI unit
 85: - scale - The value used to scale all quantities with this unit

 87:   Level: advanced

 89: .seealso: DMPlexGetScale(), PetscUnit
 90: @*/
 91: PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale)
 92: {
 93:   DM_Plex *mesh = (DM_Plex*) dm->data;

 97:   mesh->scale[unit] = scale;
 98:   return(0);
 99: }

101: static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nf, PetscScalar *mode, void *ctx)
102: {
103:   const PetscInt eps[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
104:   PetscInt *ctxInt  = (PetscInt *) ctx;
105:   PetscInt  dim2    = ctxInt[0];
106:   PetscInt  d       = ctxInt[1];
107:   PetscInt  i, j, k = dim > 2 ? d - dim : d;

110:   if (dim != dim2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %d does not match context dimension %d", dim, dim2);
111:   for (i = 0; i < dim; i++) mode[i] = 0.;
112:   if (d < dim) {
113:     mode[d] = 1.; /* Translation along axis d */
114:   } else {
115:     for (i = 0; i < dim; i++) {
116:       for (j = 0; j < dim; j++) {
117:         mode[j] += eps[i][j][k]*X[i]; /* Rotation about axis d */
118:       }
119:     }
120:   }
121:   return(0);
122: }

124: /*@
125:   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation

127:   Collective on DM

129:   Input Arguments:
130: . dm - the DM

132:   Output Argument:
133: . sp - the null space

135:   Note: This is necessary to provide a suitable coarse space for algebraic multigrid

137:   Level: advanced

139: .seealso: MatNullSpaceCreate(), PCGAMG
140: @*/
141: PetscErrorCode DMPlexCreateRigidBody(DM dm, MatNullSpace *sp)
142: {
143:   MPI_Comm       comm;
144:   Vec            mode[6];
145:   PetscSection   section, globalSection;
146:   PetscInt       dim, dimEmbed, n, m, mmin, d, i, j;

150:   PetscObjectGetComm((PetscObject)dm,&comm);
151:   DMGetDimension(dm, &dim);
152:   DMGetCoordinateDim(dm, &dimEmbed);
153:   if (dim == 1) {
154:     MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp);
155:     return(0);
156:   }
157:   DMGetSection(dm, &section);
158:   DMGetGlobalSection(dm, &globalSection);
159:   PetscSectionGetConstrainedStorageSize(globalSection, &n);
160:   m    = (dim*(dim+1))/2;
161:   VecCreate(comm, &mode[0]);
162:   VecSetSizes(mode[0], n, PETSC_DETERMINE);
163:   VecSetUp(mode[0]);
164:   VecGetSize(mode[0], &n);
165:   mmin = PetscMin(m, n);
166:   for (i = 1; i < m; ++i) {VecDuplicate(mode[0], &mode[i]);}
167:   for (d = 0; d < m; d++) {
168:     PetscInt         ctx[2];
169:     PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
170:     void            *voidctx = (void *) (&ctx[0]);

172:     ctx[0] = dimEmbed;
173:     ctx[1] = d;
174:     DMProjectFunction(dm, 0.0, &func, &voidctx, INSERT_VALUES, mode[d]);
175:   }
176:   for (i = 0; i < PetscMin(dim, mmin); ++i) {VecNormalize(mode[i], NULL);}
177:   /* Orthonormalize system */
178:   for (i = dim; i < mmin; ++i) {
179:     PetscScalar dots[6];

181:     VecMDot(mode[i], i, mode, dots);
182:     for (j = 0; j < i; ++j) dots[j] *= -1.0;
183:     VecMAXPY(mode[i], i, dots, mode);
184:     VecNormalize(mode[i], NULL);
185:   }
186:   MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp);
187:   for (i = 0; i < m; ++i) {VecDestroy(&mode[i]);}
188:   return(0);
189: }

191: /*@
192:   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation

194:   Collective on DM

196:   Input Arguments:
197: + dm    - the DM
198: . nb    - The number of bodies
199: . label - The DMLabel marking each domain
200: . nids  - The number of ids per body
201: - ids   - An array of the label ids in sequence for each domain

203:   Output Argument:
204: . sp - the null space

206:   Note: This is necessary to provide a suitable coarse space for algebraic multigrid

208:   Level: advanced

210: .seealso: MatNullSpaceCreate()
211: @*/
212: PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp)
213: {
214:   MPI_Comm       comm;
215:   PetscSection   section, globalSection;
216:   Vec           *mode;
217:   PetscScalar   *dots;
218:   PetscInt       dim, dimEmbed, n, m, b, d, i, j, off;

222:   PetscObjectGetComm((PetscObject)dm,&comm);
223:   DMGetDimension(dm, &dim);
224:   DMGetCoordinateDim(dm, &dimEmbed);
225:   DMGetSection(dm, &section);
226:   DMGetGlobalSection(dm, &globalSection);
227:   PetscSectionGetConstrainedStorageSize(globalSection, &n);
228:   m    = nb * (dim*(dim+1))/2;
229:   PetscMalloc2(m, &mode, m, &dots);
230:   VecCreate(comm, &mode[0]);
231:   VecSetSizes(mode[0], n, PETSC_DETERMINE);
232:   VecSetUp(mode[0]);
233:   for (i = 1; i < m; ++i) {VecDuplicate(mode[0], &mode[i]);}
234:   for (b = 0, off = 0; b < nb; ++b) {
235:     for (d = 0; d < m/nb; ++d) {
236:       PetscInt         ctx[2];
237:       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
238:       void            *voidctx = (void *) (&ctx[0]);

240:       ctx[0] = dimEmbed;
241:       ctx[1] = d;
242:       DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]);
243:       off   += nids[b];
244:     }
245:   }
246:   for (i = 0; i < dim; ++i) {VecNormalize(mode[i], NULL);}
247:   /* Orthonormalize system */
248:   for (i = 0; i < m; ++i) {
249:     VecMDot(mode[i], i, mode, dots);
250:     for (j = 0; j < i; ++j) dots[j] *= -1.0;
251:     VecMAXPY(mode[i], i, dots, mode);
252:     VecNormalize(mode[i], NULL);
253:   }
254:   MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp);
255:   for (i = 0; i< m; ++i) {VecDestroy(&mode[i]);}
256:   PetscFree2(mode, dots);
257:   return(0);
258: }

260: /*@
261:   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
262:   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
263:   evaluating the dual space basis of that point.  A basis function is associated with the point in its
264:   transitively-closed support whose mesh height is highest (w.r.t. DAG height), but not greater than the maximum
265:   projection height, which is set with this function.  By default, the maximum projection height is zero, which means
266:   that only mesh cells are used to project basis functions.  A height of one, for example, evaluates a cell-interior
267:   basis functions using its cells dual space basis, but all other basis functions with the dual space basis of a face.

269:   Input Parameters:
270: + dm - the DMPlex object
271: - height - the maximum projection height >= 0

273:   Level: advanced

275: .seealso: DMPlexGetMaxProjectionHeight(), DMProjectFunctionLocal(), DMProjectFunctionLabelLocal()
276: @*/
277: PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height)
278: {
279:   DM_Plex *plex = (DM_Plex *) dm->data;

283:   plex->maxProjectionHeight = height;
284:   return(0);
285: }

287: /*@
288:   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
289:   DMPlexProjectXXXLocal() functions.

291:   Input Parameters:
292: . dm - the DMPlex object

294:   Output Parameters:
295: . height - the maximum projection height

297:   Level: intermediate

299: .seealso: DMPlexSetMaxProjectionHeight(), DMProjectFunctionLocal(), DMProjectFunctionLabelLocal()
300: @*/
301: PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height)
302: {
303:   DM_Plex *plex = (DM_Plex *) dm->data;

307:   *height = plex->maxProjectionHeight;
308:   return(0);
309: }

311: /*@C
312:   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector

314:   Input Parameters:
315: + dm     - The DM, with a PetscDS that matches the problem being constrained
316: . time   - The time
317: . field  - The field to constrain
318: . Nc     - The number of constrained field components, or 0 for all components
319: . comps  - An array of constrained component numbers, or NULL for all components
320: . label  - The DMLabel defining constrained points
321: . numids - The number of DMLabel ids for constrained points
322: . ids    - An array of ids for constrained points
323: . func   - A pointwise function giving boundary values
324: - ctx    - An optional user context for bcFunc

326:   Output Parameter:
327: . locX   - A local vector to receives the boundary values

329:   Level: developer

331: .seealso: DMPlexInsertBoundaryValuesEssentialField(), DMAddBoundary()
332: @*/
333: PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX)
334: {
335:   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
336:   void            **ctxs;
337:   PetscInt          numFields;
338:   PetscErrorCode    ierr;

341:   DMGetNumFields(dm, &numFields);
342:   PetscCalloc2(numFields,&funcs,numFields,&ctxs);
343:   funcs[field] = func;
344:   ctxs[field]  = ctx;
345:   DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX);
346:   PetscFree2(funcs,ctxs);
347:   return(0);
348: }

350: /*@C
351:   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector

353:   Input Parameters:
354: + dm     - The DM, with a PetscDS that matches the problem being constrained
355: . time   - The time
356: . locU   - A local vector with the input solution values
357: . field  - The field to constrain
358: . Nc     - The number of constrained field components, or 0 for all components
359: . comps  - An array of constrained component numbers, or NULL for all components
360: . label  - The DMLabel defining constrained points
361: . numids - The number of DMLabel ids for constrained points
362: . ids    - An array of ids for constrained points
363: . func   - A pointwise function giving boundary values
364: - ctx    - An optional user context for bcFunc

366:   Output Parameter:
367: . locX   - A local vector to receives the boundary values

369:   Level: developer

371: .seealso: DMPlexInsertBoundaryValuesEssential(), DMAddBoundary()
372: @*/
373: PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
374:                                                         void (*func)(PetscInt, PetscInt, PetscInt,
375:                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
376:                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
377:                                                                      PetscReal, const PetscReal[], PetscInt, const PetscScalar[],
378:                                                                      PetscScalar[]),
379:                                                         void *ctx, Vec locX)
380: {
381:   void (**funcs)(PetscInt, PetscInt, PetscInt,
382:                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
383:                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
384:                  PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
385:   void            **ctxs;
386:   PetscInt          numFields;
387:   PetscErrorCode    ierr;

390:   DMGetNumFields(dm, &numFields);
391:   PetscCalloc2(numFields,&funcs,numFields,&ctxs);
392:   funcs[field] = func;
393:   ctxs[field]  = ctx;
394:   DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX);
395:   PetscFree2(funcs,ctxs);
396:   return(0);
397: }

399: /*@C
400:   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector

402:   Input Parameters:
403: + dm     - The DM, with a PetscDS that matches the problem being constrained
404: . time   - The time
405: . faceGeometry - A vector with the FVM face geometry information
406: . cellGeometry - A vector with the FVM cell geometry information
407: . Grad         - A vector with the FVM cell gradient information
408: . field  - The field to constrain
409: . Nc     - The number of constrained field components, or 0 for all components
410: . comps  - An array of constrained component numbers, or NULL for all components
411: . label  - The DMLabel defining constrained points
412: . numids - The number of DMLabel ids for constrained points
413: . ids    - An array of ids for constrained points
414: . func   - A pointwise function giving boundary values
415: - ctx    - An optional user context for bcFunc

417:   Output Parameter:
418: . locX   - A local vector to receives the boundary values

420:   Note: This implementation currently ignores the numcomps/comps argument from DMAddBoundary()

422:   Level: developer

424: .seealso: DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialField(), DMAddBoundary()
425: @*/
426: PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
427:                                                  PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*), void *ctx, Vec locX)
428: {
429:   PetscDS            prob;
430:   PetscSF            sf;
431:   DM                 dmFace, dmCell, dmGrad;
432:   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
433:   const PetscInt    *leaves;
434:   PetscScalar       *x, *fx;
435:   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
436:   PetscErrorCode     ierr, ierru = 0;

439:   DMGetPointSF(dm, &sf);
440:   PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);
441:   nleaves = PetscMax(0, nleaves);
442:   DMGetDimension(dm, &dim);
443:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
444:   DMGetDS(dm, &prob);
445:   VecGetDM(faceGeometry, &dmFace);
446:   VecGetArrayRead(faceGeometry, &facegeom);
447:   if (cellGeometry) {
448:     VecGetDM(cellGeometry, &dmCell);
449:     VecGetArrayRead(cellGeometry, &cellgeom);
450:   }
451:   if (Grad) {
452:     PetscFV fv;

454:     PetscDSGetDiscretization(prob, field, (PetscObject *) &fv);
455:     VecGetDM(Grad, &dmGrad);
456:     VecGetArrayRead(Grad, &grad);
457:     PetscFVGetNumComponents(fv, &pdim);
458:     DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx);
459:   }
460:   VecGetArray(locX, &x);
461:   for (i = 0; i < numids; ++i) {
462:     IS              faceIS;
463:     const PetscInt *faces;
464:     PetscInt        numFaces, f;

466:     DMLabelGetStratumIS(label, ids[i], &faceIS);
467:     if (!faceIS) continue; /* No points with that id on this process */
468:     ISGetLocalSize(faceIS, &numFaces);
469:     ISGetIndices(faceIS, &faces);
470:     for (f = 0; f < numFaces; ++f) {
471:       const PetscInt         face = faces[f], *cells;
472:       PetscFVFaceGeom        *fg;

474:       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
475:       PetscFindInt(face, nleaves, (PetscInt *) leaves, &loc);
476:       if (loc >= 0) continue;
477:       DMPlexPointLocalRead(dmFace, face, facegeom, &fg);
478:       DMPlexGetSupport(dm, face, &cells);
479:       if (Grad) {
480:         PetscFVCellGeom       *cg;
481:         PetscScalar           *cx, *cgrad;
482:         PetscScalar           *xG;
483:         PetscReal              dx[3];
484:         PetscInt               d;

486:         DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg);
487:         DMPlexPointLocalRead(dm, cells[0], x, &cx);
488:         DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad);
489:         DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG);
490:         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
491:         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d*dim], dx);
492:         ierru = (*func)(time, fg->centroid, fg->normal, fx, xG, ctx);
493:         if (ierru) {
494:           ISRestoreIndices(faceIS, &faces);
495:           ISDestroy(&faceIS);
496:           goto cleanup;
497:         }
498:       } else {
499:         PetscScalar       *xI;
500:         PetscScalar       *xG;

502:         DMPlexPointLocalRead(dm, cells[0], x, &xI);
503:         DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG);
504:         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
505:         if (ierru) {
506:           ISRestoreIndices(faceIS, &faces);
507:           ISDestroy(&faceIS);
508:           goto cleanup;
509:         }
510:       }
511:     }
512:     ISRestoreIndices(faceIS, &faces);
513:     ISDestroy(&faceIS);
514:   }
515:   cleanup:
516:   VecRestoreArray(locX, &x);
517:   if (Grad) {
518:     DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx);
519:     VecRestoreArrayRead(Grad, &grad);
520:   }
521:   if (cellGeometry) {VecRestoreArrayRead(cellGeometry, &cellgeom);}
522:   VecRestoreArrayRead(faceGeometry, &facegeom);
523:   CHKERRQ(ierru);
524:   return(0);
525: }

527: PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
528: {
529:   PetscDS        prob;
530:   PetscInt       numBd, b;

534:   DMGetDS(dm, &prob);
535:   PetscDSGetNumBoundary(prob, &numBd);
536:   for (b = 0; b < numBd; ++b) {
537:     DMBoundaryConditionType type;
538:     const char             *labelname;
539:     DMLabel                 label;
540:     PetscInt                field, Nc;
541:     const PetscInt         *comps;
542:     PetscObject             obj;
543:     PetscClassId            id;
544:     void                    (*func)(void);
545:     PetscInt                numids;
546:     const PetscInt         *ids;
547:     void                   *ctx;

549:     DMGetBoundary(dm, b, &type, NULL, &labelname, &field, &Nc, &comps, &func, &numids, &ids, &ctx);
550:     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
551:     DMGetLabel(dm, labelname, &label);
552:     DMGetField(dm, field, NULL, &obj);
553:     PetscObjectGetClassId(obj, &id);
554:     if (id == PETSCFE_CLASSID) {
555:       switch (type) {
556:         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
557:       case DM_BC_ESSENTIAL:
558:         DMPlexLabelAddCells(dm,label);
559:         DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, (PetscErrorCode (*)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *)) func, ctx, locX);
560:         DMPlexLabelClearCells(dm,label);
561:         break;
562:       case DM_BC_ESSENTIAL_FIELD:
563:         DMPlexLabelAddCells(dm,label);
564:         DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids,
565:                                                         (void (*)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
566:                                                                   const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
567:                                                                   PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[])) func, ctx, locX);
568:         DMPlexLabelClearCells(dm,label);
569:         break;
570:       default: break;
571:       }
572:     } else if (id == PETSCFV_CLASSID) {
573:       if (!faceGeomFVM) continue;
574:       DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids,
575:                                                (PetscErrorCode (*)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*)) func, ctx, locX);
576:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
577:   }
578:   return(0);
579: }

581: /*@
582:   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector

584:   Input Parameters:
585: + dm - The DM
586: . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
587: . time - The time
588: . faceGeomFVM - Face geometry data for FV discretizations
589: . cellGeomFVM - Cell geometry data for FV discretizations
590: - gradFVM - Gradient reconstruction data for FV discretizations

592:   Output Parameters:
593: . locX - Solution updated with boundary values

595:   Level: developer

597: .seealso: DMProjectFunctionLabelLocal()
598: @*/
599: PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
600: {

609:   PetscTryMethod(dm,"DMPlexInsertBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX,time,faceGeomFVM,cellGeomFVM,gradFVM));
610:   return(0);
611: }

613: PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
614: {
615:   Vec              localX;
616:   PetscErrorCode   ierr;

619:   DMGetLocalVector(dm, &localX);
620:   DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL);
621:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);
622:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);
623:   DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff);
624:   DMRestoreLocalVector(dm, &localX);
625:   return(0);
626: }

628: /*@C
629:   DMComputeL2Diff - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.

631:   Input Parameters:
632: + dm     - The DM
633: . time   - The time
634: . funcs  - The functions to evaluate for each field component
635: . ctxs   - Optional array of contexts to pass to each function, or NULL.
636: - localX - The coefficient vector u_h, a local vector

638:   Output Parameter:
639: . diff - The diff ||u - u_h||_2

641:   Level: developer

643: .seealso: DMProjectFunction(), DMComputeL2FieldDiff(), DMComputeL2GradientDiff()
644: @*/
645: PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff)
646: {
647:   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
648:   PetscSection     section;
649:   PetscQuadrature  quad;
650:   PetscScalar     *funcVal, *interpolant;
651:   PetscReal       *coords, *detJ, *J;
652:   PetscReal        localDiff = 0.0;
653:   const PetscReal *quadWeights;
654:   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, cEndInterior, c, field, fieldOffset;
655:   PetscErrorCode   ierr;

658:   DMGetDimension(dm, &dim);
659:   DMGetCoordinateDim(dm, &coordDim);
660:   DMGetSection(dm, &section);
661:   PetscSectionGetNumFields(section, &numFields);
662:   for (field = 0; field < numFields; ++field) {
663:     PetscObject  obj;
664:     PetscClassId id;
665:     PetscInt     Nc;

667:     DMGetField(dm, field, NULL, &obj);
668:     PetscObjectGetClassId(obj, &id);
669:     if (id == PETSCFE_CLASSID) {
670:       PetscFE fe = (PetscFE) obj;

672:       PetscFEGetQuadrature(fe, &quad);
673:       PetscFEGetNumComponents(fe, &Nc);
674:     } else if (id == PETSCFV_CLASSID) {
675:       PetscFV fv = (PetscFV) obj;

677:       PetscFVGetQuadrature(fv, &quad);
678:       PetscFVGetNumComponents(fv, &Nc);
679:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
680:     numComponents += Nc;
681:   }
682:   PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights);
683:   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
684:   PetscMalloc5(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&detJ,coordDim*coordDim*Nq,&J);
685:   DMPlexGetVTKCellHeight(dm, &cellHeight);
686:   DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);
687:   DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);
688:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
689:   for (c = cStart; c < cEnd; ++c) {
690:     PetscScalar *x = NULL;
691:     PetscReal    elemDiff = 0.0;
692:     PetscInt     qc = 0;

694:     DMPlexComputeCellGeometryFEM(dm, c, quad, coords, J, NULL, detJ);
695:     DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);

697:     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
698:       PetscObject  obj;
699:       PetscClassId id;
700:       void * const ctx = ctxs ? ctxs[field] : NULL;
701:       PetscInt     Nb, Nc, q, fc;

703:       DMGetField(dm, field, NULL, &obj);
704:       PetscObjectGetClassId(obj, &id);
705:       if (id == PETSCFE_CLASSID)      {PetscFEGetNumComponents((PetscFE) obj, &Nc);PetscFEGetDimension((PetscFE) obj, &Nb);}
706:       else if (id == PETSCFV_CLASSID) {PetscFVGetNumComponents((PetscFV) obj, &Nc);Nb = 1;}
707:       else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
708:       if (debug) {
709:         char title[1024];
710:         PetscSNPrintf(title, 1023, "Solution for Field %d", field);
711:         DMPrintCellVector(c, title, Nb, &x[fieldOffset]);
712:       }
713:       for (q = 0; q < Nq; ++q) {
714:         if (detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, point %D", detJ[q], c, q);
715:         (*funcs[field])(coordDim, time, &coords[coordDim * q], Nc, funcVal, ctx);
716:         if (ierr) {
717:           PetscErrorCode ierr2;
718:           ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
719:           ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
720:           ierr2 = PetscFree5(funcVal,interpolant,coords,detJ,J);CHKERRQ(ierr2);
721: 
722:         }
723:         if (id == PETSCFE_CLASSID)      {PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], q, interpolant);}
724:         else if (id == PETSCFV_CLASSID) {PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant);}
725:         else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
726:         for (fc = 0; fc < Nc; ++fc) {
727:           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
728:           if (debug) {PetscPrintf(PETSC_COMM_SELF, "    elem %d field %d diff %g\n", c, field, PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q]);}
729:           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q];
730:         }
731:       }
732:       fieldOffset += Nb;
733:       qc += Nc;
734:     }
735:     DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);
736:     if (debug) {PetscPrintf(PETSC_COMM_SELF, "  elem %d diff %g\n", c, elemDiff);}
737:     localDiff += elemDiff;
738:   }
739:   PetscFree5(funcVal,interpolant,coords,detJ,J);
740:   MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm));
741:   *diff = PetscSqrtReal(*diff);
742:   return(0);
743: }

745: PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff)
746: {
747:   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
748:   PetscSection     section;
749:   PetscQuadrature  quad;
750:   Vec              localX;
751:   PetscScalar     *funcVal, *interpolant;
752:   const PetscReal *quadPoints, *quadWeights;
753:   PetscReal       *coords, *realSpaceDer, *J, *invJ, *detJ;
754:   PetscReal        localDiff = 0.0;
755:   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, cEndInterior, c, field, fieldOffset;
756:   PetscErrorCode   ierr;

759:   DMGetDimension(dm, &dim);
760:   DMGetCoordinateDim(dm, &coordDim);
761:   DMGetSection(dm, &section);
762:   PetscSectionGetNumFields(section, &numFields);
763:   DMGetLocalVector(dm, &localX);
764:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);
765:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);
766:   for (field = 0; field < numFields; ++field) {
767:     PetscFE  fe;
768:     PetscInt Nc;

770:     DMGetField(dm, field, NULL, (PetscObject *) &fe);
771:     PetscFEGetQuadrature(fe, &quad);
772:     PetscFEGetNumComponents(fe, &Nc);
773:     numComponents += Nc;
774:   }
775:   PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);
776:   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
777:   /* DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX); */
778:   PetscMalloc7(numComponents,&funcVal,coordDim*Nq,&coords,coordDim*Nq,&realSpaceDer,coordDim*coordDim*Nq,&J,coordDim*coordDim*Nq,&invJ,numComponents,&interpolant,Nq,&detJ);
779:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
780:   DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);
781:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
782:   for (c = cStart; c < cEnd; ++c) {
783:     PetscScalar *x = NULL;
784:     PetscReal    elemDiff = 0.0;
785:     PetscInt     qc = 0;

787:     DMPlexComputeCellGeometryFEM(dm, c, quad, coords, J, invJ, detJ);
788:     DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);

790:     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
791:       PetscFE          fe;
792:       void * const     ctx = ctxs ? ctxs[field] : NULL;
793:       PetscReal       *basisDer;
794:       PetscInt         Nb, Nc, q, fc;

796:       DMGetField(dm, field, NULL, (PetscObject *) &fe);
797:       PetscFEGetDimension(fe, &Nb);
798:       PetscFEGetNumComponents(fe, &Nc);
799:       PetscFEGetDefaultTabulation(fe, NULL, &basisDer, NULL);
800:       if (debug) {
801:         char title[1024];
802:         PetscSNPrintf(title, 1023, "Solution for Field %d", field);
803:         DMPrintCellVector(c, title, Nb, &x[fieldOffset]);
804:       }
805:       for (q = 0; q < Nq; ++q) {
806:         if (detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature points %D", detJ[q], c, q);
807:         (*funcs[field])(coordDim, time, &coords[q*coordDim], n, numFields, funcVal, ctx);
808:         if (ierr) {
809:           PetscErrorCode ierr2;
810:           ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
811:           ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
812:           ierr2 = PetscFree7(funcVal,coords,realSpaceDer,J,invJ,interpolant,detJ);CHKERRQ(ierr2);
813: 
814:         }
815:         PetscFEInterpolateGradient_Static(fe, &x[fieldOffset], coordDim, invJ, n, q, interpolant);
816:         for (fc = 0; fc < Nc; ++fc) {
817:           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
818:           if (debug) {PetscPrintf(PETSC_COMM_SELF, "    elem %d fieldDer %d diff %g\n", c, field, PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q]);}
819:           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q];
820:         }
821:       }
822:       fieldOffset += Nb;
823:       qc          += Nc;
824:     }
825:     DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);
826:     if (debug) {PetscPrintf(PETSC_COMM_SELF, "  elem %d diff %g\n", c, elemDiff);}
827:     localDiff += elemDiff;
828:   }
829:   PetscFree7(funcVal,coords,realSpaceDer,J,invJ,interpolant,detJ);
830:   DMRestoreLocalVector(dm, &localX);
831:   MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm));
832:   *diff = PetscSqrtReal(*diff);
833:   return(0);
834: }

836: PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
837: {
838:   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
839:   PetscSection     section;
840:   PetscQuadrature  quad;
841:   Vec              localX;
842:   PetscScalar     *funcVal, *interpolant;
843:   PetscReal       *coords, *detJ, *J;
844:   PetscReal       *localDiff;
845:   const PetscReal *quadPoints, *quadWeights;
846:   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, cEndInterior, c, field, fieldOffset;
847:   PetscErrorCode   ierr;

850:   DMGetDimension(dm, &dim);
851:   DMGetCoordinateDim(dm, &coordDim);
852:   DMGetSection(dm, &section);
853:   PetscSectionGetNumFields(section, &numFields);
854:   DMGetLocalVector(dm, &localX);
855:   DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX);
856:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);
857:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);
858:   for (field = 0; field < numFields; ++field) {
859:     PetscObject  obj;
860:     PetscClassId id;
861:     PetscInt     Nc;

863:     DMGetField(dm, field, NULL, &obj);
864:     PetscObjectGetClassId(obj, &id);
865:     if (id == PETSCFE_CLASSID) {
866:       PetscFE fe = (PetscFE) obj;

868:       PetscFEGetQuadrature(fe, &quad);
869:       PetscFEGetNumComponents(fe, &Nc);
870:     } else if (id == PETSCFV_CLASSID) {
871:       PetscFV fv = (PetscFV) obj;

873:       PetscFVGetQuadrature(fv, &quad);
874:       PetscFVGetNumComponents(fv, &Nc);
875:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
876:     numComponents += Nc;
877:   }
878:   PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);
879:   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
880:   PetscCalloc6(numFields,&localDiff,numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&detJ,coordDim*coordDim*Nq,&J);
881:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
882:   DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);
883:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
884:   for (c = cStart; c < cEnd; ++c) {
885:     PetscScalar *x = NULL;
886:     PetscInt     qc = 0;

888:     DMPlexComputeCellGeometryFEM(dm, c, quad, coords, J, NULL, detJ);
889:     DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);

891:     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
892:       PetscObject  obj;
893:       PetscClassId id;
894:       void * const ctx = ctxs ? ctxs[field] : NULL;
895:       PetscInt     Nb, Nc, q, fc;

897:       PetscReal       elemDiff = 0.0;

899:       DMGetField(dm, field, NULL, &obj);
900:       PetscObjectGetClassId(obj, &id);
901:       if (id == PETSCFE_CLASSID)      {PetscFEGetNumComponents((PetscFE) obj, &Nc);PetscFEGetDimension((PetscFE) obj, &Nb);}
902:       else if (id == PETSCFV_CLASSID) {PetscFVGetNumComponents((PetscFV) obj, &Nc);Nb = 1;}
903:       else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
904:       if (debug) {
905:         char title[1024];
906:         PetscSNPrintf(title, 1023, "Solution for Field %d", field);
907:         DMPrintCellVector(c, title, Nb*Nc, &x[fieldOffset]);
908:       }
909:       for (q = 0; q < Nq; ++q) {
910:         if (detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature point %D", detJ, c, q);
911:         (*funcs[field])(coordDim, time, &coords[coordDim*q], numFields, funcVal, ctx);
912:         if (ierr) {
913:           PetscErrorCode ierr2;
914:           ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
915:           ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
916:           ierr2 = PetscFree6(localDiff,funcVal,interpolant,coords,detJ,J);CHKERRQ(ierr2);
917: 
918:         }
919:         if (id == PETSCFE_CLASSID)      {PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], q, interpolant);}
920:         else if (id == PETSCFV_CLASSID) {PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant);}
921:         else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
922:         for (fc = 0; fc < Nc; ++fc) {
923:           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
924:           if (debug) {PetscPrintf(PETSC_COMM_SELF, "    elem %d field %d point %g %g %g diff %g\n", c, field, coordDim > 0 ? coords[0] : 0., coordDim > 1 ? coords[1] : 0., coordDim > 2 ? coords[2] : 0., PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q]);}
925:           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q];
926:         }
927:       }
928:       fieldOffset += Nb;
929:       qc          += Nc;
930:       localDiff[field] += elemDiff;
931:     }
932:     DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);
933:   }
934:   DMRestoreLocalVector(dm, &localX);
935:   MPIU_Allreduce(localDiff, diff, numFields, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm));
936:   for (field = 0; field < numFields; ++field) diff[field] = PetscSqrtReal(diff[field]);
937:   PetscFree6(localDiff,funcVal,interpolant,coords,detJ,J);
938:   return(0);
939: }

941: /*@C
942:   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.

944:   Input Parameters:
945: + dm    - The DM
946: . time  - The time
947: . funcs - The functions to evaluate for each field component: NULL means that component does not contribute to error calculation
948: . ctxs  - Optional array of contexts to pass to each function, or NULL.
949: - X     - The coefficient vector u_h

951:   Output Parameter:
952: . D - A Vec which holds the difference ||u - u_h||_2 for each cell

954:   Level: developer

956: .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
957: @*/
958: PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D)
959: {
960:   PetscSection     section;
961:   PetscQuadrature  quad;
962:   Vec              localX;
963:   PetscScalar     *funcVal, *interpolant;
964:   PetscReal       *coords, *detJ, *J;
965:   const PetscReal *quadPoints, *quadWeights;
966:   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, cEndInterior, c, field, fieldOffset;
967:   PetscErrorCode   ierr;

970:   VecSet(D, 0.0);
971:   DMGetDimension(dm, &dim);
972:   DMGetCoordinateDim(dm, &coordDim);
973:   DMGetSection(dm, &section);
974:   PetscSectionGetNumFields(section, &numFields);
975:   DMGetLocalVector(dm, &localX);
976:   DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX);
977:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX);
978:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX);
979:   for (field = 0; field < numFields; ++field) {
980:     PetscObject  obj;
981:     PetscClassId id;
982:     PetscInt     Nc;

984:     DMGetField(dm, field, NULL, &obj);
985:     PetscObjectGetClassId(obj, &id);
986:     if (id == PETSCFE_CLASSID) {
987:       PetscFE fe = (PetscFE) obj;

989:       PetscFEGetQuadrature(fe, &quad);
990:       PetscFEGetNumComponents(fe, &Nc);
991:     } else if (id == PETSCFV_CLASSID) {
992:       PetscFV fv = (PetscFV) obj;

994:       PetscFVGetQuadrature(fv, &quad);
995:       PetscFVGetNumComponents(fv, &Nc);
996:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
997:     numComponents += Nc;
998:   }
999:   PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);
1000:   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
1001:   PetscMalloc5(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&detJ,coordDim*coordDim*Nq,&J);
1002:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
1003:   DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);
1004:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
1005:   for (c = cStart; c < cEnd; ++c) {
1006:     PetscScalar *x = NULL;
1007:     PetscScalar  elemDiff = 0.0;
1008:     PetscInt     qc = 0;

1010:     DMPlexComputeCellGeometryFEM(dm, c, quad, coords, J, NULL, detJ);
1011:     DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x);

1013:     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1014:       PetscObject  obj;
1015:       PetscClassId id;
1016:       void * const ctx = ctxs ? ctxs[field] : NULL;
1017:       PetscInt     Nb, Nc, q, fc;

1019:       DMGetField(dm, field, NULL, &obj);
1020:       PetscObjectGetClassId(obj, &id);
1021:       if (id == PETSCFE_CLASSID)      {PetscFEGetNumComponents((PetscFE) obj, &Nc);PetscFEGetDimension((PetscFE) obj, &Nb);}
1022:       else if (id == PETSCFV_CLASSID) {PetscFVGetNumComponents((PetscFV) obj, &Nc);Nb = 1;}
1023:       else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
1024:       if (funcs[field]) {
1025:         for (q = 0; q < Nq; ++q) {
1026:           if (detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature points %D", (double)detJ[q], c, q);
1027:           (*funcs[field])(coordDim, time, &coords[q*coordDim], Nc, funcVal, ctx);
1028:           if (ierr) {
1029:             PetscErrorCode ierr2;
1030:             ierr2 = DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);CHKERRQ(ierr2);
1031:             ierr2 = PetscFree5(funcVal,interpolant,coords,detJ,J);CHKERRQ(ierr2);
1032:             ierr2 = DMRestoreLocalVector(dm, &localX);CHKERRQ(ierr2);
1033: 
1034:           }
1035:           if (id == PETSCFE_CLASSID)      {PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], q, interpolant);}
1036:           else if (id == PETSCFV_CLASSID) {PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant);}
1037:           else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
1038:           for (fc = 0; fc < Nc; ++fc) {
1039:             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
1040:             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*detJ[q];
1041:           }
1042:         }
1043:       }
1044:       fieldOffset += Nb;
1045:       qc          += Nc;
1046:     }
1047:     DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x);
1048:     VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES);
1049:   }
1050:   PetscFree5(funcVal,interpolant,coords,detJ,J);
1051:   DMRestoreLocalVector(dm, &localX);
1052:   VecSqrtAbs(D);
1053:   return(0);
1054: }

1056: /*@C
1057:   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1, and stores it in a Vec.

1059:   Input Parameters:
1060: + dm - The DM
1061: - LocX  - The coefficient vector u_h

1063:   Output Parameter:
1064: . locC - A Vec which holds the Clement interpolant of the gradient

1066:   Notes:
1067:     Add citation to (Clement, 1975) and definition of the interpolant
1068:   \nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume

1070:   Level: developer

1072: .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
1073: @*/
1074: PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC)
1075: {
1076:   DM_Plex         *mesh  = (DM_Plex *) dm->data;
1077:   PetscInt         debug = mesh->printFEM;
1078:   DM               dmC;
1079:   PetscSection     section;
1080:   PetscQuadrature  quad;
1081:   PetscScalar     *interpolant, *gradsum;
1082:   PetscReal       *coords, *detJ, *J, *invJ;
1083:   const PetscReal *quadPoints, *quadWeights;
1084:   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, cEndInterior, vStart, vEnd, v, field, fieldOffset;
1085:   PetscErrorCode   ierr;

1088:   VecGetDM(locC, &dmC);
1089:   VecSet(locC, 0.0);
1090:   DMGetDimension(dm, &dim);
1091:   DMGetCoordinateDim(dm, &coordDim);
1092:   DMGetSection(dm, &section);
1093:   PetscSectionGetNumFields(section, &numFields);
1094:   for (field = 0; field < numFields; ++field) {
1095:     PetscObject  obj;
1096:     PetscClassId id;
1097:     PetscInt     Nc;

1099:     DMGetField(dm, field, NULL, &obj);
1100:     PetscObjectGetClassId(obj, &id);
1101:     if (id == PETSCFE_CLASSID) {
1102:       PetscFE fe = (PetscFE) obj;

1104:       PetscFEGetQuadrature(fe, &quad);
1105:       PetscFEGetNumComponents(fe, &Nc);
1106:     } else if (id == PETSCFV_CLASSID) {
1107:       PetscFV fv = (PetscFV) obj;

1109:       PetscFVGetQuadrature(fv, &quad);
1110:       PetscFVGetNumComponents(fv, &Nc);
1111:     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
1112:     numComponents += Nc;
1113:   }
1114:   PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights);
1115:   if ((qNc != 1) && (qNc != numComponents)) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %D != %D field components", qNc, numComponents);
1116:   PetscMalloc6(coordDim*numComponents*2,&gradsum,coordDim*numComponents,&interpolant,coordDim*Nq,&coords,Nq,&detJ,coordDim*coordDim*Nq,&J,coordDim*coordDim*Nq,&invJ);
1117:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
1118:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
1119:   DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);
1120:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
1121:   for (v = vStart; v < vEnd; ++v) {
1122:     PetscScalar volsum = 0.0;
1123:     PetscInt   *star = NULL;
1124:     PetscInt    starSize, st, d, fc;

1126:     PetscMemzero(gradsum, coordDim*numComponents * sizeof(PetscScalar));
1127:     DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star);
1128:     for (st = 0; st < starSize*2; st += 2) {
1129:       const PetscInt cell = star[st];
1130:       PetscScalar   *grad = &gradsum[coordDim*numComponents];
1131:       PetscScalar   *x    = NULL;
1132:       PetscReal      vol  = 0.0;

1134:       if ((cell < cStart) || (cell >= cEnd)) continue;
1135:       DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, J, invJ, detJ);
1136:       DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x);
1137:       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1138:         PetscObject  obj;
1139:         PetscClassId id;
1140:         PetscInt     Nb, Nc, q, qc = 0;

1142:         PetscMemzero(grad, coordDim*numComponents * sizeof(PetscScalar));
1143:         DMGetField(dm, field, NULL, &obj);
1144:         PetscObjectGetClassId(obj, &id);
1145:         if (id == PETSCFE_CLASSID)      {PetscFEGetNumComponents((PetscFE) obj, &Nc);PetscFEGetDimension((PetscFE) obj, &Nb);}
1146:         else if (id == PETSCFV_CLASSID) {PetscFVGetNumComponents((PetscFV) obj, &Nc);Nb = 1;}
1147:         else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
1148:         for (q = 0; q < Nq; ++q) {
1149:           if (detJ[q] <= 0.0) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %D, quadrature points %D", (double)detJ[q], cell, q);
1150:           if (ierr) {
1151:             PetscErrorCode ierr2;
1152:             ierr2 = DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x);CHKERRQ(ierr2);
1153:             ierr2 = DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr2);
1154:             ierr2 = PetscFree4(interpolant,coords,detJ,J);CHKERRQ(ierr2);
1155: 
1156:           }
1157:           if (id == PETSCFE_CLASSID)      {PetscFEInterpolateGradient_Static((PetscFE) obj, &x[fieldOffset], coordDim, invJ, NULL, q, interpolant);}
1158:           else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", field);
1159:           for (fc = 0; fc < Nc; ++fc) {
1160:             const PetscReal wt = quadWeights[q*qNc+qc+fc];

1162:             for (d = 0; d < coordDim; ++d) grad[fc*coordDim+d] += interpolant[fc*dim+d]*wt*detJ[q];
1163:           }
1164:           vol += quadWeights[q*qNc]*detJ[q];
1165:         }
1166:         fieldOffset += Nb;
1167:         qc          += Nc;
1168:       }
1169:       DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x);
1170:       for (fc = 0; fc < numComponents; ++fc) {
1171:         for (d = 0; d < coordDim; ++d) {
1172:           gradsum[fc*coordDim+d] += grad[fc*coordDim+d];
1173:         }
1174:       }
1175:       volsum += vol;
1176:       if (debug) {
1177:         PetscPrintf(PETSC_COMM_SELF, "Cell %D gradient: [", cell);
1178:         for (fc = 0; fc < numComponents; ++fc) {
1179:           for (d = 0; d < coordDim; ++d) {
1180:             if (fc || d > 0) {PetscPrintf(PETSC_COMM_SELF, ", ");}
1181:             PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc*coordDim+d]));
1182:           }
1183:         }
1184:         PetscPrintf(PETSC_COMM_SELF, "]\n");
1185:       }
1186:     }
1187:     for (fc = 0; fc < numComponents; ++fc) {
1188:       for (d = 0; d < coordDim; ++d) gradsum[fc*coordDim+d] /= volsum;
1189:     }
1190:     DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star);
1191:     DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES);
1192:   }
1193:   PetscFree6(gradsum,interpolant,coords,detJ,J,invJ);
1194:   return(0);
1195: }

1197: static PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec X, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user)
1198: {
1199:   DM                 dmAux = NULL;
1200:   PetscDS            prob,    probAux = NULL;
1201:   PetscSection       section, sectionAux;
1202:   Vec                locX,    locA;
1203:   PetscInt           dim, numCells = cEnd - cStart, c, f;
1204:   PetscBool          useFVM = PETSC_FALSE;
1205:   /* DS */
1206:   PetscInt           Nf,    totDim,    *uOff, *uOff_x, numConstants;
1207:   PetscInt           NfAux, totDimAux, *aOff;
1208:   PetscScalar       *u, *a;
1209:   const PetscScalar *constants;
1210:   /* Geometry */
1211:   PetscFEGeom       *cgeomFEM;
1212:   DM                 dmGrad;
1213:   PetscQuadrature    affineQuad = NULL;
1214:   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
1215:   PetscFVCellGeom   *cgeomFVM;
1216:   const PetscScalar *lgrad;
1217:   PetscInt           maxDegree;
1218:   DMField            coordField;
1219:   IS                 cellIS;
1220:   PetscErrorCode     ierr;

1223:   DMGetDS(dm, &prob);
1224:   DMGetDimension(dm, &dim);
1225:   DMGetSection(dm, &section);
1226:   PetscSectionGetNumFields(section, &Nf);
1227:   /* Determine which discretizations we have */
1228:   for (f = 0; f < Nf; ++f) {
1229:     PetscObject  obj;
1230:     PetscClassId id;

1232:     PetscDSGetDiscretization(prob, f, &obj);
1233:     PetscObjectGetClassId(obj, &id);
1234:     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
1235:   }
1236:   /* Get local solution with boundary values */
1237:   DMGetLocalVector(dm, &locX);
1238:   DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL);
1239:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX);
1240:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX);
1241:   /* Read DS information */
1242:   PetscDSGetTotalDimension(prob, &totDim);
1243:   PetscDSGetComponentOffsets(prob, &uOff);
1244:   PetscDSGetComponentDerivativeOffsets(prob, &uOff_x);
1245:   ISCreateStride(PETSC_COMM_SELF,numCells,cStart,1,&cellIS);
1246:   PetscDSGetConstants(prob, &numConstants, &constants);
1247:   /* Read Auxiliary DS information */
1248:   PetscObjectQuery((PetscObject) dm, "dmAux", (PetscObject *) &dmAux);
1249:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &locA);
1250:   if (dmAux) {
1251:     DMGetDS(dmAux, &probAux);
1252:     PetscDSGetNumFields(probAux, &NfAux);
1253:     DMGetSection(dmAux, &sectionAux);
1254:     PetscDSGetTotalDimension(probAux, &totDimAux);
1255:     PetscDSGetComponentOffsets(probAux, &aOff);
1256:   }
1257:   /* Allocate data  arrays */
1258:   PetscCalloc1(numCells*totDim, &u);
1259:   if (dmAux) {PetscMalloc1(numCells*totDimAux, &a);}
1260:   /* Read out geometry */
1261:   DMGetCoordinateField(dm,&coordField);
1262:   DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);
1263:   if (maxDegree <= 1) {
1264:     DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad);
1265:     if (affineQuad) {
1266:       DMFieldCreateFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&cgeomFEM);
1267:     }
1268:   }
1269:   if (useFVM) {
1270:     PetscFV   fv = NULL;
1271:     Vec       grad;
1272:     PetscInt  fStart, fEnd;
1273:     PetscBool compGrad;

1275:     for (f = 0; f < Nf; ++f) {
1276:       PetscObject  obj;
1277:       PetscClassId id;

1279:       PetscDSGetDiscretization(prob, f, &obj);
1280:       PetscObjectGetClassId(obj, &id);
1281:       if (id == PETSCFV_CLASSID) {fv = (PetscFV) obj; break;}
1282:     }
1283:     PetscFVGetComputeGradients(fv, &compGrad);
1284:     PetscFVSetComputeGradients(fv, PETSC_TRUE);
1285:     DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM);
1286:     DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad);
1287:     PetscFVSetComputeGradients(fv, compGrad);
1288:     VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM);
1289:     /* Reconstruct and limit cell gradients */
1290:     DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
1291:     DMGetGlobalVector(dmGrad, &grad);
1292:     DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad);
1293:     /* Communicate gradient values */
1294:     DMGetLocalVector(dmGrad, &locGrad);
1295:     DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad);
1296:     DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad);
1297:     DMRestoreGlobalVector(dmGrad, &grad);
1298:     /* Handle non-essential (e.g. outflow) boundary values */
1299:     DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad);
1300:     VecGetArrayRead(locGrad, &lgrad);
1301:   }
1302:   /* Read out data from inputs */
1303:   for (c = cStart; c < cEnd; ++c) {
1304:     PetscScalar *x = NULL;
1305:     PetscInt     i;

1307:     DMPlexVecGetClosure(dm, section, locX, c, NULL, &x);
1308:     for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
1309:     DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x);
1310:     if (dmAux) {
1311:       DMPlexVecGetClosure(dmAux, sectionAux, locA, c, NULL, &x);
1312:       for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
1313:       DMPlexVecRestoreClosure(dmAux, sectionAux, locA, c, NULL, &x);
1314:     }
1315:   }
1316:   /* Do integration for each field */
1317:   for (f = 0; f < Nf; ++f) {
1318:     PetscObject  obj;
1319:     PetscClassId id;
1320:     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;

1322:     PetscDSGetDiscretization(prob, f, &obj);
1323:     PetscObjectGetClassId(obj, &id);
1324:     if (id == PETSCFE_CLASSID) {
1325:       PetscFE         fe = (PetscFE) obj;
1326:       PetscQuadrature q;
1327:       PetscFEGeom     *chunkGeom = NULL;
1328:       PetscInt        Nq, Nb;

1330:       PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
1331:       PetscFEGetQuadrature(fe, &q);
1332:       PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL);
1333:       PetscFEGetDimension(fe, &Nb);
1334:       blockSize = Nb*Nq;
1335:       batchSize = numBlocks * blockSize;
1336:       PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
1337:       numChunks = numCells / (numBatches*batchSize);
1338:       Ne        = numChunks*numBatches*batchSize;
1339:       Nr        = numCells % (numBatches*batchSize);
1340:       offset    = numCells - Nr;
1341:       if (!affineQuad) {
1342:         DMFieldCreateFEGeom(coordField,cellIS,q,PETSC_FALSE,&cgeomFEM);
1343:       }
1344:       PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);
1345:       PetscFEIntegrate(fe, prob, f, Ne, chunkGeom, u, probAux, a, cintegral);
1346:       PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&chunkGeom);
1347:       PetscFEIntegrate(fe, prob, f, Nr, chunkGeom, &u[offset*totDim], probAux, &a[offset*totDimAux], &cintegral[offset*Nf]);
1348:       PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&chunkGeom);
1349:       if (!affineQuad) {
1350:         PetscFEGeomDestroy(&cgeomFEM);
1351:       }
1352:     } else if (id == PETSCFV_CLASSID) {
1353:       PetscInt       foff;
1354:       PetscPointFunc obj_func;
1355:       PetscScalar    lint;

1357:       PetscDSGetObjective(prob, f, &obj_func);
1358:       PetscDSGetFieldOffset(prob, f, &foff);
1359:       if (obj_func) {
1360:         for (c = 0; c < numCells; ++c) {
1361:           PetscScalar *u_x;

1363:           DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x);
1364:           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim*c+foff], NULL, u_x, aOff, NULL, &a[totDimAux*c], NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
1365:           cintegral[c*Nf+f] += PetscRealPart(lint)*cgeomFVM[c].volume;
1366:         }
1367:       }
1368:     } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", f);
1369:   }
1370:   /* Cleanup data arrays */
1371:   if (useFVM) {
1372:     VecRestoreArrayRead(locGrad, &lgrad);
1373:     VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM);
1374:     DMRestoreLocalVector(dmGrad, &locGrad);
1375:     VecDestroy(&faceGeometryFVM);
1376:     VecDestroy(&cellGeometryFVM);
1377:     DMDestroy(&dmGrad);
1378:   }
1379:   if (dmAux) {PetscFree(a);}
1380:   PetscFree(u);
1381:   /* Cleanup */
1382:   if (affineQuad) {
1383:     PetscFEGeomDestroy(&cgeomFEM);
1384:   }
1385:   PetscQuadratureDestroy(&affineQuad);
1386:   ISDestroy(&cellIS);
1387:   DMRestoreLocalVector(dm, &locX);
1388:   return(0);
1389: }

1391: /*@
1392:   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user

1394:   Input Parameters:
1395: + dm - The mesh
1396: . X  - Global input vector
1397: - user - The user context

1399:   Output Parameter:
1400: . integral - Integral for each field

1402:   Level: developer

1404: .seealso: DMPlexComputeResidualFEM()
1405: @*/
1406: PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user)
1407: {
1408:   DM_Plex       *mesh = (DM_Plex *) dm->data;
1409:   PetscScalar   *cintegral, *lintegral;
1410:   PetscInt       Nf, f, cellHeight, cStart, cEnd, cEndInterior[4], cell;

1417:   PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0);
1418:   DMGetNumFields(dm, &Nf);
1419:   DMPlexGetVTKCellHeight(dm, &cellHeight);
1420:   DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);
1421:   DMPlexGetHybridBounds(dm, &cEndInterior[0], &cEndInterior[1], &cEndInterior[2], &cEndInterior[3]);
1422:   cEnd = cEndInterior[cellHeight] < 0 ? cEnd : cEndInterior[cellHeight];
1423:   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
1424:   PetscCalloc2(Nf, &lintegral, (cEnd-cStart)*Nf, &cintegral);
1425:   DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user);
1426:   /* Sum up values */
1427:   for (cell = cStart; cell < cEnd; ++cell) {
1428:     const PetscInt c = cell - cStart;

1430:     if (mesh->printFEM > 1) {DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]);}
1431:     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c*Nf+f];
1432:   }
1433:   MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject) dm));
1434:   if (mesh->printFEM) {
1435:     PetscPrintf(PetscObjectComm((PetscObject) dm), "Integral:");
1436:     for (f = 0; f < Nf; ++f) {PetscPrintf(PetscObjectComm((PetscObject) dm), " %g", (double) PetscRealPart(integral[f]));}
1437:     PetscPrintf(PetscObjectComm((PetscObject) dm), "\n");
1438:   }
1439:   PetscFree2(lintegral, cintegral);
1440:   PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0);
1441:   return(0);
1442: }

1444: /*@
1445:   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user

1447:   Input Parameters:
1448: + dm - The mesh
1449: . X  - Global input vector
1450: - user - The user context

1452:   Output Parameter:
1453: . integral - Cellwise integrals for each field

1455:   Level: developer

1457: .seealso: DMPlexComputeResidualFEM()
1458: @*/
1459: PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user)
1460: {
1461:   DM_Plex       *mesh = (DM_Plex *) dm->data;
1462:   DM             dmF;
1463:   PetscSection   sectionF;
1464:   PetscScalar   *cintegral, *af;
1465:   PetscInt       Nf, f, cellHeight, cStart, cEnd, cEndInterior[4], cell;

1472:   PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0);
1473:   DMGetNumFields(dm, &Nf);
1474:   DMPlexGetVTKCellHeight(dm, &cellHeight);
1475:   DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);
1476:   DMPlexGetHybridBounds(dm, &cEndInterior[0], &cEndInterior[1], &cEndInterior[2], &cEndInterior[3]);
1477:   cEnd = cEndInterior[cellHeight] < 0 ? cEnd : cEndInterior[cellHeight];
1478:   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
1479:   PetscCalloc1((cEnd-cStart)*Nf, &cintegral);
1480:   DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user);
1481:   /* Put values in F*/
1482:   VecGetDM(F, &dmF);
1483:   DMGetSection(dmF, &sectionF);
1484:   VecGetArray(F, &af);
1485:   for (cell = cStart; cell < cEnd; ++cell) {
1486:     const PetscInt c = cell - cStart;
1487:     PetscInt       dof, off;

1489:     if (mesh->printFEM > 1) {DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]);}
1490:     PetscSectionGetDof(sectionF, cell, &dof);
1491:     PetscSectionGetOffset(sectionF, cell, &off);
1492:     if (dof != Nf) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %D != %D", dof, Nf);
1493:     for (f = 0; f < Nf; ++f) af[off+f] = cintegral[c*Nf+f];
1494:   }
1495:   VecRestoreArray(F, &af);
1496:   PetscFree(cintegral);
1497:   PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0);
1498:   return(0);
1499: }

1501: static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS,
1502:                                                        void (*func)(PetscInt, PetscInt, PetscInt,
1503:                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
1504:                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
1505:                                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
1506:                                                        PetscScalar *fintegral, void *user)
1507: {
1508:   DM                 plex = NULL, plexA = NULL;
1509:   PetscDS            prob, probAux = NULL;
1510:   PetscSection       section, sectionAux = NULL;
1511:   Vec                locA = NULL;
1512:   DMField            coordField;
1513:   PetscInt           Nf,        totDim,        *uOff, *uOff_x;
1514:   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
1515:   PetscScalar       *u, *a = NULL;
1516:   const PetscScalar *constants;
1517:   PetscInt           numConstants, f;
1518:   PetscErrorCode     ierr;

1521:   DMGetCoordinateField(dm, &coordField);
1522:   DMConvert(dm, DMPLEX, &plex);
1523:   DMGetDS(dm, &prob);
1524:   DMGetDefaultSection(dm, &section);
1525:   PetscSectionGetNumFields(section, &Nf);
1526:   /* Determine which discretizations we have */
1527:   for (f = 0; f < Nf; ++f) {
1528:     PetscObject  obj;
1529:     PetscClassId id;

1531:     PetscDSGetDiscretization(prob, f, &obj);
1532:     PetscObjectGetClassId(obj, &id);
1533:     if (id == PETSCFV_CLASSID) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Not supported for FVM (field %D)", f);
1534:   }
1535:   /* Read DS information */
1536:   PetscDSGetTotalDimension(prob, &totDim);
1537:   PetscDSGetComponentOffsets(prob, &uOff);
1538:   PetscDSGetComponentDerivativeOffsets(prob, &uOff_x);
1539:   PetscDSGetConstants(prob, &numConstants, &constants);
1540:   /* Read Auxiliary DS information */
1541:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &locA);
1542:   if (locA) {
1543:     DM dmAux;

1545:     VecGetDM(locA, &dmAux);
1546:     DMConvert(dmAux, DMPLEX, &plexA);
1547:     DMGetDS(dmAux, &probAux);
1548:     PetscDSGetNumFields(probAux, &NfAux);
1549:     DMGetDefaultSection(dmAux, &sectionAux);
1550:     PetscDSGetTotalDimension(probAux, &totDimAux);
1551:     PetscDSGetComponentOffsets(probAux, &aOff);
1552:   }
1553:   /* Integrate over points */
1554:   {
1555:     PetscFEGeom    *fgeom, *chunkGeom = NULL;
1556:     PetscInt        maxDegree;
1557:     PetscQuadrature qGeom = NULL;
1558:     const PetscInt *points;
1559:     PetscInt        numFaces, face, Nq, field;
1560:     PetscInt        numChunks, chunkSize, chunk, Nr, offset;

1562:     ISGetLocalSize(pointIS, &numFaces);
1563:     ISGetIndices(pointIS, &points);
1564:     PetscCalloc2(numFaces*totDim, &u, locA ? numFaces*totDimAux : 0, &a);
1565:     DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree);
1566:     for (field = 0; field < Nf; ++field) {
1567:       PetscFE fe;

1569:       PetscDSGetDiscretization(prob, field, (PetscObject *) &fe);
1570:       if (maxDegree <= 1) {DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom);}
1571:       if (!qGeom) {
1572:         PetscFEGetFaceQuadrature(fe, &qGeom);
1573:         PetscObjectReference((PetscObject) qGeom);
1574:       }
1575:       PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);
1576:       DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom);
1577:       for (face = 0; face < numFaces; ++face) {
1578:         const PetscInt point = points[face], *support, *cone;
1579:         PetscScalar    *x    = NULL;
1580:         PetscInt       i, coneSize, faceLoc;

1582:         DMPlexGetSupport(dm, point, &support);
1583:         DMPlexGetConeSize(dm, support[0], &coneSize);
1584:         DMPlexGetCone(dm, support[0], &cone);
1585:         for (faceLoc = 0; faceLoc < coneSize; ++faceLoc) if (cone[faceLoc] == point) break;
1586:         if (faceLoc == coneSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %D in cone of support[0] %D", face, support[0]);
1587:         fgeom->face[face][0] = faceLoc;
1588:         DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x);
1589:         for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
1590:         DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x);
1591:         if (locA) {
1592:           PetscInt subp;
1593:           DMPlexGetSubpoint(plexA, support[0], &subp);
1594:           DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x);
1595:           for (i = 0; i < totDimAux; ++i) a[f*totDimAux+i] = x[i];
1596:           DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x);
1597:         }
1598:       }
1599:       /* Get blocking */
1600:       {
1601:         PetscQuadrature q;
1602:         PetscInt        numBatches, batchSize, numBlocks, blockSize;
1603:         PetscInt        Nq, Nb;

1605:         PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
1606:         PetscFEGetQuadrature(fe, &q);
1607:         PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL);
1608:         PetscFEGetDimension(fe, &Nb);
1609:         blockSize = Nb*Nq;
1610:         batchSize = numBlocks * blockSize;
1611:         chunkSize = numBatches*batchSize;
1612:         PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
1613:         numChunks = numFaces / chunkSize;
1614:         Nr        = numFaces % chunkSize;
1615:         offset    = numFaces - Nr;
1616:       }
1617:       /* Do integration for each field */
1618:       for (chunk = 0; chunk < numChunks; ++chunk) {
1619:         PetscFEGeomGetChunk(fgeom, chunk*chunkSize, (chunk+1)*chunkSize, &chunkGeom);
1620:         PetscFEIntegrateBd(fe, prob, field, func, chunkSize, chunkGeom, u, probAux, a, fintegral);
1621:         PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom);
1622:       }
1623:       PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom);
1624:       PetscFEIntegrateBd(fe, prob, field, func, Nr, chunkGeom, &u[offset*totDim], probAux, a ? &a[offset*totDimAux] : NULL, &fintegral[offset*Nf]);
1625:       PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom);
1626:       /* Cleanup data arrays */
1627:       DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom);
1628:       PetscQuadratureDestroy(&qGeom);
1629:       PetscFree2(u, a);
1630:       ISRestoreIndices(pointIS, &points);
1631:     }
1632:   }
1633:   if (plex)  {DMDestroy(&plex);}
1634:   if (plexA) {DMDestroy(&plexA);}
1635:   return(0);
1636: }

1638: /*@
1639:   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user

1641:   Input Parameters:
1642: + dm      - The mesh
1643: . X       - Global input vector
1644: . label   - The boundary DMLabel
1645: . numVals - The number of label values to use, or PETSC_DETERMINE for all values
1646: . vals    - The label values to use, or PETSC_NULL for all values
1647: . func    = The function to integrate along the boundary
1648: - user    - The user context

1650:   Output Parameter:
1651: . integral - Integral for each field

1653:   Level: developer

1655: .seealso: DMPlexComputeIntegralFEM(), DMPlexComputeBdResidualFEM()
1656: @*/
1657: PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[],
1658:                                        void (*func)(PetscInt, PetscInt, PetscInt,
1659:                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
1660:                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
1661:                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
1662:                                        PetscScalar *integral, void *user)
1663: {
1664:   Vec            locX;
1665:   PetscSection   section;
1666:   DMLabel        depthLabel;
1667:   IS             facetIS;
1668:   PetscInt       dim, Nf, f, v;

1677:   PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0);
1678:   DMPlexGetDepthLabel(dm, &depthLabel);
1679:   DMGetDimension(dm, &dim);
1680:   DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);
1681:   DMGetDefaultSection(dm, &section);
1682:   PetscSectionGetNumFields(section, &Nf);
1683:   /* Get local solution with boundary values */
1684:   DMGetLocalVector(dm, &locX);
1685:   DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL);
1686:   DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX);
1687:   DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX);
1688:   /* Loop over label values */
1689:   PetscMemzero(integral, Nf * sizeof(PetscScalar));
1690:   for (v = 0; v < numVals; ++v) {
1691:     IS           pointIS;
1692:     PetscInt     numFaces, face;
1693:     PetscScalar *fintegral;

1695:     DMLabelGetStratumIS(label, vals[v], &pointIS);
1696:     if (!pointIS) continue; /* No points with that id on this process */
1697:     {
1698:       IS isectIS;

1700:       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
1701:       ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS);
1702:       ISDestroy(&pointIS);
1703:       pointIS = isectIS;
1704:     }
1705:     ISGetLocalSize(pointIS, &numFaces);
1706:     PetscCalloc1(numFaces*Nf, &fintegral);
1707:     DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, func, fintegral, user);
1708:     /* Sum point contributions into integral */
1709:     for (f = 0; f < Nf; ++f) for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face*Nf+f];
1710:     PetscFree(fintegral);
1711:     ISDestroy(&pointIS);
1712:   }
1713:   DMRestoreLocalVector(dm, &locX);
1714:   ISDestroy(&facetIS);
1715:   PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0);
1716:   return(0);
1717: }

1719: /*@
1720:   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix I from the coarse DM to the uniformly refined DM.

1722:   Input Parameters:
1723: + dmf  - The fine mesh
1724: . dmc  - The coarse mesh
1725: - user - The user context

1727:   Output Parameter:
1728: . In  - The interpolation matrix

1730:   Level: developer

1732: .seealso: DMPlexComputeInterpolatorGeneral(), DMPlexComputeJacobianFEM()
1733: @*/
1734: PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, Mat In, void *user)
1735: {
1736:   DM_Plex          *mesh  = (DM_Plex *) dmc->data;
1737:   const char       *name  = "Interpolator";
1738:   PetscDS           prob;
1739:   PetscFE          *feRef;
1740:   PetscFV          *fvRef;
1741:   PetscSection      fsection, fglobalSection;
1742:   PetscSection      csection, cglobalSection;
1743:   PetscScalar      *elemMat;
1744:   PetscInt          dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, cEndInterior, c;
1745:   PetscInt          cTotDim, rTotDim = 0;
1746:   PetscErrorCode    ierr;

1749:   PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);
1750:   DMGetDimension(dmf, &dim);
1751:   DMGetSection(dmf, &fsection);
1752:   DMGetGlobalSection(dmf, &fglobalSection);
1753:   DMGetSection(dmc, &csection);
1754:   DMGetGlobalSection(dmc, &cglobalSection);
1755:   PetscSectionGetNumFields(fsection, &Nf);
1756:   DMPlexGetHeightStratum(dmc, 0, &cStart, &cEnd);
1757:   DMPlexGetHybridBounds(dmc, &cEndInterior, NULL, NULL, NULL);
1758:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
1759:   DMGetDS(dmf, &prob);
1760:   PetscCalloc2(Nf,&feRef,Nf,&fvRef);
1761:   for (f = 0; f < Nf; ++f) {
1762:     PetscObject  obj;
1763:     PetscClassId id;
1764:     PetscInt     rNb = 0, Nc = 0;

1766:     PetscDSGetDiscretization(prob, f, &obj);
1767:     PetscObjectGetClassId(obj, &id);
1768:     if (id == PETSCFE_CLASSID) {
1769:       PetscFE fe = (PetscFE) obj;

1771:       PetscFERefine(fe, &feRef[f]);
1772:       PetscFEGetDimension(feRef[f], &rNb);
1773:       PetscFEGetNumComponents(fe, &Nc);
1774:     } else if (id == PETSCFV_CLASSID) {
1775:       PetscFV        fv = (PetscFV) obj;
1776:       PetscDualSpace Q;

1778:       PetscFVRefine(fv, &fvRef[f]);
1779:       PetscFVGetDualSpace(fvRef[f], &Q);
1780:       PetscDualSpaceGetDimension(Q, &rNb);
1781:       PetscFVGetNumComponents(fv, &Nc);
1782:     }
1783:     rTotDim += rNb;
1784:   }
1785:   PetscDSGetTotalDimension(prob, &cTotDim);
1786:   PetscMalloc1(rTotDim*cTotDim,&elemMat);
1787:   PetscMemzero(elemMat, rTotDim*cTotDim * sizeof(PetscScalar));
1788:   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
1789:     PetscDualSpace   Qref;
1790:     PetscQuadrature  f;
1791:     const PetscReal *qpoints, *qweights;
1792:     PetscReal       *points;
1793:     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;

1795:     /* Compose points from all dual basis functionals */
1796:     if (feRef[fieldI]) {
1797:       PetscFEGetDualSpace(feRef[fieldI], &Qref);
1798:       PetscFEGetNumComponents(feRef[fieldI], &Nc);
1799:     } else {
1800:       PetscFVGetDualSpace(fvRef[fieldI], &Qref);
1801:       PetscFVGetNumComponents(fvRef[fieldI], &Nc);
1802:     }
1803:     PetscDualSpaceGetDimension(Qref, &fpdim);
1804:     for (i = 0; i < fpdim; ++i) {
1805:       PetscDualSpaceGetFunctional(Qref, i, &f);
1806:       PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL);
1807:       npoints += Np;
1808:     }
1809:     PetscMalloc1(npoints*dim,&points);
1810:     for (i = 0, k = 0; i < fpdim; ++i) {
1811:       PetscDualSpaceGetFunctional(Qref, i, &f);
1812:       PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL);
1813:       for (p = 0; p < Np; ++p, ++k) for (d = 0; d < dim; ++d) points[k*dim+d] = qpoints[p*dim+d];
1814:     }

1816:     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
1817:       PetscObject  obj;
1818:       PetscClassId id;
1819:       PetscReal   *B;
1820:       PetscInt     NcJ = 0, cpdim = 0, j, qNc;

1822:       PetscDSGetDiscretization(prob, fieldJ, &obj);
1823:       PetscObjectGetClassId(obj, &id);
1824:       if (id == PETSCFE_CLASSID) {
1825:         PetscFE fe = (PetscFE) obj;

1827:         /* Evaluate basis at points */
1828:         PetscFEGetNumComponents(fe, &NcJ);
1829:         PetscFEGetDimension(fe, &cpdim);
1830:         /* For now, fields only interpolate themselves */
1831:         if (fieldI == fieldJ) {
1832:           if (Nc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %D does not match coarse field %D", Nc, NcJ);
1833:           PetscFEGetTabulation(fe, npoints, points, &B, NULL, NULL);
1834:           for (i = 0, k = 0; i < fpdim; ++i) {
1835:             PetscDualSpaceGetFunctional(Qref, i, &f);
1836:             PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights);
1837:             if (qNc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %D does not match coarse field %D", qNc, NcJ);
1838:             for (p = 0; p < Np; ++p, ++k) {
1839:               for (j = 0; j < cpdim; ++j) {
1840:                 /*
1841:                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
1842:                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
1843:                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
1844:                    qNC, Nc, Ncj, c:    Number of components in this field
1845:                    Np, p:              Number of quad points in the fine grid functional i
1846:                    k:                  i*Np + p, overall point number for the interpolation
1847:                 */
1848:                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += B[k*cpdim*NcJ+j*Nc+c]*qweights[p*qNc+c];
1849:               }
1850:             }
1851:           }
1852:           PetscFERestoreTabulation(fe, npoints, points, &B, NULL, NULL);
1853:         }
1854:       } else if (id == PETSCFV_CLASSID) {
1855:         PetscFV        fv = (PetscFV) obj;

1857:         /* Evaluate constant function at points */
1858:         PetscFVGetNumComponents(fv, &NcJ);
1859:         cpdim = 1;
1860:         /* For now, fields only interpolate themselves */
1861:         if (fieldI == fieldJ) {
1862:           if (Nc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %d does not match coarse field %d", Nc, NcJ);
1863:           for (i = 0, k = 0; i < fpdim; ++i) {
1864:             PetscDualSpaceGetFunctional(Qref, i, &f);
1865:             PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights);
1866:             if (qNc != NcJ) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %D does not match coarse field %D", qNc, NcJ);
1867:             for (p = 0; p < Np; ++p, ++k) {
1868:               for (j = 0; j < cpdim; ++j) {
1869:                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += 1.0*qweights[p*qNc+c];
1870:               }
1871:             }
1872:           }
1873:         }
1874:       }
1875:       offsetJ += cpdim;
1876:     }
1877:     offsetI += fpdim;
1878:     PetscFree(points);
1879:   }
1880:   if (mesh->printFEM > 1) {DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat);}
1881:   /* Preallocate matrix */
1882:   {
1883:     Mat          preallocator;
1884:     PetscScalar *vals;
1885:     PetscInt    *cellCIndices, *cellFIndices;
1886:     PetscInt     locRows, locCols, cell;

1888:     MatGetLocalSize(In, &locRows, &locCols);
1889:     MatCreate(PetscObjectComm((PetscObject) In), &preallocator);
1890:     MatSetType(preallocator, MATPREALLOCATOR);
1891:     MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE);
1892:     MatSetUp(preallocator);
1893:     PetscCalloc3(rTotDim*cTotDim, &vals,cTotDim,&cellCIndices,rTotDim,&cellFIndices);
1894:     for (cell = cStart; cell < cEnd; ++cell) {
1895:       DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices);
1896:       MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES);
1897:     }
1898:     PetscFree3(vals,cellCIndices,cellFIndices);
1899:     MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY);
1900:     MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY);
1901:     MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In);
1902:     MatDestroy(&preallocator);
1903:   }
1904:   /* Fill matrix */
1905:   MatZeroEntries(In);
1906:   for (c = cStart; c < cEnd; ++c) {
1907:     DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES);
1908:   }
1909:   for (f = 0; f < Nf; ++f) {PetscFEDestroy(&feRef[f]);}
1910:   PetscFree2(feRef,fvRef);
1911:   PetscFree(elemMat);
1912:   MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY);
1913:   MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY);
1914:   if (mesh->printFEM) {
1915:     PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name);
1916:     MatChop(In, 1.0e-10);
1917:     MatView(In, NULL);
1918:   }
1919:   PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);
1920:   return(0);
1921: }

1923: PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user)
1924: {
1925:   SETERRQ(PetscObjectComm((PetscObject) dmc), PETSC_ERR_SUP, "Laziness");
1926: }

1928: /*@
1929:   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix I from the coarse DM to a non-nested fine DM.

1931:   Input Parameters:
1932: + dmf  - The fine mesh
1933: . dmc  - The coarse mesh
1934: - user - The user context

1936:   Output Parameter:
1937: . In  - The interpolation matrix

1939:   Level: developer

1941: .seealso: DMPlexComputeInterpolatorNested(), DMPlexComputeJacobianFEM()
1942: @*/
1943: PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user)
1944: {
1945:   DM_Plex       *mesh = (DM_Plex *) dmf->data;
1946:   const char    *name = "Interpolator";
1947:   PetscDS        prob;
1948:   PetscSection   fsection, csection, globalFSection, globalCSection;
1949:   PetscHSetIJ    ht;
1950:   PetscLayout    rLayout;
1951:   PetscInt      *dnz, *onz;
1952:   PetscInt       locRows, rStart, rEnd;
1953:   PetscReal     *x, *v0, *J, *invJ, detJ;
1954:   PetscReal     *v0c, *Jc, *invJc, detJc;
1955:   PetscScalar   *elemMat;
1956:   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;

1960:   PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);
1961:   DMGetCoordinateDim(dmc, &dim);
1962:   DMGetDS(dmc, &prob);
1963:   PetscDSGetRefCoordArrays(prob, &x, NULL);
1964:   PetscDSGetNumFields(prob, &Nf);
1965:   PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ);
1966:   PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc);
1967:   DMGetSection(dmf, &fsection);
1968:   DMGetGlobalSection(dmf, &globalFSection);
1969:   DMGetSection(dmc, &csection);
1970:   DMGetGlobalSection(dmc, &globalCSection);
1971:   DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd);
1972:   PetscDSGetTotalDimension(prob, &totDim);
1973:   PetscMalloc1(totDim, &elemMat);

1975:   MatGetLocalSize(In, &locRows, NULL);
1976:   PetscLayoutCreate(PetscObjectComm((PetscObject) In), &rLayout);
1977:   PetscLayoutSetLocalSize(rLayout, locRows);
1978:   PetscLayoutSetBlockSize(rLayout, 1);
1979:   PetscLayoutSetUp(rLayout);
1980:   PetscLayoutGetRange(rLayout, &rStart, &rEnd);
1981:   PetscLayoutDestroy(&rLayout);
1982:   PetscCalloc2(locRows,&dnz,locRows,&onz);
1983:   PetscHSetIJCreate(&ht);
1984:   for (field = 0; field < Nf; ++field) {
1985:     PetscObject      obj;
1986:     PetscClassId     id;
1987:     PetscDualSpace   Q = NULL;
1988:     PetscQuadrature  f;
1989:     const PetscReal *qpoints;
1990:     PetscInt         Nc, Np, fpdim, i, d;

1992:     PetscDSGetDiscretization(prob, field, &obj);
1993:     PetscObjectGetClassId(obj, &id);
1994:     if (id == PETSCFE_CLASSID) {
1995:       PetscFE fe = (PetscFE) obj;

1997:       PetscFEGetDualSpace(fe, &Q);
1998:       PetscFEGetNumComponents(fe, &Nc);
1999:     } else if (id == PETSCFV_CLASSID) {
2000:       PetscFV fv = (PetscFV) obj;

2002:       PetscFVGetDualSpace(fv, &Q);
2003:       Nc   = 1;
2004:     }
2005:     PetscDualSpaceGetDimension(Q, &fpdim);
2006:     /* For each fine grid cell */
2007:     for (cell = cStart; cell < cEnd; ++cell) {
2008:       PetscInt *findices,   *cindices;
2009:       PetscInt  numFIndices, numCIndices;

2011:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2012:       DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);
2013:       if (numFIndices != fpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %d != %d dual basis vecs", numFIndices, fpdim);
2014:       for (i = 0; i < fpdim; ++i) {
2015:         Vec             pointVec;
2016:         PetscScalar    *pV;
2017:         PetscSF         coarseCellSF = NULL;
2018:         const PetscSFNode *coarseCells;
2019:         PetscInt        numCoarseCells, q, c;

2021:         /* Get points from the dual basis functional quadrature */
2022:         PetscDualSpaceGetFunctional(Q, i, &f);
2023:         PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL);
2024:         VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec);
2025:         VecSetBlockSize(pointVec, dim);
2026:         VecGetArray(pointVec, &pV);
2027:         for (q = 0; q < Np; ++q) {
2028:           const PetscReal xi0[3] = {-1., -1., -1.};

2030:           /* Transform point to real space */
2031:           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
2032:           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
2033:         }
2034:         VecRestoreArray(pointVec, &pV);
2035:         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
2036:         /* OPT: Pack all quad points from fine cell */
2037:         DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);
2038:         PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view");
2039:         /* Update preallocation info */
2040:         PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);
2041:         if (numCoarseCells != Np) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
2042:         {
2043:           PetscHashIJKey key;
2044:           PetscBool      missing;

2046:           key.i = findices[i];
2047:           if (key.i >= 0) {
2048:             /* Get indices for coarse elements */
2049:             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2050:               DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2051:               for (c = 0; c < numCIndices; ++c) {
2052:                 key.j = cindices[c];
2053:                 if (key.j < 0) continue;
2054:                 PetscHSetIJQueryAdd(ht, key, &missing);
2055:                 if (missing) {
2056:                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
2057:                   else                                     ++onz[key.i-rStart];
2058:                 }
2059:               }
2060:               DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2061:             }
2062:           }
2063:         }
2064:         PetscSFDestroy(&coarseCellSF);
2065:         VecDestroy(&pointVec);
2066:       }
2067:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2068:     }
2069:   }
2070:   PetscHSetIJDestroy(&ht);
2071:   MatXAIJSetPreallocation(In, 1, dnz, onz, NULL, NULL);
2072:   MatSetOption(In, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);
2073:   PetscFree2(dnz,onz);
2074:   for (field = 0; field < Nf; ++field) {
2075:     PetscObject      obj;
2076:     PetscClassId     id;
2077:     PetscDualSpace   Q = NULL;
2078:     PetscQuadrature  f;
2079:     const PetscReal *qpoints, *qweights;
2080:     PetscInt         Nc, qNc, Np, fpdim, i, d;

2082:     PetscDSGetDiscretization(prob, field, &obj);
2083:     PetscObjectGetClassId(obj, &id);
2084:     if (id == PETSCFE_CLASSID) {
2085:       PetscFE fe = (PetscFE) obj;

2087:       PetscFEGetDualSpace(fe, &Q);
2088:       PetscFEGetNumComponents(fe, &Nc);
2089:     } else if (id == PETSCFV_CLASSID) {
2090:       PetscFV fv = (PetscFV) obj;

2092:       PetscFVGetDualSpace(fv, &Q);
2093:       Nc   = 1;
2094:     } else SETERRQ1(PetscObjectComm((PetscObject)dmc),PETSC_ERR_ARG_WRONG,"Unknown discretization type for field %d",field);
2095:     PetscDualSpaceGetDimension(Q, &fpdim);
2096:     /* For each fine grid cell */
2097:     for (cell = cStart; cell < cEnd; ++cell) {
2098:       PetscInt *findices,   *cindices;
2099:       PetscInt  numFIndices, numCIndices;

2101:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2102:       DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);
2103:       if (numFIndices != fpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %d != %d dual basis vecs", numFIndices, fpdim);
2104:       for (i = 0; i < fpdim; ++i) {
2105:         Vec             pointVec;
2106:         PetscScalar    *pV;
2107:         PetscSF         coarseCellSF = NULL;
2108:         const PetscSFNode *coarseCells;
2109:         PetscInt        numCoarseCells, cpdim, q, c, j;

2111:         /* Get points from the dual basis functional quadrature */
2112:         PetscDualSpaceGetFunctional(Q, i, &f);
2113:         PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights);
2114:         if (qNc != Nc) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %D does not match coarse field %D", qNc, Nc);
2115:         VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec);
2116:         VecSetBlockSize(pointVec, dim);
2117:         VecGetArray(pointVec, &pV);
2118:         for (q = 0; q < Np; ++q) {
2119:           const PetscReal xi0[3] = {-1., -1., -1.};

2121:           /* Transform point to real space */
2122:           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
2123:           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
2124:         }
2125:         VecRestoreArray(pointVec, &pV);
2126:         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
2127:         /* OPT: Read this out from preallocation information */
2128:         DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);
2129:         /* Update preallocation info */
2130:         PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);
2131:         if (numCoarseCells != Np) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
2132:         VecGetArray(pointVec, &pV);
2133:         for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2134:           PetscReal pVReal[3];
2135:           const PetscReal xi0[3] = {-1., -1., -1.};

2137:           DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2138:           /* Transform points from real space to coarse reference space */
2139:           DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc);
2140:           for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
2141:           CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);

2143:           if (id == PETSCFE_CLASSID) {
2144:             PetscFE    fe = (PetscFE) obj;
2145:             PetscReal *B;

2147:             /* Evaluate coarse basis on contained point */
2148:             PetscFEGetDimension(fe, &cpdim);
2149:             PetscFEGetTabulation(fe, 1, x, &B, NULL, NULL);
2150:             PetscMemzero(elemMat, cpdim * sizeof(PetscScalar));
2151:             /* Get elemMat entries by multiplying by weight */
2152:             for (j = 0; j < cpdim; ++j) {
2153:               for (c = 0; c < Nc; ++c) elemMat[j] += B[j*Nc + c]*qweights[ccell*qNc + c];
2154:             }
2155:             PetscFERestoreTabulation(fe, 1, x, &B, NULL, NULL);
2156:           } else {
2157:             cpdim = 1;
2158:             for (j = 0; j < cpdim; ++j) {
2159:               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*qweights[ccell*qNc + c];
2160:             }
2161:           }
2162:           /* Update interpolator */
2163:           if (mesh->printFEM > 1) {DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);}
2164:           if (numCIndices != cpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %D != %D", numCIndices, cpdim);
2165:           MatSetValues(In, 1, &findices[i], numCIndices, cindices, elemMat, INSERT_VALUES);
2166:           DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2167:         }
2168:         VecRestoreArray(pointVec, &pV);
2169:         PetscSFDestroy(&coarseCellSF);
2170:         VecDestroy(&pointVec);
2171:       }
2172:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2173:     }
2174:   }
2175:   PetscFree3(v0,J,invJ);
2176:   PetscFree3(v0c,Jc,invJc);
2177:   PetscFree(elemMat);
2178:   MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY);
2179:   MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY);
2180:   PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0);
2181:   return(0);
2182: }

2184: /*@
2185:   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix M from the coarse DM to a non-nested fine DM.

2187:   Input Parameters:
2188: + dmf  - The fine mesh
2189: . dmc  - The coarse mesh
2190: - user - The user context

2192:   Output Parameter:
2193: . mass  - The mass matrix

2195:   Level: developer

2197: .seealso: DMPlexComputeMassMatrixNested(), DMPlexComputeInterpolatorNested(), DMPlexComputeInterpolatorGeneral(), DMPlexComputeJacobianFEM()
2198: @*/
2199: PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user)
2200: {
2201:   DM_Plex       *mesh = (DM_Plex *) dmf->data;
2202:   const char    *name = "Mass Matrix";
2203:   PetscDS        prob;
2204:   PetscSection   fsection, csection, globalFSection, globalCSection;
2205:   PetscHSetIJ    ht;
2206:   PetscLayout    rLayout;
2207:   PetscInt      *dnz, *onz;
2208:   PetscInt       locRows, rStart, rEnd;
2209:   PetscReal     *x, *v0, *J, *invJ, detJ;
2210:   PetscReal     *v0c, *Jc, *invJc, detJc;
2211:   PetscScalar   *elemMat;
2212:   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;

2216:   DMGetCoordinateDim(dmc, &dim);
2217:   DMGetDS(dmc, &prob);
2218:   PetscDSGetRefCoordArrays(prob, &x, NULL);
2219:   PetscDSGetNumFields(prob, &Nf);
2220:   PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ);
2221:   PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc);
2222:   DMGetSection(dmf, &fsection);
2223:   DMGetGlobalSection(dmf, &globalFSection);
2224:   DMGetSection(dmc, &csection);
2225:   DMGetGlobalSection(dmc, &globalCSection);
2226:   DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd);
2227:   PetscDSGetTotalDimension(prob, &totDim);
2228:   PetscMalloc1(totDim, &elemMat);

2230:   MatGetLocalSize(mass, &locRows, NULL);
2231:   PetscLayoutCreate(PetscObjectComm((PetscObject) mass), &rLayout);
2232:   PetscLayoutSetLocalSize(rLayout, locRows);
2233:   PetscLayoutSetBlockSize(rLayout, 1);
2234:   PetscLayoutSetUp(rLayout);
2235:   PetscLayoutGetRange(rLayout, &rStart, &rEnd);
2236:   PetscLayoutDestroy(&rLayout);
2237:   PetscCalloc2(locRows,&dnz,locRows,&onz);
2238:   PetscHSetIJCreate(&ht);
2239:   for (field = 0; field < Nf; ++field) {
2240:     PetscObject      obj;
2241:     PetscClassId     id;
2242:     PetscQuadrature  quad;
2243:     const PetscReal *qpoints;
2244:     PetscInt         Nq, Nc, i, d;

2246:     PetscDSGetDiscretization(prob, field, &obj);
2247:     PetscObjectGetClassId(obj, &id);
2248:     if (id == PETSCFE_CLASSID) {PetscFEGetQuadrature((PetscFE) obj, &quad);}
2249:     else                       {PetscFVGetQuadrature((PetscFV) obj, &quad);}
2250:     PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL);
2251:     /* For each fine grid cell */
2252:     for (cell = cStart; cell < cEnd; ++cell) {
2253:       Vec                pointVec;
2254:       PetscScalar       *pV;
2255:       PetscSF            coarseCellSF = NULL;
2256:       const PetscSFNode *coarseCells;
2257:       PetscInt           numCoarseCells, q, c;
2258:       PetscInt          *findices,   *cindices;
2259:       PetscInt           numFIndices, numCIndices;

2261:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2262:       DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);
2263:       /* Get points from the quadrature */
2264:       VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec);
2265:       VecSetBlockSize(pointVec, dim);
2266:       VecGetArray(pointVec, &pV);
2267:       for (q = 0; q < Nq; ++q) {
2268:         const PetscReal xi0[3] = {-1., -1., -1.};

2270:         /* Transform point to real space */
2271:         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
2272:         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
2273:       }
2274:       VecRestoreArray(pointVec, &pV);
2275:       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
2276:       DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);
2277:       PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view");
2278:       /* Update preallocation info */
2279:       PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);
2280:       if (numCoarseCells != Nq) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
2281:       {
2282:         PetscHashIJKey key;
2283:         PetscBool      missing;

2285:         for (i = 0; i < numFIndices; ++i) {
2286:           key.i = findices[i];
2287:           if (key.i >= 0) {
2288:             /* Get indices for coarse elements */
2289:             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2290:               DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2291:               for (c = 0; c < numCIndices; ++c) {
2292:                 key.j = cindices[c];
2293:                 if (key.j < 0) continue;
2294:                 PetscHSetIJQueryAdd(ht, key, &missing);
2295:                 if (missing) {
2296:                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
2297:                   else                                     ++onz[key.i-rStart];
2298:                 }
2299:               }
2300:               DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2301:             }
2302:           }
2303:         }
2304:       }
2305:       PetscSFDestroy(&coarseCellSF);
2306:       VecDestroy(&pointVec);
2307:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2308:     }
2309:   }
2310:   PetscHSetIJDestroy(&ht);
2311:   MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL);
2312:   MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);
2313:   PetscFree2(dnz,onz);
2314:   for (field = 0; field < Nf; ++field) {
2315:     PetscObject      obj;
2316:     PetscClassId     id;
2317:     PetscQuadrature  quad;
2318:     PetscReal       *Bfine;
2319:     const PetscReal *qpoints, *qweights;
2320:     PetscInt         Nq, Nc, i, d;

2322:     PetscDSGetDiscretization(prob, field, &obj);
2323:     PetscObjectGetClassId(obj, &id);
2324:     if (id == PETSCFE_CLASSID) {PetscFEGetQuadrature((PetscFE) obj, &quad);PetscFEGetDefaultTabulation((PetscFE) obj, &Bfine, NULL, NULL);}
2325:     else                       {PetscFVGetQuadrature((PetscFV) obj, &quad);}
2326:     PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights);
2327:     /* For each fine grid cell */
2328:     for (cell = cStart; cell < cEnd; ++cell) {
2329:       Vec                pointVec;
2330:       PetscScalar       *pV;
2331:       PetscSF            coarseCellSF = NULL;
2332:       const PetscSFNode *coarseCells;
2333:       PetscInt           numCoarseCells, cpdim, q, c, j;
2334:       PetscInt          *findices,   *cindices;
2335:       PetscInt           numFIndices, numCIndices;

2337:       DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2338:       DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ);
2339:       /* Get points from the quadrature */
2340:       VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec);
2341:       VecSetBlockSize(pointVec, dim);
2342:       VecGetArray(pointVec, &pV);
2343:       for (q = 0; q < Nq; ++q) {
2344:         const PetscReal xi0[3] = {-1., -1., -1.};

2346:         /* Transform point to real space */
2347:         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
2348:         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
2349:       }
2350:       VecRestoreArray(pointVec, &pV);
2351:       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
2352:       DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF);
2353:       /* Update matrix */
2354:       PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells);
2355:       if (numCoarseCells != Nq) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
2356:       VecGetArray(pointVec, &pV);
2357:       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2358:         PetscReal pVReal[3];
2359:         const PetscReal xi0[3] = {-1., -1., -1.};


2362:         DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2363:         /* Transform points from real space to coarse reference space */
2364:         DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc);
2365:         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
2366:         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);

2368:         if (id == PETSCFE_CLASSID) {
2369:           PetscFE    fe = (PetscFE) obj;
2370:           PetscReal *B;

2372:           /* Evaluate coarse basis on contained point */
2373:           PetscFEGetDimension(fe, &cpdim);
2374:           PetscFEGetTabulation(fe, 1, x, &B, NULL, NULL);
2375:           /* Get elemMat entries by multiplying by weight */
2376:           for (i = 0; i < numFIndices; ++i) {
2377:             PetscMemzero(elemMat, cpdim * sizeof(PetscScalar));
2378:             for (j = 0; j < cpdim; ++j) {
2379:               for (c = 0; c < Nc; ++c) elemMat[j] += B[j*Nc + c]*Bfine[(ccell*numFIndices + i)*Nc + c]*qweights[ccell*Nc + c]*detJ;
2380:             }
2381:             /* Update interpolator */
2382:             if (mesh->printFEM > 1) {DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);}
2383:             if (numCIndices != cpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %D != %D", numCIndices, cpdim);
2384:             MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES);
2385:           }
2386:           PetscFERestoreTabulation(fe, 1, x, &B, NULL, NULL);
2387:         } else {
2388:           cpdim = 1;
2389:           for (i = 0; i < numFIndices; ++i) {
2390:             PetscMemzero(elemMat, cpdim * sizeof(PetscScalar));
2391:             for (j = 0; j < cpdim; ++j) {
2392:               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*1.0*qweights[ccell*Nc + c]*detJ;
2393:             }
2394:             /* Update interpolator */
2395:             if (mesh->printFEM > 1) {DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat);}
2396:             PetscPrintf(PETSC_COMM_SELF, "Nq: %d %d Nf: %d %d Nc: %d %d\n", ccell, Nq, i, numFIndices, j, numCIndices);
2397:             if (numCIndices != cpdim) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %D != %D", numCIndices, cpdim);
2398:             MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES);
2399:           }
2400:         }
2401:         DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, &numCIndices, &cindices, NULL);
2402:       }
2403:       VecRestoreArray(pointVec, &pV);
2404:       PetscSFDestroy(&coarseCellSF);
2405:       VecDestroy(&pointVec);
2406:       DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, &numFIndices, &findices, NULL);
2407:     }
2408:   }
2409:   PetscFree3(v0,J,invJ);
2410:   PetscFree3(v0c,Jc,invJc);
2411:   PetscFree(elemMat);
2412:   MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY);
2413:   MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY);
2414:   return(0);
2415: }

2417: /*@
2418:   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns

2420:   Input Parameters:
2421: + dmc  - The coarse mesh
2422: - dmf  - The fine mesh
2423: - user - The user context

2425:   Output Parameter:
2426: . sc   - The mapping

2428:   Level: developer

2430: .seealso: DMPlexComputeInterpolatorNested(), DMPlexComputeJacobianFEM()
2431: @*/
2432: PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user)
2433: {
2434:   PetscDS        prob;
2435:   PetscFE       *feRef;
2436:   PetscFV       *fvRef;
2437:   Vec            fv, cv;
2438:   IS             fis, cis;
2439:   PetscSection   fsection, fglobalSection, csection, cglobalSection;
2440:   PetscInt      *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
2441:   PetscInt       cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, cEndInterior, c, dim, d, startC, endC, offsetC, offsetF, m;
2442:   PetscBool     *needAvg;

2446:   PetscLogEventBegin(DMPLEX_InjectorFEM,dmc,dmf,0,0);
2447:   DMGetDimension(dmf, &dim);
2448:   DMGetSection(dmf, &fsection);
2449:   DMGetGlobalSection(dmf, &fglobalSection);
2450:   DMGetSection(dmc, &csection);
2451:   DMGetGlobalSection(dmc, &cglobalSection);
2452:   PetscSectionGetNumFields(fsection, &Nf);
2453:   DMPlexGetHeightStratum(dmc, 0, &cStart, &cEnd);
2454:   DMPlexGetHybridBounds(dmc, &cEndInterior, NULL, NULL, NULL);
2455:   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
2456:   DMGetDS(dmc, &prob);
2457:   PetscCalloc3(Nf,&feRef,Nf,&fvRef,Nf,&needAvg);
2458:   for (f = 0; f < Nf; ++f) {
2459:     PetscObject  obj;
2460:     PetscClassId id;
2461:     PetscInt     fNb = 0, Nc = 0;

2463:     PetscDSGetDiscretization(prob, f, &obj);
2464:     PetscObjectGetClassId(obj, &id);
2465:     if (id == PETSCFE_CLASSID) {
2466:       PetscFE    fe = (PetscFE) obj;
2467:       PetscSpace sp;
2468:       PetscInt   maxDegree;

2470:       PetscFERefine(fe, &feRef[f]);
2471:       PetscFEGetDimension(feRef[f], &fNb);
2472:       PetscFEGetNumComponents(fe, &Nc);
2473:       PetscFEGetBasisSpace(fe, &sp);
2474:       PetscSpaceGetDegree(sp, NULL, &maxDegree);
2475:       if (!maxDegree) needAvg[f] = PETSC_TRUE;
2476:     } else if (id == PETSCFV_CLASSID) {
2477:       PetscFV        fv = (PetscFV) obj;
2478:       PetscDualSpace Q;

2480:       PetscFVRefine(fv, &fvRef[f]);
2481:       PetscFVGetDualSpace(fvRef[f], &Q);
2482:       PetscDualSpaceGetDimension(Q, &fNb);
2483:       PetscFVGetNumComponents(fv, &Nc);
2484:       needAvg[f] = PETSC_TRUE;
2485:     }
2486:     fTotDim += fNb;
2487:   }
2488:   PetscDSGetTotalDimension(prob, &cTotDim);
2489:   PetscMalloc1(cTotDim,&cmap);
2490:   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
2491:     PetscFE        feC;
2492:     PetscFV        fvC;
2493:     PetscDualSpace QF, QC;
2494:     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;

2496:     if (feRef[field]) {
2497:       PetscDSGetDiscretization(prob, field, (PetscObject *) &feC);
2498:       PetscFEGetNumComponents(feC, &NcC);
2499:       PetscFEGetNumComponents(feRef[field], &NcF);
2500:       PetscFEGetDualSpace(feRef[field], &QF);
2501:       PetscDualSpaceGetOrder(QF, &order);
2502:       PetscDualSpaceGetDimension(QF, &fpdim);
2503:       PetscFEGetDualSpace(feC, &QC);
2504:       PetscDualSpaceGetDimension(QC, &cpdim);
2505:     } else {
2506:       PetscDSGetDiscretization(prob, field, (PetscObject *) &fvC);
2507:       PetscFVGetNumComponents(fvC, &NcC);
2508:       PetscFVGetNumComponents(fvRef[field], &NcF);
2509:       PetscFVGetDualSpace(fvRef[field], &QF);
2510:       PetscDualSpaceGetDimension(QF, &fpdim);
2511:       PetscFVGetDualSpace(fvC, &QC);
2512:       PetscDualSpaceGetDimension(QC, &cpdim);
2513:     }
2514:     if (NcF != NcC) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %d does not match coarse field %d", NcF, NcC);
2515:     for (c = 0; c < cpdim; ++c) {
2516:       PetscQuadrature  cfunc;
2517:       const PetscReal *cqpoints, *cqweights;
2518:       PetscInt         NqcC, NpC;
2519:       PetscBool        found = PETSC_FALSE;

2521:       PetscDualSpaceGetFunctional(QC, c, &cfunc);
2522:       PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights);
2523:       if (NqcC != NcC) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %D must match number of field components", NqcC, NcC);
2524:       if (NpC != 1 && feRef[field]) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
2525:       for (f = 0; f < fpdim; ++f) {
2526:         PetscQuadrature  ffunc;
2527:         const PetscReal *fqpoints, *fqweights;
2528:         PetscReal        sum = 0.0;
2529:         PetscInt         NqcF, NpF;

2531:         PetscDualSpaceGetFunctional(QF, f, &ffunc);
2532:         PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights);
2533:         if (NqcF != NcF) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %D must match number of field components", NqcF, NcF);
2534:         if (NpC != NpF) continue;
2535:         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
2536:         if (sum > 1.0e-9) continue;
2537:         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d]*fqweights[d]);
2538:         if (sum < 1.0e-9) continue;
2539:         cmap[offsetC+c] = offsetF+f;
2540:         found = PETSC_TRUE;
2541:         break;
2542:       }
2543:       if (!found) {
2544:         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
2545:         if (fvRef[field] || (feRef[field] && order == 0)) {
2546:           cmap[offsetC+c] = offsetF+0;
2547:         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
2548:       }
2549:     }
2550:     offsetC += cpdim;
2551:     offsetF += fpdim;
2552:   }
2553:   for (f = 0; f < Nf; ++f) {PetscFEDestroy(&feRef[f]);PetscFVDestroy(&fvRef[f]);}
2554:   PetscFree3(feRef,fvRef,needAvg);

2556:   DMGetGlobalVector(dmf, &fv);
2557:   DMGetGlobalVector(dmc, &cv);
2558:   VecGetOwnershipRange(cv, &startC, &endC);
2559:   PetscSectionGetConstrainedStorageSize(cglobalSection, &m);
2560:   PetscMalloc2(cTotDim,&cellCIndices,fTotDim,&cellFIndices);
2561:   PetscMalloc1(m,&cindices);
2562:   PetscMalloc1(m,&findices);
2563:   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
2564:   for (c = cStart; c < cEnd; ++c) {
2565:     DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices);
2566:     for (d = 0; d < cTotDim; ++d) {
2567:       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
2568:       if ((findices[cellCIndices[d]-startC] >= 0) && (findices[cellCIndices[d]-startC] != cellFIndices[cmap[d]])) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Coarse dof %d maps to both %d and %d", cindices[cellCIndices[d]-startC], findices[cellCIndices[d]-startC], cellFIndices[cmap[d]]);
2569:       cindices[cellCIndices[d]-startC] = cellCIndices[d];
2570:       findices[cellCIndices[d]-startC] = cellFIndices[cmap[d]];
2571:     }
2572:   }
2573:   PetscFree(cmap);
2574:   PetscFree2(cellCIndices,cellFIndices);

2576:   ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis);
2577:   ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis);
2578:   VecScatterCreate(cv, cis, fv, fis, sc);
2579:   ISDestroy(&cis);
2580:   ISDestroy(&fis);
2581:   DMRestoreGlobalVector(dmf, &fv);
2582:   DMRestoreGlobalVector(dmc, &cv);
2583:   PetscLogEventEnd(DMPLEX_InjectorFEM,dmc,dmf,0,0);
2584:   return(0);
2585: }

2587: PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
2588: {
2589:   char            composeStr[33] = {0};
2590:   PetscObjectId   id;
2591:   PetscContainer  container;
2592:   PetscErrorCode  ierr;

2595:   PetscObjectGetId((PetscObject)quad,&id);
2596:   PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%x\n", id);
2597:   PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container);
2598:   if (container) {
2599:     PetscContainerGetPointer(container, (void **) geom);
2600:   } else {
2601:     DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom);
2602:     PetscContainerCreate(PETSC_COMM_SELF,&container);
2603:     PetscContainerSetPointer(container, (void *) *geom);
2604:     PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom);
2605:     PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container);
2606:     PetscContainerDestroy(&container);
2607:   }
2608:   return(0);
2609: }

2611: PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
2612: {
2614:   *geom = NULL;
2615:   return(0);
2616: }

2618: PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user)
2619: {
2620:   DM_Plex         *mesh       = (DM_Plex *) dm->data;
2621:   const char      *name       = "Residual";
2622:   DM               dmAux      = NULL;
2623:   DMLabel          ghostLabel = NULL;
2624:   PetscDS          prob       = NULL;
2625:   PetscDS          probAux    = NULL;
2626:   PetscBool        useFEM     = PETSC_FALSE;
2627:   PetscBool        isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
2628:   DMField          coordField = NULL;
2629:   Vec              locA;
2630:   PetscScalar     *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
2631:   IS               chunkIS;
2632:   const PetscInt  *cells;
2633:   PetscInt         cStart, cEnd, numCells;
2634:   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
2635:   PetscInt         maxDegree = PETSC_MAX_INT;
2636:   PetscQuadrature  affineQuad = NULL, *quads = NULL;
2637:   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
2638:   PetscErrorCode   ierr;

2641:   PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0);
2642:   /* FEM+FVM */
2643:   /* 1: Get sizes from dm and dmAux */
2644:   DMGetLabel(dm, "ghost", &ghostLabel);
2645:   DMGetDS(dm, &prob);
2646:   PetscDSGetNumFields(prob, &Nf);
2647:   PetscDSGetTotalDimension(prob, &totDim);
2648:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &locA);
2649:   if (locA) {
2650:     VecGetDM(locA, &dmAux);
2651:     DMGetDS(dmAux, &probAux);
2652:     PetscDSGetTotalDimension(probAux, &totDimAux);
2653:   }
2654:   /* 2: Get geometric data */
2655:   for (f = 0; f < Nf; ++f) {
2656:     PetscObject  obj;
2657:     PetscClassId id;
2658:     PetscBool    fimp;

2660:     PetscDSGetImplicit(prob, f, &fimp);
2661:     if (isImplicit != fimp) continue;
2662:     PetscDSGetDiscretization(prob, f, &obj);
2663:     PetscObjectGetClassId(obj, &id);
2664:     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
2665:     if (id == PETSCFV_CLASSID) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Use of FVM with PCPATCH not yet implemented");
2666:   }
2667:   if (useFEM) {
2668:     DMGetCoordinateField(dm, &coordField);
2669:     DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);
2670:     if (maxDegree <= 1) {
2671:       DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad);
2672:       if (affineQuad) {
2673:         DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);
2674:       }
2675:     } else {
2676:       PetscCalloc2(Nf,&quads,Nf,&geoms);
2677:       for (f = 0; f < Nf; ++f) {
2678:         PetscObject  obj;
2679:         PetscClassId id;
2680:         PetscBool    fimp;

2682:         PetscDSGetImplicit(prob, f, &fimp);
2683:         if (isImplicit != fimp) continue;
2684:         PetscDSGetDiscretization(prob, f, &obj);
2685:         PetscObjectGetClassId(obj, &id);
2686:         if (id == PETSCFE_CLASSID) {
2687:           PetscFE fe = (PetscFE) obj;

2689:           PetscFEGetQuadrature(fe, &quads[f]);
2690:           PetscObjectReference((PetscObject)quads[f]);
2691:           DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);
2692:         }
2693:       }
2694:     }
2695:   }
2696:   /* Loop over chunks */
2697:   ISGetPointRange(cellIS, &cStart, &cEnd, &cells);
2698:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
2699:   if (useFEM) {ISCreate(PETSC_COMM_SELF, &chunkIS);}
2700:   numCells      = cEnd - cStart;
2701:   numChunks     = 1;
2702:   cellChunkSize = numCells/numChunks;
2703:   numChunks     = PetscMin(1,numCells);
2704:   for (chunk = 0; chunk < numChunks; ++chunk) {
2705:     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
2706:     PetscReal       *vol = NULL;
2707:     PetscFVFaceGeom *fgeom = NULL;
2708:     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
2709:     PetscInt         numFaces = 0;

2711:     /* Extract field coefficients */
2712:     if (useFEM) {
2713:       ISGetPointSubrange(chunkIS, cS, cE, cells);
2714:       DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);
2715:       DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);
2716:       PetscMemzero(elemVec, numCells*totDim * sizeof(PetscScalar));
2717:     }
2718:     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
2719:     /* Loop over fields */
2720:     for (f = 0; f < Nf; ++f) {
2721:       PetscObject  obj;
2722:       PetscClassId id;
2723:       PetscBool    fimp;
2724:       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;

2726:       PetscDSGetImplicit(prob, f, &fimp);
2727:       if (isImplicit != fimp) continue;
2728:       PetscDSGetDiscretization(prob, f, &obj);
2729:       PetscObjectGetClassId(obj, &id);
2730:       if (id == PETSCFE_CLASSID) {
2731:         PetscFE         fe = (PetscFE) obj;
2732:         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
2733:         PetscFEGeom    *chunkGeom = NULL;
2734:         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
2735:         PetscInt        Nq, Nb;

2737:         PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
2738:         PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);
2739:         PetscFEGetDimension(fe, &Nb);
2740:         blockSize = Nb;
2741:         batchSize = numBlocks * blockSize;
2742:         PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
2743:         numChunks = numCells / (numBatches*batchSize);
2744:         Ne        = numChunks*numBatches*batchSize;
2745:         Nr        = numCells % (numBatches*batchSize);
2746:         offset    = numCells - Nr;
2747:         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
2748:         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
2749:         PetscFEGeomGetChunk(geom,0,offset,&chunkGeom);
2750:         PetscFEIntegrateResidual(fe, prob, f, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec);
2751:         PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom);
2752:         PetscFEIntegrateResidual(fe, prob, f, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]);
2753:         PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom);
2754:       } else if (id == PETSCFV_CLASSID) {
2755:         PetscFV fv = (PetscFV) obj;

2757:         Ne = numFaces;
2758:         /* Riemann solve over faces (need fields at face centroids) */
2759:         /*   We need to evaluate FE fields at those coordinates */
2760:         PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR);
2761:       } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", f);
2762:     }
2763:     /* Loop over domain */
2764:     if (useFEM) {
2765:       /* Add elemVec to locX */
2766:       for (c = cS; c < cE; ++c) {
2767:         const PetscInt cell = cells ? cells[c] : c;
2768:         const PetscInt cind = c - cStart;

2770:         if (mesh->printFEM > 1) {DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]);}
2771:         if (ghostLabel) {
2772:           PetscInt ghostVal;

2774:           DMLabelGetValue(ghostLabel,cell,&ghostVal);
2775:           if (ghostVal > 0) continue;
2776:         }
2777:         DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES);
2778:       }
2779:     }
2780:     /* Handle time derivative */
2781:     if (locX_t) {
2782:       PetscScalar *x_t, *fa;

2784:       VecGetArray(locF, &fa);
2785:       VecGetArray(locX_t, &x_t);
2786:       for (f = 0; f < Nf; ++f) {
2787:         PetscFV      fv;
2788:         PetscObject  obj;
2789:         PetscClassId id;
2790:         PetscInt     pdim, d;

2792:         PetscDSGetDiscretization(prob, f, &obj);
2793:         PetscObjectGetClassId(obj, &id);
2794:         if (id != PETSCFV_CLASSID) continue;
2795:         fv   = (PetscFV) obj;
2796:         PetscFVGetNumComponents(fv, &pdim);
2797:         for (c = cS; c < cE; ++c) {
2798:           const PetscInt cell = cells ? cells[c] : c;
2799:           PetscScalar   *u_t, *r;

2801:           if (ghostLabel) {
2802:             PetscInt ghostVal;

2804:             DMLabelGetValue(ghostLabel, cell, &ghostVal);
2805:             if (ghostVal > 0) continue;
2806:           }
2807:           DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t);
2808:           DMPlexPointLocalFieldRef(dm, cell, f, fa, &r);
2809:           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
2810:         }
2811:       }
2812:       VecRestoreArray(locX_t, &x_t);
2813:       VecRestoreArray(locF, &fa);
2814:     }
2815:     if (useFEM) {
2816:       DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);
2817:       DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);
2818:     }
2819:   }
2820:   if (useFEM) {ISDestroy(&chunkIS);}
2821:   ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);
2822:   /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */
2823:   if (useFEM) {
2824:     if (maxDegree <= 1) {
2825:       DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);
2826:       PetscQuadratureDestroy(&affineQuad);
2827:     } else {
2828:       for (f = 0; f < Nf; ++f) {
2829:         DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);
2830:         PetscQuadratureDestroy(&quads[f]);
2831:       }
2832:       PetscFree2(quads,geoms);
2833:     }
2834:   }
2835:   PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0);
2836:   return(0);
2837: }

2839: /*
2840:   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac

2842:   X   - The local solution vector
2843:   X_t - The local solution time derviative vector, or NULL
2844: */
2845: PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS,
2846:                                                     PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx)
2847: {
2848:   DM_Plex         *mesh  = (DM_Plex *) dm->data;
2849:   const char      *name = "Jacobian", *nameP = "JacobianPre";
2850:   DM               dmAux = NULL;
2851:   PetscDS          prob,   probAux = NULL;
2852:   PetscSection     sectionAux = NULL;
2853:   Vec              A;
2854:   DMField          coordField;
2855:   PetscFEGeom     *cgeomFEM;
2856:   PetscQuadrature  qGeom = NULL;
2857:   Mat              J = Jac, JP = JacP;
2858:   PetscScalar     *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
2859:   PetscBool        hasJac, hasPrec, hasDyn, assembleJac, isMatIS, isMatISP, *isFE, hasFV = PETSC_FALSE;
2860:   const PetscInt  *cells;
2861:   PetscInt         Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
2862:   PetscErrorCode   ierr;

2865:   CHKMEMQ;
2866:   ISGetLocalSize(cellIS, &numCells);
2867:   ISGetPointRange(cellIS, &cStart, &cEnd, &cells);
2868:   PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);
2869:   DMGetDS(dm, &prob);
2870:   PetscObjectQuery((PetscObject) dm, "dmAux", (PetscObject *) &dmAux);
2871:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &A);
2872:   if (dmAux) {
2873:     DMGetDefaultSection(dmAux, &sectionAux);
2874:     DMGetDS(dmAux, &probAux);
2875:   }
2876:   /* Get flags */
2877:   PetscDSGetNumFields(prob, &Nf);
2878:   DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE);
2879:   for (fieldI = 0; fieldI < Nf; ++fieldI) {
2880:     PetscObject  disc;
2881:     PetscClassId id;
2882:     PetscDSGetDiscretization(prob, fieldI, &disc);
2883:     PetscObjectGetClassId(disc, &id);
2884:     if (id == PETSCFE_CLASSID)      {isFE[fieldI] = PETSC_TRUE;}
2885:     else if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; isFE[fieldI] = PETSC_FALSE;}
2886:   }
2887:   PetscDSHasJacobian(prob, &hasJac);
2888:   PetscDSHasJacobianPreconditioner(prob, &hasPrec);
2889:   PetscDSHasDynamicJacobian(prob, &hasDyn);
2890:   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
2891:   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
2892:   PetscObjectTypeCompare((PetscObject) Jac,  MATIS, &isMatIS);
2893:   PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);
2894:   /* Setup input data and temp arrays (should be DMGetWorkArray) */
2895:   if (isMatISP || isMatISP) {DMPlexGetSubdomainSection(dm, &globalSection);}
2896:   if (isMatIS)  {MatISGetLocalMat(Jac,  &J);}
2897:   if (isMatISP) {MatISGetLocalMat(JacP, &JP);}
2898:   if (hasFV)    {MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE);} /* No allocated space for FV stuff, so ignore the zero entries */
2899:   PetscObjectQuery((PetscObject) dm, "dmAux", (PetscObject *) &dmAux);
2900:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &A);
2901:   PetscDSGetTotalDimension(prob, &totDim);
2902:   if (probAux) {PetscDSGetTotalDimension(probAux, &totDimAux);}
2903:   CHKMEMQ;
2904:   /* Compute batch sizes */
2905:   if (isFE[0]) {
2906:     PetscFE         fe;
2907:     PetscQuadrature q;
2908:     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;

2910:     PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe);
2911:     PetscFEGetQuadrature(fe, &q);
2912:     PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL);
2913:     PetscFEGetDimension(fe, &Nb);
2914:     PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
2915:     blockSize = Nb*numQuadPoints;
2916:     batchSize = numBlocks  * blockSize;
2917:     chunkSize = numBatches * batchSize;
2918:     numChunks = numCells / chunkSize + numCells % chunkSize;
2919:     PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
2920:   } else {
2921:     chunkSize = numCells;
2922:     numChunks = 1;
2923:   }
2924:   /* Get work space */
2925:   wsz  = (((X?1:0) + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize;
2926:   DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work);
2927:   PetscMemzero(work, wsz * sizeof(PetscScalar));
2928:   off      = 0;
2929:   u        = X       ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
2930:   u_t      = X_t     ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
2931:   a        = dmAux   ? (sz = chunkSize*totDimAux,     off += sz, work+off-sz) : NULL;
2932:   elemMat  = hasJac  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
2933:   elemMatP = hasPrec ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
2934:   elemMatD = hasDyn  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
2935:   if (off != wsz) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %D should be %D", off, wsz);
2936:   /* Setup geometry */
2937:   DMGetCoordinateField(dm, &coordField);
2938:   DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree);
2939:   if (maxDegree <= 1) {DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom);}
2940:   if (!qGeom) {
2941:     PetscFE fe;

2943:     PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe);
2944:     PetscFEGetQuadrature(fe, &qGeom);
2945:     PetscObjectReference((PetscObject) qGeom);
2946:   }
2947:   DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM);
2948:   /* Compute volume integrals */
2949:   if (assembleJac) {MatZeroEntries(J);}
2950:   MatZeroEntries(JP);
2951:   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
2952:     const PetscInt   Ncell = PetscMin(chunkSize, numCells - offCell);
2953:     PetscInt         c;

2955:     /* Extract values */
2956:     for (c = 0; c < Ncell; ++c) {
2957:       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
2958:       PetscScalar   *x = NULL,  *x_t = NULL;
2959:       PetscInt       i;

2961:       if (X) {
2962:         DMPlexVecGetClosure(dm, section, X, cell, NULL, &x);
2963:         for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
2964:         DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x);
2965:       }
2966:       if (X_t) {
2967:         DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t);
2968:         for (i = 0; i < totDim; ++i) u_t[c*totDim+i] = x_t[i];
2969:         DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t);
2970:       }
2971:       if (dmAux) {
2972:         DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x);
2973:         for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
2974:         DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x);
2975:       }
2976:     }
2977:     CHKMEMQ;
2978:     for (fieldI = 0; fieldI < Nf; ++fieldI) {
2979:       PetscFE fe;
2980:       PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);
2981:       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
2982:         if (hasJac)  {PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN,     fieldI, fieldJ, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat);}
2983:         if (hasPrec) {PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_PRE, fieldI, fieldJ, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP);}
2984:         if (hasDyn)  {PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_DYN, fieldI, fieldJ, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD);}
2985:       }
2986:       /* For finite volume, add the identity */
2987:       if (!isFE[fieldI]) {
2988:         PetscFV  fv;
2989:         PetscInt eOffset = 0, Nc, fc, foff;

2991:         PetscDSGetFieldOffset(prob, fieldI, &foff);
2992:         PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv);
2993:         PetscFVGetNumComponents(fv, &Nc);
2994:         for (c = 0; c < chunkSize; ++c, eOffset += totDim*totDim) {
2995:           for (fc = 0; fc < Nc; ++fc) {
2996:             const PetscInt i = foff + fc;
2997:             if (hasJac)  {elemMat [eOffset+i*totDim+i] = 1.0;}
2998:             if (hasPrec) {elemMatP[eOffset+i*totDim+i] = 1.0;}
2999:           }
3000:         }
3001:       }
3002:     }
3003:     CHKMEMQ;
3004:     /*   Add contribution from X_t */
3005:     if (hasDyn) {for (c = 0; c < chunkSize*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
3006:     /* Insert values into matrix */
3007:     for (c = 0; c < Ncell; ++c) {
3008:       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
3009:       if (mesh->printFEM > 1) {
3010:         if (hasJac)  {DMPrintCellMatrix(cell, name,  totDim, totDim, &elemMat[(c-cStart)*totDim*totDim]);}
3011:         if (hasPrec) {DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c-cStart)*totDim*totDim]);}
3012:       }
3013:       if (assembleJac) {DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES);}
3014:       DMPlexMatSetClosure(dm, section, globalSection, JP, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES);
3015:     }
3016:     CHKMEMQ;
3017:   }
3018:   /* Cleanup */
3019:   DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM);
3020:   PetscQuadratureDestroy(&qGeom);
3021:   if (hasFV) {MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE);}
3022:   DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE);
3023:   DMRestoreWorkArray(dm, ((1 + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize, MPIU_SCALAR, &work);
3024:   /* Compute boundary integrals */
3025:   /* DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx); */
3026:   /* Assemble matrix */
3027:   if (assembleJac) {MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY);MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY);}
3028:   MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY);MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY);
3029:   PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);
3030:   CHKMEMQ;
3031:   return(0);
3032: }