Actual source code: dmplexsnes.c

petsc-3.11.0 2019-03-29
Report Typos and Errors
  1:  #include <petsc/private/dmpleximpl.h>
  2:  #include <petsc/private/snesimpl.h>
  3:  #include <petscds.h>
  4:  #include <petscblaslapack.h>
  5:  #include <petsc/private/petscimpl.h>
  6:  #include <petsc/private/petscfeimpl.h>

  8: /************************** Interpolation *******************************/

 10: static PetscErrorCode DMSNESConvertPlex(DM dm, DM *plex, PetscBool copy)
 11: {
 12:   PetscBool      isPlex;

 16:   PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex);
 17:   if (isPlex) {
 18:     *plex = dm;
 19:     PetscObjectReference((PetscObject) dm);
 20:   } else {
 21:     PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex);
 22:     if (!*plex) {
 23:       DMConvert(dm,DMPLEX,plex);
 24:       PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex);
 25:       if (copy) {
 26:         PetscInt    i;
 27:         PetscObject obj;
 28:         const char *comps[3] = {"A","dmAux","dmCh"};

 30:         DMCopyDMSNES(dm, *plex);
 31:         for (i = 0; i < 3; i++) {
 32:           PetscObjectQuery((PetscObject) dm, comps[i], &obj);
 33:           PetscObjectCompose((PetscObject) *plex, comps[i], obj);
 34:         }
 35:       }
 36:     } else {
 37:       PetscObjectReference((PetscObject) *plex);
 38:     }
 39:   }
 40:   return(0);
 41: }

 43: /*@C
 44:   DMInterpolationCreate - Creates a DMInterpolationInfo context

 46:   Collective on comm

 48:   Input Parameter:
 49: . comm - the communicator

 51:   Output Parameter:
 52: . ctx - the context

 54:   Level: beginner

 56: .seealso: DMInterpolationEvaluate(), DMInterpolationAddPoints(), DMInterpolationDestroy()
 57: @*/
 58: PetscErrorCode DMInterpolationCreate(MPI_Comm comm, DMInterpolationInfo *ctx)
 59: {

 64:   PetscNew(ctx);

 66:   (*ctx)->comm   = comm;
 67:   (*ctx)->dim    = -1;
 68:   (*ctx)->nInput = 0;
 69:   (*ctx)->points = NULL;
 70:   (*ctx)->cells  = NULL;
 71:   (*ctx)->n      = -1;
 72:   (*ctx)->coords = NULL;
 73:   return(0);
 74: }

 76: /*@C
 77:   DMInterpolationSetDim - Sets the spatial dimension for the interpolation context

 79:   Not collective

 81:   Input Parameters:
 82: + ctx - the context
 83: - dim - the spatial dimension

 85:   Level: intermediate

 87: .seealso: DMInterpolationGetDim(), DMInterpolationEvaluate(), DMInterpolationAddPoints()
 88: @*/
 89: PetscErrorCode DMInterpolationSetDim(DMInterpolationInfo ctx, PetscInt dim)
 90: {
 92:   if ((dim < 1) || (dim > 3)) SETERRQ1(ctx->comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid dimension for points: %d", dim);
 93:   ctx->dim = dim;
 94:   return(0);
 95: }

 97: /*@C
 98:   DMInterpolationGetDim - Gets the spatial dimension for the interpolation context

100:   Not collective

102:   Input Parameter:
103: . ctx - the context

105:   Output Parameter:
106: . dim - the spatial dimension

108:   Level: intermediate

110: .seealso: DMInterpolationSetDim(), DMInterpolationEvaluate(), DMInterpolationAddPoints()
111: @*/
112: PetscErrorCode DMInterpolationGetDim(DMInterpolationInfo ctx, PetscInt *dim)
113: {
116:   *dim = ctx->dim;
117:   return(0);
118: }

120: /*@C
121:   DMInterpolationSetDof - Sets the number of fields interpolated at a point for the interpolation context

123:   Not collective

125:   Input Parameters:
126: + ctx - the context
127: - dof - the number of fields

129:   Level: intermediate

131: .seealso: DMInterpolationGetDof(), DMInterpolationEvaluate(), DMInterpolationAddPoints()
132: @*/
133: PetscErrorCode DMInterpolationSetDof(DMInterpolationInfo ctx, PetscInt dof)
134: {
136:   if (dof < 1) SETERRQ1(ctx->comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of components: %d", dof);
137:   ctx->dof = dof;
138:   return(0);
139: }

141: /*@C
142:   DMInterpolationGetDof - Gets the number of fields interpolated at a point for the interpolation context

144:   Not collective

146:   Input Parameter:
147: . ctx - the context

149:   Output Parameter:
150: . dof - the number of fields

152:   Level: intermediate

154: .seealso: DMInterpolationSetDof(), DMInterpolationEvaluate(), DMInterpolationAddPoints()
155: @*/
156: PetscErrorCode DMInterpolationGetDof(DMInterpolationInfo ctx, PetscInt *dof)
157: {
160:   *dof = ctx->dof;
161:   return(0);
162: }

164: /*@C
165:   DMInterpolationAddPoints - Add points at which we will interpolate the fields

167:   Not collective

169:   Input Parameters:
170: + ctx    - the context
171: . n      - the number of points
172: - points - the coordinates for each point, an array of size n * dim

174:   Note: The coordinate information is copied.

176:   Level: intermediate

178: .seealso: DMInterpolationSetDim(), DMInterpolationEvaluate(), DMInterpolationCreate()
179: @*/
180: PetscErrorCode DMInterpolationAddPoints(DMInterpolationInfo ctx, PetscInt n, PetscReal points[])
181: {

185:   if (ctx->dim < 0) SETERRQ(ctx->comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set");
186:   if (ctx->points)  SETERRQ(ctx->comm, PETSC_ERR_ARG_WRONGSTATE, "Cannot add points multiple times yet");
187:   ctx->nInput = n;

189:   PetscMalloc1(n*ctx->dim, &ctx->points);
190:   PetscMemcpy(ctx->points, points, n*ctx->dim * sizeof(PetscReal));
191:   return(0);
192: }

194: /*@C
195:   DMInterpolationSetUp - Computea spatial indices that add in point location during interpolation

197:   Collective on ctx

199:   Input Parameters:
200: + ctx - the context
201: . dm  - the DM for the function space used for interpolation
202: - redundantPoints - If PETSC_TRUE, all processes are passing in the same array of points. Otherwise, points need to be communicated among processes.

204:   Level: intermediate

206: .seealso: DMInterpolationEvaluate(), DMInterpolationAddPoints(), DMInterpolationCreate()
207: @*/
208: PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints)
209: {
210:   MPI_Comm          comm = ctx->comm;
211:   PetscScalar       *a;
212:   PetscInt          p, q, i;
213:   PetscMPIInt       rank, size;
214:   PetscErrorCode    ierr;
215:   Vec               pointVec;
216:   PetscSF           cellSF;
217:   PetscLayout       layout;
218:   PetscReal         *globalPoints;
219:   PetscScalar       *globalPointsScalar;
220:   const PetscInt    *ranges;
221:   PetscMPIInt       *counts, *displs;
222:   const PetscSFNode *foundCells;
223:   const PetscInt    *foundPoints;
224:   PetscMPIInt       *foundProcs, *globalProcs;
225:   PetscInt          n, N, numFound;

229:   MPI_Comm_size(comm, &size);
230:   MPI_Comm_rank(comm, &rank);
231:   if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set");
232:   /* Locate points */
233:   n = ctx->nInput;
234:   if (!redundantPoints) {
235:     PetscLayoutCreate(comm, &layout);
236:     PetscLayoutSetBlockSize(layout, 1);
237:     PetscLayoutSetLocalSize(layout, n);
238:     PetscLayoutSetUp(layout);
239:     PetscLayoutGetSize(layout, &N);
240:     /* Communicate all points to all processes */
241:     PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);
242:     PetscLayoutGetRanges(layout, &ranges);
243:     for (p = 0; p < size; ++p) {
244:       counts[p] = (ranges[p+1] - ranges[p])*ctx->dim;
245:       displs[p] = ranges[p]*ctx->dim;
246:     }
247:     MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);
248:   } else {
249:     N = n;
250:     globalPoints = ctx->points;
251:     counts = displs = NULL;
252:     layout = NULL;
253:   }
254: #if 0
255:   PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);
256:   /* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */
257: #else
258: #if defined(PETSC_USE_COMPLEX)
259:   PetscMalloc1(N*ctx->dim,&globalPointsScalar);
260:   for (i=0; i<N*ctx->dim; i++) globalPointsScalar[i] = globalPoints[i];
261: #else
262:   globalPointsScalar = globalPoints;
263: #endif
264:   VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);
265:   PetscMalloc2(N,&foundProcs,N,&globalProcs);
266:   for (p = 0; p < N; ++p) {foundProcs[p] = size;}
267:   cellSF = NULL;
268:   DMLocatePoints(dm, pointVec, DM_POINTLOCATION_REMOVE, &cellSF);
269:   PetscSFGetGraph(cellSF,NULL,&numFound,&foundPoints,&foundCells);
270: #endif
271:   for (p = 0; p < numFound; ++p) {
272:     if (foundCells[p].index >= 0) foundProcs[foundPoints ? foundPoints[p] : p] = rank;
273:   }
274:   /* Let the lowest rank process own each point */
275:   MPIU_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);
276:   ctx->n = 0;
277:   for (p = 0; p < N; ++p) {
278:     if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, (double)globalPoints[p*ctx->dim+0], (double)(ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0), (double)(ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0));
279:     else if (globalProcs[p] == rank) ctx->n++;
280:   }
281:   /* Create coordinates vector and array of owned cells */
282:   PetscMalloc1(ctx->n, &ctx->cells);
283:   VecCreate(comm, &ctx->coords);
284:   VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);
285:   VecSetBlockSize(ctx->coords, ctx->dim);
286:   VecSetType(ctx->coords,VECSTANDARD);
287:   VecGetArray(ctx->coords, &a);
288:   for (p = 0, q = 0, i = 0; p < N; ++p) {
289:     if (globalProcs[p] == rank) {
290:       PetscInt d;

292:       for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d];
293:       ctx->cells[q] = foundCells[q].index;
294:       ++q;
295:     }
296:   }
297:   VecRestoreArray(ctx->coords, &a);
298: #if 0
299:   PetscFree3(foundCells,foundProcs,globalProcs);
300: #else
301:   PetscFree2(foundProcs,globalProcs);
302:   PetscSFDestroy(&cellSF);
303:   VecDestroy(&pointVec);
304: #endif
305:   if ((void*)globalPointsScalar != (void*)globalPoints) {PetscFree(globalPointsScalar);}
306:   if (!redundantPoints) {PetscFree3(globalPoints,counts,displs);}
307:   PetscLayoutDestroy(&layout);
308:   return(0);
309: }

311: /*@C
312:   DMInterpolationGetCoordinates - Gets a Vec with the coordinates of each interpolation point

314:   Collective on ctx

316:   Input Parameter:
317: . ctx - the context

319:   Output Parameter:
320: . coordinates  - the coordinates of interpolation points

322:   Note: The local vector entries correspond to interpolation points lying on this process, according to the associated DM. This is a borrowed vector that the user should not destroy.

324:   Level: intermediate

326: .seealso: DMInterpolationEvaluate(), DMInterpolationAddPoints(), DMInterpolationCreate()
327: @*/
328: PetscErrorCode DMInterpolationGetCoordinates(DMInterpolationInfo ctx, Vec *coordinates)
329: {
332:   if (!ctx->coords) SETERRQ(ctx->comm, PETSC_ERR_ARG_WRONGSTATE, "The interpolation context has not been setup.");
333:   *coordinates = ctx->coords;
334:   return(0);
335: }

337: /*@C
338:   DMInterpolationGetVector - Gets a Vec which can hold all the interpolated field values

340:   Collective on ctx

342:   Input Parameter:
343: . ctx - the context

345:   Output Parameter:
346: . v  - a vector capable of holding the interpolated field values

348:   Note: This vector should be returned using DMInterpolationRestoreVector().

350:   Level: intermediate

352: .seealso: DMInterpolationRestoreVector(), DMInterpolationEvaluate(), DMInterpolationAddPoints(), DMInterpolationCreate()
353: @*/
354: PetscErrorCode DMInterpolationGetVector(DMInterpolationInfo ctx, Vec *v)
355: {

360:   if (!ctx->coords) SETERRQ(ctx->comm, PETSC_ERR_ARG_WRONGSTATE, "The interpolation context has not been setup.");
361:   VecCreate(ctx->comm, v);
362:   VecSetSizes(*v, ctx->n*ctx->dof, PETSC_DECIDE);
363:   VecSetBlockSize(*v, ctx->dof);
364:   VecSetType(*v,VECSTANDARD);
365:   return(0);
366: }

368: /*@C
369:   DMInterpolationRestoreVector - Returns a Vec which can hold all the interpolated field values

371:   Collective on ctx

373:   Input Parameters:
374: + ctx - the context
375: - v  - a vector capable of holding the interpolated field values

377:   Level: intermediate

379: .seealso: DMInterpolationGetVector(), DMInterpolationEvaluate(), DMInterpolationAddPoints(), DMInterpolationCreate()
380: @*/
381: PetscErrorCode DMInterpolationRestoreVector(DMInterpolationInfo ctx, Vec *v)
382: {

387:   if (!ctx->coords) SETERRQ(ctx->comm, PETSC_ERR_ARG_WRONGSTATE, "The interpolation context has not been setup.");
388:   VecDestroy(v);
389:   return(0);
390: }

392: PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Triangle_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v)
393: {
394:   PetscReal      *v0, *J, *invJ, detJ;
395:   const PetscScalar *coords;
396:   PetscScalar    *a;
397:   PetscInt       p;

401:   PetscMalloc3(ctx->dim,&v0,ctx->dim*ctx->dim,&J,ctx->dim*ctx->dim,&invJ);
402:   VecGetArrayRead(ctx->coords, &coords);
403:   VecGetArray(v, &a);
404:   for (p = 0; p < ctx->n; ++p) {
405:     PetscInt     c = ctx->cells[p];
406:     PetscScalar *x = NULL;
407:     PetscReal    xi[4];
408:     PetscInt     d, f, comp;

410:     DMPlexComputeCellGeometryFEM(dm, c, NULL, v0, J, invJ, &detJ);
411:     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %d", (double)detJ, c);
412:     DMPlexVecGetClosure(dm, NULL, xLocal, c, NULL, &x);
413:     for (comp = 0; comp < ctx->dof; ++comp) a[p*ctx->dof+comp] = x[0*ctx->dof+comp];

415:     for (d = 0; d < ctx->dim; ++d) {
416:       xi[d] = 0.0;
417:       for (f = 0; f < ctx->dim; ++f) xi[d] += invJ[d*ctx->dim+f]*0.5*PetscRealPart(coords[p*ctx->dim+f] - v0[f]);
418:       for (comp = 0; comp < ctx->dof; ++comp) a[p*ctx->dof+comp] += PetscRealPart(x[(d+1)*ctx->dof+comp] - x[0*ctx->dof+comp])*xi[d];
419:     }
420:     DMPlexVecRestoreClosure(dm, NULL, xLocal, c, NULL, &x);
421:   }
422:   VecRestoreArray(v, &a);
423:   VecRestoreArrayRead(ctx->coords, &coords);
424:   PetscFree3(v0, J, invJ);
425:   return(0);
426: }

428: PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Tetrahedron_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v)
429: {
430:   PetscReal      *v0, *J, *invJ, detJ;
431:   const PetscScalar *coords;
432:   PetscScalar    *a;
433:   PetscInt       p;

437:   PetscMalloc3(ctx->dim,&v0,ctx->dim*ctx->dim,&J,ctx->dim*ctx->dim,&invJ);
438:   VecGetArrayRead(ctx->coords, &coords);
439:   VecGetArray(v, &a);
440:   for (p = 0; p < ctx->n; ++p) {
441:     PetscInt       c = ctx->cells[p];
442:     const PetscInt order[3] = {2, 1, 3};
443:     PetscScalar   *x = NULL;
444:     PetscReal      xi[4];
445:     PetscInt       d, f, comp;

447:     DMPlexComputeCellGeometryFEM(dm, c, NULL, v0, J, invJ, &detJ);
448:     if (detJ <= 0.0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %d", (double)detJ, c);
449:     DMPlexVecGetClosure(dm, NULL, xLocal, c, NULL, &x);
450:     for (comp = 0; comp < ctx->dof; ++comp) a[p*ctx->dof+comp] = x[0*ctx->dof+comp];

452:     for (d = 0; d < ctx->dim; ++d) {
453:       xi[d] = 0.0;
454:       for (f = 0; f < ctx->dim; ++f) xi[d] += invJ[d*ctx->dim+f]*0.5*PetscRealPart(coords[p*ctx->dim+f] - v0[f]);
455:       for (comp = 0; comp < ctx->dof; ++comp) a[p*ctx->dof+comp] += PetscRealPart(x[order[d]*ctx->dof+comp] - x[0*ctx->dof+comp])*xi[d];
456:     }
457:     DMPlexVecRestoreClosure(dm, NULL, xLocal, c, NULL, &x);
458:   }
459:   VecRestoreArray(v, &a);
460:   VecRestoreArrayRead(ctx->coords, &coords);
461:   PetscFree3(v0, J, invJ);
462:   return(0);
463: }

465: PETSC_STATIC_INLINE PetscErrorCode QuadMap_Private(SNES snes, Vec Xref, Vec Xreal, void *ctx)
466: {
467:   const PetscScalar *vertices = (const PetscScalar*) ctx;
468:   const PetscScalar x0        = vertices[0];
469:   const PetscScalar y0        = vertices[1];
470:   const PetscScalar x1        = vertices[2];
471:   const PetscScalar y1        = vertices[3];
472:   const PetscScalar x2        = vertices[4];
473:   const PetscScalar y2        = vertices[5];
474:   const PetscScalar x3        = vertices[6];
475:   const PetscScalar y3        = vertices[7];
476:   const PetscScalar f_1       = x1 - x0;
477:   const PetscScalar g_1       = y1 - y0;
478:   const PetscScalar f_3       = x3 - x0;
479:   const PetscScalar g_3       = y3 - y0;
480:   const PetscScalar f_01      = x2 - x1 - x3 + x0;
481:   const PetscScalar g_01      = y2 - y1 - y3 + y0;
482:   const PetscScalar *ref;
483:   PetscScalar       *real;
484:   PetscErrorCode    ierr;

487:   VecGetArrayRead(Xref,  &ref);
488:   VecGetArray(Xreal, &real);
489:   {
490:     const PetscScalar p0 = ref[0];
491:     const PetscScalar p1 = ref[1];

493:     real[0] = x0 + f_1 * p0 + f_3 * p1 + f_01 * p0 * p1;
494:     real[1] = y0 + g_1 * p0 + g_3 * p1 + g_01 * p0 * p1;
495:   }
496:   PetscLogFlops(28);
497:   VecRestoreArrayRead(Xref,  &ref);
498:   VecRestoreArray(Xreal, &real);
499:   return(0);
500: }

502:  #include <petsc/private/dmimpl.h>
503: PETSC_STATIC_INLINE PetscErrorCode QuadJacobian_Private(SNES snes, Vec Xref, Mat J, Mat M, void *ctx)
504: {
505:   const PetscScalar *vertices = (const PetscScalar*) ctx;
506:   const PetscScalar x0        = vertices[0];
507:   const PetscScalar y0        = vertices[1];
508:   const PetscScalar x1        = vertices[2];
509:   const PetscScalar y1        = vertices[3];
510:   const PetscScalar x2        = vertices[4];
511:   const PetscScalar y2        = vertices[5];
512:   const PetscScalar x3        = vertices[6];
513:   const PetscScalar y3        = vertices[7];
514:   const PetscScalar f_01      = x2 - x1 - x3 + x0;
515:   const PetscScalar g_01      = y2 - y1 - y3 + y0;
516:   const PetscScalar *ref;
517:   PetscErrorCode    ierr;

520:   VecGetArrayRead(Xref,  &ref);
521:   {
522:     const PetscScalar x       = ref[0];
523:     const PetscScalar y       = ref[1];
524:     const PetscInt    rows[2] = {0, 1};
525:     PetscScalar       values[4];

527:     values[0] = (x1 - x0 + f_01*y) * 0.5; values[1] = (x3 - x0 + f_01*x) * 0.5;
528:     values[2] = (y1 - y0 + g_01*y) * 0.5; values[3] = (y3 - y0 + g_01*x) * 0.5;
529:     MatSetValues(J, 2, rows, 2, rows, values, INSERT_VALUES);
530:   }
531:   PetscLogFlops(30);
532:   VecRestoreArrayRead(Xref,  &ref);
533:   MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY);
534:   MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY);
535:   return(0);
536: }

538: PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Quad_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v)
539: {
540:   DM             dmCoord;
541:   PetscFE        fem = NULL;
542:   SNES           snes;
543:   KSP            ksp;
544:   PC             pc;
545:   Vec            coordsLocal, r, ref, real;
546:   Mat            J;
547:   const PetscScalar *coords;
548:   PetscScalar    *a;
549:   PetscInt       Nf, p;
550:   const PetscInt dof = ctx->dof;

554:   DMGetNumFields(dm, &Nf);
555:   if (Nf) {DMGetField(dm, 0, NULL, (PetscObject *) &fem);}
556:   DMGetCoordinatesLocal(dm, &coordsLocal);
557:   DMGetCoordinateDM(dm, &dmCoord);
558:   SNESCreate(PETSC_COMM_SELF, &snes);
559:   SNESSetOptionsPrefix(snes, "quad_interp_");
560:   VecCreate(PETSC_COMM_SELF, &r);
561:   VecSetSizes(r, 2, 2);
562:   VecSetType(r,dm->vectype);
563:   VecDuplicate(r, &ref);
564:   VecDuplicate(r, &real);
565:   MatCreate(PETSC_COMM_SELF, &J);
566:   MatSetSizes(J, 2, 2, 2, 2);
567:   MatSetType(J, MATSEQDENSE);
568:   MatSetUp(J);
569:   SNESSetFunction(snes, r, QuadMap_Private, NULL);
570:   SNESSetJacobian(snes, J, J, QuadJacobian_Private, NULL);
571:   SNESGetKSP(snes, &ksp);
572:   KSPGetPC(ksp, &pc);
573:   PCSetType(pc, PCLU);
574:   SNESSetFromOptions(snes);

576:   VecGetArrayRead(ctx->coords, &coords);
577:   VecGetArray(v, &a);
578:   for (p = 0; p < ctx->n; ++p) {
579:     PetscScalar *x = NULL, *vertices = NULL;
580:     PetscScalar *xi;
581:     PetscReal    xir[2];
582:     PetscInt     c = ctx->cells[p], comp, coordSize, xSize;

584:     /* Can make this do all points at once */
585:     DMPlexVecGetClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);
586:     if (4*2 != coordSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", coordSize, 4*2);
587:     DMPlexVecGetClosure(dm, NULL, xLocal, c, &xSize, &x);
588:     SNESSetFunction(snes, NULL, NULL, (void*) vertices);
589:     SNESSetJacobian(snes, NULL, NULL, NULL, (void*) vertices);
590:     VecGetArray(real, &xi);
591:     xi[0]  = coords[p*ctx->dim+0];
592:     xi[1]  = coords[p*ctx->dim+1];
593:     VecRestoreArray(real, &xi);
594:     SNESSolve(snes, real, ref);
595:     VecGetArray(ref, &xi);
596:     xir[0] = PetscRealPart(xi[0]);
597:     xir[1] = PetscRealPart(xi[1]);
598:     if (4*dof != xSize) {
599:       PetscReal *B;
600:       PetscInt   d;

602:       xir[0] = 2.0*xir[0] - 1.0; xir[1] = 2.0*xir[1] - 1.0;
603:       PetscFEGetTabulation(fem, 1, xir, &B, NULL, NULL);
604:       for (comp = 0; comp < dof; ++comp) {
605:         a[p*dof+comp] = 0.0;
606:         for (d = 0; d < xSize/dof; ++d) {
607:           a[p*dof+comp] += x[d*dof+comp]*B[d*dof+comp];
608:         }
609:       }
610:       PetscFERestoreTabulation(fem, 1, xir, &B, NULL, NULL);
611:     } else {
612:       for (comp = 0; comp < dof; ++comp)
613:         a[p*dof+comp] = x[0*dof+comp]*(1 - xir[0])*(1 - xir[1]) + x[1*dof+comp]*xir[0]*(1 - xir[1]) + x[2*dof+comp]*xir[0]*xir[1] + x[3*dof+comp]*(1 - xir[0])*xir[1];
614:     }
615:     VecRestoreArray(ref, &xi);
616:     DMPlexVecRestoreClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);
617:     DMPlexVecRestoreClosure(dm, NULL, xLocal, c, &xSize, &x);
618:   }
619:   VecRestoreArray(v, &a);
620:   VecRestoreArrayRead(ctx->coords, &coords);

622:   SNESDestroy(&snes);
623:   VecDestroy(&r);
624:   VecDestroy(&ref);
625:   VecDestroy(&real);
626:   MatDestroy(&J);
627:   return(0);
628: }

630: PETSC_STATIC_INLINE PetscErrorCode HexMap_Private(SNES snes, Vec Xref, Vec Xreal, void *ctx)
631: {
632:   const PetscScalar *vertices = (const PetscScalar*) ctx;
633:   const PetscScalar x0        = vertices[0];
634:   const PetscScalar y0        = vertices[1];
635:   const PetscScalar z0        = vertices[2];
636:   const PetscScalar x1        = vertices[9];
637:   const PetscScalar y1        = vertices[10];
638:   const PetscScalar z1        = vertices[11];
639:   const PetscScalar x2        = vertices[6];
640:   const PetscScalar y2        = vertices[7];
641:   const PetscScalar z2        = vertices[8];
642:   const PetscScalar x3        = vertices[3];
643:   const PetscScalar y3        = vertices[4];
644:   const PetscScalar z3        = vertices[5];
645:   const PetscScalar x4        = vertices[12];
646:   const PetscScalar y4        = vertices[13];
647:   const PetscScalar z4        = vertices[14];
648:   const PetscScalar x5        = vertices[15];
649:   const PetscScalar y5        = vertices[16];
650:   const PetscScalar z5        = vertices[17];
651:   const PetscScalar x6        = vertices[18];
652:   const PetscScalar y6        = vertices[19];
653:   const PetscScalar z6        = vertices[20];
654:   const PetscScalar x7        = vertices[21];
655:   const PetscScalar y7        = vertices[22];
656:   const PetscScalar z7        = vertices[23];
657:   const PetscScalar f_1       = x1 - x0;
658:   const PetscScalar g_1       = y1 - y0;
659:   const PetscScalar h_1       = z1 - z0;
660:   const PetscScalar f_3       = x3 - x0;
661:   const PetscScalar g_3       = y3 - y0;
662:   const PetscScalar h_3       = z3 - z0;
663:   const PetscScalar f_4       = x4 - x0;
664:   const PetscScalar g_4       = y4 - y0;
665:   const PetscScalar h_4       = z4 - z0;
666:   const PetscScalar f_01      = x2 - x1 - x3 + x0;
667:   const PetscScalar g_01      = y2 - y1 - y3 + y0;
668:   const PetscScalar h_01      = z2 - z1 - z3 + z0;
669:   const PetscScalar f_12      = x7 - x3 - x4 + x0;
670:   const PetscScalar g_12      = y7 - y3 - y4 + y0;
671:   const PetscScalar h_12      = z7 - z3 - z4 + z0;
672:   const PetscScalar f_02      = x5 - x1 - x4 + x0;
673:   const PetscScalar g_02      = y5 - y1 - y4 + y0;
674:   const PetscScalar h_02      = z5 - z1 - z4 + z0;
675:   const PetscScalar f_012     = x6 - x0 + x1 - x2 + x3 + x4 - x5 - x7;
676:   const PetscScalar g_012     = y6 - y0 + y1 - y2 + y3 + y4 - y5 - y7;
677:   const PetscScalar h_012     = z6 - z0 + z1 - z2 + z3 + z4 - z5 - z7;
678:   const PetscScalar *ref;
679:   PetscScalar       *real;
680:   PetscErrorCode    ierr;

683:   VecGetArrayRead(Xref,  &ref);
684:   VecGetArray(Xreal, &real);
685:   {
686:     const PetscScalar p0 = ref[0];
687:     const PetscScalar p1 = ref[1];
688:     const PetscScalar p2 = ref[2];

690:     real[0] = x0 + f_1*p0 + f_3*p1 + f_4*p2 + f_01*p0*p1 + f_12*p1*p2 + f_02*p0*p2 + f_012*p0*p1*p2;
691:     real[1] = y0 + g_1*p0 + g_3*p1 + g_4*p2 + g_01*p0*p1 + g_01*p0*p1 + g_12*p1*p2 + g_02*p0*p2 + g_012*p0*p1*p2;
692:     real[2] = z0 + h_1*p0 + h_3*p1 + h_4*p2 + h_01*p0*p1 + h_01*p0*p1 + h_12*p1*p2 + h_02*p0*p2 + h_012*p0*p1*p2;
693:   }
694:   PetscLogFlops(114);
695:   VecRestoreArrayRead(Xref,  &ref);
696:   VecRestoreArray(Xreal, &real);
697:   return(0);
698: }

700: PETSC_STATIC_INLINE PetscErrorCode HexJacobian_Private(SNES snes, Vec Xref, Mat J, Mat M, void *ctx)
701: {
702:   const PetscScalar *vertices = (const PetscScalar*) ctx;
703:   const PetscScalar x0        = vertices[0];
704:   const PetscScalar y0        = vertices[1];
705:   const PetscScalar z0        = vertices[2];
706:   const PetscScalar x1        = vertices[9];
707:   const PetscScalar y1        = vertices[10];
708:   const PetscScalar z1        = vertices[11];
709:   const PetscScalar x2        = vertices[6];
710:   const PetscScalar y2        = vertices[7];
711:   const PetscScalar z2        = vertices[8];
712:   const PetscScalar x3        = vertices[3];
713:   const PetscScalar y3        = vertices[4];
714:   const PetscScalar z3        = vertices[5];
715:   const PetscScalar x4        = vertices[12];
716:   const PetscScalar y4        = vertices[13];
717:   const PetscScalar z4        = vertices[14];
718:   const PetscScalar x5        = vertices[15];
719:   const PetscScalar y5        = vertices[16];
720:   const PetscScalar z5        = vertices[17];
721:   const PetscScalar x6        = vertices[18];
722:   const PetscScalar y6        = vertices[19];
723:   const PetscScalar z6        = vertices[20];
724:   const PetscScalar x7        = vertices[21];
725:   const PetscScalar y7        = vertices[22];
726:   const PetscScalar z7        = vertices[23];
727:   const PetscScalar f_xy      = x2 - x1 - x3 + x0;
728:   const PetscScalar g_xy      = y2 - y1 - y3 + y0;
729:   const PetscScalar h_xy      = z2 - z1 - z3 + z0;
730:   const PetscScalar f_yz      = x7 - x3 - x4 + x0;
731:   const PetscScalar g_yz      = y7 - y3 - y4 + y0;
732:   const PetscScalar h_yz      = z7 - z3 - z4 + z0;
733:   const PetscScalar f_xz      = x5 - x1 - x4 + x0;
734:   const PetscScalar g_xz      = y5 - y1 - y4 + y0;
735:   const PetscScalar h_xz      = z5 - z1 - z4 + z0;
736:   const PetscScalar f_xyz     = x6 - x0 + x1 - x2 + x3 + x4 - x5 - x7;
737:   const PetscScalar g_xyz     = y6 - y0 + y1 - y2 + y3 + y4 - y5 - y7;
738:   const PetscScalar h_xyz     = z6 - z0 + z1 - z2 + z3 + z4 - z5 - z7;
739:   const PetscScalar *ref;
740:   PetscErrorCode    ierr;

743:   VecGetArrayRead(Xref,  &ref);
744:   {
745:     const PetscScalar x       = ref[0];
746:     const PetscScalar y       = ref[1];
747:     const PetscScalar z       = ref[2];
748:     const PetscInt    rows[3] = {0, 1, 2};
749:     PetscScalar       values[9];

751:     values[0] = (x1 - x0 + f_xy*y + f_xz*z + f_xyz*y*z) / 2.0;
752:     values[1] = (x3 - x0 + f_xy*x + f_yz*z + f_xyz*x*z) / 2.0;
753:     values[2] = (x4 - x0 + f_yz*y + f_xz*x + f_xyz*x*y) / 2.0;
754:     values[3] = (y1 - y0 + g_xy*y + g_xz*z + g_xyz*y*z) / 2.0;
755:     values[4] = (y3 - y0 + g_xy*x + g_yz*z + g_xyz*x*z) / 2.0;
756:     values[5] = (y4 - y0 + g_yz*y + g_xz*x + g_xyz*x*y) / 2.0;
757:     values[6] = (z1 - z0 + h_xy*y + h_xz*z + h_xyz*y*z) / 2.0;
758:     values[7] = (z3 - z0 + h_xy*x + h_yz*z + h_xyz*x*z) / 2.0;
759:     values[8] = (z4 - z0 + h_yz*y + h_xz*x + h_xyz*x*y) / 2.0;

761:     MatSetValues(J, 3, rows, 3, rows, values, INSERT_VALUES);
762:   }
763:   PetscLogFlops(152);
764:   VecRestoreArrayRead(Xref,  &ref);
765:   MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY);
766:   MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY);
767:   return(0);
768: }

770: PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Hex_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v)
771: {
772:   DM             dmCoord;
773:   SNES           snes;
774:   KSP            ksp;
775:   PC             pc;
776:   Vec            coordsLocal, r, ref, real;
777:   Mat            J;
778:   const PetscScalar *coords;
779:   PetscScalar    *a;
780:   PetscInt       p;

784:   DMGetCoordinatesLocal(dm, &coordsLocal);
785:   DMGetCoordinateDM(dm, &dmCoord);
786:   SNESCreate(PETSC_COMM_SELF, &snes);
787:   SNESSetOptionsPrefix(snes, "hex_interp_");
788:   VecCreate(PETSC_COMM_SELF, &r);
789:   VecSetSizes(r, 3, 3);
790:   VecSetType(r,dm->vectype);
791:   VecDuplicate(r, &ref);
792:   VecDuplicate(r, &real);
793:   MatCreate(PETSC_COMM_SELF, &J);
794:   MatSetSizes(J, 3, 3, 3, 3);
795:   MatSetType(J, MATSEQDENSE);
796:   MatSetUp(J);
797:   SNESSetFunction(snes, r, HexMap_Private, NULL);
798:   SNESSetJacobian(snes, J, J, HexJacobian_Private, NULL);
799:   SNESGetKSP(snes, &ksp);
800:   KSPGetPC(ksp, &pc);
801:   PCSetType(pc, PCLU);
802:   SNESSetFromOptions(snes);

804:   VecGetArrayRead(ctx->coords, &coords);
805:   VecGetArray(v, &a);
806:   for (p = 0; p < ctx->n; ++p) {
807:     PetscScalar *x = NULL, *vertices = NULL;
808:     PetscScalar *xi;
809:     PetscReal    xir[3];
810:     PetscInt     c = ctx->cells[p], comp, coordSize, xSize;

812:     /* Can make this do all points at once */
813:     DMPlexVecGetClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);
814:     if (8*3 != coordSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", coordSize, 8*3);
815:     DMPlexVecGetClosure(dm, NULL, xLocal, c, &xSize, &x);
816:     if (8*ctx->dof != xSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", xSize, 8*ctx->dof);
817:     SNESSetFunction(snes, NULL, NULL, (void*) vertices);
818:     SNESSetJacobian(snes, NULL, NULL, NULL, (void*) vertices);
819:     VecGetArray(real, &xi);
820:     xi[0]  = coords[p*ctx->dim+0];
821:     xi[1]  = coords[p*ctx->dim+1];
822:     xi[2]  = coords[p*ctx->dim+2];
823:     VecRestoreArray(real, &xi);
824:     SNESSolve(snes, real, ref);
825:     VecGetArray(ref, &xi);
826:     xir[0] = PetscRealPart(xi[0]);
827:     xir[1] = PetscRealPart(xi[1]);
828:     xir[2] = PetscRealPart(xi[2]);
829:     for (comp = 0; comp < ctx->dof; ++comp) {
830:       a[p*ctx->dof+comp] =
831:         x[0*ctx->dof+comp]*(1-xir[0])*(1-xir[1])*(1-xir[2]) +
832:         x[3*ctx->dof+comp]*    xir[0]*(1-xir[1])*(1-xir[2]) +
833:         x[2*ctx->dof+comp]*    xir[0]*    xir[1]*(1-xir[2]) +
834:         x[1*ctx->dof+comp]*(1-xir[0])*    xir[1]*(1-xir[2]) +
835:         x[4*ctx->dof+comp]*(1-xir[0])*(1-xir[1])*   xir[2] +
836:         x[5*ctx->dof+comp]*    xir[0]*(1-xir[1])*   xir[2] +
837:         x[6*ctx->dof+comp]*    xir[0]*    xir[1]*   xir[2] +
838:         x[7*ctx->dof+comp]*(1-xir[0])*    xir[1]*   xir[2];
839:     }
840:     VecRestoreArray(ref, &xi);
841:     DMPlexVecRestoreClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);
842:     DMPlexVecRestoreClosure(dm, NULL, xLocal, c, &xSize, &x);
843:   }
844:   VecRestoreArray(v, &a);
845:   VecRestoreArrayRead(ctx->coords, &coords);

847:   SNESDestroy(&snes);
848:   VecDestroy(&r);
849:   VecDestroy(&ref);
850:   VecDestroy(&real);
851:   MatDestroy(&J);
852:   return(0);
853: }

855: /*@C
856:   DMInterpolationEvaluate - Using the input from dm and x, calculates interpolated field values at the interpolation points.

858:   Input Parameters:
859: + ctx - The DMInterpolationInfo context
860: . dm  - The DM
861: - x   - The local vector containing the field to be interpolated

863:   Output Parameters:
864: . v   - The vector containing the interpolated values

866:   Note: A suitable v can be obtained using DMInterpolationGetVector().

868:   Level: beginner

870: .seealso: DMInterpolationGetVector(), DMInterpolationAddPoints(), DMInterpolationCreate()
871: @*/
872: PetscErrorCode DMInterpolationEvaluate(DMInterpolationInfo ctx, DM dm, Vec x, Vec v)
873: {
874:   PetscInt       dim, coneSize, n;

881:   VecGetLocalSize(v, &n);
882:   if (n != ctx->n*ctx->dof) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid input vector size %d should be %d", n, ctx->n*ctx->dof);
883:   if (n) {
884:     DMGetDimension(dm, &dim);
885:     DMPlexGetConeSize(dm, ctx->cells[0], &coneSize);
886:     if (dim == 2) {
887:       if (coneSize == 3) {
888:         DMInterpolate_Triangle_Private(ctx, dm, x, v);
889:       } else if (coneSize == 4) {
890:         DMInterpolate_Quad_Private(ctx, dm, x, v);
891:       } else SETERRQ1(ctx->comm, PETSC_ERR_ARG_OUTOFRANGE, "Unsupported dimension %d for point interpolation", dim);
892:     } else if (dim == 3) {
893:       if (coneSize == 4) {
894:         DMInterpolate_Tetrahedron_Private(ctx, dm, x, v);
895:       } else {
896:         DMInterpolate_Hex_Private(ctx, dm, x, v);
897:       }
898:     } else SETERRQ1(ctx->comm, PETSC_ERR_ARG_OUTOFRANGE, "Unsupported dimension %d for point interpolation", dim);
899:   }
900:   return(0);
901: }

903: /*@C
904:   DMInterpolationDestroy - Destroys a DMInterpolationInfo context

906:   Collective on ctx

908:   Input Parameter:
909: . ctx - the context

911:   Level: beginner

913: .seealso: DMInterpolationEvaluate(), DMInterpolationAddPoints(), DMInterpolationCreate()
914: @*/
915: PetscErrorCode DMInterpolationDestroy(DMInterpolationInfo *ctx)
916: {

921:   VecDestroy(&(*ctx)->coords);
922:   PetscFree((*ctx)->points);
923:   PetscFree((*ctx)->cells);
924:   PetscFree(*ctx);
925:   *ctx = NULL;
926:   return(0);
927: }

929: /*@C
930:   SNESMonitorFields - Monitors the residual for each field separately

932:   Collective on SNES

934:   Input Parameters:
935: + snes   - the SNES context
936: . its    - iteration number
937: . fgnorm - 2-norm of residual
938: - vf  - PetscViewerAndFormat of type ASCII

940:   Notes:
941:   This routine prints the residual norm at each iteration.

943:   Level: intermediate

945: .keywords: SNES, nonlinear, default, monitor, norm
946: .seealso: SNESMonitorSet(), SNESMonitorDefault()
947: @*/
948: PetscErrorCode SNESMonitorFields(SNES snes, PetscInt its, PetscReal fgnorm, PetscViewerAndFormat *vf)
949: {
950:   PetscViewer        viewer = vf->viewer;
951:   Vec                res;
952:   DM                 dm;
953:   PetscSection       s;
954:   const PetscScalar *r;
955:   PetscReal         *lnorms, *norms;
956:   PetscInt           numFields, f, pStart, pEnd, p;
957:   PetscErrorCode     ierr;

961:   SNESGetFunction(snes, &res, 0, 0);
962:   SNESGetDM(snes, &dm);
963:   DMGetSection(dm, &s);
964:   PetscSectionGetNumFields(s, &numFields);
965:   PetscSectionGetChart(s, &pStart, &pEnd);
966:   PetscCalloc2(numFields, &lnorms, numFields, &norms);
967:   VecGetArrayRead(res, &r);
968:   for (p = pStart; p < pEnd; ++p) {
969:     for (f = 0; f < numFields; ++f) {
970:       PetscInt fdof, foff, d;

972:       PetscSectionGetFieldDof(s, p, f, &fdof);
973:       PetscSectionGetFieldOffset(s, p, f, &foff);
974:       for (d = 0; d < fdof; ++d) lnorms[f] += PetscRealPart(PetscSqr(r[foff+d]));
975:     }
976:   }
977:   VecRestoreArrayRead(res, &r);
978:   MPIU_Allreduce(lnorms, norms, numFields, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject) dm));
979:   PetscViewerPushFormat(viewer,vf->format);
980:   PetscViewerASCIIAddTab(viewer, ((PetscObject) snes)->tablevel);
981:   PetscViewerASCIIPrintf(viewer, "%3D SNES Function norm %14.12e [", its, (double) fgnorm);
982:   for (f = 0; f < numFields; ++f) {
983:     if (f > 0) {PetscViewerASCIIPrintf(viewer, ", ");}
984:     PetscViewerASCIIPrintf(viewer, "%14.12e", (double) PetscSqrtReal(norms[f]));
985:   }
986:   PetscViewerASCIIPrintf(viewer, "]\n");
987:   PetscViewerASCIISubtractTab(viewer, ((PetscObject) snes)->tablevel);
988:   PetscViewerPopFormat(viewer);
989:   PetscFree2(lnorms, norms);
990:   return(0);
991: }

993: /********************* Residual Computation **************************/


996: /*@
997:   DMPlexSNESGetGeometryFVM - Return precomputed geometric data

999:   Input Parameter:
1000: . dm - The DM

1002:   Output Parameters:
1003: + facegeom - The values precomputed from face geometry
1004: . cellgeom - The values precomputed from cell geometry
1005: - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell

1007:   Level: developer

1009: .seealso: DMPlexTSSetRHSFunctionLocal()
1010: @*/
1011: PetscErrorCode DMPlexSNESGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius)
1012: {
1013:   DM             plex;

1018:   DMSNESConvertPlex(dm,&plex,PETSC_TRUE);
1019:   DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL);
1020:   if (minRadius) {DMPlexGetMinRadius(plex, minRadius);}
1021:   DMDestroy(&plex);
1022:   return(0);
1023: }

1025: /*@
1026:   DMPlexSNESGetGradientDM - Return gradient data layout

1028:   Input Parameters:
1029: + dm - The DM
1030: - fv - The PetscFV

1032:   Output Parameter:
1033: . dmGrad - The layout for gradient values

1035:   Level: developer

1037: .seealso: DMPlexSNESGetGeometryFVM()
1038: @*/
1039: PetscErrorCode DMPlexSNESGetGradientDM(DM dm, PetscFV fv, DM *dmGrad)
1040: {
1041:   DM             plex;
1042:   PetscBool      computeGradients;

1049:   PetscFVGetComputeGradients(fv, &computeGradients);
1050:   if (!computeGradients) {*dmGrad = NULL; return(0);}
1051:   DMSNESConvertPlex(dm,&plex,PETSC_TRUE);
1052:   DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad);
1053:   DMDestroy(&plex);
1054:   return(0);
1055: }

1057: /*@C
1058:   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells

1060:   Input Parameters:
1061: + dm     - The DM
1062: . cellIS - The cells to include
1063: . locX   - A local vector with the solution fields
1064: . locX_t - A local vector with solution field time derivatives, or NULL
1065: - locA   - A local vector with auxiliary fields, or NULL

1067:   Output Parameters:
1068: + u   - The field coefficients
1069: . u_t - The fields derivative coefficients
1070: - a   - The auxiliary field coefficients

1072:   Level: developer

1074: .seealso: DMPlexGetFaceFields()
1075: @*/
1076: PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
1077: {
1078:   DM              plex, plexA = NULL;
1079:   PetscSection    section, sectionAux;
1080:   PetscDS         prob;
1081:   const PetscInt *cells;
1082:   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
1083:   PetscErrorCode  ierr;

1093:   DMSNESConvertPlex(dm, &plex, PETSC_FALSE);
1094:   ISGetPointRange(cellIS, &cStart, &cEnd, &cells);
1095:   DMGetSection(dm, &section);
1096:   DMGetCellDS(dm, cStart, &prob);
1097:   PetscDSGetTotalDimension(prob, &totDim);
1098:   if (locA) {
1099:     DM      dmAux;
1100:     PetscDS probAux;

1102:     VecGetDM(locA, &dmAux);
1103:     DMSNESConvertPlex(dmAux, &plexA, PETSC_FALSE);
1104:     DMGetSection(dmAux, &sectionAux);
1105:     DMGetDS(dmAux, &probAux);
1106:     PetscDSGetTotalDimension(probAux, &totDimAux);
1107:   }
1108:   numCells = cEnd - cStart;
1109:   DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u);
1110:   if (locX_t) {DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u_t);} else {*u_t = NULL;}
1111:   if (locA)   {DMGetWorkArray(dm, numCells*totDimAux, MPIU_SCALAR, a);} else {*a = NULL;}
1112:   for (c = cStart; c < cEnd; ++c) {
1113:     const PetscInt cell = cells ? cells[c] : c;
1114:     const PetscInt cind = c - cStart;
1115:     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
1116:     PetscInt       i;

1118:     DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x);
1119:     for (i = 0; i < totDim; ++i) ul[cind*totDim+i] = x[i];
1120:     DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x);
1121:     if (locX_t) {
1122:       DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t);
1123:       for (i = 0; i < totDim; ++i) ul_t[cind*totDim+i] = x_t[i];
1124:       DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t);
1125:     }
1126:     if (locA) {
1127:       PetscInt subcell;
1128:       DMPlexGetAuxiliaryPoint(plex, plexA, cell, &subcell);
1129:       DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x);
1130:       for (i = 0; i < totDimAux; ++i) al[cind*totDimAux+i] = x[i];
1131:       DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x);
1132:     }
1133:   }
1134:   DMDestroy(&plex);
1135:   if (locA) {DMDestroy(&plexA);}
1136:   ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);
1137:   return(0);
1138: }

1140: /*@C
1141:   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells

1143:   Input Parameters:
1144: + dm     - The DM
1145: . cellIS - The cells to include
1146: . locX   - A local vector with the solution fields
1147: . locX_t - A local vector with solution field time derivatives, or NULL
1148: - locA   - A local vector with auxiliary fields, or NULL

1150:   Output Parameters:
1151: + u   - The field coefficients
1152: . u_t - The fields derivative coefficients
1153: - a   - The auxiliary field coefficients

1155:   Level: developer

1157: .seealso: DMPlexGetFaceFields()
1158: @*/
1159: PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
1160: {

1164:   DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u);
1165:   if (locX_t) {DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t);}
1166:   if (locA)   {DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a);}
1167:   return(0);
1168: }

1170: /*@C
1171:   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces

1173:   Input Parameters:
1174: + dm     - The DM
1175: . fStart - The first face to include
1176: . fEnd   - The first face to exclude
1177: . locX   - A local vector with the solution fields
1178: . locX_t - A local vector with solution field time derivatives, or NULL
1179: . faceGeometry - A local vector with face geometry
1180: . cellGeometry - A local vector with cell geometry
1181: - locaGrad - A local vector with field gradients, or NULL

1183:   Output Parameters:
1184: + Nface - The number of faces with field values
1185: . uL - The field values at the left side of the face
1186: - uR - The field values at the right side of the face

1188:   Level: developer

1190: .seealso: DMPlexGetCellFields()
1191: @*/
1192: PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
1193: {
1194:   DM                 dmFace, dmCell, dmGrad = NULL;
1195:   PetscSection       section;
1196:   PetscDS            prob;
1197:   DMLabel            ghostLabel;
1198:   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
1199:   PetscBool         *isFE;
1200:   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
1201:   PetscErrorCode     ierr;

1212:   DMGetDimension(dm, &dim);
1213:   DMGetDS(dm, &prob);
1214:   DMGetSection(dm, &section);
1215:   PetscDSGetNumFields(prob, &Nf);
1216:   PetscDSGetTotalComponents(prob, &Nc);
1217:   PetscMalloc1(Nf, &isFE);
1218:   for (f = 0; f < Nf; ++f) {
1219:     PetscObject  obj;
1220:     PetscClassId id;

1222:     PetscDSGetDiscretization(prob, f, &obj);
1223:     PetscObjectGetClassId(obj, &id);
1224:     if (id == PETSCFE_CLASSID)      {isFE[f] = PETSC_TRUE;}
1225:     else if (id == PETSCFV_CLASSID) {isFE[f] = PETSC_FALSE;}
1226:     else                            {isFE[f] = PETSC_FALSE;}
1227:   }
1228:   DMGetLabel(dm, "ghost", &ghostLabel);
1229:   VecGetArrayRead(locX, &x);
1230:   VecGetDM(faceGeometry, &dmFace);
1231:   VecGetArrayRead(faceGeometry, &facegeom);
1232:   VecGetDM(cellGeometry, &dmCell);
1233:   VecGetArrayRead(cellGeometry, &cellgeom);
1234:   if (locGrad) {
1235:     VecGetDM(locGrad, &dmGrad);
1236:     VecGetArrayRead(locGrad, &lgrad);
1237:   }
1238:   DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uL);
1239:   DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uR);
1240:   /* Right now just eat the extra work for FE (could make a cell loop) */
1241:   for (face = fStart, iface = 0; face < fEnd; ++face) {
1242:     const PetscInt        *cells;
1243:     PetscFVFaceGeom       *fg;
1244:     PetscFVCellGeom       *cgL, *cgR;
1245:     PetscScalar           *xL, *xR, *gL, *gR;
1246:     PetscScalar           *uLl = *uL, *uRl = *uR;
1247:     PetscInt               ghost, nsupp, nchild;

1249:     DMLabelGetValue(ghostLabel, face, &ghost);
1250:     DMPlexGetSupportSize(dm, face, &nsupp);
1251:     DMPlexGetTreeChildren(dm, face, &nchild, NULL);
1252:     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
1253:     DMPlexPointLocalRead(dmFace, face, facegeom, &fg);
1254:     DMPlexGetSupport(dm, face, &cells);
1255:     DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);
1256:     DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);
1257:     for (f = 0; f < Nf; ++f) {
1258:       PetscInt off;

1260:       PetscDSGetComponentOffset(prob, f, &off);
1261:       if (isFE[f]) {
1262:         const PetscInt *cone;
1263:         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;

1265:         xL = xR = NULL;
1266:         PetscSectionGetFieldComponents(section, f, &comp);
1267:         DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL);
1268:         DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR);
1269:         DMPlexGetCone(dm, cells[0], &cone);
1270:         DMPlexGetConeSize(dm, cells[0], &coneSizeL);
1271:         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL) if (cone[faceLocL] == face) break;
1272:         DMPlexGetCone(dm, cells[1], &cone);
1273:         DMPlexGetConeSize(dm, cells[1], &coneSizeR);
1274:         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR) if (cone[faceLocR] == face) break;
1275:         if (faceLocL == coneSizeL && faceLocR == coneSizeR) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %d in cone of cell %d or cell %d", face, cells[0], cells[1]);
1276:         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
1277:         /* TODO: this is a hack that might not be right for nonconforming */
1278:         if (faceLocL < coneSizeL) {
1279:           EvaluateFaceFields(prob, f, faceLocL, xL, &uLl[iface*Nc+off]);
1280:           if (rdof == ldof && faceLocR < coneSizeR) {EvaluateFaceFields(prob, f, faceLocR, xR, &uRl[iface*Nc+off]);}
1281:           else              {for(d = 0; d < comp; ++d) uRl[iface*Nc+off+d] = uLl[iface*Nc+off+d];}
1282:         }
1283:         else {
1284:           EvaluateFaceFields(prob, f, faceLocR, xR, &uRl[iface*Nc+off]);
1285:           PetscSectionGetFieldComponents(section, f, &comp);
1286:           for(d = 0; d < comp; ++d) uLl[iface*Nc+off+d] = uRl[iface*Nc+off+d];
1287:         }
1288:         DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL);
1289:         DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR);
1290:       } else {
1291:         PetscFV  fv;
1292:         PetscInt numComp, c;

1294:         PetscDSGetDiscretization(prob, f, (PetscObject *) &fv);
1295:         PetscFVGetNumComponents(fv, &numComp);
1296:         DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL);
1297:         DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR);
1298:         if (dmGrad) {
1299:           PetscReal dxL[3], dxR[3];

1301:           DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL);
1302:           DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR);
1303:           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
1304:           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
1305:           for (c = 0; c < numComp; ++c) {
1306:             uLl[iface*Nc+off+c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c*dim], dxL);
1307:             uRl[iface*Nc+off+c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c*dim], dxR);
1308:           }
1309:         } else {
1310:           for (c = 0; c < numComp; ++c) {
1311:             uLl[iface*Nc+off+c] = xL[c];
1312:             uRl[iface*Nc+off+c] = xR[c];
1313:           }
1314:         }
1315:       }
1316:     }
1317:     ++iface;
1318:   }
1319:   *Nface = iface;
1320:   VecRestoreArrayRead(locX, &x);
1321:   VecRestoreArrayRead(faceGeometry, &facegeom);
1322:   VecRestoreArrayRead(cellGeometry, &cellgeom);
1323:   if (locGrad) {
1324:     VecRestoreArrayRead(locGrad, &lgrad);
1325:   }
1326:   PetscFree(isFE);
1327:   return(0);
1328: }

1330: /*@C
1331:   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces

1333:   Input Parameters:
1334: + dm     - The DM
1335: . fStart - The first face to include
1336: . fEnd   - The first face to exclude
1337: . locX   - A local vector with the solution fields
1338: . locX_t - A local vector with solution field time derivatives, or NULL
1339: . faceGeometry - A local vector with face geometry
1340: . cellGeometry - A local vector with cell geometry
1341: - locaGrad - A local vector with field gradients, or NULL

1343:   Output Parameters:
1344: + Nface - The number of faces with field values
1345: . uL - The field values at the left side of the face
1346: - uR - The field values at the right side of the face

1348:   Level: developer

1350: .seealso: DMPlexGetFaceFields()
1351: @*/
1352: PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
1353: {

1357:   DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL);
1358:   DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR);
1359:   return(0);
1360: }

1362: /*@C
1363:   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces

1365:   Input Parameters:
1366: + dm     - The DM
1367: . fStart - The first face to include
1368: . fEnd   - The first face to exclude
1369: . faceGeometry - A local vector with face geometry
1370: - cellGeometry - A local vector with cell geometry

1372:   Output Parameters:
1373: + Nface - The number of faces with field values
1374: . fgeom - The extract the face centroid and normal
1375: - vol   - The cell volume

1377:   Level: developer

1379: .seealso: DMPlexGetCellFields()
1380: @*/
1381: PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
1382: {
1383:   DM                 dmFace, dmCell;
1384:   DMLabel            ghostLabel;
1385:   const PetscScalar *facegeom, *cellgeom;
1386:   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
1387:   PetscErrorCode     ierr;

1395:   DMGetDimension(dm, &dim);
1396:   DMGetLabel(dm, "ghost", &ghostLabel);
1397:   VecGetDM(faceGeometry, &dmFace);
1398:   VecGetArrayRead(faceGeometry, &facegeom);
1399:   VecGetDM(cellGeometry, &dmCell);
1400:   VecGetArrayRead(cellGeometry, &cellgeom);
1401:   PetscMalloc1(numFaces, fgeom);
1402:   DMGetWorkArray(dm, numFaces*2, MPIU_SCALAR, vol);
1403:   for (face = fStart, iface = 0; face < fEnd; ++face) {
1404:     const PetscInt        *cells;
1405:     PetscFVFaceGeom       *fg;
1406:     PetscFVCellGeom       *cgL, *cgR;
1407:     PetscFVFaceGeom       *fgeoml = *fgeom;
1408:     PetscReal             *voll   = *vol;
1409:     PetscInt               ghost, d, nchild, nsupp;

1411:     DMLabelGetValue(ghostLabel, face, &ghost);
1412:     DMPlexGetSupportSize(dm, face, &nsupp);
1413:     DMPlexGetTreeChildren(dm, face, &nchild, NULL);
1414:     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
1415:     DMPlexPointLocalRead(dmFace, face, facegeom, &fg);
1416:     DMPlexGetSupport(dm, face, &cells);
1417:     DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);
1418:     DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);
1419:     for (d = 0; d < dim; ++d) {
1420:       fgeoml[iface].centroid[d] = fg->centroid[d];
1421:       fgeoml[iface].normal[d]   = fg->normal[d];
1422:     }
1423:     voll[iface*2+0] = cgL->volume;
1424:     voll[iface*2+1] = cgR->volume;
1425:     ++iface;
1426:   }
1427:   *Nface = iface;
1428:   VecRestoreArrayRead(faceGeometry, &facegeom);
1429:   VecRestoreArrayRead(cellGeometry, &cellgeom);
1430:   return(0);
1431: }

1433: /*@C
1434:   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces

1436:   Input Parameters:
1437: + dm     - The DM
1438: . fStart - The first face to include
1439: . fEnd   - The first face to exclude
1440: . faceGeometry - A local vector with face geometry
1441: - cellGeometry - A local vector with cell geometry

1443:   Output Parameters:
1444: + Nface - The number of faces with field values
1445: . fgeom - The extract the face centroid and normal
1446: - vol   - The cell volume

1448:   Level: developer

1450: .seealso: DMPlexGetFaceFields()
1451: @*/
1452: PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
1453: {

1457:   PetscFree(*fgeom);
1458:   DMRestoreWorkArray(dm, 0, MPIU_REAL, vol);
1459:   return(0);
1460: }

1462: static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS)
1463: {
1464:   DM_Plex         *mesh = (DM_Plex *) dm->data;
1465:   DM               plex = NULL, plexA = NULL;
1466:   PetscDS          prob, probAux = NULL;
1467:   PetscSection     section, sectionAux = NULL;
1468:   Vec              locA = NULL;
1469:   PetscScalar     *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
1470:   PetscInt         v;
1471:   PetscInt         totDim, totDimAux = 0;
1472:   PetscErrorCode   ierr;

1475:   DMConvert(dm, DMPLEX, &plex);
1476:   DMGetSection(dm, &section);
1477:   DMGetDS(dm, &prob);
1478:   PetscDSGetTotalDimension(prob, &totDim);
1479:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &locA);
1480:   if (locA) {
1481:     DM dmAux;

1483:     VecGetDM(locA, &dmAux);
1484:     DMConvert(dmAux, DMPLEX, &plexA);
1485:     DMGetDS(plexA, &probAux);
1486:     PetscDSGetTotalDimension(probAux, &totDimAux);
1487:     DMGetSection(plexA, &sectionAux);
1488:   }
1489:   for (v = 0; v < numValues; ++v) {
1490:     PetscFEGeom    *fgeom;
1491:     PetscInt        maxDegree;
1492:     PetscQuadrature qGeom = NULL;
1493:     IS              pointIS;
1494:     const PetscInt *points;
1495:     PetscInt        numFaces, face, Nq;

1497:     DMLabelGetStratumIS(label, values[v], &pointIS);
1498:     if (!pointIS) continue; /* No points with that id on this process */
1499:     {
1500:       IS isectIS;

1502:       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
1503:       ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS);
1504:       ISDestroy(&pointIS);
1505:       pointIS = isectIS;
1506:     }
1507:     ISGetLocalSize(pointIS,&numFaces);
1508:     ISGetIndices(pointIS,&points);
1509:     PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim, &elemVec, locA ? numFaces*totDimAux : 0, &a);
1510:     DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree);
1511:     if (maxDegree <= 1) {
1512:       DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom);
1513:     }
1514:     if (!qGeom) {
1515:       PetscFE fe;

1517:       PetscDSGetDiscretization(prob, field, (PetscObject *) &fe);
1518:       PetscFEGetFaceQuadrature(fe, &qGeom);
1519:       PetscObjectReference((PetscObject)qGeom);
1520:     }
1521:     PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);
1522:     DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);
1523:     for (face = 0; face < numFaces; ++face) {
1524:       const PetscInt point = points[face], *support, *cone;
1525:       PetscScalar   *x     = NULL;
1526:       PetscInt       i, coneSize, faceLoc;

1528:       DMPlexGetSupport(dm, point, &support);
1529:       DMPlexGetConeSize(dm, support[0], &coneSize);
1530:       DMPlexGetCone(dm, support[0], &cone);
1531:       for (faceLoc = 0; faceLoc < coneSize; ++faceLoc) if (cone[faceLoc] == point) break;
1532:       if (faceLoc == coneSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %D in cone of support[0] %D", point, support[0]);
1533:       fgeom->face[face][0] = faceLoc;
1534:       DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x);
1535:       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
1536:       DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x);
1537:       if (locX_t) {
1538:         DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x);
1539:         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
1540:         DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x);
1541:       }
1542:       if (locA) {
1543:         PetscInt subp;

1545:         DMPlexGetAuxiliaryPoint(plex, plexA, support[0], &subp);
1546:         DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x);
1547:         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
1548:         DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x);
1549:       }
1550:     }
1551:     PetscMemzero(elemVec, numFaces*totDim * sizeof(PetscScalar));
1552:     {
1553:       PetscFE         fe;
1554:       PetscInt        Nb;
1555:       PetscFEGeom     *chunkGeom = NULL;
1556:       /* Conforming batches */
1557:       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
1558:       /* Remainder */
1559:       PetscInt        Nr, offset;

1561:       PetscDSGetDiscretization(prob, field, (PetscObject *) &fe);
1562:       PetscFEGetDimension(fe, &Nb);
1563:       PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
1564:       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
1565:       blockSize = Nb;
1566:       batchSize = numBlocks * blockSize;
1567:        PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
1568:       numChunks = numFaces / (numBatches*batchSize);
1569:       Ne        = numChunks*numBatches*batchSize;
1570:       Nr        = numFaces % (numBatches*batchSize);
1571:       offset    = numFaces - Nr;
1572:       PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom);
1573:       PetscFEIntegrateBdResidual(fe, prob, field, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec);
1574:       PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom);
1575:       PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom);
1576:       PetscFEIntegrateBdResidual(fe, prob, field, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, &elemVec[offset*totDim]);
1577:       PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom);
1578:     }
1579:     for (face = 0; face < numFaces; ++face) {
1580:       const PetscInt point = points[face], *support;

1582:       if (mesh->printFEM > 1) {DMPrintCellVector(point, "BdResidual", totDim, &elemVec[face*totDim]);}
1583:       DMPlexGetSupport(plex, point, &support);
1584:       DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face*totDim], ADD_ALL_VALUES);
1585:     }
1586:     DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);
1587:     PetscQuadratureDestroy(&qGeom);
1588:     ISRestoreIndices(pointIS, &points);
1589:     ISDestroy(&pointIS);
1590:     PetscFree4(u, u_t, elemVec, a);
1591:   }
1592:   if (plex)  {DMDestroy(&plex);}
1593:   if (plexA) {DMDestroy(&plexA);}
1594:   return(0);
1595: }

1597: PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, Vec locF)
1598: {
1599:   DMField        coordField;
1600:   DMLabel        depthLabel;
1601:   IS             facetIS;
1602:   PetscInt       dim;

1606:   DMGetDimension(dm, &dim);
1607:   DMPlexGetDepthLabel(dm, &depthLabel);
1608:   DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);
1609:   DMGetCoordinateField(dm, &coordField);
1610:   DMPlexComputeBdResidual_Single_Internal(dm, t, label, numValues, values, field, locX, locX_t, locF, coordField, facetIS);
1611:   return(0);
1612: }

1614: PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
1615: {
1616:   PetscDS        prob;
1617:   PetscInt       numBd, bd;
1618:   DMField        coordField = NULL;
1619:   IS             facetIS    = NULL;
1620:   DMLabel        depthLabel;
1621:   PetscInt       dim;

1625:   DMGetDS(dm, &prob);
1626:   DMPlexGetDepthLabel(dm, &depthLabel);
1627:   DMGetDimension(dm, &dim);
1628:   DMLabelGetStratumIS(depthLabel,dim - 1,&facetIS);
1629:   PetscDSGetNumBoundary(prob, &numBd);
1630:   for (bd = 0; bd < numBd; ++bd) {
1631:     DMBoundaryConditionType type;
1632:     const char             *bdLabel;
1633:     DMLabel                 label;
1634:     const PetscInt         *values;
1635:     PetscInt                field, numValues;
1636:     PetscObject             obj;
1637:     PetscClassId            id;

1639:     PetscDSGetBoundary(prob, bd, &type, NULL, &bdLabel, &field, NULL, NULL, NULL, &numValues, &values, NULL);
1640:     PetscDSGetDiscretization(prob, field, &obj);
1641:     PetscObjectGetClassId(obj, &id);
1642:     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
1643:     if (!facetIS) {
1644:       DMLabel  depthLabel;
1645:       PetscInt dim;

1647:       DMPlexGetDepthLabel(dm, &depthLabel);
1648:       DMGetDimension(dm, &dim);
1649:       DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS);
1650:     }
1651:     DMGetCoordinateField(dm, &coordField);
1652:     DMGetLabel(dm, bdLabel, &label);
1653:     DMPlexComputeBdResidual_Single_Internal(dm, t, label, numValues, values, field, locX, locX_t, locF, coordField, facetIS);
1654:   }
1655:   ISDestroy(&facetIS);
1656:   return(0);
1657: }

1659: PetscErrorCode DMPlexComputeResidual_Internal(DM dm, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
1660: {
1661:   DM_Plex         *mesh       = (DM_Plex *) dm->data;
1662:   const char      *name       = "Residual";
1663:   DM               dmAux      = NULL;
1664:   DM               dmGrad     = NULL;
1665:   DMLabel          ghostLabel = NULL;
1666:   PetscDS          prob       = NULL;
1667:   PetscDS          probAux    = NULL;
1668:   PetscSection     section    = NULL;
1669:   PetscBool        useFEM     = PETSC_FALSE;
1670:   PetscBool        useFVM     = PETSC_FALSE;
1671:   PetscBool        isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
1672:   PetscFV          fvm        = NULL;
1673:   PetscFVCellGeom *cgeomFVM   = NULL;
1674:   PetscFVFaceGeom *fgeomFVM   = NULL;
1675:   DMField          coordField = NULL;
1676:   Vec              locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, grad, locGrad = NULL;
1677:   PetscScalar     *u = NULL, *u_t, *a, *uL, *uR;
1678:   IS               chunkIS;
1679:   const PetscInt  *cells;
1680:   PetscInt         cStart, cEnd, numCells;
1681:   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
1682:   PetscInt         maxDegree = PETSC_MAX_INT;
1683:   PetscQuadrature  affineQuad = NULL, *quads = NULL;
1684:   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
1685:   PetscErrorCode   ierr;

1688:   PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0);
1689:   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
1690:   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
1691:   /* FEM+FVM */
1692:   ISGetPointRange(cellIS, &cStart, &cEnd, &cells);
1693:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
1694:   /* 1: Get sizes from dm and dmAux */
1695:   DMGetSection(dm, &section);
1696:   DMGetLabel(dm, "ghost", &ghostLabel);
1697:   DMGetCellDS(dm, cStart, &prob);
1698:   PetscDSGetNumFields(prob, &Nf);
1699:   PetscDSGetTotalDimension(prob, &totDim);
1700:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &locA);
1701:   if (locA) {
1702:     PetscInt subcell;
1703:     DMPlexGetAuxiliaryPoint(dm, dmAux, cStart, &subcell);
1704:     VecGetDM(locA, &dmAux);
1705:     DMGetCellDS(dmAux, subcell, &probAux);
1706:     PetscDSGetTotalDimension(probAux, &totDimAux);
1707:   }
1708:   /* 2: Get geometric data */
1709:   for (f = 0; f < Nf; ++f) {
1710:     PetscObject  obj;
1711:     PetscClassId id;
1712:     PetscBool    fimp;

1714:     PetscDSGetImplicit(prob, f, &fimp);
1715:     if (isImplicit != fimp) continue;
1716:     PetscDSGetDiscretization(prob, f, &obj);
1717:     PetscObjectGetClassId(obj, &id);
1718:     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
1719:     if (id == PETSCFV_CLASSID) {useFVM = PETSC_TRUE; fvm = (PetscFV) obj;}
1720:   }
1721:   if (useFEM) {
1722:     DMGetCoordinateField(dm, &coordField);
1723:     DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);
1724:     if (maxDegree <= 1) {
1725:       DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad);
1726:       if (affineQuad) {
1727:         DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);
1728:       }
1729:     } else {
1730:       PetscCalloc2(Nf,&quads,Nf,&geoms);
1731:       for (f = 0; f < Nf; ++f) {
1732:         PetscObject  obj;
1733:         PetscClassId id;
1734:         PetscBool    fimp;

1736:         PetscDSGetImplicit(prob, f, &fimp);
1737:         if (isImplicit != fimp) continue;
1738:         PetscDSGetDiscretization(prob, f, &obj);
1739:         PetscObjectGetClassId(obj, &id);
1740:         if (id == PETSCFE_CLASSID) {
1741:           PetscFE fe = (PetscFE) obj;

1743:           PetscFEGetQuadrature(fe, &quads[f]);
1744:           PetscObjectReference((PetscObject)quads[f]);
1745:           DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);
1746:         }
1747:       }
1748:     }
1749:   }
1750:   if (useFVM) {
1751:     DMPlexSNESGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL);
1752:     VecGetArrayRead(faceGeometryFVM, (const PetscScalar **) &fgeomFVM);
1753:     VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM);
1754:     /* Reconstruct and limit cell gradients */
1755:     DMPlexSNESGetGradientDM(dm, fvm, &dmGrad);
1756:     if (dmGrad) {
1757:       DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
1758:       DMGetGlobalVector(dmGrad, &grad);
1759:       DMPlexReconstructGradients_Internal(dm, fvm, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad);
1760:       /* Communicate gradient values */
1761:       DMGetLocalVector(dmGrad, &locGrad);
1762:       DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad);
1763:       DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad);
1764:       DMRestoreGlobalVector(dmGrad, &grad);
1765:     }
1766:     /* Handle non-essential (e.g. outflow) boundary values */
1767:     DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad);
1768:   }
1769:   /* Loop over chunks */
1770:   if (useFEM) {ISCreate(PETSC_COMM_SELF, &chunkIS);}
1771:   numCells      = cEnd - cStart;
1772:   numChunks     = 1;
1773:   cellChunkSize = numCells/numChunks;
1774:   faceChunkSize = (fEnd - fStart)/numChunks;
1775:   numChunks     = PetscMin(1,numCells);
1776:   for (chunk = 0; chunk < numChunks; ++chunk) {
1777:     PetscScalar     *elemVec, *fluxL, *fluxR;
1778:     PetscReal       *vol;
1779:     PetscFVFaceGeom *fgeom;
1780:     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
1781:     PetscInt         fS = fStart+chunk*faceChunkSize, fE = PetscMin(fS+faceChunkSize, fEnd), numFaces = 0, face;

1783:     /* Extract field coefficients */
1784:     if (useFEM) {
1785:       ISGetPointSubrange(chunkIS, cS, cE, cells);
1786:       DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);
1787:       DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);
1788:       PetscMemzero(elemVec, numCells*totDim * sizeof(PetscScalar));
1789:     }
1790:     if (useFVM) {
1791:       DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR);
1792:       DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol);
1793:       DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL);
1794:       DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR);
1795:       PetscMemzero(fluxL, numFaces*totDim * sizeof(PetscScalar));
1796:       PetscMemzero(fluxR, numFaces*totDim * sizeof(PetscScalar));
1797:     }
1798:     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
1799:     /* Loop over fields */
1800:     for (f = 0; f < Nf; ++f) {
1801:       PetscObject  obj;
1802:       PetscClassId id;
1803:       PetscBool    fimp;
1804:       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;

1806:       PetscDSGetImplicit(prob, f, &fimp);
1807:       if (isImplicit != fimp) continue;
1808:       PetscDSGetDiscretization(prob, f, &obj);
1809:       PetscObjectGetClassId(obj, &id);
1810:       if (id == PETSCFE_CLASSID) {
1811:         PetscFE         fe = (PetscFE) obj;
1812:         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
1813:         PetscFEGeom    *chunkGeom = NULL;
1814:         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
1815:         PetscInt        Nq, Nb;

1817:         PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
1818:         PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL);
1819:         PetscFEGetDimension(fe, &Nb);
1820:         blockSize = Nb;
1821:         batchSize = numBlocks * blockSize;
1822:         PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
1823:         numChunks = numCells / (numBatches*batchSize);
1824:         Ne        = numChunks*numBatches*batchSize;
1825:         Nr        = numCells % (numBatches*batchSize);
1826:         offset    = numCells - Nr;
1827:         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
1828:         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
1829:         PetscFEGeomGetChunk(geom,0,offset,&chunkGeom);
1830:         PetscFEIntegrateResidual(fe, prob, f, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec);
1831:         PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom);
1832:         PetscFEIntegrateResidual(fe, prob, f, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]);
1833:         PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom);
1834:       } else if (id == PETSCFV_CLASSID) {
1835:         PetscFV fv = (PetscFV) obj;

1837:         Ne = numFaces;
1838:         /* Riemann solve over faces (need fields at face centroids) */
1839:         /*   We need to evaluate FE fields at those coordinates */
1840:         PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR);
1841:       } else SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %d", f);
1842:     }
1843:     /* Loop over domain */
1844:     if (useFEM) {
1845:       /* Add elemVec to locX */
1846:       for (c = cS; c < cE; ++c) {
1847:         const PetscInt cell = cells ? cells[c] : c;
1848:         const PetscInt cind = c - cStart;

1850:         if (mesh->printFEM > 1) {DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]);}
1851:         if (ghostLabel) {
1852:           PetscInt ghostVal;

1854:           DMLabelGetValue(ghostLabel,cell,&ghostVal);
1855:           if (ghostVal > 0) continue;
1856:         }
1857:         DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES);
1858:       }
1859:     }
1860:     if (useFVM) {
1861:       PetscScalar *fa;
1862:       PetscInt     iface;

1864:       VecGetArray(locF, &fa);
1865:       for (f = 0; f < Nf; ++f) {
1866:         PetscFV      fv;
1867:         PetscObject  obj;
1868:         PetscClassId id;
1869:         PetscInt     foff, pdim;

1871:         PetscDSGetDiscretization(prob, f, &obj);
1872:         PetscDSGetFieldOffset(prob, f, &foff);
1873:         PetscObjectGetClassId(obj, &id);
1874:         if (id != PETSCFV_CLASSID) continue;
1875:         fv   = (PetscFV) obj;
1876:         PetscFVGetNumComponents(fv, &pdim);
1877:         /* Accumulate fluxes to cells */
1878:         for (face = fS, iface = 0; face < fE; ++face) {
1879:           const PetscInt *scells;
1880:           PetscScalar    *fL = NULL, *fR = NULL;
1881:           PetscInt        ghost, d, nsupp, nchild;

1883:           DMLabelGetValue(ghostLabel, face, &ghost);
1884:           DMPlexGetSupportSize(dm, face, &nsupp);
1885:           DMPlexGetTreeChildren(dm, face, &nchild, NULL);
1886:           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
1887:           DMPlexGetSupport(dm, face, &scells);
1888:           DMLabelGetValue(ghostLabel,scells[0],&ghost);
1889:           if (ghost <= 0) {DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL);}
1890:           DMLabelGetValue(ghostLabel,scells[1],&ghost);
1891:           if (ghost <= 0) {DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR);}
1892:           for (d = 0; d < pdim; ++d) {
1893:             if (fL) fL[d] -= fluxL[iface*totDim+foff+d];
1894:             if (fR) fR[d] += fluxR[iface*totDim+foff+d];
1895:           }
1896:           ++iface;
1897:         }
1898:       }
1899:       VecRestoreArray(locF, &fa);
1900:     }
1901:     /* Handle time derivative */
1902:     if (locX_t) {
1903:       PetscScalar *x_t, *fa;

1905:       VecGetArray(locF, &fa);
1906:       VecGetArray(locX_t, &x_t);
1907:       for (f = 0; f < Nf; ++f) {
1908:         PetscFV      fv;
1909:         PetscObject  obj;
1910:         PetscClassId id;
1911:         PetscInt     pdim, d;

1913:         PetscDSGetDiscretization(prob, f, &obj);
1914:         PetscObjectGetClassId(obj, &id);
1915:         if (id != PETSCFV_CLASSID) continue;
1916:         fv   = (PetscFV) obj;
1917:         PetscFVGetNumComponents(fv, &pdim);
1918:         for (c = cS; c < cE; ++c) {
1919:           const PetscInt cell = cells ? cells[c] : c;
1920:           PetscScalar   *u_t, *r;

1922:           if (ghostLabel) {
1923:             PetscInt ghostVal;

1925:             DMLabelGetValue(ghostLabel, cell, &ghostVal);
1926:             if (ghostVal > 0) continue;
1927:           }
1928:           DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t);
1929:           DMPlexPointLocalFieldRef(dm, cell, f, fa, &r);
1930:           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
1931:         }
1932:       }
1933:       VecRestoreArray(locX_t, &x_t);
1934:       VecRestoreArray(locF, &fa);
1935:     }
1936:     if (useFEM) {
1937:       DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a);
1938:       DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec);
1939:     }
1940:     if (useFVM) {
1941:       DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR);
1942:       DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol);
1943:       DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL);
1944:       DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR);
1945:       if (dmGrad) {DMRestoreLocalVector(dmGrad, &locGrad);}
1946:     }
1947:   }
1948:   if (useFEM) {ISDestroy(&chunkIS);}
1949:   ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);

1951:   if (useFEM) {
1952:     DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user);

1954:     if (maxDegree <= 1) {
1955:       DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom);
1956:       PetscQuadratureDestroy(&affineQuad);
1957:     } else {
1958:       for (f = 0; f < Nf; ++f) {
1959:         DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]);
1960:         PetscQuadratureDestroy(&quads[f]);
1961:       }
1962:       PetscFree2(quads,geoms);
1963:     }
1964:   }

1966:   /* FEM */
1967:   /* 1: Get sizes from dm and dmAux */
1968:   /* 2: Get geometric data */
1969:   /* 3: Handle boundary values */
1970:   /* 4: Loop over domain */
1971:   /*   Extract coefficients */
1972:   /* Loop over fields */
1973:   /*   Set tiling for FE*/
1974:   /*   Integrate FE residual to get elemVec */
1975:   /*     Loop over subdomain */
1976:   /*       Loop over quad points */
1977:   /*         Transform coords to real space */
1978:   /*         Evaluate field and aux fields at point */
1979:   /*         Evaluate residual at point */
1980:   /*         Transform residual to real space */
1981:   /*       Add residual to elemVec */
1982:   /* Loop over domain */
1983:   /*   Add elemVec to locX */

1985:   /* FVM */
1986:   /* Get geometric data */
1987:   /* If using gradients */
1988:   /*   Compute gradient data */
1989:   /*   Loop over domain faces */
1990:   /*     Count computational faces */
1991:   /*     Reconstruct cell gradient */
1992:   /*   Loop over domain cells */
1993:   /*     Limit cell gradients */
1994:   /* Handle boundary values */
1995:   /* Loop over domain faces */
1996:   /*   Read out field, centroid, normal, volume for each side of face */
1997:   /* Riemann solve over faces */
1998:   /* Loop over domain faces */
1999:   /*   Accumulate fluxes to cells */
2000:   /* TODO Change printFEM to printDisc here */
2001:   if (mesh->printFEM) {
2002:     Vec         locFbc;
2003:     PetscInt    pStart, pEnd, p, maxDof;
2004:     PetscScalar *zeroes;

2006:     VecDuplicate(locF,&locFbc);
2007:     VecCopy(locF,locFbc);
2008:     PetscSectionGetChart(section,&pStart,&pEnd);
2009:     PetscSectionGetMaxDof(section,&maxDof);
2010:     PetscCalloc1(maxDof,&zeroes);
2011:     for (p = pStart; p < pEnd; p++) {
2012:       VecSetValuesSection(locFbc,section,p,zeroes,INSERT_BC_VALUES);
2013:     }
2014:     PetscFree(zeroes);
2015:     DMPrintLocalVec(dm, name, mesh->printTol, locFbc);
2016:     VecDestroy(&locFbc);
2017:   }
2018:   PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0);
2019:   return(0);
2020: }

2022: static PetscErrorCode DMPlexComputeResidualFEM_Check_Internal(DM dm, Vec X, Vec X_t, PetscReal t, Vec F, void *user)
2023: {
2024:   DM                dmCh, dmAux;
2025:   Vec               A;
2026:   DMField           coordField = NULL;
2027:   PetscDS           prob, probCh, probAux = NULL;
2028:   PetscSection      section, sectionAux;
2029:   PetscScalar      *elemVec, *elemVecCh, *u, *u_t, *a = NULL;
2030:   PetscInt          Nf, f, numCells, cStart, cEnd, c;
2031:   PetscInt          totDim, totDimAux = 0, diffCell = 0;
2032:   PetscInt          depth;
2033:   PetscInt          maxDegree;
2034:   IS                cellIS;
2035:   DMLabel           depthLabel;
2036:   PetscErrorCode    ierr;

2039:   DMGetSection(dm, &section);
2040:   DMGetDS(dm, &prob);
2041:   PetscDSGetTotalDimension(prob, &totDim);
2042:   PetscSectionGetNumFields(section, &Nf);
2043:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
2044:   numCells = cEnd - cStart;
2045:   PetscObjectQuery((PetscObject) dm, "dmCh", (PetscObject *) &dmCh);
2046:   DMGetDS(dmCh, &probCh);
2047:   PetscObjectQuery((PetscObject) dm, "dmAux", (PetscObject *) &dmAux);
2048:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &A);
2049:   if (dmAux) {
2050:     DMGetSection(dmAux, &sectionAux);
2051:     DMGetDS(dmAux, &probAux);
2052:     PetscDSGetTotalDimension(probAux, &totDimAux);
2053:   }
2054:   VecSet(F, 0.0);
2055:   PetscMalloc3(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,numCells*totDim,&elemVec);
2056:   PetscMalloc1(numCells*totDim,&elemVecCh);
2057:   if (dmAux) {PetscMalloc1(numCells*totDimAux, &a);}
2058:   DMPlexGetDepthLabel(dm, &depthLabel);
2059:   DMPlexGetDepth(dm,&depth);
2060:   DMLabelGetStratumIS(depthLabel,depth,&cellIS);
2061:   DMGetCoordinateField(dm, &coordField);
2062:   for (c = cStart; c < cEnd; ++c) {
2063:     PetscScalar *x = NULL, *x_t = NULL;
2064:     PetscInt     i;

2066:     DMPlexVecGetClosure(dm, section, X, c, NULL, &x);
2067:     for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
2068:     DMPlexVecRestoreClosure(dm, section, X, c, NULL, &x);
2069:     if (X_t) {
2070:       DMPlexVecGetClosure(dm, section, X_t, c, NULL, &x_t);
2071:       for (i = 0; i < totDim; ++i) u_t[c*totDim+i] = x_t[i];
2072:       DMPlexVecRestoreClosure(dm, section, X_t, c, NULL, &x_t);
2073:     }
2074:     if (dmAux) {
2075:       DM dmAuxPlex;

2077:       DMSNESConvertPlex(dmAux,&dmAuxPlex, PETSC_FALSE);
2078:       DMPlexVecGetClosure(dmAuxPlex, sectionAux, A, c, NULL, &x);
2079:       for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
2080:       DMPlexVecRestoreClosure(dmAuxPlex, sectionAux, A, c, NULL, &x);
2081:       DMDestroy(&dmAuxPlex);
2082:     }
2083:   }
2084:   for (f = 0; f < Nf; ++f) {
2085:     PetscFE  fe, feCh;
2086:     PetscInt Nq, Nb;
2087:     /* Conforming batches */
2088:     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
2089:     /* Remainder */
2090:     PetscInt Nr, offset;
2091:     PetscQuadrature qGeom = NULL;
2092:     PetscFEGeom *cgeomFEM, *chunkGeom = NULL;

2094:     PetscDSGetDiscretization(prob, f, (PetscObject *) &fe);
2095:     PetscDSGetDiscretization(probCh, f, (PetscObject *) &feCh);
2096:     PetscFEGetDimension(fe, &Nb);
2097:     PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
2098:     DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);
2099:     if (maxDegree <= 1) {
2100:       DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom);
2101:     }
2102:     if (!qGeom) {
2103:       PetscFEGetQuadrature(fe, &qGeom);
2104:       PetscObjectReference((PetscObject)qGeom);
2105:     }
2106:     PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);
2107:     DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);
2108:     blockSize = Nb;
2109:     batchSize = numBlocks * blockSize;
2110:      PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
2111:     numChunks = numCells / (numBatches*batchSize);
2112:     Ne        = numChunks*numBatches*batchSize;
2113:     Nr        = numCells % (numBatches*batchSize);
2114:     offset    = numCells - Nr;
2115:     PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);
2116:     PetscFEIntegrateResidual(fe, prob, f, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec);
2117:     PetscFEIntegrateResidual(feCh, prob, f, Ne, chunkGeom, u, u_t, probAux, a, t, elemVecCh);
2118:     PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&chunkGeom);
2119:     PetscFEIntegrateResidual(fe, prob, f, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]);
2120:     PetscFEIntegrateResidual(feCh, prob, f, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVecCh[offset*totDim]);
2121:     PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&chunkGeom);
2122:     DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);
2123:     PetscQuadratureDestroy(&qGeom);
2124:   }
2125:   ISDestroy(&cellIS);
2126:   for (c = cStart; c < cEnd; ++c) {
2127:     PetscBool diff = PETSC_FALSE;
2128:     PetscInt  d;

2130:     for (d = 0; d < totDim; ++d) if (PetscAbsScalar(elemVec[c*totDim+d] - elemVecCh[c*totDim+d]) > 1.0e-7) {diff = PETSC_TRUE;break;}
2131:     if (diff) {
2132:       PetscPrintf(PetscObjectComm((PetscObject) dm), "Different cell %d\n", c);
2133:       DMPrintCellVector(c, "Residual", totDim, &elemVec[c*totDim]);
2134:       DMPrintCellVector(c, "Check Residual", totDim, &elemVecCh[c*totDim]);
2135:       ++diffCell;
2136:     }
2137:     if (diffCell > 9) break;
2138:     DMPlexVecSetClosure(dm, section, F, c, &elemVec[c*totDim], ADD_ALL_VALUES);
2139:   }
2140:   PetscFree3(u,u_t,elemVec);
2141:   PetscFree(elemVecCh);
2142:   if (dmAux) {PetscFree(a);}
2143:   return(0);
2144: }

2146: /*@
2147:   DMPlexSNESComputeResidualFEM - Form the local residual F from the local input X using pointwise functions specified by the user

2149:   Input Parameters:
2150: + dm - The mesh
2151: . X  - Local solution
2152: - user - The user context

2154:   Output Parameter:
2155: . F  - Local output vector

2157:   Level: developer

2159: .seealso: DMPlexComputeJacobianAction()
2160: @*/
2161: PetscErrorCode DMPlexSNESComputeResidualFEM(DM dm, Vec X, Vec F, void *user)
2162: {
2163:   PetscObject    check;
2164:   DM             plex;
2165:   IS             cellIS;
2166:   PetscInt       depth;

2170:   DMSNESConvertPlex(dm,&plex,PETSC_TRUE);
2171:   DMPlexGetDepth(plex, &depth);
2172:   DMGetStratumIS(plex, "dim", depth, &cellIS);
2173:   if (!cellIS) {
2174:     DMGetStratumIS(plex, "depth", depth, &cellIS);
2175:   }
2176:   /* The dmCh is used to check two mathematically equivalent discretizations for computational equivalence */
2177:   PetscObjectQuery((PetscObject) plex, "dmCh", &check);
2178:   if (check) {DMPlexComputeResidualFEM_Check_Internal(plex, X, NULL, 0.0, F, user);}
2179:   else       {DMPlexComputeResidual_Internal(plex, cellIS, PETSC_MIN_REAL, X, NULL, 0.0, F, user);}
2180:   ISDestroy(&cellIS);
2181:   DMDestroy(&plex);
2182:   return(0);
2183: }

2185: /*@
2186:   DMPlexSNESComputeBoundaryFEM - Form the boundary values for the local input X

2188:   Input Parameters:
2189: + dm - The mesh
2190: - user - The user context

2192:   Output Parameter:
2193: . X  - Local solution

2195:   Level: developer

2197: .seealso: DMPlexComputeJacobianAction()
2198: @*/
2199: PetscErrorCode DMPlexSNESComputeBoundaryFEM(DM dm, Vec X, void *user)
2200: {
2201:   DM             plex;

2205:   DMSNESConvertPlex(dm,&plex,PETSC_TRUE);
2206:   DMPlexInsertBoundaryValues(plex, PETSC_TRUE, X, PETSC_MIN_REAL, NULL, NULL, NULL);
2207:   DMDestroy(&plex);
2208:   return(0);
2209: }

2211: PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS)
2212: {
2213:   DM_Plex       *mesh = (DM_Plex *) dm->data;
2214:   DM             plex = NULL, plexA = NULL;
2215:   PetscDS        prob, probAux = NULL;
2216:   PetscSection   section, sectionAux = NULL;
2217:   PetscSection   globalSection, subSection = NULL;
2218:   Vec            locA = NULL;
2219:   PetscScalar   *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL;
2220:   PetscInt       v;
2221:   PetscInt       Nf, totDim, totDimAux = 0;
2222:   PetscBool      isMatISP;

2226:   DMConvert(dm, DMPLEX, &plex);
2227:   DMGetSection(dm, &section);
2228:   DMGetDS(dm, &prob);
2229:   PetscDSGetNumFields(prob, &Nf);
2230:   PetscDSGetTotalDimension(prob, &totDim);
2231:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &locA);
2232:   if (locA) {
2233:     DM dmAux;

2235:     VecGetDM(locA, &dmAux);
2236:     DMConvert(dmAux, DMPLEX, &plexA);
2237:     DMGetDS(plexA, &probAux);
2238:     PetscDSGetTotalDimension(probAux, &totDimAux);
2239:     DMGetSection(plexA, &sectionAux);
2240:   }

2242:   PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);
2243:   DMGetGlobalSection(dm, &globalSection);
2244:   if (isMatISP) {DMPlexGetSubdomainSection(dm, &subSection);}
2245:   for (v = 0; v < numValues; ++v) {
2246:     PetscFEGeom    *fgeom;
2247:     PetscInt        maxDegree;
2248:     PetscQuadrature qGeom = NULL;
2249:     IS              pointIS;
2250:     const PetscInt *points;
2251:     PetscInt        numFaces, face, Nq;

2253:     DMLabelGetStratumIS(label, values[v], &pointIS);
2254:     if (!pointIS) continue; /* No points with that id on this process */
2255:     {
2256:       IS isectIS;

2258:       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
2259:       ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS);
2260:       ISDestroy(&pointIS);
2261:       pointIS = isectIS;
2262:     }
2263:     ISGetLocalSize(pointIS, &numFaces);
2264:     ISGetIndices(pointIS, &points);
2265:     PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim*totDim, &elemMat, locA ? numFaces*totDimAux : 0, &a);
2266:     DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree);
2267:     if (maxDegree <= 1) {
2268:       DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom);
2269:     }
2270:     if (!qGeom) {
2271:       PetscFE fe;

2273:       PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);
2274:       PetscFEGetFaceQuadrature(fe, &qGeom);
2275:       PetscObjectReference((PetscObject)qGeom);
2276:     }
2277:     PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);
2278:     DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);
2279:     for (face = 0; face < numFaces; ++face) {
2280:       const PetscInt point = points[face], *support, *cone;
2281:       PetscScalar   *x     = NULL;
2282:       PetscInt       i, coneSize, faceLoc;

2284:       DMPlexGetSupport(dm, point, &support);
2285:       DMPlexGetConeSize(dm, support[0], &coneSize);
2286:       DMPlexGetCone(dm, support[0], &cone);
2287:       for (faceLoc = 0; faceLoc < coneSize; ++faceLoc) if (cone[faceLoc] == point) break;
2288:       if (faceLoc == coneSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %d in cone of support[0] %d", point, support[0]);
2289:       fgeom->face[face][0] = faceLoc;
2290:       DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x);
2291:       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
2292:       DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x);
2293:       if (locX_t) {
2294:         DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x);
2295:         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
2296:         DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x);
2297:       }
2298:       if (locA) {
2299:         PetscInt subp;
2300:         DMPlexGetSubpoint(plexA, support[0], &subp);
2301:         DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x);
2302:         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
2303:         DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x);
2304:       }
2305:     }
2306:     PetscMemzero(elemMat, numFaces*totDim*totDim * sizeof(PetscScalar));
2307:     {
2308:       PetscFE         fe;
2309:       PetscInt        Nb;
2310:       /* Conforming batches */
2311:       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
2312:       /* Remainder */
2313:       PetscFEGeom    *chunkGeom = NULL;
2314:       PetscInt        fieldJ, Nr, offset;

2316:       PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);
2317:       PetscFEGetDimension(fe, &Nb);
2318:       PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
2319:       blockSize = Nb;
2320:       batchSize = numBlocks * blockSize;
2321:       PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
2322:       numChunks = numFaces / (numBatches*batchSize);
2323:       Ne        = numChunks*numBatches*batchSize;
2324:       Nr        = numFaces % (numBatches*batchSize);
2325:       offset    = numFaces - Nr;
2326:       PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom);
2327:       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
2328:         PetscFEIntegrateBdJacobian(fe, prob, fieldI, fieldJ, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat);
2329:       }
2330:       PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom);
2331:       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
2332:         PetscFEIntegrateBdJacobian(fe, prob, fieldI, fieldJ, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, X_tShift, &elemMat[offset*totDim*totDim]);
2333:       }
2334:       PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom);
2335:     }
2336:     for (face = 0; face < numFaces; ++face) {
2337:       const PetscInt point = points[face], *support;

2339:       if (mesh->printFEM > 1) {DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face*totDim*totDim]);}
2340:       DMPlexGetSupport(plex, point, &support);
2341:       if (!isMatISP) {
2342:         DMPlexMatSetClosure(plex, section, globalSection, JacP, support[0], &elemMat[face*totDim*totDim], ADD_VALUES);
2343:       } else {
2344:         Mat lJ;

2346:         MatISGetLocalMat(JacP, &lJ);
2347:         DMPlexMatSetClosure(plex, section, subSection, lJ, support[0], &elemMat[face*totDim*totDim], ADD_VALUES);
2348:       }
2349:     }
2350:     DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom);
2351:     PetscQuadratureDestroy(&qGeom);
2352:     ISRestoreIndices(pointIS, &points);
2353:     ISDestroy(&pointIS);
2354:     PetscFree4(u, u_t, elemMat, a);
2355:   }
2356:   if (plex)  {DMDestroy(&plex);}
2357:   if (plexA) {DMDestroy(&plexA);}
2358:   return(0);
2359: }

2361: PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP)
2362: {
2363:   DMField        coordField;
2364:   DMLabel        depthLabel;
2365:   IS             facetIS;
2366:   PetscInt       dim;

2370:   DMGetDimension(dm, &dim);
2371:   DMPlexGetDepthLabel(dm, &depthLabel);
2372:   DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);
2373:   DMGetCoordinateField(dm, &coordField);
2374:   DMPlexComputeBdJacobian_Single_Internal(dm, t, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS);
2375:   return(0);
2376: }

2378: PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user)
2379: {
2380:   PetscDS          prob;
2381:   PetscInt         dim, numBd, bd;
2382:   DMLabel          depthLabel;
2383:   DMField          coordField = NULL;
2384:   IS               facetIS;
2385:   PetscErrorCode   ierr;

2388:   DMGetDS(dm, &prob);
2389:   DMPlexGetDepthLabel(dm, &depthLabel);
2390:   DMGetDimension(dm, &dim);
2391:   DMLabelGetStratumIS(depthLabel, dim-1, &facetIS);
2392:   PetscDSGetNumBoundary(prob, &numBd);
2393:   DMGetCoordinateField(dm, &coordField);
2394:   for (bd = 0; bd < numBd; ++bd) {
2395:     DMBoundaryConditionType type;
2396:     const char             *bdLabel;
2397:     DMLabel                 label;
2398:     const PetscInt         *values;
2399:     PetscInt                fieldI, numValues;
2400:     PetscObject             obj;
2401:     PetscClassId            id;

2403:     PetscDSGetBoundary(prob, bd, &type, NULL, &bdLabel, &fieldI, NULL, NULL, NULL, &numValues, &values, NULL);
2404:     PetscDSGetDiscretization(prob, fieldI, &obj);
2405:     PetscObjectGetClassId(obj, &id);
2406:     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
2407:     DMGetLabel(dm, bdLabel, &label);
2408:     DMPlexComputeBdJacobian_Single_Internal(dm, t, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS);
2409:   }
2410:   ISDestroy(&facetIS);
2411:   return(0);
2412: }

2414: PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP,void *user)
2415: {
2416:   DM_Plex        *mesh  = (DM_Plex *) dm->data;
2417:   const char     *name  = "Jacobian";
2418:   DM              dmAux, plex;
2419:   Vec             A;
2420:   DMField         coordField;
2421:   PetscDS         prob, probAux = NULL;
2422:   PetscSection    section, globalSection, subSection, sectionAux;
2423:   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
2424:   const PetscInt *cells;
2425:   PetscInt        Nf, fieldI, fieldJ;
2426:   PetscInt        totDim, totDimAux, cStart, cEnd, numCells, c;
2427:   PetscBool       isMatIS, isMatISP, hasJac, hasPrec, hasDyn, hasFV = PETSC_FALSE;
2428:   PetscErrorCode  ierr;

2431:   PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);
2432:   DMGetSection(dm, &section);
2433:   PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatISP);
2434:   DMGetGlobalSection(dm, &globalSection);
2435:   if (isMatISP) {DMPlexGetSubdomainSection(dm, &subSection);}
2436:   DMGetDS(dm, &prob);
2437:   PetscDSGetTotalDimension(prob, &totDim);
2438:   PetscDSHasJacobian(prob, &hasJac);
2439:   PetscDSHasJacobianPreconditioner(prob, &hasPrec);
2440:   PetscDSHasDynamicJacobian(prob, &hasDyn);
2441:   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
2442:   PetscSectionGetNumFields(section, &Nf);
2443:   ISGetLocalSize(cellIS, &numCells);
2444:   ISGetPointRange(cellIS, &cStart, &cEnd, &cells);
2445:   PetscObjectQuery((PetscObject) dm, "dmAux", (PetscObject *) &dmAux);
2446:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &A);
2447:   if (dmAux) {
2448:     DMConvert(dmAux, DMPLEX, &plex);
2449:     DMGetSection(plex, &sectionAux);
2450:     DMGetDS(dmAux, &probAux);
2451:     PetscDSGetTotalDimension(probAux, &totDimAux);
2452:   }
2453:   PetscMalloc5(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,hasJac ? numCells*totDim*totDim : 0,&elemMat,hasPrec ? numCells*totDim*totDim : 0, &elemMatP,hasDyn ? numCells*totDim*totDim : 0, &elemMatD);
2454:   if (dmAux) {PetscMalloc1(numCells*totDimAux, &a);}
2455:   DMGetCoordinateField(dm, &coordField);
2456:   for (c = cStart; c < cEnd; ++c) {
2457:     const PetscInt cell = cells ? cells[c] : c;
2458:     const PetscInt cind = c - cStart;
2459:     PetscScalar   *x = NULL,  *x_t = NULL;
2460:     PetscInt       i;

2462:     DMPlexVecGetClosure(dm, section, X, cell, NULL, &x);
2463:     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
2464:     DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x);
2465:     if (X_t) {
2466:       DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t);
2467:       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
2468:       DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t);
2469:     }
2470:     if (dmAux) {
2471:       PetscInt subcell;
2472:       DMPlexGetAuxiliaryPoint(dm, dmAux, cell, &subcell);
2473:       DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x);
2474:       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
2475:       DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x);
2476:     }
2477:   }
2478:   if (hasJac)  {PetscMemzero(elemMat,  numCells*totDim*totDim * sizeof(PetscScalar));}
2479:   if (hasPrec) {PetscMemzero(elemMatP, numCells*totDim*totDim * sizeof(PetscScalar));}
2480:   if (hasDyn)  {PetscMemzero(elemMatD, numCells*totDim*totDim * sizeof(PetscScalar));}
2481:   for (fieldI = 0; fieldI < Nf; ++fieldI) {
2482:     PetscClassId    id;
2483:     PetscFE         fe;
2484:     PetscQuadrature qGeom = NULL;
2485:     PetscInt        Nb;
2486:     /* Conforming batches */
2487:     PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
2488:     /* Remainder */
2489:     PetscInt        Nr, offset, Nq;
2490:     PetscInt        maxDegree;
2491:     PetscFEGeom     *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;

2493:     PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);
2494:     PetscObjectGetClassId((PetscObject) fe, &id);
2495:     if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; continue;}
2496:     PetscFEGetDimension(fe, &Nb);
2497:     PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
2498:     DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);
2499:     if (maxDegree <= 1) {
2500:       DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom);
2501:     }
2502:     if (!qGeom) {
2503:       PetscFEGetQuadrature(fe,&qGeom);
2504:       PetscObjectReference((PetscObject)qGeom);
2505:     }
2506:     PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);
2507:     DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);
2508:     blockSize = Nb;
2509:     batchSize = numBlocks * blockSize;
2510:     PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
2511:     numChunks = numCells / (numBatches*batchSize);
2512:     Ne        = numChunks*numBatches*batchSize;
2513:     Nr        = numCells % (numBatches*batchSize);
2514:     offset    = numCells - Nr;
2515:     PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);
2516:     PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom);
2517:     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
2518:       if (hasJac) {
2519:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN, fieldI, fieldJ, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat);
2520:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN, fieldI, fieldJ, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]);
2521:       }
2522:       if (hasPrec) {
2523:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_PRE, fieldI, fieldJ, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP);
2524:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_PRE, fieldI, fieldJ, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatP[offset*totDim*totDim]);
2525:       }
2526:       if (hasDyn) {
2527:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_DYN, fieldI, fieldJ, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD);
2528:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_DYN, fieldI, fieldJ, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]);
2529:       }
2530:     }
2531:     PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom);
2532:     PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom);
2533:     DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);
2534:     PetscQuadratureDestroy(&qGeom);
2535:   }
2536:   /*   Add contribution from X_t */
2537:   if (hasDyn) {for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
2538:   if (hasFV) {
2539:     PetscClassId id;
2540:     PetscFV      fv;
2541:     PetscInt     offsetI, NcI, NbI = 1, fc, f;

2543:     for (fieldI = 0; fieldI < Nf; ++fieldI) {
2544:       PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv);
2545:       PetscDSGetFieldOffset(prob, fieldI, &offsetI);
2546:       PetscObjectGetClassId((PetscObject) fv, &id);
2547:       if (id != PETSCFV_CLASSID) continue;
2548:       /* Put in the identity */
2549:       PetscFVGetNumComponents(fv, &NcI);
2550:       for (c = cStart; c < cEnd; ++c) {
2551:         const PetscInt cind    = c - cStart;
2552:         const PetscInt eOffset = cind*totDim*totDim;
2553:         for (fc = 0; fc < NcI; ++fc) {
2554:           for (f = 0; f < NbI; ++f) {
2555:             const PetscInt i = offsetI + f*NcI+fc;
2556:             if (hasPrec) {
2557:               if (hasJac) {elemMat[eOffset+i*totDim+i] = 1.0;}
2558:               elemMatP[eOffset+i*totDim+i] = 1.0;
2559:             } else {elemMat[eOffset+i*totDim+i] = 1.0;}
2560:           }
2561:         }
2562:       }
2563:     }
2564:     /* No allocated space for FV stuff, so ignore the zero entries */
2565:     MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE);
2566:   }
2567:   /* Insert values into matrix */
2568:   isMatIS = PETSC_FALSE;
2569:   if (hasPrec && hasJac) {
2570:     PetscObjectTypeCompare((PetscObject) JacP, MATIS, &isMatIS);
2571:   }
2572:   if (isMatIS && !subSection) {
2573:     DMPlexGetSubdomainSection(dm, &subSection);
2574:   }
2575:   for (c = cStart; c < cEnd; ++c) {
2576:     const PetscInt cell = cells ? cells[c] : c;
2577:     const PetscInt cind = c - cStart;

2579:     if (hasPrec) {
2580:       if (hasJac) {
2581:         if (mesh->printFEM > 1) {DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]);}
2582:         if (!isMatIS) {
2583:           DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);
2584:         } else {
2585:           Mat lJ;

2587:           MatISGetLocalMat(Jac,&lJ);
2588:           DMPlexMatSetClosure(dm, section, subSection, lJ, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);
2589:         }
2590:       }
2591:       if (mesh->printFEM > 1) {DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]);}
2592:       if (!isMatISP) {
2593:         DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES);
2594:       } else {
2595:         Mat lJ;

2597:         MatISGetLocalMat(JacP,&lJ);
2598:         DMPlexMatSetClosure(dm, section, subSection, lJ, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES);
2599:       }
2600:     } else {
2601:       if (mesh->printFEM > 1) {DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]);}
2602:       if (!isMatISP) {
2603:         DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);
2604:       } else {
2605:         Mat lJ;

2607:         MatISGetLocalMat(JacP,&lJ);
2608:         DMPlexMatSetClosure(dm, section, subSection, lJ, cell, &elemMat[cind*totDim*totDim], ADD_VALUES);
2609:       }
2610:     }
2611:   }
2612:   ISRestorePointRange(cellIS, &cStart, &cEnd, &cells);
2613:   if (hasFV) {MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE);}
2614:   PetscFree5(u,u_t,elemMat,elemMatP,elemMatD);
2615:   if (dmAux) {
2616:     PetscFree(a);
2617:     DMDestroy(&plex);
2618:   }
2619:   /* Compute boundary integrals */
2620:   DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user);
2621:   /* Assemble matrix */
2622:   if (hasJac && hasPrec) {
2623:     MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY);
2624:     MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY);
2625:   }
2626:   MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY);
2627:   MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY);
2628:   PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);
2629:   return(0);
2630: }

2632: /*@
2633:   DMPlexComputeJacobianAction - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user.

2635:   Input Parameters:
2636: + dm - The mesh
2637: . cellIS - 
2638: . t  - The time
2639: . X_tShift - The multiplier for the Jacobian with repsect to X_t
2640: . X  - Local solution vector
2641: . X_t  - Time-derivative of the local solution vector
2642: . Y  - Local input vector
2643: - user - The user context

2645:   Output Parameter:
2646: . Z - Local output vector

2648:   Note:
2649:   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
2650:   like a GPU, or vectorize on a multicore machine.

2652:   Level: developer

2654: .seealso: FormFunctionLocal()
2655: @*/
2656: PetscErrorCode DMPlexComputeJacobianAction(DM dm, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user)
2657: {
2658:   DM_Plex          *mesh  = (DM_Plex *) dm->data;
2659:   const char       *name  = "Jacobian";
2660:   DM                dmAux, plex, plexAux = NULL;
2661:   Vec               A;
2662:   PetscDS           prob, probAux = NULL;
2663:   PetscQuadrature   quad;
2664:   PetscSection      section, globalSection, sectionAux;
2665:   PetscScalar      *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
2666:   PetscInt          Nf, fieldI, fieldJ;
2667:   PetscInt          totDim, totDimAux = 0;
2668:   const PetscInt   *cells;
2669:   PetscInt          cStart, cEnd, numCells, c;
2670:   PetscBool         hasDyn;
2671:   DMField           coordField;
2672:   PetscErrorCode    ierr;

2675:   PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0);
2676:   DMSNESConvertPlex(dm, &plex, PETSC_TRUE);
2677:   if (!cellIS) {
2678:     PetscInt depth;

2680:     DMPlexGetDepth(plex, &depth);
2681:     DMGetStratumIS(plex, "dim", depth, &cellIS);
2682:     if (!cellIS) {DMGetStratumIS(plex, "depth", depth, &cellIS);}
2683:   } else {
2684:     PetscObjectReference((PetscObject) cellIS);
2685:   }
2686:   DMGetSection(dm, &section);
2687:   DMGetGlobalSection(dm, &globalSection);
2688:   DMGetDS(dm, &prob);
2689:   PetscDSGetTotalDimension(prob, &totDim);
2690:   PetscDSHasDynamicJacobian(prob, &hasDyn);
2691:   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
2692:   PetscSectionGetNumFields(section, &Nf);
2693:   ISGetLocalSize(cellIS, &numCells);
2694:   ISGetPointRange(cellIS, &cStart, &cEnd, &cells);
2695:   PetscObjectQuery((PetscObject) dm, "dmAux", (PetscObject *) &dmAux);
2696:   PetscObjectQuery((PetscObject) dm, "A", (PetscObject *) &A);
2697:   if (dmAux) {
2698:     DMConvert(dmAux, DMPLEX, &plexAux);
2699:     DMGetSection(plexAux, &sectionAux);
2700:     DMGetDS(dmAux, &probAux);
2701:     PetscDSGetTotalDimension(probAux, &totDimAux);
2702:   }
2703:   VecSet(Z, 0.0);
2704:   PetscMalloc6(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,numCells*totDim*totDim,&elemMat,hasDyn ? numCells*totDim*totDim : 0, &elemMatD,numCells*totDim,&y,totDim,&z);
2705:   if (dmAux) {PetscMalloc1(numCells*totDimAux, &a);}
2706:   DMGetCoordinateField(dm, &coordField);
2707:   for (c = cStart; c < cEnd; ++c) {
2708:     const PetscInt cell = cells ? cells[c] : c;
2709:     const PetscInt cind = c - cStart;
2710:     PetscScalar   *x = NULL,  *x_t = NULL;
2711:     PetscInt       i;

2713:     DMPlexVecGetClosure(dm, section, X, cell, NULL, &x);
2714:     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
2715:     DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x);
2716:     if (X_t) {
2717:       DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t);
2718:       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
2719:       DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t);
2720:     }
2721:     if (dmAux) {
2722:       PetscInt subcell;
2723:       DMPlexGetAuxiliaryPoint(dm, dmAux, cell, &subcell);
2724:       DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x);
2725:       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
2726:       DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x);
2727:     }
2728:     DMPlexVecGetClosure(dm, section, Y, cell, NULL, &x);
2729:     for (i = 0; i < totDim; ++i) y[cind*totDim+i] = x[i];
2730:     DMPlexVecRestoreClosure(dm, section, Y, cell, NULL, &x);
2731:   }
2732:   PetscMemzero(elemMat, numCells*totDim*totDim * sizeof(PetscScalar));
2733:   if (hasDyn)  {PetscMemzero(elemMatD, numCells*totDim*totDim * sizeof(PetscScalar));}
2734:   for (fieldI = 0; fieldI < Nf; ++fieldI) {
2735:     PetscFE  fe;
2736:     PetscInt Nb;
2737:     /* Conforming batches */
2738:     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
2739:     /* Remainder */
2740:     PetscInt Nr, offset, Nq;
2741:     PetscQuadrature qGeom = NULL;
2742:     PetscInt    maxDegree;
2743:     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;

2745:     PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe);
2746:     PetscFEGetQuadrature(fe, &quad);
2747:     PetscFEGetDimension(fe, &Nb);
2748:     PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches);
2749:     DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree);
2750:     if (maxDegree <= 1) {DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom);}
2751:     if (!qGeom) {
2752:       PetscFEGetQuadrature(fe,&qGeom);
2753:       PetscObjectReference((PetscObject)qGeom);
2754:     }
2755:     PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL);
2756:     DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);
2757:     blockSize = Nb;
2758:     batchSize = numBlocks * blockSize;
2759:     PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches);
2760:     numChunks = numCells / (numBatches*batchSize);
2761:     Ne        = numChunks*numBatches*batchSize;
2762:     Nr        = numCells % (numBatches*batchSize);
2763:     offset    = numCells - Nr;
2764:     PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom);
2765:     PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom);
2766:     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
2767:       PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN, fieldI, fieldJ, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat);
2768:       PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN, fieldI, fieldJ, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]);
2769:       if (hasDyn) {
2770:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_DYN, fieldI, fieldJ, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD);
2771:         PetscFEIntegrateJacobian(fe, prob, PETSCFE_JACOBIAN_DYN, fieldI, fieldJ, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]);
2772:       }
2773:     }
2774:     PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom);
2775:     PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom);
2776:     DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM);
2777:     PetscQuadratureDestroy(&qGeom);
2778:   }
2779:   if (hasDyn) {
2780:     for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];
2781:   }
2782:   for (c = cStart; c < cEnd; ++c) {
2783:     const PetscInt     cell = cells ? cells[c] : c;
2784:     const PetscInt     cind = c - cStart;
2785:     const PetscBLASInt M = totDim, one = 1;
2786:     const PetscScalar  a = 1.0, b = 0.0;

2788:     PetscStackCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind*totDim*totDim], &M, &y[cind*totDim], &one, &b, z, &one));
2789:     if (mesh->printFEM > 1) {
2790:       DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind*totDim*totDim]);
2791:       DMPrintCellVector(c, "Y",  totDim, &y[cind*totDim]);
2792:       DMPrintCellVector(c, "Z",  totDim, z);
2793:     }
2794:     DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES);
2795:   }
2796:   PetscFree6(u,u_t,elemMat,elemMatD,y,z);
2797:   if (mesh->printFEM) {
2798:     PetscPrintf(PETSC_COMM_WORLD, "Z:\n");
2799:     VecView(Z, PETSC_VIEWER_STDOUT_WORLD);
2800:   }
2801:   PetscFree(a);
2802:   ISDestroy(&cellIS);
2803:   DMDestroy(&plexAux);
2804:   DMDestroy(&plex);
2805:   PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0);
2806:   return(0);
2807: }

2809: /*@
2810:   DMPlexSNESComputeJacobianFEM - Form the local portion of the Jacobian matrix J at the local solution X using pointwise functions specified by the user.

2812:   Input Parameters:
2813: + dm - The mesh
2814: . X  - Local input vector
2815: - user - The user context

2817:   Output Parameter:
2818: . Jac  - Jacobian matrix

2820:   Note:
2821:   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
2822:   like a GPU, or vectorize on a multicore machine.

2824:   Level: developer

2826: .seealso: FormFunctionLocal()
2827: @*/
2828: PetscErrorCode DMPlexSNESComputeJacobianFEM(DM dm, Vec X, Mat Jac, Mat JacP,void *user)
2829: {
2830:   DM             plex;
2831:   PetscDS        prob;
2832:   IS             cellIS;
2833:   PetscBool      hasJac, hasPrec;
2834:   PetscInt       depth;

2838:   DMSNESConvertPlex(dm,&plex,PETSC_TRUE);
2839:   DMPlexGetDepth(plex, &depth);
2840:   DMGetStratumIS(plex, "dim", depth, &cellIS);
2841:   if (!cellIS) {DMGetStratumIS(plex, "depth", depth, &cellIS);}
2842:   DMGetDS(dm, &prob);
2843:   PetscDSHasJacobian(prob, &hasJac);
2844:   PetscDSHasJacobianPreconditioner(prob, &hasPrec);
2845:   if (hasJac && hasPrec) {MatZeroEntries(Jac);}
2846:   MatZeroEntries(JacP);
2847:   DMPlexComputeJacobian_Internal(plex, cellIS, 0.0, 0.0, X, NULL, Jac, JacP, user);
2848:   ISDestroy(&cellIS);
2849:   DMDestroy(&plex);
2850:   return(0);
2851: }

2853: /*@
2854:   DMPlexSetSNESLocalFEM - Use DMPlex's internal FEM routines to compute SNES boundary values, residual, and Jacobian.

2856:   Input Parameters:
2857: + dm - The DM object
2858: . boundaryctx - the user context that will be passed to pointwise evaluation of boundary values (see PetscDSAddBoundary())
2859: . residualctx - the user context that will be passed to pointwise evaluation of finite element residual computations (see PetscDSSetResidual())
2860: - jacobianctx - the user context that will be passed to pointwise evaluation of finite element Jacobian construction (see PetscDSSetJacobian())

2862:   Level: developer
2863: @*/
2864: PetscErrorCode DMPlexSetSNESLocalFEM(DM dm, void *boundaryctx, void *residualctx, void *jacobianctx)
2865: {

2869:   DMSNESSetBoundaryLocal(dm,DMPlexSNESComputeBoundaryFEM,boundaryctx);
2870:   DMSNESSetFunctionLocal(dm,DMPlexSNESComputeResidualFEM,residualctx);
2871:   DMSNESSetJacobianLocal(dm,DMPlexSNESComputeJacobianFEM,jacobianctx);
2872:   return(0);
2873: }

2875: PetscErrorCode DMSNESCheckFromOptions_Internal(SNES snes, DM dm, Vec u, PetscErrorCode (**exactFuncs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx), void **ctxs)
2876: {
2877:   PetscErrorCode (**exacts)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
2878:   PetscDS        prob;
2879:   Mat            J, M;
2880:   Vec            r, b;
2881:   MatNullSpace   nullSpace;
2882:   PetscReal     *error, res = 0.0;
2883:   PetscInt       numFields;
2884:   PetscBool      hasJac, hasPrec;
2885:   PetscInt       Nf, f;

2889:   DMGetNumFields(dm, &Nf);
2890:   DMGetDS(dm, &prob);
2891:   PetscMalloc1(Nf, &exacts);
2892:   for (f = 0; f < Nf; ++f) {PetscDSGetExactSolution(prob, f, &exacts[f]);}
2893:   VecDuplicate(u, &r);
2894:   DMCreateMatrix(dm, &J);
2895:   /* TODO Null space for J */
2896:   /* Check discretization error */
2897:   DMGetNumFields(dm, &numFields);
2898:   PetscMalloc1(PetscMax(1, numFields), &error);
2899:   DMProjectFunction(dm, 0.0, exactFuncs ? exactFuncs : exacts, ctxs, INSERT_ALL_VALUES, u);
2900:   if (numFields > 1) {
2901:     PetscInt f;

2903:     DMComputeL2FieldDiff(dm, 0.0, exactFuncs ? exactFuncs : exacts, ctxs, u, error);
2904:     PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: [");
2905:     for (f = 0; f < numFields; ++f) {
2906:       if (f) {PetscPrintf(PETSC_COMM_WORLD, ", ");}
2907:       if (error[f] >= 1.0e-11) {PetscPrintf(PETSC_COMM_WORLD, "%g", (double)error[f]);}
2908:       else                     {PetscPrintf(PETSC_COMM_WORLD, "< 1.0e-11");}
2909:     }
2910:     PetscPrintf(PETSC_COMM_WORLD, "]\n");
2911:   } else {
2912:     DMComputeL2Diff(dm, 0.0, exactFuncs ? exactFuncs : exacts, ctxs, u, &error[0]);
2913:     if (error[0] >= 1.0e-11) {PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %g\n", (double)error[0]);}
2914:     else                     {PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: < 1.0e-11\n");}
2915:   }
2916:   PetscFree(error);
2917:   /* Check residual */
2918:   SNESComputeFunction(snes, u, r);
2919:   VecNorm(r, NORM_2, &res);
2920:   PetscPrintf(PETSC_COMM_WORLD, "L_2 Residual: %g\n", (double)res);
2921:   VecChop(r, 1.0e-10);
2922:   PetscObjectSetName((PetscObject) r, "Initial Residual");
2923:   PetscObjectSetOptionsPrefix((PetscObject)r,"res_");
2924:   VecViewFromOptions(r, NULL, "-vec_view");
2925:   /* Check Jacobian */
2926:   PetscDSHasJacobian(prob, &hasJac);
2927:   PetscDSHasJacobianPreconditioner(prob, &hasPrec);
2928:   if (hasJac && hasPrec) {
2929:     DMCreateMatrix(dm, &M);
2930:     SNESComputeJacobian(snes, u, J, M);
2931:     PetscObjectSetOptionsPrefix((PetscObject) M, "jacpre_");
2932:     MatViewFromOptions(M, NULL, "-mat_view");
2933:     MatDestroy(&M);
2934:   } else {
2935:     SNESComputeJacobian(snes, u, J, J);
2936:   }
2937:   PetscObjectSetOptionsPrefix((PetscObject) J, "jac_");
2938:   MatViewFromOptions(J, NULL, "-mat_view");
2939:   MatGetNullSpace(J, &nullSpace);
2940:   if (nullSpace) {
2941:     PetscBool isNull;
2942:     MatNullSpaceTest(nullSpace, J, &isNull);
2943:     if (!isNull) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_PLIB, "The null space calculated for the system operator is invalid.");
2944:   }
2945:   VecDuplicate(u, &b);
2946:   VecSet(r, 0.0);
2947:   SNESComputeFunction(snes, r, b);
2948:   MatMult(J, u, r);
2949:   VecAXPY(r, 1.0, b);
2950:   VecDestroy(&b);
2951:   VecNorm(r, NORM_2, &res);
2952:   PetscPrintf(PETSC_COMM_WORLD, "Linear L_2 Residual: %g\n", (double)res);
2953:   VecChop(r, 1.0e-10);
2954:   PetscObjectSetName((PetscObject) r, "Au - b = Au + F(0)");
2955:   PetscObjectSetOptionsPrefix((PetscObject)r,"linear_res_");
2956:   VecViewFromOptions(r, NULL, "-vec_view");
2957:   VecDestroy(&r);
2958:   MatNullSpaceDestroy(&nullSpace);
2959:   MatDestroy(&J);
2960:   PetscFree(exacts);
2961:   return(0);
2962: }

2964: /*@C
2965:   DMSNESCheckFromOptions - Check the residual and Jacobian functions using the exact solution by outputting some diagnostic information

2967:   Input Parameters:
2968: + snes - the SNES object
2969: . u    - representative SNES vector
2970: . exactFuncs - pointwise functions of the exact solution for each field
2971: - ctxs - contexts for the functions

2973:   Level: developer
2974: @*/
2975: PetscErrorCode DMSNESCheckFromOptions(SNES snes, Vec u, PetscErrorCode (**exactFuncs)(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf, PetscScalar *u, void *ctx), void **ctxs)
2976: {
2977:   PetscErrorCode (**exact)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *) = NULL;
2978:   DM             dm;
2979:   PetscDS        prob;
2980:   Vec            sol;
2981:   PetscBool      check;
2982:   PetscInt       Nf, f;

2986:   PetscOptionsHasName(((PetscObject)snes)->options,((PetscObject)snes)->prefix, "-dmsnes_check", &check);
2987:   if (!check) return(0);
2988:   SNESGetDM(snes, &dm);
2989:   DMGetDS(dm, &prob);
2990:   if (!exactFuncs) {
2991:     PetscDSGetNumFields(prob, &Nf);
2992:     PetscMalloc1(Nf, &exact);
2993:     for (f = 0; f < Nf; ++f) {PetscDSGetExactSolution(prob, f, &exact[f]);}
2994:   }
2995:   VecDuplicate(u, &sol);
2996:   SNESSetSolution(snes, sol);
2997:   DMSNESCheckFromOptions_Internal(snes, dm, sol, exactFuncs ? exactFuncs : exact, ctxs);
2998:   VecDestroy(&sol);
2999:   PetscFree(exact);
3000:   return(0);
3001: }