Actual source code: plexhdf5.c

petsc-3.8.3 2017-12-09
Report Typos and Errors
  1:  #include <petsc/private/dmpleximpl.h>
  2:  #include <petsc/private/isimpl.h>
  3:  #include <petsc/private/vecimpl.h>
  4: #include <petscviewerhdf5.h>

  6: PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);

  8: #if defined(PETSC_HAVE_HDF5)
  9: static PetscErrorCode DMSequenceView_HDF5(DM dm, const char *seqname, PetscInt seqnum, PetscScalar value, PetscViewer viewer)
 10: {
 11:   Vec            stamp;
 12:   PetscMPIInt    rank;

 16:   if (seqnum < 0) return(0);
 17:   MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);
 18:   VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);
 19:   VecSetBlockSize(stamp, 1);
 20:   PetscObjectSetName((PetscObject) stamp, seqname);
 21:   if (!rank) {
 22:     PetscReal timeScale;
 23:     PetscBool istime;

 25:     PetscStrncmp(seqname, "time", 5, &istime);
 26:     if (istime) {DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale); value *= timeScale;}
 27:     VecSetValue(stamp, 0, value, INSERT_VALUES);
 28:   }
 29:   VecAssemblyBegin(stamp);
 30:   VecAssemblyEnd(stamp);
 31:   PetscViewerHDF5PushGroup(viewer, "/");
 32:   PetscViewerHDF5SetTimestep(viewer, seqnum);
 33:   VecView(stamp, viewer);
 34:   PetscViewerHDF5PopGroup(viewer);
 35:   VecDestroy(&stamp);
 36:   return(0);
 37: }

 39: PetscErrorCode DMSequenceLoad_HDF5_Internal(DM dm, const char *seqname, PetscInt seqnum, PetscScalar *value, PetscViewer viewer)
 40: {
 41:   Vec            stamp;
 42:   PetscMPIInt    rank;

 46:   if (seqnum < 0) return(0);
 47:   MPI_Comm_rank(PetscObjectComm((PetscObject) viewer), &rank);
 48:   VecCreateMPI(PetscObjectComm((PetscObject) viewer), rank ? 0 : 1, 1, &stamp);
 49:   VecSetBlockSize(stamp, 1);
 50:   PetscObjectSetName((PetscObject) stamp, seqname);
 51:   PetscViewerHDF5PushGroup(viewer, "/");
 52:   PetscViewerHDF5SetTimestep(viewer, seqnum);
 53:   VecLoad(stamp, viewer);
 54:   PetscViewerHDF5PopGroup(viewer);
 55:   if (!rank) {
 56:     const PetscScalar *a;
 57:     PetscReal timeScale;
 58:     PetscBool istime;

 60:     VecGetArrayRead(stamp, &a);
 61:     *value = a[0];
 62:     VecRestoreArrayRead(stamp, &a);
 63:     PetscStrncmp(seqname, "time", 5, &istime);
 64:     if (istime) {DMPlexGetScale(dm, PETSC_UNIT_TIME, &timeScale); *value /= timeScale;}
 65:   }
 66:   VecDestroy(&stamp);
 67:   return(0);
 68: }

 70: PetscErrorCode VecView_Plex_Local_HDF5_Internal(Vec v, PetscViewer viewer)
 71: {
 72:   DM                      dm;
 73:   DM                      dmBC;
 74:   PetscSection            section, sectionGlobal;
 75:   Vec                     gv;
 76:   const char             *name;
 77:   PetscViewerVTKFieldType ft;
 78:   PetscViewerFormat       format;
 79:   PetscInt                seqnum;
 80:   PetscReal               seqval;
 81:   PetscBool               isseq;
 82:   PetscErrorCode          ierr;

 85:   PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);
 86:   VecGetDM(v, &dm);
 87:   DMGetDefaultSection(dm, &section);
 88:   DMGetOutputSequenceNumber(dm, &seqnum, &seqval);
 89:   PetscViewerHDF5SetTimestep(viewer, seqnum);
 90:   DMSequenceView_HDF5(dm, "time", seqnum, (PetscScalar) seqval, viewer);
 91:   PetscViewerGetFormat(viewer, &format);
 92:   DMGetOutputDM(dm, &dmBC);
 93:   DMGetDefaultGlobalSection(dmBC, &sectionGlobal);
 94:   DMGetGlobalVector(dmBC, &gv);
 95:   PetscObjectGetName((PetscObject) v, &name);
 96:   PetscObjectSetName((PetscObject) gv, name);
 97:   DMLocalToGlobalBegin(dmBC, v, INSERT_VALUES, gv);
 98:   DMLocalToGlobalEnd(dmBC, v, INSERT_VALUES, gv);
 99:   PetscObjectTypeCompare((PetscObject) gv, VECSEQ, &isseq);
100:   if (isseq) {VecView_Seq(gv, viewer);}
101:   else       {VecView_MPI(gv, viewer);}
102:   if (format == PETSC_VIEWER_HDF5_VIZ) {
103:     /* Output visualization representation */
104:     PetscInt numFields, f;
105:     DMLabel  cutLabel;

107:     PetscSectionGetNumFields(section, &numFields);
108:     DMGetLabel(dm, "periodic_cut", &cutLabel);
109:     for (f = 0; f < numFields; ++f) {
110:       Vec         subv;
111:       IS          is;
112:       const char *fname, *fgroup;
113:       char        subname[PETSC_MAX_PATH_LEN];
114:       char        group[PETSC_MAX_PATH_LEN];
115:       PetscInt    pStart, pEnd;
116:       PetscBool   flag;

118:       DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft);
119:       fgroup = (ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_POINT_FIELD) ? "/vertex_fields" : "/cell_fields";
120:       PetscSectionGetFieldName(section, f, &fname);
121:       if (!fname) continue;
122:       PetscViewerHDF5PushGroup(viewer, fgroup);
123:       if (cutLabel) {
124:         const PetscScalar *ga;
125:         PetscScalar       *suba;
126:         PetscInt           Nc, gstart, subSize = 0, extSize = 0, subOff = 0, newOff = 0, p;

128:         PetscSectionGetFieldComponents(section, f, &Nc);
129:         for (p = pStart; p < pEnd; ++p) {
130:           PetscInt gdof, fdof = 0, val;

132:           PetscSectionGetDof(sectionGlobal, p, &gdof);
133:           if (gdof > 0) {PetscSectionGetFieldDof(section, p, f, &fdof);}
134:           subSize += fdof;
135:           DMLabelGetValue(cutLabel, p, &val);
136:           if (val == 1) extSize += fdof;
137:         }
138:         VecCreate(PetscObjectComm((PetscObject) gv), &subv);
139:         VecSetSizes(subv, subSize+extSize, PETSC_DETERMINE);
140:         VecSetBlockSize(subv, Nc);
141:         VecSetType(subv, VECSTANDARD);
142:         VecGetOwnershipRange(gv, &gstart, NULL);
143:         VecGetArrayRead(gv, &ga);
144:         VecGetArray(subv, &suba);
145:         for (p = pStart; p < pEnd; ++p) {
146:           PetscInt gdof, goff, val;

148:           PetscSectionGetDof(sectionGlobal, p, &gdof);
149:           if (gdof > 0) {
150:             PetscInt fdof, fc, f2, poff = 0;

152:             PetscSectionGetOffset(sectionGlobal, p, &goff);
153:             /* Can get rid of this loop by storing field information in the global section */
154:             for (f2 = 0; f2 < f; ++f2) {
155:               PetscSectionGetFieldDof(section, p, f2, &fdof);
156:               poff += fdof;
157:             }
158:             PetscSectionGetFieldDof(section, p, f, &fdof);
159:             for (fc = 0; fc < fdof; ++fc, ++subOff) suba[subOff] = ga[goff+poff+fc - gstart];
160:             DMLabelGetValue(cutLabel, p, &val);
161:             if (val == 1) {
162:               for (fc = 0; fc < fdof; ++fc, ++newOff) suba[subSize+newOff] = ga[goff+poff+fc - gstart];
163:             }
164:           }
165:         }
166:         VecRestoreArrayRead(gv, &ga);
167:         VecRestoreArray(subv, &suba);
168:       } else {
169:         PetscSectionGetField_Internal(section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);
170:       }
171:       PetscStrcpy(subname, name);
172:       PetscStrcat(subname, "_");
173:       PetscStrcat(subname, fname);
174:       PetscObjectSetName((PetscObject) subv, subname);
175:       if (isseq) {VecView_Seq(subv, viewer);}
176:       else       {VecView_MPI(subv, viewer);}
177:       if (cutLabel) {
178:         VecDestroy(&subv);
179:       } else {
180:         PetscSectionRestoreField_Internal(section, sectionGlobal, gv, f, pStart, pEnd, &is, &subv);
181:       }
182:       PetscViewerHDF5PopGroup(viewer);
183:       PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "%s/%s", fgroup, subname);
184:       PetscViewerHDF5HasAttribute(viewer, group, "vector_field_type", &flag);
185:       if (!flag) {
186:         if ((ft == PETSC_VTK_POINT_VECTOR_FIELD) || (ft == PETSC_VTK_CELL_VECTOR_FIELD)) {
187:           PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "vector");
188:         } else {
189:           PetscViewerHDF5WriteAttribute(viewer, group, "vector_field_type", PETSC_STRING, "scalar");
190:         }
191:       }
192:     }
193:   }
194:   DMRestoreGlobalVector(dmBC, &gv);
195:   return(0);
196: }

198: PetscErrorCode VecView_Plex_HDF5_Internal(Vec v, PetscViewer viewer)
199: {
200:   DM             dm;
201:   Vec            locv;
202:   const char    *name;

206:   VecGetDM(v, &dm);
207:   DMGetLocalVector(dm, &locv);
208:   PetscObjectGetName((PetscObject) v, &name);
209:   PetscObjectSetName((PetscObject) locv, name);
210:   DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);
211:   DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);
212:   PetscViewerHDF5PushGroup(viewer, "/fields");
213:   PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);
214:   VecView_Plex_Local(locv, viewer);
215:   PetscViewerPopFormat(viewer);
216:   PetscViewerHDF5PopGroup(viewer);
217:   DMRestoreLocalVector(dm, &locv);
218:   return(0);
219: }

221: PetscErrorCode VecView_Plex_HDF5_Native_Internal(Vec v, PetscViewer viewer)
222: {
223:   PetscBool      isseq;

227:   PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);
228:   PetscViewerHDF5PushGroup(viewer, "/fields");
229:   if (isseq) {VecView_Seq(v, viewer);}
230:   else       {VecView_MPI(v, viewer);}
231:   PetscViewerHDF5PopGroup(viewer);
232:   return(0);
233: }

235: PetscErrorCode VecLoad_Plex_HDF5_Internal(Vec v, PetscViewer viewer)
236: {
237:   DM             dm;
238:   Vec            locv;
239:   const char    *name;
240:   PetscInt       seqnum;

244:   VecGetDM(v, &dm);
245:   DMGetLocalVector(dm, &locv);
246:   PetscObjectGetName((PetscObject) v, &name);
247:   PetscObjectSetName((PetscObject) locv, name);
248:   DMGetOutputSequenceNumber(dm, &seqnum, NULL);
249:   PetscViewerHDF5SetTimestep(viewer, seqnum);
250:   PetscViewerHDF5PushGroup(viewer, "/fields");
251:   VecLoad_Plex_Local(locv, viewer);
252:   PetscViewerHDF5PopGroup(viewer);
253:   DMLocalToGlobalBegin(dm, locv, INSERT_VALUES, v);
254:   DMLocalToGlobalEnd(dm, locv, INSERT_VALUES, v);
255:   DMRestoreLocalVector(dm, &locv);
256:   return(0);
257: }

259: PetscErrorCode VecLoad_Plex_HDF5_Native_Internal(Vec v, PetscViewer viewer)
260: {
261:   DM             dm;
262:   PetscInt       seqnum;

266:   VecGetDM(v, &dm);
267:   DMGetOutputSequenceNumber(dm, &seqnum, NULL);
268:   PetscViewerHDF5SetTimestep(viewer, seqnum);
269:   PetscViewerHDF5PushGroup(viewer, "/fields");
270:   VecLoad_Default(v, viewer);
271:   PetscViewerHDF5PopGroup(viewer);
272:   return(0);
273: }

275: static PetscErrorCode DMPlexWriteTopology_HDF5_Static(DM dm, IS globalPointNumbers, PetscViewer viewer)
276: {
277:   IS              orderIS, conesIS, cellsIS, orntsIS;
278:   const PetscInt *gpoint;
279:   PetscInt       *order, *sizes, *cones, *ornts;
280:   PetscInt        dim, pStart, pEnd, p, conesSize = 0, cellsSize = 0, c = 0, s = 0;
281:   PetscErrorCode  ierr;

284:   ISGetIndices(globalPointNumbers, &gpoint);
285:   DMGetDimension(dm, &dim);
286:   DMPlexGetChart(dm, &pStart, &pEnd);
287:   for (p = pStart; p < pEnd; ++p) {
288:     if (gpoint[p] >= 0) {
289:       PetscInt coneSize;

291:       DMPlexGetConeSize(dm, p, &coneSize);
292:       conesSize += 1;
293:       cellsSize += coneSize;
294:     }
295:   }
296:   PetscMalloc1(conesSize, &order);
297:   PetscMalloc1(conesSize, &sizes);
298:   PetscMalloc1(cellsSize, &cones);
299:   PetscMalloc1(cellsSize, &ornts);
300:   for (p = pStart; p < pEnd; ++p) {
301:     if (gpoint[p] >= 0) {
302:       const PetscInt *cone, *ornt;
303:       PetscInt        coneSize, cp;

305:       DMPlexGetConeSize(dm, p, &coneSize);
306:       DMPlexGetCone(dm, p, &cone);
307:       DMPlexGetConeOrientation(dm, p, &ornt);
308:       order[s]   = gpoint[p];
309:       sizes[s++] = coneSize;
310:       for (cp = 0; cp < coneSize; ++cp, ++c) {cones[c] = gpoint[cone[cp]] < 0 ? -(gpoint[cone[cp]]+1) : gpoint[cone[cp]]; ornts[c] = ornt[cp];}
311:     }
312:   }
313:   if (s != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of points %d != %d", s, conesSize);
314:   if (c != cellsSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cone points %d != %d", c, cellsSize);
315:   ISCreateGeneral(PetscObjectComm((PetscObject) dm), conesSize, order, PETSC_OWN_POINTER, &orderIS);
316:   PetscObjectSetName((PetscObject) orderIS, "order");
317:   ISCreateGeneral(PetscObjectComm((PetscObject) dm), conesSize, sizes, PETSC_OWN_POINTER, &conesIS);
318:   PetscObjectSetName((PetscObject) conesIS, "cones");
319:   ISCreateGeneral(PetscObjectComm((PetscObject) dm), cellsSize, cones, PETSC_OWN_POINTER, &cellsIS);
320:   PetscObjectSetName((PetscObject) cellsIS, "cells");
321:   ISCreateGeneral(PetscObjectComm((PetscObject) dm), cellsSize, ornts, PETSC_OWN_POINTER, &orntsIS);
322:   PetscObjectSetName((PetscObject) orntsIS, "orientation");
323:   PetscViewerHDF5PushGroup(viewer, "/topology");
324:   ISView(orderIS, viewer);
325:   ISView(conesIS, viewer);
326:   ISView(cellsIS, viewer);
327:   ISView(orntsIS, viewer);
328:   PetscViewerHDF5PopGroup(viewer);
329:   ISDestroy(&orderIS);
330:   ISDestroy(&conesIS);
331:   ISDestroy(&cellsIS);
332:   ISDestroy(&orntsIS);
333:   ISRestoreIndices(globalPointNumbers, &gpoint);

335:   PetscViewerHDF5WriteAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);
336:   return(0);
337: }

339: static PetscErrorCode DMPlexWriteTopology_Vertices_HDF5_Static(DM dm, DMLabel label, PetscInt labelId, PetscViewer viewer)
340: {
341:   DMLabel         cutLabel;
342:   PetscSection    cSection;
343:   IS              cellIS, globalVertexNumbers;
344:   const PetscInt *gvertex;
345:   PetscInt       *vertices;
346:   IS              cutvertices;
347:   const PetscInt *cutverts;
348:   PetscInt        dim, depth, vStart, vEnd, vExtra = 0, v, cellHeight, cStart, cMax, cEnd, cell, conesSize = 0, numCornersLocal = 0, numCorners;
349:   hid_t           fileId, groupId;
350:   PetscErrorCode  ierr;

353:   DMGetDimension(dm, &dim);
354:   DMPlexGetDepth(dm, &depth);
355:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
356:   DMGetCoordinateSection(dm, &cSection);
357:   DMPlexGetVTKCellHeight(dm, &cellHeight);
358:   DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);
359:   DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);
360:   if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
361:   for (cell = cStart; cell < cEnd; ++cell) {
362:     PetscInt *closure = NULL;
363:     PetscInt  closureSize, v, Nc = 0;

365:     if (label) {
366:       PetscInt value;
367:       DMLabelGetValue(label, cell, &value);
368:       if (value == labelId) continue;
369:     }
370:     DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);
371:     for (v = 0; v < closureSize*2; v += 2) {
372:       if ((closure[v] >= vStart) && (closure[v] < vEnd)) ++Nc;
373:     }
374:     DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);
375:     conesSize += Nc;
376:     if (!numCornersLocal)           numCornersLocal = Nc;
377:     else if (numCornersLocal != Nc) numCornersLocal = 1;
378:   }
379:   MPIU_Allreduce(&numCornersLocal, &numCorners, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm));
380:   if (numCornersLocal && (numCornersLocal != numCorners || numCorners == 1)) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Visualization topology currently only supports identical cell shapes");

382:   PetscViewerHDF5PushGroup(viewer, "/viz");
383:   PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);
384:   PetscStackCallHDF5(H5Gclose,(groupId));
385:   PetscViewerHDF5PopGroup(viewer);

387:   DMGetLabel(dm, "periodic_cut", &cutLabel);
388:   if (cutLabel) {
389:     DMLabelGetStratumIS(cutLabel, 1, &cutvertices);
390:     ISGetIndices(cutvertices, &cutverts);
391:     ISGetLocalSize(cutvertices, &vExtra);
392:   }
393:   DMPlexGetVertexNumbering(dm, &globalVertexNumbers);
394:   ISGetIndices(globalVertexNumbers, &gvertex);
395:   PetscMalloc1(conesSize, &vertices);
396:   for (cell = cStart, v = 0; cell < cEnd; ++cell) {
397:     PetscInt *closure = NULL;
398:     PetscInt  closureSize, Nc = 0, p;
399:     PetscBool replace = PETSC_FALSE;

401:     if (label) {
402:       PetscInt value;
403:       DMLabelGetValue(label, cell, &value);
404:       if (value == labelId) continue;
405:     }
406:     if (cutLabel) {
407:       PetscInt value;
408:       DMLabelGetValue(cutLabel, cell, &value);
409:       if (value == 2) replace = PETSC_TRUE;
410:     }
411:     DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);
412:     for (p = 0; p < closureSize*2; p += 2) {
413:       if ((closure[p] >= vStart) && (closure[p] < vEnd)) {
414:         closure[Nc++] = closure[p];
415:       }
416:     }
417:     DMPlexInvertCell_Internal(dim, Nc, closure);
418:     for (p = 0; p < Nc; ++p) {
419:       const PetscInt gv = gvertex[closure[p] - vStart];
420:       vertices[v++] = gv < 0 ? -(gv+1) : gv;
421:       if (replace) {
422:         PetscInt newv;
423:         PetscFindInt(closure[p], vExtra, cutverts, &newv);
424:         if (newv >= 0) vertices[v-1] = vEnd - vStart + newv;
425:       }
426:     }
427:     DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);
428:   }
429:   if (cutLabel) {
430:     ISRestoreIndices(cutvertices, &cutverts);
431:     ISDestroy(&cutvertices);
432:   }
433:   if (v != conesSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of cell vertices %d != %d", v, conesSize);
434:   ISCreateGeneral(PetscObjectComm((PetscObject) dm), conesSize, vertices, PETSC_OWN_POINTER, &cellIS);
435:   PetscLayoutSetBlockSize(cellIS->map, numCorners);
436:   PetscObjectSetName((PetscObject) cellIS, "cells");
437:   PetscViewerHDF5PushGroup(viewer, "/viz/topology");
438:   ISView(cellIS, viewer);
439:   PetscViewerHDF5WriteAttribute(viewer, "/viz/topology/cells", "cell_corners", PETSC_INT, (void *) &numCorners);
440:   ISDestroy(&cellIS);

442:   PetscViewerHDF5WriteAttribute(viewer, "/viz/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);
443:   return(0);
444: }

446: static PetscErrorCode DMPlexWriteCoordinates_HDF5_Static(DM dm, PetscViewer viewer)
447: {
448:   DM             cdm;
449:   Vec            coordinates, newcoords;
450:   PetscReal      lengthScale;
451:   PetscInt       m, M, bs;

455:   DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);
456:   DMGetCoordinateDM(dm, &cdm);
457:   DMGetCoordinates(dm, &coordinates);
458:   VecCreate(PetscObjectComm((PetscObject) coordinates), &newcoords);
459:   PetscObjectSetName((PetscObject) newcoords, "vertices");
460:   VecGetSize(coordinates, &M);
461:   VecGetLocalSize(coordinates, &m);
462:   VecSetSizes(newcoords, m, M);
463:   VecGetBlockSize(coordinates, &bs);
464:   VecSetBlockSize(newcoords, bs);
465:   VecSetType(newcoords,VECSTANDARD);
466:   VecCopy(coordinates, newcoords);
467:   VecScale(newcoords, lengthScale);
468:   /* Did not use DMGetGlobalVector() in order to bypass default group assignment */
469:   PetscViewerHDF5PushGroup(viewer, "/geometry");
470:   PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE);
471:   VecView(newcoords, viewer);
472:   PetscViewerPopFormat(viewer);
473:   PetscViewerHDF5PopGroup(viewer);
474:   VecDestroy(&newcoords);
475:   return(0);
476: }

478: static PetscErrorCode DMPlexWriteCoordinates_Vertices_HDF5_Static(DM dm, PetscViewer viewer)
479: {
480:   Vec              coordinates, newcoords;
481:   PetscSection     cSection;
482:   PetscScalar     *coords, *ncoords;
483:   DMLabel          cutLabel;
484:   const PetscReal *L;
485:   const DMBoundaryType *bd;
486:   PetscReal        lengthScale;
487:   PetscInt         vStart, vEnd, vExtra = 0, v, bs, coordSize, dof, off, d;
488:   PetscBool        localized, embedded;
489:   hid_t            fileId, groupId;
490:   PetscErrorCode   ierr;

493:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
494:   DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);
495:   DMGetCoordinatesLocal(dm, &coordinates);
496:   VecGetBlockSize(coordinates, &bs);
497:   VecGetLocalSize(coordinates, &coordSize);
498:   DMGetCoordinatesLocalized(dm,&localized);
499:   if (localized == PETSC_FALSE) return(0);
500:   DMGetLabel(dm, "periodic_cut", &cutLabel);
501:   if (cutLabel) {
502:     IS              vertices;
503:     const PetscInt *verts;
504:     PetscInt        n;

506:     DMLabelGetStratumIS(cutLabel, 1, &vertices);
507:     ISGetIndices(vertices, &verts);
508:     ISGetLocalSize(vertices, &n);
509:     for (v = 0; v < n; ++v) {
510:       if ((verts[v] >= vStart) && (verts[v] < vEnd)) ++vExtra;
511:     }
512:     ISRestoreIndices(vertices, &verts);
513:     ISDestroy(&vertices);
514:   }
515:   DMGetPeriodicity(dm, NULL, NULL, &L, &bd);
516:   DMGetCoordinateSection(dm, &cSection);
517:   VecCreate(PetscObjectComm((PetscObject) coordinates), &newcoords);
518:   PetscSectionGetDof(cSection, vStart, &dof);
519:   embedded  = (PetscBool) (L && dof == 2 && !cutLabel);
520:   coordSize = 0;
521:   coordSize += dof*vExtra;
522:   for (v = vStart; v < vEnd; ++v) {
523:     PetscSectionGetDof(cSection, v, &dof);
524:     if (embedded) coordSize += dof+1;
525:     else          coordSize += dof;
526:   }
527:   if (embedded) {VecSetBlockSize(newcoords, bs+1);}
528:   else          {VecSetBlockSize(newcoords, bs);}
529:   VecSetSizes(newcoords, coordSize, PETSC_DETERMINE);
530:   VecSetType(newcoords, VECSTANDARD);
531:   VecGetArray(coordinates, &coords);
532:   VecGetArray(newcoords,   &ncoords);
533:   coordSize = 0;
534:   for (v = vStart; v < vEnd; ++v) {
535:     PetscSectionGetDof(cSection, v, &dof);
536:     PetscSectionGetOffset(cSection, v, &off);
537:     if (embedded) {
538:       if ((bd[0] == DM_BOUNDARY_PERIODIC) && (bd[1] == DM_BOUNDARY_PERIODIC)) {
539:         PetscReal theta, phi, r, R;
540:         /* XY-periodic */
541:         /* Suppose its an y-z circle, then
542:              \hat r = (0, cos(th), sin(th)) \hat x = (1, 0, 0)
543:            and the circle in that plane is
544:              \hat r cos(phi) + \hat x sin(phi) */
545:         theta = 2.0*PETSC_PI*PetscRealPart(coords[off+1])/L[1];
546:         phi   = 2.0*PETSC_PI*PetscRealPart(coords[off+0])/L[0];
547:         r     = L[0]/(2.0*PETSC_PI * 2.0*L[1]);
548:         R     = L[1]/(2.0*PETSC_PI);
549:         ncoords[coordSize++] =  PetscSinReal(phi) * r;
550:         ncoords[coordSize++] = -PetscCosReal(theta) * (R + r * PetscCosReal(phi));
551:         ncoords[coordSize++] =  PetscSinReal(theta) * (R + r * PetscCosReal(phi));
552:       } else if ((bd[0] == DM_BOUNDARY_PERIODIC)) {
553:         /* X-periodic */
554:         ncoords[coordSize++] = -PetscCosReal(2.0*PETSC_PI*PetscRealPart(coords[off+0])/L[0])*(L[0]/(2.0*PETSC_PI));
555:         ncoords[coordSize++] = coords[off+1];
556:         ncoords[coordSize++] = PetscSinReal(2.0*PETSC_PI*PetscRealPart(coords[off+0])/L[0])*(L[0]/(2.0*PETSC_PI));
557:       } else if ((bd[1] == DM_BOUNDARY_PERIODIC)) {
558:         /* Y-periodic */
559:         ncoords[coordSize++] = coords[off+0];
560:         ncoords[coordSize++] = PetscSinReal(2.0*PETSC_PI*PetscRealPart(coords[off+1])/L[1])*(L[1]/(2.0*PETSC_PI));
561:         ncoords[coordSize++] = -PetscCosReal(2.0*PETSC_PI*PetscRealPart(coords[off+1])/L[1])*(L[1]/(2.0*PETSC_PI));
562:       } else if ((bd[0] == DM_BOUNDARY_TWIST)) {
563:         PetscReal phi, r, R;
564:         /* Mobius strip */
565:         /* Suppose its an x-z circle, then
566:              \hat r = (-cos(phi), 0, sin(phi)) \hat y = (0, 1, 0)
567:            and in that plane we rotate by pi as we go around the circle
568:              \hat r cos(phi/2) + \hat y sin(phi/2) */
569:         phi   = 2.0*PETSC_PI*PetscRealPart(coords[off+0])/L[0];
570:         R     = L[0];
571:         r     = PetscRealPart(coords[off+1]) - L[1]/2.0;
572:         ncoords[coordSize++] = -PetscCosReal(phi) * (R + r * PetscCosReal(phi/2.0));
573:         ncoords[coordSize++] =  PetscSinReal(phi/2.0) * r;
574:         ncoords[coordSize++] =  PetscSinReal(phi) * (R + r * PetscCosReal(phi/2.0));
575:       } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Cannot handle periodicity in this domain");
576:     } else {
577:       if (cutLabel) {
578:         DMLocalizeCoordinate(dm, &coords[off], PETSC_TRUE, &ncoords[coordSize]);
579:         coordSize += dof;
580:       } else {
581:         for (d = 0; d < dof; ++d, ++coordSize) ncoords[coordSize] = coords[off+d];
582:       }
583:     }
584:   }
585:   if (cutLabel) {
586:     IS              vertices;
587:     const PetscInt *verts;
588:     PetscInt        n;

590:     DMLabelGetStratumIS(cutLabel, 1, &vertices);
591:     ISGetIndices(vertices, &verts);
592:     ISGetLocalSize(vertices, &n);
593:     for (v = 0; v < n; ++v) {
594:       if ((verts[v] < vStart) || (verts[v] >= vEnd)) continue;
595:       PetscSectionGetDof(cSection, verts[v], &dof);
596:       PetscSectionGetOffset(cSection, verts[v], &off);
597:       for (d = 0; d < dof; ++d, ++coordSize) ncoords[coordSize] = coords[off+d] + ((bd[d] == DM_BOUNDARY_PERIODIC) ? L[d] : 0.0);
598:     }
599:     ISRestoreIndices(vertices, &verts);
600:     ISDestroy(&vertices);
601:   }
602:   VecRestoreArray(coordinates, &coords);
603:   VecRestoreArray(newcoords,   &ncoords);
604:   PetscObjectSetName((PetscObject) newcoords, "vertices");
605:   VecScale(newcoords, lengthScale);
606:   PetscViewerHDF5PushGroup(viewer, "/viz");
607:   PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);
608:   PetscStackCallHDF5(H5Gclose,(groupId));
609:   PetscViewerHDF5PopGroup(viewer);
610:   PetscViewerHDF5PushGroup(viewer, "/viz/geometry");
611:   VecView(newcoords, viewer);
612:   PetscViewerHDF5PopGroup(viewer);
613:   VecDestroy(&newcoords);
614:   return(0);
615: }

617: /* We only write cells and vertices. Does this screw up parallel reading? */
618: PetscErrorCode DMPlexView_HDF5_Internal(DM dm, PetscViewer viewer)
619: {
620:   DMLabel           label   = NULL;
621:   PetscInt          labelId = 0;
622:   IS                globalPointNumbers;
623:   const PetscInt   *gpoint;
624:   PetscInt          numLabels, l;
625:   hid_t             fileId, groupId;
626:   PetscViewerFormat format;
627:   PetscErrorCode    ierr;

630:   DMPlexCreatePointNumbering(dm, &globalPointNumbers);
631:   PetscViewerGetFormat(viewer, &format);
632:   DMPlexWriteCoordinates_HDF5_Static(dm, viewer);
633:   if (format == PETSC_VIEWER_HDF5_VIZ) {DMPlexWriteCoordinates_Vertices_HDF5_Static(dm, viewer);}
634:   DMPlexWriteTopology_HDF5_Static(dm, globalPointNumbers, viewer);
635:   if (format == PETSC_VIEWER_HDF5_VIZ) {DMPlexWriteTopology_Vertices_HDF5_Static(dm, label, labelId, viewer);}
636:   /* Write Labels*/
637:   ISGetIndices(globalPointNumbers, &gpoint);
638:   PetscViewerHDF5PushGroup(viewer, "/labels");
639:   PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);
640:   if (groupId != fileId) PetscStackCallHDF5(H5Gclose,(groupId));
641:   PetscViewerHDF5PopGroup(viewer);
642:   DMGetNumLabels(dm, &numLabels);
643:   for (l = 0; l < numLabels; ++l) {
644:     DMLabel         label;
645:     const char     *name;
646:     IS              valueIS, pvalueIS, globalValueIS;
647:     const PetscInt *values;
648:     PetscInt        numValues, v;
649:     PetscBool       isDepth, output;
650:     char            group[PETSC_MAX_PATH_LEN];

652:     DMGetLabelName(dm, l, &name);
653:     DMGetLabelOutput(dm, name, &output);
654:     PetscStrncmp(name, "depth", 10, &isDepth);
655:     if (isDepth || !output) continue;
656:     DMGetLabel(dm, name, &label);
657:     PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s", name);
658:     PetscViewerHDF5PushGroup(viewer, group);
659:     PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);
660:     if (groupId != fileId) PetscStackCallHDF5(H5Gclose,(groupId));
661:     PetscViewerHDF5PopGroup(viewer);
662:     DMLabelGetValueIS(label, &valueIS);
663:     /* Must copy to a new IS on the global comm */
664:     ISGetLocalSize(valueIS, &numValues);
665:     ISGetIndices(valueIS, &values);
666:     ISCreateGeneral(PetscObjectComm((PetscObject) dm), numValues, values, PETSC_COPY_VALUES, &pvalueIS);
667:     ISRestoreIndices(valueIS, &values);
668:     ISAllGather(pvalueIS, &globalValueIS);
669:     ISDestroy(&pvalueIS);
670:     ISSortRemoveDups(globalValueIS);
671:     ISGetLocalSize(globalValueIS, &numValues);
672:     ISGetIndices(globalValueIS, &values);
673:     for (v = 0; v < numValues; ++v) {
674:       IS              stratumIS, globalStratumIS;
675:       const PetscInt *spoints = NULL;
676:       PetscInt       *gspoints, n = 0, gn, p;
677:       const char     *iname = "indices";

679:       PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%d", name, values[v]);
680:       DMLabelGetStratumIS(label, values[v], &stratumIS);

682:       if (stratumIS) {ISGetLocalSize(stratumIS, &n);}
683:       if (stratumIS) {ISGetIndices(stratumIS, &spoints);}
684:       for (gn = 0, p = 0; p < n; ++p) if (gpoint[spoints[p]] >= 0) ++gn;
685:       PetscMalloc1(gn,&gspoints);
686:       for (gn = 0, p = 0; p < n; ++p) if (gpoint[spoints[p]] >= 0) gspoints[gn++] = gpoint[spoints[p]];
687:       if (stratumIS) {ISRestoreIndices(stratumIS, &spoints);}
688:       ISCreateGeneral(PetscObjectComm((PetscObject) dm), gn, gspoints, PETSC_OWN_POINTER, &globalStratumIS);
689:       if (stratumIS) {PetscObjectGetName((PetscObject) stratumIS, &iname);}
690:       PetscObjectSetName((PetscObject) globalStratumIS, iname);

692:       PetscViewerHDF5PushGroup(viewer, group);
693:       ISView(globalStratumIS, viewer);
694:       PetscViewerHDF5PopGroup(viewer);
695:       ISDestroy(&globalStratumIS);
696:       ISDestroy(&stratumIS);
697:     }
698:     ISRestoreIndices(globalValueIS, &values);
699:     ISDestroy(&globalValueIS);
700:     ISDestroy(&valueIS);
701:   }
702:   ISRestoreIndices(globalPointNumbers, &gpoint);
703:   ISDestroy(&globalPointNumbers);
704:   return(0);
705: }

707: typedef struct {
708:   PetscMPIInt rank;
709:   DM          dm;
710:   PetscViewer viewer;
711:   DMLabel     label;
712: } LabelCtx;

714: static herr_t ReadLabelStratumHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
715: {
716:   PetscViewer     viewer = ((LabelCtx *) op_data)->viewer;
717:   DMLabel         label  = ((LabelCtx *) op_data)->label;
718:   IS              stratumIS;
719:   const PetscInt *ind;
720:   PetscInt        value, N, i;
721:   const char     *lname;
722:   char            group[PETSC_MAX_PATH_LEN];
723:   PetscErrorCode  ierr;

725:   PetscOptionsStringToInt(name, &value);
726:   ISCreate(PetscObjectComm((PetscObject) viewer), &stratumIS);
727:   PetscObjectSetName((PetscObject) stratumIS, "indices");
728:   DMLabelGetName(label, &lname);
729:   PetscSNPrintf(group, PETSC_MAX_PATH_LEN, "/labels/%s/%s", lname, name);
730:   PetscViewerHDF5PushGroup(viewer, group);
731:   {
732:     /* Force serial load */
733:     PetscViewerHDF5ReadSizes(viewer, "indices", NULL, &N);
734:     PetscLayoutSetLocalSize(stratumIS->map, !((LabelCtx *) op_data)->rank ? N : 0);
735:     PetscLayoutSetSize(stratumIS->map, N);
736:   }
737:   ISLoad(stratumIS, viewer);
738:   PetscViewerHDF5PopGroup(viewer);
739:   ISGetLocalSize(stratumIS, &N);
740:   ISGetIndices(stratumIS, &ind);
741:   for (i = 0; i < N; ++i) {DMLabelSetValue(label, ind[i], value);}
742:   ISRestoreIndices(stratumIS, &ind);
743:   ISDestroy(&stratumIS);
744:   return 0;
745: }

747: static herr_t ReadLabelHDF5_Static(hid_t g_id, const char *name, const H5L_info_t *info, void *op_data)
748: {
749:   DM             dm  = ((LabelCtx *) op_data)->dm;
750:   hsize_t        idx = 0;
752:   herr_t         err;

754:   DMCreateLabel(dm, name); if (ierr) return (herr_t) ierr;
755:   DMGetLabel(dm, name, &((LabelCtx *) op_data)->label); if (ierr) return (herr_t) ierr;
756:   PetscStackCall("H5Literate_by_name",err = H5Literate_by_name(g_id, name, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelStratumHDF5_Static, op_data, 0));
757:   return err;
758: }

760: /* The first version will read everything onto proc 0, letting the user distribute
761:    The next will create a naive partition, and then rebalance after reading
762: */
763: PetscErrorCode DMPlexLoad_HDF5_Internal(DM dm, PetscViewer viewer)
764: {
765:   LabelCtx        ctx;
766:   PetscSection    coordSection;
767:   Vec             coordinates;
768:   IS              orderIS, conesIS, cellsIS, orntsIS;
769:   const PetscInt *order, *cones, *cells, *ornts;
770:   PetscReal       lengthScale;
771:   PetscInt       *cone, *ornt;
772:   PetscInt        dim, spatialDim, N, numVertices, vStart, vEnd, v, pEnd, p, q, maxConeSize = 0, c;
773:   hid_t           fileId, groupId;
774:   hsize_t         idx = 0;
775:   PetscMPIInt     rank;
776:   PetscErrorCode  ierr;

779:   MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);
780:   /* Read toplogy */
781:   PetscViewerHDF5ReadAttribute(viewer, "/topology/cells", "cell_dim", PETSC_INT, (void *) &dim);
782:   DMSetDimension(dm, dim);
783:   PetscViewerHDF5PushGroup(viewer, "/topology");

785:   ISCreate(PetscObjectComm((PetscObject) dm), &orderIS);
786:   PetscObjectSetName((PetscObject) orderIS, "order");
787:   ISCreate(PetscObjectComm((PetscObject) dm), &conesIS);
788:   PetscObjectSetName((PetscObject) conesIS, "cones");
789:   ISCreate(PetscObjectComm((PetscObject) dm), &cellsIS);
790:   PetscObjectSetName((PetscObject) cellsIS, "cells");
791:   ISCreate(PetscObjectComm((PetscObject) dm), &orntsIS);
792:   PetscObjectSetName((PetscObject) orntsIS, "orientation");
793:   {
794:     /* Force serial load */
795:     PetscViewerHDF5ReadSizes(viewer, "order", NULL, &pEnd);
796:     PetscLayoutSetLocalSize(orderIS->map, !rank ? pEnd : 0);
797:     PetscLayoutSetSize(orderIS->map, pEnd);
798:     PetscViewerHDF5ReadSizes(viewer, "cones", NULL, &pEnd);
799:     PetscLayoutSetLocalSize(conesIS->map, !rank ? pEnd : 0);
800:     PetscLayoutSetSize(conesIS->map, pEnd);
801:     pEnd = !rank ? pEnd : 0;
802:     PetscViewerHDF5ReadSizes(viewer, "cells", NULL, &N);
803:     PetscLayoutSetLocalSize(cellsIS->map, !rank ? N : 0);
804:     PetscLayoutSetSize(cellsIS->map, N);
805:     PetscViewerHDF5ReadSizes(viewer, "orientation", NULL, &N);
806:     PetscLayoutSetLocalSize(orntsIS->map, !rank ? N : 0);
807:     PetscLayoutSetSize(orntsIS->map, N);
808:   }
809:   ISLoad(orderIS, viewer);
810:   ISLoad(conesIS, viewer);
811:   ISLoad(cellsIS, viewer);
812:   ISLoad(orntsIS, viewer);
813:   PetscViewerHDF5PopGroup(viewer);
814:   /* Read geometry */
815:   PetscViewerHDF5PushGroup(viewer, "/geometry");
816:   VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);
817:   PetscObjectSetName((PetscObject) coordinates, "vertices");
818:   {
819:     /* Force serial load */
820:     PetscViewerHDF5ReadSizes(viewer, "vertices", &spatialDim, &N);
821:     VecSetSizes(coordinates, !rank ? N : 0, N);
822:     VecSetBlockSize(coordinates, spatialDim);
823:   }
824:   VecLoad(coordinates, viewer);
825:   PetscViewerHDF5PopGroup(viewer);
826:   DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);
827:   VecScale(coordinates, 1.0/lengthScale);
828:   VecGetLocalSize(coordinates, &numVertices);
829:   VecGetBlockSize(coordinates, &spatialDim);
830:   numVertices /= spatialDim;
831:   /* Create Plex */
832:   DMPlexSetChart(dm, 0, pEnd);
833:   ISGetIndices(orderIS, &order);
834:   ISGetIndices(conesIS, &cones);
835:   for (p = 0; p < pEnd; ++p) {
836:     DMPlexSetConeSize(dm, order[p], cones[p]);
837:     maxConeSize = PetscMax(maxConeSize, cones[p]);
838:   }
839:   DMSetUp(dm);
840:   ISGetIndices(cellsIS, &cells);
841:   ISGetIndices(orntsIS, &ornts);
842:   PetscMalloc2(maxConeSize,&cone,maxConeSize,&ornt);
843:   for (p = 0, q = 0; p < pEnd; ++p) {
844:     for (c = 0; c < cones[p]; ++c, ++q) {cone[c] = cells[q]; ornt[c] = ornts[q];}
845:     DMPlexSetCone(dm, order[p], cone);
846:     DMPlexSetConeOrientation(dm, order[p], ornt);
847:   }
848:   PetscFree2(cone,ornt);
849:   ISRestoreIndices(orderIS, &order);
850:   ISRestoreIndices(conesIS, &cones);
851:   ISRestoreIndices(cellsIS, &cells);
852:   ISRestoreIndices(orntsIS, &ornts);
853:   ISDestroy(&orderIS);
854:   ISDestroy(&conesIS);
855:   ISDestroy(&cellsIS);
856:   ISDestroy(&orntsIS);
857:   DMPlexSymmetrize(dm);
858:   DMPlexStratify(dm);
859:   /* Create coordinates */
860:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
861:   if (numVertices != vEnd - vStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of coordinates loaded %d does not match number of vertices %d", numVertices, vEnd - vStart);
862:   DMGetCoordinateSection(dm, &coordSection);
863:   PetscSectionSetNumFields(coordSection, 1);
864:   PetscSectionSetFieldComponents(coordSection, 0, spatialDim);
865:   PetscSectionSetChart(coordSection, vStart, vEnd);
866:   for (v = vStart; v < vEnd; ++v) {
867:     PetscSectionSetDof(coordSection, v, spatialDim);
868:     PetscSectionSetFieldDof(coordSection, v, 0, spatialDim);
869:   }
870:   PetscSectionSetUp(coordSection);
871:   DMSetCoordinates(dm, coordinates);
872:   VecDestroy(&coordinates);
873:   /* Read Labels*/
874:   ctx.rank   = rank;
875:   ctx.dm     = dm;
876:   ctx.viewer = viewer;
877:   PetscViewerHDF5PushGroup(viewer, "/labels");
878:   PetscViewerHDF5OpenGroup(viewer, &fileId, &groupId);
879:   PetscStackCallHDF5(H5Literate,(groupId, H5_INDEX_NAME, H5_ITER_NATIVE, &idx, ReadLabelHDF5_Static, &ctx));
880:   PetscStackCallHDF5(H5Gclose,(groupId));
881:   PetscViewerHDF5PopGroup(viewer);
882:   return(0);
883: }
884: #endif