Actual source code: init.c

petsc-3.12.0 2019-09-29
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9:  #include <petscsys.h>
 10:  #include <petsc/private/petscimpl.h>
 11:  #include <petscvalgrind.h>
 12:  #include <petscviewer.h>
 13: #if defined(PETSC_USE_LOG)
 14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
 15: #endif

 17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 18: #include <sys/sysinfo.h>
 19: #endif
 20: #if defined(PETSC_HAVE_UNISTD_H)
 21: #include <unistd.h>
 22: #endif
 23: #if defined(PETSC_HAVE_CUDA)
 24: #include <cuda_runtime.h>
 25: PETSC_EXTERN PetscErrorCode PetscCUBLASInitializeHandle(void);
 26: PETSC_EXTERN PetscErrorCode PetscCUSOLVERDnInitializeHandle(void);
 27: #endif

 29: #if defined(PETSC_HAVE_VIENNACL)
 30: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 31: #endif

 33: /* ------------------------Nasty global variables -------------------------------*/
 34: /*
 35:      Indicates if PETSc started up MPI, or it was
 36:    already started before PETSc was initialized.
 37: */
 38: PetscBool   PetscBeganMPI         = PETSC_FALSE;
 39: PetscBool   PetscInitializeCalled = PETSC_FALSE;
 40: PetscBool   PetscFinalizeCalled   = PETSC_FALSE;
 41: PetscBool   PetscCUDAInitialized  = PETSC_FALSE;

 43: PetscMPIInt PetscGlobalRank       = -1;
 44: PetscMPIInt PetscGlobalSize       = -1;

 46: #if defined(PETSC_HAVE_COMPLEX)
 47: #if defined(PETSC_COMPLEX_INSTANTIATE)
 48: template <> class std::complex<double>; /* instantiate complex template class */
 49: #endif
 50: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 51: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 52: MPI_Datatype MPIU_C_COMPLEX;
 53: #endif

 55: /*MC
 56:    PETSC_i - the imaginary number i

 58:    Synopsis:
 59:    #include <petscsys.h>
 60:    PetscComplex PETSC_i;

 62:    Level: beginner

 64:    Note:
 65:    Complex numbers are automatically available if PETSc located a working complex implementation

 67: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 68: M*/
 69: PetscComplex PETSC_i;
 70: #endif
 71: #if defined(PETSC_USE_REAL___FLOAT128)
 72: MPI_Datatype MPIU___FLOAT128 = 0;
 73: #if defined(PETSC_HAVE_COMPLEX)
 74: MPI_Datatype MPIU___COMPLEX128 = 0;
 75: #endif
 76: #elif defined(PETSC_USE_REAL___FP16)
 77: MPI_Datatype MPIU___FP16 = 0;
 78: #endif
 79: MPI_Datatype MPIU_2SCALAR = 0;
 80: #if defined(PETSC_USE_64BIT_INDICES)
 81: MPI_Datatype MPIU_2INT = 0;
 82: #endif
 83: MPI_Datatype MPIU_BOOL;
 84: MPI_Datatype MPIU_ENUM;
 85: MPI_Datatype MPIU_FORTRANADDR;
 86: MPI_Datatype MPIU_SIZE_T;

 88: /*
 89:        Function that is called to display all error messages
 90: */
 91: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
 92: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
 93: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
 94: /*
 95:   This is needed to turn on/off GPU synchronization
 96: */
 97: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
 98: PetscBool PetscCUDASynchronize = PETSC_FALSE;

100: /* ------------------------------------------------------------------------------*/
101: /*
102:    Optional file where all PETSc output from various prints is saved
103: */
104: PETSC_INTERN FILE *petsc_history;
105: FILE *petsc_history = NULL;

107: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
108: {
110:   PetscMPIInt    rank,size;
111:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
112:   char           version[256];

115:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
116:   if (!rank) {
117:     char        arch[10];
118:     int         err;

120:     PetscGetArchType(arch,10);
121:     PetscGetDate(date,64);
122:     PetscGetVersion(version,256);
123:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
124:     if (filename) {
125:       PetscFixFilename(filename,fname);
126:     } else {
127:       PetscGetHomeDirectory(pfile,240);
128:       PetscStrcat(pfile,"/.petschistory");
129:       PetscFixFilename(pfile,fname);
130:     }

132:     *fd = fopen(fname,"a");
133:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

135:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
136:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
137:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
138:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
139:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");

141:     err = fflush(*fd);
142:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
143:   }
144:   return(0);
145: }

147: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
148: {
150:   PetscMPIInt    rank;
151:   char           date[64];
152:   int            err;

155:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
156:   if (!rank) {
157:     PetscGetDate(date,64);
158:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
159:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
160:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
161:     err  = fflush(*fd);
162:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
163:     err = fclose(*fd);
164:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
165:   }
166:   return(0);
167: }

169: /* ------------------------------------------------------------------------------*/

171: /*
172:    This is ugly and probably belongs somewhere else, but I want to
173:   be able to put a true MPI abort error handler with command line args.

175:     This is so MPI errors in the debugger will leave all the stack
176:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
177:   in the debugger hence we call abort() instead of MPI_Abort().
178: */

180: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
181: {
183:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
184:   abort();
185: }

187: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
188: {

192:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
193:   PetscAttachDebugger();
194:   if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
195: }

197: #if defined(PETSC_HAVE_CUDA)
198: /*@C
199:      PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device

201:      Logically collective

203:   Input Parameter:
204:   comm - the MPI communicator that will utilize the CUDA devices

206:   Options Database:
207: +  -cuda_initialize <default yes,no> - do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically
208:                                when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed
209: .  -cuda_view - view information about the CUDA devices
210: .  -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view
211: -  -cuda_set_device <gpu> - integer number of the device

213:   Level: beginner

215:   Notes:
216:    Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins

218: @*/
219: PetscErrorCode PetscCUDAInitialize(MPI_Comm comm)
220: {
221:   PetscErrorCode        ierr;
222:   PetscInt              deviceOpt = 0;
223:   PetscBool             cuda_view_flag = PETSC_FALSE,flg;
224:   struct cudaDeviceProp prop;
225:   int                   devCount,device,devicecnt;
226:   cudaError_t           err = cudaSuccess;
227:   PetscMPIInt           rank,size;

230:   /*
231:      If collecting logging information, by default, wait for GPU to complete its operations
232:      before returning to the CPU in order to get accurate timings of each event
233:   */
234:   PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);
235:   if (!PetscCUDASynchronize) {
236:     PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);
237:   }

239:   PetscOptionsBegin(comm,NULL,"CUDA options","Sys");
240:   PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);
241:   device = (int)deviceOpt;
242:   PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);
243:   PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);
244:   PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);
245:   PetscOptionsEnd();
246:   if (!PetscCUDAInitialized) {
247:     MPI_Comm_size(comm,&size);

249:     if (size>1 && !flg) {
250:       /* check to see if we force multiple ranks to hit the same GPU */
251:       /* we're not using the same GPU on multiple MPI threads. So try to allocated different   GPUs to different processes */

253:       /* First get the device count */
254:       err   = cudaGetDeviceCount(&devCount);
255:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));

257:       /* next determine the rank and then set the device via a mod */
258:       MPI_Comm_rank(comm,&rank);
259:       device = rank % devCount;
260:     }
261:     err = cudaSetDevice(device);
262:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));

264:     /* set the device flags so that it can map host memory */
265:     err = cudaSetDeviceFlags(cudaDeviceMapHost);
266:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));

268:     PetscCUBLASInitializeHandle();
269:     PetscCUSOLVERDnInitializeHandle();
270:     PetscCUDAInitialized = PETSC_TRUE;
271:   }
272:   if (cuda_view_flag) {
273:     MPI_Comm_rank(comm,&rank);
274:     err  = cudaGetDeviceCount(&devCount);
275:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
276:     for (devicecnt = 0; devicecnt < devCount; ++devicecnt) {
277:       err = cudaGetDeviceProperties(&prop,devicecnt);
278:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
279:       PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);
280:     }
281:     PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);
282:     PetscSynchronizedFlush(comm,PETSC_STDOUT);
283:   }
284:   return(0);
285: }
286: #endif

288: /*@C
289:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
290:      wishes a clean exit somewhere deep in the program.

292:    Collective on PETSC_COMM_WORLD

294:    Options Database Keys are the same as for PetscFinalize()

296:    Level: advanced

298:    Note:
299:    See PetscInitialize() for more general runtime options.

301: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
302: @*/
303: PetscErrorCode  PetscEnd(void)
304: {
306:   PetscFinalize();
307:   exit(0);
308:   return 0;
309: }

311: PetscBool PetscOptionsPublish = PETSC_FALSE;
312: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
313: PETSC_INTERN PetscBool      petscsetmallocvisited;
314: static       char           emacsmachinename[256];

316: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
317: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

319: /*@C
320:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
321:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
322:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

324:    Input Parameter:
325: +  help - the help function (may be NULL)
326: -  version - the version function (may be NULL)

328:    Level: developer

330: @*/
331: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
332: {
334:   PetscExternalHelpFunction    = help;
335:   PetscExternalVersionFunction = version;
336:   return(0);
337: }

339: #if defined(PETSC_USE_LOG)
340: PETSC_INTERN PetscBool   PetscObjectsLog;
341: #endif

343: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(void)
344: {
345:   char              string[64],mname[PETSC_MAX_PATH_LEN],*f;
346:   MPI_Comm          comm = PETSC_COMM_WORLD;
347:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
348:   PetscErrorCode    ierr;
349:   PetscReal         si;
350:   PetscInt          intensity;
351:   int               i;
352:   PetscMPIInt       rank;
353:   char              version[256],helpoptions[256];
354: #if defined(PETSC_USE_LOG)
355:   PetscViewerFormat format;
356:   PetscBool         flg4 = PETSC_FALSE;
357: #endif
358: #if defined(PETSC_HAVE_CUDA)
359:   PetscBool         initCuda = PETSC_TRUE;
360: #endif

363:   MPI_Comm_rank(comm,&rank);

365: #if !defined(PETSC_HAVE_THREADSAFETY)
366:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
367:     /*
368:       Setup the memory management; support for tracing malloc() usage
369:     */
370:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

372: #if defined(PETSC_USE_DEBUG)
373:     mdebug        = PETSC_TRUE;
374:     initializenan = PETSC_TRUE;
375:     PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
376: #else
377:     /* don't warn about unused option */
378:     PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
379:     flg1 = PETSC_FALSE;
380: #endif
381:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
382:     if (flg1 || flg2) {
383:       mdebug        = PETSC_TRUE;
384:       eachcall      = PETSC_TRUE;
385:       initializenan = PETSC_TRUE;
386:     } else if (flg3 && !flg2) {
387:       mdebug        = PETSC_FALSE;
388:       eachcall      = PETSC_FALSE;
389:       initializenan = PETSC_FALSE;
390:     }

392:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
393:     if (mlog) {
394:       mdebug = PETSC_TRUE;
395:     }
396:     /* the next line is deprecated */
397:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
398:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
399:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
400:     if (mdebug) {
401:       PetscMallocSetDebug(eachcall,initializenan);
402:     }
403:     if (mlog) {
404:       PetscReal logthreshold = 0;
405:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
406:       PetscMallocViewSet(logthreshold);
407:     }
408:   }

410:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
411:   if (flg2) {PetscMallocSetCoalesce(flg1);}
412:   flg1 = PETSC_FALSE;
413:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
414:   /* ignore this option if malloc is already set */
415:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

417:   flg1 = PETSC_FALSE;
418:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
419:   if (!flg1) {
420:     flg1 = PETSC_FALSE;
421:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
422:   }
423:   if (flg1) {
424:     PetscMemorySetGetMaximumUsage();
425:   }
426: #endif

428: #if defined(PETSC_USE_LOG)
429:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
430: #endif

432:   /*
433:       Set the display variable for graphics
434:   */
435:   PetscSetDisplay();

437:   /*
438:       Print the PETSc version information
439:   */
440:   PetscOptionsHasName(NULL,NULL,"-v",&flg1);
441:   PetscOptionsHasName(NULL,NULL,"-version",&flg2);
442:   PetscOptionsHasHelp(NULL,&flg3);
443:   if (flg1 || flg2 || flg3) {

445:     /*
446:        Print "higher-level" package version message
447:     */
448:     if (PetscExternalVersionFunction) {
449:       (*PetscExternalVersionFunction)(comm);
450:     }

452:     PetscGetVersion(version,256);
453:     (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
454:     (*PetscHelpPrintf)(comm,"%s\n",version);
455:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
456:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
457:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
458:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
459:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
460:     (*PetscHelpPrintf)(comm,"--------------------------------------------------------------------------\n");
461:   }

463:   /*
464:        Print "higher-level" package help message
465:   */
466:   if (flg3) {
467:     if (PetscExternalHelpFunction) {
468:       (*PetscExternalHelpFunction)(comm);
469:     }
470:   }

472:   PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
473:   if (flg1) {
474:     PetscStrcmp(helpoptions,"intro",&flg2);
475:     if (flg2) {
476:       PetscOptionsDestroyDefault();
477:       PetscFreeMPIResources();
478:       MPI_Finalize();
479:       exit(0);
480:     }
481:   }

483:   /*
484:       Setup the error handling
485:   */
486:   flg1 = PETSC_FALSE;
487:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
488:   if (flg1) {
489:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
490:     PetscPushErrorHandler(PetscAbortErrorHandler,0);
491:   }
492:   flg1 = PETSC_FALSE;
493:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
494:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
495:   flg1 = PETSC_FALSE;
496:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
497:   if (flg1) {
498:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
499:   }
500:   flg1 = PETSC_FALSE;
501:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
502:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
503:   flg1 = PETSC_FALSE;
504:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
505:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
506:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);

509:   /*
510:       Setup debugger information
511:   */
512:   PetscSetDefaultDebugger();
513:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
514:   if (flg1) {
515:     MPI_Errhandler err_handler;

517:     PetscSetDebuggerFromString(string);
518:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
519:     MPI_Comm_set_errhandler(comm,err_handler);
520:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
521:   }
522:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
523:   if (flg1) { PetscSetDebugTerminal(string); }
524:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
525:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
526:   if (flg1 || flg2) {
527:     PetscMPIInt    size;
528:     PetscInt       lsize,*nodes;
529:     MPI_Errhandler err_handler;
530:     /*
531:        we have to make sure that all processors have opened
532:        connections to all other processors, otherwise once the
533:        debugger has stated it is likely to receive a SIGUSR1
534:        and kill the program.
535:     */
536:     MPI_Comm_size(comm,&size);
537:     if (size > 2) {
538:       PetscMPIInt dummy = 0;
539:       MPI_Status  status;
540:       for (i=0; i<size; i++) {
541:         if (rank != i) {
542:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
543:         }
544:       }
545:       for (i=0; i<size; i++) {
546:         if (rank != i) {
547:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
548:         }
549:       }
550:     }
551:     /* check if this processor node should be in debugger */
552:     PetscMalloc1(size,&nodes);
553:     lsize = size;
554:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
555:     if (flag) {
556:       for (i=0; i<lsize; i++) {
557:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
558:       }
559:     }
560:     if (!flag) {
561:       PetscSetDebuggerFromString(string);
562:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
563:       if (flg1) {
564:         PetscAttachDebugger();
565:       } else {
566:         PetscStopForDebugger();
567:       }
568:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
569:       MPI_Comm_set_errhandler(comm,err_handler);
570:     }
571:     PetscFree(nodes);
572:   }

574:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
575:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

577:   /*
578:         Setup profiling and logging
579:   */
580: #if defined(PETSC_USE_INFO)
581:   {
582:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
583:     PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
584:     if (flg1 && logname[0]) {
585:       PetscInfoAllow(PETSC_TRUE,logname);
586:     } else if (flg1) {
587:       PetscInfoAllow(PETSC_TRUE,NULL);
588:     }
589:   }
590: #endif
591: #if defined(PETSC_USE_LOG)
592:   mname[0] = 0;
593:   PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
594:   if (flg1) {
595:     if (mname[0]) {
596:       PetscOpenHistoryFile(mname,&petsc_history);
597:     } else {
598:       PetscOpenHistoryFile(NULL,&petsc_history);
599:     }
600:   }

602:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

604: #if defined(PETSC_HAVE_MPE)
605:   flg1 = PETSC_FALSE;
606:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
607:   if (flg1) {PetscLogMPEBegin();}
608: #endif
609:   flg1 = PETSC_FALSE;
610:   flg3 = PETSC_FALSE;
611:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
612:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
613:   if (flg1)                      { PetscLogAllBegin(); }
614:   else if (flg3)                 { PetscLogDefaultBegin();}

616:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
617:   if (flg1) {
618:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
619:     FILE *file;
620:     if (mname[0]) {
621:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
622:       PetscFixFilename(name,fname);
623:       file = fopen(fname,"w");
624:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
625:     } else file = PETSC_STDOUT;
626:     PetscLogTraceBegin(file);
627:   }

629:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
630:   if (flg4) {
631:     if (format == PETSC_VIEWER_ASCII_XML) {
632:       PetscLogNestedBegin();
633:     } else {
634:       PetscLogDefaultBegin();
635:     }
636:   }
637:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
638:     PetscReal threshold = PetscRealConstant(0.01);
639:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
640:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
641:   }
642: #endif

644:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);

646: #if defined(PETSC_HAVE_CUDA)
647:   PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");
648:   PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCuda,&initCuda,NULL);
649:   PetscOptionsEnd();
650:   if (initCuda) {PetscCUDAInitialize(PETSC_COMM_WORLD);}
651: #endif

653:   /*
654:        Print basic help message
655:   */
656:   PetscOptionsHasHelp(NULL,&flg1);
657:   if (flg1) {
658:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
659:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
660:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
661:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
662:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
663:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
664:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
665:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
666:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
667:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
668:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
669:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
670:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
671:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
672:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
673:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
674:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
675:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
676:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
677:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
678:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
679:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
680:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
681:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
682:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
683:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
684:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
685:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
686:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
687:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
688:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
689:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
690:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
691: #if defined(PETSC_USE_LOG)
692:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
693:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
694:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
695: #if defined(PETSC_HAVE_MPE)
696:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
697: #endif
698:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
699: #endif
700:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
701:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
702:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
703:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
704:   }

706: #if defined(PETSC_HAVE_POPEN)
707:   {
708:   char machine[128];
709:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
710:   if (flg1) {
711:     PetscPOpenSetMachine(machine);
712:   }
713:   }
714: #endif

716:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
717:   if (flg1) {
718:     PetscSleep(si);
719:   }

721:   PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
722:   if (flg1) {
723:     PetscStrstr(mname,"null",&f);
724:     if (f) {
725:       PetscInfoDeactivateClass(0);
726:     }
727:   }

729: #if defined(PETSC_HAVE_VIENNACL)
730:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
731:   if (!flg3) {
732:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
733:   }
734:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
735:   PetscViennaCLSynchronize = flg3;
736:   PetscViennaCLInit();
737: #endif

739:   /*
740:      Creates the logging data structures; this is enabled even if logging is not turned on
741:      This is the last thing we do before returning to the user code to prevent having the
742:      logging numbers contaminated by any startup time associated with MPI and the GPUs
743:   */
744: #if defined(PETSC_USE_LOG)
745:   PetscLogInitialize();
746: #endif

748:   return(0);
749: }