Actual source code: init.c

petsc-3.14.0 2020-09-29
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot be known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9: #include <petscsys.h>
 10: #include <petsc/private/petscimpl.h>
 11: #include <petscvalgrind.h>
 12: #include <petscviewer.h>
 13: #if defined(PETSC_USE_LOG)
 14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
 15: #endif

 17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 18: #include <sys/sysinfo.h>
 19: #endif
 20: #if defined(PETSC_HAVE_UNISTD_H)
 21: #include <unistd.h>
 22: #endif

 24: #if defined(PETSC_HAVE_CUDA)
 25:   #include <cuda_runtime.h>
 26: #include <petsccublas.h>
 27: #endif

 29: #if defined(PETSC_HAVE_HIP)
 30:   #include <hip/hip_runtime.h>
 31: #endif

 33: #if defined(PETSC_HAVE_DEVICE)
 34:   #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
 35:     #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
 36:   #endif
 37: #endif

 39: #if defined(PETSC_HAVE_VIENNACL)
 40: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 41: #endif


 44: /* ------------------------Nasty global variables -------------------------------*/
 45: /*
 46:      Indicates if PETSc started up MPI, or it was
 47:    already started before PETSc was initialized.
 48: */
 49: PetscBool   PetscBeganMPI                 = PETSC_FALSE;
 50: PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
 51: PetscBool   PetscInitializeCalled         = PETSC_FALSE;
 52: PetscBool   PetscFinalizeCalled           = PETSC_FALSE;

 54: PetscMPIInt PetscGlobalRank               = -1;
 55: PetscMPIInt PetscGlobalSize               = -1;

 57: #if defined(PETSC_HAVE_KOKKOS)
 58: PetscBool   PetscBeganKokkos              = PETSC_FALSE;
 59: #endif

 61: PetscBool   use_gpu_aware_mpi             = PETSC_TRUE;

 63: #if defined(PETSC_HAVE_COMPLEX)
 64: #if defined(PETSC_COMPLEX_INSTANTIATE)
 65: template <> class std::complex<double>; /* instantiate complex template class */
 66: #endif
 67: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 68: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 69: MPI_Datatype MPIU_C_COMPLEX;
 70: #endif

 72: /*MC
 73:    PETSC_i - the imaginary number i

 75:    Synopsis:
 76: #include <petscsys.h>
 77:    PetscComplex PETSC_i;

 79:    Level: beginner

 81:    Note:
 82:    Complex numbers are automatically available if PETSc located a working complex implementation

 84: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 85: M*/
 86: PetscComplex PETSC_i;
 87: #endif
 88: #if defined(PETSC_USE_REAL___FLOAT128)
 89: MPI_Datatype MPIU___FLOAT128 = 0;
 90: #if defined(PETSC_HAVE_COMPLEX)
 91: MPI_Datatype MPIU___COMPLEX128 = 0;
 92: #endif
 93: #elif defined(PETSC_USE_REAL___FP16)
 94: MPI_Datatype MPIU___FP16 = 0;
 95: #endif
 96: MPI_Datatype MPIU_2SCALAR = 0;
 97: #if defined(PETSC_USE_64BIT_INDICES)
 98: MPI_Datatype MPIU_2INT = 0;
 99: #endif
100: MPI_Datatype MPIU_BOOL;
101: MPI_Datatype MPIU_ENUM;
102: MPI_Datatype MPIU_FORTRANADDR;
103: MPI_Datatype MPIU_SIZE_T;

105: /*
106:        Function that is called to display all error messages
107: */
108: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
109: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
110: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
111: /*
112:   This is needed to turn on/off GPU synchronization
113: */
114: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;

116: /* ------------------------------------------------------------------------------*/
117: /*
118:    Optional file where all PETSc output from various prints is saved
119: */
120: PETSC_INTERN FILE *petsc_history;
121: FILE *petsc_history = NULL;

123: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
124: {
126:   PetscMPIInt    rank,size;
127:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
128:   char           version[256];

131:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
132:   if (!rank) {
133:     char        arch[10];
134:     int         err;

136:     PetscGetArchType(arch,10);
137:     PetscGetDate(date,64);
138:     PetscGetVersion(version,256);
139:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
140:     if (filename) {
141:       PetscFixFilename(filename,fname);
142:     } else {
143:       PetscGetHomeDirectory(pfile,sizeof(pfile));
144:       PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
145:       PetscFixFilename(pfile,fname);
146:     }

148:     *fd = fopen(fname,"a");
149:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

151:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
153:     PetscGetProgramName(pname,sizeof(pname));
154:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
155:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

157:     err = fflush(*fd);
158:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
159:   }
160:   return(0);
161: }

163: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
164: {
166:   PetscMPIInt    rank;
167:   char           date[64];
168:   int            err;

171:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
172:   if (!rank) {
173:     PetscGetDate(date,64);
174:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
175:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
176:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
177:     err  = fflush(*fd);
178:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
179:     err = fclose(*fd);
180:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
181:   }
182:   return(0);
183: }

185: /* ------------------------------------------------------------------------------*/

187: /*
188:    This is ugly and probably belongs somewhere else, but I want to
189:   be able to put a true MPI abort error handler with command line args.

191:     This is so MPI errors in the debugger will leave all the stack
192:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
193:   in the debugger hence we call abort() instead of MPI_Abort().
194: */

196: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
197: {
199:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
200:   abort();
201: }

203: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
204: {

208:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
209:   PetscAttachDebugger();
210:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
211: }

213: /*@C
214:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
215:      wishes a clean exit somewhere deep in the program.

217:    Collective on PETSC_COMM_WORLD

219:    Options Database Keys are the same as for PetscFinalize()

221:    Level: advanced

223:    Note:
224:    See PetscInitialize() for more general runtime options.

226: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
227: @*/
228: PetscErrorCode  PetscEnd(void)
229: {
231:   PetscFinalize();
232:   exit(0);
233:   return 0;
234: }

236: PetscBool PetscOptionsPublish = PETSC_FALSE;
237: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
238: PETSC_INTERN PetscBool      petscsetmallocvisited;
239: static       char           emacsmachinename[256];

241: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
242: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

244: /*@C
245:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
246:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
247:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

249:    Input Parameter:
250: +  help - the help function (may be NULL)
251: -  version - the version function (may be NULL)

253:    Level: developer

255: @*/
256: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
257: {
259:   PetscExternalHelpFunction    = help;
260:   PetscExternalVersionFunction = version;
261:   return(0);
262: }

264: #if defined(PETSC_USE_LOG)
265: PETSC_INTERN PetscBool   PetscObjectsLog;
266: #endif

268: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
269: {
270:   if (PetscUnlikely(*err)) {
271:     PetscMPIInt len;
272:     char        errstring[MPI_MAX_ERROR_STRING];

274:     MPI_Error_string(*err,errstring,&len);
275:     PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
276:   }
277:   return;
278: }

280: /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
281:    Use the following macros to define CUDA/HIP initialization related vars/routines.
282:  */
283: #if defined(PETSC_HAVE_CUDA)
284:   typedef cudaError_t                             cupmError_t;
285:   typedef struct cudaDeviceProp                   cupmDeviceProp;
286:   #define cupmGetDeviceCount(x)                   cudaGetDeviceCount(x)
287:   #define cupmGetDevice(x)                        cudaGetDevice(x)
288:   #define cupmSetDevice(x)                        cudaSetDevice(x)
289:   #define cupmSetDeviceFlags(x)                   cudaSetDeviceFlags(x)
290:   #define cupmGetDeviceProperties(x,y)            cudaGetDeviceProperties(x,y)
291:   #define cupmGetLastError()                      cudaGetLastError()
292:   #define cupmDeviceMapHost                       cudaDeviceMapHost
293:   #define cupmSuccess                             cudaSuccess
294:   #define cupmErrorMemoryAllocation               cudaErrorMemoryAllocation
295:   #define cupmErrorLaunchOutOfResources           cudaErrorLaunchOutOfResources
296:   #define cupmErrorSetOnActiveProcess             cudaErrorSetOnActiveProcess
297:   #define CHKERRCUPM(x)                           CHKERRCUDA(x)
298:   #define PetscCUPMBLASInitializeHandle()         PetscCUBLASInitializeHandle()
299:   #define PetscCUPMSOLVERDnInitializeHandle()     PetscCUSOLVERDnInitializeHandle()
300:   #define PetscCUPMInitialize                     PetscCUDAInitialize
301:   #define PetscCUPMInitialized                    PetscCUDAInitialized
302:   #define PetscCUPMInitializeCheck                PetscCUDAInitializeCheck
303:   #define PetscCUPMInitializeAndView              PetscCUDAInitializeAndView
304:   #define PetscCUPMSynchronize                    PetscCUDASynchronize
305:   #define PetscNotUseCUPM                         PetscNotUseCUDA
306:   #define cupmOptionsStr                          "CUDA options"
307:   #define cupmSetDeviceStr                        "-cuda_device"
308:   #define cupmViewStr                             "-cuda_view"
309:   #define cupmSynchronizeStr                      "-cuda_synchronize"
310:   #define PetscCUPMInitializeStr                  "PetscCUDAInitialize"
311:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckCUDA
312:   #define PetscMPICUPMAwarenessCheck              PetscMPICUDAAwarenessCheck
313:   #include "cupminit.inc"
314: #endif

316: #if defined(PETSC_HAVE_HIP)
317:   typedef hipError_t                              cupmError_t;
318:   typedef hipDeviceProp_t                         cupmDeviceProp;
319:   #define cupmGetDeviceCount(x)                   hipGetDeviceCount(x)
320:   #define cupmGetDevice(x)                        hipGetDevice(x)
321:   #define cupmSetDevice(x)                        hipSetDevice(x)
322:   #define cupmSetDeviceFlags(x)                   hipSetDeviceFlags(x)
323:   #define cupmGetDeviceProperties(x,y)            hipGetDeviceProperties(x,y)
324:   #define cupmGetLastError()                      hipGetLastError()
325:   #define cupmDeviceMapHost                       hipDeviceMapHost
326:   #define cupmSuccess                             hipSuccess
327:   #define cupmErrorMemoryAllocation               hipErrorMemoryAllocation
328:   #define cupmErrorLaunchOutOfResources           hipErrorLaunchOutOfResources
329:   #define cupmErrorSetOnActiveProcess             hipErrorSetOnActiveProcess
330:   #define CHKERRCUPM(x)                           CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
331:   #define PetscCUPMBLASInitializeHandle()         0
332:   #define PetscCUPMSOLVERDnInitializeHandle()     0
333:   #define PetscCUPMInitialize                     PetscHIPInitialize
334:   #define PetscCUPMInitialized                    PetscHIPInitialized
335:   #define PetscCUPMInitializeCheck                PetscHIPInitializeCheck
336:   #define PetscCUPMInitializeAndView              PetscHIPInitializeAndView
337:   #define PetscCUPMSynchronize                    PetscHIPSynchronize
338:   #define PetscNotUseCUPM                         PetscNotUseHIP
339:   #define cupmOptionsStr                          "HIP options"
340:   #define cupmSetDeviceStr                        "-hip_device"
341:   #define cupmViewStr                             "-hip_view"
342:   #define cupmSynchronizeStr                      "-hip_synchronize"
343:   #define PetscCUPMInitializeStr                  "PetscHIPInitialize"
344:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckHIP
345:   #define PetscMPICUPMAwarenessCheck              PetscMPIHIPAwarenessCheck
346:   #include "cupminit.inc"
347: #endif

349: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
350: {
351:   char              string[64];
352:   MPI_Comm          comm = PETSC_COMM_WORLD;
353:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
354:   PetscErrorCode    ierr;
355:   PetscReal         si;
356:   PetscInt          intensity;
357:   int               i;
358:   PetscMPIInt       rank;
359:   char              version[256];
360: #if defined(PETSC_USE_LOG)
361:   char              mname[PETSC_MAX_PATH_LEN];
362:   PetscViewerFormat format;
363:   PetscBool         flg4 = PETSC_FALSE;
364: #endif

367:   MPI_Comm_rank(comm,&rank);

369: #if !defined(PETSC_HAVE_THREADSAFETY)
370:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
371:     /*
372:       Setup the memory management; support for tracing malloc() usage
373:     */
374:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

376:     if (PetscDefined(USE_DEBUG)) {
377:       mdebug        = PETSC_TRUE;
378:       initializenan = PETSC_TRUE;
379:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
380:     } else {
381:       /* don't warn about unused option */
382:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
383:       flg1 = PETSC_FALSE;
384:     }
385:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
386:     if (flg1 || flg2) {
387:       mdebug        = PETSC_TRUE;
388:       eachcall      = PETSC_TRUE;
389:       initializenan = PETSC_TRUE;
390:     } else if (flg3 && !flg2) {
391:       mdebug        = PETSC_FALSE;
392:       eachcall      = PETSC_FALSE;
393:       initializenan = PETSC_FALSE;
394:     }

396:     PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);
397:     if (flg2) {PetscMallocLogRequestedSizeSet(flg1);}

399:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
400:     if (mlog) {
401:       mdebug = PETSC_TRUE;
402:     }
403:     /* the next line is deprecated */
404:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
405:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
406:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
407:     if (mdebug) {
408:       PetscMallocSetDebug(eachcall,initializenan);
409:     }
410:     if (mlog) {
411:       PetscReal logthreshold = 0;
412:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
413:       PetscMallocViewSet(logthreshold);
414:     }
415: #if defined(PETSC_USE_LOG)
416:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
417: #endif
418:   }

420:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
421:   if (flg2) {PetscMallocSetCoalesce(flg1);}
422:   flg1 = PETSC_FALSE;
423:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
424:   /* ignore this option if malloc is already set */
425:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

427:   flg1 = PETSC_FALSE;
428:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
429:   if (!flg1) {
430:     flg1 = PETSC_FALSE;
431:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
432:   }
433:   if (flg1) {
434:     PetscMemorySetGetMaximumUsage();
435:   }
436: #endif

438: #if defined(PETSC_USE_LOG)
439:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
440: #endif

442:   /*
443:       Set the display variable for graphics
444:   */
445:   PetscSetDisplay();

447:   /*
448:      Print main application help message
449:   */
450:   PetscOptionsHasHelp(NULL,&hasHelp);
451:   if (help && hasHelp) {
452:     PetscPrintf(comm,help);
453:     PetscPrintf(comm,"----------------------------------------\n");
454:   }

456:   /*
457:       Print the PETSc version information
458:   */
459:   PetscOptionsHasName(NULL,NULL,"-version",&flg1);
460:   if (flg1 || hasHelp) {
461:     /*
462:        Print "higher-level" package version message
463:     */
464:     if (PetscExternalVersionFunction) {
465:       (*PetscExternalVersionFunction)(comm);
466:     }

468:     PetscGetVersion(version,256);
469:     (*PetscHelpPrintf)(comm,"%s\n",version);
470:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
471:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
472:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
473:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
474:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
475:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
476:   }

478:   /*
479:        Print "higher-level" package help message
480:   */
481:   if (hasHelp) {
482:     PetscBool hasHelpIntro;

484:     if (PetscExternalHelpFunction) {
485:       (*PetscExternalHelpFunction)(comm);
486:     }
487:     PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
488:     if (hasHelpIntro) {
489:       PetscOptionsDestroyDefault();
490:       PetscFreeMPIResources();
491:       MPI_Finalize();
492:       exit(0);
493:     }
494:   }

496:   /*
497:       Setup the error handling
498:   */
499:   flg1 = PETSC_FALSE;
500:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
501:   if (flg1) {
502:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
503:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
504:   }
505:   flg1 = PETSC_FALSE;
506:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
507:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
508:   flg1 = PETSC_FALSE;
509:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
510:   if (flg1) {
511:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
512:   }
513:   /* experimental */
514:   flg1 = PETSC_FALSE;
515:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
516:   if (flg1) {
517:     MPI_Errhandler eh;

519:     MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
520:     MPI_Comm_set_errhandler(comm,eh);
521:     MPI_Errhandler_free(&eh);
522:   }
523:   flg1 = PETSC_FALSE;
524:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
525:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}

527:   /*
528:       Setup debugger information
529:   */
530:   PetscSetDefaultDebugger();
531:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
532:   if (flg1) {
533:     MPI_Errhandler err_handler;

535:     PetscSetDebuggerFromString(string);
536:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
537:     MPI_Comm_set_errhandler(comm,err_handler);
538:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
539:   }
540:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
541:   if (flg1) { PetscSetDebugTerminal(string); }
542:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
543:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
544:   if (flg1 || flg2) {
545:     PetscMPIInt    size;
546:     PetscInt       lsize,*ranks;
547:     MPI_Errhandler err_handler;
548:     /*
549:        we have to make sure that all processors have opened
550:        connections to all other processors, otherwise once the
551:        debugger has stated it is likely to receive a SIGUSR1
552:        and kill the program.
553:     */
554:     MPI_Comm_size(comm,&size);
555:     if (size > 2) {
556:       PetscMPIInt dummy = 0;
557:       MPI_Status  status;
558:       for (i=0; i<size; i++) {
559:         if (rank != i) {
560:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
561:         }
562:       }
563:       for (i=0; i<size; i++) {
564:         if (rank != i) {
565:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
566:         }
567:       }
568:     }
569:     /* check if this processor node should be in debugger */
570:     PetscMalloc1(size,&ranks);
571:     lsize = size;
572:     /* Deprecated in 3.14 */
573:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);
574:     if (flag) {
575:       const char * const quietopt="-options_suppress_deprecated_warnings";
576:       char               msg[4096];
577:       PetscBool          quiet = PETSC_FALSE;

579:       PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);
580:       if (!quiet) {
581:         PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");
582:         PetscStrcat(msg,"-debugger_nodes");
583:         PetscStrcat(msg," is deprecated as of version ");
584:         PetscStrcat(msg,"3.14");
585:         PetscStrcat(msg," and will be removed in a future release.");
586:         PetscStrcat(msg," Please use the option ");
587:         PetscStrcat(msg,"-debugger_ranks");
588:         PetscStrcat(msg," instead.");
589:         PetscStrcat(msg," (Silence this warning with ");
590:         PetscStrcat(msg,quietopt);
591:         PetscStrcat(msg,")\n");
592:         PetscPrintf(comm,msg);
593:       }
594:     } else {
595:       lsize = size;
596:       PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);
597:     }
598:     if (flag) {
599:       for (i=0; i<lsize; i++) {
600:         if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
601:       }
602:     }
603:     if (!flag) {
604:       PetscSetDebuggerFromString(string);
605:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
606:       if (flg1) {
607:         PetscAttachDebugger();
608:       } else {
609:         PetscStopForDebugger();
610:       }
611:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
612:       MPI_Comm_set_errhandler(comm,err_handler);
613:     } else {
614:       PetscWaitOnError();
615:     }
616:     PetscFree(ranks);
617:   }

619:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
620:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

622:   /*
623:         Setup profiling and logging
624:   */
625: #if defined(PETSC_USE_INFO)
626:   {
627:     PetscInfoSetFromOptions(NULL);
628:   }
629: #endif
630:   PetscDetermineInitialFPTrap();
631:   flg1 = PETSC_FALSE;
632:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
633:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
634:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
636: #if defined(PETSC_USE_LOG)
637:   mname[0] = 0;
638:   PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
639:   if (flg1) {
640:     if (mname[0]) {
641:       PetscOpenHistoryFile(mname,&petsc_history);
642:     } else {
643:       PetscOpenHistoryFile(NULL,&petsc_history);
644:     }
645:   }

647:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

649: #if defined(PETSC_HAVE_MPE)
650:   flg1 = PETSC_FALSE;
651:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
652:   if (flg1) {PetscLogMPEBegin();}
653: #endif
654:   flg1 = PETSC_FALSE;
655:   flg3 = PETSC_FALSE;
656:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
657:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
658:   if (flg1)                      { PetscLogAllBegin(); }
659:   else if (flg3)                 { PetscLogDefaultBegin();}

661:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
662:   if (flg1) {
663:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
664:     FILE *file;
665:     if (mname[0]) {
666:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
667:       PetscFixFilename(name,fname);
668:       file = fopen(fname,"w");
669:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
670:     } else file = PETSC_STDOUT;
671:     PetscLogTraceBegin(file);
672:   }

674:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
675:   if (flg4) {
676:     if (format == PETSC_VIEWER_ASCII_XML) {
677:       PetscLogNestedBegin();
678:     } else {
679:       PetscLogDefaultBegin();
680:     }
681:   }
682:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
683:     PetscReal threshold = PetscRealConstant(0.01);
684:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
685:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
686:   }
687: #endif

689:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
690:   PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);
691:   /*
692:     If collecting logging information, by default, wait for device to complete its operations
693:     before returning to the CPU in order to get accurate timings of each event
694:   */
695:   PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
696:   if (!logView) {PetscOptionsHasName(NULL,NULL,"-log_view",&logView);}

698: #if defined(PETSC_HAVE_CUDA)
699:   PetscOptionsCheckCUDA(logView);
700: #endif

702: #if defined(PETSC_HAVE_HIP)
703:   PetscOptionsCheckHIP(logView);
704: #endif

706:   /*
707:        Print basic help message
708:   */
709:   if (hasHelp) {
710:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
711:     (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
712:     (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
713:     (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
714:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
715:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
716:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
717:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
718:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
719:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
720:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
721:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
722:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
723:     (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");
724:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
725:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
726:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
727:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
728:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
729:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
730:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
731:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
732:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
733:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
734:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
735:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
736:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
737:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
738:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
739:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
740:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
741:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
742:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
743:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
744:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
745: #if defined(PETSC_USE_LOG)
746:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
747:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
748:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
749:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
750: #if defined(PETSC_HAVE_MPE)
751:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
752: #endif
753: #endif
754: #if defined(PETSC_USE_INFO)
755:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
756: #endif
757:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
758:     (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
759:     (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
760:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
761:   }

763: #if defined(PETSC_HAVE_POPEN)
764:   {
765:   char machine[128];
766:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
767:   if (flg1) {
768:     PetscPOpenSetMachine(machine);
769:   }
770:   }
771: #endif

773:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
774:   if (flg1) {
775:     PetscSleep(si);
776:   }

778: #if defined(PETSC_HAVE_VIENNACL)
779:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
780:   if (!flg3) {
781:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
782:   }
783:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
784:   PetscViennaCLSynchronize = flg3;
785:   PetscViennaCLInit();
786: #endif

788:   /*
789:      Creates the logging data structures; this is enabled even if logging is not turned on
790:      This is the last thing we do before returning to the user code to prevent having the
791:      logging numbers contaminated by any startup time associated with MPI and the GPUs
792:   */
793: #if defined(PETSC_USE_LOG)
794:   PetscLogInitialize();
795: #endif

797:   return(0);
798: }