56 char control_filename[PETSC_MAX_PATH_LEN] =
"";
57 PetscBool control_flg;
58 PetscBool particle_console_output_freq_flg = PETSC_FALSE;
60 PetscFunctionBeginUser;
65 ierr = PetscNew(p_simCtx); CHKERRQ(ierr);
131 simCtx->
max_angle = -54. * 3.1415926 / 180.;
141 strcpy(simCtx->
grid_file,
"config/grid.run");
149 ierr = PetscMalloc1(1, &simCtx->
bcs_files); CHKERRQ(ierr);
150 ierr = PetscStrallocpy(
"config/bcs.run", &simCtx->
bcs_files[0]); CHKERRQ(ierr);
155 simCtx->
U_bc = 0.0; simCtx->
ccc = 0;
197 simCtx->
ibm = NULL; simCtx->
ibmv = NULL; simCtx->
fsi = NULL;
201 strcpy(simCtx->
allowedFile,
"config/whitelist.run");
202 simCtx->
useCfg = PETSC_FALSE;
232 ierr = PetscNew(&simCtx->
pps); CHKERRQ(ierr);
236 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &simCtx->
rank); CHKERRQ(ierr);
237 ierr = MPI_Comm_size(PETSC_COMM_WORLD, &simCtx->
size); CHKERRQ(ierr);
240 ierr = PetscOptionsGetString(NULL, NULL,
"-control_file", control_filename,
sizeof(control_filename), &control_flg); CHKERRQ(ierr);
243 if (!control_flg || strlen(control_filename) == 0) {
244 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
245 "\n\n*** MANDATORY ARGUMENT MISSING ***\n"
246 "The -control_file argument was not provided.\n"
247 "This program must be launched with a configuration file.\n"
248 "Example: mpiexec -n 4 ./simulator -control_file /path/to/your/config.control\n"
249 "This is typically handled automatically by the 'picurv' script.\n");
253 LOG(
GLOBAL,
LOG_INFO,
"Loading mandatory configuration from: %s\n", control_filename);
254 ierr = PetscOptionsInsertFile(PETSC_COMM_WORLD, NULL, control_filename, PETSC_FALSE);
255 if (ierr == PETSC_ERR_FILE_OPEN) {
256 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_FILE_OPEN,
"The specified control file was not found or could not be opened: %s", control_filename);
263 ierr = PetscOptionsGetString(NULL, NULL,
"-whitelist_config_file", simCtx->
allowedFile, PETSC_MAX_PATH_LEN, &simCtx->
useCfg); CHKERRQ(ierr);
269 PetscPrintf(PETSC_COMM_SELF,
"[%s] WARNING: Failed to load allowed functions from '%s'. Falling back to default list.\n", __func__, simCtx->
allowedFile);
270 simCtx->
useCfg = PETSC_FALSE;
273 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
274 "Whitelist config file '%s' is empty. Omit -whitelist_config_file to use the default allow-list, or list at least one function.",
282 ierr = PetscStrallocpy(
"main", &simCtx->
allowedFuncs[0]); CHKERRQ(ierr);
283 ierr = PetscStrallocpy(
"CreateSimulationContext", &simCtx->
allowedFuncs[1]); CHKERRQ(ierr);
295 ierr = PetscOptionsGetString(NULL, NULL,
"-profiling_timestep_file", simCtx->
profilingTimestepFile, PETSC_MAX_PATH_LEN, NULL); CHKERRQ(ierr);
296 ierr = PetscOptionsGetBool(NULL, NULL,
"-profiling_final_summary", &simCtx->
profilingFinalSummary, NULL); CHKERRQ(ierr);
300 PetscPrintf(PETSC_COMM_SELF,
"[%s] WARNING: Unknown profiling timestep mode '%s'. Falling back to 'selected'.\n", __func__, simCtx->
profilingTimestepMode);
309 PetscPrintf(PETSC_COMM_SELF,
"[%s] WARNING: Failed to load selected profiling functions from '%s'. Falling back to default list.\n", __func__, simCtx->
profilingSelectedFuncsFile);
335 ierr = PetscOptionsGetInt(NULL, NULL,
"-start_step", &simCtx->
StartStep, NULL); CHKERRQ(ierr);
336 ierr = PetscOptionsGetInt(NULL,NULL,
"-totalsteps", &simCtx->
StepsToRun, NULL); CHKERRQ(ierr);
337 ierr = PetscOptionsGetBool(NULL, NULL,
"-only_setup", &simCtx->
OnlySetup, NULL); CHKERRQ(ierr);
338 ierr = PetscOptionsGetBool(NULL, NULL,
"-continue_mode", &simCtx->
continueMode, NULL); CHKERRQ(ierr);
339 ierr = PetscOptionsGetReal(NULL, NULL,
"-dt", &simCtx->
dt, NULL); CHKERRQ(ierr);
340 ierr = PetscOptionsGetInt(NULL, NULL,
"-tio", &simCtx->
tiout, NULL); CHKERRQ(ierr);
341 ierr = PetscOptionsGetInt(NULL, NULL,
"-particle_console_output_freq", &simCtx->
particleConsoleOutputFreq, &particle_console_output_freq_flg); CHKERRQ(ierr);
342 if (!particle_console_output_freq_flg) {
346 ierr = PetscOptionsGetString(NULL,NULL,
"-output_dir",simCtx->
output_dir,
sizeof(simCtx->
output_dir),NULL);CHKERRQ(ierr);
347 ierr = PetscOptionsGetString(NULL,NULL,
"-restart_dir",simCtx->
restart_dir,
sizeof(simCtx->
restart_dir),NULL);CHKERRQ(ierr);
348 ierr = PetscOptionsGetString(NULL,NULL,
"-log_dir",simCtx->
log_dir,
sizeof(simCtx->
log_dir),NULL);CHKERRQ(ierr);
349 ierr = PetscOptionsGetString(NULL,NULL,
"-euler_subdir",simCtx->
euler_subdir,
sizeof(simCtx->
euler_subdir),NULL);CHKERRQ(ierr);
351 ierr = PetscOptionsGetBool(NULL, NULL,
"-walltime_guard_enabled", &simCtx->
walltimeGuardEnabled, NULL); CHKERRQ(ierr);
352 ierr = PetscOptionsGetInt(NULL, NULL,
"-walltime_guard_warmup_steps", &simCtx->
walltimeGuardWarmupSteps, NULL); CHKERRQ(ierr);
353 ierr = PetscOptionsGetReal(NULL, NULL,
"-walltime_guard_multiplier", &simCtx->
walltimeGuardMultiplier, NULL); CHKERRQ(ierr);
354 ierr = PetscOptionsGetReal(NULL, NULL,
"-walltime_guard_min_seconds", &simCtx->
walltimeGuardMinSeconds, NULL); CHKERRQ(ierr);
355 ierr = PetscOptionsGetReal(NULL, NULL,
"-walltime_guard_estimator_alpha", &simCtx->
walltimeGuardEstimatorAlpha, NULL); CHKERRQ(ierr);
358 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
"Invalid value for -walltime_guard_warmup_steps: %d. Must be > 0.", simCtx->
walltimeGuardWarmupSteps);
361 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
"Invalid value for -walltime_guard_multiplier: %.6f. Must be in (0, 5].", (
double)simCtx->
walltimeGuardMultiplier);
364 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
"Invalid value for -walltime_guard_min_seconds: %.6f. Must be > 0.", (
double)simCtx->
walltimeGuardMinSeconds);
367 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
"Invalid value for -walltime_guard_estimator_alpha: %.6f. Must be in (0, 1].", (
double)simCtx->
walltimeGuardEstimatorAlpha);
371 SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,
"Invalid value for -euler_field_source. Must be 'load','analytical' or 'solve'. You provided '%s'.",simCtx->
eulerianSource);
375 const char *job_start_env = getenv(
"PICURV_JOB_START_EPOCH");
376 const char *limit_env = getenv(
"PICURV_WALLTIME_LIMIT_SECONDS");
380 if (!job_start_ok || !limit_ok) {
387 "Runtime walltime guard enabled but %s/%s are missing or invalid. Falling back to external shutdown signals only.\n",
388 "PICURV_JOB_START_EPOCH",
389 "PICURV_WALLTIME_LIMIT_SECONDS"
398 ierr = PetscOptionsGetInt(NULL, NULL,
"-imm", &simCtx->
immersed, NULL); CHKERRQ(ierr);
399 ierr = PetscOptionsGetInt(NULL, NULL,
"-fsi", &simCtx->
movefsi, NULL); CHKERRQ(ierr);
400 ierr = PetscOptionsGetInt(NULL, NULL,
"-rfsi", &simCtx->
rotatefsi, NULL); CHKERRQ(ierr);
401 ierr = PetscOptionsGetInt(NULL, NULL,
"-sediment", &simCtx->
sediment, NULL); CHKERRQ(ierr);
402 ierr = PetscOptionsGetInt(NULL, NULL,
"-rheology", &simCtx->
rheology, NULL); CHKERRQ(ierr);
403 ierr = PetscOptionsGetInt(NULL, NULL,
"-inv", &simCtx->
invicid, NULL); CHKERRQ(ierr);
404 ierr = PetscOptionsGetInt(NULL, NULL,
"-TwoD", &simCtx->
TwoD, NULL); CHKERRQ(ierr);
405 ierr = PetscOptionsGetInt(NULL, NULL,
"-thin", &simCtx->
thin, NULL); CHKERRQ(ierr);
406 ierr = PetscOptionsGetInt(NULL, NULL,
"-mframe", &simCtx->
moveframe, NULL); CHKERRQ(ierr);
407 ierr = PetscOptionsGetInt(NULL, NULL,
"-rframe", &simCtx->
rotateframe, NULL); CHKERRQ(ierr);
408 ierr = PetscOptionsGetInt(NULL, NULL,
"-blk", &simCtx->
blank, NULL); CHKERRQ(ierr);
409 ierr = PetscOptionsGetInt(NULL, NULL,
"-dgf_z", &simCtx->
dgf_z, NULL); CHKERRQ(ierr);
410 ierr = PetscOptionsGetInt(NULL, NULL,
"-dgf_y", &simCtx->
dgf_y, NULL); CHKERRQ(ierr);
411 ierr = PetscOptionsGetInt(NULL, NULL,
"-dgf_x", &simCtx->
dgf_x, NULL); CHKERRQ(ierr);
412 ierr = PetscOptionsGetInt(NULL, NULL,
"-dgf_az", &simCtx->
dgf_az, NULL); CHKERRQ(ierr);
413 ierr = PetscOptionsGetInt(NULL, NULL,
"-dgf_ay", &simCtx->
dgf_ay, NULL); CHKERRQ(ierr);
414 ierr = PetscOptionsGetInt(NULL, NULL,
"-dgf_ax", &simCtx->
dgf_ax, NULL); CHKERRQ(ierr);
419 ierr = PetscOptionsGetInt(NULL, NULL,
"-cop", &simCtx->
cop, NULL); CHKERRQ(ierr);
420 ierr = PetscOptionsGetInt(NULL, NULL,
"-fish", &simCtx->
fish, NULL); CHKERRQ(ierr);
421 ierr = PetscOptionsGetInt(NULL, NULL,
"-pizza", &simCtx->
pizza, NULL); CHKERRQ(ierr);
422 ierr = PetscOptionsGetInt(NULL, NULL,
"-turbine", &simCtx->
turbine, NULL); CHKERRQ(ierr);
423 ierr = PetscOptionsGetInt(NULL, NULL,
"-fishcyl", &simCtx->
fishcyl, NULL); CHKERRQ(ierr);
424 ierr = PetscOptionsGetInt(NULL, NULL,
"-eel", &simCtx->
eel, NULL); CHKERRQ(ierr);
425 ierr = PetscOptionsGetInt(NULL, NULL,
"-cstart", &simCtx->
fish_c, NULL); CHKERRQ(ierr);
426 ierr = PetscOptionsGetInt(NULL, NULL,
"-wing", &simCtx->
wing, NULL); CHKERRQ(ierr);
427 ierr = PetscOptionsGetInt(NULL, NULL,
"-mhv", &simCtx->
MHV, NULL); CHKERRQ(ierr);
428 ierr = PetscOptionsGetInt(NULL, NULL,
"-hydro", &simCtx->
hydro, NULL); CHKERRQ(ierr);
429 ierr = PetscOptionsGetInt(NULL, NULL,
"-lv", &simCtx->
LV, NULL); CHKERRQ(ierr);
430 ierr = PetscOptionsGetInt(NULL, NULL,
"-Pipe", &simCtx->
Pipe, NULL); CHKERRQ(ierr);
431 ierr = PetscOptionsGetInt(NULL, NULL,
"-Turbulent_Channel_z", &simCtx->
channelz, NULL); CHKERRQ(ierr);
432 ierr = PetscOptionsGetReal(NULL,NULL,
"-Turbulent_Channel_z_Driving_Force",&simCtx->
drivingForceMagnitude,NULL);CHKERRQ(ierr);
433 ierr = PetscOptionsGetReal(NULL,NULL,
"-Turbulent_Channel_z_Scaling_Factor",&simCtx->
forceScalingFactor,NULL);CHKERRQ(ierr);
436 char mom_solver_type_char[PETSC_MAX_PATH_LEN];
437 PetscBool mom_solver_type_flg = PETSC_FALSE;
438 ierr = PetscOptionsGetString(NULL, NULL,
"-mom_solver_type", mom_solver_type_char,
sizeof(mom_solver_type_char), &mom_solver_type_flg); CHKERRQ(ierr);
439 ierr = PetscOptionsGetInt(NULL, NULL,
"-mom_max_pseudo_steps", &simCtx->
mom_max_pseudo_steps, NULL); CHKERRQ(ierr);
440 ierr = PetscOptionsGetReal(NULL, NULL,
"-mom_atol", &simCtx->
mom_atol, NULL); CHKERRQ(ierr);
441 ierr = PetscOptionsGetReal(NULL, NULL,
"-mom_rtol", &simCtx->
mom_rtol, NULL); CHKERRQ(ierr);
442 ierr = PetscOptionsGetReal(NULL, NULL,
"-imp_stol", &simCtx->
imp_stol, NULL); CHKERRQ(ierr);
443 ierr = PetscOptionsGetInt(NULL, NULL,
"-central", &simCtx->
central, NULL); CHKERRQ(ierr);
446 if (mom_solver_type_flg) {
447 if(strcmp(mom_solver_type_char,
"DUALTIME_PICARD_RK4") == 0) {
449 }
else if (strcmp(mom_solver_type_char,
"EXPLICIT_RK") == 0) {
452 LOG(
GLOBAL,
LOG_ERROR,
"Invalid value for -mom_solver_type: '%s'. Valid options are: 'DUALTIME_PICARD_RK4', 'EXPLICIT_RK'.\n", mom_solver_type_char);
453 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
"Invalid value for -mom_solver_type: '%s'.", mom_solver_type_char);
458 ierr = PetscOptionsGetInt(NULL, NULL,
"-mg_level", &simCtx->
mglevels, NULL); CHKERRQ(ierr);
459 ierr = PetscOptionsGetInt(NULL, NULL,
"-mg_max_it", &simCtx->
mg_MAX_IT, NULL); CHKERRQ(ierr);
460 ierr = PetscOptionsGetInt(NULL, NULL,
"-mg_idx", &simCtx->
mg_idx, NULL); CHKERRQ(ierr);
461 ierr = PetscOptionsGetInt(NULL, NULL,
"-mg_pre_it", &simCtx->
mg_preItr, NULL); CHKERRQ(ierr);
462 ierr = PetscOptionsGetInt(NULL, NULL,
"-mg_post_it", &simCtx->
mg_poItr, NULL); CHKERRQ(ierr);
465 ierr = PetscOptionsGetInt(NULL, NULL,
"-poisson", &simCtx->
poisson, NULL); CHKERRQ(ierr);
466 ierr = PetscOptionsGetReal(NULL, NULL,
"-poisson_tol", &simCtx->
poisson_tol, NULL); CHKERRQ(ierr);
467 ierr = PetscOptionsGetInt(NULL, NULL,
"-str", &simCtx->
STRONG_COUPLING, NULL); CHKERRQ(ierr);
468 ierr = PetscOptionsGetReal(NULL, NULL,
"-ren", &simCtx->
ren, NULL); CHKERRQ(ierr);
469 ierr = PetscOptionsGetReal(NULL, NULL,
"-pseudo_cfl", &simCtx->
pseudo_cfl, NULL); CHKERRQ(ierr);
470 ierr = PetscOptionsGetReal(NULL, NULL,
"-max_pseudo_cfl", &simCtx->
max_pseudo_cfl, NULL); CHKERRQ(ierr);
471 ierr = PetscOptionsGetReal(NULL, NULL,
"-min_pseudo_cfl", &simCtx->
min_pseudo_cfl, NULL); CHKERRQ(ierr);
473 ierr = PetscOptionsGetReal(NULL, NULL,
"-pseudo_cfl_growth_factor", &simCtx->
pseudo_cfl_growth_factor, NULL); CHKERRQ(ierr);
476 ierr = PetscOptionsGetInt(NULL, NULL,
"-finit", &simCtx->
FieldInitialization, NULL); CHKERRQ(ierr);
477 ierr = PetscOptionsGetReal(NULL, NULL,
"-ucont_x", &simCtx->
InitialConstantContra.
x, NULL); CHKERRQ(ierr);
478 ierr = PetscOptionsGetReal(NULL, NULL,
"-ucont_y", &simCtx->
InitialConstantContra.
y, NULL); CHKERRQ(ierr);
479 ierr = PetscOptionsGetReal(NULL, NULL,
"-ucont_z", &simCtx->
InitialConstantContra.
z, NULL); CHKERRQ(ierr);
483 PetscBool verification_scalar_value_set = PETSC_FALSE;
484 PetscBool verification_scalar_phi0_set = PETSC_FALSE;
485 PetscBool verification_scalar_slope_x_set = PETSC_FALSE;
486 PetscBool verification_scalar_amplitude_set = PETSC_FALSE;
487 PetscBool verification_scalar_kx_set = PETSC_FALSE;
488 PetscBool verification_scalar_ky_set = PETSC_FALSE;
489 PetscBool verification_scalar_kz_set = PETSC_FALSE;
490 ierr = PetscOptionsGetString(NULL, NULL,
"-verification_diffusivity_mode",
493 ierr = PetscOptionsGetString(NULL, NULL,
"-verification_diffusivity_profile",
496 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_diffusivity_gamma0",
498 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_diffusivity_slope_x",
500 ierr = PetscOptionsGetString(NULL, NULL,
"-verification_scalar_mode",
503 ierr = PetscOptionsGetString(NULL, NULL,
"-verification_scalar_profile",
506 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_value",
508 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_phi0",
510 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_slope_x",
512 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_amplitude",
514 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_kx",
516 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_ky",
518 ierr = PetscOptionsGetReal(NULL, NULL,
"-verification_scalar_kz",
528 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONGSTATE,
529 "verification diffusivity overrides require -euler_field_source \"analytical\".");
532 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
533 "Unsupported -verification_diffusivity_mode '%s'. Only 'analytical' is supported.",
537 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
538 "Unsupported -verification_diffusivity_profile '%s'. Only 'LINEAR_X' is supported.",
544 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONGSTATE,
545 "verification scalar overrides require -euler_field_source \"analytical\".");
548 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
549 "Unsupported -verification_scalar_mode '%s'. Only 'analytical' is supported.",
553 if (!verification_scalar_value_set) {
554 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
555 "verification scalar profile CONSTANT requires -verification_scalar_value.");
558 if (!verification_scalar_phi0_set || !verification_scalar_slope_x_set) {
559 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
560 "verification scalar profile LINEAR_X requires -verification_scalar_phi0 and -verification_scalar_slope_x.");
563 if (!verification_scalar_amplitude_set || !verification_scalar_kx_set ||
564 !verification_scalar_ky_set || !verification_scalar_kz_set) {
565 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
566 "verification scalar profile SIN_PRODUCT requires -verification_scalar_amplitude, -verification_scalar_kx, -verification_scalar_ky, and -verification_scalar_kz.");
569 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
570 "Unsupported -verification_scalar_profile '%s'. Supported profiles: CONSTANT, LINEAR_X, SIN_PRODUCT.",
578 ierr = PetscOptionsGetReal(NULL,NULL,
"-schmidt_number",&simCtx->
schmidt_number,NULL);CHKERRQ(ierr);
580 ierr = PetscOptionsGetInt(NULL, NULL,
"-no_of_bodies", &simCtx->
NumberOfBodies, NULL); CHKERRQ(ierr);
581 ierr = PetscOptionsGetReal(NULL,NULL,
"-wall_roughness",&simCtx->
wall_roughness_height,NULL);CHKERRQ(ierr);
590 ierr = PetscOptionsGetInt(NULL, NULL,
"-nblk", &simCtx->
block_number, NULL); CHKERRQ(ierr);
591 ierr = PetscOptionsGetInt(NULL, NULL,
"-inlet", &simCtx->
inletprofile, NULL); CHKERRQ(ierr);
592 ierr = PetscOptionsGetInt(NULL, NULL,
"-Ogrid", &simCtx->
Ogrid, NULL); CHKERRQ(ierr);
594 ierr = PetscOptionsGetInt(NULL, NULL,
"-grid1d", &simCtx->
grid1d, NULL); CHKERRQ(ierr);
595 ierr = PetscOptionsGetBool(NULL, NULL,
"-grid", &simCtx->
generate_grid, NULL); CHKERRQ(ierr);
596 ierr = PetscOptionsGetString(NULL, NULL,
"-grid_file", simCtx->
grid_file, PETSC_MAX_PATH_LEN, NULL); CHKERRQ(ierr);
597 ierr = PetscOptionsGetInt(NULL, NULL,
"-da_processors_x", &simCtx->
da_procs_x, NULL); CHKERRQ(ierr);
598 ierr = PetscOptionsGetInt(NULL, NULL,
"-da_processors_y", &simCtx->
da_procs_y, NULL); CHKERRQ(ierr);
599 ierr = PetscOptionsGetInt(NULL, NULL,
"-da_processors_z", &simCtx->
da_procs_z, NULL); CHKERRQ(ierr);
600 ierr = PetscOptionsGetInt(NULL, NULL,
"-i_periodic", &simCtx->
i_periodic, NULL); CHKERRQ(ierr);
601 ierr = PetscOptionsGetInt(NULL, NULL,
"-j_periodic", &simCtx->
j_periodic, NULL); CHKERRQ(ierr);
602 ierr = PetscOptionsGetInt(NULL, NULL,
"-k_periodic", &simCtx->
k_periodic, NULL); CHKERRQ(ierr);
603 ierr = PetscOptionsGetInt(NULL, NULL,
"-pbc_domain", &simCtx->
blkpbc, NULL); CHKERRQ(ierr);
605 ierr = PetscOptionsGetReal(NULL, NULL,
"-grid_rotation_angle", &simCtx->
grid_rotation_angle, NULL); CHKERRQ(ierr);
606 ierr = PetscOptionsGetReal(NULL, NULL,
"-Croty", &simCtx->
Croty, NULL); CHKERRQ(ierr);
607 ierr = PetscOptionsGetReal(NULL, NULL,
"-Crotz", &simCtx->
Crotz, NULL); CHKERRQ(ierr);
609 char file_list_str[PETSC_MAX_PATH_LEN * 10];
611 ierr = PetscOptionsGetString(NULL, NULL,
"-bcs_files", file_list_str,
sizeof(file_list_str), &bcs_flg); CHKERRQ(ierr);
612 ierr = PetscOptionsGetReal(NULL, NULL,
"-U_bc", &simCtx->
U_bc, NULL); CHKERRQ(ierr);
618 ierr = PetscFree(simCtx->
bcs_files[0]); CHKERRQ(ierr);
619 ierr = PetscFree(simCtx->
bcs_files); CHKERRQ(ierr);
626 ierr = PetscStrallocpy(file_list_str, &str_copy); CHKERRQ(ierr);
629 token = strtok(str_copy,
",");
632 token = strtok(NULL,
",");
634 ierr = PetscFree(str_copy); CHKERRQ(ierr);
638 ierr = PetscStrallocpy(file_list_str, &str_copy); CHKERRQ(ierr);
639 token = strtok(str_copy,
",");
641 ierr = PetscStrallocpy(token, &simCtx->
bcs_files[i]); CHKERRQ(ierr);
642 token = strtok(NULL,
",");
644 ierr = PetscFree(str_copy); CHKERRQ(ierr);
650 PetscInt temp_les_model;
651 ierr = PetscOptionsGetInt(NULL, NULL,
"-les", &temp_les_model, NULL); CHKERRQ(ierr);
653 ierr = PetscOptionsGetInt(NULL, NULL,
"-rans", &simCtx->
rans, NULL); CHKERRQ(ierr);
654 ierr = PetscOptionsGetInt(NULL, NULL,
"-wallfunction", &simCtx->
wallfunction, NULL); CHKERRQ(ierr);
655 ierr = PetscOptionsGetInt(NULL, NULL,
"-mixed", &simCtx->
mixed, NULL); CHKERRQ(ierr);
656 ierr = PetscOptionsGetInt(NULL, NULL,
"-clark", &simCtx->
clark, NULL); CHKERRQ(ierr);
657 ierr = PetscOptionsGetInt(NULL, NULL,
"-dynamic_freq", &simCtx->
dynamic_freq, NULL); CHKERRQ(ierr);
658 ierr = PetscOptionsGetReal(NULL, NULL,
"-max_cs", &simCtx->
max_cs, NULL); CHKERRQ(ierr);
659 ierr = PetscOptionsGetReal(NULL, NULL,
"-const_cs", &simCtx->
Const_CS, NULL); CHKERRQ(ierr);
660 ierr = PetscOptionsGetInt(NULL, NULL,
"-testfilter_ik", &simCtx->
testfilter_ik, NULL); CHKERRQ(ierr);
661 ierr = PetscOptionsGetInt(NULL, NULL,
"-testfilter_1d", &simCtx->
testfilter_1d, NULL); CHKERRQ(ierr);
662 ierr = PetscOptionsGetInt(NULL, NULL,
"-i_homo_filter", &simCtx->
i_homo_filter, NULL); CHKERRQ(ierr);
663 ierr = PetscOptionsGetInt(NULL, NULL,
"-j_homo_filter", &simCtx->
j_homo_filter, NULL); CHKERRQ(ierr);
664 ierr = PetscOptionsGetInt(NULL, NULL,
"-k_homo_filter", &simCtx->
k_homo_filter, NULL); CHKERRQ(ierr);
665 ierr = PetscOptionsGetBool(NULL, NULL,
"-averaging", &simCtx->
averaging, NULL); CHKERRQ(ierr);
669 ierr = PetscOptionsGetInt(NULL, NULL,
"-numParticles", &simCtx->
np, NULL); CHKERRQ(ierr);
670 ierr = PetscOptionsGetBool(NULL, NULL,
"-read_fields", &simCtx->
readFields, NULL); CHKERRQ(ierr);
672 ierr = PetscOptionsGetInt(NULL, NULL,
"-pinit", &temp_pinit, NULL); CHKERRQ(ierr);
675 ierr = PetscOptionsGetInt(NULL, NULL,
"-interpolation_method", &temp_interp, NULL); CHKERRQ(ierr);
679 ierr = PetscOptionsGetReal(NULL, NULL,
"-psrc_x", &simCtx->
psrc_x, NULL); CHKERRQ(ierr);
680 ierr = PetscOptionsGetReal(NULL, NULL,
"-psrc_y", &simCtx->
psrc_y, NULL); CHKERRQ(ierr);
681 ierr = PetscOptionsGetReal(NULL, NULL,
"-psrc_z", &simCtx->
psrc_z, NULL); CHKERRQ(ierr);
688 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG,
"Invalid value for -particle_restart_mode. Must be 'load' or 'init'. You provided '%s'.", simCtx->
particleRestartMode);
693 ierr = PetscOptionsGetBool(NULL, NULL,
"-rs_fsi", &simCtx->
rstart_fsi, NULL); CHKERRQ(ierr);
694 ierr = PetscOptionsGetInt(NULL, NULL,
"-duplicate", &simCtx->
duplicate, NULL); CHKERRQ(ierr);
698 ierr = PetscOptionsGetInt(NULL, NULL,
"-logfreq", &simCtx->
LoggingFrequency, NULL); CHKERRQ(ierr);
701 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_INCOMP,
"Number of BC files (%d) does not match number of blocks (%d). Use -bcs_files \"file1.dat,file2.dat,...\".", simCtx->
num_bcs_files, simCtx->
block_number);
707 ierr = PetscOptionsGetString(NULL,NULL,
"-postprocessing_config_file",simCtx->
PostprocessingControlFile,PETSC_MAX_PATH_LEN,NULL); CHKERRQ(ierr);
719 for (PetscInt i = 0; i < simCtx->
nAllowed; ++i) {
727 if (simCtx->
tiout > 0) {
734 if (simCtx->
np > 0) {
747 ierr = PetscLogDefaultBegin(); CHKERRQ(ierr);
752 PetscFunctionReturn(0);
819 PetscFunctionBeginUser;
820 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
835 ierr = PetscSNPrintf(desc,
sizeof(desc),
"BCS file #%d", i + 1); CHKERRQ(ierr);
879 ierr = PetscRMTree(simCtx->
log_dir);
881 PetscError(PETSC_COMM_SELF, __LINE__,
__FUNCT__, __FILE__, ierr, PETSC_ERROR_INITIAL,
"Could not remove existing log directory '%s'. Check permissions.", simCtx->
log_dir);
883 ierr = PetscMkdir(simCtx->
log_dir); CHKERRQ(ierr);
887 ierr = PetscMkdir(simCtx->
log_dir); CHKERRQ(ierr);
891 char path_buffer[PETSC_MAX_PATH_LEN];
895 ierr = PetscTestDirectory(simCtx->
output_dir,
'r', &exists); CHKERRQ(ierr);
898 ierr = PetscMkdir(simCtx->
output_dir); CHKERRQ(ierr);
902 ierr = PetscSNPrintf(path_buffer,
sizeof(path_buffer),
"%s/%s", simCtx->
output_dir, simCtx->
euler_subdir); CHKERRQ(ierr);
904 ierr = PetscTestDirectory(path_buffer,
'r', &exists); CHKERRQ(ierr);
907 ierr = PetscMkdir(path_buffer); CHKERRQ(ierr);
911 if (simCtx->
np > 0) {
912 ierr = PetscSNPrintf(path_buffer,
sizeof(path_buffer),
"%s/%s", simCtx->
output_dir, simCtx->
particle_subdir); CHKERRQ(ierr);
914 ierr = PetscTestDirectory(path_buffer,
'r', &exists); CHKERRQ(ierr);
917 ierr = PetscMkdir(path_buffer); CHKERRQ(ierr);
924 char path_buffer[PETSC_MAX_PATH_LEN];
926 const char *last_slash_euler = strrchr(pps->
output_prefix,
'/');
927 if(last_slash_euler){
930 if(dir_len >=
sizeof(path_buffer)) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,
"Post-processing output prefix path is too long.");
932 path_buffer[dir_len] =
'\0';
934 ierr = PetscTestDirectory(path_buffer,
'r', &exists); CHKERRQ(ierr);
945 if(last_slash_particle){
948 if(dir_len >
sizeof(path_buffer)) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,
"Post-processing particle output prefix path is too long.");
950 path_buffer[dir_len] =
'\0';
952 ierr = PetscTestDirectory(path_buffer,
'r', &exists); CHKERRQ(ierr);
965 if(last_slash_stats){
968 if(dir_len >=
sizeof(path_buffer)) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,
"Post-processing statistics output prefix path is too long.");
970 path_buffer[dir_len] =
'\0';
972 ierr = PetscTestDirectory(path_buffer,
'r', &exists); CHKERRQ(ierr);
984 ierr = MPI_Barrier(PETSC_COMM_WORLD); CHKERRMPI(ierr);
988 PetscFunctionReturn(0);
1170 PetscErrorCode ierr;
1175 PetscFunctionBeginUser;
1181 for (PetscInt level = usermg->
mglevels-1; level >=0; level--) {
1182 for (PetscInt bi = 0; bi < nblk; bi++) {
1185 if(!user->
da || !user->
fda) {
1186 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONGSTATE,
"DMs not properly initialized in UserCtx before vector creation.");
1193 ierr = DMCreateGlobalVector(user->
fda, &user->
Ucont); CHKERRQ(ierr); ierr = VecSet(user->
Ucont, 0.0); CHKERRQ(ierr);
1194 ierr = DMCreateGlobalVector(user->
fda, &user->
Ucat); CHKERRQ(ierr); ierr = VecSet(user->
Ucat, 0.0); CHKERRQ(ierr);
1195 ierr = DMCreateGlobalVector(user->
da, &user->
P); CHKERRQ(ierr); ierr = VecSet(user->
P, 0.0); CHKERRQ(ierr);
1196 ierr = DMCreateGlobalVector(user->
da, &user->
Nvert); CHKERRQ(ierr); ierr = VecSet(user->
Nvert, 0.0); CHKERRQ(ierr);
1198 ierr = DMCreateLocalVector(user->
fda, &user->
lUcont); CHKERRQ(ierr); ierr = VecSet(user->
lUcont, 0.0); CHKERRQ(ierr);
1199 ierr = DMCreateLocalVector(user->
fda, &user->
lUcat); CHKERRQ(ierr); ierr = VecSet(user->
lUcat, 0.0); CHKERRQ(ierr);
1200 ierr = DMCreateLocalVector(user->
da, &user->
lP); CHKERRQ(ierr); ierr = VecSet(user->
lP, 0.0); CHKERRQ(ierr);
1201 ierr = DMCreateLocalVector(user->
da, &user->
lNvert); CHKERRQ(ierr); ierr = VecSet(user->
lNvert, 0.0); CHKERRQ(ierr);
1204 ierr = VecDuplicate(user->
P,&user->
Diffusivity); CHKERRQ(ierr); ierr = VecSet(user->
Diffusivity, 0.0); CHKERRQ(ierr);
1210 ierr = VecDuplicate(user->
P, &user->
Phi); CHKERRQ(ierr); ierr = VecSet(user->
Phi, 0.0); CHKERRQ(ierr);
1211 ierr = VecDuplicate(user->
lP, &user->
lPhi); CHKERRQ(ierr); ierr = VecSet(user->
lPhi, 0.0); CHKERRQ(ierr);
1214 if (level == usermg->
mglevels - 1) {
1215 ierr = VecDuplicate(user->
Ucont, &user->
Ucont_o); CHKERRQ(ierr); ierr = VecSet(user->
Ucont_o, 0.0); CHKERRQ(ierr);
1216 ierr = VecDuplicate(user->
Ucont, &user->
Ucont_rm1); CHKERRQ(ierr); ierr = VecSet(user->
Ucont_rm1, 0.0); CHKERRQ(ierr);
1217 ierr = VecDuplicate(user->
Ucat, &user->
Ucat_o); CHKERRQ(ierr); ierr = VecSet(user->
Ucat_o, 0.0); CHKERRQ(ierr);
1218 ierr = VecDuplicate(user->
P, &user->
P_o); CHKERRQ(ierr); ierr = VecSet(user->
P_o, 0.0); CHKERRQ(ierr);
1219 ierr = VecDuplicate(user->
lUcont, &user->
lUcont_o); CHKERRQ(ierr); ierr = VecSet(user->
lUcont_o, 0.0); CHKERRQ(ierr);
1221 ierr = DMCreateLocalVector(user->
da, &user->
lNvert_o); CHKERRQ(ierr); ierr = VecSet(user->
lNvert_o, 0.0); CHKERRQ(ierr);
1222 ierr = VecDuplicate(user->
Nvert, &user->
Nvert_o); CHKERRQ(ierr); ierr = VecSet(user->
Nvert_o, 0.0); CHKERRQ(ierr);
1226 ierr = DMCreateGlobalVector(user->
fda, &user->
Csi); CHKERRQ(ierr); ierr = VecSet(user->
Csi, 0.0); CHKERRQ(ierr);
1227 ierr = VecDuplicate(user->
Csi, &user->
Eta); CHKERRQ(ierr); ierr = VecSet(user->
Eta, 0.0); CHKERRQ(ierr);
1228 ierr = VecDuplicate(user->
Csi, &user->
Zet); CHKERRQ(ierr); ierr = VecSet(user->
Zet, 0.0); CHKERRQ(ierr);
1229 ierr = DMCreateGlobalVector(user->
da, &user->
Aj); CHKERRQ(ierr); ierr = VecSet(user->
Aj, 0.0); CHKERRQ(ierr);
1231 ierr = DMCreateLocalVector(user->
fda, &user->
lCsi); CHKERRQ(ierr); ierr = VecSet(user->
lCsi, 0.0); CHKERRQ(ierr);
1232 ierr = VecDuplicate(user->
lCsi, &user->
lEta); CHKERRQ(ierr); ierr = VecSet(user->
lEta, 0.0); CHKERRQ(ierr);
1233 ierr = VecDuplicate(user->
lCsi, &user->
lZet); CHKERRQ(ierr); ierr = VecSet(user->
lZet, 0.0); CHKERRQ(ierr);
1234 ierr = DMCreateLocalVector(user->
da, &user->
lAj); CHKERRQ(ierr); ierr = VecSet(user->
lAj, 0.0); CHKERRQ(ierr);
1239 ierr = VecDuplicate(user->
Csi, &user->
ICsi); CHKERRQ(ierr); ierr = VecSet(user->
ICsi, 0.0); CHKERRQ(ierr);
1240 ierr = VecDuplicate(user->
Csi, &user->
IEta); CHKERRQ(ierr); ierr = VecSet(user->
IEta, 0.0); CHKERRQ(ierr);
1241 ierr = VecDuplicate(user->
Csi, &user->
IZet); CHKERRQ(ierr); ierr = VecSet(user->
IZet, 0.0); CHKERRQ(ierr);
1242 ierr = VecDuplicate(user->
Csi, &user->
JCsi); CHKERRQ(ierr); ierr = VecSet(user->
JCsi, 0.0); CHKERRQ(ierr);
1243 ierr = VecDuplicate(user->
Csi, &user->
JEta); CHKERRQ(ierr); ierr = VecSet(user->
JEta, 0.0); CHKERRQ(ierr);
1244 ierr = VecDuplicate(user->
Csi, &user->
JZet); CHKERRQ(ierr); ierr = VecSet(user->
JZet, 0.0); CHKERRQ(ierr);
1245 ierr = VecDuplicate(user->
Csi, &user->
KCsi); CHKERRQ(ierr); ierr = VecSet(user->
KCsi, 0.0); CHKERRQ(ierr);
1246 ierr = VecDuplicate(user->
Csi, &user->
KEta); CHKERRQ(ierr); ierr = VecSet(user->
KEta, 0.0); CHKERRQ(ierr);
1247 ierr = VecDuplicate(user->
Csi, &user->
KZet); CHKERRQ(ierr); ierr = VecSet(user->
KZet, 0.0); CHKERRQ(ierr);
1249 ierr = VecDuplicate(user->
Aj, &user->
IAj); CHKERRQ(ierr); ierr = VecSet(user->
IAj, 0.0); CHKERRQ(ierr);
1250 ierr = VecDuplicate(user->
Aj, &user->
JAj); CHKERRQ(ierr); ierr = VecSet(user->
JAj, 0.0); CHKERRQ(ierr);
1251 ierr = VecDuplicate(user->
Aj, &user->
KAj); CHKERRQ(ierr); ierr = VecSet(user->
KAj, 0.0); CHKERRQ(ierr);
1253 ierr = VecDuplicate(user->
lCsi, &user->
lICsi); CHKERRQ(ierr); ierr = VecSet(user->
lICsi, 0.0); CHKERRQ(ierr);
1254 ierr = VecDuplicate(user->
lCsi, &user->
lIEta); CHKERRQ(ierr); ierr = VecSet(user->
lIEta, 0.0); CHKERRQ(ierr);
1255 ierr = VecDuplicate(user->
lCsi, &user->
lIZet); CHKERRQ(ierr); ierr = VecSet(user->
lIZet, 0.0); CHKERRQ(ierr);
1256 ierr = VecDuplicate(user->
lCsi, &user->
lJCsi); CHKERRQ(ierr); ierr = VecSet(user->
lJCsi, 0.0); CHKERRQ(ierr);
1257 ierr = VecDuplicate(user->
lCsi, &user->
lJEta); CHKERRQ(ierr); ierr = VecSet(user->
lJEta, 0.0); CHKERRQ(ierr);
1258 ierr = VecDuplicate(user->
lCsi, &user->
lJZet); CHKERRQ(ierr); ierr = VecSet(user->
lJZet, 0.0); CHKERRQ(ierr);
1259 ierr = VecDuplicate(user->
lCsi, &user->
lKCsi); CHKERRQ(ierr); ierr = VecSet(user->
lKCsi, 0.0); CHKERRQ(ierr);
1260 ierr = VecDuplicate(user->
lCsi, &user->
lKEta); CHKERRQ(ierr); ierr = VecSet(user->
lKEta, 0.0); CHKERRQ(ierr);
1261 ierr = VecDuplicate(user->
lCsi, &user->
lKZet); CHKERRQ(ierr); ierr = VecSet(user->
lKZet, 0.0); CHKERRQ(ierr);
1263 ierr = VecDuplicate(user->
lAj, &user->
lIAj); CHKERRQ(ierr); ierr = VecSet(user->
lIAj, 0.0); CHKERRQ(ierr);
1264 ierr = VecDuplicate(user->
lAj, &user->
lJAj); CHKERRQ(ierr); ierr = VecSet(user->
lJAj, 0.0); CHKERRQ(ierr);
1265 ierr = VecDuplicate(user->
lAj, &user->
lKAj); CHKERRQ(ierr); ierr = VecSet(user->
lKAj, 0.0); CHKERRQ(ierr);
1268 ierr = DMCreateGlobalVector(user->
fda, &user->
Cent); CHKERRQ(ierr); ierr = VecSet(user->
Cent, 0.0); CHKERRQ(ierr);
1269 ierr = DMCreateLocalVector(user->
fda, &user->
lCent); CHKERRQ(ierr); ierr = VecSet(user->
lCent, 0.0); CHKERRQ(ierr);
1271 ierr = VecDuplicate(user->
Cent, &user->
GridSpace); CHKERRQ(ierr); ierr = VecSet(user->
GridSpace, 0.0); CHKERRQ(ierr);
1272 ierr = VecDuplicate(user->
lCent, &user->
lGridSpace); CHKERRQ(ierr); ierr = VecSet(user->
lGridSpace, 0.0); CHKERRQ(ierr);
1275 ierr = VecDuplicate(user->
lCent, &user->
Centx); CHKERRQ(ierr); ierr = VecSet(user->
Centx, 0.0); CHKERRQ(ierr);
1276 ierr = VecDuplicate(user->
lCent, &user->
Centy); CHKERRQ(ierr); ierr = VecSet(user->
Centy, 0.0); CHKERRQ(ierr);
1277 ierr = VecDuplicate(user->
lCent, &user->
Centz); CHKERRQ(ierr); ierr = VecSet(user->
Centz, 0.0); CHKERRQ(ierr);
1281 if (simCtx->
les || simCtx->
rans) {
1282 ierr = DMCreateGlobalVector(user->
da, &user->
Nu_t); CHKERRQ(ierr); ierr = VecSet(user->
Nu_t, 0.0); CHKERRQ(ierr);
1283 ierr = DMCreateLocalVector(user->
da, &user->
lNu_t); CHKERRQ(ierr); ierr = VecSet(user->
lNu_t, 0.0); CHKERRQ(ierr);
1286 ierr = DMCreateGlobalVector(user->
da,&user->
CS); CHKERRQ(ierr); ierr = VecSet(user->
CS,0.0); CHKERRQ(ierr);
1287 ierr = DMCreateLocalVector(user->
da,&user->
lCs); CHKERRQ(ierr); ierr = VecSet(user->
lCs,0.0); CHKERRQ(ierr);
1305 ierr = DMCreateGlobalVector(user->
da,&user->
Psi); CHKERRQ(ierr); ierr = VecSet(user->
Psi,0.0); CHKERRQ(ierr);
1306 ierr = DMCreateLocalVector(user->
da,&user->
lPsi); CHKERRQ(ierr); ierr = VecSet(user->
lPsi,0.0); CHKERRQ(ierr);
1311 ierr = DMCreateGlobalVector(user->
fda, &user->
Bcs.
Ubcs); CHKERRQ(ierr);
1312 ierr = VecSet(user->
Bcs.
Ubcs, 0.0); CHKERRQ(ierr);
1313 ierr = DMCreateGlobalVector(user->
fda, &user->
Bcs.
Uch); CHKERRQ(ierr);
1314 ierr = VecSet(user->
Bcs.
Uch, 0.0); CHKERRQ(ierr);
1320 ierr = VecDuplicate(user->
P, &user->
P_nodal); CHKERRQ(ierr);
1321 ierr = VecSet(user->
P_nodal, 0.0); CHKERRQ(ierr);
1323 ierr = VecDuplicate(user->
Ucat, &user->
Ucat_nodal); CHKERRQ(ierr);
1324 ierr = VecSet(user->
Ucat_nodal, 0.0); CHKERRQ(ierr);
1326 ierr = VecDuplicate(user->
P, &user->
Qcrit); CHKERRQ(ierr);
1327 ierr = VecSet(user->
Qcrit, 0.0); CHKERRQ(ierr);
1332 ierr = VecDuplicate(user->
Psi, &user->
Psi_nodal); CHKERRQ(ierr);
1333 ierr = VecSet(user->
Psi_nodal, 0.0); CHKERRQ(ierr);
1352 PetscFunctionReturn(0);
1363 PetscErrorCode ierr;
1365 Vec globalVec = NULL;
1366 Vec localVec = NULL;
1369 PetscFunctionBeginUser;
1371 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1375 if (strcmp(fieldName,
"Ucat") == 0) {
1376 globalVec = user->
Ucat;
1377 localVec = user->
lUcat;
1379 }
else if (strcmp(fieldName,
"Ucont") == 0) {
1380 globalVec = user->
Ucont;
1383 }
else if (strcmp(fieldName,
"P") == 0) {
1384 globalVec = user->
P;
1385 localVec = user->
lP;
1387 }
else if (strcmp(fieldName,
"Diffusivity") == 0) {
1391 }
else if (strcmp(fieldName,
"DiffusivityGradient") == 0) {
1395 }
else if (strcmp(fieldName,
"Csi") == 0) {
1396 globalVec = user->
Csi;
1397 localVec = user->
lCsi;
1399 }
else if (strcmp(fieldName,
"Eta") == 0) {
1400 globalVec = user->
Eta;
1401 localVec = user->
lEta;
1403 }
else if (strcmp(fieldName,
"Zet") == 0) {
1404 globalVec = user->
Zet;
1405 localVec = user->
lZet;
1407 }
else if (strcmp(fieldName,
"Nvert") == 0) {
1408 globalVec = user->
Nvert;
1412 }
else if (strcmp(fieldName,
"Aj") == 0) {
1413 globalVec = user->
Aj;
1414 localVec = user->
lAj;
1416 }
else if (strcmp(fieldName,
"Cent") == 0) {
1417 globalVec = user->
Cent;
1418 localVec = user->
lCent;
1420 }
else if (strcmp(fieldName,
"GridSpace") == 0) {
1424 }
else if (strcmp(fieldName,
"ICsi") == 0){
1425 globalVec = user->
ICsi;
1426 localVec = user->
lICsi;
1428 }
else if (strcmp(fieldName,
"IEta") == 0){
1429 globalVec = user->
IEta;
1430 localVec = user->
lIEta;
1432 }
else if (strcmp(fieldName,
"IZet") == 0){
1433 globalVec = user->
IZet;
1434 localVec = user->
lIZet;
1436 }
else if (strcmp(fieldName,
"JCsi") == 0){
1437 globalVec = user->
JCsi;
1438 localVec = user->
lJCsi;
1440 }
else if (strcmp(fieldName,
"JEta") == 0){
1441 globalVec = user->
JEta;
1442 localVec = user->
lJEta;
1444 }
else if (strcmp(fieldName,
"JZet") == 0){
1445 globalVec = user->
JZet;
1446 localVec = user->
lJZet;
1448 }
else if (strcmp(fieldName,
"KCsi") == 0){
1449 globalVec = user->
KCsi;
1450 localVec = user->
lKCsi;
1452 }
else if (strcmp(fieldName,
"KEta") == 0){
1453 globalVec = user->
KEta;
1454 localVec = user->
lKEta;
1456 }
else if (strcmp(fieldName,
"KZet") == 0){
1457 globalVec = user->
KZet;
1458 localVec = user->
lKZet;
1460 }
else if (strcmp(fieldName,
"IAj") == 0){
1461 globalVec = user->
IAj;
1462 localVec = user->
lIAj;
1464 }
else if (strcmp(fieldName,
"JAj") == 0){
1465 globalVec = user->
JAj;
1466 localVec = user->
lJAj;
1468 }
else if (strcmp(fieldName,
"KAj") == 0){
1469 globalVec = user->
KAj;
1470 localVec = user->
lKAj;
1472 }
else if (strcmp(fieldName,
"Phi") == 0){
1473 globalVec = user->
Phi;
1474 localVec = user->
lPhi;
1476 }
else if (strcmp(fieldName,
"Psi") == 0){
1477 globalVec = user->
Psi;
1478 localVec = user->
lPsi;
1481 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_UNKNOWN_TYPE,
"Field '%s' not recognized for ghost update.", fieldName);
1486 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Global vector for field '%s' is NULL.", fieldName);
1489 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Local vector for field '%s' is NULL.", fieldName);
1492 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"DM for field '%s' is NULL.", fieldName);
1496 rank, fieldName, (
void*)dm, (
void*)globalVec, (
void*)localVec);
1502 PetscReal norm_global_before;
1503 ierr = VecNorm(globalVec, NORM_INFINITY, &norm_global_before); CHKERRQ(ierr);
1504 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Max norm '%s' (Global) BEFORE Ghost Update: %g\n", fieldName, norm_global_before);
1513 ierr = DMGlobalToLocalBegin(dm, globalVec, INSERT_VALUES, localVec); CHKERRQ(ierr);
1514 ierr = DMGlobalToLocalEnd(dm, globalVec, INSERT_VALUES, localVec); CHKERRQ(ierr);
1521 PetscReal norm_local_after;
1522 ierr = VecNorm(localVec, NORM_INFINITY, &norm_local_after); CHKERRQ(ierr);
1527 if (strcmp(fieldName,
"Ucat") == 0) {
1528 PetscMPIInt rank_test;
1529 MPI_Comm_rank(PETSC_COMM_WORLD, &rank_test);
1532 DMDALocalInfo info_check;
1533 ierr = DMDAGetLocalInfo(dm, &info_check); CHKERRQ(ierr);
1536 Cmpnts ***lUcat_arr_test = NULL;
1537 PetscErrorCode ierr_test = 0;
1539 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Testing '%s' access immediately after ghost update...\n", rank_test, fieldName);
1540 ierr_test = DMDAVecGetArrayDOFRead(dm, localVec, &lUcat_arr_test);
1543 LOG_ALLOW(
LOCAL,
LOG_ERROR,
"Rank %d: ERROR %d getting '%s' array after ghost update!\n", rank_test, ierr_test, fieldName);
1544 }
else if (!lUcat_arr_test) {
1545 LOG_ALLOW(
LOCAL,
LOG_ERROR,
"Rank %d: ERROR NULL pointer getting '%s' array after ghost update!\n", rank_test, fieldName);
1549 PetscInt k_int = info_check.zs + (info_check.zm > 1 ? 1 : 0);
1550 PetscInt j_int = info_check.ys + (info_check.ym > 1 ? 1 : 0);
1551 PetscInt i_int = info_check.xs + (info_check.xm > 1 ? 1 : 0);
1557 if (k_int >= info_check.mz - 1) {
1558 k_int = info_check.mz - 2;
1565 if (j_int >= info_check.my - 1) {
1566 j_int = info_check.my - 2;
1573 if (i_int >= info_check.mx - 1) {
1574 i_int = info_check.mx - 2;
1581 if (k_int >= info_check.zs && k_int < info_check.zs + info_check.zm &&
1582 j_int >= info_check.ys && j_int < info_check.ys + info_check.ym &&
1583 i_int >= info_check.xs && i_int < info_check.xs + info_check.xm)
1585 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Attempting test read OWNED INTERIOR [%d][%d][%d] (Global)\n", rank_test, k_int, j_int, i_int);
1586 Cmpnts test_val_owned_interior = lUcat_arr_test[k_int][j_int][i_int];
1587 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: SUCCESS reading owned interior: x=%g\n", rank_test, test_val_owned_interior.
x);
1589 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Skipping interior test read for non-owned index [%d][%d][%d].\n", rank_test, k_int, j_int, i_int);
1594 PetscInt k_bnd = info_check.zs;
1595 PetscInt j_bnd = info_check.ys;
1596 PetscInt i_bnd = info_check.xs;
1597 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Attempting test read OWNED BOUNDARY [%d][%d][%d] (Global)\n", rank_test, k_bnd, j_bnd, i_bnd);
1598 Cmpnts test_val_owned_boundary = lUcat_arr_test[k_bnd][j_bnd][i_bnd];
1599 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: SUCCESS reading owned boundary: x=%g\n", rank_test, test_val_owned_boundary.
x);
1603 if (info_check.zs > 0) {
1604 PetscInt k_ghost = info_check.zs - 1;
1605 PetscInt j_ghost = info_check.ys;
1606 PetscInt i_ghost = info_check.xs;
1607 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Attempting test read GHOST [%d][%d][%d] (Global)\n", rank_test, k_ghost, j_ghost, i_ghost);
1608 Cmpnts test_val_ghost = lUcat_arr_test[k_ghost][j_ghost][i_ghost];
1615 ierr_test = DMDAVecRestoreArrayDOFRead(dm, localVec, &lUcat_arr_test);
1616 if(ierr_test){
LOG_ALLOW(
LOCAL,
LOG_ERROR,
"Rank %d: ERROR %d restoring '%s' array after test read!\n", rank_test, ierr_test, fieldName); }
1624 PetscFunctionReturn(0);
2249 PetscErrorCode ierr;
2251 Cmpnts ***lcsi_arr, ***leta_arr, ***lzet_arr;
2254 PetscReal ***lnvert_arr;
2255 PetscReal ***laj_arr;
2257 PetscFunctionBeginUser;
2264 ierr = DMDAGetLocalInfo(user->
fda, &info); CHKERRQ(ierr);
2266 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Contra2Cart requires lUcont, lCsi/Eta/Zet, lNvert, and Ucat to be non-NULL.");
2271 ierr = DMDAVecGetArrayRead(user->
fda, user->
lUcont, &lucont_arr); CHKERRQ(ierr);
2272 ierr = DMDAVecGetArrayRead(user->
fda, user->
lCsi, &lcsi_arr); CHKERRQ(ierr);
2273 ierr = DMDAVecGetArrayRead(user->
fda, user->
lEta, &leta_arr); CHKERRQ(ierr);
2274 ierr = DMDAVecGetArrayRead(user->
fda, user->
lZet, &lzet_arr); CHKERRQ(ierr);
2275 ierr = DMDAVecGetArrayRead(user->
da, user->
lNvert, &lnvert_arr); CHKERRQ(ierr);
2276 ierr = DMDAVecGetArrayRead(user->
da, user->
lAj, &laj_arr); CHKERRQ(ierr);
2281 ierr = DMDAVecGetArray(user->
fda, user->
Ucat, &gucat_arr); CHKERRQ(ierr);
2288 PetscInt i_start = (info.xs == 0) ? info.xs + 1 : info.xs;
2289 PetscInt i_end = (info.xs + info.xm == info.mx) ? info.xs + info.xm - 1 : info.xs + info.xm;
2291 PetscInt j_start = (info.ys == 0) ? info.ys + 1 : info.ys;
2292 PetscInt j_end = (info.ys + info.ym == info.my) ? info.ys + info.ym - 1 : info.ys + info.ym;
2294 PetscInt k_start = (info.zs == 0) ? info.zs + 1 : info.zs;
2295 PetscInt k_end = (info.zs + info.zm == info.mz) ? info.zs + info.zm - 1 : info.zs + info.zm;
2299 for (PetscInt k_cell = k_start; k_cell < k_end; ++k_cell) {
2300 for (PetscInt j_cell = j_start; j_cell < j_end; ++j_cell) {
2301 for (PetscInt i_cell = i_start; i_cell < i_end; ++i_cell) {
2308 PetscReal mat[3][3];
2312 mat[0][0] = 0.5 * (lcsi_arr[k_cell][j_cell][i_cell-1].
x + lcsi_arr[k_cell][j_cell][i_cell].
x);
2313 mat[0][1] = 0.5 * (lcsi_arr[k_cell][j_cell][i_cell-1].
y + lcsi_arr[k_cell][j_cell][i_cell].
y);
2314 mat[0][2] = 0.5 * (lcsi_arr[k_cell][j_cell][i_cell-1].
z + lcsi_arr[k_cell][j_cell][i_cell].
z);
2316 mat[1][0] = 0.5 * (leta_arr[k_cell][j_cell-1][i_cell].
x + leta_arr[k_cell][j_cell][i_cell].
x);
2317 mat[1][1] = 0.5 * (leta_arr[k_cell][j_cell-1][i_cell].
y + leta_arr[k_cell][j_cell][i_cell].
y);
2318 mat[1][2] = 0.5 * (leta_arr[k_cell][j_cell-1][i_cell].
z + leta_arr[k_cell][j_cell][i_cell].
z);
2320 mat[2][0] = 0.5 * (lzet_arr[k_cell-1][j_cell][i_cell].
x + lzet_arr[k_cell][j_cell][i_cell].
x);
2321 mat[2][1] = 0.5 * (lzet_arr[k_cell-1][j_cell][i_cell].
y + lzet_arr[k_cell][j_cell][i_cell].
y);
2322 mat[2][2] = 0.5 * (lzet_arr[k_cell-1][j_cell][i_cell].
z + lzet_arr[k_cell][j_cell][i_cell].
z);
2327 q[0] = 0.5 * (lucont_arr[k_cell][j_cell][i_cell-1].
x + lucont_arr[k_cell][j_cell][i_cell].
x);
2328 q[1] = 0.5 * (lucont_arr[k_cell][j_cell-1][i_cell].
y + lucont_arr[k_cell][j_cell][i_cell].
y);
2329 q[2] = 0.5 * (lucont_arr[k_cell-1][j_cell][i_cell].
z + lucont_arr[k_cell][j_cell][i_cell].
z);
2332 PetscReal det = mat[0][0] * (mat[1][1] * mat[2][2] - mat[1][2] * mat[2][1]) -
2333 mat[0][1] * (mat[1][0] * mat[2][2] - mat[1][2] * mat[2][0]) +
2334 mat[0][2] * (mat[1][0] * mat[2][1] - mat[1][1] * mat[2][0]);
2336 if (PetscAbsReal(det) < 1.0e-18) {
2337 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_FLOP_COUNT,
"Transformation matrix determinant is near zero at cell (%d,%d,%d) \n", i_cell, j_cell, k_cell);
2340 PetscReal det_inv = 1.0 / det;
2342 PetscReal det0 = q[0] * (mat[1][1] * mat[2][2] - mat[1][2] * mat[2][1]) -
2343 q[1] * (mat[0][1] * mat[2][2] - mat[0][2] * mat[2][1]) +
2344 q[2] * (mat[0][1] * mat[1][2] - mat[0][2] * mat[1][1]);
2346 PetscReal det1 = -q[0] * (mat[1][0] * mat[2][2] - mat[1][2] * mat[2][0]) +
2347 q[1] * (mat[0][0] * mat[2][2] - mat[0][2] * mat[2][0]) -
2348 q[2] * (mat[0][0] * mat[1][2] - mat[0][2] * mat[1][0]);
2350 PetscReal det2 = q[0] * (mat[1][0] * mat[2][1] - mat[1][1] * mat[2][0]) -
2351 q[1] * (mat[0][0] * mat[2][1] - mat[0][1] * mat[2][0]) +
2352 q[2] * (mat[0][0] * mat[1][1] - mat[0][1] * mat[1][0]);
2356 gucat_arr[k_cell][j_cell][i_cell].
x = det0 * det_inv;
2357 gucat_arr[k_cell][j_cell][i_cell].
y = det1 * det_inv;
2358 gucat_arr[k_cell][j_cell][i_cell].
z = det2 * det_inv;
2364 ierr = DMDAVecRestoreArrayRead(user->
fda, user->
lUcont, &lucont_arr); CHKERRQ(ierr);
2365 ierr = DMDAVecRestoreArrayRead(user->
fda, user->
lCsi, &lcsi_arr); CHKERRQ(ierr);
2366 ierr = DMDAVecRestoreArrayRead(user->
fda, user->
lEta, &leta_arr); CHKERRQ(ierr);
2367 ierr = DMDAVecRestoreArrayRead(user->
fda, user->
lZet, &lzet_arr); CHKERRQ(ierr);
2368 ierr = DMDAVecRestoreArrayRead(user->
da, user->
lNvert, &lnvert_arr); CHKERRQ(ierr);
2369 ierr = DMDAVecRestoreArrayRead(user->
da, user->
lAj, &laj_arr); CHKERRQ(ierr);
2370 ierr = DMDAVecRestoreArray(user->
fda, user->
Ucat, &gucat_arr); CHKERRQ(ierr);
2374 PetscFunctionReturn(0);
2956 PetscErrorCode ierr;
2957 PetscFunctionBeginUser;
2960 if (user->
Ucont) { ierr = VecDestroy(&user->
Ucont); CHKERRQ(ierr); }
2961 if (user->
lUcont) { ierr = VecDestroy(&user->
lUcont); CHKERRQ(ierr); }
2962 if (user->
Ucat) { ierr = VecDestroy(&user->
Ucat); CHKERRQ(ierr); }
2963 if (user->
lUcat) { ierr = VecDestroy(&user->
lUcat); CHKERRQ(ierr); }
2964 if (user->
P) { ierr = VecDestroy(&user->
P); CHKERRQ(ierr); }
2965 if (user->
lP) { ierr = VecDestroy(&user->
lP); CHKERRQ(ierr); }
2966 if (user->
Nvert) { ierr = VecDestroy(&user->
Nvert); CHKERRQ(ierr); }
2967 if (user->
lNvert) { ierr = VecDestroy(&user->
lNvert); CHKERRQ(ierr); }
2976 if (user->
Phi) { ierr = VecDestroy(&user->
Phi); CHKERRQ(ierr); }
2977 if (user->
lPhi) { ierr = VecDestroy(&user->
lPhi); CHKERRQ(ierr); }
2980 if (user->
Ucont_o) { ierr = VecDestroy(&user->
Ucont_o); CHKERRQ(ierr); }
2982 if (user->
Ucat_o) { ierr = VecDestroy(&user->
Ucat_o); CHKERRQ(ierr); }
2983 if (user->
P_o) { ierr = VecDestroy(&user->
P_o); CHKERRQ(ierr); }
2984 if (user->
Nvert_o) { ierr = VecDestroy(&user->
Nvert_o); CHKERRQ(ierr); }
2990 if (user->
Csi) { ierr = VecDestroy(&user->
Csi); CHKERRQ(ierr); }
2991 if (user->
Eta) { ierr = VecDestroy(&user->
Eta); CHKERRQ(ierr); }
2992 if (user->
Zet) { ierr = VecDestroy(&user->
Zet); CHKERRQ(ierr); }
2993 if (user->
Aj) { ierr = VecDestroy(&user->
Aj); CHKERRQ(ierr); }
2994 if (user->
lCsi) { ierr = VecDestroy(&user->
lCsi); CHKERRQ(ierr); }
2995 if (user->
lEta) { ierr = VecDestroy(&user->
lEta); CHKERRQ(ierr); }
2996 if (user->
lZet) { ierr = VecDestroy(&user->
lZet); CHKERRQ(ierr); }
2997 if (user->
lAj) { ierr = VecDestroy(&user->
lAj); CHKERRQ(ierr); }
3000 if (user->
ICsi) { ierr = VecDestroy(&user->
ICsi); CHKERRQ(ierr); }
3001 if (user->
IEta) { ierr = VecDestroy(&user->
IEta); CHKERRQ(ierr); }
3002 if (user->
IZet) { ierr = VecDestroy(&user->
IZet); CHKERRQ(ierr); }
3003 if (user->
JCsi) { ierr = VecDestroy(&user->
JCsi); CHKERRQ(ierr); }
3004 if (user->
JEta) { ierr = VecDestroy(&user->
JEta); CHKERRQ(ierr); }
3005 if (user->
JZet) { ierr = VecDestroy(&user->
JZet); CHKERRQ(ierr); }
3006 if (user->
KCsi) { ierr = VecDestroy(&user->
KCsi); CHKERRQ(ierr); }
3007 if (user->
KEta) { ierr = VecDestroy(&user->
KEta); CHKERRQ(ierr); }
3008 if (user->
KZet) { ierr = VecDestroy(&user->
KZet); CHKERRQ(ierr); }
3009 if (user->
IAj) { ierr = VecDestroy(&user->
IAj); CHKERRQ(ierr); }
3010 if (user->
JAj) { ierr = VecDestroy(&user->
JAj); CHKERRQ(ierr); }
3011 if (user->
KAj) { ierr = VecDestroy(&user->
KAj); CHKERRQ(ierr); }
3012 if (user->
lICsi) { ierr = VecDestroy(&user->
lICsi); CHKERRQ(ierr); }
3013 if (user->
lIEta) { ierr = VecDestroy(&user->
lIEta); CHKERRQ(ierr); }
3014 if (user->
lIZet) { ierr = VecDestroy(&user->
lIZet); CHKERRQ(ierr); }
3015 if (user->
lJCsi) { ierr = VecDestroy(&user->
lJCsi); CHKERRQ(ierr); }
3016 if (user->
lJEta) { ierr = VecDestroy(&user->
lJEta); CHKERRQ(ierr); }
3017 if (user->
lJZet) { ierr = VecDestroy(&user->
lJZet); CHKERRQ(ierr); }
3018 if (user->
lKCsi) { ierr = VecDestroy(&user->
lKCsi); CHKERRQ(ierr); }
3019 if (user->
lKEta) { ierr = VecDestroy(&user->
lKEta); CHKERRQ(ierr); }
3020 if (user->
lKZet) { ierr = VecDestroy(&user->
lKZet); CHKERRQ(ierr); }
3021 if (user->
lIAj) { ierr = VecDestroy(&user->
lIAj); CHKERRQ(ierr); }
3022 if (user->
lJAj) { ierr = VecDestroy(&user->
lJAj); CHKERRQ(ierr); }
3023 if (user->
lKAj) { ierr = VecDestroy(&user->
lKAj); CHKERRQ(ierr); }
3026 if (user->
Cent) { ierr = VecDestroy(&user->
Cent); CHKERRQ(ierr); }
3027 if (user->
lCent) { ierr = VecDestroy(&user->
lCent); CHKERRQ(ierr); }
3030 if (user->
Centx) { ierr = VecDestroy(&user->
Centx); CHKERRQ(ierr); }
3031 if (user->
Centy) { ierr = VecDestroy(&user->
Centy); CHKERRQ(ierr); }
3032 if (user->
Centz) { ierr = VecDestroy(&user->
Centz); CHKERRQ(ierr); }
3035 if (user->
Nu_t) { ierr = VecDestroy(&user->
Nu_t); CHKERRQ(ierr); }
3036 if (user->
lNu_t) { ierr = VecDestroy(&user->
lNu_t); CHKERRQ(ierr); }
3037 if (user->
CS) { ierr = VecDestroy(&user->
CS); CHKERRQ(ierr); }
3038 if (user->
lCs) { ierr = VecDestroy(&user->
lCs); CHKERRQ(ierr); }
3040 if (user->
K_Omega) { ierr = VecDestroy(&user->
K_Omega); CHKERRQ(ierr); }
3048 if (user->
Psi) { ierr = VecDestroy(&user->
Psi); CHKERRQ(ierr); }
3049 if (user->
lPsi) { ierr = VecDestroy(&user->
lPsi); CHKERRQ(ierr); }
3052 if (user->
Bcs.
Ubcs) { ierr = VecDestroy(&user->
Bcs.
Ubcs); CHKERRQ(ierr); }
3053 if (user->
Bcs.
Uch) { ierr = VecDestroy(&user->
Bcs.
Uch); CHKERRQ(ierr); }
3056 if (user->
P_nodal) { ierr = VecDestroy(&user->
P_nodal); CHKERRQ(ierr); }
3058 if (user->
Qcrit) { ierr = VecDestroy(&user->
Qcrit); CHKERRQ(ierr); }
3069 if (user->
P_sum) { ierr = VecDestroy(&user->
P_sum); CHKERRQ(ierr); }
3072 if (user->
Rhs) { ierr = VecDestroy(&user->
Rhs); CHKERRQ(ierr); }
3073 if (user->
dUcont) { ierr = VecDestroy(&user->
dUcont); CHKERRQ(ierr); }
3074 if (user->
pUcont) { ierr = VecDestroy(&user->
pUcont); CHKERRQ(ierr); }
3077 if (user->
B) { ierr = VecDestroy(&user->
B); CHKERRQ(ierr); }
3078 if (user->
R) { ierr = VecDestroy(&user->
R); CHKERRQ(ierr); }
3081 PetscFunctionReturn(0);