Solves the Momentum Equations using Dual-Time Stepping with a Fixed-Point RK4 Smoother.
Local to this translation unit.
167{
168 (void)ibm;
169 (void)fsi;
170
173
174
175 const PetscInt ti = simCtx->
step;
176 const PetscReal dt = simCtx->
dt;
178 const PetscReal alfa[] = {0.25, 1.0/3.0, 0.5, 1.0};
179 Vec *pRhs;
180
181
182 PetscBool force_restart = PETSC_FALSE;
183
184
186 const PetscReal tol_abs_delta = simCtx->
mom_atol;
187 const PetscReal tol_rtol_delta = simCtx->
mom_rtol;
188
189
190 PetscErrorCode ierr;
191 PetscMPIInt rank;
192 PetscInt istage, pseudo_iter;
193 PetscReal ts, te, cput;
194
195
196 PetscReal global_norm_delta = 10.0;
197 PetscReal global_rel_delta = 1.0;
198 PetscReal global_rel_resid = 1.0;
199
200
201 PetscReal *delta_sol_norm_init, *delta_sol_norm_prev, *delta_sol_norm_curr, *delta_sol_rel_curr;
202 PetscReal *resid_norm_init, *resid_norm_prev, *resid_norm_curr, *resid_rel_curr;
203 PetscReal *pseudo_dt_scaling;
204
205 PetscFunctionBeginUser;
207 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
209
210
211 ierr = PetscMalloc2(block_number, &delta_sol_norm_init, block_number, &delta_sol_norm_prev); CHKERRQ(ierr);
212 ierr = PetscMalloc2(block_number, &delta_sol_norm_curr, block_number, &delta_sol_rel_curr); CHKERRQ(ierr);
213 ierr = PetscMalloc2(block_number, &resid_norm_init, block_number, &resid_norm_prev); CHKERRQ(ierr);
214 ierr = PetscMalloc2(block_number, &resid_norm_curr, block_number, &pseudo_dt_scaling); CHKERRQ(ierr);
215 ierr = PetscMalloc1(block_number, &resid_rel_curr); CHKERRQ(ierr);
216 ierr = PetscMalloc1(block_number, &pRhs); CHKERRQ(ierr);
217
218 ierr = PetscTime(&ts); CHKERRQ(ierr);
219
220
221
222 if (block_number > 1) {
223
224 }
225
226 for (PetscInt bi = 0; bi < block_number; bi++) {
228
229
230
231
232 ierr = VecDuplicate(user[bi].Ucont, &user[bi].Rhs); CHKERRQ(ierr);
233 ierr = VecDuplicate(user[bi].Rhs, &pRhs[bi]); CHKERRQ(ierr);
234 ierr = VecDuplicate(user[bi].Ucont, &user[bi].dUcont); CHKERRQ(ierr);
235 ierr = VecDuplicate(user[bi].Ucont, &user[bi].pUcont); CHKERRQ(ierr);
236
237
238 ierr = VecCopy(user[bi].Ucont, user[bi].pUcont); CHKERRQ(ierr);
239
240
242
243
244 ierr = VecCopy(user[bi].Rhs, pRhs[bi]); CHKERRQ(ierr);
245
246
247 ierr = VecNorm(user[bi].Rhs, NORM_INFINITY, &resid_norm_init[bi]); CHKERRQ(ierr);
248
249
250 resid_norm_prev[bi] = resid_norm_init[bi];
251 delta_sol_norm_prev[bi] = 1000.0;
252 pseudo_dt_scaling[bi] = cfl;
253
255 }
256
257
258 pseudo_iter = 0;
259 while ((force_restart || (global_norm_delta > tol_abs_delta || global_rel_delta > tol_rtol_delta) || pseudo_iter < 1)
260 && pseudo_iter < max_pseudo_steps)
261 {
262 pseudo_iter++;
263 force_restart = PETSC_FALSE;
264
265 for (PetscInt bi = 0; bi < block_number; bi++) {
266
267
268 for (istage = 0; istage < 4; istage++) {
269
271
272
273 ierr = VecWAXPY(user[bi].Ucont,
274 pseudo_dt_scaling[bi] * alfa[istage] * dt,
275 user[bi].Rhs,
276 user[bi].pUcont); CHKERRQ(ierr);
277
278
280
281
282
283
285
286
288
289 }
290
291
292
293
294
295
296 ierr = VecWAXPY(user[bi].dUcont, -1.0, user[bi].pUcont, user[bi].Ucont); CHKERRQ(ierr);
297
298
299 ierr = VecNorm(user[bi].dUcont, NORM_INFINITY, &delta_sol_norm_curr[bi]); CHKERRQ(ierr);
300 ierr = VecNorm(user[bi].Rhs, NORM_INFINITY, &resid_norm_curr[bi]); CHKERRQ(ierr);
301
302
303 if (pseudo_iter == 1) {
304 delta_sol_norm_init[bi] = delta_sol_norm_curr[bi];
305 delta_sol_rel_curr[bi] = 1.0;
306 resid_rel_curr[bi] = 1.0;
307
308 } else {
309 if (delta_sol_norm_init[bi] > 1.0e-10)
310 delta_sol_rel_curr[bi] = delta_sol_norm_curr[bi] / delta_sol_norm_init[bi];
311 else
312 delta_sol_rel_curr[bi] = 0.0;
313
314 if(resid_norm_init[bi] > 1.0e-10)
315 resid_rel_curr[bi] = resid_norm_curr[bi] / resid_norm_init[bi];
316 else
317 resid_rel_curr[bi] = 0.0;
318 }
319
320
321 if (!rank) {
322 FILE *f;
323 char filen[PETSC_MAX_PATH_LEN + 128];
324 ierr = PetscSNPrintf(filen,
sizeof(filen),
"%s/Momentum_Solver_Convergence_History_Block_%1d.log", simCtx->
log_dir, bi); CHKERRQ(ierr);
326 else f = fopen(filen, "a");
327
328 PetscFPrintf(PETSC_COMM_WORLD, f, "Step: %d | PseudoIter(k): %d| | Pseudo-cfl: %.4f |dUk|: %le | |dUk|/|dUprev|: %le | |Rk|: %le | |Rk|/|Rprev|: %le \n",
329 (int)ti, (int)pseudo_iter, pseudo_dt_scaling[bi], delta_sol_norm_curr[bi], delta_sol_rel_curr[bi], resid_norm_curr[bi],resid_rel_curr[bi]);
330 fclose(f);
331 }
332 }
333
334
335 global_norm_delta = -1.0e20;
336 global_rel_delta = -1.0e20;
337 global_rel_resid = -1.0e20;
338
339 for (PetscInt bi = 0; bi < block_number; bi++) {
340 global_norm_delta = PetscMax(delta_sol_norm_curr[bi], global_norm_delta);
341 global_rel_delta = PetscMax(delta_sol_rel_curr[bi], global_rel_delta);
342 global_rel_resid = PetscMax(resid_rel_curr[bi], global_rel_resid);
343 }
344 ierr = PetscTime(&te); CHKERRQ(ierr);
345 cput = te - ts;
347 pseudo_iter, global_norm_delta, global_rel_delta, global_rel_resid, cput);
348
349
350 for (PetscInt bi = 0; bi < block_number; bi++) {
351
352 PetscReal resid_ratio = resid_norm_curr[bi] / resid_norm_prev[bi];
353
354 PetscBool is_sol_nan, is_resid_nan, is_diverging;
355
356 is_sol_nan = (PetscBool)PetscIsNanReal(delta_sol_norm_curr[bi]);
357 is_resid_nan = (PetscBool)PetscIsNanReal(resid_norm_curr[bi]);
358
359
360 is_diverging = (PetscBool)(pseudo_iter > 1 &&
362 delta_sol_norm_curr[bi] > delta_sol_norm_prev[bi]
363 );
364
365 if(is_diverging || is_sol_nan || is_resid_nan) {
366
367 if(is_sol_nan){
369 }
370 if(is_resid_nan){
372 }
373 if(is_diverging){
375 }
376
378 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_CONV_FAILED,
379 "Block %d: Solver diverged (NaN or explosion) and min time-step reached. Simulation aborted.", bi);
380 }
381
382
383 force_restart = PETSC_TRUE;
384
385
387
388
389 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
" Block %d: Backtracking triggered. Reducing scaling to %f.\n", bi, pseudo_dt_scaling[bi]);
390
391
392 ierr = VecCopy(user[bi].pUcont, user[bi].Ucont); CHKERRQ(ierr);
393
394 ierr = VecCopy(pRhs[bi], user[bi].Rhs); CHKERRQ(ierr);
395
396
398
399
400
401
402 delta_sol_norm_curr[bi] = 1000.0;
403 resid_norm_curr[bi] = 1000.0;
404
405 }
406 else {
407
408 ierr = VecCopy(user[bi].Ucont, user[bi].pUcont); CHKERRQ(ierr);
409 ierr = VecCopy(user[bi].Rhs, pRhs[bi]); CHKERRQ(ierr);
410 resid_norm_prev[bi] = resid_norm_curr[bi];
411 delta_sol_norm_prev[bi] = delta_sol_norm_curr[bi];
412
413
414
415 PetscBool is_converging_fast;
416 is_converging_fast = (PetscBool)(resid_ratio < 0.9);
417 PetscBool is_stagnating;
418 is_stagnating = (PetscBool)(resid_ratio >= 0.98 && resid_ratio < 1.0);
419 PetscBool is_noisy;
420 is_noisy = (PetscBool)(resid_ratio >= 1.0 && resid_ratio < simCtx->mom_dt_rk4_residual_norm_noise_allowance_factor);
421
422 PetscReal current_cfl = pseudo_dt_scaling[bi];
425
426 if(is_converging_fast){
427
429 } else if(is_stagnating && current_cfl < cfl_floor){
430
432 } else if(is_stagnating && current_cfl >= cfl_floor && current_cfl < cfl_ceiling && !is_noisy){
433
435 } else if(is_stagnating && current_cfl >= cfl_ceiling && !is_noisy){
436
438 } else if(is_noisy){
439
440 if(current_cfl > cfl_ceiling){
441
443 } else if(current_cfl > cfl_floor && current_cfl <= cfl_ceiling){
444
446 } else if(current_cfl <= cfl_floor){
447
448
450 }
451 }
452
453 pseudo_dt_scaling[bi] = PetscMin(pseudo_dt_scaling[bi], simCtx->
max_pseudo_cfl);
454 pseudo_dt_scaling[bi] = PetscMax(pseudo_dt_scaling[bi], simCtx->
min_pseudo_cfl);
455 }
456 }
457
458
459 if (force_restart) pseudo_iter--;
460
461 if (block_number > 1) {
462
463 }
464 }
465
466
467
468
469 PetscReal local_min_cfl = 1.0e20;
470 PetscReal global_min_cfl;
471
473
474
475 for (PetscInt bi = 0; bi < block_number; bi++) {
476 if (pseudo_dt_scaling[bi] < local_min_cfl) {
477 local_min_cfl = pseudo_dt_scaling[bi];
478 }
479 }
480
481
482
483 ierr = MPI_Allreduce(&local_min_cfl, &global_min_cfl, 1, MPIU_REAL, MPI_MIN, PETSC_COMM_WORLD); CHKERRQ(ierr);
484
485
486
487
488
489 PetscReal cfl_carry_over_relaxation_factor = 1.0;
490
491 PetscReal next_start_cfl;
492
493 if(global_min_cfl < cfl_floor){
494
495 next_start_cfl = cfl_floor * 1.5;
496 } else{
497 next_start_cfl = global_min_cfl * cfl_carry_over_relaxation_factor;
498 }
499
500 next_start_cfl = PetscMin(next_start_cfl, simCtx->
max_pseudo_cfl);
501 next_start_cfl = PetscMax(next_start_cfl, simCtx->
min_pseudo_cfl);
502
503
504 if (!rank) {
505 LOG_ALLOW(
GLOBAL,
LOG_INFO,
" CFL Adaptation: Step %d finished at CFL %.3f. Next step will start at CFL %.3f.\n",
506 simCtx->
step, global_min_cfl, next_start_cfl);
507 }
508
509
511
512
513 for (PetscInt bi = 0; bi < block_number; bi++) {
514 ierr = VecDestroy(&user[bi].Rhs); CHKERRQ(ierr);
515 ierr = VecDestroy(&user[bi].dUcont); CHKERRQ(ierr);
516 ierr = VecDestroy(&user[bi].pUcont); CHKERRQ(ierr);
517 ierr = VecDestroy(&pRhs[bi]); CHKERRQ(ierr);
518 }
519 ierr = PetscFree(pRhs); CHKERRQ(ierr);
520
521 ierr = PetscFree2(delta_sol_norm_init, delta_sol_norm_prev);CHKERRQ(ierr);
522 ierr = PetscFree2(delta_sol_norm_curr, delta_sol_rel_curr);CHKERRQ(ierr);
523 ierr = PetscFree2(resid_norm_init, resid_norm_prev);CHKERRQ(ierr);
524 ierr = PetscFree2(resid_norm_curr, pseudo_dt_scaling); CHKERRQ(ierr);
525 ierr = PetscFree(resid_rel_curr); CHKERRQ(ierr);
526
529 PetscFunctionReturn(0);
530}
@ LOG_TRACE
Very fine-grained tracing information for in-depth debugging.
@ LOG_WARNING
Non-critical issues that warrant attention.
static PetscErrorCode ComputeTotalResidual(UserCtx *user)
Internal helper implementation: ComputeTotalResidual().
PetscErrorCode UpdateLocalGhosts(UserCtx *user, const char *fieldName)
Updates the local vector (including ghost points) from its corresponding global vector.
PetscReal pseudo_cfl_reduction_factor
char log_dir[PETSC_MAX_PATH_LEN]
PetscReal pseudo_cfl_growth_factor
PetscReal mom_dt_rk4_residual_norm_noise_allowance_factor
PetscInt mom_max_pseudo_steps