136 DM swarm = user->
swarm;
138 PetscReal *pos = NULL;
139 PetscReal *vel = NULL;
140 PetscReal *diffusivity = NULL;
141 Cmpnts *diffusivitygradient = NULL;
142 PetscReal *psi = NULL;
143 PetscReal *weights = NULL;
144 PetscInt *cell = NULL;
145 PetscInt *status = NULL;
146 PetscInt64 *pid = NULL;
149 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
151 PetscFunctionBeginUser;
156 ierr = DMSwarmGetLocalSize(swarm, &nLocal); CHKERRQ(ierr);
159 PetscFunctionReturn(0);
162 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void**)&pos); CHKERRQ(ierr);
163 ierr = DMSwarmGetField(swarm,
"velocity", NULL, NULL, (
void**)&vel); CHKERRQ(ierr);
164 ierr = DMSwarmGetField(swarm,
"Diffusivity", NULL, NULL, (
void**)&diffusivity); CHKERRQ(ierr);
165 ierr = DMSwarmGetField(swarm,
"DiffusivityGradient", NULL, NULL, (
void**)&diffusivitygradient); CHKERRQ(ierr);
166 ierr = DMSwarmGetField(swarm,
"Psi", NULL, NULL, (
void**)&psi); CHKERRQ(ierr);
167 ierr = DMSwarmGetField(swarm,
"weight", NULL, NULL, (
void**)&weights); CHKERRQ(ierr);
168 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell); CHKERRQ(ierr);
169 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status); CHKERRQ(ierr);
170 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid); CHKERRQ(ierr);
175 for (p = 0; p < nLocal; p++) {
180 ierr =
UnpackSwarmFields(p, pid, weights, pos, cell, vel, status, diffusivity, diffusivitygradient, psi, &particle); CHKERRQ(ierr);
186 ierr =
UpdateSwarmFields(p, &particle, pos, vel, weights, cell, status, diffusivity, diffusivitygradient, psi); CHKERRQ(ierr);
190 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void**)&pos); CHKERRQ(ierr);
191 ierr = DMSwarmRestoreField(swarm,
"velocity", NULL, NULL, (
void**)&vel); CHKERRQ(ierr);
192 ierr = DMSwarmRestoreField(swarm,
"Diffusivity", NULL, NULL, (
void**)&diffusivity); CHKERRQ(ierr);
193 ierr = DMSwarmRestoreField(swarm,
"DiffusivityGradient", NULL, NULL, (
void**)&diffusivitygradient); CHKERRQ(ierr);
194 ierr = DMSwarmRestoreField(swarm,
"Psi", NULL, NULL, (
void**)&psi); CHKERRQ(ierr);
195 ierr = DMSwarmRestoreField(swarm,
"weight", NULL, NULL, (
void**)&weights); CHKERRQ(ierr);
196 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell); CHKERRQ(ierr);
197 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status); CHKERRQ(ierr);
198 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid); CHKERRQ(ierr);
205 PetscFunctionReturn(0);
228 PetscInt *removedCountLocal,
229 PetscInt *removedCountGlobal,
233 DM swarm = user->
swarm;
234 PetscInt nLocalInitial;
235 PetscReal *pos_p = NULL;
236 PetscInt64 *pid_p = NULL;
237 PetscInt local_removed_count = 0;
238 PetscMPIInt global_removed_count_mpi = 0;
239 PetscMPIInt rank, size;
241 PetscFunctionBeginUser;
242 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
243 ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size); CHKERRQ(ierr);
247 *removedCountLocal = 0;
248 if (removedCountGlobal) *removedCountGlobal = 0;
250 ierr = DMSwarmGetLocalSize(swarm, &nLocalInitial); CHKERRQ(ierr);
254 if (nLocalInitial > 0) {
256 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
257 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
260 for (PetscInt p = nLocalInitial - 1; p >= 0; p--) {
261 PetscBool isInsideAnyBox = PETSC_FALSE;
262 Cmpnts current_pos = {pos_p[3*p + 0], pos_p[3*p + 1], pos_p[3*p + 2]};
265 for (PetscMPIInt proc = 0; proc < size; proc++) {
267 isInsideAnyBox = PETSC_TRUE;
272 if (!isInsideAnyBox) {
273 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Removing out-of-bounds particle [PID %lld] at local index %d. Pos: (%g, %g, %g)\n",
274 rank, (
long long)pid_p[p], p, current_pos.
x, current_pos.
y, current_pos.
z);
281 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
282 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
285 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
286 local_removed_count++;
289 PetscInt nLocalCurrent;
290 ierr = DMSwarmGetLocalSize(swarm, &nLocalCurrent); CHKERRQ(ierr);
292 if (nLocalCurrent > 0 && p > 0) {
293 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
294 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
309 if (pos_p) { ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr); }
310 if (pid_p) { ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr); }
313 PetscInt nLocalFinal;
314 ierr = DMSwarmGetLocalSize(swarm, &nLocalFinal); CHKERRQ(ierr);
315 LOG_ALLOW(
LOCAL,
LOG_INFO,
"[Rank %d] Finished removing %d out-of-bounds particles. Final local size: %d.\n", rank, local_removed_count, nLocalFinal);
318 *removedCountLocal = local_removed_count;
319 if (removedCountGlobal) {
320 ierr = MPI_Allreduce(&local_removed_count, &global_removed_count_mpi, 1, MPI_INT, MPI_SUM, PetscObjectComm((PetscObject)swarm)); CHKERRQ(ierr);
321 *removedCountGlobal = global_removed_count_mpi;
326 PetscFunctionReturn(0);
336 PetscInt *removedCountLocal,
337 PetscInt *removedCountGlobal)
340 DM swarm = user->
swarm;
341 PetscInt nLocalInitial;
342 PetscInt *status_p = NULL;
343 PetscInt64 *pid_p = NULL;
344 PetscReal *pos_p = NULL;
345 PetscInt local_removed_count = 0;
346 PetscMPIInt global_removed_count_mpi = 0;
349 PetscFunctionBeginUser;
351 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
355 *removedCountLocal = 0;
356 if (removedCountGlobal) *removedCountGlobal = 0;
358 ierr = DMSwarmGetLocalSize(swarm, &nLocalInitial); CHKERRQ(ierr);
362 if (nLocalInitial > 0) {
364 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr);
365 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
366 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
369 for (PetscInt p = nLocalInitial - 1; p >= 0; p--) {
370 if (status_p[p] ==
LOST) {
371 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Removing LOST particle [PID %lld] at local index %d. Position: (%.4f, %.4f, %.4f).\n",
372 rank, (
long long)pid_p[p], p, pos_p[3*p], pos_p[3*p+1], pos_p[3*p+2]);
379 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr);
380 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
381 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
384 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
385 local_removed_count++;
388 PetscInt nLocalCurrent;
389 ierr = DMSwarmGetLocalSize(swarm, &nLocalCurrent); CHKERRQ(ierr);
391 if (nLocalCurrent > 0 && p > 0) {
392 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr);
393 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
394 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
410 if (status_p) { ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr); }
411 if (pid_p) { ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr); }
412 if (pos_p) { ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr); }
415 PetscInt nLocalFinal;
416 ierr = DMSwarmGetLocalSize(swarm, &nLocalFinal); CHKERRQ(ierr);
417 LOG_ALLOW(
LOCAL,
LOG_INFO,
"Rank %d: Finished removing %d LOST particles. Final local size: %d.\n", rank, local_removed_count, nLocalFinal);
420 *removedCountLocal = local_removed_count;
421 if (removedCountGlobal) {
422 ierr = MPI_Allreduce(&local_removed_count, &global_removed_count_mpi, 1, MPI_INT, MPI_SUM, PetscObjectComm((PetscObject)swarm)); CHKERRQ(ierr);
423 *removedCountGlobal = global_removed_count_mpi;
429 PetscFunctionReturn(0);
507 DM swarm = user->
swarm;
511 PetscInt *global_cell_id_arr;
512 PetscScalar ***count_arr_3d;
516 PetscInt particles_counted_locally = 0;
518 PetscFunctionBeginUser;
520 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
523 if (!da) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->da is NULL.");
524 if (!swarm) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->swarm is NULL.");
525 if (!countVec) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->ParticleCount is NULL.");
528 ierr = DMDAGetInfo(da, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &count_dof, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
529 if (count_dof != 1) {
530 PetscSNPrintf(msg,
sizeof(msg),
"countDM must have DOF=1, got %" PetscInt_FMT
".", count_dof);
531 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG,
"%s", msg);
535 ierr = VecSet(localcountVec, 0.0); CHKERRQ(ierr);
539 ierr = DMSwarmGetLocalSize(swarm, &nlocal); CHKERRQ(ierr);
540 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void **)&global_cell_id_arr); CHKERRQ(ierr);
541 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid",NULL,NULL,(
void **)&PID_arr);CHKERRQ(ierr);
545 ierr = DMDAVecGetArray(da, localcountVec, &count_arr_3d); CHKERRQ(ierr);
548 PetscInt gxs, gys, gzs, gxm, gym, gzm;
549 ierr = DMDAGetGhostCorners(da, &gxs, &gys, &gzs, &gxm, &gym, &gzm); CHKERRQ(ierr);
552 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"CalculateParticleCountPerCell (Rank %d): Processing %" PetscInt_FMT
" local particles using GLOBAL CellIDs.\n",rank,nlocal);
553 for (p = 0; p < nlocal; p++) {
555 PetscInt i_geom = global_cell_id_arr[p * 3 + 0];
556 PetscInt j_geom = global_cell_id_arr[p * 3 + 1];
557 PetscInt k_geom = global_cell_id_arr[p * 3 + 2];
560 PetscInt i = (PetscInt)i_geom + 1;
561 PetscInt j = (PetscInt)j_geom + 1;
562 PetscInt k = (PetscInt)k_geom + 1;
570 "[Rank %d] Read CellID for p=%" PetscInt_FMT
", PID = %" PetscInt64_FMT
": (%" PetscInt_FMT
", %" PetscInt_FMT
", %" PetscInt_FMT
")\n",
571 rank, p, PID_arr[p], i, j, k);
574 if (i >= gxs && i < gxs + gxm &&
575 j >= gys && j < gys + gym &&
576 k >= gzs && k < gzs + gzm )
581 count_arr_3d[k][j][i] += 1.0;
582 particles_counted_locally++;
587 "(Rank %d): Skipping particle %" PetscInt64_FMT
" with global CellID (%" PetscInt_FMT
", %" PetscInt_FMT
", %" PetscInt_FMT
") - likely outside local+ghost range.\n",
588 rank, PID_arr[p], i, j, k);
591 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"(Rank %d): Local counting finished. Processed %" PetscInt_FMT
" particles locally.\n", rank, particles_counted_locally);
594 ierr = DMDAVecRestoreArray(da, localcountVec, &count_arr_3d); CHKERRQ(ierr);
595 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void **)&global_cell_id_arr); CHKERRQ(ierr);
596 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid",NULL,NULL,(
void **)&PID_arr);CHKERRQ(ierr);
600 ierr = VecZeroEntries(countVec); CHKERRQ(ierr);
601 ierr = DMLocalToGlobalBegin(da, localcountVec, ADD_VALUES, countVec); CHKERRQ(ierr);
602 ierr = DMLocalToGlobalEnd(da, localcountVec, ADD_VALUES, countVec); CHKERRQ(ierr);
613 PetscReal total_counted_particles = 0.0, max_count_in_cell = 0.0;
614 ierr = VecSum(countVec, &total_counted_particles); CHKERRQ(ierr);
615 PetscInt max_idx_global = -1;
616 ierr = VecMax(countVec, &max_idx_global, &max_count_in_cell); CHKERRQ(ierr);
618 total_counted_particles, max_count_in_cell);
621 if (max_idx_global >= 0) {
625 ierr = DMDAGetInfo(da, NULL, &M, &N, &P, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
628 PetscInt Kmax = max_idx_global / (M * N);
629 PetscInt Jmax = (max_idx_global % (M * N)) / M;
630 PetscInt Imax = max_idx_global % M;
632 (
int)Imax, (
int)Jmax, (
int)Kmax, (
int)max_idx_global);
635 PetscScalar count_at_origin = 0.0;
636 PetscScalar ***count_arr_for_check;
637 ierr = DMDAVecGetArrayRead(da, countVec, &count_arr_for_check); CHKERRQ(ierr);
639 PetscInt xs, ys, zs, xm, ym, zm;
640 ierr = DMDAGetCorners(da, &xs, &ys, &zs, &xm, &ym, &zm); CHKERRQ(ierr);
641 if (0 >= xs && 0 < xs+xm && 0 >= ys && 0 < ys+ym && 0 >= zs && 0 < zs+zm) {
642 count_at_origin = count_arr_for_check[0][0][0];
645 count_at_origin = -999.0;
647 ierr = DMDAVecRestoreArrayRead(da, countVec, &count_arr_for_check); CHKERRQ(ierr);
659 PetscFunctionReturn(0);
676 PetscInt N_current, nlocal_current;
680 PetscFunctionBeginUser;
682 ierr = PetscObjectGetComm((PetscObject)swarm, &comm); CHKERRQ(ierr);
683 ierr = MPI_Comm_rank(comm, &rank); CHKERRQ(ierr);
684 ierr = DMSwarmGetSize(swarm, &N_current); CHKERRQ(ierr);
685 ierr = DMSwarmGetLocalSize(swarm, &nlocal_current); CHKERRQ(ierr);
687 PetscInt delta = N_target - N_current;
691 PetscFunctionReturn(0);
695 PetscInt num_to_remove_global = -delta;
696 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Current size %d > target size %d. Removing %d particles globally.\n", N_current, N_target, num_to_remove_global);
706 ierr = MPI_Exscan(&nlocal_current, &rstart, 1, MPIU_INT, MPI_SUM, comm); CHKERRMPI(ierr);
708 rend = rstart + nlocal_current;
712 PetscInt nlocal_remove_count = 0;
713 if (rend > N_target) {
714 PetscInt start_remove_local_idx = (N_target > rstart) ? (N_target - rstart) : 0;
715 nlocal_remove_count = nlocal_current - start_remove_local_idx;
718 if (nlocal_remove_count < 0) nlocal_remove_count = 0;
719 if (nlocal_remove_count > nlocal_current) nlocal_remove_count = nlocal_current;
721 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Global range [%d, %d). Target size %d. Need to remove %d local particles (from end).\n", rank, rstart, rend, N_target, nlocal_remove_count);
724 PetscInt removal_ops_done = 0;
725 for (PetscInt p = nlocal_current - 1; p >= 0 && removal_ops_done < nlocal_remove_count; --p) {
726 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
730 if (removal_ops_done != nlocal_remove_count) {
731 SETERRQ(comm, PETSC_ERR_PLIB,
"Rank %d: Failed to remove the expected number of local particles (%d != %d)", rank, removal_ops_done, nlocal_remove_count);
736 ierr = MPI_Barrier(comm); CHKERRMPI(ierr);
739 PetscInt num_to_add_global = delta;
740 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Current size %d < target size %d. Adding %d particles globally.\n", N_current, N_target, num_to_add_global);
741 ierr = DMSwarmAddNPoints(swarm, num_to_add_global); CHKERRQ(ierr);
747 ierr = DMSwarmGetSize(swarm, &N_final); CHKERRQ(ierr);
748 if (N_final != N_target) {
749 SETERRQ(comm, PETSC_ERR_PLIB,
"Failed to resize swarm: expected %d particles, got %d", N_target, N_final);
753 PetscFunctionReturn(0);
767 char filename[PETSC_MAX_PATH_LEN];
769 PetscInt N_current = 0;
772 const char *refFieldName =
"position";
773 const PetscInt bs = 3;
781 const int placeholder_int = 0;
786 PetscFunctionBeginUser;
788 ierr = PetscObjectGetComm((PetscObject)user->
swarm, &comm); CHKERRQ(ierr);
789 ierr = MPI_Comm_rank(comm, &rank); CHKERRQ(ierr);
797 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONGSTATE,
"Invalid execution mode for reading simulation fields.");
807 ierr = PetscSNPrintf(filename,
sizeof(filename),
"%s/%s%05" PetscInt_FMT
"_%d.%s",
814 PetscBool fileExists = PETSC_FALSE;
815 ierr = PetscTestFile(filename,
'r', &fileExists); CHKERRQ(ierr);
826 ierr = VecCreate(PETSC_COMM_SELF, &tmpVec); CHKERRQ(ierr);
827 ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF, filename, FILE_MODE_READ, &viewer); CHKERRQ(ierr);
828 ierr = VecLoad(tmpVec, viewer); CHKERRQ(ierr);
829 ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr);
831 ierr = VecGetSize(tmpVec, &vecSize); CHKERRQ(ierr);
832 ierr = VecDestroy(&tmpVec); CHKERRQ(ierr);
834 if (vecSize % bs != 0) {
836 LOG_ALLOW(
GLOBAL,
LOG_ERROR,
"Rank 0: Vector size %d from file '%s' is not divisible by block size %d.\n", vecSize, filename, bs);
838 N_file = vecSize / bs;
845 ierr = MPI_Bcast(&N_file, 1, MPIU_INT, 0, comm); CHKERRMPI(ierr);
849 SETERRQ(comm, PETSC_ERR_FILE_OPEN,
"Mandatory reference file '%s' not found for timestep %d (as determined by Rank 0).", filename, ti);
852 SETERRQ(comm, PETSC_ERR_FILE_READ,
"Reference file '%s' has incorrect format (as determined by Rank 0).", filename);
855 SETERRQ(comm, PETSC_ERR_PLIB,
"Received invalid particle count %d from Rank 0.", N_file);
860 ierr = DMSwarmGetSize(user->
swarm, &N_current); CHKERRQ(ierr);
862 if (N_file != N_current) {
863 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Swarm size %d differs from file size %d. Resizing swarm globally.\n", N_current, N_file);
873 PetscFunctionReturn(0);
887 DM swarm = user->
swarm;
888 PetscReal *positions_field = NULL;
889 PetscInt64 *particleIDs = NULL;
890 PetscInt *cell_ID_field = NULL;
891 const Cmpnts ***coor_nodes_local_array;
894 PetscInt xs_gnode_rank, ys_gnode_rank, zs_gnode_rank;
895 PetscInt IM_nodes_global, JM_nodes_global, KM_nodes_global;
897 PetscRandom rand_logic_reinit_i, rand_logic_reinit_j, rand_logic_reinit_k;
898 PetscInt nlocal_current;
899 PetscInt particles_actually_reinitialized_count = 0;
900 PetscBool can_this_rank_service_inlet = PETSC_FALSE;
902 PetscFunctionBeginUser;
908 PetscFunctionReturn(0);
911 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
912 ierr = DMSwarmGetLocalSize(swarm, &nlocal_current); CHKERRQ(ierr);
915 if (nlocal_current == 0) {
916 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[T=%.4f, Step=%d] Rank %d has no local particles to re-initialize on inlet.\n", currentTime, step, rank);
917 PetscFunctionReturn(0);
921 ierr = DMDAGetLocalInfo(user->
da, &info); CHKERRQ(ierr);
922 ierr = DMDAGetInfo(user->
da, NULL, &IM_nodes_global, &JM_nodes_global, &KM_nodes_global, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL); CHKERRQ(ierr);
923 ierr = DMDAGetCorners(user->
da, &xs_gnode_rank, &ys_gnode_rank, &zs_gnode_rank, NULL, NULL, NULL); CHKERRQ(ierr);
926 IM_nodes_global -= 1; JM_nodes_global -= 1; KM_nodes_global -= 1;
928 const PetscInt IM_cells_global = IM_nodes_global > 0 ? IM_nodes_global - 1 : 0;
929 const PetscInt JM_cells_global = JM_nodes_global > 0 ? JM_nodes_global - 1 : 0;
930 const PetscInt KM_cells_global = KM_nodes_global > 0 ? KM_nodes_global - 1 : 0;
935 ierr =
CanRankServiceInletFace(user, &info, IM_nodes_global, JM_nodes_global, KM_nodes_global, &can_this_rank_service_inlet); CHKERRQ(ierr);
938 ierr = DMGetCoordinatesLocal(user->
da, &Coor_local); CHKERRQ(ierr);
939 ierr = DMDAVecGetArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
940 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void**)&positions_field); CHKERRQ(ierr);
941 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&particleIDs); CHKERRQ(ierr);
942 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_ID_field); CHKERRQ(ierr);
944 if (!can_this_rank_service_inlet) {
948 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[T=%.4f, Step=%d] Rank %d is resetting %d local particles to inlet center (%.6f, %.6f, %.6f) for migration.\n", currentTime, step, rank, nlocal_current, user->
simCtx->
CMx_c, user->
simCtx->
CMy_c, user->
simCtx->
CMz_c);
950 for(PetscInt p = 0; p < nlocal_current; p++){
955 cell_ID_field[3*p+0] = -1;
956 cell_ID_field[3*p+1] = -1;
957 cell_ID_field[3*p+2] = -1;
961 ierr = DMDAVecRestoreArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
962 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void**)&positions_field); CHKERRQ(ierr);
963 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&particleIDs); CHKERRQ(ierr);
964 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_ID_field); CHKERRQ(ierr);
966 PetscFunctionReturn(0);
977 for (PetscInt p = 0; p < nlocal_current; p++) {
978 PetscInt ci_metric_lnode, cj_metric_lnode, ck_metric_lnode;
979 PetscReal xi_metric_logic, eta_metric_logic, zta_metric_logic;
980 Cmpnts phys_coords = {0.0,0.0,0.0};
981 PetscBool particle_was_placed = PETSC_FALSE;
986 IM_nodes_global, JM_nodes_global, KM_nodes_global,
987 &rand_logic_reinit_i, &rand_logic_reinit_j, &rand_logic_reinit_k,
988 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
989 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic); CHKERRQ(ierr);
994 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
995 xi_metric_logic, eta_metric_logic, zta_metric_logic,
996 &phys_coords); CHKERRQ(ierr);
999 positions_field[3*p+0] = phys_coords.
x;
1000 positions_field[3*p+1] = phys_coords.
y;
1001 positions_field[3*p+2] = phys_coords.
z;
1002 particle_was_placed = PETSC_TRUE;
1005 PetscBool placement_flag = PETSC_FALSE;
1007 IM_cells_global, JM_cells_global, KM_cells_global,
1009 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
1010 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic,&placement_flag); CHKERRQ(ierr);
1016 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1017 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1018 &phys_coords); CHKERRQ(ierr);
1021 positions_field[3*p+0] = phys_coords.
x;
1022 positions_field[3*p+1] = phys_coords.
y;
1023 positions_field[3*p+2] = phys_coords.
z;
1024 particle_was_placed = PETSC_TRUE;
1028 LOG_ALLOW(
GLOBAL,
LOG_WARNING,
"Rank %d: Particle PID %ld deterministic placement failed (belongs to different rank). Falling back to random placement.\n", rank, particleIDs[p]);
1031 IM_nodes_global, JM_nodes_global, KM_nodes_global,
1032 &rand_logic_reinit_i, &rand_logic_reinit_j, &rand_logic_reinit_k,
1033 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
1034 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic); CHKERRQ(ierr);
1038 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1039 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1040 &phys_coords); CHKERRQ(ierr);
1043 positions_field[3*p+0] = phys_coords.
x;
1044 positions_field[3*p+1] = phys_coords.
y;
1045 positions_field[3*p+2] = phys_coords.
z;
1046 particle_was_placed = PETSC_TRUE;
1050 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE,
"ReinitializeParticlesOnInletSurface only supports ParticleInitialization modes 0 and 3.");
1053 if(particle_was_placed){
1054 particles_actually_reinitialized_count++;
1056 cell_ID_field[3*p+0] = -1;
1057 cell_ID_field[3*p+1] = -1;
1058 cell_ID_field[3*p+2] = -1;
1061 "Rank %d: PID %ld (idx %ld) RE-PLACED. CellOriginNode(locDAIdx):(%d,%d,%d). LogicCoords: (%.2e,%.2f,%.2f). PhysCoords: (%.6f,%.6f,%.6f).\n",
1062 rank, particleIDs[p], (
long)p,
1063 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1064 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1065 phys_coords.
x, phys_coords.
y, phys_coords.
z);
1070 if (particles_actually_reinitialized_count > 0) {
1071 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"[T=%.4f, Step=%d] Rank %d (on inlet face %d) successfully re-initialized %d of %d local particles.\n", currentTime, step, rank, user->
identifiedInletBCFace, particles_actually_reinitialized_count, nlocal_current);
1072 }
else if (nlocal_current > 0) {
1073 LOG_ALLOW(
GLOBAL,
LOG_WARNING,
"[T=%.4f, Step=%d] Rank %d claimed to service inlet face %d, but re-initialized 0 of %d local particles. This may indicate an issue if particles were expected to be re-placed.\n", currentTime, step, rank, user->
identifiedInletBCFace, nlocal_current);
1077 ierr = PetscRandomDestroy(&rand_logic_reinit_i); CHKERRQ(ierr);
1078 ierr = PetscRandomDestroy(&rand_logic_reinit_j); CHKERRQ(ierr);
1079 ierr = PetscRandomDestroy(&rand_logic_reinit_k); CHKERRQ(ierr);
1081 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void**)&positions_field); CHKERRQ(ierr);
1082 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&particleIDs); CHKERRQ(ierr);
1083 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_ID_field); CHKERRQ(ierr);
1084 ierr = DMDAVecRestoreArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
1088 PetscFunctionReturn(0);
1211 PetscInt n_local_before,
1212 const PetscInt64 pids_before[])
1214 PetscErrorCode ierr;
1216 PetscInt n_local_after;
1217 PetscInt newcomer_count = 0;
1220 PetscInt64 *pid_field_after = NULL;
1221 PetscInt *status_field_after = NULL;
1222 PetscInt *cell_field_after = NULL;
1224 PetscFunctionBeginUser;
1230 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Input DMSwarm is NULL in FlagNewcomersForLocation.");
1233 if (n_local_before > 0 && !pids_before) {
1234 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"Input pids_before array is NULL for n_local_before > 0.");
1237 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1240 ierr = DMSwarmGetLocalSize(swarm, &n_local_after); CHKERRQ(ierr);
1243 rank, n_local_before, n_local_after);
1246 if (n_local_after == 0) {
1247 PetscFunctionReturn(0);
1252 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_field_after); CHKERRQ(ierr);
1253 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_field_after); CHKERRQ(ierr);
1254 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_field_after); CHKERRQ(ierr);
1255 if (!pid_field_after || !status_field_after) {
1256 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Failed to get required swarm fields in FlagNewcomersForLocation.");
1261 for (PetscInt p_idx = 0; p_idx < n_local_after; ++p_idx) {
1262 PetscInt64 current_pid = pid_field_after[p_idx];
1263 PetscBool is_found_in_before_list;
1266 ierr =
BinarySearchInt64(n_local_before, pids_before, current_pid, &is_found_in_before_list); CHKERRQ(ierr);
1269 if (!is_found_in_before_list) {
1278 rank, current_pid, p_idx);
1284 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_field_after); CHKERRQ(ierr);
1285 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_field_after); CHKERRQ(ierr);
1286 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_field_after); CHKERRQ(ierr);
1288 if (newcomer_count > 0) {
1294 PetscFunctionReturn(0);
1305 PetscErrorCode ierr;
1306 DM swarm = user->
swarm;
1308 PetscInt *cell_p = NULL;
1309 PetscInt64 *pid_p = NULL;
1313 PetscInt local_migration_count = 0;
1314 PetscInt migrationListCapacity = 0;
1315 PetscInt global_migration_count = 0;
1317 PetscFunctionBeginUser;
1319 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1321 ierr = DMSwarmGetLocalSize(swarm, &nlocal); CHKERRQ(ierr);
1325 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1326 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1331 for (PetscInt p_idx = 0; p_idx < nlocal; ++p_idx) {
1332 PetscInt ci = cell_p[3*p_idx + 0];
1333 PetscInt cj = cell_p[3*p_idx + 1];
1334 PetscInt ck = cell_p[3*p_idx + 2];
1337 if (ci < 0 || cj < 0 || ck < 0) {
1341 PetscMPIInt owner_rank;
1344 if (owner_rank != -1 && owner_rank != rank) {
1346 ierr =
AddToMigrationList(&migrationList, &migrationListCapacity, &local_migration_count,
1347 p_idx, owner_rank); CHKERRQ(ierr);
1350 (
long)pid_p[p_idx], ci, cj, ck, owner_rank, rank);
1354 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1355 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1359 ierr = MPI_Allreduce(&local_migration_count, &global_migration_count, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD); CHKERRQ(ierr);
1361 if (global_migration_count > 0) {
1362 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Fast restart migration: Directly migrating %d particles using CellIDs.\n", global_migration_count);
1363 ierr =
SetMigrationRanks(user, migrationList, local_migration_count); CHKERRQ(ierr);
1371 ierr = PetscFree(migrationList); CHKERRQ(ierr);
1374 PetscFunctionReturn(0);
1508 PetscErrorCode ierr;
1509 PetscInt passes = 0;
1510 const PetscInt MAX_MIGRATION_PASSES = 50;
1511 PetscInt global_migrations_this_pass;
1513 PetscInt total_migrated_this_timestep = 0;
1515 PetscFunctionBeginUser;
1517 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1519 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"LocateAllParticlesInGrid (Orchestrator) - Beginning particle settlement process.\n");
1530 PetscInt local_migration_count = 0;
1531 PetscInt migrationListCapacity = 0;
1532 PetscInt nlocal_before;
1533 PetscInt64 *pids_before_snapshot = NULL;
1534 PetscInt local_lost_count = 0;
1536 ierr = DMSwarmGetLocalSize(user->
swarm, &nlocal_before); CHKERRQ(ierr);
1540 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[Rank %d] Pass %d begins with %d local particles.\n", rank, passes, nlocal_before);
1544 if (nlocal_before > 0) {
1546 PetscReal *pos_p, *weights_p, *vel_p;
1547 PetscInt *cell_p, *status_p;
1549 ierr = DMSwarmGetField(user->
swarm,
"position", NULL, NULL, (
void**)&pos_p); CHKERRQ(ierr);
1550 ierr = DMSwarmGetField(user->
swarm,
"velocity", NULL, NULL, (
void**)&vel_p); CHKERRQ(ierr);
1551 ierr = DMSwarmGetField(user->
swarm,
"weight", NULL, NULL, (
void**)&weights_p); CHKERRQ(ierr);
1552 ierr = DMSwarmGetField(user->
swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1553 ierr = DMSwarmGetField(user->
swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1554 ierr = DMSwarmGetField(user->
swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_p); CHKERRQ(ierr);
1560 for (PetscInt p_idx = 0; p_idx < nlocal_before; p_idx++) {
1565 "Local Particle idx=%d, PID=%ld, status=%s, cell=(%d, %d, %d)\n",
1574 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
" [rank %d][PID %ld] skipped in pass %d as it is already located at (%d,%d,%d).\n",rank,pid_p[p_idx],passes,cell_p[3*p_idx],cell_p[3*p_idx + 1],cell_p[3*p_idx + 2]);
1583 ierr =
UnpackSwarmFields(p_idx, pid_p, weights_p, pos_p, cell_p, vel_p, status_p,NULL,NULL,NULL,¤t_particle); CHKERRQ(ierr);
1592 if (current_particle.
cell[0] >= 0) {
1635 PetscMPIInt guessed_owner_rank = MPI_PROC_NULL;
1639 if (guessed_owner_rank != MPI_PROC_NULL && guessed_owner_rank != rank) {
1641 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
"[PID %ld] Guess SUCCESS: Found migration target Rank %d. Finalizing.\n", current_particle.
PID, guessed_owner_rank);
1650 if (guessed_owner_rank == rank) {
1651 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[PID %ld] Guess determined particle is local. Proceeding to robust walk to find cell.\n", current_particle.
PID);
1653 LOG_ALLOW(
LOCAL,
LOG_WARNING,
"[PID %ld] Guess FAILED to find an owner. Proceeding to robust walk for definitive search.\n", current_particle.
PID);
1662 ierr =
UpdateSwarmFields(p_idx, ¤t_particle, pos_p, vel_p, weights_p, cell_p, status_p,NULL,NULL,NULL); CHKERRQ(ierr);
1666 }
else if (final_status ==
LOST) {
1676 ierr = DMSwarmRestoreField(user->
swarm,
"position", NULL, NULL, (
void**)&pos_p); CHKERRQ(ierr);
1677 ierr = DMSwarmRestoreField(user->
swarm,
"velocity", NULL, NULL, (
void**)&vel_p); CHKERRQ(ierr);
1678 ierr = DMSwarmRestoreField(user->
swarm,
"weight", NULL, NULL, (
void**)&weights_p); CHKERRQ(ierr);
1679 ierr = DMSwarmRestoreField(user->
swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1680 ierr = DMSwarmRestoreField(user->
swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1681 ierr = DMSwarmRestoreField(user->
swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_p); CHKERRQ(ierr);
1685 LOG_ALLOW(
LOCAL,
LOG_INFO,
"[Rank %d] Pass %d: Identified %d particles to migrate out.\n", rank, passes, local_migration_count);
1690 ierr = MPI_Allreduce(&local_migration_count, &global_migrations_this_pass, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD); CHKERRQ(ierr);
1692 total_migrated_this_timestep += global_migrations_this_pass;
1694 if(global_migrations_this_pass > 0 ){
1696 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Pass %d: Migrating %d particles globally.\n", passes, global_migrations_this_pass);
1698 ierr =
SetMigrationRanks(user, migrationList, local_migration_count); CHKERRQ(ierr);
1708 ierr = PetscFree(pids_before_snapshot);
1709 ierr = PetscFree(migrationList);
1711 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"End of pass %d. Total particles migrated globally: %d.\n", passes, global_migrations_this_pass);
1713 }
while (global_migrations_this_pass > 0 && passes < MAX_MIGRATION_PASSES);
1716 if (passes >= MAX_MIGRATION_PASSES) {
1717 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_CONV_FAILED,
"Particle migration failed to converge after %d passes. Check for particles oscillating between ranks.", MAX_MIGRATION_PASSES);
1728 PetscFunctionReturn(0);
PetscErrorCode UnpackSwarmFields(PetscInt i, const PetscInt64 *PIDs, const PetscReal *weights, const PetscReal *positions, const PetscInt *cellIndices, PetscReal *velocities, PetscInt *LocStatus, PetscReal *diffusivity, Cmpnts *diffusivitygradient, PetscReal *psi, Particle *particle)
Initializes a Particle struct with data from DMSwarm fields.