137 DM swarm = user->
swarm;
139 PetscReal *pos = NULL;
140 PetscReal *vel = NULL;
141 PetscReal *diffusivity = NULL;
142 Cmpnts *diffusivitygradient = NULL;
143 PetscReal *psi = NULL;
144 PetscReal *weights = NULL;
145 PetscInt *cell = NULL;
146 PetscInt *status = NULL;
147 PetscInt64 *pid = NULL;
150 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
152 PetscFunctionBeginUser;
157 ierr = DMSwarmGetLocalSize(swarm, &nLocal); CHKERRQ(ierr);
161 PetscFunctionReturn(0);
177 for (p = 0; p < nLocal; p++) {
182 ierr =
UnpackSwarmFields(p, pid, weights, pos, cell, vel, status, diffusivity, diffusivitygradient, psi, &particle); CHKERRQ(ierr);
188 ierr =
UpdateSwarmFields(p, &particle, pos, vel, weights, cell, status, diffusivity, diffusivitygradient, psi); CHKERRQ(ierr);
207 PetscFunctionReturn(0);
229 PetscInt *removedCountLocal,
230 PetscInt *removedCountGlobal,
234 DM swarm = user->
swarm;
235 PetscInt nLocalInitial;
236 PetscReal *pos_p = NULL;
237 PetscInt64 *pid_p = NULL;
238 PetscInt local_removed_count = 0;
239 PetscMPIInt global_removed_count_mpi = 0;
240 PetscMPIInt rank, size;
242 PetscFunctionBeginUser;
243 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
244 ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size); CHKERRQ(ierr);
248 *removedCountLocal = 0;
249 if (removedCountGlobal) *removedCountGlobal = 0;
251 ierr = DMSwarmGetLocalSize(swarm, &nLocalInitial); CHKERRQ(ierr);
255 if (nLocalInitial > 0) {
261 for (PetscInt p = nLocalInitial - 1; p >= 0; p--) {
262 PetscBool isInsideAnyBox = PETSC_FALSE;
263 Cmpnts current_pos = {pos_p[3*p + 0], pos_p[3*p + 1], pos_p[3*p + 2]};
266 for (PetscMPIInt proc = 0; proc < size; proc++) {
268 isInsideAnyBox = PETSC_TRUE;
273 if (!isInsideAnyBox) {
274 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Removing out-of-bounds particle [PID %lld] at local index %d. Pos: (%g, %g, %g)\n",
275 rank, (
long long)pid_p[p], p, current_pos.
x, current_pos.
y, current_pos.
z);
286 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
287 local_removed_count++;
290 PetscInt nLocalCurrent;
291 ierr = DMSwarmGetLocalSize(swarm, &nLocalCurrent); CHKERRQ(ierr);
293 if (nLocalCurrent > 0 && p > 0) {
314 PetscInt nLocalFinal;
315 ierr = DMSwarmGetLocalSize(swarm, &nLocalFinal); CHKERRQ(ierr);
316 LOG_ALLOW(
LOCAL,
LOG_INFO,
"[Rank %d] Finished removing %d out-of-bounds particles. Final local size: %d.\n", rank, local_removed_count, nLocalFinal);
319 *removedCountLocal = local_removed_count;
320 if (removedCountGlobal) {
321 ierr = MPI_Allreduce(&local_removed_count, &global_removed_count_mpi, 1, MPI_INT, MPI_SUM, PetscObjectComm((PetscObject)swarm)); CHKERRQ(ierr);
322 *removedCountGlobal = global_removed_count_mpi;
327 PetscFunctionReturn(0);
337 PetscInt *removedCountLocal,
338 PetscInt *removedCountGlobal)
341 DM swarm = user->
swarm;
342 PetscInt nLocalInitial;
343 PetscInt *status_p = NULL;
344 PetscInt64 *pid_p = NULL;
345 PetscReal *pos_p = NULL;
346 PetscInt local_removed_count = 0;
347 PetscMPIInt global_removed_count_mpi = 0;
350 PetscFunctionBeginUser;
352 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
356 *removedCountLocal = 0;
357 if (removedCountGlobal) *removedCountGlobal = 0;
359 ierr = DMSwarmGetLocalSize(swarm, &nLocalInitial); CHKERRQ(ierr);
363 if (nLocalInitial > 0) {
370 for (PetscInt p = nLocalInitial - 1; p >= 0; p--) {
371 if (status_p[p] ==
LOST) {
372 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Removing LOST particle [PID %lld] at local index %d. Position: (%.4f, %.4f, %.4f).\n",
373 rank, (
long long)pid_p[p], p, pos_p[3*p], pos_p[3*p+1], pos_p[3*p+2]);
385 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
386 local_removed_count++;
389 PetscInt nLocalCurrent;
390 ierr = DMSwarmGetLocalSize(swarm, &nLocalCurrent); CHKERRQ(ierr);
392 if (nLocalCurrent > 0 && p > 0) {
416 PetscInt nLocalFinal;
417 ierr = DMSwarmGetLocalSize(swarm, &nLocalFinal); CHKERRQ(ierr);
418 LOG_ALLOW(
LOCAL,
LOG_INFO,
"Rank %d: Finished removing %d LOST particles. Final local size: %d.\n", rank, local_removed_count, nLocalFinal);
421 *removedCountLocal = local_removed_count;
422 if (removedCountGlobal) {
423 ierr = MPI_Allreduce(&local_removed_count, &global_removed_count_mpi, 1, MPI_INT, MPI_SUM, PetscObjectComm((PetscObject)swarm)); CHKERRQ(ierr);
424 *removedCountGlobal = global_removed_count_mpi;
430 PetscFunctionReturn(0);
508 DM swarm = user->
swarm;
512 PetscInt *global_cell_id_arr;
513 PetscScalar ***count_arr_3d;
517 PetscInt particles_counted_locally = 0;
519 PetscFunctionBeginUser;
521 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
524 if (!da) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->da is NULL.");
525 if (!swarm) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->swarm is NULL.");
526 if (!countVec) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->ParticleCount is NULL.");
529 ierr = DMDAGetInfo(da, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &count_dof, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
530 if (count_dof != 1) {
531 PetscSNPrintf(msg,
sizeof(msg),
"countDM must have DOF=1, got %" PetscInt_FMT
".", count_dof);
532 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG,
"%s", msg);
536 ierr = VecSet(localcountVec, 0.0); CHKERRQ(ierr);
540 ierr = DMSwarmGetLocalSize(swarm, &nlocal); CHKERRQ(ierr);
546 ierr = DMDAVecGetArray(da, localcountVec, &count_arr_3d); CHKERRQ(ierr);
549 PetscInt gxs, gys, gzs, gxm, gym, gzm;
550 ierr = DMDAGetGhostCorners(da, &gxs, &gys, &gzs, &gxm, &gym, &gzm); CHKERRQ(ierr);
553 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"CalculateParticleCountPerCell (Rank %d): Processing %" PetscInt_FMT
" local particles using GLOBAL CellIDs.\n",rank,nlocal);
554 for (p = 0; p < nlocal; p++) {
556 PetscInt i_geom = global_cell_id_arr[p * 3 + 0];
557 PetscInt j_geom = global_cell_id_arr[p * 3 + 1];
558 PetscInt k_geom = global_cell_id_arr[p * 3 + 2];
561 PetscInt i = (PetscInt)i_geom + 1;
562 PetscInt j = (PetscInt)j_geom + 1;
563 PetscInt k = (PetscInt)k_geom + 1;
571 "[Rank %d] Read CellID for p=%" PetscInt_FMT
", PID = %" PetscInt64_FMT
": (%" PetscInt_FMT
", %" PetscInt_FMT
", %" PetscInt_FMT
")\n",
572 rank, p, PID_arr[p], i, j, k);
575 if (i >= gxs && i < gxs + gxm &&
576 j >= gys && j < gys + gym &&
577 k >= gzs && k < gzs + gzm )
582 count_arr_3d[k][j][i] += 1.0;
583 particles_counted_locally++;
588 "(Rank %d): Skipping particle %" PetscInt64_FMT
" with global CellID (%" PetscInt_FMT
", %" PetscInt_FMT
", %" PetscInt_FMT
") - likely outside local+ghost range.\n",
589 rank, PID_arr[p], i, j, k);
592 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"(Rank %d): Local counting finished. Processed %" PetscInt_FMT
" particles locally.\n", rank, particles_counted_locally);
595 ierr = DMDAVecRestoreArray(da, localcountVec, &count_arr_3d); CHKERRQ(ierr);
601 ierr = VecZeroEntries(countVec); CHKERRQ(ierr);
602 ierr = DMLocalToGlobalBegin(da, localcountVec, ADD_VALUES, countVec); CHKERRQ(ierr);
603 ierr = DMLocalToGlobalEnd(da, localcountVec, ADD_VALUES, countVec); CHKERRQ(ierr);
614 PetscReal total_counted_particles = 0.0, max_count_in_cell = 0.0;
615 ierr = VecSum(countVec, &total_counted_particles); CHKERRQ(ierr);
616 PetscInt max_idx_global = -1;
617 ierr = VecMax(countVec, &max_idx_global, &max_count_in_cell); CHKERRQ(ierr);
619 total_counted_particles, max_count_in_cell);
622 if (max_idx_global >= 0) {
626 ierr = DMDAGetInfo(da, NULL, &M, &N, &P, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
629 PetscInt Kmax = max_idx_global / (M * N);
630 PetscInt Jmax = (max_idx_global % (M * N)) / M;
631 PetscInt Imax = max_idx_global % M;
633 (
int)Imax, (
int)Jmax, (
int)Kmax, (
int)max_idx_global);
636 PetscScalar count_at_origin = 0.0;
637 PetscScalar ***count_arr_for_check;
638 ierr = DMDAVecGetArrayRead(da, countVec, &count_arr_for_check); CHKERRQ(ierr);
640 PetscInt xs, ys, zs, xm, ym, zm;
641 ierr = DMDAGetCorners(da, &xs, &ys, &zs, &xm, &ym, &zm); CHKERRQ(ierr);
642 if (0 >= xs && 0 < xs+xm && 0 >= ys && 0 < ys+ym && 0 >= zs && 0 < zs+zm) {
643 count_at_origin = count_arr_for_check[0][0][0];
646 count_at_origin = -999.0;
648 ierr = DMDAVecRestoreArrayRead(da, countVec, &count_arr_for_check); CHKERRQ(ierr);
660 PetscFunctionReturn(0);
764 DM swarm = user->
swarm;
765 PetscReal *positions_field = NULL;
766 PetscInt64 *particleIDs = NULL;
767 PetscInt *cell_ID_field = NULL;
768 const Cmpnts ***coor_nodes_local_array;
771 PetscInt xs_gnode_rank, ys_gnode_rank, zs_gnode_rank;
772 PetscInt IM_nodes_global, JM_nodes_global, KM_nodes_global;
774 PetscRandom rand_logic_reinit_i, rand_logic_reinit_j, rand_logic_reinit_k;
775 PetscInt nlocal_current;
776 PetscInt particles_actually_reinitialized_count = 0;
777 PetscBool can_this_rank_service_inlet = PETSC_FALSE;
779 PetscFunctionBeginUser;
786 PetscFunctionReturn(0);
789 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
790 ierr = DMSwarmGetLocalSize(swarm, &nlocal_current); CHKERRQ(ierr);
793 if (nlocal_current == 0) {
794 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[T=%.4f, Step=%d] Rank %d has no local particles to re-initialize on inlet.\n", currentTime, step, rank);
796 PetscFunctionReturn(0);
800 ierr = DMDAGetLocalInfo(user->
da, &info); CHKERRQ(ierr);
801 ierr = DMDAGetInfo(user->
da, NULL, &IM_nodes_global, &JM_nodes_global, &KM_nodes_global, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL); CHKERRQ(ierr);
802 ierr = DMDAGetCorners(user->
da, &xs_gnode_rank, &ys_gnode_rank, &zs_gnode_rank, NULL, NULL, NULL); CHKERRQ(ierr);
805 IM_nodes_global -= 1; JM_nodes_global -= 1; KM_nodes_global -= 1;
807 const PetscInt IM_cells_global = IM_nodes_global > 0 ? IM_nodes_global - 1 : 0;
808 const PetscInt JM_cells_global = JM_nodes_global > 0 ? JM_nodes_global - 1 : 0;
809 const PetscInt KM_cells_global = KM_nodes_global > 0 ? KM_nodes_global - 1 : 0;
814 ierr =
CanRankServiceInletFace(user, &info, IM_nodes_global, JM_nodes_global, KM_nodes_global, &can_this_rank_service_inlet); CHKERRQ(ierr);
817 ierr = DMGetCoordinatesLocal(user->
da, &Coor_local); CHKERRQ(ierr);
818 ierr = DMDAVecGetArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
823 if (!can_this_rank_service_inlet) {
827 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[T=%.4f, Step=%d] Rank %d is resetting %d local particles to inlet center (%.6f, %.6f, %.6f) for migration.\n", currentTime, step, rank, nlocal_current, user->
simCtx->
CMx_c, user->
simCtx->
CMy_c, user->
simCtx->
CMz_c);
829 for(PetscInt p = 0; p < nlocal_current; p++){
834 cell_ID_field[3*p+0] = -1;
835 cell_ID_field[3*p+1] = -1;
836 cell_ID_field[3*p+2] = -1;
840 ierr = DMDAVecRestoreArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
845 PetscFunctionReturn(0);
856 for (PetscInt p = 0; p < nlocal_current; p++) {
857 PetscInt ci_metric_lnode, cj_metric_lnode, ck_metric_lnode;
858 PetscReal xi_metric_logic, eta_metric_logic, zta_metric_logic;
859 Cmpnts phys_coords = {0.0,0.0,0.0};
860 PetscBool particle_was_placed = PETSC_FALSE;
865 IM_nodes_global, JM_nodes_global, KM_nodes_global,
866 &rand_logic_reinit_i, &rand_logic_reinit_j, &rand_logic_reinit_k,
867 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
868 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic); CHKERRQ(ierr);
873 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
874 xi_metric_logic, eta_metric_logic, zta_metric_logic,
875 &phys_coords); CHKERRQ(ierr);
878 positions_field[3*p+0] = phys_coords.
x;
879 positions_field[3*p+1] = phys_coords.
y;
880 positions_field[3*p+2] = phys_coords.
z;
881 particle_was_placed = PETSC_TRUE;
884 PetscBool placement_flag = PETSC_FALSE;
886 IM_cells_global, JM_cells_global, KM_cells_global,
888 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
889 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic,&placement_flag); CHKERRQ(ierr);
895 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
896 xi_metric_logic, eta_metric_logic, zta_metric_logic,
897 &phys_coords); CHKERRQ(ierr);
900 positions_field[3*p+0] = phys_coords.
x;
901 positions_field[3*p+1] = phys_coords.
y;
902 positions_field[3*p+2] = phys_coords.
z;
903 particle_was_placed = PETSC_TRUE;
907 LOG_ALLOW(
GLOBAL,
LOG_WARNING,
"Rank %d: Particle PID %ld deterministic placement failed (belongs to different rank). Falling back to random placement.\n", rank, particleIDs[p]);
910 IM_nodes_global, JM_nodes_global, KM_nodes_global,
911 &rand_logic_reinit_i, &rand_logic_reinit_j, &rand_logic_reinit_k,
912 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
913 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic); CHKERRQ(ierr);
917 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
918 xi_metric_logic, eta_metric_logic, zta_metric_logic,
919 &phys_coords); CHKERRQ(ierr);
922 positions_field[3*p+0] = phys_coords.
x;
923 positions_field[3*p+1] = phys_coords.
y;
924 positions_field[3*p+2] = phys_coords.
z;
925 particle_was_placed = PETSC_TRUE;
929 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE,
"ReinitializeParticlesOnInletSurface only supports ParticleInitialization modes 0 and 3.");
932 if(particle_was_placed){
933 particles_actually_reinitialized_count++;
935 cell_ID_field[3*p+0] = -1;
936 cell_ID_field[3*p+1] = -1;
937 cell_ID_field[3*p+2] = -1;
940 "Rank %d: PID %ld (idx %ld) RE-PLACED. CellOriginNode(locDAIdx):(%d,%d,%d). LogicCoords: (%.2e,%.2f,%.2f). PhysCoords: (%.6f,%.6f,%.6f).\n",
941 rank, particleIDs[p], (
long)p,
942 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
943 xi_metric_logic, eta_metric_logic, zta_metric_logic,
944 phys_coords.
x, phys_coords.
y, phys_coords.
z);
949 if (particles_actually_reinitialized_count > 0) {
950 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"[T=%.4f, Step=%d] Rank %d (on inlet face %d) successfully re-initialized %d of %d local particles.\n", currentTime, step, rank, user->
identifiedInletBCFace, particles_actually_reinitialized_count, nlocal_current);
951 }
else if (nlocal_current > 0) {
952 LOG_ALLOW(
GLOBAL,
LOG_WARNING,
"[T=%.4f, Step=%d] Rank %d claimed to service inlet face %d, but re-initialized 0 of %d local particles. This may indicate an issue if particles were expected to be re-placed.\n", currentTime, step, rank, user->
identifiedInletBCFace, nlocal_current);
956 ierr = PetscRandomDestroy(&rand_logic_reinit_i); CHKERRQ(ierr);
957 ierr = PetscRandomDestroy(&rand_logic_reinit_j); CHKERRQ(ierr);
958 ierr = PetscRandomDestroy(&rand_logic_reinit_k); CHKERRQ(ierr);
963 ierr = DMDAVecRestoreArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
967 PetscFunctionReturn(0);
1090 PetscInt n_local_before,
1091 const PetscInt64 pids_before[])
1093 PetscErrorCode ierr;
1095 PetscInt n_local_after;
1096 PetscInt newcomer_count = 0;
1099 PetscInt64 *pid_field_after = NULL;
1100 PetscInt *status_field_after = NULL;
1101 PetscInt *cell_field_after = NULL;
1103 PetscFunctionBeginUser;
1109 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Input DMSwarm is NULL in FlagNewcomersForLocation.");
1112 if (n_local_before > 0 && !pids_before) {
1113 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"Input pids_before array is NULL for n_local_before > 0.");
1116 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1119 ierr = DMSwarmGetLocalSize(swarm, &n_local_after); CHKERRQ(ierr);
1122 rank, n_local_before, n_local_after);
1125 if (n_local_after == 0) {
1127 PetscFunctionReturn(0);
1135 if (!pid_field_after || !status_field_after) {
1136 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Failed to get required swarm fields in FlagNewcomersForLocation.");
1141 for (PetscInt p_idx = 0; p_idx < n_local_after; ++p_idx) {
1142 PetscInt64 current_pid = pid_field_after[p_idx];
1143 PetscBool is_found_in_before_list;
1146 ierr =
BinarySearchInt64(n_local_before, pids_before, current_pid, &is_found_in_before_list); CHKERRQ(ierr);
1149 if (!is_found_in_before_list) {
1158 rank, current_pid, p_idx);
1168 if (newcomer_count > 0) {
1174 PetscFunctionReturn(0);
1185 PetscErrorCode ierr;
1186 DM swarm = user->
swarm;
1188 PetscInt *cell_p = NULL;
1189 PetscInt64 *pid_p = NULL;
1193 PetscInt local_migration_count = 0;
1194 PetscInt migrationListCapacity = 0;
1195 PetscInt global_migration_count = 0;
1197 PetscFunctionBeginUser;
1199 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1201 ierr = DMSwarmGetLocalSize(swarm, &nlocal); CHKERRQ(ierr);
1211 for (PetscInt p_idx = 0; p_idx < nlocal; ++p_idx) {
1212 PetscInt ci = cell_p[3*p_idx + 0];
1213 PetscInt cj = cell_p[3*p_idx + 1];
1214 PetscInt ck = cell_p[3*p_idx + 2];
1217 if (ci < 0 || cj < 0 || ck < 0) {
1221 PetscMPIInt owner_rank;
1224 if (owner_rank != -1 && owner_rank != rank) {
1226 ierr =
AddToMigrationList(&migrationList, &migrationListCapacity, &local_migration_count,
1227 p_idx, owner_rank); CHKERRQ(ierr);
1230 (
long)pid_p[p_idx], ci, cj, ck, owner_rank, rank);
1239 ierr = MPI_Allreduce(&local_migration_count, &global_migration_count, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD); CHKERRQ(ierr);
1241 if (global_migration_count > 0) {
1242 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Fast restart migration: Directly migrating %d particles using CellIDs.\n", global_migration_count);
1243 ierr =
SetMigrationRanks(user, migrationList, local_migration_count); CHKERRQ(ierr);
1251 ierr = PetscFree(migrationList); CHKERRQ(ierr);
1254 PetscFunctionReturn(0);
1387 PetscErrorCode ierr;
1388 PetscInt passes = 0;
1389 const PetscInt MAX_MIGRATION_PASSES = 50;
1390 PetscInt global_migrations_this_pass;
1392 PetscInt total_migrated_this_timestep = 0;
1394 PetscFunctionBeginUser;
1396 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1398 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"LocateAllParticlesInGrid (Orchestrator) - Beginning particle settlement process.\n");
1409 PetscInt local_migration_count = 0;
1410 PetscInt migrationListCapacity = 0;
1411 PetscInt nlocal_before;
1412 PetscInt64 *pids_before_snapshot = NULL;
1413 PetscInt local_lost_count = 0;
1415 ierr = DMSwarmGetLocalSize(user->
swarm, &nlocal_before); CHKERRQ(ierr);
1419 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[Rank %d] Pass %d begins with %d local particles.\n", rank, passes, nlocal_before);
1423 if (nlocal_before > 0) {
1425 PetscReal *pos_p, *weights_p, *vel_p;
1426 PetscInt *cell_p, *status_p;
1439 for (PetscInt p_idx = 0; p_idx < nlocal_before; p_idx++) {
1444 "Local Particle idx=%d, PID=%ld, status=%s, cell=(%d, %d, %d)\n",
1453 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
" [rank %d][PID %ld] skipped in pass %d as it is already located at (%d,%d,%d).\n",rank,pid_p[p_idx],passes,cell_p[3*p_idx],cell_p[3*p_idx + 1],cell_p[3*p_idx + 2]);
1462 ierr =
UnpackSwarmFields(p_idx, pid_p, weights_p, pos_p, cell_p, vel_p, status_p,NULL,NULL,NULL,¤t_particle); CHKERRQ(ierr);
1471 if (current_particle.
cell[0] >= 0) {
1514 PetscMPIInt guessed_owner_rank = MPI_PROC_NULL;
1518 if (guessed_owner_rank != MPI_PROC_NULL && guessed_owner_rank != rank) {
1520 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
"[PID %ld] Guess SUCCESS: Found migration target Rank %d. Finalizing.\n", current_particle.
PID, guessed_owner_rank);
1529 if (guessed_owner_rank == rank) {
1530 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[PID %ld] Guess determined particle is local. Proceeding to robust walk to find cell.\n", current_particle.
PID);
1532 LOG_ALLOW(
LOCAL,
LOG_WARNING,
"[PID %ld] Guess FAILED to find an owner. Proceeding to robust walk for definitive search.\n", current_particle.
PID);
1541 ierr =
UpdateSwarmFields(p_idx, ¤t_particle, pos_p, vel_p, weights_p, cell_p, status_p,NULL,NULL,NULL); CHKERRQ(ierr);
1545 }
else if (final_status ==
LOST) {
1564 LOG_ALLOW(
LOCAL,
LOG_INFO,
"[Rank %d] Pass %d: Identified %d particles to migrate out.\n", rank, passes, local_migration_count);
1569 ierr = MPI_Allreduce(&local_migration_count, &global_migrations_this_pass, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD); CHKERRQ(ierr);
1571 total_migrated_this_timestep += global_migrations_this_pass;
1573 if(global_migrations_this_pass > 0 ){
1575 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Pass %d: Migrating %d particles globally.\n", passes, global_migrations_this_pass);
1577 ierr =
SetMigrationRanks(user, migrationList, local_migration_count); CHKERRQ(ierr);
1587 ierr = PetscFree(pids_before_snapshot);
1588 ierr = PetscFree(migrationList);
1590 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"End of pass %d. Total particles migrated globally: %d.\n", passes, global_migrations_this_pass);
1592 }
while (global_migrations_this_pass > 0 && passes < MAX_MIGRATION_PASSES);
1595 if (passes >= MAX_MIGRATION_PASSES) {
1596 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_CONV_FAILED,
"Particle migration failed to converge after %d passes. Check for particles oscillating between ranks.", MAX_MIGRATION_PASSES);
1607 PetscFunctionReturn(0);
PetscErrorCode UnpackSwarmFields(PetscInt i, const PetscInt64 *PIDs, const PetscReal *weights, const PetscReal *positions, const PetscInt *cellIndices, PetscReal *velocities, PetscInt *LocStatus, PetscReal *diffusivity, Cmpnts *diffusivitygradient, PetscReal *psi, Particle *particle)
Initializes a Particle struct with data from DMSwarm fields.
PetscErrorCode UpdateSwarmFields(PetscInt i, const Particle *particle, PetscReal *positions, PetscReal *velocities, PetscReal *weights, PetscInt *cellIndices, PetscInt *status, PetscReal *diffusivity, Cmpnts *diffusivitygradient, PetscReal *psi)
Updates DMSwarm data arrays from a Particle struct.