155 DM swarm = user->
swarm;
157 PetscReal *pos = NULL;
158 PetscReal *vel = NULL;
159 PetscReal *diffusivity = NULL;
160 PetscReal *diffusivitygradient = NULL;
161 PetscReal *psi = NULL;
162 PetscReal *weights = NULL;
163 PetscInt *cell = NULL;
164 PetscInt *status = NULL;
165 PetscInt *pid = NULL;
168 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
170 PetscFunctionBeginUser;
175 ierr = DMSwarmGetLocalSize(swarm, &nLocal); CHKERRQ(ierr);
178 PetscFunctionReturn(0);
181 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void**)&pos); CHKERRQ(ierr);
182 ierr = DMSwarmGetField(swarm,
"velocity", NULL, NULL, (
void**)&vel); CHKERRQ(ierr);
183 ierr = DMSwarmGetField(swarm,
"Diffusivity", NULL, NULL, (
void**)&diffusivity); CHKERRQ(ierr);
184 ierr = DMSwarmGetField(swarm,
"DiffusivityGradient", NULL, NULL, (
void**)&diffusivitygradient); CHKERRQ(ierr);
185 ierr = DMSwarmGetField(swarm,
"Psi", NULL, NULL, (
void**)&psi); CHKERRQ(ierr);
186 ierr = DMSwarmGetField(swarm,
"weight", NULL, NULL, (
void**)&weights); CHKERRQ(ierr);
187 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell); CHKERRQ(ierr);
188 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status); CHKERRQ(ierr);
189 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid); CHKERRQ(ierr);
194 for (p = 0; p < nLocal; p++) {
199 ierr =
UnpackSwarmFields(p, pid, weights, pos, cell, vel, status, diffusivity, diffusivitygradient, psi, &particle); CHKERRQ(ierr);
205 ierr =
UpdateSwarmFields(p, &particle, pos, vel, weights, cell, status, diffusivity, diffusivitygradient, psi); CHKERRQ(ierr);
209 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void**)&pos); CHKERRQ(ierr);
210 ierr = DMSwarmRestoreField(swarm,
"velocity", NULL, NULL, (
void**)&vel); CHKERRQ(ierr);
211 ierr = DMSwarmRestoreField(swarm,
"Diffusivity", NULL, NULL, (
void**)&diffusivity); CHKERRQ(ierr);
212 ierr = DMSwarmRestoreField(swarm,
"DiffusivityGradient", NULL, NULL, (
void**)&diffusivitygradient); CHKERRQ(ierr);
213 ierr = DMSwarmRestoreField(swarm,
"Psi", NULL, NULL, (
void**)&psi); CHKERRQ(ierr);
214 ierr = DMSwarmRestoreField(swarm,
"weight", NULL, NULL, (
void**)&weights); CHKERRQ(ierr);
215 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell); CHKERRQ(ierr);
216 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status); CHKERRQ(ierr);
217 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid); CHKERRQ(ierr);
224 PetscFunctionReturn(0);
282 PetscInt *removedCountLocal,
283 PetscInt *removedCountGlobal,
287 DM swarm = user->
swarm;
288 PetscInt nLocalInitial;
289 PetscReal *pos_p = NULL;
290 PetscInt64 *pid_p = NULL;
291 PetscInt local_removed_count = 0;
292 PetscMPIInt global_removed_count_mpi = 0;
293 PetscMPIInt rank, size;
295 PetscFunctionBeginUser;
296 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
297 ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size); CHKERRQ(ierr);
301 *removedCountLocal = 0;
302 if (removedCountGlobal) *removedCountGlobal = 0;
304 ierr = DMSwarmGetLocalSize(swarm, &nLocalInitial); CHKERRQ(ierr);
308 if (nLocalInitial > 0) {
310 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
311 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
314 for (PetscInt p = nLocalInitial - 1; p >= 0; p--) {
315 PetscBool isInsideAnyBox = PETSC_FALSE;
316 Cmpnts current_pos = {pos_p[3*p + 0], pos_p[3*p + 1], pos_p[3*p + 2]};
319 for (PetscMPIInt proc = 0; proc < size; proc++) {
321 isInsideAnyBox = PETSC_TRUE;
326 if (!isInsideAnyBox) {
327 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Removing out-of-bounds particle [PID %lld] at local index %d. Pos: (%g, %g, %g)\n",
328 rank, (
long long)pid_p[p], p, current_pos.
x, current_pos.
y, current_pos.
z);
335 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
336 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
339 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
340 local_removed_count++;
343 PetscInt nLocalCurrent;
344 ierr = DMSwarmGetLocalSize(swarm, &nLocalCurrent); CHKERRQ(ierr);
346 if (nLocalCurrent > 0 && p > 0) {
347 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
348 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
363 if (pos_p) { ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr); }
364 if (pid_p) { ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr); }
367 PetscInt nLocalFinal;
368 ierr = DMSwarmGetLocalSize(swarm, &nLocalFinal); CHKERRQ(ierr);
369 LOG_ALLOW(
LOCAL,
LOG_INFO,
"[Rank %d] Finished removing %d out-of-bounds particles. Final local size: %d.\n", rank, local_removed_count, nLocalFinal);
372 *removedCountLocal = local_removed_count;
373 if (removedCountGlobal) {
374 ierr = MPI_Allreduce(&local_removed_count, &global_removed_count_mpi, 1, MPI_INT, MPI_SUM, PetscObjectComm((PetscObject)swarm)); CHKERRQ(ierr);
375 *removedCountGlobal = global_removed_count_mpi;
380 PetscFunctionReturn(0);
416 PetscInt *removedCountLocal,
417 PetscInt *removedCountGlobal)
420 DM swarm = user->
swarm;
421 PetscInt nLocalInitial;
422 PetscInt *status_p = NULL;
423 PetscInt64 *pid_p = NULL;
424 PetscReal *pos_p = NULL;
425 PetscInt local_removed_count = 0;
426 PetscMPIInt global_removed_count_mpi = 0;
429 PetscFunctionBeginUser;
431 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
435 *removedCountLocal = 0;
436 if (removedCountGlobal) *removedCountGlobal = 0;
438 ierr = DMSwarmGetLocalSize(swarm, &nLocalInitial); CHKERRQ(ierr);
442 if (nLocalInitial > 0) {
444 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr);
445 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
446 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
449 for (PetscInt p = nLocalInitial - 1; p >= 0; p--) {
450 if (status_p[p] ==
LOST) {
451 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Removing LOST particle [PID %lld] at local index %d. Position: (%.4f, %.4f, %.4f).\n",
452 rank, (
long long)pid_p[p], p, pos_p[3*p], pos_p[3*p+1], pos_p[3*p+2]);
459 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr);
460 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
461 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
464 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
465 local_removed_count++;
468 PetscInt nLocalCurrent;
469 ierr = DMSwarmGetLocalSize(swarm, &nLocalCurrent); CHKERRQ(ierr);
471 if (nLocalCurrent > 0 && p > 0) {
472 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr);
473 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr);
474 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr);
490 if (status_p) { ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void **)&status_p); CHKERRQ(ierr); }
491 if (pid_p) { ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void **)&pid_p); CHKERRQ(ierr); }
492 if (pos_p) { ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void **)&pos_p); CHKERRQ(ierr); }
495 PetscInt nLocalFinal;
496 ierr = DMSwarmGetLocalSize(swarm, &nLocalFinal); CHKERRQ(ierr);
497 LOG_ALLOW(
LOCAL,
LOG_INFO,
"Rank %d: Finished removing %d LOST particles. Final local size: %d.\n", rank, local_removed_count, nLocalFinal);
500 *removedCountLocal = local_removed_count;
501 if (removedCountGlobal) {
502 ierr = MPI_Allreduce(&local_removed_count, &global_removed_count_mpi, 1, MPI_INT, MPI_SUM, PetscObjectComm((PetscObject)swarm)); CHKERRQ(ierr);
503 *removedCountGlobal = global_removed_count_mpi;
509 PetscFunctionReturn(0);
604 DM swarm = user->
swarm;
608 PetscInt *global_cell_id_arr;
609 PetscScalar ***count_arr_3d;
613 PetscInt particles_counted_locally = 0;
615 PetscFunctionBeginUser;
617 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
620 if (!da) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->da is NULL.");
621 if (!swarm) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->swarm is NULL.");
622 if (!countVec) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"UserCtx->ParticleCount is NULL.");
625 ierr = DMDAGetInfo(da, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &count_dof, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
626 if (count_dof != 1) { PetscSNPrintf(msg,
sizeof(msg),
"countDM must have DOF=1, got %d.", count_dof); SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, msg); }
629 ierr = VecSet(localcountVec, 0.0); CHKERRQ(ierr);
633 ierr = DMSwarmGetLocalSize(swarm, &nlocal); CHKERRQ(ierr);
634 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void **)&global_cell_id_arr); CHKERRQ(ierr);
635 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid",NULL,NULL,(
void **)&PID_arr);CHKERRQ(ierr);
639 ierr = DMDAVecGetArray(da, localcountVec, &count_arr_3d); CHKERRQ(ierr);
642 PetscInt gxs, gys, gzs, gxm, gym, gzm;
643 ierr = DMDAGetGhostCorners(da, &gxs, &gys, &gzs, &gxm, &gym, &gzm); CHKERRQ(ierr);
646 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"CalculateParticleCountPerCell (Rank %d): Processing %d local particles using GLOBAL CellIDs.\n",rank,nlocal);
647 for (p = 0; p < nlocal; p++) {
649 PetscInt i_geom = global_cell_id_arr[p * 3 + 0];
650 PetscInt j_geom = global_cell_id_arr[p * 3 + 1];
651 PetscInt k_geom = global_cell_id_arr[p * 3 + 2];
654 PetscInt i = (PetscInt)i_geom + 1;
655 PetscInt j = (PetscInt)j_geom + 1;
656 PetscInt k = (PetscInt)k_geom + 1;
663 LOG_LOOP_ALLOW(
LOCAL,
LOG_VERBOSE,p,100,
"[Rank %d] Read CellID for p=%d, PID = %ld: (%ld, %ld, %ld)\n", rank, p,PID_arr[p],i, j, k);
666 if (i >= gxs && i < gxs + gxm &&
667 j >= gys && j < gys + gym &&
668 k >= gzs && k < gzs + gzm )
673 count_arr_3d[k][j][i] += 1.0;
674 particles_counted_locally++;
678 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
"(Rank %d): Skipping particle %ld with global CellID (%ld, %ld, %ld) - likely outside local+ghost range.\n",rank, PID_arr[p] , i, j, k);
681 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"(Rank %d): Local counting finished. Processed %d particles locally.\n", rank, particles_counted_locally);
684 ierr = DMDAVecRestoreArray(da, localcountVec, &count_arr_3d); CHKERRQ(ierr);
685 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void **)&global_cell_id_arr); CHKERRQ(ierr);
686 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid",NULL,NULL,(
void **)&PID_arr);CHKERRQ(ierr);
690 ierr = VecZeroEntries(countVec); CHKERRQ(ierr);
691 ierr = DMLocalToGlobalBegin(da, localcountVec, ADD_VALUES, countVec); CHKERRQ(ierr);
692 ierr = DMLocalToGlobalEnd(da, localcountVec, ADD_VALUES, countVec); CHKERRQ(ierr);
703 PetscReal total_counted_particles = 0.0, max_count_in_cell = 0.0;
704 ierr = VecSum(countVec, &total_counted_particles); CHKERRQ(ierr);
705 PetscInt max_idx_global = -1;
706 ierr = VecMax(countVec, &max_idx_global, &max_count_in_cell); CHKERRQ(ierr);
708 total_counted_particles, max_count_in_cell);
711 if (max_idx_global >= 0) {
715 ierr = DMDAGetInfo(da, NULL, &M, &N, &P, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
718 PetscInt Kmax = max_idx_global / (M * N);
719 PetscInt Jmax = (max_idx_global % (M * N)) / M;
720 PetscInt Imax = max_idx_global % M;
722 (
int)Imax, (
int)Jmax, (
int)Kmax, (
int)max_idx_global);
725 PetscScalar count_at_origin = 0.0;
726 PetscScalar ***count_arr_for_check;
727 ierr = DMDAVecGetArrayRead(da, countVec, &count_arr_for_check); CHKERRQ(ierr);
729 PetscInt xs, ys, zs, xm, ym, zm;
730 ierr = DMDAGetCorners(da, &xs, &ys, &zs, &xm, &ym, &zm); CHKERRQ(ierr);
731 if (0 >= xs && 0 < xs+xm && 0 >= ys && 0 < ys+ym && 0 >= zs && 0 < zs+zm) {
732 count_at_origin = count_arr_for_check[0][0][0];
735 count_at_origin = -999.0;
737 ierr = DMDAVecRestoreArrayRead(da, countVec, &count_arr_for_check); CHKERRQ(ierr);
749 PetscFunctionReturn(0);
761 PetscInt N_current, nlocal_current;
765 PetscFunctionBeginUser;
767 ierr = PetscObjectGetComm((PetscObject)swarm, &comm); CHKERRQ(ierr);
768 ierr = MPI_Comm_rank(comm, &rank); CHKERRQ(ierr);
769 ierr = DMSwarmGetSize(swarm, &N_current); CHKERRQ(ierr);
770 ierr = DMSwarmGetLocalSize(swarm, &nlocal_current); CHKERRQ(ierr);
772 PetscInt delta = N_target - N_current;
776 PetscFunctionReturn(0);
780 PetscInt num_to_remove_global = -delta;
781 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Current size %d > target size %d. Removing %d particles globally.\n", N_current, N_target, num_to_remove_global);
791 ierr = MPI_Exscan(&nlocal_current, &rstart, 1, MPIU_INT, MPI_SUM, comm); CHKERRMPI(ierr);
793 rend = rstart + nlocal_current;
797 PetscInt nlocal_remove_count = 0;
798 if (rend > N_target) {
799 PetscInt start_remove_local_idx = (N_target > rstart) ? (N_target - rstart) : 0;
800 nlocal_remove_count = nlocal_current - start_remove_local_idx;
803 if (nlocal_remove_count < 0) nlocal_remove_count = 0;
804 if (nlocal_remove_count > nlocal_current) nlocal_remove_count = nlocal_current;
806 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"Rank %d: Global range [%d, %d). Target size %d. Need to remove %d local particles (from end).\n", rank, rstart, rend, N_target, nlocal_remove_count);
809 PetscInt removal_ops_done = 0;
810 for (PetscInt p = nlocal_current - 1; p >= 0 && removal_ops_done < nlocal_remove_count; --p) {
811 ierr = DMSwarmRemovePointAtIndex(swarm, p); CHKERRQ(ierr);
815 if (removal_ops_done != nlocal_remove_count) {
816 SETERRQ(comm, PETSC_ERR_PLIB,
"Rank %d: Failed to remove the expected number of local particles (%d != %d)", rank, removal_ops_done, nlocal_remove_count);
821 ierr = MPI_Barrier(comm); CHKERRMPI(ierr);
824 PetscInt num_to_add_global = delta;
825 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Current size %d < target size %d. Adding %d particles globally.\n", N_current, N_target, num_to_add_global);
826 ierr = DMSwarmAddNPoints(swarm, num_to_add_global); CHKERRQ(ierr);
832 ierr = DMSwarmGetSize(swarm, &N_final); CHKERRQ(ierr);
833 if (N_final != N_target) {
834 SETERRQ(comm, PETSC_ERR_PLIB,
"Failed to resize swarm: expected %d particles, got %d", N_target, N_final);
838 PetscFunctionReturn(0);
862 char filename[PETSC_MAX_PATH_LEN];
864 PetscInt N_current = 0;
867 const char *refFieldName =
"position";
868 const PetscInt bs = 3;
876 const int placeholder_int = 0;
881 PetscFunctionBeginUser;
883 ierr = PetscObjectGetComm((PetscObject)user->
swarm, &comm); CHKERRQ(ierr);
884 ierr = MPI_Comm_rank(comm, &rank); CHKERRQ(ierr);
892 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONGSTATE,
"Invalid execution mode for reading simulation fields.");
902 ierr = PetscSNPrintf(filename,
sizeof(filename),
"%s/%s%05" PetscInt_FMT
"_%d.%s",
909 PetscBool fileExists = PETSC_FALSE;
910 ierr = PetscTestFile(filename,
'r', &fileExists); CHKERRQ(ierr);
921 ierr = VecCreate(PETSC_COMM_SELF, &tmpVec); CHKERRQ(ierr);
922 ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF, filename, FILE_MODE_READ, &viewer); CHKERRQ(ierr);
923 ierr = VecLoad(tmpVec, viewer); CHKERRQ(ierr);
924 ierr = PetscViewerDestroy(&viewer); CHKERRQ(ierr);
926 ierr = VecGetSize(tmpVec, &vecSize); CHKERRQ(ierr);
927 ierr = VecDestroy(&tmpVec); CHKERRQ(ierr);
929 if (vecSize % bs != 0) {
931 LOG_ALLOW(
GLOBAL,
LOG_ERROR,
"Rank 0: Vector size %d from file '%s' is not divisible by block size %d.\n", vecSize, filename, bs);
933 N_file = vecSize / bs;
940 ierr = MPI_Bcast(&N_file, 1, MPIU_INT, 0, comm); CHKERRMPI(ierr);
944 SETERRQ(comm, PETSC_ERR_FILE_OPEN,
"Mandatory reference file '%s' not found for timestep %d (as determined by Rank 0).", filename, ti);
947 SETERRQ(comm, PETSC_ERR_FILE_READ,
"Reference file '%s' has incorrect format (as determined by Rank 0).", filename);
950 SETERRQ(comm, PETSC_ERR_PLIB,
"Received invalid particle count %d from Rank 0.", N_file);
955 ierr = DMSwarmGetSize(user->
swarm, &N_current); CHKERRQ(ierr);
957 if (N_file != N_current) {
958 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Swarm size %d differs from file size %d. Resizing swarm globally.\n", N_current, N_file);
968 PetscFunctionReturn(0);
992 DM swarm = user->
swarm;
993 PetscReal *positions_field = NULL;
994 PetscInt64 *particleIDs = NULL;
995 PetscInt *cell_ID_field = NULL;
996 const Cmpnts ***coor_nodes_local_array;
999 PetscInt xs_gnode_rank, ys_gnode_rank, zs_gnode_rank;
1000 PetscInt IM_nodes_global, JM_nodes_global, KM_nodes_global;
1002 PetscRandom rand_logic_reinit_i, rand_logic_reinit_j, rand_logic_reinit_k;
1003 PetscInt nlocal_current;
1004 PetscInt particles_actually_reinitialized_count = 0;
1005 PetscBool can_this_rank_service_inlet = PETSC_FALSE;
1007 PetscFunctionBeginUser;
1013 PetscFunctionReturn(0);
1016 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1017 ierr = DMSwarmGetLocalSize(swarm, &nlocal_current); CHKERRQ(ierr);
1020 if (nlocal_current == 0) {
1021 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[T=%.4f, Step=%d] Rank %d has no local particles to re-initialize on inlet.\n", currentTime, step, rank);
1022 PetscFunctionReturn(0);
1026 ierr = DMDAGetLocalInfo(user->
da, &info); CHKERRQ(ierr);
1027 ierr = DMDAGetInfo(user->
da, NULL, &IM_nodes_global, &JM_nodes_global, &KM_nodes_global, NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL); CHKERRQ(ierr);
1028 ierr = DMDAGetCorners(user->
da, &xs_gnode_rank, &ys_gnode_rank, &zs_gnode_rank, NULL, NULL, NULL); CHKERRQ(ierr);
1031 IM_nodes_global -= 1; JM_nodes_global -= 1; KM_nodes_global -= 1;
1033 const PetscInt IM_cells_global = IM_nodes_global > 0 ? IM_nodes_global - 1 : 0;
1034 const PetscInt JM_cells_global = JM_nodes_global > 0 ? JM_nodes_global - 1 : 0;
1035 const PetscInt KM_cells_global = KM_nodes_global > 0 ? KM_nodes_global - 1 : 0;
1040 ierr =
CanRankServiceInletFace(user, &info, IM_nodes_global, JM_nodes_global, KM_nodes_global, &can_this_rank_service_inlet); CHKERRQ(ierr);
1043 ierr = DMGetCoordinatesLocal(user->
da, &Coor_local); CHKERRQ(ierr);
1044 ierr = DMDAVecGetArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
1045 ierr = DMSwarmGetField(swarm,
"position", NULL, NULL, (
void**)&positions_field); CHKERRQ(ierr);
1046 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&particleIDs); CHKERRQ(ierr);
1047 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_ID_field); CHKERRQ(ierr);
1049 if (!can_this_rank_service_inlet) {
1053 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[T=%.4f, Step=%d] Rank %d is resetting %d local particles to inlet center (%.6f, %.6f, %.6f) for migration.\n", currentTime, step, rank, nlocal_current, user->
simCtx->
CMx_c, user->
simCtx->
CMy_c, user->
simCtx->
CMz_c);
1055 for(PetscInt p = 0; p < nlocal_current; p++){
1060 cell_ID_field[3*p+0] = -1;
1061 cell_ID_field[3*p+1] = -1;
1062 cell_ID_field[3*p+2] = -1;
1066 ierr = DMDAVecRestoreArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
1067 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void**)&positions_field); CHKERRQ(ierr);
1068 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&particleIDs); CHKERRQ(ierr);
1069 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_ID_field); CHKERRQ(ierr);
1071 PetscFunctionReturn(0);
1082 for (PetscInt p = 0; p < nlocal_current; p++) {
1083 PetscInt ci_metric_lnode, cj_metric_lnode, ck_metric_lnode;
1084 PetscReal xi_metric_logic, eta_metric_logic, zta_metric_logic;
1085 Cmpnts phys_coords = {0.0,0.0,0.0};
1086 PetscBool particle_was_placed = PETSC_FALSE;
1091 IM_nodes_global, JM_nodes_global, KM_nodes_global,
1092 &rand_logic_reinit_i, &rand_logic_reinit_j, &rand_logic_reinit_k,
1093 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
1094 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic); CHKERRQ(ierr);
1099 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1100 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1101 &phys_coords); CHKERRQ(ierr);
1104 positions_field[3*p+0] = phys_coords.
x;
1105 positions_field[3*p+1] = phys_coords.
y;
1106 positions_field[3*p+2] = phys_coords.
z;
1107 particle_was_placed = PETSC_TRUE;
1110 PetscBool placement_flag = PETSC_FALSE;
1112 IM_cells_global, JM_cells_global, KM_cells_global,
1114 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
1115 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic,&placement_flag); CHKERRQ(ierr);
1121 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1122 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1123 &phys_coords); CHKERRQ(ierr);
1126 positions_field[3*p+0] = phys_coords.
x;
1127 positions_field[3*p+1] = phys_coords.
y;
1128 positions_field[3*p+2] = phys_coords.
z;
1129 particle_was_placed = PETSC_TRUE;
1133 LOG_ALLOW(
GLOBAL,
LOG_WARNING,
"Rank %d: Particle PID %ld deterministic placement failed (belongs to different rank). Falling back to random placement.\n", rank, particleIDs[p]);
1136 IM_nodes_global, JM_nodes_global, KM_nodes_global,
1137 &rand_logic_reinit_i, &rand_logic_reinit_j, &rand_logic_reinit_k,
1138 &ci_metric_lnode, &cj_metric_lnode, &ck_metric_lnode,
1139 &xi_metric_logic, &eta_metric_logic, &zta_metric_logic); CHKERRQ(ierr);
1143 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1144 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1145 &phys_coords); CHKERRQ(ierr);
1148 positions_field[3*p+0] = phys_coords.
x;
1149 positions_field[3*p+1] = phys_coords.
y;
1150 positions_field[3*p+2] = phys_coords.
z;
1151 particle_was_placed = PETSC_TRUE;
1155 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE,
"ReinitializeParticlesOnInletSurface only supports ParticleInitialization modes 0 and 3.");
1158 if(particle_was_placed){
1159 particles_actually_reinitialized_count++;
1161 cell_ID_field[3*p+0] = -1;
1162 cell_ID_field[3*p+1] = -1;
1163 cell_ID_field[3*p+2] = -1;
1166 "Rank %d: PID %ld (idx %ld) RE-PLACED. CellOriginNode(locDAIdx):(%d,%d,%d). LogicCoords: (%.2e,%.2f,%.2f). PhysCoords: (%.6f,%.6f,%.6f).\n",
1167 rank, particleIDs[p], (
long)p,
1168 ci_metric_lnode, cj_metric_lnode, ck_metric_lnode,
1169 xi_metric_logic, eta_metric_logic, zta_metric_logic,
1170 phys_coords.
x, phys_coords.
y, phys_coords.
z);
1175 if (particles_actually_reinitialized_count > 0) {
1176 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"[T=%.4f, Step=%d] Rank %d (on inlet face %d) successfully re-initialized %d of %d local particles.\n", currentTime, step, rank, user->
identifiedInletBCFace, particles_actually_reinitialized_count, nlocal_current);
1177 }
else if (nlocal_current > 0) {
1178 LOG_ALLOW(
GLOBAL,
LOG_WARNING,
"[T=%.4f, Step=%d] Rank %d claimed to service inlet face %d, but re-initialized 0 of %d local particles. This may indicate an issue if particles were expected to be re-placed.\n", currentTime, step, rank, user->
identifiedInletBCFace, nlocal_current);
1182 ierr = PetscRandomDestroy(&rand_logic_reinit_i); CHKERRQ(ierr);
1183 ierr = PetscRandomDestroy(&rand_logic_reinit_j); CHKERRQ(ierr);
1184 ierr = PetscRandomDestroy(&rand_logic_reinit_k); CHKERRQ(ierr);
1186 ierr = DMSwarmRestoreField(swarm,
"position", NULL, NULL, (
void**)&positions_field); CHKERRQ(ierr);
1187 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&particleIDs); CHKERRQ(ierr);
1188 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_ID_field); CHKERRQ(ierr);
1189 ierr = DMDAVecRestoreArrayRead(user->
fda, Coor_local, (
void*)&coor_nodes_local_array); CHKERRQ(ierr);
1193 PetscFunctionReturn(0);
1383 PetscInt n_local_before,
1384 const PetscInt64 pids_before[])
1386 PetscErrorCode ierr;
1388 PetscInt n_local_after;
1389 PetscInt newcomer_count = 0;
1392 PetscInt64 *pid_field_after = NULL;
1393 PetscInt *status_field_after = NULL;
1394 PetscInt *cell_field_after = NULL;
1396 PetscFunctionBeginUser;
1402 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Input DMSwarm is NULL in FlagNewcomersForLocation.");
1405 if (n_local_before > 0 && !pids_before) {
1406 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_NULL,
"Input pids_before array is NULL for n_local_before > 0.");
1409 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1412 ierr = DMSwarmGetLocalSize(swarm, &n_local_after); CHKERRQ(ierr);
1415 rank, n_local_before, n_local_after);
1418 if (n_local_after == 0) {
1419 PetscFunctionReturn(0);
1424 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_field_after); CHKERRQ(ierr);
1425 ierr = DMSwarmGetField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_field_after); CHKERRQ(ierr);
1426 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_field_after); CHKERRQ(ierr);
1427 if (!pid_field_after || !status_field_after) {
1428 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE,
"Failed to get required swarm fields in FlagNewcomersForLocation.");
1433 for (PetscInt p_idx = 0; p_idx < n_local_after; ++p_idx) {
1434 PetscInt64 current_pid = pid_field_after[p_idx];
1435 PetscBool is_found_in_before_list;
1438 ierr =
BinarySearchInt64(n_local_before, pids_before, current_pid, &is_found_in_before_list); CHKERRQ(ierr);
1441 if (!is_found_in_before_list) {
1450 rank, current_pid, p_idx);
1456 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_field_after); CHKERRQ(ierr);
1457 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_field_after); CHKERRQ(ierr);
1458 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_field_after); CHKERRQ(ierr);
1460 if (newcomer_count > 0) {
1466 PetscFunctionReturn(0);
1495 PetscErrorCode ierr;
1496 DM swarm = user->
swarm;
1498 PetscInt *cell_p = NULL;
1499 PetscInt64 *pid_p = NULL;
1503 PetscInt local_migration_count = 0;
1504 PetscInt migrationListCapacity = 0;
1505 PetscInt global_migration_count = 0;
1507 PetscFunctionBeginUser;
1509 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1511 ierr = DMSwarmGetLocalSize(swarm, &nlocal); CHKERRQ(ierr);
1515 ierr = DMSwarmGetField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1516 ierr = DMSwarmGetField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1521 for (PetscInt p_idx = 0; p_idx < nlocal; ++p_idx) {
1522 PetscInt ci = cell_p[3*p_idx + 0];
1523 PetscInt cj = cell_p[3*p_idx + 1];
1524 PetscInt ck = cell_p[3*p_idx + 2];
1527 if (ci < 0 || cj < 0 || ck < 0) {
1531 PetscMPIInt owner_rank;
1534 if (owner_rank != -1 && owner_rank != rank) {
1536 ierr =
AddToMigrationList(&migrationList, &migrationListCapacity, &local_migration_count,
1537 p_idx, owner_rank); CHKERRQ(ierr);
1540 (
long)pid_p[p_idx], ci, cj, ck, owner_rank, rank);
1544 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1545 ierr = DMSwarmRestoreField(swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1549 ierr = MPI_Allreduce(&local_migration_count, &global_migration_count, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD); CHKERRQ(ierr);
1551 if (global_migration_count > 0) {
1552 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Fast restart migration: Directly migrating %d particles using CellIDs.\n", global_migration_count);
1553 ierr =
SetMigrationRanks(user, migrationList, local_migration_count); CHKERRQ(ierr);
1561 ierr = PetscFree(migrationList); CHKERRQ(ierr);
1564 PetscFunctionReturn(0);
1739 PetscErrorCode ierr;
1740 PetscInt passes = 0;
1741 const PetscInt MAX_MIGRATION_PASSES = 50;
1742 PetscInt global_migrations_this_pass;
1744 PetscInt total_migrated_this_timestep = 0;
1746 PetscFunctionBeginUser;
1748 ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
1749 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"LocateAllParticlesInGrid (Orchestrator) - Beginning particle settlement process.\n");
1759 PetscInt local_migration_count = 0;
1760 PetscInt migrationListCapacity = 0;
1761 PetscInt nlocal_before;
1762 PetscInt64 *pids_before_snapshot = NULL;
1763 PetscInt local_lost_count = 0;
1765 ierr = DMSwarmGetLocalSize(user->
swarm, &nlocal_before); CHKERRQ(ierr);
1766 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[Rank %d] Pass %d begins with %d local particles.\n", rank, passes, nlocal_before);
1770 if (nlocal_before > 0) {
1772 PetscReal *pos_p, *weights_p, *vel_p;
1773 PetscInt *cell_p, *status_p;
1775 ierr = DMSwarmGetField(user->
swarm,
"position", NULL, NULL, (
void**)&pos_p); CHKERRQ(ierr);
1776 ierr = DMSwarmGetField(user->
swarm,
"velocity", NULL, NULL, (
void**)&vel_p); CHKERRQ(ierr);
1777 ierr = DMSwarmGetField(user->
swarm,
"weight", NULL, NULL, (
void**)&weights_p); CHKERRQ(ierr);
1778 ierr = DMSwarmGetField(user->
swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1779 ierr = DMSwarmGetField(user->
swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1780 ierr = DMSwarmGetField(user->
swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_p); CHKERRQ(ierr);
1786 for (PetscInt p_idx = 0; p_idx < nlocal_before; p_idx++) {
1791 "Local Particle idx=%d, PID=%ld, status=%s, cell=(%d, %d, %d)\n",
1800 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
" [rank %d][PID %ld] skipped in pass %d as it is already located at (%d,%d,%d).\n",rank,pid_p[p_idx],passes,cell_p[3*p_idx],cell_p[3*p_idx + 1],cell_p[3*p_idx + 2]);
1809 ierr =
UnpackSwarmFields(p_idx, pid_p, weights_p, pos_p, cell_p, vel_p, status_p,NULL,NULL,NULL,¤t_particle); CHKERRQ(ierr);
1818 if (current_particle.
cell[0] >= 0) {
1861 PetscMPIInt guessed_owner_rank = MPI_PROC_NULL;
1865 if (guessed_owner_rank != MPI_PROC_NULL && guessed_owner_rank != rank) {
1866 LOG_ALLOW(
LOCAL,
LOG_VERBOSE,
"[PID %ld] Guess SUCCESS: Found migration target Rank %d. Finalizing.\n", current_particle.
PID, guessed_owner_rank);
1874 if (guessed_owner_rank == rank) {
1875 LOG_ALLOW(
LOCAL,
LOG_DEBUG,
"[PID %ld] Guess determined particle is local. Proceeding to robust walk to find cell.\n", current_particle.
PID);
1877 LOG_ALLOW(
LOCAL,
LOG_WARNING,
"[PID %ld] Guess FAILED to find an owner. Proceeding to robust walk for definitive search.\n", current_particle.
PID);
1886 ierr =
UpdateSwarmFields(p_idx, ¤t_particle, pos_p, vel_p, weights_p, cell_p, status_p,NULL,NULL,NULL); CHKERRQ(ierr);
1890 }
else if (final_status ==
LOST) {
1897 ierr = DMSwarmRestoreField(user->
swarm,
"position", NULL, NULL, (
void**)&pos_p); CHKERRQ(ierr);
1898 ierr = DMSwarmRestoreField(user->
swarm,
"velocity", NULL, NULL, (
void**)&vel_p); CHKERRQ(ierr);
1899 ierr = DMSwarmRestoreField(user->
swarm,
"weight", NULL, NULL, (
void**)&weights_p); CHKERRQ(ierr);
1900 ierr = DMSwarmRestoreField(user->
swarm,
"DMSwarm_CellID", NULL, NULL, (
void**)&cell_p); CHKERRQ(ierr);
1901 ierr = DMSwarmRestoreField(user->
swarm,
"DMSwarm_pid", NULL, NULL, (
void**)&pid_p); CHKERRQ(ierr);
1902 ierr = DMSwarmRestoreField(user->
swarm,
"DMSwarm_location_status", NULL, NULL, (
void**)&status_p); CHKERRQ(ierr);
1906 LOG_ALLOW(
LOCAL,
LOG_INFO,
"[Rank %d] Pass %d: Identified %d particles to migrate out.\n", rank, passes, local_migration_count);
1911 ierr = MPI_Allreduce(&local_migration_count, &global_migrations_this_pass, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD); CHKERRQ(ierr);
1913 total_migrated_this_timestep += global_migrations_this_pass;
1915 if(global_migrations_this_pass > 0 ){
1917 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"Pass %d: Migrating %d particles globally.\n", passes, global_migrations_this_pass);
1919 ierr =
SetMigrationRanks(user, migrationList, local_migration_count); CHKERRQ(ierr);
1929 ierr = PetscFree(pids_before_snapshot);
1930 ierr = PetscFree(migrationList);
1932 LOG_ALLOW(
GLOBAL,
LOG_INFO,
"End of pass %d. Total particles migrated globally: %d.\n", passes, global_migrations_this_pass);
1934 }
while (global_migrations_this_pass > 0 && passes < MAX_MIGRATION_PASSES);
1937 if (passes >= MAX_MIGRATION_PASSES) {
1938 SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_CONV_FAILED,
"Particle migration failed to converge after %d passes. Check for particles oscillating between ranks.", MAX_MIGRATION_PASSES);
1947 PetscFunctionReturn(0);
PetscErrorCode UnpackSwarmFields(PetscInt i, const PetscInt64 *PIDs, const PetscReal *weights, const PetscReal *positions, const PetscInt *cellIndices, PetscReal *velocities, PetscInt *LocStatus, PetscReal *diffusivity, Cmpnts *diffusivitygradient, PetscReal *psi, Particle *particle)
Initializes a Particle struct with data from DMSwarm fields.