diff --git a/src/Patch/Patch.cpp b/src/Patch/Patch.cpp index 621eb0e93..f648de86d 100644 --- a/src/Patch/Patch.cpp +++ b/src/Patch/Patch.cpp @@ -1,21 +1,21 @@ -// CLOBAL COORDINATES: +// GLOBAL COORDINATES: // Patch_minGlobal Patch_maxGlobal -// --------<===================================== gs ===================================>------------ +// --------<==========================================================================>------------ // GLOBAL INDICES: 0 . nspace_global // ix+oversize ix+oversize -// ------------------------------------ . ------------------------------------ -// | | | ... | | | . | | | | ... | | | | -// | | | ... | | | . | | | | ... | | | | -// ------------------------------------ . ------------------------------------ +// ------------------------------------ . ------------------------------- +// | | | ... | | | . | | | ... | | | +// | | | ... | | | . | | | ... | | | +// ------------------------------------ . ------------------------------- // Patch_minLocal Patch_maxLocal . Patch_minLocal Patch_maxLocal // ---------------------------------------- // | | | . | | | // | | | . | | | // ---------------------------------------- -// LOCAL COORDINATES: x(0) rlb x(ix) rub x(nspace) +// LOCAL COORDINATES: x(0) x(ix) x(nspace) // ----<============= length =========>---- -// LOCAL INDICES: 0 lb ub nspace +// LOCAL INDICES: 0 nspace #include "Patch.h" @@ -51,8 +51,6 @@ Patch::Patch(Params& params, SmileiMPI* smpi, unsigned int ipatch, unsigned int } // END Patch::Patch - - // Cloning patch constructor Patch::Patch(Patch* patch, Params& params, SmileiMPI* smpi, unsigned int ipatch, unsigned int n_moved, bool with_particles = true) { @@ -66,15 +64,8 @@ Patch::Patch(Patch* patch, Params& params, SmileiMPI* smpi, unsigned int ipatch, void Patch::initStep1(Params& params) { - // for nDim_fields = 1 : bug if Pcoordinates.size = 1 !! - //Pcoordinates.resize(nDim_fields_); Pcoordinates.resize( 2 ); - // else if ( params.geometry == "2d3v" ) { - // Pcoordinates.resize(3); - // generalhilbertindexinv(params.mi[0], params.mi[1], params.mi[2], &Pcoordinates[0], &Pcoordinates[1], &Pcoordinates[2], hindex); - // } - nbNeighbors_ = 2; neighbor_.resize(nDim_fields_); corner_neighbor_.resize(params.nDim_field); @@ -197,20 +188,6 @@ void Patch::updateMPIenv(SmileiMPI* smpi) for (int iDim = 0 ; iDim < nDim_fields_ ; iDim++) for (int iNeighbor=0 ; iNeighborhrank(neighbor_[iDim][iNeighbor]); - -// cout << "\n\tPatch Corner decomp : " << corner_neighbor_[0][1] << "\t" << neighbor_[1][1] << "\t" << corner_neighbor_[1][1] << endl; -// cout << "\tPatch Corner decomp : " << neighbor_[0][0] << "\t" << hindex << "\t" << neighbor_[0][1] << endl; -// cout << "\tPatch Corner decomp : " << corner_neighbor_[0][0] << "\t" << neighbor_[1][0] << "\t" << corner_neighbor_[1][0] << endl; -// - -// cout << "\n\tMPI Corner decomp : " << "MPI_PROC_NULL" << "\t" << MPI_neighbor_[2][1] << "\t" << "MPI_PROC_NULL" << endl << endl; -// -// cout << "\n\tMPI Corner decomp : " << "MPI_PROC_NULL" << "\t" << MPI_neighbor_[1][1] << "\t" << "MPI_PROC_NULL" << endl; -// cout << "\tMPI Corner decomp : " << MPI_neighbor_[0][0] << "\t" << smpi->getRank() << "\t" << MPI_neighbor_[0][1] << endl; -// cout << "\tMPI Corner decomp : " << "MPI_PROC_NULL" << "\t" << MPI_neighbor_[1][0] << "\t" << "MPI_PROC_NULL" << endl << endl; -// -// cout << "\n\tMPI Corner decomp : " << "MPI_PROC_NULL" << "\t" << MPI_neighbor_[2][0] << "\t" << "MPI_PROC_NULL" << endl; - for (int iDim=0 ; iDim< (int)neighbor_.size() ; iDim++) for (int iNeighbor=0 ; iNeighbor<2 ; iNeighbor++) { @@ -223,7 +200,6 @@ void Patch::updateMPIenv(SmileiMPI* smpi) // --------------------------------------------------------------------------------------------------------------------- // Split particles Id to send in per direction and per patch neighbor dedicated buffers -// Apply periodicity if necessary // --------------------------------------------------------------------------------------------------------------------- void Patch::initExchParticles(SmileiMPI* smpi, int ispec, Params& params) { @@ -231,7 +207,6 @@ void Patch::initExchParticles(SmileiMPI* smpi, int ispec, Params& params) int ndim = params.nDim_field; int idim,check; std::vector* indexes_of_particles_to_exchange = &vecSpecies[ispec]->indexes_of_particles_to_exchange; -// double xmax[3]; for (int iDim=0 ; iDim < ndim ; iDim++){ for (int iNeighbor=0 ; iNeighborMPIbuff.partSend[iDim][iNeighbor].initialize(0,cuParticles); vecSpecies[ispec]->MPIbuff.part_index_send[iDim][iNeighbor].resize(0); vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][iNeighbor] = 0; + vecSpecies[ispec]->MPIbuff.part_index_send_sz[iDim][iNeighbor] = 0; } } @@ -270,8 +246,7 @@ void Patch::initExchParticles(SmileiMPI* smpi, int ispec, Params& params) idim++; } } - - + } // initExchParticles(... iDim) @@ -289,7 +264,7 @@ void Patch::initCommParticles(SmileiMPI* smpi, int ispec, Params& params, int iD /********************************************************************************/ for (int iNeighbor=0 ; iNeighborMPIbuff.part_index_send_sz[iDim][iNeighbor] = (vecSpecies[ispec]->MPIbuff.part_index_send[iDim][iNeighbor]).size(); + vecSpecies[ispec]->MPIbuff.part_index_send_sz[iDim][iNeighbor] += (vecSpecies[ispec]->MPIbuff.part_index_send[iDim][iNeighbor]).size(); if (is_a_MPI_neighbor(iDim, iNeighbor)) { //If neighbour is MPI ==> I send him the number of particles I'll send later. @@ -324,9 +299,9 @@ void Patch::CommParticles(SmileiMPI* smpi, int ispec, Params& params, int iDim, MPI_Datatype typePartSend, typePartRecv; Particles &cuParticles = (*vecSpecies[ispec]->particles); - int n_part_send, n_part_recv; + int n_part_send; int h0 = (*vecPatch)(0)->hindex; - double x_max = params.cell_length[iDim]*( params.n_space_global[iDim] ); + double x_max = params.sim_length[iDim]; /********************************************************************************/ // Wait for end of communications over number of particles @@ -352,13 +327,11 @@ void Patch::CommParticles(SmileiMPI* smpi, int ispec, Params& params, int iDim, // Proceed to effective Particles' communications /********************************************************************************/ - // Number of properties per particles = nDim_Particles + 3 + 1 + 1 - for (int iNeighbor=0 ; iNeighborMPIbuff.part_index_send[iDim][iNeighbor]).size(); - if ( (neighbor_[iDim][iNeighbor]!=MPI_PROC_NULL) && (n_part_send!=0) ) { + + //If I have to send particles + if ( vecSpecies[ispec]->MPIbuff.part_index_send_sz[iDim][iNeighbor] != 0) { + n_part_send = (vecSpecies[ispec]->MPIbuff.part_index_send[iDim][iNeighbor]).size(); // Enabled periodicity if (smpi->periods_[iDim]==1) { for (int iPart=0 ; iPartMPIbuff.part_index_send[iDim][iNeighbor][iPart], vecSpecies[ispec]->MPIbuff.partSend[iDim][iNeighbor]); - // Then send particles + // ... then send particles int tag = buildtag( hindex, iDim+1, iNeighbor+3 ); typePartSend = smpi->createMPIparticles( &(vecSpecies[ispec]->MPIbuff.partSend[iDim][iNeighbor]) ); MPI_Isend( &((vecSpecies[ispec]->MPIbuff.partSend[iDim][iNeighbor]).position(0,0)), 1, typePartSend, MPI_neighbor_[iDim][iNeighbor], tag, MPI_COMM_WORLD, &(vecSpecies[ispec]->MPIbuff.srequest[iDim][iNeighbor]) ); MPI_Type_free( &typePartSend ); } else { - //If not MPI comm, copy particles directly in the receive buffer + //If (this is not an MPI comm): copy particles directly in the receive buffer for (int iPart=0 ; iPartMPIbuff.part_index_send[iDim][iNeighbor][iPart],((*vecPatch)( neighbor_[iDim][iNeighbor]- h0 )->vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]) ); } } // END of Send - n_part_recv = vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][(iNeighbor+1)%2]; - if ( (neighbor_[iDim][(iNeighbor+1)%2]!=MPI_PROC_NULL) && (n_part_recv!=0) ) { - if (is_a_MPI_neighbor(iDim, (iNeighbor+1)%2)) { - // If MPI comm, receive particles in the recv buffer previously initialized. + // If (# of particles received not 0) and (this is an MPI comm): receive particles in the recv buffer previously initialized. + if ( vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][(iNeighbor+1)%2] != 0 && is_a_MPI_neighbor(iDim, (iNeighbor+1)%2) ) { typePartRecv = smpi->createMPIparticles( &(vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]) ); int tag = buildtag( neighbor_[iDim][(iNeighbor+1)%2], iDim+1 ,iNeighbor+3 ); MPI_Irecv( &((vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]).position(0,0)), 1, typePartRecv, MPI_neighbor_[iDim][(iNeighbor+1)%2], tag, MPI_COMM_WORLD, &(vecSpecies[ispec]->MPIbuff.rrequest[iDim][(iNeighbor+1)%2]) ); MPI_Type_free( &typePartRecv ); - } - } // END of Recv } // END for iNeighbor @@ -417,6 +386,7 @@ void Patch::finalizeCommParticles(SmileiMPI* smpi, int ispec, Params& params, in int ndim = params.nDim_field; int idim, check; + int h0 = (*vecPatch)(0)->hindex; Particles &cuParticles = (*vecSpecies[ispec]->particles); @@ -443,18 +413,16 @@ void Patch::finalizeCommParticles(SmileiMPI* smpi, int ispec, Params& params, in MPI_Status sstat [2]; MPI_Status rstat [2]; - n_part_send = vecSpecies[ispec]->MPIbuff.part_index_send[iDim][iNeighbor].size(); - n_part_recv = vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][(iNeighbor+1)%2]; + n_part_send = vecSpecies[ispec]->MPIbuff.part_index_send_sz[iDim][(iNeighbor+1)%2]; + n_part_recv = vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][iNeighbor]; + if ( n_part_send!=0 && is_a_MPI_neighbor(iDim, (iNeighbor+1)%2) ) + MPI_Wait( &(vecSpecies[ispec]->MPIbuff.srequest[iDim][(iNeighbor+1)%2]), &(sstat[(iNeighbor+1)%2]) ); - - if ( (neighbor_[iDim][iNeighbor]!=MPI_PROC_NULL) && (n_part_send!=0) ) { + + if ( n_part_recv!=0 ) { if (is_a_MPI_neighbor(iDim, iNeighbor)) - MPI_Wait( &(vecSpecies[ispec]->MPIbuff.srequest[iDim][iNeighbor]), &(sstat[iNeighbor]) ); - } - if ( (neighbor_[iDim][(iNeighbor+1)%2]!=MPI_PROC_NULL) && (n_part_recv!=0) ) { - if (is_a_MPI_neighbor(iDim, (iNeighbor+1)%2)) - MPI_Wait( &(vecSpecies[ispec]->MPIbuff.rrequest[iDim][(iNeighbor+1)%2]), &(rstat[(iNeighbor+1)%2]) ); + MPI_Wait( &(vecSpecies[ispec]->MPIbuff.rrequest[iDim][iNeighbor]), &(rstat[iNeighbor]) ); // Treat diagonalParticles if (iDim < ndim-1){ // No need to treat diag particles at last dimension. @@ -463,39 +431,51 @@ void Patch::finalizeCommParticles(SmileiMPI* smpi, int ispec, Params& params, in idim = iDim+1;//We check next dimension while (check == 0 && idimMPIbuff.partRecv[iDim][(iNeighbor+1)%2]).position(idim,iPart) < min_local[idim] ){ - if (neighbor_[idim][0]!=MPI_PROC_NULL){ //if neighbour exists - //... copy it at the back of the local particle vector ... - (vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]).cp_particle(iPart, cuParticles); - //...adjust bmax ... - (*cubmax)[(*cubmax).size()-1]++; - //... and add its index to the particles to be sent later... - vecSpecies[ispec]->MPIbuff.part_index_send[idim][0].push_back( cuParticles.size()-1 ); - //..without forgeting to add it to the list of particles to clean. - vecSpecies[ispec]->addPartInExchList(cuParticles.size()-1); + if ( (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).position(idim,iPart) < min_local[idim] ){ + // Enabled periodicity + if ( smpi->periods_[idim]==1) + if ( Pcoordinates[idim] == 0 ) + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).position(idim,iPart) += params.sim_length[idim] ; + //Move particle + if (is_a_MPI_neighbor(idim, 0)) { + //... copy it at the back of the send particle vector ... + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).cp_particle(iPart, vecSpecies[ispec]->MPIbuff.partSend[idim][0]); + } else { + //Copy directly in the receive buffer of the next dimension + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).cp_particle( iPart, ((*vecPatch)( neighbor_[idim][0]- h0 )->vecSpecies[ispec]->MPIbuff.partRecv[idim][1]) ); } + // Adjust send size + vecSpecies[ispec]->MPIbuff.part_index_send_sz[idim][0] ++; //Remove it from receive buffer. - (vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]).erase_particle(iPart); - vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][(iNeighbor+1)%2]--; + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).erase_particle(iPart); + vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][iNeighbor]--; check = 1; } //Other side of idim - else if ( (vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]).position(idim,iPart) >= max_local[idim]) { - if (neighbor_[idim][1]!=MPI_PROC_NULL){ //if neighbour exists - (vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]).cp_particle(iPart, cuParticles); - (*cubmax)[(*cubmax).size()-1]++; - vecSpecies[ispec]->MPIbuff.part_index_send[idim][1].push_back( cuParticles.size()-1 ); - vecSpecies[ispec]->addPartInExchList(cuParticles.size()-1); + else if ( (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).position(idim,iPart) >= max_local[idim]) { + // Enabled periodicity + if ( smpi->periods_[idim]==1) + if ( Pcoordinates[idim] == params.number_of_patches[idim]-1 ) + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).position(idim,iPart) -= params.sim_length[idim] ; + //Move particle + if (is_a_MPI_neighbor(idim, 1)) { + //... copy it at the back of the send particle vector ... + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).cp_particle(iPart, vecSpecies[ispec]->MPIbuff.partSend[idim][1]); + } else { + //Copy directly in the receive buffer of the next dimension + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).cp_particle( iPart, ((*vecPatch)( neighbor_[idim][1]- h0 )->vecSpecies[ispec]->MPIbuff.partRecv[idim][0]) ); } - (vecSpecies[ispec]->MPIbuff.partRecv[iDim][(iNeighbor+1)%2]).erase_particle(iPart); - vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][(iNeighbor+1)%2]--; + // Adjust send size + vecSpecies[ispec]->MPIbuff.part_index_send_sz[idim][1] ++; + //Remove it from receive buffer. + (vecSpecies[ispec]->MPIbuff.partRecv[iDim][iNeighbor]).erase_particle(iPart); + vecSpecies[ispec]->MPIbuff.part_index_recv_sz[iDim][iNeighbor]--; check = 1; } idim++; } } }//If not last dim for diagonal particles. - } //If received something } //loop i Neighbor @@ -533,11 +513,8 @@ void Patch::finalizeCommParticles(SmileiMPI* smpi, int ispec, Params& params, in shift[j]+=shift[j-1]; } //Make room for new particles - if (shift[(*cubmax).size()]) { - //! vecor::resize of Charge crashed ! Temporay solution : push_back / Particle - //cuParticles.initialize( cuParticles.size()+shift[(*cubmax).size()], cuParticles.Position.size() ); - for (int inewpart=0 ; inewpart=1; j--){ @@ -561,22 +538,19 @@ void Patch::finalizeCommParticles(SmileiMPI* smpi, int ispec, Params& params, in } //idim > 0; this is the difficult case, when particles can arrive in any bin. for (idim = 1; idim < ndim; idim++){ - //if (idim!=iDim) continue; for (int iNeighbor=0 ; iNeighborMPIbuff.part_index_recv_sz[idim][iNeighbor]; if ( (neighbor_[idim][iNeighbor]!=MPI_PROC_NULL) && (n_part_recv!=0) ) { - for(unsigned int j=0; j<(unsigned int)n_part_recv; j++){ - ii = int((vecSpecies[ispec]->MPIbuff.partRecv[idim][iNeighbor].position(0,j)-min_local[0])/dbin);//bin in which the particle goes. - vecSpecies[ispec]->MPIbuff.partRecv[idim][iNeighbor].overwrite_part(j, cuParticles,(*cubmax)[ii]); - (*cubmax)[ii] ++ ; - } + for(unsigned int j=0; j<(unsigned int)n_part_recv; j++){ + ii = int((vecSpecies[ispec]->MPIbuff.partRecv[idim][iNeighbor].position(0,j)-min_local[0])/dbin);//bin in which the particle goes. + vecSpecies[ispec]->MPIbuff.partRecv[idim][iNeighbor].overwrite_part(j, cuParticles,(*cubmax)[ii]); + (*cubmax)[ii] ++ ; + } } } } - }//End Recv_buffers ==> particles - } // finalizeCommParticles(... iDim) @@ -636,7 +610,7 @@ void Patch::cleanup_sent_particles(int ispec, std::vector* indexes_of_parti ii--; iPart = (*indexes_of_particles_to_exchange)[ii]; } - if (iPart >= (*cubmin)[ibin] && iPart < (*cubmax)[ibin]) { //On traite la dernière particule (qui peut aussi etre la premiere) + if (iPart >= (*cubmin)[ibin] && iPart < (*cubmax)[ibin]) { //Treat last particle (which can also be the first if there is only 1) cuParticles.overwrite_part((*cubmax)[ibin]-1, iPart ); (*cubmax)[ibin]--; }