Skip to content

Commit

Permalink
Merge pull request trilinos#13538 from iyamazaki/basker-vovs
Browse files Browse the repository at this point in the history 
ShyLU - Basker : add SequentialHostInit to View-of-Views
  • Loading branch information
@iyamazaki
iyamazaki authored Oct 30, 2024
2 parents c19339e + 218676e commit 6f6cb99
Show file tree
Hide file tree
Showing 19 changed files with 424 additions and 390 deletions.
7 changes: 0 additions & 7 deletions packages/shylu/shylu_node/basker/src/shylubasker_decl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1494,13 +1494,6 @@ namespace BaskerNS

//end NDE


//RHS and solutions (These are not used anymore)
ENTRY_2DARRAY rhs;
ENTRY_2DARRAY sol;
Int nrhs;


BASKER_TREE part_tree;
BASKER_TREE tree;
BASKER_SYMBOLIC_TREE stree;
Expand Down
45 changes: 24 additions & 21 deletions packages/shylu/shylu_node/basker/src/shylubasker_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,8 @@ namespace BaskerNS
BASKER_INLINE
void Basker<Int,Entry,Exe_Space>::Finalize()
{
//finalize all matrices
A.Finalize();
At.Finalize(); //??? is At even used
BTF_A.Finalize();
BTF_C.Finalize();
BTF_B.Finalize();
BTF_D.Finalize();
BTF_E.Finalize();

//finalize array of 2d matrics
// Actuall Finalize is called by desctructor
FREE_MATRIX_2DARRAY(AVM, tree.nblks);
FREE_MATRIX_2DARRAY(ALM, tree.nblks);

Expand All @@ -120,7 +112,6 @@ namespace BaskerNS

//Thread Array
FREE_THREAD_1DARRAY(thread_array);
basker_barrier.Finalize();

//S (Check on this)
FREE_INT_2DARRAY(S, tree.nblks);
Expand Down Expand Up @@ -187,12 +178,6 @@ namespace BaskerNS

FREE_ENTRY_1DARRAY(x_view_ptr_scale);
FREE_ENTRY_1DARRAY(y_view_ptr_scale);

//Structures
part_tree.Finalize();
tree.Finalize();
stree.Finalize();
stats.Finalize();
}//end Finalize()


Expand Down Expand Up @@ -239,7 +224,7 @@ namespace BaskerNS
//Option = 2, BTF BASKER

if(option == 1)
{
{
default_order();
}
else if(option == 2)
Expand Down Expand Up @@ -475,12 +460,16 @@ namespace BaskerNS
//Find BTF ordering
if(btf_order2() != BASKER_SUCCESS)
{
if(Options.verbose == BASKER_TRUE)
{
printf("Basker Ordering Failed \n"); fflush(stdout);
}
return BASKER_ERROR;
}

if(Options.verbose == BASKER_TRUE)
{
printf("Basker Ordering Found \n");
printf("Basker Ordering Found \n"); fflush(stdout);
}

/*if((Options.btf == BASKER_TRUE) && (btf_tabs_offset != 0))
Expand Down Expand Up @@ -512,7 +501,7 @@ namespace BaskerNS
if(symb_flag == BASKER_TRUE)
{
if(Options.verbose == BASKER_TRUE) {
printf("BASKER: YOU CANNOT RERUN SFACTOR\n");
printf("BASKER: YOU CANNOT RERUN SFACTOR\n"); fflush(stdout);
}
return BASKER_ERROR;
}
Expand Down Expand Up @@ -547,7 +536,7 @@ namespace BaskerNS

if(Options.verbose == BASKER_TRUE)
{
printf(" == Basker Symbolic Done ==\n\n");
printf(" == Basker Symbolic Done ==\n\n"); fflush(stdout);
}

#ifdef BASKER_TIMER
Expand Down Expand Up @@ -1573,7 +1562,7 @@ namespace BaskerNS
#endif
}

// ----------------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------------
// 'sort' rows of BTF_A into ND structure
#if 0
for (Int i = 0; i < BTF_A.nnz; ++i) {
Expand Down Expand Up @@ -1621,23 +1610,31 @@ namespace BaskerNS
symmetric_sfactor();
if(Options.verbose == BASKER_TRUE) {
std::cout<< " > Basker Factor: Time for symbolic after ND on a big block A: " << nd_symbolic_timer.seconds() << std::endl;
fflush(stdout);
}

Kokkos::Timer nd_last_dense_timer;
bool flag = true;
btf_last_dense(flag);
if(Options.verbose == BASKER_TRUE) {
std::cout<< " > Basker Factor: Time for last-dense after ND on a big block A: " << nd_last_dense_timer.seconds() << std::endl;
fflush(stdout);
}


#ifdef BASKER_KOKKOS
// ----------------------------------------------------------------------------------------------
// Allocate & Initialize blocks
#ifdef BASKER_PARALLEL_INIT_FACTOR
kokkos_sfactor_init_factor<Int,Entry,Exe_Space>
iF(this);
Kokkos::parallel_for(TeamPolicy(num_threads,1), iF);
Kokkos::fence();
#else
for (Int p = 0; p < num_threads; p++) {
this->t_init_factor(p);
}
#endif

/*kokkos_sfactor_init_workspace<Int,Entry,Exe_Space>
iWS(flag, this);
Expand Down Expand Up @@ -1950,10 +1947,16 @@ namespace BaskerNS
}*/

Kokkos::Timer nd_setup2_timer;
#ifdef BASKER_PARALLEL_INIT_WORKSPACE
kokkos_sfactor_init_workspace<Int,Entry,Exe_Space>
iWS(flag, this);
Kokkos::parallel_for(TeamPolicy(num_threads,1), iWS);
Kokkos::fence();
#else
for (Int p = 0; p < num_threads; p++) {
this->t_init_workspace(flag, p);
}
#endif
if(Options.verbose == BASKER_TRUE) {
std::cout<< " > Basker Factor: Time for workspace allocation after ND on a big block A: " << nd_setup2_timer.seconds() << std::endl;
}
Expand Down
2 changes: 1 addition & 1 deletion packages/shylu/shylu_node/basker/src/shylubasker_error_manager.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ namespace BaskerNS
Int blkcol = thread_array(ti).error_blk;
Int blkUrow = LU_size(blkcol)-1;
if(LL(blkcol)(0).nnz >=
LU(blkcol)(blkUrow).nnz)
LU(blkcol)(blkUrow).nnz)
{
resize_U = thread_array(ti).error_info;
}
Expand Down
3 changes: 3 additions & 0 deletions packages/shylu/shylu_node/basker/src/shylubasker_matrix_decl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ namespace BaskerNS
BASKER_INLINE
int fill();

BASKER_INLINE
void init_ptr();

BASKER_INLINE
void init_inc_lvl();

Expand Down
9 changes: 8 additions & 1 deletion packages/shylu/shylu_node/basker/src/shylubasker_matrix_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ namespace BaskerNS
if(nnz == _nnz)
{
copy_vec(_row_idx, _nnz, row_idx);
copy_vec(_val,_nnz, val);
copy_vec(_val, _nnz, val);
}
else
{
Expand Down Expand Up @@ -498,6 +498,13 @@ namespace BaskerNS
return 0;
}

template <class Int, class Entry, class Exe_Space>
BASKER_INLINE
void BaskerMatrix<Int,Entry,Exe_Space>::init_ptr()
{
for (Int i = 0; i < ncol+1; i ++) col_ptr(i) = 0;
}

template <class Int, class Entry, class Exe_Space>
BASKER_INLINE
void BaskerMatrix<Int,Entry,Exe_Space>::convert2D
Expand Down
17 changes: 3 additions & 14 deletions packages/shylu/shylu_node/basker/src/shylubasker_nfactor.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,13 +171,9 @@ namespace BaskerNS
}//end while
if(Options.verbose == BASKER_TRUE)
{
printf("Time DOMAIN: %lf \n", timer.seconds());
printf("Time DOMAIN: %lf \n\n", timer.seconds());
timer.reset();
}
#ifdef BASKER_TIMER
printf("Time DOMAIN: %lf \n", timer.seconds());
timer.reset();
#endif

#else// else basker_kokkos
#pragma omp parallel
Expand Down Expand Up @@ -282,13 +278,9 @@ namespace BaskerNS
//printf( " End Sep: info = %d (%d, %d)\n",info,BASKER_SUCCESS,BASKER_ERROR );
if(Options.verbose == BASKER_TRUE)
{
printf("Time SEP: %lf \n", timer.seconds());
printf("Time SEP: %lf \n\n", timer.seconds());
timer.reset();
}
#ifdef BASKER_TIMER
printf("Time SEP: %lf \n", timer.seconds());
timer.reset();
#endif
}

// ---------------------------------------------------------------------------------------- //
Expand Down Expand Up @@ -363,11 +355,8 @@ namespace BaskerNS

if(Options.verbose == BASKER_TRUE)
{
printf("Time BTF: %lf \n", timer.seconds());
printf("Time BTF: %lf \n\n", timer.seconds());
}
#ifdef BASKER_TIMER
printf("Time BTF: %lf \n", timer.seconds());
#endif
}//end btf call

Kokkos::Timer tzback;
Expand Down
24 changes: 12 additions & 12 deletions packages/shylu/shylu_node/basker/src/shylubasker_nfactor_blk.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ namespace BaskerNS
const Mag normA = BTF_A.gnorm;
const Mag normA_blk = BTF_A.anorm;

Int b = S[0][kid]; //Which blk from schedule
Int b = S(0)(kid); //Which blk from schedule
BASKER_MATRIX &L = LL(b)(0);
BASKER_MATRIX &U = LU(b)(LU_size(b)-1);
BASKER_MATRIX &M = ALM(b)(0); //A->blk
Expand All @@ -159,9 +159,9 @@ namespace BaskerNS
ENTRY_1DARRAY X = LL(b)(0).ews;
Int ws_size = LL(b)(0).iws_size;
#else //else if BASKER_2DL
INT_1DARRAY ws = thread_array[kid].iws;
ENTRY_1DARRAY X = thread_array[kid].ews;
Int ws_size = thread_array[kid].iws_size;
INT_1DARRAY ws = thread_array(kid).iws;
ENTRY_1DARRAY X = thread_array(kid).ews;
Int ws_size = thread_array(kid).iws_size;
#endif
//Int bcol = L.scol; //begining col //NOT UD
Int scol_top = btf_tabs[btf_top_tabs_offset]; // the first column index of A
Expand Down Expand Up @@ -1286,8 +1286,8 @@ namespace BaskerNS
INT_1DARRAY ws = LL(wsb)(l).iws;
const Int ws_size = LL(wsb)(l).iws_size;
#else
INT_1DARRAY ws = thread_array[kid].iws;
Int ws_size = thread_array[kid].iws_size;
INT_1DARRAY ws = thread_array(kid).iws;
Int ws_size = thread_array(kid).iws_size;
#endif

const Int scol_top = btf_tabs[btf_top_tabs_offset]; // the first column index of A
Expand Down Expand Up @@ -1460,9 +1460,9 @@ namespace BaskerNS
ENTRY_1DARRAY X = LL(wsb)(l).ews;
Int ws_size = LL(wsb)(l).iws_size;
#else
INT_1DARRAY ws = thread_array[kid].iws;
ENTRY_1DARRAY X = thread_array[kid].ews;
Int ws_size = thread_array[kid].iws_size;
INT_1DARRAY ws = thread_array(kid).iws;
ENTRY_1DARRAY X = thread_array(kid).ews;
Int ws_size = thread_array(kid).iws_size;
#endif

const Entry zero (0.0);
Expand Down Expand Up @@ -1607,7 +1607,7 @@ namespace BaskerNS
if (blkcol == 2 && blkrow == 1) printf( " L.colptr(%d) = %d\n",k+1,lnnz );
#endif

//LL[X_col][X_row].p_size = 0;
//LL(X_col)(X_row).p_size = 0;
LL(X_col)(X_row).p_size = 0;

return 0;
Expand Down Expand Up @@ -1831,7 +1831,7 @@ namespace BaskerNS
}//over all nonzero in left

#ifdef BASKER_2DL
//LL[X_col][X_row].p_size = nnz;
//LL(X_col)(X_row).p_size = nnz;
LL(X_col)(X_row).p_size = nnz;
#endif

Expand Down Expand Up @@ -2056,7 +2056,7 @@ namespace BaskerNS
nnz, kid, X_col, X_row);
printf("kid %d Ending nnz: %d \n",kid, nnz);
#endif
//LL[X_col][X_row].p_size = nnz;
//LL(X_col)(X_row).p_size = nnz;
LL(X_col)(X_row).p_size = nnz;
#endif

Expand Down
Loading

0 comments on commit 6f6cb99

Please sign in to comment.