51#ifdef HAVE_IFPACK_PARALLEL_SUBDOMAIN_SOLVERS
54#include "Teuchos_Hashtable.hpp"
55#include "Teuchos_Array.hpp"
59# ifdef IFPACK_NODE_AWARE_CODE
62# include "Teuchos_Hashtable.hpp"
63# include "Teuchos_Array.hpp"
66 extern int ML_NODE_ID;
75template<
typename int_type>
78 RCP<Epetra_Map> TmpMap;
79 RCP<Epetra_CrsMatrix> TmpMatrix;
80 RCP<Epetra_Import> TmpImporter;
87 std::vector<int_type> ExtElements;
89 for (
int overlap = 0 ; overlap < OverlapLevel_in ; ++overlap) {
90 if (TmpMatrix != Teuchos::null) {
91 RowMap = &(TmpMatrix->RowMatrixRowMap());
92 ColMap = &(TmpMatrix->RowMatrixColMap());
95 RowMap = &(
A().RowMatrixRowMap());
96 ColMap = &(
A().RowMatrixColMap());
100 std::vector<int_type> list(size);
106 int_type GID = (int_type) ColMap->
GID64(i);
107 if (
A().RowMatrixRowMap().LID(GID) == -1) {
108 typename std::vector<int_type>::iterator pos
109 =
find(ExtElements.begin(),ExtElements.end(),GID);
110 if (pos == ExtElements.end()) {
111 ExtElements.push_back(GID);
118 const int_type *listptr = NULL;
119 if ( ! list.empty() ) listptr = &list[0];
126 TmpMatrix->Import(
A(),*TmpImporter,
Insert);
133 std::vector<int_type> list(
NumMyRowsA_ + ExtElements.size());
135 list[i] = (int_type)
A().RowMatrixRowMap().GID64(i);
136 for (
int i = 0 ; i < (int)ExtElements.size() ; ++i)
139 const int_type *listptr = NULL;
140 if ( ! list.empty() ) listptr = &list[0];
143 listptr, 0,
Comm()) );
145#ifdef HAVE_IFPACK_PARALLEL_SUBDOMAIN_SOLVERS
148# ifdef IFPACK_NODE_AWARE_CODE
155 const int_type * extelsptr = NULL;
156 if ( ! ExtElements.empty() ) extelsptr = &ExtElements[0];
158 extelsptr,0,
A().
Comm()) );
166 int OverlapLevel_in) :
168 OverlapLevel_(OverlapLevel_in)
171 if (OverlapLevel_in == 0)
175 if (
Comm().NumProc() == 1)
182#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
184 BuildMap<int>(OverlapLevel_in);
188#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
190 BuildMap<long long>(OverlapLevel_in);
194 throw "Ifpack_OverlappingRowMatrix::Ifpack_OverlappingRowMatrix: GlobalIndices type unknown";
221#ifdef HAVE_IFPACK_PARALLEL_SUBDOMAIN_SOLVERS
227 int OverlapLevel_in,
int subdomainID) :
229 OverlapLevel_(OverlapLevel_in)
233#ifdef IFPACK_OVA_TIME_BUILD
234 double t0 = MPI_Wtime();
235 double t1, true_t0=t0;
239 const int votesMax = INT_MAX;
242 if (OverlapLevel_in == 0)
246 if (
Comm().NumProc() == 1)
254 assert(pComm != NULL);
255 MPI_Comm nodeMPIComm;
256 MPI_Comm_split(pComm->Comm(),subdomainID,pComm->MyPID(),&nodeMPIComm);
264 RCP<Epetra_Map> TmpMap;
265 RCP<Epetra_CrsMatrix> TmpMatrix;
266 RCP<Epetra_Import> TmpImporter;
278#ifdef IFPACK_OVA_TIME_BUILD
280 fprintf(stderr,
"[node %d]: time for initialization %2.3e\n", subdomainID, t1-t0);
285#ifdef IFPACK_OVA_TIME_BUILD
287 fprintf(stderr,
"[node %d]: overlap hash table allocs %2.3e\n", subdomainID, t1-t0);
291 Teuchos::Hashtable<int,int> colMapTable(3 *
A().
RowMatrixColMap().NumMyElements() );
295 Teuchos::Hashtable<int,int> ghostTable(3 *
A().
RowMatrixColMap().NumMyElements() );
300 for (
int overlap = 0 ; overlap < OverlapLevel_in ; ++overlap)
305 if (TmpMatrix != Teuchos::null) {
306 RowMap = &(TmpMatrix->RowMatrixRowMap());
307 ColMap = &(TmpMatrix->RowMatrixColMap());
308 DomainMap = &(TmpMatrix->OperatorDomainMap());
311 RowMap = &(
A().RowMatrixRowMap());
312 ColMap = &(
A().RowMatrixColMap());
313 DomainMap = &(
A().OperatorDomainMap());
316#ifdef IFPACK_OVA_TIME_BUILD
318 fprintf(stderr,
"[node %d]: overlap pointer pulls %2.3e\n", subdomainID, t1-t0);
326 rowIdList.PutValue(subdomainID);
327 Teuchos::RCP<Epetra_Import> nodeIdImporter = rcp(
new Epetra_Import( *ColMap, *DomainMap ));
329#ifdef IFPACK_OVA_TIME_BUILD
331 fprintf(stderr,
"[node %d]: overlap intvector/importer allocs %2.3e\n", subdomainID, t1-t0);
335 colIdList.Import(rowIdList,*nodeIdImporter,Insert);
340#ifdef IFPACK_OVA_TIME_BUILD
342 fprintf(stderr,
"[node %d]: overlap col/row ID imports %2.3e\n", subdomainID, t1-t0);
350 int GID = ColMap->
GID(i);
352 if (ghostTable.containsKey(GID)) myvotes = ghostTable.get(GID);
353 if ( colIdList[i] != subdomainID && myvotes < votesMax)
356 if (ghostTable.containsKey(GID)) {
357 votes = ghostTable.get(GID);
359 ghostTable.put(GID,votes);
361 ghostTable.put(GID,1);
366 Teuchos::Array<int> gidsarray,votesarray;
367 ghostTable.arrayify(gidsarray,votesarray);
368 int *gids = gidsarray.getRawPtr();
369 int *votes = votesarray.getRawPtr();
386#ifdef IFPACK_OVA_TIME_BUILD
388 fprintf(stderr,
"[node %d]: overlap before culling %2.3e\n", subdomainID, t1-t0);
394 if (nodeComm->MyPID() == 0)
398 int *lengths =
new int[nodeComm->NumProc()+1];
400 lengths[1] = ghostTable.size();
401 for (
int i=1; i<nodeComm->NumProc(); i++) {
403 MPI_Recv( &leng, 1, MPI_INT, i, MPI_ANY_TAG, nodeComm->Comm(), &status);
404 lengths[i+1] = lengths[i] + leng;
406 int total = lengths[nodeComm->NumProc()];
408 int* ghosts =
new int[total];
409 for (
int i=0; i<total; i++) ghosts[i] = -9;
410 int *round =
new int[total];
411 int *owningpid =
new int[total];
413 for (
int i=1; i<nodeComm->NumProc(); i++) {
414 int count = lengths[i+1] - lengths[i];
415 MPI_Recv( ghosts+lengths[i], count, MPI_INT, i, MPI_ANY_TAG, nodeComm->Comm(), &status);
416 MPI_Recv( round+lengths[i], count, MPI_INT, i, MPI_ANY_TAG, nodeComm->Comm(), &status);
420 for (
int i=0; i<lengths[1]; i++) {
427 for (
int j=1; j<nodeComm->NumProc(); j++) {
428 for (
int i=lengths[j]; i<lengths[j+1]; i++) {
435 roundpid[0] = round; roundpid[1] = owningpid;
437 epetraUtil.
Sort(
true,total,ghosts,0,0,2,roundpid);
440 int *nlosers =
new int[nodeComm->NumProc()];
441 int** losers =
new int*[nodeComm->NumProc()];
442 for (
int i=0; i<nodeComm->NumProc(); i++) {
444 losers[i] =
new int[ lengths[i+1]-lengths[i] ];
456 for (
int i=1; i<total; i++) {
457 if (ghosts[i] == ghosts[i-1]) {
459 if (round[i] > round[
max]) {
463 int j = owningpid[idx];
464 losers[j][nlosers[j]++] = ghosts[idx];
475 for (
int i=1; i<nodeComm->NumProc(); i++) {
476 MPI_Send( nlosers+i, 1, MPI_INT, i, 8675, nodeComm->Comm());
477 MPI_Send( losers[i], nlosers[i], MPI_INT, i, 8675, nodeComm->Comm());
484 cullList =
new int[ncull+1];
485 for (
int i=0; i<nlosers[0]; i++)
486 cullList[i] = losers[0][i];
488 for (
int i=0; i<nodeComm->NumProc(); i++)
498 int hashsize = ghostTable.size();
499 MPI_Send( &hashsize, 1, MPI_INT, 0, 8675, nodeComm->Comm());
500 MPI_Send( gids, hashsize, MPI_INT, 0, 8675, nodeComm->Comm());
501 MPI_Send( votes, hashsize, MPI_INT, 0, 8675, nodeComm->Comm());
505 MPI_Recv( &ncull, 1, MPI_INT, 0, 8675, nodeComm->Comm(), &status);
506 cullList =
new int[ncull+1];
507 MPI_Recv( cullList, ncull, MPI_INT, 0, 8675, nodeComm->Comm(), &status);
514 for (
int i=0; i<ncull; i++) {
515 try{ghostTable.remove(cullList[i]);}
518 printf(
"pid %d: In OverlappingRowMatrix ctr, problem removing ghost elt %d from ghostTable\n",
519 Comm().MyPID(),cullList[i]);
528 gidsarray.clear(); votesarray.clear();
529 ghostTable.arrayify(gidsarray,votesarray);
531 std::vector<int> list(size);
533 for (
int i=0; i<ghostTable.size(); i++) {
535 if (votesarray[i] < votesMax) {
536 list[count] = gidsarray[i];
537 ghostTable.put(gidsarray[i],votesMax);
543#ifdef IFPACK_OVA_TIME_BUILD
545 fprintf(stderr,
"[node %d]: overlap duplicate removal %2.3e\n", subdomainID, t1-t0);
558 TmpMatrix->Import(
A(),*TmpImporter,Insert);
562#ifdef IFPACK_OVA_TIME_BUILD
564 fprintf(stderr,
"[node %d]: overlap TmpMatrix fillcomplete %2.3e\n", subdomainID, t1-t0);
576 ov_colIdList.PutValue(-1);
578 ov_rowIdList.PutValue(subdomainID);
579 Teuchos::RCP<Epetra_Import> ov_nodeIdImporter = rcp(
new Epetra_Import( TmpMatrix->ColMap(), TmpMatrix->RowMap()));
580 ov_colIdList.Import(ov_rowIdList,*ov_nodeIdImporter,Insert);
582 for (
int i=0 ; i<ov_colIdList.MyLength(); i++) {
583 if (ov_colIdList[i] == subdomainID) {
584 int GID = (ov_colIdList.Map()).GID(i);
585 colMapTable.put(GID,1);
592 ov_colIdList.PutValue(-1);
594 ArowIdList.PutValue(subdomainID);
596 ov_colIdList.Import(ArowIdList,*nodeIdImporter,Insert);
598 for (
int i=0 ; i<ov_colIdList.MyLength(); i++) {
599 if (ov_colIdList[i] == subdomainID) {
600 int GID = (ov_colIdList.Map()).GID(i);
601 colMapTable.put(GID,1);
606#ifdef IFPACK_OVA_TIME_BUILD
608 fprintf(stderr,
"[node %d]: overlap 2 imports to fix up colmap %2.3e\n", subdomainID, t1-t0);
620 std::vector<int> ghostElements;
622 Teuchos::Array<int> gidsarray,votesarray;
623 ghostTable.arrayify(gidsarray,votesarray);
624 for (
int i=0; i<ghostTable.size(); i++) {
625 ghostElements.push_back(gidsarray[i]);
628 for (
int i = 0 ; i <
A().RowMatrixColMap().NumMyElements() ; ++i) {
629 int GID =
A().RowMatrixColMap().GID(i);
631 if (colMapTable.containsKey(GID)) {
632 try{colMapTable.remove(GID);}
634 printf(
"pid %d: In OverlappingRowMatrix ctr, problem removing entry %d from colMapTable\n",
Comm().MyPID(),GID);
641 std::vector<int> colMapElements;
643 gidsarray.clear(); votesarray.clear();
644 colMapTable.arrayify(gidsarray,votesarray);
645 for (
int i=0; i<colMapTable.size(); i++)
646 colMapElements.push_back(gidsarray[i]);
659 std::vector<int> rowList(
NumMyRowsA_ + ghostElements.size());
661 rowList[i] =
A().RowMatrixRowMap().GID(i);
662 for (
int i = 0 ; i < (int)ghostElements.size() ; ++i)
674 std::vector<int> colList(
A().
RowMatrixColMap().NumMyElements() + colMapElements.size());
675 int nc =
A().RowMatrixColMap().NumMyElements();
676 for (
int i = 0 ; i < nc; i++)
677 colList[i] =
A().RowMatrixColMap().GID(i);
678 for (
int i = 0 ; i < (int)colMapElements.size() ; i++)
679 colList[nc+i] = colMapElements[i];
687#ifdef IFPACK_OVA_TIME_BUILD
689 fprintf(stderr,
"[node %d]: time to init B() col/row maps %2.3e\n", subdomainID, t1-t0);
702 int numMyElts = colMap_->NumMyElements();
703 assert(numMyElts!=0);
709 int* myGlobalElts =
new int[numMyElts];
710 colMap_->MyGlobalElements(myGlobalElts);
711 int* pidList =
new int[numMyElts];
712 nodeMap->RemoteIDList(numMyElts, myGlobalElts, pidList, 0);
717 tt[0] = myGlobalElts;
718 Util.
Sort(
true, numMyElts, pidList, 0, (
double**)0, 1, tt);
723 while (localStart<numMyElts) {
724 int currPID = (pidList)[localStart];
726 while (i<numMyElts && (pidList)[i] == currPID) i++;
727 if (currPID != nodeComm->MyPID())
728 Util.Sort(
true, i-localStart, (myGlobalElts)+localStart, 0, 0, 0, 0);
734 while (localStart<numMyElts && (pidList)[localStart] != nodeComm->MyPID()) localStart++;
735 assert(localStart != numMyElts);
736 int localEnd=localStart;
737 while (localEnd<numMyElts && (pidList)[localEnd] == nodeComm->MyPID()) localEnd++;
738 int* mySortedGlobalElts =
new int[numMyElts];
739 int leng = localEnd - localStart;
743 int *rowGlobalElts = nodeMap->MyGlobalElements();
763 for (
int i=0; i<leng; i++) {
765 (mySortedGlobalElts)[i] = rowGlobalElts[i];
769 for (
int i=leng; i< localEnd; i++) {
770 (mySortedGlobalElts)[i] = (myGlobalElts)[i-leng];
772 for (
int i=localEnd; i<numMyElts; i++) {
773 (mySortedGlobalElts)[i] = (myGlobalElts)[i];
777#ifdef IFPACK_OVA_TIME_BUILD
779 fprintf(stderr,
"[node %d]: time to sort col map arrays (cp=1) %2.3e\n", subdomainID, t1-t0);
784 int indexBase = colMap_->IndexBase();
785 if (colMap_)
delete colMap_;
786 colMap_ =
new Epetra_Map(-1,numMyElts,mySortedGlobalElts,indexBase,
Comm());
789 delete [] myGlobalElts;
791 delete [] mySortedGlobalElts;
795 printf(
"** * Ifpack_OverlappingRowmatrix ctor: problem creating column map * **\n\n");
803#ifdef IFPACK_OVA_TIME_BUILD
805 fprintf(stderr,
"[node %d]: time to sort col map arrays (cp=2) %2.3e\n", subdomainID, t1-t0);
842#ifdef IFPACK_OVA_TIME_BUILD
844 fprintf(stderr,
"[node %d]: time to import and setup ExtMap_ %2.3e\n", subdomainID, t1-t0);
860#ifdef IFPACK_OVA_TIME_BUILD
862 fprintf(stderr,
"[node %d]: time to FillComplete B() %2.3e\n", subdomainID, t1-t0);
896#ifdef IFPACK_OVA_TIME_BUILD
898 fprintf(stderr,
"[node %d]: time for final calcs %2.3e\n", subdomainID, t1-t0);
899 fprintf(stderr,
"[node %d]: total IORM ctor time %2.3e\n", subdomainID, t1-true_t0);
907# ifdef IFPACK_NODE_AWARE_CODE
913 int OverlapLevel_in,
int nodeID) :
915 OverlapLevel_(OverlapLevel_in)
919#ifdef IFPACK_OVA_TIME_BUILD
920 double t0 = MPI_Wtime();
921 double t1, true_t0=t0;
925 const int votesMax = INT_MAX;
928 if (OverlapLevel_in == 0)
932 if (
Comm().NumProc() == 1)
940 assert(pComm != NULL);
941 MPI_Comm nodeMPIComm;
942 MPI_Comm_split(pComm->Comm(),nodeID,pComm->MyPID(),&nodeMPIComm);
950 RCP<Epetra_Map> TmpMap;
951 RCP<Epetra_CrsMatrix> TmpMatrix;
952 RCP<Epetra_Import> TmpImporter;
964#ifdef IFPACK_OVA_TIME_BUILD
966 fprintf(stderr,
"[node %d]: time for initialization %2.3e\n", nodeID, t1-t0);
971#ifdef IFPACK_OVA_TIME_BUILD
973 fprintf(stderr,
"[node %d]: overlap hash table allocs %2.3e\n", nodeID, t1-t0);
977 Teuchos::Hashtable<int,int> colMapTable(3 *
A().
RowMatrixColMap().NumMyElements() );
981 Teuchos::Hashtable<int,int> ghostTable(3 *
A().
RowMatrixColMap().NumMyElements() );
986 for (
int overlap = 0 ; overlap < OverlapLevel_in ; ++overlap)
991 if (TmpMatrix != Teuchos::null) {
992 RowMap = &(TmpMatrix->RowMatrixRowMap());
993 ColMap = &(TmpMatrix->RowMatrixColMap());
994 DomainMap = &(TmpMatrix->OperatorDomainMap());
997 RowMap = &(
A().RowMatrixRowMap());
998 ColMap = &(
A().RowMatrixColMap());
999 DomainMap = &(
A().OperatorDomainMap());
1002#ifdef IFPACK_OVA_TIME_BUILD
1004 fprintf(stderr,
"[node %d]: overlap pointer pulls %2.3e\n", nodeID, t1-t0);
1012 rowIdList.PutValue(nodeID);
1013 Teuchos::RCP<Epetra_Import> nodeIdImporter = rcp(
new Epetra_Import( *ColMap, *DomainMap ));
1015#ifdef IFPACK_OVA_TIME_BUILD
1017 fprintf(stderr,
"[node %d]: overlap intvector/importer allocs %2.3e\n", nodeID, t1-t0);
1021 colIdList.Import(rowIdList,*nodeIdImporter,Insert);
1026#ifdef IFPACK_OVA_TIME_BUILD
1028 fprintf(stderr,
"[node %d]: overlap col/row ID imports %2.3e\n", nodeID, t1-t0);
1036 int GID = ColMap->
GID(i);
1038 if (ghostTable.containsKey(GID)) myvotes = ghostTable.get(GID);
1039 if ( colIdList[i] != nodeID && myvotes < votesMax)
1042 if (ghostTable.containsKey(GID)) {
1043 votes = ghostTable.get(GID);
1045 ghostTable.put(GID,votes);
1047 ghostTable.put(GID,1);
1052 Teuchos::Array<int> gidsarray,votesarray;
1053 ghostTable.arrayify(gidsarray,votesarray);
1054 int *gids = gidsarray.getRawPtr();
1055 int *votes = votesarray.getRawPtr();
1072#ifdef IFPACK_OVA_TIME_BUILD
1074 fprintf(stderr,
"[node %d]: overlap before culling %2.3e\n", nodeID, t1-t0);
1080 if (nodeComm->MyPID() == 0)
1084 int *lengths =
new int[nodeComm->NumProc()+1];
1086 lengths[1] = ghostTable.size();
1087 for (
int i=1; i<nodeComm->NumProc(); i++) {
1089 MPI_Recv( &leng, 1, MPI_INT, i, MPI_ANY_TAG, nodeComm->Comm(), &status);
1090 lengths[i+1] = lengths[i] + leng;
1092 int total = lengths[nodeComm->NumProc()];
1094 int* ghosts =
new int[total];
1095 for (
int i=0; i<total; i++) ghosts[i] = -9;
1096 int *round =
new int[total];
1097 int *owningpid =
new int[total];
1099 for (
int i=1; i<nodeComm->NumProc(); i++) {
1100 int count = lengths[i+1] - lengths[i];
1101 MPI_Recv( ghosts+lengths[i], count, MPI_INT, i, MPI_ANY_TAG, nodeComm->Comm(), &status);
1102 MPI_Recv( round+lengths[i], count, MPI_INT, i, MPI_ANY_TAG, nodeComm->Comm(), &status);
1106 for (
int i=0; i<lengths[1]; i++) {
1107 ghosts[i] = gids[i];
1108 round[i] = votes[i];
1113 for (
int j=1; j<nodeComm->NumProc(); j++) {
1114 for (
int i=lengths[j]; i<lengths[j+1]; i++) {
1121 roundpid[0] = round; roundpid[1] = owningpid;
1123 epetraUtil.
Sort(
true,total,ghosts,0,0,2,roundpid);
1126 int *nlosers =
new int[nodeComm->NumProc()];
1127 int** losers =
new int*[nodeComm->NumProc()];
1128 for (
int i=0; i<nodeComm->NumProc(); i++) {
1130 losers[i] =
new int[ lengths[i+1]-lengths[i] ];
1142 for (
int i=1; i<total; i++) {
1143 if (ghosts[i] == ghosts[i-1]) {
1145 if (round[i] > round[
max]) {
1149 int j = owningpid[idx];
1150 losers[j][nlosers[j]++] = ghosts[idx];
1158 delete [] owningpid;
1161 for (
int i=1; i<nodeComm->NumProc(); i++) {
1162 MPI_Send( nlosers+i, 1, MPI_INT, i, 8675, nodeComm->Comm());
1163 MPI_Send( losers[i], nlosers[i], MPI_INT, i, 8675, nodeComm->Comm());
1170 cullList =
new int[ncull+1];
1171 for (
int i=0; i<nlosers[0]; i++)
1172 cullList[i] = losers[0][i];
1174 for (
int i=0; i<nodeComm->NumProc(); i++)
1175 delete [] losers[i];
1184 int hashsize = ghostTable.size();
1185 MPI_Send( &hashsize, 1, MPI_INT, 0, 8675, nodeComm->Comm());
1186 MPI_Send( gids, hashsize, MPI_INT, 0, 8675, nodeComm->Comm());
1187 MPI_Send( votes, hashsize, MPI_INT, 0, 8675, nodeComm->Comm());
1191 MPI_Recv( &ncull, 1, MPI_INT, 0, 8675, nodeComm->Comm(), &status);
1192 cullList =
new int[ncull+1];
1193 MPI_Recv( cullList, ncull, MPI_INT, 0, 8675, nodeComm->Comm(), &status);
1200 for (
int i=0; i<ncull; i++) {
1201 try{ghostTable.remove(cullList[i]);}
1204 printf(
"pid %d: In OverlappingRowMatrix ctr, problem removing ghost elt %d from ghostTable\n",
1205 Comm().MyPID(),cullList[i]);
1214 gidsarray.clear(); votesarray.clear();
1215 ghostTable.arrayify(gidsarray,votesarray);
1217 std::vector<int> list(size);
1219 for (
int i=0; i<ghostTable.size(); i++) {
1221 if (votesarray[i] < votesMax) {
1222 list[count] = gidsarray[i];
1223 ghostTable.put(gidsarray[i],votesMax);
1229#ifdef IFPACK_OVA_TIME_BUILD
1231 fprintf(stderr,
"[node %d]: overlap duplicate removal %2.3e\n", nodeID, t1-t0);
1244 TmpMatrix->Import(
A(),*TmpImporter,Insert);
1248#ifdef IFPACK_OVA_TIME_BUILD
1250 fprintf(stderr,
"[node %d]: overlap TmpMatrix fillcomplete %2.3e\n", nodeID, t1-t0);
1262 ov_colIdList.PutValue(-1);
1264 ov_rowIdList.PutValue(nodeID);
1265 Teuchos::RCP<Epetra_Import> ov_nodeIdImporter = rcp(
new Epetra_Import( TmpMatrix->ColMap(), TmpMatrix->RowMap()));
1266 ov_colIdList.Import(ov_rowIdList,*ov_nodeIdImporter,Insert);
1268 for (
int i=0 ; i<ov_colIdList.MyLength(); i++) {
1269 if (ov_colIdList[i] == nodeID) {
1270 int GID = (ov_colIdList.Map()).GID(i);
1271 colMapTable.put(GID,1);
1278 ov_colIdList.PutValue(-1);
1280 ArowIdList.PutValue(nodeID);
1282 ov_colIdList.Import(ArowIdList,*nodeIdImporter,Insert);
1284 for (
int i=0 ; i<ov_colIdList.MyLength(); i++) {
1285 if (ov_colIdList[i] == nodeID) {
1286 int GID = (ov_colIdList.Map()).GID(i);
1287 colMapTable.put(GID,1);
1292#ifdef IFPACK_OVA_TIME_BUILD
1294 fprintf(stderr,
"[node %d]: overlap 2 imports to fix up colmap %2.3e\n", nodeID, t1-t0);
1306 std::vector<int> ghostElements;
1308 Teuchos::Array<int> gidsarray,votesarray;
1309 ghostTable.arrayify(gidsarray,votesarray);
1310 for (
int i=0; i<ghostTable.size(); i++) {
1311 ghostElements.push_back(gidsarray[i]);
1314 for (
int i = 0 ; i <
A().RowMatrixColMap().NumMyElements() ; ++i) {
1315 int GID =
A().RowMatrixColMap().GID(i);
1317 if (colMapTable.containsKey(GID)) {
1318 try{colMapTable.remove(GID);}
1320 printf(
"pid %d: In OverlappingRowMatrix ctr, problem removing entry %d from colMapTable\n",
Comm().MyPID(),GID);
1327 std::vector<int> colMapElements;
1329 gidsarray.clear(); votesarray.clear();
1330 colMapTable.arrayify(gidsarray,votesarray);
1331 for (
int i=0; i<colMapTable.size(); i++)
1332 colMapElements.push_back(gidsarray[i]);
1345 std::vector<int> rowList(
NumMyRowsA_ + ghostElements.size());
1347 rowList[i] =
A().RowMatrixRowMap().GID(i);
1348 for (
int i = 0 ; i < (int)ghostElements.size() ; ++i)
1360 std::vector<int> colList(
A().
RowMatrixColMap().NumMyElements() + colMapElements.size());
1361 int nc =
A().RowMatrixColMap().NumMyElements();
1362 for (
int i = 0 ; i < nc; i++)
1363 colList[i] =
A().RowMatrixColMap().GID(i);
1364 for (
int i = 0 ; i < (int)colMapElements.size() ; i++)
1365 colList[nc+i] = colMapElements[i];
1373#ifdef IFPACK_OVA_TIME_BUILD
1375 fprintf(stderr,
"[node %d]: time to init B() col/row maps %2.3e\n", nodeID, t1-t0);
1388 int numMyElts = colMap_->NumMyElements();
1389 assert(numMyElts!=0);
1395 int* myGlobalElts =
new int[numMyElts];
1396 colMap_->MyGlobalElements(myGlobalElts);
1397 int* pidList =
new int[numMyElts];
1398 nodeMap->RemoteIDList(numMyElts, myGlobalElts, pidList, 0);
1403 tt[0] = myGlobalElts;
1404 Util.
Sort(
true, numMyElts, pidList, 0, (
double**)0, 1, tt);
1409 while (localStart<numMyElts) {
1410 int currPID = (pidList)[localStart];
1412 while (i<numMyElts && (pidList)[i] == currPID) i++;
1413 if (currPID != nodeComm->MyPID())
1414 Util.Sort(
true, i-localStart, (myGlobalElts)+localStart, 0, 0, 0, 0);
1420 while (localStart<numMyElts && (pidList)[localStart] != nodeComm->MyPID()) localStart++;
1421 assert(localStart != numMyElts);
1422 int localEnd=localStart;
1423 while (localEnd<numMyElts && (pidList)[localEnd] == nodeComm->MyPID()) localEnd++;
1424 int* mySortedGlobalElts =
new int[numMyElts];
1425 int leng = localEnd - localStart;
1429 int *rowGlobalElts = nodeMap->MyGlobalElements();
1449 for (
int i=0; i<leng; i++) {
1451 (mySortedGlobalElts)[i] = rowGlobalElts[i];
1455 for (
int i=leng; i< localEnd; i++) {
1456 (mySortedGlobalElts)[i] = (myGlobalElts)[i-leng];
1458 for (
int i=localEnd; i<numMyElts; i++) {
1459 (mySortedGlobalElts)[i] = (myGlobalElts)[i];
1463#ifdef IFPACK_OVA_TIME_BUILD
1465 fprintf(stderr,
"[node %d]: time to sort col map arrays (cp=1) %2.3e\n", nodeID, t1-t0);
1470 int indexBase = colMap_->IndexBase();
1471 if (colMap_)
delete colMap_;
1472 colMap_ =
new Epetra_Map(-1,numMyElts,mySortedGlobalElts,indexBase,
Comm());
1475 delete [] myGlobalElts;
1477 delete [] mySortedGlobalElts;
1481 printf(
"** * Ifpack_OverlappingRowmatrix ctor: problem creating column map * **\n\n");
1489#ifdef IFPACK_OVA_TIME_BUILD
1491 fprintf(stderr,
"[node %d]: time to sort col map arrays (cp=2) %2.3e\n", nodeID, t1-t0);
1528#ifdef IFPACK_OVA_TIME_BUILD
1530 fprintf(stderr,
"[node %d]: time to import and setup ExtMap_ %2.3e\n", nodeID, t1-t0);
1546#ifdef IFPACK_OVA_TIME_BUILD
1548 fprintf(stderr,
"[node %d]: time to FillComplete B() %2.3e\n", nodeID, t1-t0);
1581#ifdef IFPACK_OVA_TIME_BUILD
1583 fprintf(stderr,
"[node %d]: time for final calcs %2.3e\n", nodeID, t1-t0);
1584 fprintf(stderr,
"[node %d]: total IORM ctor time %2.3e\n", nodeID, t1-true_t0);
1610#ifdef HAVE_IFPACK_PARALLEL_SUBDOMAIN_SOLVERS
1614 int * Indices)
const
1620 ierr =
A().ExtractMyRowCopy(LocRow,Length,NumEntries,Values,Indices);
1621 Themap=&
A().RowMatrixColMap();
1632int Ifpack_OverlappingRowMatrix::
1633ExtractGlobalRowCopy(
int GlobRow,
int Length,
int & NumEntries,
double *Values,
1634 int * Indices)
const
1638 int LocRow =
A().RowMatrixRowMap().LID(GlobRow);
1640 ierr =
A().ExtractMyRowCopy(LocRow,Length,NumEntries,Values,Indices);
1641 Themap=&
A().RowMatrixColMap();
1651 for (
int i=0; i<NumEntries; i++) {
1652 Indices[i]=Themap->GID(Indices[i]);
1653 assert(Indices[i] != -1);
1659# ifdef IFPACK_NODE_AWARE_CODE
1663 int * Indices)
const
1669 ierr =
A().ExtractMyRowCopy(LocRow,Length,NumEntries,Values,Indices);
1670 Themap=&
A().RowMatrixColMap();
1681int Ifpack_OverlappingRowMatrix::
1682ExtractGlobalRowCopy(
int GlobRow,
int Length,
int & NumEntries,
double *Values,
1683 int * Indices)
const
1687 int LocRow =
A().RowMatrixRowMap().LID(GlobRow);
1689 ierr =
A().ExtractMyRowCopy(LocRow,Length,NumEntries,Values,Indices);
1690 Themap=&
A().RowMatrixColMap();
1700 for (
int i=0; i<NumEntries; i++) {
1701 Indices[i]=Themap->GID(Indices[i]);
1702 assert(Indices[i] != -1);
1712 int * Indices)
const
1716 ierr =
A().ExtractMyRowCopy(MyRow,Length,NumEntries,Values,Indices);
1750 for (
int j = 0 ; j < Nnz ; ++j) {
1751 Y[k][i] += Val[j] * X[k][Ind[j]];
1762 for (
int j = 0 ; j < Nnz ; ++j) {
1786#ifndef HAVE_IFPACK_PARALLEL_SUBDOMAIN_SOLVERS
1787# ifndef IFPACK_NODE_AWARE_CODE
#define IFPACK_RETURN(ifpack_err)
#define IFPACK_CHK_ERR(ifpack_err)
#define IFPACK_CHK_ERRV(ifpack_err)
static bool find(Parser_dh p, char *option, OptionsNode **ptr)
long long GID64(int LID) const
int NumMyElements() const
int Export(const Epetra_SrcDistObject &A, const Epetra_Import &Importer, Epetra_CombineMode CombineMode, const Epetra_OffsetIndex *Indexor=0)
virtual int NumMyRows() const=0
virtual int NumMyCols() const=0
virtual const Epetra_Map & RowMatrixColMap() const=0
virtual const Epetra_Map & RowMatrixRowMap() const=0
virtual int NumMyNonzeros() const=0
virtual int NumMyDiagonals() const=0
virtual int MaxNumEntries() const=0
virtual int ExtractMyRowCopy(int MyRow, int Length, int &NumEntries, double *Values, int *Indices) const=0
void Sort(bool SortAscending, int NumKeys, T *Keys, int NumDoubleCompanions, double **DoubleCompanions, int NumIntCompanions, int **IntCompanions, int NumLongLongCompanions, long long **LongLongCompanions)
virtual int ExtractMyRowCopy(int MyRow, int Length, int &NumEntries, double *Values, int *Indices) const
Returns a copy of the specified local row in user-provided arrays.
int ExportMultiVector(const Epetra_MultiVector &OvX, Epetra_MultiVector &X, Epetra_CombineMode CM=Add)
int ImportMultiVector(const Epetra_MultiVector &X, Epetra_MultiVector &OvX, Epetra_CombineMode CM=Insert)
const Epetra_Comm & Comm() const
Returns a pointer to the Epetra_Comm communicator associated with this operator.
void BuildMap(int OverlapLevel_in)
const Epetra_RowMatrix & A() const
virtual int ApplyInverse(const Epetra_MultiVector &X, Epetra_MultiVector &Y) const
bool UseTranspose() const
Returns the current UseTranspose setting.
virtual const Epetra_Map & RowMatrixColMap() const
Returns the Epetra_Map object associated with the columns of this matrix.
~Ifpack_OverlappingRowMatrix()
Ifpack_OverlappingRowMatrix(const Teuchos::RefCountPtr< const Epetra_RowMatrix > &Matrix_in, int OverlapLevel_in)
virtual int MaxNumEntries() const
Returns the maximum of NumMyRowEntries() over all rows.
virtual const Epetra_Map & RowMatrixRowMap() const
Returns the Epetra_Map object associated with the rows of this matrix.
virtual int Multiply(bool TransA, const Epetra_MultiVector &X, Epetra_MultiVector &Y) const
Returns the result of a Epetra_RowMatrix multiplied by a Epetra_MultiVector X in Y.
virtual int NumMyRowEntries(int MyRow, int &NumEntries) const
Returns the number of nonzero entries in MyRow.
Teuchos::RefCountPtr< Epetra_CrsMatrix > ExtMatrix_
const Epetra_Map & OperatorDomainMap() const
Returns the Epetra_Map object associated with the domain of this operator.
virtual int ExtractDiagonalCopy(Epetra_Vector &Diagonal) const
Returns a copy of the main diagonal in a user-provided vector.
virtual int Apply(const Epetra_MultiVector &X, Epetra_MultiVector &Y) const
long long NumGlobalNonzeros_
const Epetra_BlockMap & Map() const
Epetra_RowMatrix & B() const
Teuchos::RefCountPtr< const Epetra_Import > Importer_
Teuchos::RefCountPtr< Epetra_Map > ExtMap_
Teuchos::RefCountPtr< const Epetra_Map > Map_
Teuchos::RefCountPtr< Epetra_Import > ExtImporter_