From ab682cc0bd60057a422f82032ba22b6d4929660b Mon Sep 17 00:00:00 2001
From: Thorsten Liebig <Thorsten.Liebig@gmx.de>
Date: Fri, 2 Dec 2016 19:03:35 +0100
Subject: [PATCH] A few fixes for MPI

main.cpp:
	1. 	Check return value of ParseFDTDSetup and exit if false
	2. 	Use exit instead of return. These are almost identical. But
		in my OpenMPI installation the process with teh highes rank
		segfaults at the end when using return. This is not the case
		with exit. Probably some C++ cleanup problem (destructors).
openems.cpp:
	Give Parse_XML_FDTDSetup a deterministic return value.
openems_fdtd_mpi.cpp:
	1.	Remove the word "only" in an error message because there can
		also be too many processes.
	2.	Fix the indexing variables for SetSplitPos in SetupMPI. Otherwise
		more than one split results in an out-of-range exception and
		unexpected behavior.
RunOpenEMS_MPI.m:
	Apply Settings.MPI.GlobalArgs also to multi-host scenarios.
---
 FDTD/openems_fdtd_mpi.cpp | 6 +++---
 main.cpp                  | 8 ++++++--
 matlab/RunOpenEMS_MPI.m   | 2 +-
 openems.cpp               | 1 +
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/FDTD/openems_fdtd_mpi.cpp b/FDTD/openems_fdtd_mpi.cpp
index eaab59a..404de5c 100644
--- a/FDTD/openems_fdtd_mpi.cpp
+++ b/FDTD/openems_fdtd_mpi.cpp
@@ -210,7 +210,7 @@ bool openEMS_FDTD_MPI::SetupMPI()
 	if (numProcs!=m_NumProc)
 	{
 		if (m_MyID==0)
-			cerr << "openEMS_FDTD_MPI::SetupMPI: Error: Requested splits require " << numProcs << " processes, but only " << m_NumProc << " were found! Exit! " << endl;
+			cerr << "openEMS_FDTD_MPI::SetupMPI: Error: Requested splits require " << numProcs << " processes, but " << m_NumProc << " were found! Exit! " << endl;
 		exit(10);
 	}
 
@@ -256,8 +256,8 @@ bool openEMS_FDTD_MPI::SetupMPI()
 						grid->AddDiscLine(2, m_Original_Grid->GetLine(2,n) );
 
 					m_MPI_Op->SetSplitPos(0,m_SplitNumber[0].at(i));
-					m_MPI_Op->SetSplitPos(1,m_SplitNumber[1].at(i));
-					m_MPI_Op->SetSplitPos(2,m_SplitNumber[2].at(i));
+					m_MPI_Op->SetSplitPos(1,m_SplitNumber[1].at(j));
+					m_MPI_Op->SetSplitPos(2,m_SplitNumber[2].at(k));
 
 					if (i>0)
 						m_MPI_Op->SetNeighborDown(0,procTable[i-1][j][k]);
diff --git a/main.cpp b/main.cpp
index 1d03481..8a8000f 100644
--- a/main.cpp
+++ b/main.cpp
@@ -68,8 +68,12 @@ int main(int argc, char *argv[])
 	}
 
 	int EC = FDTD.ParseFDTDSetup(argv[1]);
+	if(!EC) {
+	  cerr << "openEMS - ParseFDTDSetup failed." << endl;
+	  exit(1);
+	}
 	EC = FDTD.SetupFDTD();
-	if (EC) return EC;
+	if (EC) exit(EC);
 	FDTD.RunFDTD();
 
 #ifdef MPI_SUPPORT
@@ -77,5 +81,5 @@ int main(int argc, char *argv[])
 	MPI::Finalize();
 #endif
 
-	return 0;
+	exit(0);
 }
diff --git a/matlab/RunOpenEMS_MPI.m b/matlab/RunOpenEMS_MPI.m
index 495f7e5..779b038 100644
--- a/matlab/RunOpenEMS_MPI.m
+++ b/matlab/RunOpenEMS_MPI.m
@@ -87,7 +87,7 @@ end
 
 if isfield(Settings.MPI,'Hosts')
     disp(['Running remote openEMS_MPI in working dir: ' work_path]);
-    [status]  = system(['mpiexec -host ' HostList ' -n ' int2str(NrProc) ' -wdir ' work_path ' ' Settings.MPI.Binary ' ' Sim_File ' ' opts ' ' append_unix]);
+    [status]  = system(['mpiexec ' Settings.MPI.GlobalArgs ' -host ' HostList ' -n ' int2str(NrProc) ' -wdir ' work_path ' ' Settings.MPI.Binary ' ' Sim_File ' ' opts ' ' append_unix]);
 else
     disp('Running local openEMS_MPI');
     [status]  = system(['mpiexec ' Settings.MPI.GlobalArgs ' -n ' int2str(NrProc) ' ' Settings.MPI.Binary ' ' Sim_File ' ' opts ' ' append_unix]);
diff --git a/openems.cpp b/openems.cpp
index 1aaad18..1d08ef7 100644
--- a/openems.cpp
+++ b/openems.cpp
@@ -821,6 +821,7 @@ bool openEMS::Parse_XML_FDTDSetup(TiXmlElement* FDTD_Opts)
 		this->SetTimeStep(dhelp);
 	if (FDTD_Opts->QueryDoubleAttribute("TimeStepFactor",&dhelp)==TIXML_SUCCESS)
 		this->SetTimeStepFactor(dhelp);
+	return true;
 }
 
 void openEMS::SetGaussExcite(double f0, double fc)