first implementation of a multithreaded engine (bugs and mem leaks present!)

FDTD/engine_multithread.cpp

@@ -38,81 +38,172 @@ Engine_Multithread::~Engine_Multithread()
 void Engine_Multithread::Init()
 {
 	Engine::Init();
+
+	numTS = 0;
+
+	// initialize threads
+	int numThreads = boost::thread::hardware_concurrency();
+	std::cout << "using " << numThreads << " threads" << std::endl;
+	m_barrier1 = new boost::barrier(numThreads+1); // numThread workers + 1 excitation thread
+	m_barrier2 = new boost::barrier(numThreads+1); // numThread workers + 1 excitation thread
+	m_barrier3 = new boost::barrier(numThreads); // numThread workers
+	m_startBarrier = new boost::barrier(numThreads+1); // numThread workers + 1 controller
+	m_stopBarrier = new boost::barrier(numThreads+1); // numThread workers + 1 controller
+
+	for (int n=0; n<numThreads; n++) {
+		unsigned int linesPerThread = (Op->numLines[0]+numThreads-1) / numThreads;
+		unsigned int start = n * linesPerThread;
+		unsigned int stop = min( (n+1) * linesPerThread - 1, Op->numLines[0]-1 );
+		//std::cout << "### " << Op->numLines[0] << " " << linesPerThread << " " << start << " " << stop << std::endl;
+		boost::thread *t = new boost::thread( thread(this,start,stop) );
+		m_thread_group.add_thread( t );
+	}
+	boost::thread *t = new boost::thread( thread_e_excitation(this) );
+	m_thread_group.add_thread( t );
 }
 
 void Engine_Multithread::Reset()
 {
+
+
 	Engine::Reset();
 }
 
 bool Engine_Multithread::IterateTS(unsigned int iterTS)
 {
+	m_iterTS = iterTS;
+	//cout << "bool Engine_Multithread::IterateTS(): starting threads ...";
+	m_startBarrier->wait(); // start the threads
+	//cout << "... threads started";
+
+	m_stopBarrier->wait(); // wait for the threads to finish <iterTS> time steps
+	return true;
+}
+
+
+
+
+
+
+
+thread::thread( Engine_Multithread* ptr, unsigned int start, unsigned int stop ) : m_enginePtr(ptr), m_start(start), m_stop(stop), m_stopThread(false)
+{
+	Op = m_enginePtr->Op;
+	volt = m_enginePtr->volt;
+	curr = m_enginePtr->curr;
+}
+
+void thread::operator()()
+{
+	//std::cout << "thread::operator() Parameters: " << m_start << " " << m_stop << std::endl;
+
 	unsigned int pos[3];
-	int exc_pos;
 	bool shift[3];
 
-	for (unsigned int iter=0;iter<iterTS;++iter)
+	while (!m_stopThread) {
-	{
+		// wait for start
-		//voltage updates
+		//cout << "Thread " << boost::this_thread::get_id() << " waiting..." << endl;
-		for (pos[0]=0;pos[0]<Op->numLines[0];++pos[0])
+		m_enginePtr->m_startBarrier->wait();
+		//cout << "Thread " << boost::this_thread::get_id() << " waiting... started." << endl;
+
+		for (unsigned int iter=0;iter<m_enginePtr->m_iterTS;++iter)
 		{
-			shift[0]=pos[0];
+			//voltage updates
-			for (pos[1]=0;pos[1]<Op->numLines[1];++pos[1])
+			for (pos[0]=m_start;pos[0]<=m_stop;++pos[0])
 			{
-				shift[1]=pos[1];
+				shift[0]=pos[0];
-				for (pos[2]=0;pos[2]<Op->numLines[2];++pos[2])
+				for (pos[1]=0;pos[1]<Op->numLines[1];++pos[1])
 				{
-					shift[2]=pos[2];
+					shift[1]=pos[1];
-					//do the updates here
+					for (pos[2]=0;pos[2]<Op->numLines[2];++pos[2])
-					//for x
+					{
-					volt[0][pos[0]][pos[1]][pos[2]] *= Op->vv[0][pos[0]][pos[1]][pos[2]];
+						shift[2]=pos[2];
-					volt[0][pos[0]][pos[1]][pos[2]] += Op->vi[0][pos[0]][pos[1]][pos[2]] * ( curr[2][pos[0]][pos[1]][pos[2]] - curr[2][pos[0]][pos[1]-shift[1]][pos[2]] - curr[1][pos[0]][pos[1]][pos[2]] + curr[1][pos[0]][pos[1]][pos[2]-shift[2]]);
+						//do the updates here
+						//for x
+						volt[0][pos[0]][pos[1]][pos[2]] *= Op->vv[0][pos[0]][pos[1]][pos[2]];
+						volt[0][pos[0]][pos[1]][pos[2]] += Op->vi[0][pos[0]][pos[1]][pos[2]] * ( curr[2][pos[0]][pos[1]][pos[2]] - curr[2][pos[0]][pos[1]-shift[1]][pos[2]] - curr[1][pos[0]][pos[1]][pos[2]] + curr[1][pos[0]][pos[1]][pos[2]-shift[2]]);
 
-					//for y
+						//for y
-					volt[1][pos[0]][pos[1]][pos[2]] *= Op->vv[1][pos[0]][pos[1]][pos[2]];
+						volt[1][pos[0]][pos[1]][pos[2]] *= Op->vv[1][pos[0]][pos[1]][pos[2]];
-					volt[1][pos[0]][pos[1]][pos[2]] += Op->vi[1][pos[0]][pos[1]][pos[2]] * ( curr[0][pos[0]][pos[1]][pos[2]] - curr[0][pos[0]][pos[1]][pos[2]-shift[2]] - curr[2][pos[0]][pos[1]][pos[2]] + curr[2][pos[0]-shift[0]][pos[1]][pos[2]]);
+						volt[1][pos[0]][pos[1]][pos[2]] += Op->vi[1][pos[0]][pos[1]][pos[2]] * ( curr[0][pos[0]][pos[1]][pos[2]] - curr[0][pos[0]][pos[1]][pos[2]-shift[2]] - curr[2][pos[0]][pos[1]][pos[2]] + curr[2][pos[0]-shift[0]][pos[1]][pos[2]]);
 
-					//for x
+						//for x
-					volt[2][pos[0]][pos[1]][pos[2]] *= Op->vv[2][pos[0]][pos[1]][pos[2]];
+						volt[2][pos[0]][pos[1]][pos[2]] *= Op->vv[2][pos[0]][pos[1]][pos[2]];
-					volt[2][pos[0]][pos[1]][pos[2]] += Op->vi[2][pos[0]][pos[1]][pos[2]] * ( curr[1][pos[0]][pos[1]][pos[2]] - curr[1][pos[0]-shift[0]][pos[1]][pos[2]] - curr[0][pos[0]][pos[1]][pos[2]] + curr[0][pos[0]][pos[1]-shift[1]][pos[2]]);
+						volt[2][pos[0]][pos[1]][pos[2]] += Op->vi[2][pos[0]][pos[1]][pos[2]] * ( curr[1][pos[0]][pos[1]][pos[2]] - curr[1][pos[0]-shift[0]][pos[1]][pos[2]] - curr[0][pos[0]][pos[1]][pos[2]] + curr[0][pos[0]][pos[1]-shift[1]][pos[2]]);
+					}
 				}
 			}
+
+			//cout << "Thread " << boost::this_thread::get_id() << " m_barrier1 waiting..." << endl;
+			m_enginePtr->m_barrier1->wait();
+
+			// e-field excitation (thread thread_e_excitation)
+
+			m_enginePtr->m_barrier2->wait();
+			// e_excitation finished
+
+			//current updates
+			for (pos[0]=m_start;pos[0]<=m_stop-1;++pos[0])
+			{
+				for (pos[1]=0;pos[1]<Op->numLines[1]-1;++pos[1])
+				{
+					for (pos[2]=0;pos[2]<Op->numLines[2]-1;++pos[2])
+					{
+						//do the updates here
+						//for x
+						curr[0][pos[0]][pos[1]][pos[2]] *= Op->ii[0][pos[0]][pos[1]][pos[2]];
+						curr[0][pos[0]][pos[1]][pos[2]] += Op->iv[0][pos[0]][pos[1]][pos[2]] * ( volt[2][pos[0]][pos[1]][pos[2]] - volt[2][pos[0]][pos[1]+1][pos[2]] - volt[1][pos[0]][pos[1]][pos[2]] + volt[1][pos[0]][pos[1]][pos[2]+1]);
+
+						//for y
+						curr[1][pos[0]][pos[1]][pos[2]] *= Op->ii[1][pos[0]][pos[1]][pos[2]];
+						curr[1][pos[0]][pos[1]][pos[2]] += Op->iv[1][pos[0]][pos[1]][pos[2]] * ( volt[0][pos[0]][pos[1]][pos[2]] - volt[0][pos[0]][pos[1]][pos[2]+1] - volt[2][pos[0]][pos[1]][pos[2]] + volt[2][pos[0]+1][pos[1]][pos[2]]);
+
+						//for x
+						curr[2][pos[0]][pos[1]][pos[2]] *= Op->ii[2][pos[0]][pos[1]][pos[2]];
+						curr[2][pos[0]][pos[1]][pos[2]] += Op->iv[2][pos[0]][pos[1]][pos[2]] * ( volt[1][pos[0]][pos[1]][pos[2]] - volt[1][pos[0]+1][pos[1]][pos[2]] - volt[0][pos[0]][pos[1]][pos[2]] + volt[0][pos[0]][pos[1]+1][pos[2]]);
+					}
+				}
+			}
+
+			m_enginePtr->m_barrier3->wait();
+
+			//soft current excitation here (H-field excite)
+
+			++m_enginePtr->numTS;   // FIXME BUG!!!!!     increases not by 1, but by the number of threads!!!!
 		}
 
+		m_enginePtr->m_stopBarrier->wait();
+	}
+}
+
+
+thread_e_excitation::thread_e_excitation( Engine_Multithread* ptr ) : m_enginePtr(ptr), m_stopThread(false)
+{
+	Op = m_enginePtr->Op;
+	volt = m_enginePtr->volt;
+	curr = m_enginePtr->curr;
+}
+
+void thread_e_excitation::operator()()
+{
+	//std::cout << "thread_e_excitation::operator()" << std::endl;
+
+	while (!m_stopThread) {
+
+		// waiting on thread
+		m_enginePtr->m_barrier1->wait();
+
+		int exc_pos;
 		//soft voltage excitation here (E-field excite)
 		for (unsigned int n=0;n<Op->E_Exc_Count;++n)
 		{
-			exc_pos = (int)numTS - (int)Op->E_Exc_delay[n];
+			exc_pos = (int)m_enginePtr->m_numTS - (int)Op->E_Exc_delay[n];
 			exc_pos*= (exc_pos>0 && exc_pos<(int)Op->ExciteLength);
-//			if (n==0) cerr << numTS << " => " << Op->ExciteSignal[exc_pos] << endl;
+	//			if (n==0) cerr << numTS << " => " << Op->ExciteSignal[exc_pos] << endl;
 			volt[Op->E_Exc_dir[n]][Op->E_Exc_index[0][n]][Op->E_Exc_index[1][n]][Op->E_Exc_index[2][n]] += Op->E_Exc_amp[n]*Op->ExciteSignal[exc_pos];
 		}
 
-		//current updates
+		// continueing thread
-		for (pos[0]=0;pos[0]<Op->numLines[0]-1;++pos[0])
+		m_enginePtr->m_barrier2->wait();
-		{
-			for (pos[1]=0;pos[1]<Op->numLines[1]-1;++pos[1])
-			{
-				for (pos[2]=0;pos[2]<Op->numLines[2]-1;++pos[2])
-				{
-					//do the updates here
-					//for x
-					curr[0][pos[0]][pos[1]][pos[2]] *= Op->ii[0][pos[0]][pos[1]][pos[2]];
-					curr[0][pos[0]][pos[1]][pos[2]] += Op->iv[0][pos[0]][pos[1]][pos[2]] * ( volt[2][pos[0]][pos[1]][pos[2]] - volt[2][pos[0]][pos[1]+1][pos[2]] - volt[1][pos[0]][pos[1]][pos[2]] + volt[1][pos[0]][pos[1]][pos[2]+1]);
-
-					//for y
-					curr[1][pos[0]][pos[1]][pos[2]] *= Op->ii[1][pos[0]][pos[1]][pos[2]];
-					curr[1][pos[0]][pos[1]][pos[2]] += Op->iv[1][pos[0]][pos[1]][pos[2]] * ( volt[0][pos[0]][pos[1]][pos[2]] - volt[0][pos[0]][pos[1]][pos[2]+1] - volt[2][pos[0]][pos[1]][pos[2]] + volt[2][pos[0]+1][pos[1]][pos[2]]);
-
-					//for x
-					curr[2][pos[0]][pos[1]][pos[2]] *= Op->ii[2][pos[0]][pos[1]][pos[2]];
-					curr[2][pos[0]][pos[1]][pos[2]] += Op->iv[2][pos[0]][pos[1]][pos[2]] * ( volt[1][pos[0]][pos[1]][pos[2]] - volt[1][pos[0]+1][pos[1]][pos[2]] - volt[0][pos[0]][pos[1]][pos[2]] + volt[0][pos[0]][pos[1]+1][pos[2]]);
-				}
-			}
-		}
-
-		//soft current excitation here (H-field excite)
-		++numTS;
 	}
-	return true;
 }

FDTD/engine_multithread.h

@@ -21,6 +21,8 @@
 #include "operator.h"
 #include "engine.h"
 
+#include <boost/thread.hpp>
+
 class Engine_Multithread : public Engine
 {
 	friend class Processing;
@@ -28,6 +30,8 @@ class Engine_Multithread : public Engine
 	friend class ProcessCurrent;
 	friend class ProcessFields;
 	friend class ProcessFieldsTD;
+	friend class thread;
+	friend class thread_e_excitation;
 public:
 	static Engine_Multithread* createEngine(Operator* op);
 	virtual ~Engine_Multithread();
@@ -38,9 +42,44 @@ public:
 	//!Iterate a number of timesteps
 	virtual bool IterateTS(unsigned int iterTS);
 
+	void doWork(unsigned int start, unsigned int stop, unsigned int iterTS);
+	void doWork_e_excitation(unsigned int start, unsigned int stop, unsigned int iterTS);
+
 
 protected:
 	Engine_Multithread(Operator* op);
+	boost::thread_group m_thread_group;
+	boost::barrier *m_barrier1, *m_barrier2, *m_barrier3, *m_startBarrier, *m_stopBarrier;
+	volatile unsigned int m_iterTS;
+	volatile unsigned int m_numTS;
+};
+
+
+class thread {
+public:
+	thread( Engine_Multithread* ptr, unsigned int start, unsigned int stop );
+	void operator()();
+
+protected:
+	unsigned int m_start, m_stop;
+	volatile bool m_stopThread;
+	Engine_Multithread *m_enginePtr;
+	Operator *Op;
+	FDTD_FLOAT**** volt;
+	FDTD_FLOAT**** curr;
+};
+
+class thread_e_excitation {
+public:
+	thread_e_excitation( Engine_Multithread* ptr);
+	void operator()();
+
+protected:
+	volatile bool m_stopThread;
+	Engine_Multithread *m_enginePtr;
+	Operator *Op;
+	FDTD_FLOAT**** volt;
+	FDTD_FLOAT**** curr;
 };
 
 #endif // ENGINE_MULTITHREAD_H

openEMS.pro

@@ -14,7 +14,8 @@ LIBS += -L../CSXCAD \
     -L../fparser \
     -lfparser \
     -L../tinyxml \
-    -ltinyxml
+	-ltinyxml \
+	-lboost_thread
 QMAKE_LFLAGS += \'-Wl,-rpath,\$$ORIGIN/../CSXCAD\'
 QMAKE_LFLAGS += \'-Wl,-rpath,\$$ORIGIN/../fparser\'
 QMAKE_LFLAGS += \'-Wl,-rpath,\$$ORIGIN/../tinyxml\'