In the meantime, I also tried an OpenCL implementation. For big matrixes the performances gain is really significative. (Works better on float than double on my GPU.)

If you have time, adding an OpenCL implemented in your benchmark can be a good idea for the sake of completeness of your new post.

As a starting point, here is my very first draft of OpenCL implementation:

namespace MatrixMultiplication
{
public class OpenCL
{
static public ComputeContext Context { get; private set; }
static public ComputeDevice Device { get; private set; }
static OpenCL()
{
ComputePlatform platform = ComputePlatform.Platforms.FirstOrDefault();
if (platform == null) throw new Exception(“ERROR – No OpenCL platform available”);
ComputeContextPropertyList properties = new ComputeContextPropertyList(platform);
Context = new ComputeContext(ComputeDeviceTypes.All, properties, null, IntPtr.Zero);
Device = Context.Devices.FirstOrDefault();
if (Device == null) throw new Exception(“ERROR – No OpenCL device available”);
if (!Device.Extensions.Contains(“cl_khr_fp64″)) throw new Exception(“ERROR – OpenCL device doesn’t support doubles”);
}

public static TimeSpan Multiply(int N)
{
Random rand = new Random();
MatrixOpenCL b = new MatrixOpenCL(N, N); b.Apply(d => rand.Next());
MatrixOpenCL c = new MatrixOpenCL(N, N); c.Apply(d => rand.Next());
Stopwatch stopwatch = Stopwatch.StartNew();
MatrixOpenCL a = b * c;
stopwatch.Stop();
return stopwatch.Elapsed;
}
}

public class MatrixOpenCL
{
const String programSource = @”
#pragma OPENCL EXTENSION cl_khr_fp64 : enable
__kernel void MatrixMultiply(__global double* C, __global double* A, __global double* B, int aCols, int bCols)
{
int i = get_global_id(0);
int j = get_global_id(1);
double value = 0;
for (int k = 0; k < bCols; ++k)
value += A[i*aCols+k] * B[k*bCols+j];
C[i*bCols+j] = value;
}";

static ComputeKernel matrixMultiplyKernel;
static MatrixOpenCL()
{
ComputeProgram program = new ComputeProgram(OpenCL.Context, new string[] { programSource });
program.Build(null, null, null, IntPtr.Zero);
matrixMultiplyKernel = program.CreateKernel("MatrixMultiply");
}

public int NumLines { get; private set; }
public int NumCols { get; private set; }
double[] elements;
public MatrixOpenCL(int numLines, int numCols)
{
NumLines = numLines; NumCols = numCols;
elements = new double[numLines*numCols];
}
public static MatrixOpenCL operator *(MatrixOpenCL a, MatrixOpenCL b)
{
if (a.NumCols != b.NumLines) throw new ArgumentException();
MatrixOpenCL c = new MatrixOpenCL(a.NumLines, b.NumCols);
ComputeBuffer bufA = new ComputeBuffer(OpenCL.Context, ComputeMemoryFlags.ReadOnly, a.elements);
ComputeBuffer bufB = new ComputeBuffer(OpenCL.Context, ComputeMemoryFlags.ReadOnly, b.elements);
ComputeBuffer bufC = new ComputeBuffer(OpenCL.Context, ComputeMemoryFlags.ReadWrite, c.elements);
matrixMultiplyKernel.SetMemoryArgument(0, bufC);
matrixMultiplyKernel.SetMemoryArgument(1, bufA);
matrixMultiplyKernel.SetMemoryArgument(2, bufB);
matrixMultiplyKernel.SetValueArgument(3, a.NumCols);
matrixMultiplyKernel.SetValueArgument(4, b.NumCols);
ComputeCommandQueue commands = new ComputeCommandQueue(OpenCL.Context, OpenCL.Device, ComputeCommandQueueFlags.None);
long y = ((long)c.NumCols + OpenCL.Device.MaxWorkGroupSize – 1) / OpenCL.Device.MaxWorkGroupSize;
long x = ((long)c.NumLines * c.NumCols / y + OpenCL.Device.MaxWorkGroupSize – 1) / OpenCL.Device.MaxWorkGroupSize;
commands.Execute(matrixMultiplyKernel, null, new long[] { c.NumLines, c.NumCols }, new long[] { c.NumLines / x, c.NumCols / y }, null);
commands.ReadFromBuffer(bufC, ref c.elements, true, null);
commands.Finish(); commands.Dispose();
bufA.Dispose(); bufB.Dispose(); bufC.Dispose();
return c;
}
public double this[int i, int j]
{
get { return elements[i * NumCols + j]; }
set { elements[i * NumCols + j] = value; }
}
public void Apply(Func function)
{
Parallel.For(0, elements.Length, i => elements[i] = function(elements[i]));
}

}

}

I made the adjustment you mentioned and the two parallel implementations are virtually identical. I will make a new post soon with those results. Thanks for reading and testing.

-Ian

The performance difference is very likely coming from the fact that in the SimpleParallel lambda, the C# compiler decided to capture the variables by reference (equivalent to the C++11 lambda closure [&]). By providing a static method, I forced those variables to be passed by value, therefore improving the performances.

To confirm this analysis, I changed the static function to the following code, forcing values to be passed as references. It gives back the original performances of SimpleParallel method, as expected.

private static void Multiply(int i, ref int N, ref double[][] A, ref double[][] B, ref double[][] C)

The next challenge was to modify the initial lamba expression, to have the good performances, without the need of a static function. The following syntax worked: it was probably explicit enough for the C# compiler to undertand that it could capture the variables by value.

Parallel.For(0, N, i => { int n = N; double[][] a = A, b = B, c = C; for (int k = 0; k < n; k++) for (int j = 0; j < n; j++) c[i][j] += a[i][k] * b[k][j]; });

The only different is that I call a static method instead of a lambda function in the Parallel.For loop.

Therefore, I’m wondering if the improvement of OptimizedParallel is really due to a better partitioning compared to the default partitioning method of Parallel.For, or because lambda functions are causing a significant overhead in the processing… and why such an overhead if it is really the case…public static TimeSpan Multiply(int N)
public static TimeSpan Multiply(int N)

{
var C = new double[N][];

var A = new double[N][];
var B = new double[N][];
Util.Initialize(N, A, B, C);
Stopwatch stopwatch = Stopwatch.StartNew();
Parallel.For(0, N, i => Multiply(i, N, A, B, C));

stopwatch.Stop();
return stopwatch.Elapsed;

}
private static void Multiply(int i, int N, double[][] A, double[][] B, double[][] C)

{
for (int k = 0; k < N; k++) for (int j = 0; j < N; j++) C[i][j] += A[i][k] * B[k][j];

}