benchmarking: updated transpiler to drastically reduce the number of transpilations at the expense of memory usage

package/src/ExpressionExecutorCuda.jl

@@ -49,19 +49,26 @@ end
 # Convert Expressions to PTX Code and execute that instead
 function evaluate_gpu(expressions::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
 	@assert axes(expressions) == axes(p)
-	variableCols = size(X, 2)
-	variableRows = size(X, 1)
+	numVariableSets = size(X, 2) # nr. of columns of X
+	variableSetSize = size(X, 1) # nr. of rows of X
 	
 	variables = CuArray(X)
 
-	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
+	largestParameterSetSize = Utils.get_max_inner_length(p) # parameters get transformed into matrix. Will be nr. of rows in parameter matrix
+
+	compiledKernels = Vector{CuFunction}(undef, length(expressions)) 
+	kernelName = "evaluate_gpu"
 	@inbounds Threads.@threads for i in eachindex(expressions)
-		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
+		ex = ExpressionProcessing.expr_to_postfix(expressions[i])
+		ptxKernel = Transpiler.transpile(ex, variableSetSize, largestParameterSetSize, numVariableSets, i-1, kernelName) # i-1 because julia is 1-based but PTX needs 0-based indexing
+		compiledKernels[i] = Transpiler.CompileKernel(ptxKernel, kernelName)
 	end
 
-	results = Matrix{Float32}(undef, variableCols, length(exprs))
+	results = Matrix{Float32}(undef, numVariableSets, length(exprs))
 	for i in 1:repetitions # Simulate parameter tuning -> local search (X remains the same, p gets changed in small steps and must be performed sequentially, which it is with this impl)
-		results = Transpiler.evaluate(exprs, variables, variableCols, variableRows, p)
+		# evaluate
+		# results = Transpiler.evaluate(exprs, variables, numVariableSets, variableSetSize, p)
+		results = Transpiler.evaluate(compiledKernels, variables, variableSetSize, p)
 	end
 
 	return results
@@ -103,7 +110,6 @@ function interpret_cpu(exprs::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector
 	res
 end
 
-
 # Flow
 # input: Vector expr    == expressions contains eg. 4 expressions
 #        Matrix X       == |expr| columns, n rows. n == number of variabls x1..xn; n is the same for all expressions --- WRONG

package/src/Transpiler.jl

@@ -14,37 +14,6 @@ const Operand = Union{Float32, String} # Operand is either fixed value or regist
 "
 function evaluate(expressions::Vector{ExpressionProcessing.PostfixType}, cudaVars::CuArray{Float32}, variableColumns::Integer, variableRows::Integer, parameters::Vector{Vector{Float32}})::Matrix{Float32}
 
-	# TODO: test this again with multiple threads. The first time I tried, I was using only one thread
-	# Test this parallel version again when doing performance tests. With the simple "functionality" tests this took 0.03 seconds while sequential took "0.00009" seconds
-	# Threads.@threads for i in eachindex(expressions)
-	# 	cacheLock = ReentrantLock()
-	# 	cacheHit = false
-	# 	lock(cacheLock) do 
-	# 		if haskey(transpilerCache, expressions[i])
-	# 			kernels[i] = transpilerCache[expressions[i]]
-	# 			cacheHit = true
-	# 		end
-	# 	end
-
-	# 	if cacheHit
-	# 		continue
-	# 	end
-
-	# 	formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
-
-	# 	kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableColumns, i-1) # i-1 because julia is 1-based but PTX needs 0-based indexing
-
-	# 	linker = CuLink()
-	# 	add_data!(linker, "ExpressionProcessing", kernel)
-
-	# 	image = complete(linker)
-	
-	# 	mod = CuModule(image)
-	# 	kernels[i] = CuFunction(mod, "ExpressionProcessing")
-
-	# 	@lock cacheLock transpilerCache[expressions[i]] = kernels[i]
-	# end
-
 	cudaParams = Utils.create_cuda_array(parameters, NaN32) # maybe make constant (see PerformanceTests.jl for more info)
 
 	# each expression has nr. of variable sets (nr. of columns of the variables) results and there are n expressions
@@ -54,33 +23,44 @@ function evaluate(expressions::Vector{ExpressionProcessing.PostfixType}, cudaVar
 	blocks = cld(variableColumns, threads)
 	
 	kernelName = "evaluate_gpu"
-	# TODO: Implement batching as a middleground between "transpile everything and then run" and "tranpile one run one" even though cudacall is async
 	@inbounds Threads.@threads for i in eachindex(expressions)
-		# if haskey(resultCache, expressions[i])
-		# 	kernels[i] = resultCache[expressions[i]]
-		# 	continue
-		# end
-		
-		# formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
 		kernel = transpile(expressions[i], variableRows, Utils.get_max_inner_length(parameters), variableColumns, i-1, kernelName) # i-1 because julia is 1-based but PTX needs 0-based indexing
-
-		linker = CuLink()
-		add_data!(linker, kernelName, kernel)
-		
-		image = complete(linker)
-		mod = CuModule(image)
-		compiledKernel = CuFunction(mod, kernelName)
-
+		compiledKernel = CompileKernel(kernel, kernelName)
 		cudacall(compiledKernel, (CuPtr{Float32},CuPtr{Float32},CuPtr{Float32}), cudaVars, cudaParams, cudaResults; threads=threads, blocks=blocks)
 	end
 
-	# for kernel in kernels
-	# 	cudacall(kernel, (CuPtr{Float32},CuPtr{Float32},CuPtr{Float32}), cudaVars, cudaParams, cudaResults; threads=threads, blocks=blocks)
-	# end
-	
 	return cudaResults
 end
 
+"
+A simplified version of the evaluate function. It takes a list of already compiled kernels to be executed. This should yield better performance, where the same expressions should be evaluated multiple times i.e. for parameter optimisation.
+"
+function evaluate(kernels::Vector{CuFunction}, cudaVars::CuArray{Float32}, nrOfVariableSets::Integer, parameters::Vector{Vector{Float32}})::Matrix{Float32}
+
+	cudaParams = Utils.create_cuda_array(parameters, NaN32) # maybe make constant (see PerformanceTests.jl for more info)
+
+	# each expression has nr. of variable sets (nr. of columns of the variables) results and there are n expressions
+	cudaResults = CuArray{Float32}(undef, nrOfVariableSets, length(expressions))
+
+	threads = min(nrOfVariableSets, 256)
+	blocks = cld(nrOfVariableSets, threads)
+	
+	@inbounds Threads.@threads for i in eachindex(kernels)
+		cudacall(kernels[i], (CuPtr{Float32},CuPtr{Float32},CuPtr{Float32}), cudaVars, cudaParams, cudaResults; threads=threads, blocks=blocks)
+	end
+
+	return cudaResults
+end
+
+function CompileKernel(ptxKernel::String, kernelName::String)::CuFunction
+	linker = CuLink()
+	add_data!(linker, kernelName, ptxKernel)
+		
+	image = complete(linker)
+	mod = CuModule(image)
+	return CuFunction(mod, kernelName)
+end
+
 # To increase performance, it would probably be best for all helper functions to return their IO Buffer and not a string
 # seekstart(buf1); write(buf2, buf1)
 "