benchmarking: moved frontend calls and sending postfixExprs+vars outside to drastically reduce amount of calculations

package/src/ExpressionExecutorCuda.jl

@@ -22,36 +22,45 @@ export evaluate_gpu
 #
 
 # Evaluate Expressions on the GPU
-function interpret_gpu(exprs::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
-	@assert axes(exprs) == axes(p)
-	ncols = size(X, 2)
+function interpret_gpu(expressions::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
+	@assert axes(expressions) == axes(p)
+	variableCols = size(X, 2)
+	variableRows = size(X, 1)
 
-	results = Matrix{Float32}(undef, ncols, length(exprs))
-	# TODO: create CuArray for variables here already, as they never change 
-	#		could/should be done even before calling this, but I guess it would be diminishing returns 
-	# 		TODO: test how this would impact performance, if it gets faster, adapt implementation section
-	# TODO: create CuArray for expressions here already. They also do not change over the course of parameter optimisation and therefore a lot of unnecessary calls to expr_to_postfix can be save (even though a cache is used, this should still be faster)
+	variables = CuArray(X)
 
+	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
+	@inbounds Threads.@threads for i in eachindex(expressions)
+		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
+	end
+	cudaExprs = Utils.create_cuda_array(exprs, ExpressionProcessing.ExpressionElement(EMPTY, 0)) # column corresponds to data for one expression;
+	exprsLength = length(exprs)
+	exprsInnerLength = Utils.get_max_inner_length(exprs)
+	
+	results = Matrix{Float32}(undef, variableCols, length(exprs))
 	for i in 1:repetitions # Simulate parameter tuning -> local search (X remains the same, p gets changed in small steps and must be performed sequentially, which it is with this impl)
-		results = Interpreter.interpret(exprs, X, p)
+		results = Interpreter.interpret(cudaExprs, exprsLength, exprsInnerLength, variables, variableCols, variableRows, p)
 	end
 
 	return results
 end
 
 # Convert Expressions to PTX Code and execute that instead
-function evaluate_gpu(exprs::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
-	@assert axes(exprs) == axes(p)
-	ncols = size(X, 2)
+function evaluate_gpu(expressions::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
+	@assert axes(expressions) == axes(p)
+	variableCols = size(X, 2)
+	variableRows = size(X, 1)
+	
+	variables = CuArray(X)
 
-	results = Matrix{Float32}(undef, ncols, length(exprs))
-	# TODO: create CuArray for variables here already, as they never change 
-	#		could/should be done even before calling this, but I guess it would be diminishing returns 
-	# 		TODO: test how this would impact performance, if it gets faster, adapt implementation section
-	# TODO: create CuArray for expressions here already. They also do not change over the course of parameter optimisation and therefore a lot of unnecessary calls to expr_to_postfix can be save (even though a cache is used, this should still be faster)
+	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
+	@inbounds Threads.@threads for i in eachindex(expressions)
+		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
+	end
 
+	results = Matrix{Float32}(undef, variableCols, length(exprs))
 	for i in 1:repetitions # Simulate parameter tuning -> local search (X remains the same, p gets changed in small steps and must be performed sequentially, which it is with this impl)
-		results = Transpiler.evaluate(exprs, X, p)
+		results = Transpiler.evaluate(exprs, variables, variableCols, variableRows, p)
 	end
 
 	return results

package/src/ExpressionProcessing.jl

@@ -22,7 +22,6 @@ const PostfixType = Vector{ExpressionElement}
 "
 Converts a julia expression to its postfix notation.
 NOTE: All 64-Bit values will be converted to 32-Bit. Be aware of the lost precision.
-NOTE: This function is not thread save, especially cache access is not thread save
 "
 function expr_to_postfix(expression::Expr)::PostfixType
 	expr = expression

package/src/Interpreter.jl

@@ -8,31 +8,25 @@ export interpret
 
 "Interprets the given expressions with the values provided.
 # Arguments
- - expressions::Vector{ExpressionProcessing.PostfixType} : The expressions to execute in postfix form
- - variables::Matrix{Float32} : The variables to use. Each column is mapped to the variables x1..xn
+ - cudaExprs::CuArray{ExpressionProcessing.PostfixType} : The expressions to execute in postfix form and already sent to the GPU. The type information in the signature is missing, because creating a CuArray{ExpressionProcessing.PostfixType} results in a mor everbose type definition
+ - cudaVars::CuArray{Float32} : The variables to use. Each column is mapped to the variables x1..xn. The type information is missing due to the same reasons as cudaExprs
  - parameters::Vector{Vector{Float32}} : The parameters to use. Each Vector contains the values for the parameters p1..pn. The number of parameters can be different for every expression
  - kwparam ```frontendCache```: The cache that stores the (partial) results of the frontend
 "
-function interpret(expressions::Vector{Expr}, variables::Matrix{Float32}, parameters::Vector{Vector{Float32}})::Matrix{Float32}
-	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
-	@inbounds for i in eachindex(expressions)
-		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
-	end
+function interpret(cudaExprs, numExprs::Integer, exprsInnerLength::Integer,
+	cudaVars, variableColumns::Integer, variableRows::Integer, parameters::Vector{Vector{Float32}})::Matrix{Float32}
 
-	variableCols = size(variables, 2) # number of variable sets to use for each expression
-	cudaVars = CuArray(variables)
 	cudaParams = Utils.create_cuda_array(parameters, NaN32) # column corresponds to data for one expression
-	cudaExprs = Utils.create_cuda_array(exprs, ExpressionElement(EMPTY, 0)) # column corresponds to data for one expression;
 	# put into seperate cuArray, as this is static and would be inefficient to send seperatly to each kernel
-	cudaStepsize = CuArray([Utils.get_max_inner_length(exprs), Utils.get_max_inner_length(parameters), size(variables, 1)]) # max num of values per expression; max nam of parameters per expression; number of variables per expression
+	cudaStepsize = CuArray([exprsInnerLength, Utils.get_max_inner_length(parameters), variableRows]) # max num of values per expression; max nam of parameters per expression; number of variables per expression
 
 	# each expression has nr. of variable sets (nr. of columns of the variables) results and there are n expressions
-	cudaResults = CuArray{Float32}(undef, variableCols, length(exprs))
+	cudaResults = CuArray{Float32}(undef, variableColumns, numExprs)
 
 	# Start kernel for each expression to ensure that no warp is working on different expressions
-	@inbounds Threads.@threads for i in eachindex(exprs)
-		numThreads = min(variableCols, 256)
-		numBlocks = cld(variableCols, numThreads)
+	@inbounds Threads.@threads for i in 1:numExprs # multithreaded to speedup dispatching (seems to have improved performance)
+		numThreads = min(variableColumns, 256)
+		numBlocks = cld(variableColumns, numThreads)
 
 		@cuda threads=numThreads blocks=numBlocks fastmath=true interpret_expression(cudaExprs, cudaVars, cudaParams, cudaResults, cudaStepsize, i)
 	end

package/src/Transpiler.jl

@@ -12,10 +12,7 @@ const Operand = Union{Float32, String} # Operand is either fixed value or regist
  - kwparam ```frontendCache```: The cache that stores the (partial) results of the frontend, to speedup the pre-processing
  - kwparam ```frontendCache```: The cache that stores the result of the transpilation. Useful for parameter optimisation, as the same expression gets executed multiple times
 "
-function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, parameters::Vector{Vector{Float32}})::Matrix{Float32}
-	varRows = size(variables, 1)
-	variableCols = size(variables, 2)
-	# kernels = Vector{CuFunction}(undef, length(expressions))
+function evaluate(expressions::Vector{ExpressionProcessing.PostfixType}, cudaVars::CuArray{Float32}, variableColumns::Integer, variableRows::Integer, parameters::Vector{Vector{Float32}})::Matrix{Float32}
 
 	# TODO: test this again with multiple threads. The first time I tried, I was using only one thread
 	# Test this parallel version again when doing performance tests. With the simple "functionality" tests this took 0.03 seconds while sequential took "0.00009" seconds
@@ -35,7 +32,7 @@ function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, paramet
 
 	# 	formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
 
-	# 	kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableCols, i-1) # i-1 because julia is 1-based but PTX needs 0-based indexing
+	# 	kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableColumns, i-1) # i-1 because julia is 1-based but PTX needs 0-based indexing
 
 	# 	linker = CuLink()
 	# 	add_data!(linker, "ExpressionProcessing", kernel)
@@ -48,14 +45,13 @@ function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, paramet
 	# 	@lock cacheLock transpilerCache[expressions[i]] = kernels[i]
 	# end
 
-	cudaVars = CuArray(variables) # maybe put in shared memory (see PerformanceTests.jl for more info)
 	cudaParams = Utils.create_cuda_array(parameters, NaN32) # maybe make constant (see PerformanceTests.jl for more info)
 
 	# each expression has nr. of variable sets (nr. of columns of the variables) results and there are n expressions
-	cudaResults = CuArray{Float32}(undef, variableCols, length(expressions))
+	cudaResults = CuArray{Float32}(undef, variableColumns, length(expressions))
 
-	threads = min(variableCols, 256)
-	blocks = cld(variableCols, threads)
+	threads = min(variableColumns, 256)
+	blocks = cld(variableColumns, threads)
 	
 	kernelName = "evaluate_gpu"
 	# TODO: Implement batching as a middleground between "transpile everything and then run" and "tranpile one run one" even though cudacall is async
@@ -65,8 +61,8 @@ function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, paramet
 		# 	continue
 		# end
 		
-		formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
-		kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableCols, i-1, kernelName) # i-1 because julia is 1-based but PTX needs 0-based indexing
+		# formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
+		kernel = transpile(expressions[i], variableRows, Utils.get_max_inner_length(parameters), variableColumns, i-1, kernelName) # i-1 because julia is 1-based but PTX needs 0-based indexing
 
 		linker = CuLink()
 		add_data!(linker, kernelName, kernel)