benchmarking: moved frontend calls and sending postfixExprs+vars outside to drastically reduce amount of calculations

package/src/ExpressionExecutorCuda.jl

@@ -22,36 +22,45 @@ export evaluate_gpu
 #
 
 # Evaluate Expressions on the GPU
-function interpret_gpu(exprs::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
-	@assert axes(exprs) == axes(p)
-	ncols = size(X, 2)
+function interpret_gpu(expressions::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
+	@assert axes(expressions) == axes(p)
+	variableCols = size(X, 2)
+	variableRows = size(X, 1)
 
-	results = Matrix{Float32}(undef, ncols, length(exprs))
-	# TODO: create CuArray for variables here already, as they never change 
-	#		could/should be done even before calling this, but I guess it would be diminishing returns 
-	# 		TODO: test how this would impact performance, if it gets faster, adapt implementation section
-	# TODO: create CuArray for expressions here already. They also do not change over the course of parameter optimisation and therefore a lot of unnecessary calls to expr_to_postfix can be save (even though a cache is used, this should still be faster)
+	variables = CuArray(X)
 
+	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
+	@inbounds Threads.@threads for i in eachindex(expressions)
+		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
+	end
+	cudaExprs = Utils.create_cuda_array(exprs, ExpressionProcessing.ExpressionElement(EMPTY, 0)) # column corresponds to data for one expression;
+	exprsLength = length(exprs)
+	exprsInnerLength = Utils.get_max_inner_length(exprs)
+	
+	results = Matrix{Float32}(undef, variableCols, length(exprs))
 	for i in 1:repetitions # Simulate parameter tuning -> local search (X remains the same, p gets changed in small steps and must be performed sequentially, which it is with this impl)
-		results = Interpreter.interpret(exprs, X, p)
+		results = Interpreter.interpret(cudaExprs, exprsLength, exprsInnerLength, variables, variableCols, variableRows, p)
 	end
 
 	return results
 end
 
 # Convert Expressions to PTX Code and execute that instead
-function evaluate_gpu(exprs::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
-	@assert axes(exprs) == axes(p)
-	ncols = size(X, 2)
+function evaluate_gpu(expressions::Vector{Expr}, X::Matrix{Float32}, p::Vector{Vector{Float32}}; repetitions=1)::Matrix{Float32}
+	@assert axes(expressions) == axes(p)
+	variableCols = size(X, 2)
+	variableRows = size(X, 1)
 	
-	results = Matrix{Float32}(undef, ncols, length(exprs))
-	# TODO: create CuArray for variables here already, as they never change 
-	#		could/should be done even before calling this, but I guess it would be diminishing returns 
-	# 		TODO: test how this would impact performance, if it gets faster, adapt implementation section
-	# TODO: create CuArray for expressions here already. They also do not change over the course of parameter optimisation and therefore a lot of unnecessary calls to expr_to_postfix can be save (even though a cache is used, this should still be faster)
+	variables = CuArray(X)
 
+	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
+	@inbounds Threads.@threads for i in eachindex(expressions)
+		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
+	end
+
+	results = Matrix{Float32}(undef, variableCols, length(exprs))
 	for i in 1:repetitions # Simulate parameter tuning -> local search (X remains the same, p gets changed in small steps and must be performed sequentially, which it is with this impl)
-		results = Transpiler.evaluate(exprs, X, p)
+		results = Transpiler.evaluate(exprs, variables, variableCols, variableRows, p)
 	end
 
 	return results

package/src/ExpressionProcessing.jl

@@ -22,7 +22,6 @@ const PostfixType = Vector{ExpressionElement}
 "
 Converts a julia expression to its postfix notation.
 NOTE: All 64-Bit values will be converted to 32-Bit. Be aware of the lost precision.
-NOTE: This function is not thread save, especially cache access is not thread save
 "
 function expr_to_postfix(expression::Expr)::PostfixType
 	expr = expression

package/src/Interpreter.jl

@@ -8,31 +8,25 @@ export interpret
 
 "Interprets the given expressions with the values provided.
 # Arguments
- - expressions::Vector{ExpressionProcessing.PostfixType} : The expressions to execute in postfix form
- - variables::Matrix{Float32} : The variables to use. Each column is mapped to the variables x1..xn
+ - cudaExprs::CuArray{ExpressionProcessing.PostfixType} : The expressions to execute in postfix form and already sent to the GPU. The type information in the signature is missing, because creating a CuArray{ExpressionProcessing.PostfixType} results in a mor everbose type definition
+ - cudaVars::CuArray{Float32} : The variables to use. Each column is mapped to the variables x1..xn. The type information is missing due to the same reasons as cudaExprs
  - parameters::Vector{Vector{Float32}} : The parameters to use. Each Vector contains the values for the parameters p1..pn. The number of parameters can be different for every expression
  - kwparam ```frontendCache```: The cache that stores the (partial) results of the frontend
 "
-function interpret(expressions::Vector{Expr}, variables::Matrix{Float32}, parameters::Vector{Vector{Float32}})::Matrix{Float32}
-	exprs = Vector{ExpressionProcessing.PostfixType}(undef, length(expressions))
-	@inbounds for i in eachindex(expressions)
-		exprs[i] = ExpressionProcessing.expr_to_postfix(expressions[i])
-	end
+function interpret(cudaExprs, numExprs::Integer, exprsInnerLength::Integer,
+	cudaVars, variableColumns::Integer, variableRows::Integer, parameters::Vector{Vector{Float32}})::Matrix{Float32}
 
-	variableCols = size(variables, 2) # number of variable sets to use for each expression
-	cudaVars = CuArray(variables)
 	cudaParams = Utils.create_cuda_array(parameters, NaN32) # column corresponds to data for one expression
-	cudaExprs = Utils.create_cuda_array(exprs, ExpressionElement(EMPTY, 0)) # column corresponds to data for one expression;
 	# put into seperate cuArray, as this is static and would be inefficient to send seperatly to each kernel
-	cudaStepsize = CuArray([Utils.get_max_inner_length(exprs), Utils.get_max_inner_length(parameters), size(variables, 1)]) # max num of values per expression; max nam of parameters per expression; number of variables per expression
+	cudaStepsize = CuArray([exprsInnerLength, Utils.get_max_inner_length(parameters), variableRows]) # max num of values per expression; max nam of parameters per expression; number of variables per expression
 
 	# each expression has nr. of variable sets (nr. of columns of the variables) results and there are n expressions
-	cudaResults = CuArray{Float32}(undef, variableCols, length(exprs))
+	cudaResults = CuArray{Float32}(undef, variableColumns, numExprs)
 
 	# Start kernel for each expression to ensure that no warp is working on different expressions
-	@inbounds Threads.@threads for i in eachindex(exprs)
-		numThreads = min(variableCols, 256)
-		numBlocks = cld(variableCols, numThreads)
+	@inbounds Threads.@threads for i in 1:numExprs # multithreaded to speedup dispatching (seems to have improved performance)
+		numThreads = min(variableColumns, 256)
+		numBlocks = cld(variableColumns, numThreads)
 
 		@cuda threads=numThreads blocks=numBlocks fastmath=true interpret_expression(cudaExprs, cudaVars, cudaParams, cudaResults, cudaStepsize, i)
 	end

package/src/Transpiler.jl

@@ -12,10 +12,7 @@ const Operand = Union{Float32, String} # Operand is either fixed value or regist
  - kwparam ```frontendCache```: The cache that stores the (partial) results of the frontend, to speedup the pre-processing
  - kwparam ```frontendCache```: The cache that stores the result of the transpilation. Useful for parameter optimisation, as the same expression gets executed multiple times
 "
-function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, parameters::Vector{Vector{Float32}})::Matrix{Float32}
-	varRows = size(variables, 1)
-	variableCols = size(variables, 2)
-	# kernels = Vector{CuFunction}(undef, length(expressions))
+function evaluate(expressions::Vector{ExpressionProcessing.PostfixType}, cudaVars::CuArray{Float32}, variableColumns::Integer, variableRows::Integer, parameters::Vector{Vector{Float32}})::Matrix{Float32}
 
 	# TODO: test this again with multiple threads. The first time I tried, I was using only one thread
 	# Test this parallel version again when doing performance tests. With the simple "functionality" tests this took 0.03 seconds while sequential took "0.00009" seconds
@@ -35,7 +32,7 @@ function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, paramet
 
 	# 	formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
 
-	# 	kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableCols, i-1) # i-1 because julia is 1-based but PTX needs 0-based indexing
+	# 	kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableColumns, i-1) # i-1 because julia is 1-based but PTX needs 0-based indexing
 
 	# 	linker = CuLink()
 	# 	add_data!(linker, "ExpressionProcessing", kernel)
@@ -48,14 +45,13 @@ function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, paramet
 	# 	@lock cacheLock transpilerCache[expressions[i]] = kernels[i]
 	# end
 
-	cudaVars = CuArray(variables) # maybe put in shared memory (see PerformanceTests.jl for more info)
 	cudaParams = Utils.create_cuda_array(parameters, NaN32) # maybe make constant (see PerformanceTests.jl for more info)
 
 	# each expression has nr. of variable sets (nr. of columns of the variables) results and there are n expressions
-	cudaResults = CuArray{Float32}(undef, variableCols, length(expressions))
+	cudaResults = CuArray{Float32}(undef, variableColumns, length(expressions))
 
-	threads = min(variableCols, 256)
-	blocks = cld(variableCols, threads)
+	threads = min(variableColumns, 256)
+	blocks = cld(variableColumns, threads)
 	
 	kernelName = "evaluate_gpu"
 	# TODO: Implement batching as a middleground between "transpile everything and then run" and "tranpile one run one" even though cudacall is async
@@ -65,8 +61,8 @@ function evaluate(expressions::Vector{Expr}, variables::Matrix{Float32}, paramet
 		# 	continue
 		# end
 		
-		formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
-		kernel = transpile(formattedExpr, varRows, Utils.get_max_inner_length(parameters), variableCols, i-1, kernelName) # i-1 because julia is 1-based but PTX needs 0-based indexing
+		# formattedExpr = ExpressionProcessing.expr_to_postfix(expressions[i])
+		kernel = transpile(expressions[i], variableRows, Utils.get_max_inner_length(parameters), variableColumns, i-1, kernelName) # i-1 because julia is 1-based but PTX needs 0-based indexing
 
 		linker = CuLink()
 		add_data!(linker, kernelName, kernel)

package/test/InterpreterTests.jl

@@ -21,8 +21,16 @@ parameters[2][1] = 5.0
 parameters[2][2] = 0.0
 
 function testHelper(expression::Expr, variables::Matrix{Float32}, parameters::Vector{Vector{Float32}}, expectedResult)
-	exprs = Vector([expression])
-	result = Interpreter.interpret(exprs, variables, parameters)
+	exprs = [ExpressionProcessing.expr_to_postfix(expression)]
+	cudaExprs = Utils.create_cuda_array(exprs, ExpressionProcessing.ExpressionElement(EMPTY, 0))
+	exprsLength = length(exprs)
+	exprsInnerLength = Utils.get_max_inner_length(exprs)
+
+	X = CuArray(variables)
+	variableCols = size(variables, 2)
+	variableRows = size(variables, 1)
+
+	result = Interpreter.interpret(cudaExprs, exprsLength, exprsInnerLength, X, variableCols, variableRows, parameters)
 
 	expectedResult32 = convert(Float32, expectedResult)
 	@test isequal(result[1,1], expectedResult32)
@@ -127,8 +135,16 @@ end
 	expr1 = :((x1 + 5) * p1 - 3 / abs(x2) + (2^4) - log(8))
 	expr2 = :(1 + 5 * x1 - 10^2 + (p1 - p2) / 9 + exp(x2))
 
-	exprs = Vector([expr1, expr2])
-	result = Interpreter.interpret(exprs, var, param)
+	exprs = [ExpressionProcessing.expr_to_postfix(expr1), ExpressionProcessing.expr_to_postfix(expr2)]
+	cudaExprs = Utils.create_cuda_array(exprs, ExpressionProcessing.ExpressionElement(EMPTY, 0))
+	exprsLength = length(exprs)
+	exprsInnerLength = Utils.get_max_inner_length(exprs)
+
+	X = CuArray(var)
+	variableCols = size(var, 2)
+	variableRows = size(var, 1)
+
+	result = Interpreter.interpret(cudaExprs, exprsLength, exprsInnerLength, X, variableCols, variableRows, param)
 
 	# var set 1
 	@test isapprox(result[1,1], 37.32, atol=0.01) # expr1

package/test/PerformanceTests.jl

@@ -10,6 +10,7 @@ using .ExpressionProcessing
 
 include("parser.jl") # to parse expressions from a file
 
+# ATTENTAION: Evaluation information at the very bottom
 const BENCHMARKS_RESULTS_PATH = "./results-fh-new"
 
 # Number of expressions can get really big (into millions)
@@ -68,7 +69,7 @@ suite["GPUT"]["nikuradse_1"] = @benchmarkable evaluate_gpu(exprs, X_t, parameter
 
 loadparams!(suite, BenchmarkTools.load("params.json")[1], :samples, :evals, :gctrial, :time_tolerance, :evals_set, :gcsample, :seconds, :overhead, :memory_tolerance)
 
-results = run(suite, verbose=true, seconds=28800) # 8 hour timeout
+results = run(suite, verbose=true, seconds=43200) # 12 hour timeout
 resultsCPU = BenchmarkTools.load("$BENCHMARKS_RESULTS_PATH/cpu.json")[1]
 
 if compareWithCPU
@@ -139,3 +140,8 @@ else
 	println(oldVsGPUT_std)
 end
 
+
+
+# Initial implementation: 
+#	- Interpreter: no cache; 256 blocksize; exprs pre-processed and sent to GPU on every call; vars sent on every call; frontend + dispatch are multithreaded
+#	- Transpiler: no cahce; 256 blocksize; exprs pre-processed and transpiled on every call; vars sent on every call; frontend + transpilation + dispatch are multithreaded

package/test/TranspilerTests.jl

@@ -41,19 +41,15 @@ parameters[2][1] = 5.0
 parameters[2][2] = 0.0
 parameters[3][1] = 16.0
 
-@testset "TEMP" begin
-	return
-	exprs = [:(x1 + p1)]
-	vars = Matrix{Float32}(undef, 1, 1)
-	params = Vector{Vector{Float32}}(undef, 1)
-
-	vars[1, 1] = 1
-	params[1] = [1]
-	Transpiler.evaluate(exprs, vars, params)
-end
 
 @testset "Test transpiler evaluation" begin
-	results = Transpiler.evaluate(expressions, variables, parameters)
+	variableCols = size(variables, 2)
+	variableRows = size(variables, 1)
+	X = CuArray(variables)
+
+	exprs = [ExpressionProcessing.expr_to_postfix(expressions[1]), ExpressionProcessing.expr_to_postfix(expressions[2]), ExpressionProcessing.expr_to_postfix(expressions[3])]
+
+	results = Transpiler.evaluate(exprs, X, variableCols, variableRows, parameters)
 
 	# dump(expressions[3]; maxdepth=10)
 	# Expr 1:

package/test/runtests.jl

@@ -12,7 +12,7 @@ include(joinpath(baseFolder, "src", "Transpiler.jl"))
 @testset "Functionality tests" begin
 	# include("ExpressionProcessingTests.jl")
 	# include("InterpreterTests.jl")
-	# include("TranspilerTests.jl")
+	include("TranspilerTests.jl")
 end
 
 
@@ -22,5 +22,5 @@ end
 
 @testset "Performance tests" begin
 	# include("PerformanceTuning.jl")
-	include("PerformanceTests.jl")
+	# include("PerformanceTests.jl")
 end