benchmarking: prepared tests for using actual data

package/test/PerformanceTests.jl

@@ -4,48 +4,40 @@ using BenchmarkTools
 using .Transpiler
 using .Interpreter
 
-const BENCHMARKS_RESULTS_PATH = "./results-fh"
+const BENCHMARKS_RESULTS_PATH = "./results-fh-new"
 
-# TODO: Expressions can get much much bigger (into millions) (will be provided by Mr. Kronberger)
-# TODO: Variable-Sets: 1000 can be considered the minimum; 100.000 can be considered the maximum (will be provided by Mr. Kronberger)
+# Number of expressions can get really big (into millions)
+# Variable-Sets: 1000 can be considered the minimum; 100.000 can be considered the maximum
 
-exprsCPU = [
-	# CPU interpreter requires an anonymous function and array ref s
-	:(p[1] * x[1] + p[2]), # 5 op
-	:((((x[1] + x[2]) + x[3]) + x[4]) + x[5]), # 9 op
-	:(log(abs(x[1]))), # 3 op
-	:(powabs(p[2] - powabs(p[1] + x[1], 1/x[1]),p[3])) # 13 op
-] # 30 op
-exprsCPU = map(e -> Expr(:->, :(x,p), e), exprsCPU)
+data,varnames = readdlm("data/nikuradse_1.csv", ',', header=true);
+X = convert(Matrix{Float32}, data)
 
-exprsGPU = [
-	# CPU interpreter requires an anonymous function and array ref s
-	:(p1 * x1 + p2), # 5 op
-	:((((x1 + x2) + x3) + x4) + x5), # 9 op
-	:(log(abs(x1))), # 3 op
-	:(powabs(p2 - powabs(p1 + x1, 1/x1),p3)) # 13 op
-] # 30 op
+exprs = Expr[]
+parameters = Vector{Vector{Float32}}()
+varnames = ["x$i" for i in 1:10]
+paramnames = ["p$i" for i in 1:20]
+# data/esr_nvar2_len10.txt.gz_9.txt.gz has  ~250_000 exprs
+# data/esr_nvar2_len10.txt.gz_10.txt.gz has ~800_000 exrps
+GZip.open("data/esr_nvar2_len10.txt.gz_9.txt.gz") do io 
+	i = 0
+	for line in eachline(io)
+		expr, p = parse_infix(line, varnames, paramnames)
+		
+		if i > 10
+			return
+		end
+		println(expr)
 
-# p is the same for CPU and GPU
-p = [randn(Float32, 10) for _ in 1:length(exprsCPU)] # generate 10 random parameter values for each expr
+		push!(exprs, expr)
+		push!(parameters, randn(Float32, length(p)))
+
+		i += 1
+	end
+end
 expr_reps = 100 # 100 parameter optimisation steps (local search; sequentially; only p changes but not X)
 
 
-@testset "CPU performance" begin
-	# warmup
-	# interpret_cpu(exprsCPU, X, p)
-	
-	# @btime interpret_cpu(exprsCPU, X, p; repetitions=expr_reps) # repetitions simulates parameter optimisation
-	# @btime test_cpu_interpreter(1000)
-	# @btime fetch.([Threads.@spawn interpret_cpu(exprsCPU, X, p; repetitions=expr_reps) for i in 1:reps])
-
-	# test_cpu_interpreter(1000, parallel=true) # start julia -t 6 for six threads
-	# @btime test_cpu_interpreter(10000)
-	# @btime test_cpu_interpreter(10000, parallel=true)
-
-end
-
-@testset "Interpreter Performance" begin
+# TODO: Tipps for tuning:
 	# Put data in shared memory: 
 	# https://cuda.juliagpu.org/v2.6/api/kernel/#Shared-memory
 
@@ -54,62 +46,31 @@ end
 
 	# Memory management like in C++ might help with performance improvements
 	# https://cuda.juliagpu.org/v2.6/lib/driver/#Memory-Management
-end
 
-@testset "Transpiler Performance" begin
-	# Put data in shared memory: 
-	# https://cuda.juliagpu.org/v2.6/api/kernel/#Shared-memory
-
-	# Make array const:
-	# https://cuda.juliagpu.org/v2.6/api/kernel/#Device-arrays
-
-	# Memory management like in C++ might help with performance improvements
-	# https://cuda.juliagpu.org/v2.6/lib/driver/#Memory-Management
-end
-
-# After these tests have been redone, use Nsight Compute/Systems as described here: 
-#https://cuda.juliagpu.org/stable/development/profiling/#NVIDIA-Nsight-Systems
+# https://cuda.juliagpu.org/stable/development/profiling/#NVIDIA-Nsight-Systems
 # Systems and Compute installable via WSL. Compute UI can even be used inside wsl
-# Add /usr/local/cuda/bin in .bashrc to PATH to access ncu and nsys (depending how well this works with my 1080 do it on my machine, otherwise re do the tests and perform them on FH PCs)
-# University setup at 10.20.1.7 if needed
-
-compareWithCPU = false
+# Add /usr/local/cuda/bin in .bashrc to PATH to access ncu and nsys (do the tests on FH PCs)
+# University setup at 10.20.1.7 and 10.20.1.13
 
+compareWithCPU = true
 
 suite = BenchmarkGroup()
 suite["CPU"] = BenchmarkGroup(["CPUInterpreter"])
 suite["GPUI"] = BenchmarkGroup(["GPUInterpreter"])
 suite["GPUT"] = BenchmarkGroup(["GPUTranspiler"])
-# TODO: see CpuInterpreterTests.jl to see how all data is loaded and implement this here
-varsets_small = 1000 # 1k should be absolute minimum
-varsets_medium = 10000
-varsets_large = 100000 # 100k should be absolute maximum (although not as strict as minimum)
 
 if compareWithCPU
-	X_small = randn(Float32, varsets_small, 5)
-	suite["CPU"]["small varset"] = @benchmarkable interpret_cpu(exprsCPU, X_small, p; repetitions=expr_reps)
-	X_medium = randn(Float32, varsets_medium, 5)
-	suite["CPU"]["medium varset"] = @benchmarkable interpret_cpu(exprsCPU, X_medium, p; repetitions=expr_reps)
-	X_large = randn(Float32, varsets_large, 5)
-	suite["CPU"]["large varset"] = @benchmarkable interpret_cpu(exprsCPU, X_large, p; repetitions=expr_reps)
+	suite["CPU"]["nikuradse_1"] = @benchmarkable interpret_cpu(exprsCPU, X, parameters; repetitions=expr_reps)
 end
 
-X_small_GPU = randn(Float32, 5, varsets_small) # column-major
-suite["GPUI"]["small varset"] = @benchmarkable interpret_gpu(exprsGPU, X_small_GPU, p; repetitions=expr_reps)
-suite["GPUT"]["small varset"] = @benchmarkable evaluate_gpu(exprsGPU, X_small_GPU, p; repetitions=expr_reps)
+# TODO: Most likely need to transpose X matrix here, as we are expecting a column major matrix for more efficient memory access
+suite["GPUI"]["nikuradse_1"] = @benchmarkable interpret_gpu(exprsGPU, X, parameters; repetitions=expr_reps)
+suite["GPUT"]["nikuradse_1"] = @benchmarkable evaluate_gpu(exprsGPU, X, parameters; repetitions=expr_reps)
 
-X_medium_GPU = randn(Float32, 5, varsets_medium) # column-major
-suite["GPUI"]["medium varset"] = @benchmarkable interpret_gpu(exprsGPU, X_medium_GPU, p; repetitions=expr_reps)
-suite["GPUT"]["medium varset"] = @benchmarkable evaluate_gpu(exprsGPU, X_medium_GPU, p; repetitions=expr_reps)
-
-X_large_GPU = randn(Float32, 5, varsets_large) # column-major
-suite["GPUI"]["large varset"] = @benchmarkable interpret_gpu(exprsGPU, X_large_GPU, p; repetitions=expr_reps)
-suite["GPUT"]["large varset"] = @benchmarkable evaluate_gpu(exprsGPU, X_large_GPU, p; repetitions=expr_reps)
-
-# interpret_gpu(exprsGPU, X_large_GPU, p; repetitions=expr_reps)
-
-# tune!(suite)
-# BenchmarkTools.save("params.json", params(suite))
+for i in 1:10
+	tune!(suite)
+end
+BenchmarkTools.save("params.json", params(suite))
 
 loadparams!(suite, BenchmarkTools.load("params.json")[1], :samples, :evals, :gctrial, :time_tolerance, :evals_set, :gcsample, :seconds, :overhead, :memory_tolerance)
 
@@ -148,7 +109,7 @@ if compareWithCPU
 	println(gpuiVsGPUT_median)
 	println(gpuiVsGPUT_std)
 	
-	BenchmarkTools.save("$BENCHMARKS_RESULTS_PATH/5-interpreter_using_fastmath.json", results)
+	BenchmarkTools.save("$BENCHMARKS_RESULTS_PATH/0_initial.json", results)
 else
 	resultsOld = BenchmarkTools.load("$BENCHMARKS_RESULTS_PATH/3-tuned-blocksize_I128_T96.json")[1]
 	# resultsOld = BenchmarkTools.load("$BENCHMARKS_RESULTS_PATH/3-tuned-blocksize_I128_T96.json")[1]