benchmarking: fixed bugs; took initial_benchmark

package/test/PerformanceTests.jl

@@ -4,6 +4,7 @@ using BenchmarkTools
 using .Transpiler
 using .Interpreter
 
+const BENCHMARKS_RESULTS_PATH = "./results"
 # University setup at 10.20.1.7 if needed
 exprsCPU = [
 	# CPU interpreter requires an anonymous function and array ref s
@@ -24,10 +25,9 @@ exprsGPU = [
 
 # p is the same for CPU and GPU
 p = [randn(Float32, 10) for _ in 1:length(exprsCPU)] # generate 10 random parameter values for each expr
-nrows = 1000
-X = randn(Float32, nrows, 5)
-
 expr_reps = 100 # 100 parameter optimisation steps basically
+
+
 @testset "CPU performance" begin
 	# warmup
 	# interpret_cpu(exprsCPU, X, p)
@@ -42,8 +42,6 @@ expr_reps = 100 # 100 parameter optimisation steps basically
 
 end
 
-ncols = 1000
-X_GPU = randn(Float32, 5, ncols)
 @testset "Interpreter Performance" begin
 	# Put data in shared memory: 
 	# https://cuda.juliagpu.org/v2.6/api/kernel/#Shared-memory
@@ -95,21 +93,54 @@ X_large_GPU = randn(Float32, 5, varsets_large)
 suite["GPUI"]["large varset"] = @benchmarkable interpret_gpu(exprsGPU, X_large_GPU, p; repetitions=expr_reps)
 suite["GPUT"]["large varset"] = @benchmarkable evaluate_gpu(exprsGPU, X_large_GPU, p; repetitions=expr_reps)
 
-tune!(suite)
+# interpret_gpu(exprsGPU, X_large_GPU, p; repetitions=expr_reps)
 
-BenchmarkTools.save("params.json", params(suite))
-# loadparams!(suite, BenchmarkTools.load("params.json")[1], :samples, :evals, :gctrial, :time_tolerance, :evals_set, :gcsample, :seconds, :overhead, :memory_tolerance)
+# tune!(suite)
+# BenchmarkTools.save("params.json", params(suite))
 
-# results = run(suite, verbose=true, seconds=180)
-# results2 = run(suite, verbose=true, seconds=180)
+loadparams!(suite, BenchmarkTools.load("params.json")[1], :samples, :evals, :gctrial, :time_tolerance, :evals_set, :gcsample, :seconds, :overhead, :memory_tolerance)
 
-# medianCPU = median(results["CPU"])
-# medianInterpreter = median(results["GPUI"])
-# medianTranspiler = median(results["GPUT"])
+results = run(suite, verbose=true, seconds=180)
 
-# jud = judge(medianCPU, medianCPU2; time_tolerance=0.001)
-# println(jud)
+# BenchmarkTools.save("$BENCHMARKS_RESULTS_PATH/initial_results.json", results)
+# initial_results = BenchmarkTools.load("$BENCHMARKS_RESULTS_PATHinitial_results.json")
 
-# judge(medianCPU, medianInterpreter; time_tolerance=0.001)
-# judge(medianCPU, medianTranspiler; time_tolerance=0.001)
-# judge(medianInterpreter, medianTranspiler; time_tolerance=0.001)
+medianCPU = median(results["CPU"])
+minimumCPU = minimum(results["CPU"])
+stdCPU = std(results["CPU"])
+
+medianInterpreter = median(results["GPUI"])
+minimumInterpreter = minimum(results["GPUI"])
+stdInterpreter = std(results["GPUI"])
+
+medianTranspiler = median(results["GPUT"])
+minimumTranspiler = minimum(results["GPUT"])
+stdTranspiler = std(results["GPUT"])
+
+cpuVsGPUI_median = judge(medianInterpreter, medianCPU) # is interpreter better than cpu?
+cpuVsGPUT_median = judge(medianTranspiler, medianCPU) # is transpiler better than cpu?
+gpuiVsGPUT_median = judge(medianTranspiler, medianInterpreter) # is tranpiler better than interpreter?
+
+cpuVsGPUI_minimum = judge(minimumInterpreter, minimumCPU) # is interpreter better than cpu?
+cpuVsGPUT_minimum = judge(minimumTranspiler, minimumCPU) # is transpiler better than cpu?
+gpuiVsGPUT_minimum = judge(minimumTranspiler, minimumInterpreter) # is tranpiler better than interpreter?
+
+cpuVsGPUI_std = judge(stdInterpreter, stdCPU) # is interpreter better than cpu?
+cpuVsGPUT_std = judge(stdTranspiler, stdCPU) # is transpiler better than cpu?
+gpuiVsGPUT_std = judge(stdTranspiler, stdInterpreter) # is tranpiler better than interpreter?
+
+
+println("Is the interpreter better than the CPU implementation:")
+println(cpuVsGPUI_median)
+println(cpuVsGPUI_minimum)
+println(cpuVsGPUI_std)
+
+println("Is the transpiler better than the CPU implementation:")
+println(cpuVsGPUT_median)
+println(cpuVsGPUT_minimum)
+println(cpuVsGPUT_std)
+
+println("Is the transpiler better than the interpreter:")
+println(gpuiVsGPUT_median)
+println(gpuiVsGPUT_minimum)
+println(gpuiVsGPUT_std)