hemera benchmark

examples/qed_bench_tape.jl

@@ -200,7 +200,7 @@ bench_qed("ke->kkke")
 bench_qed("ke->kkkke")
 bench_qed("ke->kkkkke")
 bench_qed("ke->kkkkkke")
-bench_qed("ke->kkkkkkke", true)
+bench_qed("ke->kkkkkkke")
 
 bench_abc("AB->AB")
 bench_abc("AB->ABBB")

examples/qed_gen_bench.jl

@@ -4,7 +4,7 @@ using CSV
 using BenchmarkTools
 using StatsBase
 
-results_filename = "qed_gen_results.csv"
+results_filename = "qed_gen_results_$(Threads.nthreads()).csv"
 
 df = DataFrame(
     process_name = String[],
@@ -29,26 +29,10 @@ df = DataFrame(
     graph_edges_reduced = Int[],
     graph_mem = Float64[],
     graph_mem_reduced = Float64[],
+    graph_elapsed_reduce = Float64[],
 )
 
-# if they exist, read existing results and append new ones
-if isfile(results_filename)
-    df = CSV.read(results_filename, DataFrame)
-end
-
-processes = [
-    "ke->ke",
-    "ke->kke",
-    "ke->kkke",
-    "ke->kkkke",
-    "ke->kkkkke",
-    "ke->kkkkkke",
-    "ke->kkkkkkke",
-    "ke->kkkkkkkke",
-    #"ke->kkkkkkkkke",
-]
-
-function bench_process(process::AbstractString)
+function bench_process(process::AbstractString; warmup = false, optimize = true)
     println("Benchmarking $process...")
     model = QEDModel()
 
@@ -62,12 +46,21 @@ function bench_process(process::AbstractString)
     node_dict = countmap(typeof.(graph.nodes))
     graph_size = Base.summarysize(graph)
 
-    optim = ReductionOptimizer()
-    optimize_to_fixpoint!(optim, graph)
+    reduce_elapsed = -1.0
+    node_dict_reduced = Dict()
+    graph_size_reduced = -1.0
+    props_reduced = GraphProperties()
+    if optimize
+        reduce_elapsed = @elapsed optimize_to_fixpoint!(ReductionOptimizer(), graph)
 
-    props_reduced = GraphProperties(graph)
-    node_dict_reduced = countmap(typeof.(graph.nodes))
-    graph_size_reduced = Base.summarysize(graph)
+        props_reduced = GraphProperties(graph)
+        node_dict_reduced = countmap(typeof.(graph.nodes))
+        graph_size_reduced = Base.summarysize(graph)
+    end
+
+    if warmup
+        return nothing
+    end
 
     push!(
         df,
@@ -94,13 +87,58 @@ function bench_process(process::AbstractString)
             :graph_edges_reduced => props_reduced.noEdges,
             :graph_mem => graph_size,
             :graph_mem_reduced => graph_size_reduced,
+            :graph_elapsed_reduce => reduce_elapsed,
         ),
     )
     return nothing
 end
 
-for process in processes
-    bench_process(process)
+processes = [
+    ("ke->ke", true),
+    ("ke->kke", true),
+    ("ke->kkke", true),
+    ("ke->kkkke", true),
+    ("ke->kkkkke", true),
+    ("ke->kkkkkke", true),
+    ("ke->kkkkkkke", true),
+    ("ke->kkkkkkkke", false),
+    ("ke->kkkkkkkkke", false),
+]
+
+df = DataFrame(
+    process_name = String[],
+    cpu_threads = Int[],
+    graph_gen_samples = Int[],
+    graph_gen_mean = Float64[],
+    graph_gen_std = Float64[],
+    graph_gen_median = Float64[],
+    graph_nodes = Int[],
+    graph_data_nodes = Int[],
+    graph_u_nodes = Int[],
+    graph_v_nodes = Int[],
+    graph_s1_nodes = Int[],
+    graph_s2_nodes = Int[],
+    graph_edges = Int[],
+    graph_nodes_reduced = Int[],
+    graph_data_nodes_reduced = Int[],
+    graph_u_nodes_reduced = Int[],
+    graph_v_nodes_reduced = Int[],
+    graph_s1_nodes_reduced = Int[],
+    graph_s2_nodes_reduced = Int[],
+    graph_edges_reduced = Int[],
+    graph_mem = Float64[],
+    graph_mem_reduced = Float64[],
+    graph_elapsed_reduce = Float64[],
+)
+
+# if they exist, read existing results and append new ones
+if isfile(results_filename)
+    df = CSV.read(results_filename, DataFrame)
 end
 
-CSV.write(results_filename, df)
+bench_process("ke->kke", warmup = true)
+
+for (process, opt) in processes
+    bench_process(process, optimize = opt)
+    CSV.write(results_filename, df)
+end