diff --git a/README.md b/README.md
index d27a115..a66bec4 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,9 @@ Times are from my home machine: AMD Ryzen 7900X3D, 64GB DDR5 RAM @ 6000MHz
$ julia --project examples/import_bench.jl
AB->AB:
Graph:
- Nodes: Total: 34, DataTask: 19, ComputeTaskP: 4, ComputeTaskS2: 2, ComputeTaskV: 4, ComputeTaskU: 4, ComputeTaskSum: 1
+ Nodes: Total: 34, DataTask: 19, ComputeTaskP: 4,
+ ComputeTaskS2: 2, ComputeTaskV: 4, ComputeTaskU: 4,
+ ComputeTaskSum: 1
Edges: 37
Total Compute Effort: 185
Total Data Transfer: 102
@@ -43,7 +45,9 @@ Graph:
AB->ABBB:
Graph:
- Nodes: Total: 280, DataTask: 143, ComputeTaskP: 6, ComputeTaskS2: 24, ComputeTaskV: 64, ComputeTaskU: 6, ComputeTaskSum: 1, ComputeTaskS1: 36
+ Nodes: Total: 280, DataTask: 143, ComputeTaskP: 6,
+ ComputeTaskS2: 24, ComputeTaskV: 64, ComputeTaskU: 6,
+ ComputeTaskSum: 1, ComputeTaskS1: 36
Edges: 385
Total Compute Effort: 2007
Total Data Transfer: 828
@@ -53,7 +57,9 @@ Graph:
AB->ABBBBB:
Graph:
- Nodes: Total: 7854, DataTask: 3931, ComputeTaskP: 8, ComputeTaskS2: 720, ComputeTaskV: 1956, ComputeTaskU: 8, ComputeTaskSum: 1, ComputeTaskS1: 1230
+ Nodes: Total: 7854, DataTask: 3931, ComputeTaskP: 8,
+ ComputeTaskS2: 720, ComputeTaskV: 1956, ComputeTaskU: 8,
+ ComputeTaskSum: 1, ComputeTaskS1: 1230
Edges: 11241
Total Compute Effort: 58789
Total Data Transfer: 23244
@@ -72,7 +78,9 @@ Graph:
AB->ABBBBBBB:
Graph:
- Nodes: Total: 438436, DataTask: 219223, ComputeTaskP: 10, ComputeTaskS2: 40320, ComputeTaskV: 109600, ComputeTaskU: 10, ComputeTaskSum: 1, ComputeTaskS1: 69272
+ Nodes: Total: 438436, DataTask: 219223, ComputeTaskP: 10,
+ ComputeTaskS2: 40320, ComputeTaskV: 109600, ComputeTaskU: 10,
+ ComputeTaskSum: 1, ComputeTaskS1: 69272
Edges: 628665
Total Compute Effort: 3288131
Total Data Transfer: 1297700
@@ -80,9 +88,20 @@ Graph:
Graph size in memory: 118.4037 MiB
463.082 ms (7645256 allocations: 422.57 MiB)
+AB->ABBBBBBBBB:
+Graph:
+ Nodes: Total: 39456442, DataTask: 19728227, ComputeTaskS1: 6235290, ComputeTaskP: 12, ComputeTaskU: 12, ComputeTaskV: 9864100, ComputeTaskS2: 3628800, ComputeTaskSum: 1
+ Edges: 56578129
+ Total Compute Effort: 295923153
+ Total Data Transfer: 175407750
+ Total Compute Intensity: 1.6870585991782006
+ Graph size in memory: 12.7845 GiB
+
ABAB->ABAB:
Graph:
- Nodes: Total: 3218, DataTask: 1613, ComputeTaskP: 8, ComputeTaskS2: 288, ComputeTaskV: 796, ComputeTaskU: 8, ComputeTaskSum: 1, ComputeTaskS1: 504
+ Nodes: Total: 3218, DataTask: 1613, ComputeTaskP: 8,
+ ComputeTaskS2: 288, ComputeTaskV: 796, ComputeTaskU: 8,
+ ComputeTaskSum: 1, ComputeTaskS1: 504
Edges: 4581
Total Compute Effort: 24009
Total Data Transfer: 9494
@@ -92,11 +111,13 @@ Graph:
ABAB->ABC:
Graph:
- Nodes: Total: 817, DataTask: 412, ComputeTaskP: 7, ComputeTaskS2: 72, ComputeTaskV: 198, ComputeTaskU: 7, ComputeTaskSum: 1, ComputeTaskS1: 120
+ Nodes: Total: 817, DataTask: 412, ComputeTaskP: 7,
+ ComputeTaskS2: 72, ComputeTaskV: 198, ComputeTaskU: 7,
+ ComputeTaskSum: 1, ComputeTaskS1: 120
Edges: 1151
Total Compute Effort: 6028
Total Data Transfer: 2411
Total Compute Intensity: 2.5002073828287017
Graph size in memory: 225.0625 KiB
286.583 μs (13996 allocations: 804.48 KiB)
-```
\ No newline at end of file
+```
diff --git a/src/graph.jl b/src/graph.jl
index d56fc4b..d2bc808 100644
--- a/src/graph.jl
+++ b/src/graph.jl
@@ -54,15 +54,13 @@ mutable struct PossibleOperations
nodeFusions::Set{NodeFusion}
nodeReductions::Set{NodeReduction}
nodeSplits::Set{NodeSplit}
- dirty::Bool
end
function PossibleOperations()
return PossibleOperations(
Set{NodeFusion}(),
Set{NodeReduction}(),
- Set{NodeSplit}(),
- true
+ Set{NodeSplit}()
)
end
@@ -81,11 +79,14 @@ mutable struct DAG
# The possible operations at the current state of the DAG
possibleOperations::PossibleOperations
+ # The set of nodes whose possible operations need to be reevaluated
+ dirtyNodes::Set{Node}
+
# "snapshot" system: keep track of added/removed nodes/edges since last snapshot
# these are muted in insert_node! etc.
diff::Diff
end
function DAG()
- return DAG(Set{Node}(), Stack{AppliedOperation}(), Deque{Operation}(), PossibleOperations(), Diff())
+ return DAG(Set{Node}(), Stack{AppliedOperation}(), Deque{Operation}(), PossibleOperations(), Set{Node}(), Diff())
end
diff --git a/src/graph_functions.jl b/src/graph_functions.jl
index f8ddd44..2225508 100644
--- a/src/graph_functions.jl
+++ b/src/graph_functions.jl
@@ -63,38 +63,100 @@ end
is_entry_node(node::Node) = length(children(node)) == 0
is_exit_node(node::Node) = length(parents(node)) == 0
+# function to invalidate the operation caches for a given operation
+function invalidate_caches!(graph::DAG, operation::Operation)
+ delete!(graph.possibleOperations, operation)
+
+ # delete the operation from all caches of nodes involved in the operation
+ # (we can iterate over single values, tuples and vectors just fine)
+ for node in operation.input
+ delete!(node.operations, operation)
+ end
+
+ return nothing
+end
+
+# for graph mutating functions we need to do a few things
+# 1: mute the graph (duh)
+# 2: keep track of what was changed for the diff (if track == true)
+# 3: invalidate operation caches
+
function insert_node!(graph::DAG, node::Node, track=true)
+ # 1: mute
push!(graph.nodes, node)
+ # 2: keep track
if (track) push!(graph.diff.addedNodes, node) end
- graph.possibleOperations.dirty = true
+
+ # 3: invalidate caches
+ push!(graph.dirtyNodes, node)
+
return node
end
function insert_edge!(graph::DAG, edge::Edge, track=true)
- # edge points from child to parent
- push!(edge.edge[1].parents, edge.edge[2])
- push!(edge.edge[2].children, edge.edge[1])
+ node1 = edge.edge[1]
+ node2 = edge.edge[2]
+ # 1: mute
+ # edge points from child to parent
+ push!(node1.parents, node2)
+ push!(node2.children, node1)
+
+ # 2: keep track
if (track) push!(graph.diff.addedEdges, edge) end
- graph.possibleOperations.dirty = true
+
+ # 3: invalidate caches
+ while !isempty(node1.operations)
+ invalidate_caches!(graph, first(node1.operations))
+ end
+ while !isempty(node2.operations)
+ invalidate_caches!(graph, first(node2.operations))
+ end
+ push!(graph.dirtyNodes, node1)
+ push!(graph.dirtyNodes, node2)
+
return edge
end
function remove_node!(graph::DAG, node::Node, track=true)
+ # 1: mute
delete!(graph.nodes, node)
+ # 2: keep track
if (track) push!(graph.diff.removedNodes, node) end
- graph.possibleOperations.dirty = true
+
+ # 3: invalidate caches
+ while !isempty(node)
+ invalidate_caches!(graph, first(node.operations))
+ end
+ delete!(graph.dirtyNodes, node)
+ # no need to invalidate anything else, the node is gone afterwards anyways
+
return nothing
end
function remove_edge!(graph::DAG, edge::Edge, track=true)
- filter!(x -> x != edge.edge[2], edge.edge[1].parents)
- filter!(x -> x != edge.edge[1], edge.edge[2].children)
+ node1 = edge.edge[1]
+ node2 = edge.edge[2]
+ # 1: mute
+ filter!(x -> x != node2, node1.parents)
+ filter!(x -> x != node1, node2.children)
+
+ # 2: keep track
if (track) push!(graph.diff.removedEdges, edge) end
- graph.possibleOperations.dirty = true
+
+ # 3: invalidate caches
+ while !isempty(node1.operations)
+ invalidate_caches!(graph, first(node1.operations))
+ end
+ while !isempty(node2.operations)
+ invalidate_caches!(graph, first(node2.operations))
+ end
+ push!(graph.dirtyNodes, node1)
+ push!(graph.dirtyNodes, node2)
+
return nothing
end
@@ -134,6 +196,10 @@ function get_exit_node(graph::DAG)
error("The given graph has no exit node! It is either empty or not acyclic!")
end
+function can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+ #Todo
+end
+
function can_reduce(n1::Node, n2::Node)
if (n1.task != n2.task)
return false
@@ -198,12 +264,17 @@ function show(io::IO, graph::DAG)
else
print("Total: ", length(graph.nodes), ", ")
first = true
+ i = 0
for (type, number) in zip(keys(nodeDict), values(nodeDict))
+ i += 1
if first
first = false
else
print(", ")
end
+ if (i % 3 == 0)
+ print("\n ")
+ end
print(type, ": ", number)
end
end
diff --git a/src/graph_operations.jl b/src/graph_operations.jl
index bac0285..18a3414 100644
--- a/src/graph_operations.jl
+++ b/src/graph_operations.jl
@@ -5,12 +5,7 @@ function push_operation!(graph::DAG, operation::Operation)
# 1.: Add the operation to the DAG
push!(graph.operationsToApply, operation)
- # 2.: Apply all operations in the chain
- apply_all!(graph)
-
- # 3.: Regenerate properties, possible operations from here
-
- graph.possibleOperations.dirty = true
+ return nothing
end
# reverts the latest applied operation, essentially like a ctrl+z for
@@ -24,13 +19,7 @@ function pop_operation!(graph::DAG)
else
error("No more operations to pop!")
end
-
- # 2.: Apply all (remaining) operations in the chain
- apply_all!(graph)
-
- # 3.: Regenerate properties, possible operations from here
-
- graph.possibleOperations.dirty = true
+ return nothing
end
can_pop(graph::DAG) = !isempty(graph.operationsToApply) || !isempty(graph.appliedOperations)
@@ -40,6 +29,8 @@ function reset_graph!(graph::DAG)
while (can_pop(graph))
pop_operation!(graph)
end
+
+ return nothing
end
# implementation detail functions, don't export
@@ -56,6 +47,7 @@ function apply_all!(graph::DAG)
# push to the end of the appliedOperations deque
push!(graph.appliedOperations, appliedOp)
end
+ return nothing
end
@@ -245,6 +237,11 @@ function node_split!(graph::DAG, n1::Node)
return get_snapshot_diff(graph)
end
+# function to find node fusions involving the given node
+function find_fusions(graph::DAG, node::Node)
+
+end
+
# function to generate all possible optmizations on the graph
function generate_options(graph::DAG)
options = PossibleOperations()
diff --git a/src/import.jl b/src/import.jl
index 9438946..d4be7e1 100644
--- a/src/import.jl
+++ b/src/import.jl
@@ -20,7 +20,7 @@ function parse_edges(input::AbstractString)
return output
end
-function import_txt(filename::String, verbose::Bool = isinteractive())
+function import_txt(filename::String, verbose::Bool = false)
file = open(filename, "r")
if (verbose) println("Opened file") end