Shuffle files and functions around for more consistent naming and smaller files

src/graph/compare.jl

@@ -0,0 +1,14 @@
+
+in(node::Node, graph::DAG) = node in graph.nodes
+in(edge::Edge, graph::DAG) = edge in graph.edges
+
+function ==(n1::Node, n2::Node, g::DAG)
+   if typeof(n1) != typeof(n2)
+      return false
+   end
+   if !(n1 in g) || !(n2 in g)
+      return false
+   end
+
+   return n1.task == n2.task && children(n1) == children(n2)
+end

src/graph/interface.jl

@@ -0,0 +1,35 @@
+# user interface on the DAG
+
+# applies a new operation to the end of the graph
+function push_operation!(graph::DAG, operation::Operation)
+   # 1.: Add the operation to the DAG
+   push!(graph.operationsToApply, operation)
+
+   return nothing
+end
+
+# reverts the latest applied operation, essentially like a ctrl+z for
+function pop_operation!(graph::DAG)
+   # 1.: Remove the operation from the appliedChain of the DAG
+   if !isempty(graph.operationsToApply)
+      pop!(graph.operationsToApply)
+   elseif !isempty(graph.appliedOperations)
+      appliedOp = pop!(graph.appliedOperations)
+      revert_operation!(graph, appliedOp)
+   else
+      error("No more operations to pop!")
+   end
+   
+   return nothing
+end
+
+can_pop(graph::DAG) = !isempty(graph.operationsToApply) || !isempty(graph.appliedOperations)
+
+# reset the graph to its initial state with no operations applied
+function reset_graph!(graph::DAG)
+   while (can_pop(graph))
+      pop_operation!(graph)
+   end
+
+   return nothing
+end

src/graph/mute.jl

@@ -0,0 +1,162 @@
+# for graph mutating functions we need to do a few things
+# 1: mute the graph (duh)
+# 2: keep track of what was changed for the diff (if track == true)
+# 3: invalidate operation caches
+
+function insert_node!(graph::DAG, node::Node, track=true, invalidate_cache=true)
+   # 1: mute
+   push!(graph.nodes, node)
+
+   # 2: keep track
+   if (track) push!(graph.diff.addedNodes, node) end
+
+   # 3: invalidate caches
+   if (!invalidate_cache) return node end
+   push!(graph.dirtyNodes, node)
+
+   return node
+end
+
+function insert_edge!(graph::DAG, node1::Node, node2::Node, track=true, invalidate_cache=true)
+   # @assert (node2 ∉ node1.parents) && (node1 ∉ node2.children) "Edge to insert already exists"
+
+   # 1: mute
+   # edge points from child to parent
+   push!(node1.parents, node2)
+   push!(node2.children, node1)
+
+   # 2: keep track
+   if (track) push!(graph.diff.addedEdges, make_edge(node1, node2)) end
+   
+   # 3: invalidate caches
+   if (!invalidate_cache) return nothing end
+
+   invalidate_operation_caches!(graph, node1)
+   invalidate_operation_caches!(graph, node2)
+
+   push!(graph.dirtyNodes, node1)
+   push!(graph.dirtyNodes, node2)
+
+   return nothing
+end
+
+function remove_node!(graph::DAG, node::Node, track=true, invalidate_cache=true)
+   # @assert node in graph.nodes "Trying to remove a node that's not in the graph"
+
+   # 1: mute
+   delete!(graph.nodes, node)
+
+   # 2: keep track
+   if (track) push!(graph.diff.removedNodes, node) end
+
+   # 3: invalidate caches
+   if (!invalidate_cache) return nothing end
+
+   invalidate_operation_caches!(graph, node)
+   delete!(graph.dirtyNodes, node)
+
+   return nothing
+end
+
+function remove_edge!(graph::DAG, node1::Node, node2::Node, track=true, invalidate_cache=true)
+   # 1: mute
+   pre_length1 = length(node1.parents)
+   pre_length2 = length(node2.children)
+   filter!(x -> x != node2, node1.parents)
+   filter!(x -> x != node1, node2.children)
+
+   #=@assert begin
+      removed = pre_length1 - length(node1.parents)
+      removed <= 1
+   end "removed more than one node from node1's parents"=#
+
+   #=@assert begin
+      removed = pre_length2 - length(node2.children)
+      removed <= 1
+   end "removed more than one node from node2's children"=#
+
+   # 2: keep track
+   if (track) push!(graph.diff.removedEdges, make_edge(node1, node2)) end
+
+   # 3: invalidate caches
+   if (!invalidate_cache) return nothing end
+
+   invalidate_operation_caches!(graph, node1)
+   invalidate_operation_caches!(graph, node2)
+   if (node1 in graph)
+      push!(graph.dirtyNodes, node1)
+   end
+   if (node2 in graph) 
+      push!(graph.dirtyNodes, node2)
+   end
+
+   return nothing
+end
+
+# return the graph "difference" since last time this function was called
+function get_snapshot_diff(graph::DAG)
+   return swapfield!(graph, :diff, Diff())
+end
+
+# function to invalidate the operation caches for a given NodeFusion
+function invalidate_caches!(graph::DAG, operation::NodeFusion)
+   delete!(graph.possibleOperations, operation)
+
+   # delete the operation from all caches of nodes involved in the operation
+   filter!(!=(operation), operation.input[1].nodeFusions)
+   filter!(!=(operation), operation.input[3].nodeFusions)
+   
+   operation.input[2].nodeFusion = missing
+
+   return nothing
+end
+
+# function to invalidate the operation caches for a given NodeReduction
+function invalidate_caches!(graph::DAG, operation::NodeReduction)
+   delete!(graph.possibleOperations, operation)
+
+   for node in operation.input
+      node.nodeReduction = missing
+   end
+
+   return nothing
+end
+
+# function to invalidate the operation caches for a given NodeSplit
+function invalidate_caches!(graph::DAG, operation::NodeSplit)
+   delete!(graph.possibleOperations, operation)
+
+   # delete the operation from all caches of nodes involved in the operation
+   # for node split there is only one node
+   operation.input.nodeSplit = missing
+   
+   return nothing
+end
+
+# function to invalidate the operation caches of a ComputeTaskNode
+function invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode)
+   if !ismissing(node.nodeReduction)
+      invalidate_caches!(graph, node.nodeReduction)
+   end
+   if !ismissing(node.nodeSplit)
+      invalidate_caches!(graph, node.nodeSplit)
+   end
+   while !isempty(node.nodeFusions)
+      invalidate_caches!(graph, pop!(node.nodeFusions))
+   end
+   return nothing
+end
+
+# function to invalidate the operation caches of a DataTaskNode
+function invalidate_operation_caches!(graph::DAG, node::DataTaskNode)
+   if !ismissing(node.nodeReduction)
+      invalidate_caches!(graph, node.nodeReduction)
+   end
+   if !ismissing(node.nodeSplit)
+      invalidate_caches!(graph, node.nodeSplit)
+   end
+   if !ismissing(node.nodeFusion)
+      invalidate_caches!(graph, node.nodeFusion)
+   end
+   return nothing
+end

src/graph/print.jl

@@ -0,0 +1,55 @@
+function show_nodes(io, graph::DAG)
+   print(io, "[")
+   first = true
+   for n in graph.nodes
+      if first
+         first = false
+      else
+         print(io, ", ")
+      end
+      print(io, n)
+   end
+   print(io, "]")
+end
+
+function show(io::IO, graph::DAG)
+   println(io, "Graph:")
+   print(io, "  Nodes: ")
+
+   nodeDict = Dict{Type, Int64}()
+   noEdges = 0
+   for node in graph.nodes
+      if haskey(nodeDict, typeof(node.task))
+         nodeDict[typeof(node.task)] = nodeDict[typeof(node.task)] + 1
+      else
+         nodeDict[typeof(node.task)] = 1
+      end
+      noEdges += length(parents(node))
+   end
+
+   if length(graph.nodes) <= 20
+      show_nodes(io, graph)
+   else
+      print("Total: ", length(graph.nodes), ", ")
+      first = true
+      i = 0
+      for (type, number) in zip(keys(nodeDict), values(nodeDict))
+         i += 1
+         if first
+            first = false
+         else
+            print(", ")
+         end
+         if (i % 3 == 0)
+            print("\n         ")
+         end
+         print(type, ": ", number)
+      end
+   end
+   println(io)
+   println(io, "  Edges: ", noEdges)
+   properties = graph_properties(graph)
+   println(io, "  Total Compute Effort: ", properties.compute_effort)
+   println(io, "  Total Data Transfer: ", properties.data)
+   println(io, "  Total Compute Intensity: ", properties.compute_intensity)
+end

src/graph/properties.jl

@@ -0,0 +1,31 @@
+function graph_properties(graph::DAG)
+   # make sure the graph is fully generated
+   apply_all!(graph)
+
+   d = 0
+   ce = 0
+   ed = 0
+   for node in graph.nodes
+      d += data(node.task) * length(node.parents)
+      ce += compute_effort(node.task)
+      ed += length(node.parents)
+   end
+
+   ci = ce / d
+
+   result = (data = d,
+             compute_effort = ce,
+             compute_intensity = ci,
+             nodes = length(graph.nodes),
+             edges = ed)
+   return result
+end
+
+function get_exit_node(graph::DAG)
+   for node in graph.nodes
+      if (is_exit_node(node))
+         return node
+      end
+   end
+   @assert false "The given graph has no exit node! It is either empty or not acyclic!"
+end

src/graph/type.jl

@@ -0,0 +1,41 @@
+using DataStructures
+
+mutable struct PossibleOperations
+   nodeFusions::Set{NodeFusion}
+   nodeReductions::Set{NodeReduction}
+   nodeSplits::Set{NodeSplit}
+end
+
+function PossibleOperations()
+   return PossibleOperations(
+      Set{NodeFusion}(),
+      Set{NodeReduction}(),
+      Set{NodeSplit}()
+   )
+end
+
+# The actual state of the DAG is the initial state given by the set of nodes 
+# but with all the operations in appliedChain applied in order
+mutable struct DAG
+   nodes::Set{Node}
+
+   # The operations currently applied to the set of nodes
+   appliedOperations::Stack{AppliedOperation}
+
+   # The operations not currently applied but part of the current state of the DAG
+   operationsToApply::Deque{Operation}
+
+   # The possible operations at the current state of the DAG
+   possibleOperations::PossibleOperations
+
+   # The set of nodes whose possible operations need to be reevaluated
+   dirtyNodes::Set{Node}
+
+   # "snapshot" system: keep track of added/removed nodes/edges since last snapshot
+   # these are muted in insert_node! etc.
+   diff::Diff
+end
+
+function DAG()
+   return DAG(Set{Node}(), Stack{AppliedOperation}(), Deque{Operation}(), PossibleOperations(), Set{Node}(), Diff())
+end

src/graph/validate.jl

@@ -0,0 +1,52 @@
+# check whether the given graph is connected
+function is_connected(graph::DAG)
+   nodeQueue = Deque{Node}()
+   push!(nodeQueue, get_exit_node(graph))
+   seenNodes = Set{Node}()
+
+   while !isempty(nodeQueue)
+      current = pop!(nodeQueue)
+      push!(seenNodes, current)
+
+      for child in current.children
+         push!(nodeQueue, child)
+      end
+   end
+
+   return length(seenNodes) == length(graph.nodes)
+end
+
+function is_valid(graph::DAG)
+   for node in graph.nodes
+      @assert is_valid(graph, node)
+   end
+
+   for op in graph.operationsToApply
+      @assert is_valid(graph, op)
+   end
+
+   for nr in graph.possibleOperations.nodeReductions
+      @assert is_valid(graph, nr)
+   end
+   for ns in graph.possibleOperations.nodeSplits
+      @assert is_valid(graph, ns)
+   end
+   for nf in graph.possibleOperations.nodeFusions
+      @assert is_valid(graph, nf)
+   end
+
+   for node in graph.dirtyNodes
+      @assert node in graph "Dirty Node is not part of the graph!"
+      @assert ismissing(node.nodeReduction) "Dirty Node has a NodeReduction!"
+      @assert ismissing(node.nodeSplit) "Dirty Node has a NodeSplit!"
+      if (typeof(node) <: DataTaskNode)
+         @assert ismissing(node.nodeFusion) "Dirty DataTaskNode has a Node Fusion!"
+      elseif (typeof(node) <: ComputeTaskNode)
+         @assert isempty(node.nodeFusions) "Dirty ComputeTaskNode has Node Fusions!"
+      end
+   end
+
+   @assert is_connected(graph) "Graph is not connected!"
+
+   return true
+end