From 0f5f475cb4c663a6284ace558c57a0d132c40ecf Mon Sep 17 00:00:00 2001 From: Anton Reinhard Date: Thu, 24 Aug 2023 15:11:54 +0200 Subject: [PATCH] Shuffle files and functions around for more consistent naming and smaller files --- src/MetagraphOptimization.jl | 50 ++- src/diff/print.jl | 6 + src/diff/properties.jl | 9 + src/diff/type.jl | 13 + src/graph/compare.jl | 14 + .../interface.jl} | 1 + src/graph/mute.jl | 162 ++++++++ src/graph/print.jl | 55 +++ src/graph/properties.jl | 31 ++ src/{graph.jl => graph/type.jl} | 49 --- src/graph/validate.jl | 52 +++ src/graph_functions.jl | 389 ------------------ src/{abc_model => models/abc}/parse.jl | 0 .../abc/properties.jl} | 0 .../tasks.jl => models/abc/types.jl} | 0 src/node/compare.jl | 15 + src/node/create.jl | 23 ++ src/node/print.jl | 7 + src/node/properties.jl | 52 +++ src/{nodes.jl => node/type.jl} | 4 + src/node/validate.jl | 43 ++ src/node_functions.jl | 95 ----- src/{operations => operation}/apply.jl | 0 src/{operations => operation}/clean.jl | 0 src/{operations => operation}/find.jl | 0 src/{operations => operation}/get.jl | 0 src/{operations => operation}/print.jl | 0 src/operation/type.jl | 34 ++ src/{operations => operation}/utility.jl | 0 src/{operations => operation}/validate.jl | 0 src/task/compare.jl | 11 + src/task/print.jl | 4 + src/{task_functions.jl => task/properties.jl} | 22 +- src/{tasks.jl => task/type.jl} | 3 + 34 files changed, 573 insertions(+), 571 deletions(-) create mode 100644 src/diff/print.jl create mode 100644 src/diff/properties.jl create mode 100644 src/diff/type.jl create mode 100644 src/graph/compare.jl rename src/{graph_interface.jl => graph/interface.jl} (99%) create mode 100644 src/graph/mute.jl create mode 100644 src/graph/print.jl create mode 100644 src/graph/properties.jl rename src/{graph.jl => graph/type.jl} (51%) create mode 100644 src/graph/validate.jl delete mode 100644 src/graph_functions.jl rename src/{abc_model => models/abc}/parse.jl (100%) rename src/{abc_model/task_functions.jl => models/abc/properties.jl} (100%) rename src/{abc_model/tasks.jl => models/abc/types.jl} (100%) create mode 100644 src/node/compare.jl create mode 100644 src/node/create.jl create mode 100644 src/node/print.jl create mode 100644 src/node/properties.jl rename src/{nodes.jl => node/type.jl} (82%) create mode 100644 src/node/validate.jl delete mode 100644 src/node_functions.jl rename src/{operations => operation}/apply.jl (100%) rename src/{operations => operation}/clean.jl (100%) rename src/{operations => operation}/find.jl (100%) rename src/{operations => operation}/get.jl (100%) rename src/{operations => operation}/print.jl (100%) create mode 100644 src/operation/type.jl rename src/{operations => operation}/utility.jl (100%) rename src/{operations => operation}/validate.jl (100%) create mode 100644 src/task/compare.jl create mode 100644 src/task/print.jl rename src/{task_functions.jl => task/properties.jl} (62%) rename src/{tasks.jl => task/type.jl} (70%) diff --git a/src/MetagraphOptimization.jl b/src/MetagraphOptimization.jl index 5909227..6090a24 100644 --- a/src/MetagraphOptimization.jl +++ b/src/MetagraphOptimization.jl @@ -21,29 +21,45 @@ import Base.insert! import Base.collect -include("tasks.jl") -include("nodes.jl") -include("graph.jl") +include("task/type.jl") +include("node/type.jl") +include("diff/type.jl") +include("operation/type.jl") +include("graph/type.jl") include("trie.jl") include("utility.jl") -include("task_functions.jl") -include("node_functions.jl") -include("graph_functions.jl") +include("diff/print.jl") +include("diff/properties.jl") -include("operations/utility.jl") -include("operations/apply.jl") -include("operations/clean.jl") -include("operations/find.jl") -include("operations/get.jl") -include("operations/print.jl") -include("operations/validate.jl") +include("graph/compare.jl") +include("graph/interface.jl") +include("graph/mute.jl") +include("graph/print.jl") +include("graph/properties.jl") +include("graph/validate.jl") -include("graph_interface.jl") +include("node/compare.jl") +include("node/create.jl") +include("node/print.jl") +include("node/properties.jl") +include("node/validate.jl") -include("abc_model/tasks.jl") -include("abc_model/task_functions.jl") -include("abc_model/parse.jl") +include("operation/utility.jl") +include("operation/apply.jl") +include("operation/clean.jl") +include("operation/find.jl") +include("operation/get.jl") +include("operation/print.jl") +include("operation/validate.jl") + +include("task/compare.jl") +include("task/print.jl") +include("task/properties.jl") + +include("models/abc/types.jl") +include("models/abc/properties.jl") +include("models/abc/parse.jl") end # module MetagraphOptimization diff --git a/src/diff/print.jl b/src/diff/print.jl new file mode 100644 index 0000000..7be6c65 --- /dev/null +++ b/src/diff/print.jl @@ -0,0 +1,6 @@ +function show(io::IO, diff::Diff) + print(io, "Nodes: ") + print(io, length(diff.addedNodes) + length(diff.removedNodes)) + print(io, " Edges: ") + print(io, length(diff.addedEdges) + length(diff.removedEdges)) +end diff --git a/src/diff/properties.jl b/src/diff/properties.jl new file mode 100644 index 0000000..56e8983 --- /dev/null +++ b/src/diff/properties.jl @@ -0,0 +1,9 @@ +# return a namedtuple of the lengths of the added/removed nodes/edges +function length(diff::Diff) + return ( + addedNodes = length(diff.addedNodes), + removedNodes = length(diff.removedNodes), + addedEdges = length(diff.addedEdges), + removedEdges = length(diff.removedEdges) + ) +end diff --git a/src/diff/type.jl b/src/diff/type.jl new file mode 100644 index 0000000..a573de3 --- /dev/null +++ b/src/diff/type.jl @@ -0,0 +1,13 @@ +const Diff = NamedTuple{ + (:addedNodes, :removedNodes, :addedEdges, :removedEdges), + Tuple{Vector{Node}, Vector{Node}, Vector{Edge}, Vector{Edge}} +} + +function Diff() + return ( + addedNodes = Vector{Node}(), + removedNodes = Vector{Node}(), + addedEdges = Vector{Edge}(), + removedEdges = Vector{Edge}() + )::Diff +end diff --git a/src/graph/compare.jl b/src/graph/compare.jl new file mode 100644 index 0000000..f9b5986 --- /dev/null +++ b/src/graph/compare.jl @@ -0,0 +1,14 @@ + +in(node::Node, graph::DAG) = node in graph.nodes +in(edge::Edge, graph::DAG) = edge in graph.edges + +function ==(n1::Node, n2::Node, g::DAG) + if typeof(n1) != typeof(n2) + return false + end + if !(n1 in g) || !(n2 in g) + return false + end + + return n1.task == n2.task && children(n1) == children(n2) +end diff --git a/src/graph_interface.jl b/src/graph/interface.jl similarity index 99% rename from src/graph_interface.jl rename to src/graph/interface.jl index 7cc414f..de669f6 100644 --- a/src/graph_interface.jl +++ b/src/graph/interface.jl @@ -19,6 +19,7 @@ function pop_operation!(graph::DAG) else error("No more operations to pop!") end + return nothing end diff --git a/src/graph/mute.jl b/src/graph/mute.jl new file mode 100644 index 0000000..8bc707b --- /dev/null +++ b/src/graph/mute.jl @@ -0,0 +1,162 @@ +# for graph mutating functions we need to do a few things +# 1: mute the graph (duh) +# 2: keep track of what was changed for the diff (if track == true) +# 3: invalidate operation caches + +function insert_node!(graph::DAG, node::Node, track=true, invalidate_cache=true) + # 1: mute + push!(graph.nodes, node) + + # 2: keep track + if (track) push!(graph.diff.addedNodes, node) end + + # 3: invalidate caches + if (!invalidate_cache) return node end + push!(graph.dirtyNodes, node) + + return node +end + +function insert_edge!(graph::DAG, node1::Node, node2::Node, track=true, invalidate_cache=true) + # @assert (node2 ∉ node1.parents) && (node1 ∉ node2.children) "Edge to insert already exists" + + # 1: mute + # edge points from child to parent + push!(node1.parents, node2) + push!(node2.children, node1) + + # 2: keep track + if (track) push!(graph.diff.addedEdges, make_edge(node1, node2)) end + + # 3: invalidate caches + if (!invalidate_cache) return nothing end + + invalidate_operation_caches!(graph, node1) + invalidate_operation_caches!(graph, node2) + + push!(graph.dirtyNodes, node1) + push!(graph.dirtyNodes, node2) + + return nothing +end + +function remove_node!(graph::DAG, node::Node, track=true, invalidate_cache=true) + # @assert node in graph.nodes "Trying to remove a node that's not in the graph" + + # 1: mute + delete!(graph.nodes, node) + + # 2: keep track + if (track) push!(graph.diff.removedNodes, node) end + + # 3: invalidate caches + if (!invalidate_cache) return nothing end + + invalidate_operation_caches!(graph, node) + delete!(graph.dirtyNodes, node) + + return nothing +end + +function remove_edge!(graph::DAG, node1::Node, node2::Node, track=true, invalidate_cache=true) + # 1: mute + pre_length1 = length(node1.parents) + pre_length2 = length(node2.children) + filter!(x -> x != node2, node1.parents) + filter!(x -> x != node1, node2.children) + + #=@assert begin + removed = pre_length1 - length(node1.parents) + removed <= 1 + end "removed more than one node from node1's parents"=# + + #=@assert begin + removed = pre_length2 - length(node2.children) + removed <= 1 + end "removed more than one node from node2's children"=# + + # 2: keep track + if (track) push!(graph.diff.removedEdges, make_edge(node1, node2)) end + + # 3: invalidate caches + if (!invalidate_cache) return nothing end + + invalidate_operation_caches!(graph, node1) + invalidate_operation_caches!(graph, node2) + if (node1 in graph) + push!(graph.dirtyNodes, node1) + end + if (node2 in graph) + push!(graph.dirtyNodes, node2) + end + + return nothing +end + +# return the graph "difference" since last time this function was called +function get_snapshot_diff(graph::DAG) + return swapfield!(graph, :diff, Diff()) +end + +# function to invalidate the operation caches for a given NodeFusion +function invalidate_caches!(graph::DAG, operation::NodeFusion) + delete!(graph.possibleOperations, operation) + + # delete the operation from all caches of nodes involved in the operation + filter!(!=(operation), operation.input[1].nodeFusions) + filter!(!=(operation), operation.input[3].nodeFusions) + + operation.input[2].nodeFusion = missing + + return nothing +end + +# function to invalidate the operation caches for a given NodeReduction +function invalidate_caches!(graph::DAG, operation::NodeReduction) + delete!(graph.possibleOperations, operation) + + for node in operation.input + node.nodeReduction = missing + end + + return nothing +end + +# function to invalidate the operation caches for a given NodeSplit +function invalidate_caches!(graph::DAG, operation::NodeSplit) + delete!(graph.possibleOperations, operation) + + # delete the operation from all caches of nodes involved in the operation + # for node split there is only one node + operation.input.nodeSplit = missing + + return nothing +end + +# function to invalidate the operation caches of a ComputeTaskNode +function invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode) + if !ismissing(node.nodeReduction) + invalidate_caches!(graph, node.nodeReduction) + end + if !ismissing(node.nodeSplit) + invalidate_caches!(graph, node.nodeSplit) + end + while !isempty(node.nodeFusions) + invalidate_caches!(graph, pop!(node.nodeFusions)) + end + return nothing +end + +# function to invalidate the operation caches of a DataTaskNode +function invalidate_operation_caches!(graph::DAG, node::DataTaskNode) + if !ismissing(node.nodeReduction) + invalidate_caches!(graph, node.nodeReduction) + end + if !ismissing(node.nodeSplit) + invalidate_caches!(graph, node.nodeSplit) + end + if !ismissing(node.nodeFusion) + invalidate_caches!(graph, node.nodeFusion) + end + return nothing +end diff --git a/src/graph/print.jl b/src/graph/print.jl new file mode 100644 index 0000000..119f56f --- /dev/null +++ b/src/graph/print.jl @@ -0,0 +1,55 @@ +function show_nodes(io, graph::DAG) + print(io, "[") + first = true + for n in graph.nodes + if first + first = false + else + print(io, ", ") + end + print(io, n) + end + print(io, "]") +end + +function show(io::IO, graph::DAG) + println(io, "Graph:") + print(io, " Nodes: ") + + nodeDict = Dict{Type, Int64}() + noEdges = 0 + for node in graph.nodes + if haskey(nodeDict, typeof(node.task)) + nodeDict[typeof(node.task)] = nodeDict[typeof(node.task)] + 1 + else + nodeDict[typeof(node.task)] = 1 + end + noEdges += length(parents(node)) + end + + if length(graph.nodes) <= 20 + show_nodes(io, graph) + else + print("Total: ", length(graph.nodes), ", ") + first = true + i = 0 + for (type, number) in zip(keys(nodeDict), values(nodeDict)) + i += 1 + if first + first = false + else + print(", ") + end + if (i % 3 == 0) + print("\n ") + end + print(type, ": ", number) + end + end + println(io) + println(io, " Edges: ", noEdges) + properties = graph_properties(graph) + println(io, " Total Compute Effort: ", properties.compute_effort) + println(io, " Total Data Transfer: ", properties.data) + println(io, " Total Compute Intensity: ", properties.compute_intensity) +end diff --git a/src/graph/properties.jl b/src/graph/properties.jl new file mode 100644 index 0000000..afcacc7 --- /dev/null +++ b/src/graph/properties.jl @@ -0,0 +1,31 @@ +function graph_properties(graph::DAG) + # make sure the graph is fully generated + apply_all!(graph) + + d = 0 + ce = 0 + ed = 0 + for node in graph.nodes + d += data(node.task) * length(node.parents) + ce += compute_effort(node.task) + ed += length(node.parents) + end + + ci = ce / d + + result = (data = d, + compute_effort = ce, + compute_intensity = ci, + nodes = length(graph.nodes), + edges = ed) + return result +end + +function get_exit_node(graph::DAG) + for node in graph.nodes + if (is_exit_node(node)) + return node + end + end + @assert false "The given graph has no exit node! It is either empty or not acyclic!" +end diff --git a/src/graph.jl b/src/graph/type.jl similarity index 51% rename from src/graph.jl rename to src/graph/type.jl index 92ed7f3..715bf3c 100644 --- a/src/graph.jl +++ b/src/graph/type.jl @@ -1,54 +1,5 @@ using DataStructures -const Diff = NamedTuple{ - (:addedNodes, :removedNodes, :addedEdges, :removedEdges), - Tuple{Vector{Node}, Vector{Node}, Vector{Edge}, Vector{Edge}} -} - -function Diff() - return ( - addedNodes = Vector{Node}(), - removedNodes = Vector{Node}(), - addedEdges = Vector{Edge}(), - removedEdges = Vector{Edge}() - )::Diff -end - -# An abstract base class for operations -# an operation can be applied to a DAG -abstract type Operation end - -# An abstract base class for already applied operations -# an applied operation can be reversed iff it is the last applied operation on the DAG -abstract type AppliedOperation end - -struct NodeFusion <: Operation - input::Tuple{ComputeTaskNode, DataTaskNode, ComputeTaskNode} -end - -struct AppliedNodeFusion <: AppliedOperation - operation::NodeFusion - diff::Diff -end - -struct NodeReduction <: Operation - input::Vector{Node} -end - -struct AppliedNodeReduction <: AppliedOperation - operation::NodeReduction - diff::Diff -end - -struct NodeSplit <: Operation - input::Node -end - -struct AppliedNodeSplit <: AppliedOperation - operation::NodeSplit - diff::Diff -end - mutable struct PossibleOperations nodeFusions::Set{NodeFusion} nodeReductions::Set{NodeReduction} diff --git a/src/graph/validate.jl b/src/graph/validate.jl new file mode 100644 index 0000000..062cf21 --- /dev/null +++ b/src/graph/validate.jl @@ -0,0 +1,52 @@ +# check whether the given graph is connected +function is_connected(graph::DAG) + nodeQueue = Deque{Node}() + push!(nodeQueue, get_exit_node(graph)) + seenNodes = Set{Node}() + + while !isempty(nodeQueue) + current = pop!(nodeQueue) + push!(seenNodes, current) + + for child in current.children + push!(nodeQueue, child) + end + end + + return length(seenNodes) == length(graph.nodes) +end + +function is_valid(graph::DAG) + for node in graph.nodes + @assert is_valid(graph, node) + end + + for op in graph.operationsToApply + @assert is_valid(graph, op) + end + + for nr in graph.possibleOperations.nodeReductions + @assert is_valid(graph, nr) + end + for ns in graph.possibleOperations.nodeSplits + @assert is_valid(graph, ns) + end + for nf in graph.possibleOperations.nodeFusions + @assert is_valid(graph, nf) + end + + for node in graph.dirtyNodes + @assert node in graph "Dirty Node is not part of the graph!" + @assert ismissing(node.nodeReduction) "Dirty Node has a NodeReduction!" + @assert ismissing(node.nodeSplit) "Dirty Node has a NodeSplit!" + if (typeof(node) <: DataTaskNode) + @assert ismissing(node.nodeFusion) "Dirty DataTaskNode has a Node Fusion!" + elseif (typeof(node) <: ComputeTaskNode) + @assert isempty(node.nodeFusions) "Dirty ComputeTaskNode has Node Fusions!" + end + end + + @assert is_connected(graph) "Graph is not connected!" + + return true +end diff --git a/src/graph_functions.jl b/src/graph_functions.jl deleted file mode 100644 index b95ed93..0000000 --- a/src/graph_functions.jl +++ /dev/null @@ -1,389 +0,0 @@ -using DataStructures - -in(node::Node, graph::DAG) = node in graph.nodes -in(edge::Edge, graph::DAG) = edge in graph.edges - -function is_parent(potential_parent, node) - return potential_parent in node.parents -end - -function is_child(potential_child, node) - return potential_child in node.children -end - -function ==(n1::Node, n2::Node, g::DAG) - if typeof(n1) != typeof(n2) - return false - end - if !(n1 in g) || !(n2 in g) - return false - end - - return n1.task == n2.task && children(n1) == children(n2) -end - -# children = prerequisite nodes, nodes that need to execute before the task, edges point into this task -function children(node::Node) - return copy(node.children) -end - -# parents = subsequent nodes, nodes that need this node to execute, edges point from this task -function parents(node::Node) - return copy(node.parents) -end - -# siblings = all children of any parents, no duplicates, includes the node itself -function siblings(node::Node) - result = Set{Node}() - push!(result, node) - for parent in node.parents - union!(result, parent.children) - end - - return result -end - -# partners = all parents of any children, no duplicates, includes the node itself -function partners(node::Node) - result = Set{Node}() - push!(result, node) - for child in node.children - union!(result, child.parents) - end - - return result -end - -# alternative version to partners(Node), avoiding allocation of a new set -# works on the given set and returns nothing -function partners(node::Node, set::Set{Node}) - push!(set, node) - for child in node.children - union!(set, child.parents) - end - return nothing -end - -is_entry_node(node::Node) = length(node.children) == 0 -is_exit_node(node::Node) = length(node.parents) == 0 - -# function to invalidate the operation caches for a given NodeFusion -function invalidate_caches!(graph::DAG, operation::NodeFusion) - delete!(graph.possibleOperations, operation) - - # delete the operation from all caches of nodes involved in the operation - filter!(!=(operation), operation.input[1].nodeFusions) - filter!(!=(operation), operation.input[3].nodeFusions) - - operation.input[2].nodeFusion = missing - - return nothing -end - -# function to invalidate the operation caches for a given NodeReduction -function invalidate_caches!(graph::DAG, operation::NodeReduction) - delete!(graph.possibleOperations, operation) - - for node in operation.input - node.nodeReduction = missing - end - - return nothing -end - -# function to invalidate the operation caches for a given NodeSplit -function invalidate_caches!(graph::DAG, operation::NodeSplit) - delete!(graph.possibleOperations, operation) - - # delete the operation from all caches of nodes involved in the operation - # for node split there is only one node - operation.input.nodeSplit = missing - - return nothing -end - -# function to invalidate the operation caches of a ComputeTaskNode -function invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode) - if !ismissing(node.nodeReduction) - invalidate_caches!(graph, node.nodeReduction) - end - if !ismissing(node.nodeSplit) - invalidate_caches!(graph, node.nodeSplit) - end - while !isempty(node.nodeFusions) - invalidate_caches!(graph, pop!(node.nodeFusions)) - end - return nothing -end - -# function to invalidate the operation caches of a DataTaskNode -function invalidate_operation_caches!(graph::DAG, node::DataTaskNode) - if !ismissing(node.nodeReduction) - invalidate_caches!(graph, node.nodeReduction) - end - if !ismissing(node.nodeSplit) - invalidate_caches!(graph, node.nodeSplit) - end - if !ismissing(node.nodeFusion) - invalidate_caches!(graph, node.nodeFusion) - end - return nothing -end - -# for graph mutating functions we need to do a few things -# 1: mute the graph (duh) -# 2: keep track of what was changed for the diff (if track == true) -# 3: invalidate operation caches - -function insert_node!(graph::DAG, node::Node, track=true, invalidate_cache=true) - # 1: mute - push!(graph.nodes, node) - - # 2: keep track - if (track) push!(graph.diff.addedNodes, node) end - - # 3: invalidate caches - if (!invalidate_cache) return node end - push!(graph.dirtyNodes, node) - - return node -end - -function insert_edge!(graph::DAG, node1::Node, node2::Node, track=true, invalidate_cache=true) - # @assert (node2 ∉ node1.parents) && (node1 ∉ node2.children) "Edge to insert already exists" - - # 1: mute - # edge points from child to parent - push!(node1.parents, node2) - push!(node2.children, node1) - - # 2: keep track - if (track) push!(graph.diff.addedEdges, make_edge(node1, node2)) end - - # 3: invalidate caches - if (!invalidate_cache) return nothing end - - invalidate_operation_caches!(graph, node1) - invalidate_operation_caches!(graph, node2) - - push!(graph.dirtyNodes, node1) - push!(graph.dirtyNodes, node2) - - return nothing -end - -function remove_node!(graph::DAG, node::Node, track=true, invalidate_cache=true) - # @assert node in graph.nodes "Trying to remove a node that's not in the graph" - - # 1: mute - delete!(graph.nodes, node) - - # 2: keep track - if (track) push!(graph.diff.removedNodes, node) end - - # 3: invalidate caches - if (!invalidate_cache) return nothing end - - invalidate_operation_caches!(graph, node) - delete!(graph.dirtyNodes, node) - - return nothing -end - -function remove_edge!(graph::DAG, node1::Node, node2::Node, track=true, invalidate_cache=true) - # 1: mute - pre_length1 = length(node1.parents) - pre_length2 = length(node2.children) - filter!(x -> x != node2, node1.parents) - filter!(x -> x != node1, node2.children) - - #=@assert begin - removed = pre_length1 - length(node1.parents) - removed <= 1 - end "removed more than one node from node1's parents"=# - - #=@assert begin - removed = pre_length2 - length(node2.children) - removed <= 1 - end "removed more than one node from node2's children"=# - - # 2: keep track - if (track) push!(graph.diff.removedEdges, make_edge(node1, node2)) end - - # 3: invalidate caches - if (!invalidate_cache) return nothing end - - invalidate_operation_caches!(graph, node1) - invalidate_operation_caches!(graph, node2) - if (node1 in graph) - push!(graph.dirtyNodes, node1) - end - if (node2 in graph) - push!(graph.dirtyNodes, node2) - end - - return nothing -end - -# return the graph "difference" since last time this function was called -function get_snapshot_diff(graph::DAG) - return swapfield!(graph, :diff, Diff()) -end - -function graph_properties(graph::DAG) - # make sure the graph is fully generated - apply_all!(graph) - - d = 0 - ce = 0 - ed = 0 - for node in graph.nodes - d += data(node.task) * length(node.parents) - ce += compute_effort(node.task) - ed += length(node.parents) - end - - ci = ce / d - - result = (data = d, - compute_effort = ce, - compute_intensity = ci, - nodes = length(graph.nodes), - edges = ed) - return result -end - -function get_exit_node(graph::DAG) - for node in graph.nodes - if (is_exit_node(node)) - return node - end - end - @assert false "The given graph has no exit node! It is either empty or not acyclic!" -end - -# check whether the given graph is connected -function is_connected(graph::DAG) - nodeQueue = Deque{Node}() - push!(nodeQueue, get_exit_node(graph)) - seenNodes = Set{Node}() - - while !isempty(nodeQueue) - current = pop!(nodeQueue) - push!(seenNodes, current) - - for child in current.children - push!(nodeQueue, child) - end - end - - return length(seenNodes) == length(graph.nodes) -end - -function show_nodes(io, graph::DAG) - print(io, "[") - first = true - for n in graph.nodes - if first - first = false - else - print(io, ", ") - end - print(io, n) - end - print(io, "]") -end - -function show(io::IO, graph::DAG) - println(io, "Graph:") - print(io, " Nodes: ") - - nodeDict = Dict{Type, Int64}() - noEdges = 0 - for node in graph.nodes - if haskey(nodeDict, typeof(node.task)) - nodeDict[typeof(node.task)] = nodeDict[typeof(node.task)] + 1 - else - nodeDict[typeof(node.task)] = 1 - end - noEdges += length(parents(node)) - end - - if length(graph.nodes) <= 20 - show_nodes(io, graph) - else - print("Total: ", length(graph.nodes), ", ") - first = true - i = 0 - for (type, number) in zip(keys(nodeDict), values(nodeDict)) - i += 1 - if first - first = false - else - print(", ") - end - if (i % 3 == 0) - print("\n ") - end - print(type, ": ", number) - end - end - println(io) - println(io, " Edges: ", noEdges) - properties = graph_properties(graph) - println(io, " Total Compute Effort: ", properties.compute_effort) - println(io, " Total Data Transfer: ", properties.data) - println(io, " Total Compute Intensity: ", properties.compute_intensity) -end - -function show(io::IO, diff::Diff) - print(io, "Nodes: ") - print(io, length(diff.addedNodes) + length(diff.removedNodes)) - print(io, " Edges: ") - print(io, length(diff.addedEdges) + length(diff.removedEdges)) -end - -# return a namedtuple of the lengths of the added/removed nodes/edges -function length(diff::Diff) - return ( - addedNodes = length(diff.addedNodes), - removedNodes = length(diff.removedNodes), - addedEdges = length(diff.addedEdges), - removedEdges = length(diff.removedEdges) - ) -end - -function is_valid(graph::DAG) - for node in graph.nodes - @assert is_valid(graph, node) - end - - for op in graph.operationsToApply - @assert is_valid(graph, op) - end - - for nr in graph.possibleOperations.nodeReductions - @assert is_valid(graph, nr) - end - for ns in graph.possibleOperations.nodeSplits - @assert is_valid(graph, ns) - end - for nf in graph.possibleOperations.nodeFusions - @assert is_valid(graph, nf) - end - - for node in graph.dirtyNodes - @assert node in graph "Dirty Node is not part of the graph!" - @assert ismissing(node.nodeReduction) "Dirty Node has a NodeReduction!" - @assert ismissing(node.nodeSplit) "Dirty Node has a NodeSplit!" - if (typeof(node) <: DataTaskNode) - @assert ismissing(node.nodeFusion) "Dirty DataTaskNode has a Node Fusion!" - elseif (typeof(node) <: ComputeTaskNode) - @assert isempty(node.nodeFusions) "Dirty ComputeTaskNode has Node Fusions!" - end - end - - @assert is_connected(graph) "Graph is not connected!" - - return true -end diff --git a/src/abc_model/parse.jl b/src/models/abc/parse.jl similarity index 100% rename from src/abc_model/parse.jl rename to src/models/abc/parse.jl diff --git a/src/abc_model/task_functions.jl b/src/models/abc/properties.jl similarity index 100% rename from src/abc_model/task_functions.jl rename to src/models/abc/properties.jl diff --git a/src/abc_model/tasks.jl b/src/models/abc/types.jl similarity index 100% rename from src/abc_model/tasks.jl rename to src/models/abc/types.jl diff --git a/src/node/compare.jl b/src/node/compare.jl new file mode 100644 index 0000000..3743690 --- /dev/null +++ b/src/node/compare.jl @@ -0,0 +1,15 @@ +function ==(e1::Edge, e2::Edge) + return e1.edge[1] == e2.edge[1] && e1.edge[2] == e2.edge[2] +end + +function ==(n1::Node, n2::Node) + return false +end + +function ==(n1::ComputeTaskNode, n2::ComputeTaskNode) + return n1.id == n2.id +end + +function ==(n1::DataTaskNode, n2::DataTaskNode) + return n1.id == n2.id +end diff --git a/src/node/create.jl b/src/node/create.jl new file mode 100644 index 0000000..6343274 --- /dev/null +++ b/src/node/create.jl @@ -0,0 +1,23 @@ +function make_node(t::AbstractTask) + error("Cannot make a node from this task type") +end + +function make_node(t::AbstractDataTask) + return DataTaskNode(t) +end + +function make_node(t::AbstractComputeTask) + return ComputeTaskNode(t) +end + +function make_edge(n1::Node, n2::Node) + error("Can only create edges from compute to data node or reverse") +end + +function make_edge(n1::ComputeTaskNode, n2::DataTaskNode) + return Edge((n1, n2)) +end + +function make_edge(n1::DataTaskNode, n2::ComputeTaskNode) + return Edge((n1, n2)) +end diff --git a/src/node/print.jl b/src/node/print.jl new file mode 100644 index 0000000..3731890 --- /dev/null +++ b/src/node/print.jl @@ -0,0 +1,7 @@ +function show(io::IO, n::Node) + print(io, "Node(", n.task, ")") +end + +function show(io::IO, e::Edge) + print(io, "Edge(", e.edge[1], ", ", e.edge[2], ")") +end diff --git a/src/node/properties.jl b/src/node/properties.jl new file mode 100644 index 0000000..bccb026 --- /dev/null +++ b/src/node/properties.jl @@ -0,0 +1,52 @@ +is_entry_node(node::Node) = length(node.children) == 0 +is_exit_node(node::Node) = length(node.parents) == 0 + +# children = prerequisite nodes, nodes that need to execute before the task, edges point into this task +function children(node::Node) + return copy(node.children) +end + +# parents = subsequent nodes, nodes that need this node to execute, edges point from this task +function parents(node::Node) + return copy(node.parents) +end + +# siblings = all children of any parents, no duplicates, includes the node itself +function siblings(node::Node) + result = Set{Node}() + push!(result, node) + for parent in node.parents + union!(result, parent.children) + end + + return result +end + +# partners = all parents of any children, no duplicates, includes the node itself +function partners(node::Node) + result = Set{Node}() + push!(result, node) + for child in node.children + union!(result, child.parents) + end + + return result +end + +# alternative version to partners(Node), avoiding allocation of a new set +# works on the given set and returns nothing +function partners(node::Node, set::Set{Node}) + push!(set, node) + for child in node.children + union!(set, child.parents) + end + return nothing +end + +function is_parent(potential_parent, node) + return potential_parent in node.parents +end + +function is_child(potential_child, node) + return potential_child in node.children +end diff --git a/src/nodes.jl b/src/node/type.jl similarity index 82% rename from src/nodes.jl rename to src/node/type.jl index c1e6b80..4ff3e6d 100644 --- a/src/nodes.jl +++ b/src/node/type.jl @@ -54,3 +54,7 @@ struct Edge # edge points from child to parent edge::Union{Tuple{DataTaskNode, ComputeTaskNode}, Tuple{ComputeTaskNode, DataTaskNode}} end + +copy(m::Missing) = missing +copy(n::ComputeTaskNode) = ComputeTaskNode(copy(n.task), copy(n.parents), copy(n.children), UUIDs.uuid1(rng[threadid()]), copy(n.nodeReduction), copy(n.nodeSplit), copy(n.nodeFusions)) +copy(n::DataTaskNode) = DataTaskNode(copy(n.task), copy(n.parents), copy(n.children), UUIDs.uuid1(rng[threadid()]), copy(n.nodeReduction), copy(n.nodeSplit), copy(n.nodeFusion)) diff --git a/src/node/validate.jl b/src/node/validate.jl new file mode 100644 index 0000000..a53af2c --- /dev/null +++ b/src/node/validate.jl @@ -0,0 +1,43 @@ +function is_valid_node(graph::DAG, node::Node) + @assert node in graph "Node is not part of the given graph!" + + for parent in node.parents + @assert typeof(parent) != typeof(node) "Node's type is the same as its parent's!" + @assert parent in graph "Node's parent is not in the same graph!" + @assert node in parent.children "Node is not a child of its parent!" + end + + for child in node.children + @assert typeof(child) != typeof(node) "Node's type is the same as its child's!" + @assert child in graph "Node's child is not in the same graph!" + @assert node in child.parents "Node is not a parent of its child!" + end + + if !ismissing(node.nodeReduction) + @assert is_valid(graph, node.nodeReduction) + end + if !ismissing(node.nodeSplit) + @assert is_valid(graph, node.nodeSplit) + end + return true +end + +# call with @assert +function is_valid(graph::DAG, node::ComputeTaskNode) + @assert is_valid_node(graph, node) + + for nf in node.nodeFusions + @assert is_valid(graph, nf) + end + return true +end + +# call with @assert +function is_valid(graph::DAG, node::DataTaskNode) + @assert is_valid_node(graph, node) + + if !ismissing(node.nodeFusion) + @assert is_valid(graph, node.nodeFusion) + end + return true +end diff --git a/src/node_functions.jl b/src/node_functions.jl deleted file mode 100644 index cccde82..0000000 --- a/src/node_functions.jl +++ /dev/null @@ -1,95 +0,0 @@ -function make_node(t::AbstractTask) - error("Cannot make a node from this task type") -end - -function make_node(t::AbstractDataTask) - return DataTaskNode(t) -end - -function make_node(t::AbstractComputeTask) - return ComputeTaskNode(t) -end - -function make_edge(n1::Node, n2::Node) - error("Can only create edges from compute to data node or reverse") -end - -function make_edge(n1::ComputeTaskNode, n2::DataTaskNode) - return Edge((n1, n2)) -end - -function make_edge(n1::DataTaskNode, n2::ComputeTaskNode) - return Edge((n1, n2)) -end - -function show(io::IO, n::Node) - print(io, "Node(", n.task, ")") -end - -function show(io::IO, e::Edge) - print(io, "Edge(", e.edge[1], ", ", e.edge[2], ")") -end - -function ==(e1::Edge, e2::Edge) - return e1.edge[1] == e2.edge[1] && e1.edge[2] == e2.edge[2] -end - -function ==(n1::Node, n2::Node) - return false -end - -function ==(n1::ComputeTaskNode, n2::ComputeTaskNode) - return n1.id == n2.id -end - -function ==(n1::DataTaskNode, n2::DataTaskNode) - return n1.id == n2.id -end - -function is_valid_node(graph::DAG, node::Node) - @assert node in graph "Node is not part of the given graph!" - - for parent in node.parents - @assert typeof(parent) != typeof(node) "Node's type is the same as its parent's!" - @assert parent in graph "Node's parent is not in the same graph!" - @assert node in parent.children "Node is not a child of its parent!" - end - - for child in node.children - @assert typeof(child) != typeof(node) "Node's type is the same as its child's!" - @assert child in graph "Node's child is not in the same graph!" - @assert node in child.parents "Node is not a parent of its child!" - end - - if !ismissing(node.nodeReduction) - @assert is_valid(graph, node.nodeReduction) - end - if !ismissing(node.nodeSplit) - @assert is_valid(graph, node.nodeSplit) - end - return true -end - -# call with @assert -function is_valid(graph::DAG, node::ComputeTaskNode) - @assert is_valid_node(graph, node) - - for nf in node.nodeFusions - @assert is_valid(graph, nf) - end - return true -end - -# call with @assert -function is_valid(graph::DAG, node::DataTaskNode) - @assert is_valid_node(graph, node) - - if !ismissing(node.nodeFusion) - @assert is_valid(graph, node.nodeFusion) - end - return true -end - -copy(m::Missing) = missing -copy(n::ComputeTaskNode) = ComputeTaskNode(copy(n.task), copy(n.parents), copy(n.children), UUIDs.uuid1(rng[threadid()]), copy(n.nodeReduction), copy(n.nodeSplit), copy(n.nodeFusions)) -copy(n::DataTaskNode) = DataTaskNode(copy(n.task), copy(n.parents), copy(n.children), UUIDs.uuid1(rng[threadid()]), copy(n.nodeReduction), copy(n.nodeSplit), copy(n.nodeFusion)) diff --git a/src/operations/apply.jl b/src/operation/apply.jl similarity index 100% rename from src/operations/apply.jl rename to src/operation/apply.jl diff --git a/src/operations/clean.jl b/src/operation/clean.jl similarity index 100% rename from src/operations/clean.jl rename to src/operation/clean.jl diff --git a/src/operations/find.jl b/src/operation/find.jl similarity index 100% rename from src/operations/find.jl rename to src/operation/find.jl diff --git a/src/operations/get.jl b/src/operation/get.jl similarity index 100% rename from src/operations/get.jl rename to src/operation/get.jl diff --git a/src/operations/print.jl b/src/operation/print.jl similarity index 100% rename from src/operations/print.jl rename to src/operation/print.jl diff --git a/src/operation/type.jl b/src/operation/type.jl new file mode 100644 index 0000000..bc98444 --- /dev/null +++ b/src/operation/type.jl @@ -0,0 +1,34 @@ +# An abstract base class for operations +# an operation can be applied to a DAG +abstract type Operation end + +# An abstract base class for already applied operations +# an applied operation can be reversed iff it is the last applied operation on the DAG +abstract type AppliedOperation end + +struct NodeFusion <: Operation + input::Tuple{ComputeTaskNode, DataTaskNode, ComputeTaskNode} +end + +struct AppliedNodeFusion <: AppliedOperation + operation::NodeFusion + diff::Diff +end + +struct NodeReduction <: Operation + input::Vector{Node} +end + +struct AppliedNodeReduction <: AppliedOperation + operation::NodeReduction + diff::Diff +end + +struct NodeSplit <: Operation + input::Node +end + +struct AppliedNodeSplit <: AppliedOperation + operation::NodeSplit + diff::Diff +end diff --git a/src/operations/utility.jl b/src/operation/utility.jl similarity index 100% rename from src/operations/utility.jl rename to src/operation/utility.jl diff --git a/src/operations/validate.jl b/src/operation/validate.jl similarity index 100% rename from src/operations/validate.jl rename to src/operation/validate.jl diff --git a/src/task/compare.jl b/src/task/compare.jl new file mode 100644 index 0000000..f2869b3 --- /dev/null +++ b/src/task/compare.jl @@ -0,0 +1,11 @@ +function ==(t1::AbstractTask, t2::AbstractTask) + return false +end + +function ==(t1::AbstractComputeTask, t2::AbstractComputeTask) + return typeof(t1) == typeof(t2) +end + +function ==(t1::AbstractDataTask, t2::AbstractDataTask) + return data(t1) == data(t2) +end diff --git a/src/task/print.jl b/src/task/print.jl new file mode 100644 index 0000000..b2cce41 --- /dev/null +++ b/src/task/print.jl @@ -0,0 +1,4 @@ +function show(io::IO, t::FusedComputeTask) + (T1, T2) = get_types(t) + print(io, "ComputeFuse(", T1(), ", ", T2(), ")") +end diff --git a/src/task_functions.jl b/src/task/properties.jl similarity index 62% rename from src/task_functions.jl rename to src/task/properties.jl index d0a371e..1349fbf 100644 --- a/src/task_functions.jl +++ b/src/task/properties.jl @@ -30,24 +30,4 @@ function compute_intensity(t::AbstractTask)::UInt64 return typemax(UInt64) end return compute_effort(t) / data(t) -end - -function show(io::IO, t::FusedComputeTask) - (T1, T2) = get_types(t) - print(io, "ComputeFuse(", T1(), ", ", T2(), ")") -end - -function ==(t1::AbstractTask, t2::AbstractTask) - return false -end - -function ==(t1::AbstractComputeTask, t2::AbstractComputeTask) - return typeof(t1) == typeof(t2) -end - -function ==(t1::AbstractDataTask, t2::AbstractDataTask) - return data(t1) == data(t2) -end - -copy(t::AbstractDataTask) = error("Need to implement copying for your data tasks!") -copy(t::AbstractComputeTask) = typeof(t)() +end \ No newline at end of file diff --git a/src/tasks.jl b/src/task/type.jl similarity index 70% rename from src/tasks.jl rename to src/task/type.jl index da9d7f9..ef8eac8 100644 --- a/src/tasks.jl +++ b/src/task/type.jl @@ -7,3 +7,6 @@ struct FusedComputeTask{T1<:AbstractComputeTask, T2<:AbstractComputeTask} <: Abs end get_types(::FusedComputeTask{T1, T2}) where {T1, T2} = (T1, T2) + +copy(t::AbstractDataTask) = error("Need to implement copying for your data tasks!") +copy(t::AbstractComputeTask) = typeof(t)()