Various optimizations, Readme update, Bench script for imports

src/graph_functions.jl

@@ -70,8 +70,8 @@ end
 
 function insert_edge(graph::DAG, edge::Edge)
    # edge points from child to parent
-   push!(edge.edge[1][].parents, edge.edge[2][])
-   push!(edge.edge[2][].children, edge.edge[1][])
+   push!(edge.edge[1].parents, edge.edge[2])
+   push!(edge.edge[2].children, edge.edge[1])
 
    return edge
 end
@@ -82,8 +82,8 @@ function remove_node(graph::DAG, node::Node)
 end
 
 function remove_edge(graph::DAG, edge::Edge)
-   delete!(edge.edge[1][].parents, edge.edge[2][])
-   delete!(edge.edge[2][].children, edge.edge[1][])
+   filter!(x -> x != edge.edge[2], edge.edge[1].parents)
+   filter!(x -> x != edge.edge[1], edge.edge[2].children)
    return nothing
 end
 
@@ -175,6 +175,7 @@ function show(io::IO, graph::DAG)
    if length(graph.nodes) <= 20
       show_nodes(io, graph)
    else
+      print("Total: ", length(graph.nodes), ", ")
       first = true
       for (type, number) in zip(keys(nodeDict), values(nodeDict))
          if first

src/import.jl

@@ -20,17 +20,24 @@ function parse_edges(input::AbstractString)
     return output
 end
 
-function importTxt(filename::String)
+function import_txt(filename::String, verbose::Bool = isinteractive())
     file = open(filename, "r")
 
-    println("Opened file")
+    if (verbose) println("Opened file") end
     nodes_string = readline(file)
     nodes = parse_nodes(nodes_string)
 
     close(file)
-    println("Read file")
+    if (verbose) println("Read file") end
 
     graph = DAG()
+    
+    # estimate total number of nodes
+    # try to slightly overestimate so no resizing is necessary
+    # data nodes are not included in length(nodes) and there are a few more than compute nodes
+    estimate_no_nodes = round(Int, length(nodes) * 4)
+    if (verbose) println("Estimating ", estimate_no_nodes, " Nodes") end
+    sizehint!(graph.nodes, estimate_no_nodes)
 
     sum_node = insert_node(graph, make_node(ComputeTaskSum()))
     global_data_out = insert_node(graph, make_node(DataTask(10)))
@@ -39,14 +46,14 @@ function importTxt(filename::String)
     # remember the data out nodes for connection
     dataOutNodes = Dict()
 
-    println("Building graph")
+    if (verbose) println("Building graph") end
     noNodes = 0
     nodesToRead = length(nodes)
     while !isempty(nodes)
         node = popfirst!(nodes)
         noNodes += 1
         if (noNodes % 100 == 0)
-            @printf "\rReading Nodes... %.2f%%" (100. * noNodes / nodesToRead)
+            if (verbose) @printf "\rReading Nodes... %.2f%%" (100. * noNodes / nodesToRead) end
         end
         if occursin(regex_a, node)
             # add nodes and edges for the state reading to u(P(Particle))
@@ -127,7 +134,10 @@ function importTxt(filename::String)
 
             insert_edge(graph, make_edge(data_out, sum_node))
         elseif occursin(regex_plus, node)
-            println("\rReading Nodes Complete    ")
+            if (verbose)
+                println("\rReading Nodes Complete    ") 
+                println("Added ", length(graph.nodes), " nodes")
+            end
         else
             error("Unknown node '", node, "' while reading from file ", filename)
         end

src/metagraph_optimization.jl

@@ -11,13 +11,16 @@ include("node_functions.jl")
 include("graph_functions.jl")
 include("graph_operations.jl")
 include("import.jl")
+include("utility.jl")
 
 export Node, Edge, ComputeTaskNode, DataTaskNode, DAG
 export AbstractTask, AbstractComputeTask, AbstractDataTask, DataTask, ComputeTaskP, ComputeTaskS1, ComputeTaskS2, ComputeTaskV, ComputeTaskU, ComputeTaskSum, FusedComputeTask
 export make_node, make_edge, insert_node, insert_edge, is_entry_node, is_exit_node, parents, children, compute, graph_properties, get_exit_node, is_valid
 export node_fusion, node_reduction, node_split, generate_options
-export importTxt
+export import_txt
 
 export ==, in, show
 
+export bytes_to_human_readable
+
 end # module metagraph_optimization

src/node_functions.jl

@@ -15,11 +15,11 @@ function make_edge(n1::Node, n2::Node)
 end
 
 function make_edge(n1::ComputeTaskNode, n2::DataTaskNode)
-    return Edge((Ref(n1), Ref(n2)))
+    return Edge((n1, n2))
 end
 
 function make_edge(n1::DataTaskNode, n2::ComputeTaskNode)
-    return Edge((Ref(n1), Ref(n2)))
+    return Edge((n1, n2))
 end
 
 function show(io::IO, n::Node)
@@ -27,9 +27,9 @@ function show(io::IO, n::Node)
 end
 
 function show(io::IO, e::Edge)
-    print(io, "Edge(", e.edge[1][], ", ", e.edge[2][], ")")
+    print(io, "Edge(", e.edge[1], ", ", e.edge[2], ")")
 end
 
 function ==(e1::Edge, e2::Edge)
-    return e1.edge[1][] == e2.edge[1][] && e1.edge[2][] == e2.edge[2][]
+    return e1.edge[1] == e2.edge[1] && e1.edge[2] == e2.edge[2]
 end

src/nodes.jl

@@ -1,33 +1,38 @@
 using Random
 using UUIDs
 
-rng = Random.MersenneTwister(0);
+rng = Random.MersenneTwister(0)
 
 abstract type Node end
 
 struct DataTaskNode <: Node
    task::AbstractDataTask
-   parents::Set{Node}
-   children::Set{Node}
-   id::Base.UUID  # need a unique identifier unique to every *constructed* node
+   
+   # use vectors as sets have way too much memory overhead
+   parents::Vector{Node}
+   children::Vector{Node}
+
+   # need a unique identifier unique to every *constructed* node
+   id::Base.UUID
 end
 
+# same as DataTaskNode
 struct ComputeTaskNode <: Node
    task::AbstractComputeTask
-   parents::Set{Node}
-   children::Set{Node}
-   id::Base.UUID  # need a unique identifier unique to every *constructed* node
+   parents::Vector{Node}
+   children::Vector{Node}
+   id::Base.UUID
 end
 
 function DataTaskNode(t::AbstractDataTask)
-   return DataTaskNode(t, Set{Node}(), Set{Node}(), UUIDs.uuid1(rng))
+   return DataTaskNode(t, Vector{Node}(), Vector{Node}(), UUIDs.uuid1(rng))
 end
 
 function ComputeTaskNode(t::AbstractComputeTask)
-   return ComputeTaskNode(t, Set{Node}(), Set{Node}(), UUIDs.uuid1(rng))
+   return ComputeTaskNode(t, Vector{Node}(), Vector{Node}(), UUIDs.uuid1(rng))
 end
 
 struct Edge
    # edge points from child to parent
-   edge::Union{Tuple{Ref{DataTaskNode}, Ref{ComputeTaskNode}}, Tuple{Ref{ComputeTaskNode}, Ref{DataTaskNode}}}
+   edge::Union{Tuple{DataTaskNode, ComputeTaskNode}, Tuple{ComputeTaskNode, DataTaskNode}}
 end

src/utility.jl

@@ -0,0 +1,9 @@
+function bytes_to_human_readable(bytes)
+    units = ["B", "KiB", "MiB", "GiB", "TiB"]
+    unit_index = 1
+    while bytes >= 1024 && unit_index < length(units)
+        bytes /= 1024
+        unit_index += 1
+    end
+    return string(round(bytes, digits=4), " ", units[unit_index])
+end