Fix tests

Reviewed-on: Rubydragon/MetagraphOptimization.jl#5

.JuliaFormatter.toml

@@ -0,0 +1,13 @@
+indent = 4
+margin = 80
+always_for_in = true
+for_in_replacement = "in"
+whitespace_typedefs = true
+whitespace_ops_in_indices = true
+long_to_short_function_def = false
+always_use_return = true
+whitespace_in_kwargs = true
+conditional_to_if = true
+normalize_line_endings = "unix"
+
+overwrite = true

.gitattributes

@@ -1,2 +1,2 @@
-examples/AB->ABBBBBBB.txt filter=lfs diff=lfs merge=lfs -text
-examples/AB->ABBBBBBBBB.txt filter=lfs diff=lfs merge=lfs -text
+input/AB->ABBBBBBBBB.txt filter=lfs diff=lfs merge=lfs -text
+input/AB->ABBBBBBB.txt filter=lfs diff=lfs merge=lfs -text

.gitea/workflows/julia-package-ci.yml

@@ -1,33 +1,185 @@
-name: Test
+name: MetagraphOptimization_CI
 
 on: [push]
 
+env:
+  # keep the depot directly in the repository for the cache
+  JULIA_DEPOT_PATH: './.julia'
+
 jobs:
-  test:
+  prepare:
     runs-on: arch-latest
 
     steps:
-      #- name: Get git-lfs
-      #  run: apt-get update && apt-get install git-lfs
-
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      #- name: Checkout LFS objects
-      #  run: git lfs checkout
-
       - name: Setup Julia environment
-        uses: https://github.com/julia-actions/setup-julia@v1.9.1
+        uses: https://github.com/julia-actions/setup-julia@v1.9.2
         with:
-          version: '1.9.1'
+          version: '1.9.2'
+
+      # needed for the file hashing, should be removed when ${{ hashFiles('**/Project.toml') }} is supported in gitea
+      - name: Setup go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Hash files
+        uses: https://gitea.com/actions/go-hashfiles@v0.0.1
+        id: get-hash
+        with: 
+          patterns: |-
+            **/Project.toml
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+      - name: Check cache hit
+        if: steps.cache-restore.outputs.cache-hit == 'true'
+        run: exit 0
 
       - name: Install dependencies
-        run: julia --project -e 'import Pkg; Pkg.instantiate()'
+        run: |
+          julia --project=./        -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+          julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+          julia --project=docs/     -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Cache Julia packages
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+  test:
+    needs: prepare
+    runs-on: arch-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Julia environment
+        uses: https://github.com/julia-actions/setup-julia@v1.9.2
+        with:
+          version: '1.9.2'
+
+      # needed for the file hashing, should be removed when ${{ hashFiles('**/Project.toml') }} is supported in gitea
+      - name: Setup go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Hash files
+        uses: https://gitea.com/actions/go-hashfiles@v0.0.1
+        id: get-hash
+        with: 
+          patterns: |-
+            **/Project.toml  
+
+      - name: Restore cached Julia packages
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+      - name: Install dependencies
+        run: |
+          julia --project=./        -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+          julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+          julia --project=docs/     -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Format check
+        run: |
+          julia --project=./ -e 'using JuliaFormatter; format(".", verbose=true)'
+          julia --project=./ -e '
+          out = Cmd(`git diff --name-only`) |> read |> String
+          if out == ""
+            exit(0)
+          else
+            @error "Some files have not been formatted !!!"
+            write(stdout, out)
+            exit(1)
+          end'
 
       - name: Run tests
-        run: julia --project -e 'import Pkg; Pkg.test()'
+        run: julia --project=./ -t 4 -e 'import Pkg; Pkg.test()' -O0
 
       - name: Run examples
-        run: julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); include("examples/import_bench.jl")'
+        run: julia --project=examples/ -t 4 -e 'include("examples/import_bench.jl")' -O3
+
+  docs:
+    needs: prepare
+    runs-on: arch-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Julia environment
+        uses: https://github.com/julia-actions/setup-julia@v1.9.2
+        with:
+          version: '1.9.2'
+
+      # needed for the file hashing, should be removed when ${{ hashFiles('**/Project.toml') }} is supported in gitea
+      - name: Setup go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Hash files
+        uses: https://gitea.com/actions/go-hashfiles@v0.0.1
+        id: get-hash
+        with: 
+          patterns: |-
+            **/Project.toml  
+
+      - name: Restore cached Julia packages
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+      - name: Install dependencies
+        run: |
+          julia --project=./        -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+          julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+          julia --project=docs/     -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Build docs
+        run: julia --project=docs/ docs/make.jl
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: web-doc
+          path: docs/build/
+
+      #- name: Webhook Trigger
+      #  uses: https://github.com/zzzze/webhook-trigger@master
+      #  continue-on-error: true
+      #  with:
+      #    data: "{\"event\":\"action_completed\", \"download_url\":\"deckardcain.local:8099/something\"}"
+      #    webhook_url: ${{ secrets.WEBHOOK_URL }}

.gitignore

@@ -1,10 +1,10 @@
 # ---> Julia
 # Files generated by invoking Julia with --code-coverage
-*.jl.cov
-*.jl.*.cov
+*.cov
+*.cov
 
 # Files generated by invoking Julia with --track-allocation
-*.jl.mem
+*.mem
 
 # System-specific files and directories generated by the BinaryProvider and BinDeps packages
 # They contain absolute paths specific to the host computer, and so should not be committed

Project.toml

@@ -5,7 +5,7 @@ version = "0.1.0"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 

README.md

@@ -4,15 +4,17 @@ Directed Acyclic Graph optimization for QED
 
 ## Usage
 
+For all the julia calls, use `-t n` to give julia `n` threads.
+
 Instantiate the project first:
 
-`julia --project -e 'import Pkg; Pkg.instantiate()'`
+`julia --project=./ -e 'import Pkg; Pkg.instantiate()'`
 
 ### Run Tests
 
 To run all tests, run
 
-`julia --project=. -e 'import Pkg; Pkg.test()'`
+`julia --project=./ -e 'import Pkg; Pkg.test()' -O0`
 
 ### Run Examples
 
@@ -22,7 +24,7 @@ Get the correct environment for the examples folder:
 
 Then execute a specific example:
 
-`julia --project=examples examples/<file>.jl`
+`julia --project=examples examples/<file>.jl -O3`
 
 ## Concepts
 

docs/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8"
+MetagraphOptimization = "3e869610-d48d-4942-ba70-c1b702a33ca4"

docs/make.jl

@@ -0,0 +1,29 @@
+using Documenter
+using MetagraphOptimization
+
+makedocs(
+    #format = Documenter.LaTeX(platform=""),
+
+    root = "docs",
+    source = "src",
+    build = "build",
+    clean = true,
+    doctest = true,
+    modules = Module[MetagraphOptimization],
+    repo = "https://code.woubery.com/Rubydragon/MetagraphOptimization.jl/src/branch/{commit}{path}#L{line}",
+    sitename = "MetagraphOptimization.jl",
+    pages = [
+        "index.md",
+        "Manual" => "manual.md",
+        "Library" => [
+            "Graph" => "lib/internals/graph.md",
+            "Node" => "lib/internals/node.md",
+            "Task" => "lib/internals/task.md",
+            "Operation" => "lib/internals/operation.md",
+            "Models" => "lib/internals/models.md",
+            "Diff" => "lib/internals/diff.md",
+            "Utility" => "lib/internals/utility.md",
+        ],
+        "Contribution" => "contribution.md",
+    ],
+)

docs/src/contribution.md

@@ -0,0 +1,3 @@
+# Contribution
+
+This is currently in development for a diploma thesis and is therefore private and impossible to contribute to.

docs/src/index.md

@@ -0,0 +1,26 @@
+# MetagraphOptimization.jl
+
+*A domain-specific DAG-optimizer*
+
+## Package Features
+- Read a DAG from a file
+- Analyze its properties
+- Mute the graph using the operations NodeFusion, NodeReduction and NodeSplit
+
+## Coming Soon:
+- Add Code Generation from finished DAG
+- Add optimization algorithms and strategies
+
+## Library Outline
+
+```@contents
+Pages = [
+    "lib/public.md",
+    "lib/internals.md"
+]
+```
+
+### [Index](@id main-index)
+```@index
+Pages = ["lib/public.md"]
+```

docs/src/lib/internals/diff.md

@@ -0,0 +1,22 @@
+# Diff
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["diff/type.jl"]
+Order   = [:type]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["diff/properties.jl"]
+Order   = [:function]
+```
+
+## Printing
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["diff/print.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/graph.md

@@ -0,0 +1,50 @@
+# Graph
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/type.jl"]
+Order   = [:type]
+```
+
+## Interface
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/interface.jl"]
+Order   = [:function]
+```
+
+## Compare
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/compare.jl"]
+Order   = [:function]
+```
+
+## Mute
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/mute.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/print.jl"]
+Order   = [:function]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/properties.jl"]
+Order   = [:function]
+```
+
+## Validate
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/validate.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/models.md

@@ -0,0 +1,28 @@
+# Models
+
+## ABC-Model
+
+### Types
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/types.jl"]
+Order   = [:type, :constant]
+```
+
+### Parse
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/parse.jl"]
+Order   = [:function]
+```
+
+### Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/properties.jl"]
+Order   = [:function]
+```
+
+## QED-Model
+
+*To be added*

docs/src/lib/internals/node.md

@@ -0,0 +1,43 @@
+# Node
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/type.jl"]
+Order   = [:type]
+```
+
+## Create
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/create.jl"]
+Order   = [:function]
+```
+
+## Compare
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/compare.jl"]
+Order   = [:function]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/properties.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/print.jl"]
+Order   = [:function]
+```
+
+## Validate
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/validate.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/operation.md

@@ -0,0 +1,57 @@
+# Operation
+
+## Types
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/type.jl"]
+Order   = [:type]
+```
+
+## Find
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/find.jl"]
+Order   = [:function]
+```
+
+## Apply
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/apply.jl"]
+Order   = [:function]
+```
+
+## Get
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/get.jl"]
+Order   = [:function]
+```
+
+## Clean
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/clean.jl"]
+Order   = [:function]
+```
+
+## Utility
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/utility.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/print.jl"]
+Order   = [:function]
+```
+
+## Validate
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/validate.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/task.md

@@ -0,0 +1,36 @@
+# Task
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/type.jl"]
+Order   = [:type]
+```
+
+## Create
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/create.jl"]
+Order   = [:function]
+```
+
+## Compare
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/compare.jl"]
+Order   = [:function]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/properties.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/print.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/utility.md

@@ -0,0 +1,17 @@
+# Utility
+
+## Helper Functions
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages   = ["utility.jl"]
+Order   = [:type, :function]
+```
+
+## Trie Helper
+This is a simple implementation of a [Trie Data Structure](https://en.wikipedia.org/wiki/Trie) to greatly improve the performance of the Node Reduction search.
+
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages   = ["trie.jl"]
+Order   = [:type, :function]
+```

docs/src/lib/public.md

@@ -0,0 +1,24 @@
+# Public Documentation
+
+Documentation for `MetagraphOptimization.jl`'s public interface.
+
+See the Internals section of the manual for documentation of everything else.
+
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages   = ["MetagraphOptimization.jl"]
+Order   = [:module]
+```
+
+## Contents
+
+```@contents
+Pages = ["public.md"]
+Depth = 2
+```
+
+## Index
+
+```@index
+Pages = ["public.md"]
+```

docs/src/manual.md

@@ -0,0 +1,3 @@
+# Manual
+
+This will become a manual.

examples/Project.toml

@@ -1,5 +1,3 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 MetagraphOptimization = "3e869610-d48d-4942-ba70-c1b702a33ca4"
-Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
-ProfileView = "c46f51b8-102a-5cf2-8d2c-8597cb0e0da7"

examples/import_bench.jl

@@ -7,7 +7,7 @@ function bench_txt(filepath::String, bench::Bool = true)
     name = basename(filepath)
     name, _ = splitext(name)
 
-    filepath = joinpath(@__DIR__, filepath)
+    filepath = joinpath(@__DIR__, "../input/", filepath)
     if !isfile(filepath)
         println("File ", filepath, " does not exist, skipping bench")
         return
@@ -16,12 +16,18 @@ function bench_txt(filepath::String, bench::Bool = true)
     println(name, ":")
     g = parse_abc(filepath)
     print(g)
-    println("  Graph size in memory: ", bytes_to_human_readable(Base.summarysize(g)))
+    println(
+        "  Graph size in memory: ",
+        bytes_to_human_readable(MetagraphOptimization.mem(g)),
+    )
 
     if (bench)
         @btime parse_abc($filepath)
-        println()
     end
+
+    println("  Get Operations: ")
+    @time get_operations(g)
+    return println()
 end
 
 function import_bench()
@@ -29,9 +35,9 @@ function import_bench()
     bench_txt("AB->ABBB.txt")
     bench_txt("AB->ABBBBB.txt")
     bench_txt("AB->ABBBBBBB.txt")
-    #bench_txt("AB->ABBBBBBBBB.txt", false)
+    #bench_txt("AB->ABBBBBBBBB.txt")
     bench_txt("ABAB->ABAB.txt")
-    bench_txt("ABAB->ABC.txt")
+    return bench_txt("ABAB->ABC.txt")
 end
 
 import_bench()

examples/plot_chain.jl

@@ -6,7 +6,7 @@ function gen_plot(filepath)
     name = basename(filepath)
     name, _ = splitext(name)
 
-    filepath = joinpath(@__DIR__, filepath)
+    filepath = joinpath(@__DIR__, "../input/", filepath)
     if !isfile(filepath)
         println("File ", filepath, " does not exist, skipping")
         return
@@ -21,7 +21,7 @@ function gen_plot(filepath)
     x = Vector{Float64}()
     y = Vector{Float64}()
 
-    for i = 1:30
+    for i in 1:30
         print("\r", i)
         # push
         opt = get_operations(g)
@@ -38,7 +38,7 @@ function gen_plot(filepath)
             push_operation!(g, rand(collect(opt.nodeSplits)))
             println("NS")
         else
-            i = i-1
+            i = i - 1
         end
 
         props = graph_properties(g)
@@ -48,13 +48,26 @@ function gen_plot(filepath)
 
     println("\rDone.")
 
-    plot([x[1], x[2]], [y[1], y[2]], linestyle = :solid, linewidth = 1, color = :red, legend=false)
+    plot(
+        [x[1], x[2]],
+        [y[1], y[2]],
+        linestyle = :solid,
+        linewidth = 1,
+        color = :red,
+        legend = false,
+    )
     # Create lines connecting the reference point to each data point
     for i in 3:length(x)
-        plot!([x[i-1], x[i]], [y[i-1], y[i]], linestyle = :solid, linewidth = 1, color = :red)
+        plot!(
+            [x[i - 1], x[i]],
+            [y[i - 1], y[i]],
+            linestyle = :solid,
+            linewidth = 1,
+            color = :red,
+        )
     end
 
-    gui()
+    return gui()
 end
 
 gen_plot("AB->ABBB.txt")

examples/plot_star.jl

@@ -6,7 +6,7 @@ function gen_plot(filepath)
     name = basename(filepath)
     name, _ = splitext(name)
 
-    filepath = joinpath(@__DIR__, filepath)
+    filepath = joinpath(@__DIR__, "../input/", filepath)
     if !isfile(filepath)
         println("File ", filepath, " does not exist, skipping")
         return
@@ -18,7 +18,7 @@ function gen_plot(filepath)
 
     println("Random Walking... ")
 
-    for i = 1:30
+    for i in 1:30
         print("\r", i)
         # push
         opt = get_operations(g)
@@ -35,7 +35,7 @@ function gen_plot(filepath)
             push_operation!(g, rand(collect(opt.nodeSplits)))
             println("NS")
         else
-            i = i-1
+            i = i - 1
         end
     end
 
@@ -60,7 +60,14 @@ function gen_plot(filepath)
         push!(y, props.compute_effort)
         pop_operation!(g)
 
-        push!(names, "NF: (" * string(props.data) * ", " * string(props.compute_effort) * ")")
+        push!(
+            names,
+            "NF: (" *
+            string(props.data) *
+            ", " *
+            string(props.compute_effort) *
+            ")",
+        )
     end
     for op in opt.nodeReductions
         push_operation!(g, op)
@@ -69,7 +76,14 @@ function gen_plot(filepath)
         push!(y, props.compute_effort)
         pop_operation!(g)
 
-        push!(names, "NR: (" * string(props.data) * ", " * string(props.compute_effort) * ")")
+        push!(
+            names,
+            "NR: (" *
+            string(props.data) *
+            ", " *
+            string(props.compute_effort) *
+            ")",
+        )
     end
     for op in opt.nodeSplits
         push_operation!(g, op)
@@ -78,19 +92,39 @@ function gen_plot(filepath)
         push!(y, props.compute_effort)
         pop_operation!(g)
 
-        push!(names, "NS: (" * string(props.data) * ", " * string(props.compute_effort) * ")")
+        push!(
+            names,
+            "NS: (" *
+            string(props.data) *
+            ", " *
+            string(props.compute_effort) *
+            ")",
+        )
     end
-    
-    plot([x0, x[1]], [y0, y[1]], linestyle = :solid, linewidth = 1, color = :red, legend=false)
+
+    plot(
+        [x0, x[1]],
+        [y0, y[1]],
+        linestyle = :solid,
+        linewidth = 1,
+        color = :red,
+        legend = false,
+    )
     # Create lines connecting the reference point to each data point
     for i in 2:length(x)
-        plot!([x0, x[i]], [y0, y[i]], linestyle = :solid, linewidth = 1, color = :red)
+        plot!(
+            [x0, x[i]],
+            [y0, y[i]],
+            linestyle = :solid,
+            linewidth = 1,
+            color = :red,
+        )
     end
     #scatter!(x, y, label=names)
 
     print(names)
 
-    gui()
+    return gui()
 end
 
 gen_plot("AB->ABBB.txt")

examples/profiling_utilities.jl

@@ -5,7 +5,7 @@ function test_random_walk(g::DAG, n::Int64)
 
     properties = graph_properties(g)
 
-    for i = 1:n
+    for i in 1:n
         # choose push or pop
         if rand(Bool)
             # push
@@ -32,5 +32,5 @@ function test_random_walk(g::DAG, n::Int64)
         end
     end
 
-    reset_graph!(g)
-end
+    return reset_graph!(g)
+end

results/FWKHIP8999

@@ -0,0 +1,164 @@
+Commit Hash: a7fb15c95b63eee40eb7b9324d83b748053c5e13
+
+Run with 32 Threads
+
+AB->AB:
+Graph:
+  Nodes: Total: 34, ComputeTaskS2: 2, ComputeTaskU: 4, 
+         ComputeTaskSum: 1, ComputeTaskV: 4, ComputeTaskP: 4, 
+         DataTask: 19
+  Edges: 37
+  Total Compute Effort: 185
+  Total Data Transfer: 104
+  Total Compute Intensity: 1.7788461538461537
+  28.171 μs (515 allocations: 52.06 KiB)
+  Get Operations: 
+Sorting...
+  0.218136 seconds (155.59 k allocations: 10.433 MiB, 3.34% gc time, 3175.93% compilation time)
+Node Reductions...
+  0.299127 seconds (257.04 k allocations: 16.853 MiB, 2827.94% compilation time)
+Node Fusions...
+  0.046983 seconds (16.70 k allocations: 1.120 MiB, 3048.15% compilation time)
+Node Splits...
+  0.033681 seconds (14.09 k allocations: 958.144 KiB, 3166.45% compilation time)
+Waiting...
+  0.000001 seconds
+  1.096006 seconds (581.46 k allocations: 38.180 MiB, 0.66% gc time, 1677.26% compilation time)
+
+AB->ABBB:
+Graph:
+  Nodes: Total: 280, ComputeTaskS2: 24, ComputeTaskU: 6, 
+         ComputeTaskV: 64, ComputeTaskSum: 1, ComputeTaskP: 6, 
+         ComputeTaskS1: 36, DataTask: 143
+  Edges: 385
+  Total Compute Effort: 2007
+  Total Data Transfer: 1176
+  Total Compute Intensity: 1.7066326530612246
+  207.236 μs (4324 allocations: 296.87 KiB)
+  Get Operations: 
+Sorting...
+  0.000120 seconds (167 allocations: 16.750 KiB)
+Node Reductions...
+  0.000550 seconds (1.98 k allocations: 351.234 KiB)
+Node Fusions...
+  0.000168 seconds (417 allocations: 83.797 KiB)
+Node Splits...
+  0.000150 seconds (478 allocations: 36.406 KiB)
+Waiting...
+  0.000000 seconds
+  0.039897 seconds (16.19 k allocations: 1.440 MiB, 95.31% compilation time)
+
+AB->ABBBBB:
+Graph:
+  Nodes: Total: 7854, ComputeTaskS2: 720, ComputeTaskU: 8, 
+         ComputeTaskV: 1956, ComputeTaskSum: 1, ComputeTaskP: 8, 
+         ComputeTaskS1: 1230, DataTask: 3931
+  Edges: 11241
+  Total Compute Effort: 58789
+  Total Data Transfer: 34826
+  Total Compute Intensity: 1.6880778728536152
+  5.787 ms (121839 allocations: 7.72 MiB)
+  Get Operations: 
+Sorting...
+  0.000499 seconds (175 allocations: 17.000 KiB)
+Node Reductions...
+  0.002126 seconds (45.76 k allocations: 4.477 MiB)
+Node Fusions...
+  0.000949 seconds (7.09 k allocations: 1.730 MiB)
+Node Splits...
+  0.000423 seconds (8.06 k allocations: 544.031 KiB)
+Waiting...
+  0.000000 seconds
+  0.015005 seconds (100.12 k allocations: 13.161 MiB)
+
+AB->ABBBBBBB:
+Graph:
+  Nodes: Total: 438436, ComputeTaskS2: 40320, ComputeTaskU: 10, 
+         ComputeTaskV: 109600, ComputeTaskSum: 1, ComputeTaskP: 10, 
+         ComputeTaskS1: 69272, DataTask: 219223
+  Edges: 628665
+  Total Compute Effort: 3288131
+  Total Data Transfer: 1949004
+  Total Compute Intensity: 1.687082735592128
+  1.309 s (6826397 allocations: 430.63 MiB)
+  Get Operations: 
+Sorting...
+  0.011898 seconds (197 allocations: 17.688 KiB)
+Node Reductions...
+  0.110569 seconds (2.78 M allocations: 225.675 MiB)
+Node Fusions...
+  0.022475 seconds (380.91 k allocations: 108.982 MiB)
+Node Splits...
+  0.011369 seconds (438.80 k allocations: 28.743 MiB)
+Waiting...
+  0.000001 seconds
+  2.503065 seconds (5.77 M allocations: 683.968 MiB, 48.27% gc time)
+
+AB->ABBBBBBBBB:
+Graph:
+  Nodes: Total: 39456442, ComputeTaskS2: 3628800, ComputeTaskU: 12, 
+         ComputeTaskV: 9864100, ComputeTaskSum: 1, ComputeTaskP: 12, 
+         ComputeTaskS1: 6235290, DataTask: 19728227
+  Edges: 56578129
+  Total Compute Effort: 295923153
+  Total Data Transfer: 175407750
+  Total Compute Intensity: 1.6870585991782006
+  389.495 s (626095682 allocations: 37.80 GiB)
+  Get Operations: 
+Sorting...
+  1.181713 seconds (197 allocations: 17.688 KiB)
+Node Reductions...
+ 10.057358 seconds (251.09 M allocations: 19.927 GiB)
+Node Fusions...
+  1.288635 seconds (34.24 M allocations: 6.095 GiB)
+Node Splits...
+  0.719345 seconds (39.46 M allocations: 2.522 GiB)
+Waiting...
+  0.000001 seconds
+904.138951 seconds (519.47 M allocations: 54.494 GiB, 25.03% gc time)
+
+ABAB->ABAB:
+Graph:
+  Nodes: Total: 3218, ComputeTaskS2: 288, ComputeTaskU: 8, 
+         ComputeTaskV: 796, ComputeTaskSum: 1, ComputeTaskP: 8, 
+         ComputeTaskS1: 504, DataTask: 1613
+  Edges: 4581
+  Total Compute Effort: 24009
+  Total Data Transfer: 14144
+  Total Compute Intensity: 1.697468891402715
+  2.691 ms (49557 allocations: 3.17 MiB)
+  Get Operations: 
+Sorting...
+  0.000246 seconds (171 allocations: 16.875 KiB)
+Node Reductions...
+  0.001037 seconds (19.42 k allocations: 1.751 MiB)
+Node Fusions...
+  0.001512 seconds (3.04 k allocations: 1.027 MiB)
+Node Splits...
+  0.000197 seconds (3.41 k allocations: 231.078 KiB)
+Waiting...
+  0.000000 seconds
+  0.007492 seconds (42.20 k allocations: 5.399 MiB)
+
+ABAB->ABC:
+Graph:
+  Nodes: Total: 817, ComputeTaskS2: 72, ComputeTaskU: 7, 
+         ComputeTaskV: 198, ComputeTaskSum: 1, ComputeTaskP: 7, 
+         ComputeTaskS1: 120, DataTask: 412
+  Edges: 1151
+  Total Compute Effort: 6028
+  Total Data Transfer: 3538
+  Total Compute Intensity: 1.7037874505370265
+  602.767 μs (12544 allocations: 843.16 KiB)
+  Get Operations: 
+Sorting...
+  0.000127 seconds (171 allocations: 16.875 KiB)
+Node Reductions...
+  0.000440 seconds (5.33 k allocations: 494.047 KiB)
+Node Fusions...
+  0.001761 seconds (939 allocations: 280.797 KiB)
+Node Splits...
+  0.000123 seconds (1.00 k allocations: 72.109 KiB)
+Waiting...
+  0.000000 seconds
+  0.003831 seconds (11.74 k allocations: 1.451 MiB)

results/temp.md

@@ -0,0 +1,30 @@
+(AB->ABBBBBBB, 1)   1.620 s (5909018 allocations: 656.78 MiB)
+(AB->ABBBBBBB, 2)   758.299 ms (5909088 allocations: 765.78 MiB)
+(AB->ABBBBBBB, 3)   595.788 ms (5909161 allocations: 748.89 MiB)
+(AB->ABBBBBBB, 4)   849.007 ms (5880250 allocations: 762.00 MiB)
+(AB->ABBBBBBB, 5)   563.021 ms (5880332 allocations: 781.17 MiB)
+(AB->ABBBBBBB, 6)   526.095 ms (5880419 allocations: 818.32 MiB)
+(AB->ABBBBBBB, 7)   586.057 ms (5880482 allocations: 826.36 MiB)
+(AB->ABBBBBBB, 8)   504.515 ms (5880542 allocations: 796.58 MiB)
+
+(AB->ABBBBBBB, 1)   1.537 s (5596315 allocations: 616.81 MiB)
+(AB->ABBBBBBB, 2)   826.918 ms (5596385 allocations: 725.81 MiB)
+(AB->ABBBBBBB, 3)   538.787 ms (5596457 allocations: 708.92 MiB)
+(AB->ABBBBBBB, 4)   918.853 ms (5596528 allocations: 725.08 MiB)
+(AB->ABBBBBBB, 5)   511.959 ms (5596606 allocations: 744.25 MiB)
+(AB->ABBBBBBB, 6)   887.160 ms (5596691 allocations: 763.42 MiB)
+(AB->ABBBBBBB, 7)   898.757 ms (5596762 allocations: 789.91 MiB)
+(AB->ABBBBBBB, 8)   497.545 ms (5596820 allocations: 759.66 MiB)
+
+
+Initial:
+
+$ julia --project=examples/ -e 'using BenchmarkTools; using MetagraphOptimization; parse_abc("input/AB->AB.txt"); @time g = parse_abc("input/AB->ABBBBBBBBB.txt")'
+ 65.370947 seconds (626.10 M allocations: 37.381 GiB, 53.59% gc time, 0.01% compilation time)
+
+Removing make_edge from calls in parse:
+ 50.053920 seconds (593.41 M allocations: 32.921 GiB, 49.70% gc time, 0.09% compilation time)
+
+Nodes operation storage rework (and O3):
+ 31.997128 seconds (450.66 M allocations: 25.294 GiB, 31.56% gc time, 0.14% compilation time)
+ 

scripts/bench_threads.fish

@@ -0,0 +1,25 @@
+#!/bin/fish
+set minthreads 1
+set maxthreads 8
+
+julia --project=./examples -t 4 -e 'import Pkg; Pkg.instantiate()'
+
+#for i in $(seq $minthreads $maxthreads)
+#   printf "(AB->AB, $i) "
+#   julia --project=./examples -t $i -O3 -e 'using MetagraphOptimization; using BenchmarkTools; @btime get_operations(graph) setup=(graph = parse_abc("input/AB->AB.txt"))'
+#end
+
+#for i in $(seq $minthreads $maxthreads)
+#   printf "(AB->ABBB, $i) "
+#   julia --project=./examples -t $i -O3 -e 'using MetagraphOptimization; using BenchmarkTools; @btime get_operations(graph) setup=(graph = parse_abc("input/AB->ABBB.txt"))'
+#end
+
+#for i in $(seq $minthreads $maxthreads)
+#   printf "(AB->ABBBBB, $i) "
+#   julia --project=./examples -t $i -O3 -e 'using MetagraphOptimization; using BenchmarkTools; @btime get_operations(graph) setup=(graph = parse_abc("input/AB->ABBBBB.txt"))'
+#end
+
+for i in $(seq $minthreads $maxthreads)
+   printf "(AB->ABBBBBBB, $i) "
+   julia --project=./examples -t $i -O3 -e 'using MetagraphOptimization; using BenchmarkTools; @btime get_operations(graph) setup=(graph = parse_abc("input/AB->ABBBBBBB.txt"))'
+end

src/MetagraphOptimization.jl

@@ -1,10 +1,52 @@
+"""
+    MetagraphOptimization
+
+A module containing tools to work on DAGs.
+"""
 module MetagraphOptimization
 
-export Node, Edge, ComputeTaskNode, DataTaskNode, DAG
-export AbstractTask, AbstractComputeTask, AbstractDataTask, DataTask, FusedComputeTask
-export make_node, make_edge, insert_node, insert_edge, is_entry_node, is_exit_node, parents, children, compute, graph_properties, get_exit_node, is_valid
-export NodeFusion, NodeReduction, NodeSplit, push_operation!, pop_operation!, can_pop, reset_graph!, get_operations
-export parse_abc, ComputeTaskP, ComputeTaskS1, ComputeTaskS2, ComputeTaskV, ComputeTaskU, ComputeTaskSum
+export DAG
+export Node
+export Edge
+export ComputeTaskNode
+export DataTaskNode
+export AbstractTask
+export AbstractComputeTask
+export AbstractDataTask
+export DataTask
+export FusedComputeTask
+
+export make_node
+export make_edge
+export insert_node
+export insert_edge
+export is_entry_node
+export is_exit_node
+export parents
+export children
+export compute
+export graph_properties
+export get_exit_node
+export is_valid
+
+export Operation
+export AppliedOperation
+export NodeFusion
+export NodeReduction
+export NodeSplit
+export push_operation!
+export pop_operation!
+export can_pop
+export reset_graph!
+export get_operations
+
+export parse_abc
+export ComputeTaskP
+export ComputeTaskS1
+export ComputeTaskS2
+export ComputeTaskV
+export ComputeTaskU
+export ComputeTaskSum
 
 export ==, in, show, isempty, delete!, length
 
@@ -17,20 +59,50 @@ import Base.in
 import Base.copy
 import Base.isempty
 import Base.delete!
+import Base.insert!
+import Base.collect
 
 
-include("tasks.jl")
-include("nodes.jl")
-include("graph.jl")
+include("task/type.jl")
+include("node/type.jl")
+include("diff/type.jl")
+include("operation/type.jl")
+include("graph/type.jl")
 
-include("task_functions.jl")
-include("node_functions.jl")
-include("graph_functions.jl")
-include("graph_operations.jl")
+include("trie.jl")
 include("utility.jl")
 
-include("abc_model/tasks.jl")
-include("abc_model/task_functions.jl")
-include("abc_model/parse.jl")
+include("diff/print.jl")
+include("diff/properties.jl")
+
+include("graph/compare.jl")
+include("graph/interface.jl")
+include("graph/mute.jl")
+include("graph/print.jl")
+include("graph/properties.jl")
+include("graph/validate.jl")
+
+include("node/compare.jl")
+include("node/create.jl")
+include("node/print.jl")
+include("node/properties.jl")
+include("node/validate.jl")
+
+include("operation/utility.jl")
+include("operation/apply.jl")
+include("operation/clean.jl")
+include("operation/find.jl")
+include("operation/get.jl")
+include("operation/print.jl")
+include("operation/validate.jl")
+
+include("task/create.jl")
+include("task/compare.jl")
+include("task/print.jl")
+include("task/properties.jl")
+
+include("models/abc/types.jl")
+include("models/abc/properties.jl")
+include("models/abc/parse.jl")
 
 end # module MetagraphOptimization

src/abc_model/parse.jl

@@ -1,149 +0,0 @@
-using Printf
-
-# functions for importing DAGs from a file
-regex_a = r"^[A-C]\d+$"                     # Regex for the initial particles
-regex_c = r"^[A-C]\(([^']*),([^']*)\)$"     # Regex for the combinations of 2 particles
-regex_m = r"^M\(([^']*),([^']*),([^']*)\)$" # Regex for the combinations of 3 particles
-regex_plus = r"^\+$"                        # Regex for the sum
-
-function parse_nodes(input::AbstractString)
-    regex = r"'([^']*)'"
-    matches = eachmatch(regex, input)
-    output = [match.captures[1] for match in matches]
-    return output
-end
-
-function parse_edges(input::AbstractString)
-    regex = r"\('([^']*)', '([^']*)'\)"
-    matches = eachmatch(regex, input)
-    output = [(match.captures[1], match.captures[2]) for match in matches]
-    return output
-end
-
-# reads an abc-model process from the given file
-function parse_abc(filename::String, verbose::Bool = false)
-    file = open(filename, "r")
-
-    if (verbose) println("Opened file") end
-    nodes_string = readline(file)
-    nodes = parse_nodes(nodes_string)
-
-    close(file)
-    if (verbose) println("Read file") end
-
-    graph = DAG()
-    
-    # estimate total number of nodes
-    # try to slightly overestimate so no resizing is necessary
-    # data nodes are not included in length(nodes) and there are a few more than compute nodes
-    estimate_no_nodes = round(Int, length(nodes) * 4)
-    if (verbose) println("Estimating ", estimate_no_nodes, " Nodes") end
-    sizehint!(graph.nodes, estimate_no_nodes)
-
-    sum_node = insert_node!(graph, make_node(ComputeTaskSum()), false)
-    global_data_out = insert_node!(graph, make_node(DataTask(10)), false)
-    insert_edge!(graph, make_edge(sum_node, global_data_out), false)
-
-    # remember the data out nodes for connection
-    dataOutNodes = Dict()
-
-    if (verbose) println("Building graph") end
-    noNodes = 0
-    nodesToRead = length(nodes)
-    while !isempty(nodes)
-        node = popfirst!(nodes)
-        noNodes += 1
-        if (noNodes % 100 == 0)
-            if (verbose) @printf "\rReading Nodes... %.2f%%" (100. * noNodes / nodesToRead) end
-        end
-        if occursin(regex_a, node)
-            # add nodes and edges for the state reading to u(P(Particle))
-            data_in = insert_node!(graph, make_node(DataTask(4)), false) # read particle data node
-            compute_P = insert_node!(graph, make_node(ComputeTaskP()), false) # compute P node
-            data_Pu = insert_node!(graph, make_node(DataTask(6)), false) # transfer data from P to u
-            compute_u = insert_node!(graph, make_node(ComputeTaskU()), false) # compute U node
-            data_out = insert_node!(graph, make_node(DataTask(3)), false) # transfer data out from u
-
-            insert_edge!(graph, make_edge(data_in, compute_P), false)
-            insert_edge!(graph, make_edge(compute_P, data_Pu), false)
-            insert_edge!(graph, make_edge(data_Pu, compute_u), false)
-            insert_edge!(graph, make_edge(compute_u, data_out), false)
-            
-            # remember the data_out node for future edges
-            dataOutNodes[node] = data_out
-        elseif occursin(regex_c, node)
-            capt = match(regex_c, node)
-            
-            in1 = capt.captures[1]
-            in2 = capt.captures[2]
-
-            compute_v = insert_node!(graph, make_node(ComputeTaskV()), false)
-            data_out = insert_node!(graph, make_node(DataTask(5)), false)
-
-            if (occursin(regex_c, capt.captures[1]))
-                # put an S node after this input
-                compute_S = insert_node!(graph, make_node(ComputeTaskS1()), false)
-                data_S_v = insert_node!(graph, make_node(DataTask(5)), false)
-
-                insert_edge!(graph, make_edge(dataOutNodes[capt.captures[1]], compute_S), false)
-                insert_edge!(graph, make_edge(compute_S, data_S_v), false)
-
-                insert_edge!(graph, make_edge(data_S_v, compute_v), false)
-            else
-                insert_edge!(graph, make_edge(dataOutNodes[capt.captures[1]], compute_v), false)
-            end
-
-            if (occursin(regex_c, capt.captures[2]))
-                # i think the current generator only puts the combined particles in the first space, so this case might never be entered
-                # put an S node after this input
-                compute_S = insert_node!(graph, make_node(ComputeTaskS1()), false)
-                data_S_v = insert_node!(graph, make_node(DataTask(5)), false)
-
-                insert_edge!(graph, make_edge(dataOutNodes[capt.captures[2]], compute_S), false)
-                insert_edge!(graph, make_edge(compute_S, data_S_v), false)
-
-                insert_edge!(graph, make_edge(data_S_v, compute_v), false)
-            else
-                insert_edge!(graph, make_edge(dataOutNodes[capt.captures[2]], compute_v), false)
-            end
- 
-            insert_edge!(graph, make_edge(compute_v, data_out), false)
-            dataOutNodes[node] = data_out
-
-        elseif occursin(regex_m, node)
-            # assume for now that only the first particle of the three is combined and the other two are "original" ones
-            capt = match(regex_m, node)
-            in1 = capt.captures[1]
-            in2 = capt.captures[2]
-            in3 = capt.captures[3]
-
-            # in2 + in3 with a v
-            compute_v = insert_node!(graph, make_node(ComputeTaskV()), false)
-            data_v = insert_node!(graph, make_node(DataTask(5)), false)
-
-            insert_edge!(graph, make_edge(dataOutNodes[in2], compute_v), false)
-            insert_edge!(graph, make_edge(dataOutNodes[in3], compute_v), false)
-            insert_edge!(graph, make_edge(compute_v, data_v), false)
-
-            # combine with the v of the combined other input
-            compute_S2 = insert_node!(graph, make_node(ComputeTaskS2()), false)
-            data_out = insert_node!(graph, make_node(DataTask(10)), false)
-
-            insert_edge!(graph, make_edge(data_v, compute_S2), false)
-            insert_edge!(graph, make_edge(dataOutNodes[in1], compute_S2), false)
-            insert_edge!(graph, make_edge(compute_S2, data_out), false)
-
-            insert_edge!(graph, make_edge(data_out, sum_node), false)
-        elseif occursin(regex_plus, node)
-            if (verbose)
-                println("\rReading Nodes Complete    ") 
-                println("Added ", length(graph.nodes), " nodes")
-            end
-        else
-            error("Unknown node '", node, "' while reading from file ", filename)
-        end
-    end
-
-    # don't actually need to read the edges
-    return graph
-end

src/abc_model/task_functions.jl

@@ -1,21 +0,0 @@
-# define compute_efforts tasks computation
-# put some "random" numbers here for now
-compute_effort(t::ComputeTaskS1) = 10
-compute_effort(t::ComputeTaskS2) = 10
-compute_effort(t::ComputeTaskU) = 6
-compute_effort(t::ComputeTaskV) = 20
-compute_effort(t::ComputeTaskP) = 15
-compute_effort(t::ComputeTaskSum) = 1
-
-function show(io::IO, t::DataTask)
-    print(io, "Data", t.data)
-end
-
-show(io::IO, t::ComputeTaskS1) = print("ComputeS1")
-show(io::IO, t::ComputeTaskS2) = print("ComputeS2")
-show(io::IO, t::ComputeTaskP) = print("ComputeP")
-show(io::IO, t::ComputeTaskU) = print("ComputeU")
-show(io::IO, t::ComputeTaskV) = print("ComputeV")
-show(io::IO, t::ComputeTaskSum) = print("ComputeSum")
-
-copy(t::DataTask) = DataTask(t.data)

src/abc_model/tasks.jl

@@ -1,27 +0,0 @@
-struct DataTask <: AbstractDataTask
-    data::UInt64
- end
- 
- # S task with 1 child
- struct ComputeTaskS1 <: AbstractComputeTask
- end
- 
- # S task with 2 children
- struct ComputeTaskS2 <: AbstractComputeTask
- end
- 
- # P task with 0 children
- struct ComputeTaskP <: AbstractComputeTask
- end
- 
- # v task with 2 children
- struct ComputeTaskV <: AbstractComputeTask
- end
- 
- # u task with 1 child
- struct ComputeTaskU <: AbstractComputeTask
- end
- 
- # task that sums all its inputs, n children
- struct ComputeTaskSum <: AbstractComputeTask
- end

src/diff/print.jl

@@ -0,0 +1,11 @@
+"""
+    show(io::IO, diff::Diff)
+
+Pretty-print a [`Diff`](@ref). Called via print, println and co.
+"""
+function show(io::IO, diff::Diff)
+    print(io, "Nodes: ")
+    print(io, length(diff.addedNodes) + length(diff.removedNodes))
+    print(io, " Edges: ")
+    return print(io, length(diff.addedEdges) + length(diff.removedEdges))
+end

src/diff/properties.jl

@@ -0,0 +1,14 @@
+"""
+    length(diff::Diff)
+    
+Return a named tuple of the lengths of the added/removed nodes/edges.
+The fields are `.addedNodes`, `.addedEdges`, `.removedNodes` and `.removedEdges`.
+"""
+function length(diff::Diff)
+    return (
+        addedNodes = length(diff.addedNodes),
+        removedNodes = length(diff.removedNodes),
+        addedEdges = length(diff.addedEdges),
+        removedEdges = length(diff.removedEdges),
+    )
+end

src/diff/type.jl

@@ -0,0 +1,18 @@
+"""
+    Diff
+
+A named tuple representing a difference of added and removed nodes and edges on a [`DAG`](@ref).
+"""
+const Diff = NamedTuple{
+    (:addedNodes, :removedNodes, :addedEdges, :removedEdges),
+    Tuple{Vector{Node}, Vector{Node}, Vector{Edge}, Vector{Edge}},
+}
+
+function Diff()
+    return (
+        addedNodes = Vector{Node}(),
+        removedNodes = Vector{Node}(),
+        addedEdges = Vector{Edge}(),
+        removedEdges = Vector{Edge}(),
+    )::Diff
+end

src/graph.jl

@@ -1,92 +0,0 @@
-using DataStructures
-
-const Diff = NamedTuple{
-   (:addedNodes, :removedNodes, :addedEdges, :removedEdges),
-   Tuple{Vector{Node}, Vector{Node}, Vector{Edge}, Vector{Edge}}
-}
-
-function Diff()
-   return (
-      addedNodes = Vector{Node}(), 
-      removedNodes = Vector{Node}(), 
-      addedEdges = Vector{Edge}(), 
-      removedEdges = Vector{Edge}()
-   )::Diff
-end
-
-# An abstract base class for operations
-# an operation can be applied to a DAG
-abstract type Operation end
-
-# An abstract base class for already applied operations
-# an applied operation can be reversed iff it is the last applied operation on the DAG
-abstract type AppliedOperation end
-
-struct NodeFusion <: Operation
-   input::Tuple{ComputeTaskNode, DataTaskNode, ComputeTaskNode}
-end
-
-struct AppliedNodeFusion <: AppliedOperation
-   operation::NodeFusion
-   diff::Diff
-end
-
-struct NodeReduction <: Operation
-   input::Vector{Node}
-end
-
-struct AppliedNodeReduction <: AppliedOperation
-   operation::NodeReduction
-   diff::Diff
-end
-
-struct NodeSplit <: Operation
-   input::Node
-end
-
-struct AppliedNodeSplit <: AppliedOperation
-   operation::NodeSplit
-   diff::Diff
-end
-
-
-mutable struct PossibleOperations
-   nodeFusions::Set{NodeFusion}
-   nodeReductions::Set{NodeReduction}
-   nodeSplits::Set{NodeSplit}
-end
-
-function PossibleOperations()
-   return PossibleOperations(
-      Set{NodeFusion}(),
-      Set{NodeReduction}(),
-      Set{NodeSplit}()
-   )
-end
-
-
-# The actual state of the DAG is the initial state given by the set of nodes 
-# but with all the operations in appliedChain applied in order
-mutable struct DAG
-   nodes::Set{Node}
-
-   # The operations currently applied to the set of nodes
-   appliedOperations::Stack{AppliedOperation}
-
-   # The operations not currently applied but part of the current state of the DAG
-   operationsToApply::Deque{Operation}
-
-   # The possible operations at the current state of the DAG
-   possibleOperations::PossibleOperations
-
-   # The set of nodes whose possible operations need to be reevaluated
-   dirtyNodes::Set{Node}
-
-   # "snapshot" system: keep track of added/removed nodes/edges since last snapshot
-   # these are muted in insert_node! etc.
-   diff::Diff
-end
-
-function DAG()
-   return DAG(Set{Node}(), Stack{AppliedOperation}(), Deque{Operation}(), PossibleOperations(), Set{Node}(), Diff())
-end

src/graph/compare.jl

@@ -0,0 +1,37 @@
+"""
+    in(node::Node, graph::DAG)
+
+Check whether the node is part of the graph.
+"""
+in(node::Node, graph::DAG) = node in graph.nodes
+
+"""
+    in(edge::Edge, graph::DAG)
+
+Check whether the edge is part of the graph.
+"""
+function in(edge::Edge, graph::DAG)
+    n1 = edge.edge[1]
+    n2 = edge.edge[2]
+    if !(n1 in graph) || !(n2 in graph)
+        return false
+    end
+
+    return n1 in n2.children
+end
+
+"""
+    ==(n1::Node, n2::Node, g::DAG)
+
+Check equality of two nodes in a graph.
+"""
+function ==(n1::Node, n2::Node, g::DAG)
+    if typeof(n1) != typeof(n2)
+        return false
+    end
+    if !(n1 in g) || !(n2 in g)
+        return false
+    end
+
+    return n1.task == n2.task && children(n1) == children(n2)
+end

src/graph/interface.jl

@@ -0,0 +1,55 @@
+"""
+    push_operation!(graph::DAG, operation::Operation)
+
+Apply a new operation to the graph.
+
+See also: [`DAG`](@ref), [`pop_operation!`](@ref)
+"""
+function push_operation!(graph::DAG, operation::Operation)
+    # 1.: Add the operation to the DAG
+    push!(graph.operationsToApply, operation)
+
+    return nothing
+end
+
+"""
+    pop_operation!(graph::DAG)
+
+Revert the latest applied operation on the graph.
+
+See also: [`DAG`](@ref), [`push_operation!`](@ref)
+"""
+function pop_operation!(graph::DAG)
+    # 1.: Remove the operation from the appliedChain of the DAG
+    if !isempty(graph.operationsToApply)
+        pop!(graph.operationsToApply)
+    elseif !isempty(graph.appliedOperations)
+        appliedOp = pop!(graph.appliedOperations)
+        revert_operation!(graph, appliedOp)
+    else
+        error("No more operations to pop!")
+    end
+
+    return nothing
+end
+
+"""
+    can_pop(graph::DAG)
+
+Return `true` if [`pop_operation!`](@ref) is possible, `false` otherwise.
+"""
+can_pop(graph::DAG) =
+    !isempty(graph.operationsToApply) || !isempty(graph.appliedOperations)
+
+"""
+    reset_graph!(graph::DAG)
+
+Reset the graph to its initial state with no operations applied.
+"""
+function reset_graph!(graph::DAG)
+    while (can_pop(graph))
+        pop_operation!(graph)
+    end
+
+    return nothing
+end

src/graph/mute.jl

@@ -0,0 +1,280 @@
+# for graph mutating functions we need to do a few things
+# 1: mute the graph (duh)
+# 2: keep track of what was changed for the diff (if track == true)
+# 3: invalidate operation caches
+
+"""
+    insert_node!(graph::DAG, node::Node; track = true, invalidate_cache = true)
+
+Insert the node into the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`remove_node!`](@ref), [`insert_edge!`](@ref), [`remove_edge!`](@ref)
+"""
+function insert_node!(
+    graph::DAG,
+    node::Node,
+    track = true,
+    invalidate_cache = true,
+)
+    # 1: mute
+    push!(graph.nodes, node)
+
+    # 2: keep track
+    if (track)
+        push!(graph.diff.addedNodes, node)
+    end
+
+    # 3: invalidate caches
+    if (!invalidate_cache)
+        return node
+    end
+    push!(graph.dirtyNodes, node)
+
+    return node
+end
+
+"""
+    insert_edge!(graph::DAG, node1::Node, node2::Node; track = true, invalidate_cache = true)
+
+Insert the edge between node1 (child) and node2 (parent) into the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`insert_node!`](@ref), [`remove_node!`](@ref), [`remove_edge!`](@ref)
+"""
+function insert_edge!(
+    graph::DAG,
+    node1::Node,
+    node2::Node,
+    track = true,
+    invalidate_cache = true,
+)
+    # @assert (node2 ∉ node1.parents) && (node1 ∉ node2.children) "Edge to insert already exists"
+
+    # 1: mute
+    # edge points from child to parent
+    push!(node1.parents, node2)
+    push!(node2.children, node1)
+
+    # 2: keep track
+    if (track)
+        push!(graph.diff.addedEdges, make_edge(node1, node2))
+    end
+
+    # 3: invalidate caches
+    if (!invalidate_cache)
+        return nothing
+    end
+
+    invalidate_operation_caches!(graph, node1)
+    invalidate_operation_caches!(graph, node2)
+
+    push!(graph.dirtyNodes, node1)
+    push!(graph.dirtyNodes, node2)
+
+    return nothing
+end
+
+"""
+    remove_node!(graph::DAG, node::Node; track = true, invalidate_cache = true)
+
+Remove the node from the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`insert_node!`](@ref), [`insert_edge!`](@ref), [`remove_edge!`](@ref)
+"""
+function remove_node!(
+    graph::DAG,
+    node::Node,
+    track = true,
+    invalidate_cache = true,
+)
+    # @assert node in graph.nodes "Trying to remove a node that's not in the graph"
+
+    # 1: mute
+    delete!(graph.nodes, node)
+
+    # 2: keep track
+    if (track)
+        push!(graph.diff.removedNodes, node)
+    end
+
+    # 3: invalidate caches
+    if (!invalidate_cache)
+        return nothing
+    end
+
+    invalidate_operation_caches!(graph, node)
+    delete!(graph.dirtyNodes, node)
+
+    return nothing
+end
+
+"""
+    remove_edge!(graph::DAG, node1::Node, node2::Node; track = true, invalidate_cache = true)
+
+Remove the edge between node1 (child) and node2 (parent) into the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`insert_node!`](@ref), [`remove_node!`](@ref), [`insert_edge!`](@ref)
+"""
+function remove_edge!(
+    graph::DAG,
+    node1::Node,
+    node2::Node,
+    track = true,
+    invalidate_cache = true,
+)
+    # 1: mute
+    pre_length1 = length(node1.parents)
+    pre_length2 = length(node2.children)
+    filter!(x -> x != node2, node1.parents)
+    filter!(x -> x != node1, node2.children)
+
+    #=@assert begin
+       removed = pre_length1 - length(node1.parents)
+       removed <= 1
+    end "removed more than one node from node1's parents"=#
+
+    #=@assert begin
+       removed = pre_length2 - length(node2.children)
+       removed <= 1
+    end "removed more than one node from node2's children"=#
+
+    # 2: keep track
+    if (track)
+        push!(graph.diff.removedEdges, make_edge(node1, node2))
+    end
+
+    # 3: invalidate caches
+    if (!invalidate_cache)
+        return nothing
+    end
+
+    invalidate_operation_caches!(graph, node1)
+    invalidate_operation_caches!(graph, node2)
+    if (node1 in graph)
+        push!(graph.dirtyNodes, node1)
+    end
+    if (node2 in graph)
+        push!(graph.dirtyNodes, node2)
+    end
+
+    return nothing
+end
+
+"""
+    get_snapshot_diff(graph::DAG)
+
+Return the graph's [`Diff`](@ref) since last time this function was called.
+
+See also: [`revert_diff!`](@ref), [`AppliedOperation`](@ref) and [`revert_operation!`](@ref)
+"""
+function get_snapshot_diff(graph::DAG)
+    return swapfield!(graph, :diff, Diff())
+end
+
+"""
+    invalidate_caches!(graph::DAG, operation::NodeFusion)
+
+Invalidate the operation caches for a given [`NodeFusion`](@ref).
+
+This deletes the operation from the graph's possible operations and from the involved nodes' own operation caches.
+"""
+function invalidate_caches!(graph::DAG, operation::NodeFusion)
+    delete!(graph.possibleOperations, operation)
+
+    # delete the operation from all caches of nodes involved in the operation
+    filter!(!=(operation), operation.input[1].nodeFusions)
+    filter!(!=(operation), operation.input[3].nodeFusions)
+
+    operation.input[2].nodeFusion = missing
+
+    return nothing
+end
+
+"""
+    invalidate_caches!(graph::DAG, operation::NodeReduction)
+
+Invalidate the operation caches for a given [`NodeReduction`](@ref).
+
+This deletes the operation from the graph's possible operations and from the involved nodes' own operation caches.
+"""
+function invalidate_caches!(graph::DAG, operation::NodeReduction)
+    delete!(graph.possibleOperations, operation)
+
+    for node in operation.input
+        node.nodeReduction = missing
+    end
+
+    return nothing
+end
+
+"""
+    invalidate_caches!(graph::DAG, operation::NodeSplit)
+
+Invalidate the operation caches for a given [`NodeSplit`](@ref).
+
+This deletes the operation from the graph's possible operations and from the involved nodes' own operation caches.
+"""
+function invalidate_caches!(graph::DAG, operation::NodeSplit)
+    delete!(graph.possibleOperations, operation)
+
+    # delete the operation from all caches of nodes involved in the operation
+    # for node split there is only one node
+    operation.input.nodeSplit = missing
+
+    return nothing
+end
+
+"""
+    invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode)
+
+Invalidate the operation caches of the given node through calls to the respective [`invalidate_caches!`](@ref) functions.
+"""
+function invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode)
+    if !ismissing(node.nodeReduction)
+        invalidate_caches!(graph, node.nodeReduction)
+    end
+    if !ismissing(node.nodeSplit)
+        invalidate_caches!(graph, node.nodeSplit)
+    end
+    while !isempty(node.nodeFusions)
+        invalidate_caches!(graph, pop!(node.nodeFusions))
+    end
+    return nothing
+end
+
+"""
+    invalidate_operation_caches!(graph::DAG, node::DataTaskNode)
+
+Invalidate the operation caches of the given node through calls to the respective [`invalidate_caches!`](@ref) functions.
+"""
+function invalidate_operation_caches!(graph::DAG, node::DataTaskNode)
+    if !ismissing(node.nodeReduction)
+        invalidate_caches!(graph, node.nodeReduction)
+    end
+    if !ismissing(node.nodeSplit)
+        invalidate_caches!(graph, node.nodeSplit)
+    end
+    if !ismissing(node.nodeFusion)
+        invalidate_caches!(graph, node.nodeFusion)
+    end
+    return nothing
+end

src/graph/print.jl

@@ -0,0 +1,69 @@
+"""
+    show_nodes(io::IO, graph::DAG)
+
+Print a graph's nodes. Should only be used for small graphs as it prints every node in a list.
+"""
+function show_nodes(io::IO, graph::DAG)
+    print(io, "[")
+    first = true
+    for n in graph.nodes
+        if first
+            first = false
+        else
+            print(io, ", ")
+        end
+        print(io, n)
+    end
+    return print(io, "]")
+end
+
+"""
+    show(io::IO, graph::DAG)
+
+Print the given graph to io. If there are too many nodes it will print only a summary of them.
+"""
+function show(io::IO, graph::DAG)
+    println(io, "Graph:")
+    print(io, "  Nodes: ")
+
+    nodeDict = Dict{Type, Int64}()
+    noEdges = 0
+    for node in graph.nodes
+        if haskey(nodeDict, typeof(node.task))
+            nodeDict[typeof(node.task)] = nodeDict[typeof(node.task)] + 1
+        else
+            nodeDict[typeof(node.task)] = 1
+        end
+        noEdges += length(parents(node))
+    end
+
+    if length(graph.nodes) <= 20
+        show_nodes(io, graph)
+    else
+        print("Total: ", length(graph.nodes), ", ")
+        first = true
+        i = 0
+        for (type, number) in zip(keys(nodeDict), values(nodeDict))
+            i += 1
+            if first
+                first = false
+            else
+                print(", ")
+            end
+            if (i % 3 == 0)
+                print("\n         ")
+            end
+            print(type, ": ", number)
+        end
+    end
+    println(io)
+    println(io, "  Edges: ", noEdges)
+    properties = graph_properties(graph)
+    println(io, "  Total Compute Effort: ", properties.compute_effort)
+    println(io, "  Total Data Transfer: ", properties.data)
+    return println(
+        io,
+        "  Total Compute Intensity: ",
+        properties.compute_intensity,
+    )
+end

src/graph/properties.jl

@@ -0,0 +1,43 @@
+"""
+    graph_properties(graph::DAG)
+
+Return the graph's properties, a named tuple with fields `.data`, `.compute_effort`, `.compute_intensity`, `.nodes` (number of nodes) and `.edges` (number of edges).
+"""
+function graph_properties(graph::DAG)
+    # make sure the graph is fully generated
+    apply_all!(graph)
+
+    d = 0
+    ce = 0
+    ed = 0
+    for node in graph.nodes
+        d += data(node.task) * length(node.parents)
+        ce += compute_effort(node.task)
+        ed += length(node.parents)
+    end
+
+    ci = ce / d
+
+    result = (
+        data = d,
+        compute_effort = ce,
+        compute_intensity = ci,
+        nodes = length(graph.nodes),
+        edges = ed,
+    )
+    return result
+end
+
+"""
+    get_exit_node(graph::DAG)
+
+Return the graph's exit node. This assumes the graph only has a single exit node. If the graph has multiple exit nodes, the one encountered first will be returned.
+"""
+function get_exit_node(graph::DAG)
+    for node in graph.nodes
+        if (is_exit_node(node))
+            return node
+        end
+    end
+    @assert false "The given graph has no exit node! It is either empty or not acyclic!"
+end

src/graph/type.jl

@@ -0,0 +1,73 @@
+using DataStructures
+
+"""
+    PossibleOperations
+
+A struct storing all possible operations on a [`DAG`](@ref).
+To get the [`PossibleOperations`](@ref) on a [`DAG`](@ref), use [`get_operations`](@ref).
+"""
+mutable struct PossibleOperations
+    nodeFusions::Set{NodeFusion}
+    nodeReductions::Set{NodeReduction}
+    nodeSplits::Set{NodeSplit}
+end
+
+""" 
+    DAG
+
+The representation of the graph as a set of [`Node`](@ref)s.
+
+A DAG can be loaded using the appropriate parse function, e.g. [`parse_abc`](@ref).
+
+[`Operation`](@ref)s can be applied on it using [`push_operation!`](@ref) and reverted using [`pop_operation!`](@ref) like a stack.
+To get the set of possible operations, use [`get_operations`](@ref).
+The members of the object should not be manually accessed, instead always use the provided interface functions.
+"""
+mutable struct DAG
+    nodes::Set{Node}
+
+    # The operations currently applied to the set of nodes
+    appliedOperations::Stack{AppliedOperation}
+
+    # The operations not currently applied but part of the current state of the DAG
+    operationsToApply::Deque{Operation}
+
+    # The possible operations at the current state of the DAG
+    possibleOperations::PossibleOperations
+
+    # The set of nodes whose possible operations need to be reevaluated
+    dirtyNodes::Set{Node}
+
+    # "snapshot" system: keep track of added/removed nodes/edges since last snapshot
+    # these are muted in insert_node! etc.
+    diff::Diff
+end
+
+"""
+    PossibleOperations()
+
+Construct and return an empty [`PossibleOperations`](@ref) object.
+"""
+function PossibleOperations()
+    return PossibleOperations(
+        Set{NodeFusion}(),
+        Set{NodeReduction}(),
+        Set{NodeSplit}(),
+    )
+end
+
+"""
+    DAG()
+
+Construct and return an empty [`DAG`](@ref).
+"""
+function DAG()
+    return DAG(
+        Set{Node}(),
+        Stack{AppliedOperation}(),
+        Deque{Operation}(),
+        PossibleOperations(),
+        Set{Node}(),
+        Diff(),
+    )
+end

src/graph/validate.jl

@@ -0,0 +1,61 @@
+"""
+    is_connected(graph::DAG)
+
+Return whether the given graph is connected.
+"""
+function is_connected(graph::DAG)
+    nodeQueue = Deque{Node}()
+    push!(nodeQueue, get_exit_node(graph))
+    seenNodes = Set{Node}()
+
+    while !isempty(nodeQueue)
+        current = pop!(nodeQueue)
+        push!(seenNodes, current)
+
+        for child in current.children
+            push!(nodeQueue, child)
+        end
+    end
+
+    return length(seenNodes) == length(graph.nodes)
+end
+
+"""
+    is_valid(graph::DAG)
+
+Validate the entire graph using asserts. Intended for testing with `@assert is_valid(graph)`.
+"""
+function is_valid(graph::DAG)
+    for node in graph.nodes
+        @assert is_valid(graph, node)
+    end
+
+    for op in graph.operationsToApply
+        @assert is_valid(graph, op)
+    end
+
+    for nr in graph.possibleOperations.nodeReductions
+        @assert is_valid(graph, nr)
+    end
+    for ns in graph.possibleOperations.nodeSplits
+        @assert is_valid(graph, ns)
+    end
+    for nf in graph.possibleOperations.nodeFusions
+        @assert is_valid(graph, nf)
+    end
+
+    for node in graph.dirtyNodes
+        @assert node in graph "Dirty Node is not part of the graph!"
+        @assert ismissing(node.nodeReduction) "Dirty Node has a NodeReduction!"
+        @assert ismissing(node.nodeSplit) "Dirty Node has a NodeSplit!"
+        if (typeof(node) <: DataTaskNode)
+            @assert ismissing(node.nodeFusion) "Dirty DataTaskNode has a Node Fusion!"
+        elseif (typeof(node) <: ComputeTaskNode)
+            @assert isempty(node.nodeFusions) "Dirty ComputeTaskNode has Node Fusions!"
+        end
+    end
+
+    @assert is_connected(graph) "Graph is not connected!"
+
+    return true
+end

src/graph_functions.jl

@@ -1,374 +0,0 @@
-using DataStructures
-
-in(node::Node, graph::DAG) = node in graph.nodes
-in(edge::Edge, graph::DAG) = edge in graph.edges
-
-function isempty(operations::PossibleOperations)
-   return isempty(operations.nodeFusions) && 
-          isempty(operations.nodeReductions) && 
-          isempty(operations.nodeSplits)
-end
-
-function length(operations::PossibleOperations)
-   return (nodeFusions = length(operations.nodeFusions),
-           nodeReductions = length(operations.nodeReductions),
-           nodeSplits = length(operations.nodeSplits))
-end
-
-function delete!(operations::PossibleOperations, op::NodeFusion)
-   delete!(operations.nodeFusions, op)
-   return operations
-end
-function delete!(operations::PossibleOperations, op::NodeReduction)
-   delete!(operations.nodeReductions, op)
-   return operations
-end
-function delete!(operations::PossibleOperations, op::NodeSplit)
-   delete!(operations.nodeSplits, op)
-   return operations
-end
-
-function is_parent(potential_parent, node)
-   return potential_parent in node.parents
-end
-
-function is_child(potential_child, node)
-   return potential_child in node.children
-end
-
-function ==(n1::Node, n2::Node, g::DAG)
-   if typeof(n1) != typeof(n2)
-      return false
-   end
-   if !(n1 in g) || !(n2 in g)
-      return false
-   end
-
-   return n1.task == n2.task && children(n1) == children(n2)
-end
-
-# children = prerequisite nodes, nodes that need to execute before the task, edges point into this task
-function children(node::Node)
-   return copy(node.children)
-end
-
-# parents = subsequent nodes, nodes that need this node to execute, edges point from this task
-function parents(node::Node)
-   return copy(node.parents)
-end
-
-# siblings = all children of any parents, no duplicates, does not include the node itself
-function siblings(node::Node)
-   result = Set{Node}()
-   for parent in node.parents
-      for sibling in parent.children
-         if (sibling != node)
-            push!(result, sibling)
-         end
-      end
-   end
-
-   return result
-end
-
-# partners = all parents of any children, no duplicates, does not include the node itself
-function partners(node::Node)
-   result = Set{Node}()
-   for child in node.children
-      for partner in child.parents
-         if (partner != node)
-            push!(result, partner)
-         end
-      end
-   end
-
-   return result
-end
-
-is_entry_node(node::Node) = length(node.children) == 0
-is_exit_node(node::Node) = length(node.parents) == 0
-
-# function to invalidate the operation caches for a given operation
-function invalidate_caches!(graph::DAG, operation::Operation)
-   delete!(graph.possibleOperations, operation)
-
-   # delete the operation from all caches of nodes involved in the operation
-   # (we can iterate over tuples and vectors just fine)
-   for node in operation.input
-      filter!(!=(operation), node.operations)
-   end
-
-   return nothing
-end
-
-# function to invalidate the operation caches for a given Node Split specifically
-function invalidate_caches!(graph::DAG, operation::NodeSplit)
-   delete!(graph.possibleOperations, operation)
-
-   # delete the operation from all caches of nodes involved in the operation
-   # for node split there is only one node
-   filter!(x -> x != operation, operation.input.operations)
-
-   return nothing
-end
-
-# for graph mutating functions we need to do a few things
-# 1: mute the graph (duh)
-# 2: keep track of what was changed for the diff (if track == true)
-# 3: invalidate operation caches
-
-function insert_node!(graph::DAG, node::Node, track=true)
-   # 1: mute
-   push!(graph.nodes, node)
-
-   # 2: keep track
-   if (track) push!(graph.diff.addedNodes, node) end
-
-   # 3: invalidate caches
-   push!(graph.dirtyNodes, node)
-
-   return node
-end
-
-function insert_edge!(graph::DAG, edge::Edge, track=true)
-   node1 = edge.edge[1]
-   node2 = edge.edge[2]
-
-   # 1: mute
-   #=if (node2 in node1.parents) || (node1 in node2.children)
-      if !(node2 in node1.parents && node1 in node2.children)
-         error("One-sided edge")
-      end
-      error("Edge to insert already exists")
-   end=#
-
-   # edge points from child to parent
-   push!(node1.parents, node2)
-   push!(node2.children, node1)
-
-   # 2: keep track
-   if (track) push!(graph.diff.addedEdges, edge) end
-   
-   # 3: invalidate caches
-   while !isempty(node1.operations)
-      invalidate_caches!(graph, first(node1.operations))
-   end
-   while !isempty(node2.operations)
-      invalidate_caches!(graph, first(node2.operations))
-   end
-   push!(graph.dirtyNodes, node1)
-   push!(graph.dirtyNodes, node2)
-
-   return edge
-end
-
-function remove_node!(graph::DAG, node::Node, track=true)
-   # 1: mute
-   #=if !(node in graph.nodes)
-      error("Trying to remove a node that's not in the graph")
-   end=#
-   delete!(graph.nodes, node)
-
-   # 2: keep track
-   if (track) push!(graph.diff.removedNodes, node) end
-
-   # 3: invalidate caches
-   while !isempty(node.operations)
-      invalidate_caches!(graph, first(node.operations))
-   end
-   delete!(graph.dirtyNodes, node)
-
-   return nothing
-end
-
-function remove_edge!(graph::DAG, edge::Edge, track=true)
-   node1 = edge.edge[1]
-   node2 = edge.edge[2]
-
-   # 1: mute
-   pre_length1 = length(node1.parents)
-   pre_length2 = length(node2.children)
-   filter!(x -> x != node2, node1.parents)
-   filter!(x -> x != node1, node2.children)
-
-   #=removed = pre_length1 - length(node1.parents)
-   if (removed > 1)
-      error("removed $removed from node1's parents")
-   end
-
-   removed = pre_length2 - length(node2.children)
-   if (removed > 1)
-      error("removed $removed from node2's children")
-   end=#
-
-   # 2: keep track
-   if (track) push!(graph.diff.removedEdges, edge) end
-
-   # 3: invalidate caches
-   while !isempty(node1.operations)
-      invalidate_caches!(graph, first(node1.operations))
-   end
-   while !isempty(node2.operations)
-      invalidate_caches!(graph, first(node2.operations))
-   end
-   if (node1 in graph)
-      push!(graph.dirtyNodes, node1)
-   end
-   if (node2 in graph) 
-      push!(graph.dirtyNodes, node2)
-   end
-
-   return nothing
-end
-
-# return the graph "difference" since last time this function was called
-function get_snapshot_diff(graph::DAG)
-   return swapfield!(graph, :diff, Diff())
-end
-
-function graph_properties(graph::DAG)
-   # make sure the graph is fully generated
-   apply_all!(graph)
-
-   d = 0
-   ce = 0
-   ed = 0
-   for node in graph.nodes
-      d += data(node.task) * length(node.parents)
-      ce += compute_effort(node.task)
-      ed += length(node.parents)
-   end
-
-   ci = ce / d
-
-   result = (data = d,
-             compute_effort = ce,
-             compute_intensity = ci,
-             nodes = length(graph.nodes),
-             edges = ed)
-   return result
-end
-
-function get_exit_node(graph::DAG)
-   for node in graph.nodes
-      if (is_exit_node(node))
-         return node
-      end
-   end
-   error("The given graph has no exit node! It is either empty or not acyclic!")
-end
-
-function can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
-   if !is_child(n1, n2) || !is_child(n2, n3)
-      # the checks are redundant but maybe a good sanity check
-      return false
-   end
-
-   if length(n2.parents) != 1 || length(n2.children) != 1 || length(n1.parents) != 1
-      return false
-   end
-
-   return true
-end
-
-function can_reduce(n1::Node, n2::Node)
-   if (n1.task != n2.task)
-      return false
-   end
-   return Set(n1.children) == Set(n2.children)
-end
-
-function can_split(n::Node)
-   return length(parents(n)) > 1
-end
-
-# check whether the given graph is connected
-function is_valid(graph::DAG)
-   nodeQueue = Deque{Node}()
-   push!(nodeQueue, get_exit_node(graph))
-   seenNodes = Set{Node}()
-
-   while !isempty(nodeQueue)
-      current = pop!(nodeQueue)
-      push!(seenNodes, current)
-
-      for child in current.chlidren
-         push!(nodeQueue, child)
-      end
-   end
-
-   return length(seenNodes) == length(graph.nodes)
-end
-
-function show_nodes(io, graph::DAG)
-   print(io, "[")
-   first = true
-   for n in graph.nodes
-      if first
-         first = false
-      else
-         print(io, ", ")
-      end
-      print(io, n)
-   end
-   print(io, "]")
-end
-
-function show(io::IO, graph::DAG)
-   println(io, "Graph:")
-   print(io, "  Nodes: ")
-
-   nodeDict = Dict{Type, Int64}()
-   noEdges = 0
-   for node in graph.nodes
-      if haskey(nodeDict, typeof(node.task))
-         nodeDict[typeof(node.task)] = nodeDict[typeof(node.task)] + 1
-      else
-         nodeDict[typeof(node.task)] = 1
-      end
-      noEdges += length(parents(node))
-   end
-
-   if length(graph.nodes) <= 20
-      show_nodes(io, graph)
-   else
-      print("Total: ", length(graph.nodes), ", ")
-      first = true
-      i = 0
-      for (type, number) in zip(keys(nodeDict), values(nodeDict))
-         i += 1
-         if first
-            first = false
-         else
-            print(", ")
-         end
-         if (i % 3 == 0)
-            print("\n         ")
-         end
-         print(type, ": ", number)
-      end
-   end
-   println(io)
-   println(io, "  Edges: ", noEdges)
-   properties = graph_properties(graph)
-   println(io, "  Total Compute Effort: ", properties.compute_effort)
-   println(io, "  Total Data Transfer: ", properties.data)
-   println(io, "  Total Compute Intensity: ", properties.compute_intensity)
-end
-
-function show(io::IO, diff::Diff)
-   print(io, "Nodes: ")
-   print(io, length(diff.addedNodes) + length(diff.removedNodes))
-   print(io, " Edges: ")
-   print(io, length(diff.addedEdges) + length(diff.removedEdges))
-end
-
-# return a namedtuple of the lengths of the added/removed nodes/edges
-function length(diff::Diff)
-   return (
-      addedNodes = length(diff.addedNodes), 
-      removedNodes = length(diff.removedNodes), 
-      addedEdges = length(diff.addedEdges), 
-      removedEdges = length(diff.removedEdges)
-   )
-end

src/graph_operations.jl

@@ -1,485 +0,0 @@
-# outside interface
-
-# applies a new operation to the end of the graph
-function push_operation!(graph::DAG, operation::Operation)
-   # 1.: Add the operation to the DAG
-   push!(graph.operationsToApply, operation)
-
-   return nothing
-end
-
-# reverts the latest applied operation, essentially like a ctrl+z for
-function pop_operation!(graph::DAG)
-   # 1.: Remove the operation from the appliedChain of the DAG
-   if !isempty(graph.operationsToApply)
-      pop!(graph.operationsToApply)
-   elseif !isempty(graph.appliedOperations)
-      appliedOp = pop!(graph.appliedOperations)
-      revert_operation!(graph, appliedOp)
-   else
-      error("No more operations to pop!")
-   end
-   return nothing
-end
-
-can_pop(graph::DAG) = !isempty(graph.operationsToApply) || !isempty(graph.appliedOperations)
-
-# reset the graph to its initial state with no operations applied
-function reset_graph!(graph::DAG)
-   while (can_pop(graph))
-      pop_operation!(graph)
-   end
-
-   return nothing
-end
-
-# implementation detail functions, don't export
-
-# applies all unapplied operations in the DAG
-function apply_all!(graph::DAG)
-   while !isempty(graph.operationsToApply)
-      # get next operation to apply from front of the deque
-      op = popfirst!(graph.operationsToApply)
-
-      # apply it
-      appliedOp = apply_operation!(graph, op)
-
-      # push to the end of the appliedOperations deque
-      push!(graph.appliedOperations, appliedOp)
-   end
-   return nothing
-end
-
-
-function apply_operation!(graph::DAG, operation::Operation)
-   error("Unknown operation type!")
-end
-
-function apply_operation!(graph::DAG, operation::NodeFusion)
-   diff = node_fusion!(graph, operation.input[1], operation.input[2], operation.input[3])
-   return AppliedNodeFusion(operation, diff)
-end
-
-function apply_operation!(graph::DAG, operation::NodeReduction)
-   diff = node_reduction!(graph, operation.input[1], operation.input[2])
-   return AppliedNodeReduction(operation, diff)
-end
-
-function apply_operation!(graph::DAG, operation::NodeSplit)
-   diff = node_split!(graph, operation.input)
-   return AppliedNodeSplit(operation, diff)
-end
-
-
-function revert_operation!(graph::DAG, operation::AppliedOperation)
-   error("Unknown operation type!")
-end
-
-function revert_operation!(graph::DAG, operation::AppliedNodeFusion)
-   revert_diff!(graph, operation.diff)
-   return operation.operation
-end
-
-function revert_operation!(graph::DAG, operation::AppliedNodeReduction)
-   revert_diff!(graph, operation.diff)
-   return operation.operation
-end
-
-function revert_operation!(graph::DAG, operation::AppliedNodeSplit)
-   revert_diff!(graph, operation.diff)
-   return operation.operation
-end
-
-
-function revert_diff!(graph::DAG, diff)
-   # add removed nodes, remove added nodes, same for edges
-   # note the order
-   for edge in diff.addedEdges
-      remove_edge!(graph, edge, false)
-   end
-   for node in diff.addedNodes
-      remove_node!(graph, node, false)
-   end
-
-   for node in diff.removedNodes
-      insert_node!(graph, node, false)
-   end
-   for edge in diff.removedEdges
-      insert_edge!(graph, edge, false)
-   end
-end
-
-# Fuse nodes n1 -> n2 -> n3 together into one node, return the applied difference to the graph
-function node_fusion!(graph::DAG, n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
-   # clear snapshot
-   get_snapshot_diff(graph)
-
-   if !(n1 in graph) || !(n2 in graph) || !(n3 in graph)
-      error("[Node Fusion] The given nodes are not part of the given graph")
-   end
-
-   if !is_child(n1, n2) || !is_child(n2, n3) || !is_parent(n3, n2) || !is_parent(n2, n1)
-      # the checks are redundant but maybe a good sanity check
-      error("[Node Fusion] The given nodes are not connected by edges which is required for node fusion")
-   end
-
-   # save children and parents
-   n1_children = children(n1)
-   n3_parents = parents(n3)
-   n3_children = children(n3)
-
-   if length(n2.parents) > 1
-      error("[Node Fusion] The given data node has more than one parent")
-   end
-   if length(n2.children) > 1
-      error("[Node Fusion] The given data node has more than one child")
-   end
-   if length(n1.parents) > 1
-      error("[Node Fusion] The given n1 has more than one parent")
-   end
-
-   required_edge1 = make_edge(n1, n2)
-   required_edge2 = make_edge(n2, n3)
-
-   # remove the edges and nodes that will be replaced by the fused node
-   remove_edge!(graph, required_edge1)
-   remove_edge!(graph, required_edge2)
-   remove_node!(graph, n1)
-   remove_node!(graph, n2)
-
-   # get n3's children now so it automatically excludes n2
-   n3_children = children(n3)
-   remove_node!(graph, n3)
-
-   # create new node with the fused compute task
-   new_node = ComputeTaskNode(FusedComputeTask{typeof(n1.task), typeof(n3.task)}())
-   insert_node!(graph, new_node)
-   
-   # use a set for combined children of n1 and n3 to not get duplicates
-   n1and3_children = Set{Node}()
-
-   # remove edges from n1 children to n1
-   for child in n1_children
-      remove_edge!(graph, make_edge(child, n1))
-      push!(n1and3_children, child)
-   end
-
-   # remove edges from n3 children to n3
-   for child in n3_children
-      remove_edge!(graph, make_edge(child, n3))
-      push!(n1and3_children, child)
-   end
-
-   for child in n1and3_children
-      insert_edge!(graph, make_edge(child, new_node))
-   end
-
-   # "repoint" parents of n3 from new node
-   for parent in n3_parents
-      remove_edge!(graph, make_edge(n3, parent))
-      insert_edge!(graph, make_edge(new_node, parent))
-   end
-
-   return get_snapshot_diff(graph)
-end
-
-function node_reduction!(graph::DAG, n1::Node, n2::Node)
-   # clear snapshot
-   get_snapshot_diff(graph)
-
-   #=if !(n1 in graph) || !(n2 in graph)
-      error("[Node Reduction] The given nodes are not part of the given graph")
-   end=#
-
-   #=if typeof(n1) != typeof(n2)
-      error("[Node Reduction] The given nodes are not of the same type")
-   end=#
-
-   # save n2 parents and children
-   n2_children = children(n2)
-   n2_parents = Set(n2.parents)
-
-   #=if Set(n2_children) != Set(n1.children)
-      error("[Node Reduction] The given nodes do not have equal prerequisite nodes which is required for node reduction")
-   end=#
-
-   # remove n2 and all its parents and children
-   for child in n2_children
-      remove_edge!(graph, make_edge(child, n2))
-   end
-
-
-   for parent in n2_parents
-      remove_edge!(graph, make_edge(n2, parent))
-   end
-
-   for parent in n1.parents
-      # delete parents in n1 that already exist in n2
-      delete!(n2_parents, parent)
-   end
-
-   for parent in n2_parents
-      # now add parents of n2 to n1 without duplicates
-      insert_edge!(graph, make_edge(n1, parent))
-   end
-
-   remove_node!(graph, n2)
-
-   return get_snapshot_diff(graph)
-end
-
-function node_split!(graph::DAG, n1::Node)
-   # clear snapshot
-   get_snapshot_diff(graph)
-
-   #=if !(n1 in graph)
-      error("[Node Split] The given node is not part of the given graph")
-   end=#
-
-   n1_parents = parents(n1)
-   n1_children = children(n1)
-
-   #=if length(n1_parents) <= 1
-      error("[Node Split] The given node does not have multiple parents which is required for node split")
-   end=#
-
-   for parent in n1_parents
-      remove_edge!(graph, make_edge(n1, parent))
-   end
-   for child in n1_children
-      remove_edge!(graph, make_edge(child, n1))
-   end
-   remove_node!(graph, n1)
-
-   for parent in n1_parents
-      n_copy = copy(n1)
-      insert_node!(graph, n_copy)
-      insert_edge!(graph, make_edge(n_copy, parent))
-      
-      for child in n1_children
-         insert_edge!(graph, make_edge(child, n_copy))
-      end
-   end
-
-   return get_snapshot_diff(graph)
-end
-
-# function to find node fusions involving the given node if it's a data node
-# pushes the found fusion everywhere it needs to be and returns nothing
-function find_fusions!(graph::DAG, node::DataTaskNode)
-   if length(node.parents) != 1 || length(node.children) != 1
-      return nothing
-   end
-
-   child_node = first(node.children)
-   parent_node = first(node.parents)
-
-   #=if !(child_node in graph) || !(parent_node in graph)
-      error("Parents/Children that are not in the graph!!!")
-   end=#
-
-   if length(child_node.parents) != 1
-      return nothing
-   end
-
-   nf = NodeFusion((child_node, node, parent_node))
-   push!(graph.possibleOperations.nodeFusions, nf)
-   push!(child_node.operations, nf)
-   push!(node.operations, nf)
-   push!(parent_node.operations, nf)
-
-   return nothing
-end
-
-# function to find node fusions involving the given node if it's a compute node
-# pushes the found fusion(s) everywhere it needs to be and returns nothing
-function find_fusions!(graph::DAG, node::ComputeTaskNode)
-   # for loop that always runs once for a scoped block we can break out of
-   for _ in 1:1
-      # assume this node as child of the chain
-      if length(node.parents) != 1
-         break
-      end
-      node2 = first(node.parents)
-      if length(node2.parents) != 1 || length(node2.children) != 1
-         break
-      end
-      node3 = first(node2.parents)
-
-      #=if !(node2 in graph) || !(node3 in graph)
-         error("Parents/Children that are not in the graph!!!")
-      end=#
-
-      nf = NodeFusion((node, node2, node3))
-      push!(graph.possibleOperations.nodeFusions, nf)
-      push!(node.operations, nf)
-      push!(node2.operations, nf)
-      push!(node3.operations, nf)
-   end
-
-   for _ in 1:1
-      # assume this node as parent of the chain
-      if length(node.children) < 1
-         break
-      end
-      node2 = first(node.children)
-      if length(node2.parents) != 1 || length(node2.children) != 1
-         break
-      end
-      node1 = first(node2.children)
-      if (length(node1.parents) > 1)
-         break
-      end
-
-      #=if !(node2 in graph) || !(node1 in graph)
-         error("Parents/Children that are not in the graph!!!")
-      end=#
-
-      nf = NodeFusion((node1, node2, node))
-      push!(graph.possibleOperations.nodeFusions, nf)
-      push!(node1.operations, nf)
-      push!(node2.operations, nf)
-      push!(node.operations, nf)
-   end
-
-   return nothing
-end
-
-function find_reductions!(graph::DAG, node::Node)
-   reductionVector = nothing
-   # possible reductions are with nodes that are partners, i.e. parents of children
-   for partner in partners(node)
-      if can_reduce(node, partner)
-         if reductionVector === nothing
-            # only when there's at least one reduction partner, insert the vector
-            reductionVector = Vector{Node}()
-            push!(reductionVector, node)
-         end
-
-         push!(reductionVector, partner)
-      end
-   end
-
-   if reductionVector !== nothing
-      nr = NodeReduction(reductionVector)
-      push!(graph.possibleOperations.nodeReductions, nr)
-      for node in reductionVector
-         push!(node.operations, nr)
-      end
-   end
-
-   return nothing
-end
-
-function find_splits!(graph::DAG, node::Node)
-   if (can_split(node))
-      ns = NodeSplit(node)
-      push!(graph.possibleOperations.nodeSplits, ns)
-      push!(node.operations, ns)
-   end
-
-   return nothing
-end
-
-# "clean" the operations on a dirty node
-function clean_node!(graph::DAG, node::Node)
-   find_fusions!(graph, node)
-   find_reductions!(graph, node)
-   find_splits!(graph, node)
-
-   delete!(graph.dirtyNodes, node)
-end
-
-# function to generate all possible optmizations on the graph
-function generate_options(graph::DAG)
-   options = PossibleOperations()
-
-   # make sure the graph is fully generated through
-   apply_all!(graph)
-
-   # find possible node fusions
-   for node in graph.nodes
-      if (typeof(node) <: DataTaskNode)
-         if length(node.parents) != 1
-            # data node can only have a single parent
-            continue
-         end
-         parent_node = first(node.parents)
-
-         if length(node.children) != 1
-            # this node is an entry node or has multiple children which should not be possible
-            continue
-         end
-         child_node = first(node.children)
-         if (length(child_node.parents) != 1)
-            continue
-         end
-
-         nf = NodeFusion((child_node, node, parent_node))
-         push!(options.nodeFusions, nf)
-         push!(child_node.operations, nf)
-         push!(node.operations, nf)
-         push!(parent_node.operations, nf)
-      end
-   end
-
-   # find possible node reductions
-   visitedNodes = Set{Node}()
-
-   for node in graph.nodes
-      if (node in visitedNodes)
-         continue
-      end
-
-      push!(visitedNodes, node)
-
-      reductionVector = nothing
-      # possible reductions are with nodes that are partners, i.e. parents of children
-      for partner in partners(node)
-         if can_reduce(node, partner)
-            if reductionVector === nothing
-               # only when there's at least one reduction partner, insert the vector
-               reductionVector = Vector{Node}()
-               push!(reductionVector, node)
-            end
-
-            push!(reductionVector, partner)
-            push!(visitedNodes, partner)
-         end
-      end
-
-      if reductionVector !== nothing
-         nr = NodeReduction(reductionVector)
-         push!(options.nodeReductions, nr)
-         for node in reductionVector
-            push!(node.operations, nr)
-         end
-      end
-   end
-
-   # find possible node splits
-   for node in graph.nodes
-      if (can_split(node))
-         ns = NodeSplit(node)
-         push!(options.nodeSplits, ns)
-         push!(node.operations, ns)
-      end
-   end
-
-   graph.possibleOperations = options
-   empty!(graph.dirtyNodes)
-end
-
-function get_operations(graph::DAG)
-   apply_all!(graph)
-
-   if isempty(graph.possibleOperations)
-      generate_options(graph)
-   end
-
-   while !isempty(graph.dirtyNodes)
-      clean_node!(graph, first(graph.dirtyNodes))
-   end
-
-   return graph.possibleOperations
-end

src/models/abc/parse.jl

@@ -0,0 +1,200 @@
+# functions for importing DAGs from a file
+regex_a = r"^[A-C]\d+$"                     # Regex for the initial particles
+regex_c = r"^[A-C]\(([^']*),([^']*)\)$"     # Regex for the combinations of 2 particles
+regex_m = r"^M\(([^']*),([^']*),([^']*)\)$" # Regex for the combinations of 3 particles
+regex_plus = r"^\+$"                        # Regex for the sum
+
+"""
+    parse_nodes(input::AbstractString)
+
+Parse the given string into a vector of strings containing each node.
+"""
+function parse_nodes(input::AbstractString)
+    regex = r"'([^']*)'"
+    matches = eachmatch(regex, input)
+    output = [match.captures[1] for match in matches]
+    return output
+end
+
+"""
+    parse_edges(input::AbstractString)
+
+Parse the given string into a vector of strings containing each edge. Currently unused since the entire graph can be read from just the node names.
+"""
+function parse_edges(input::AbstractString)
+    regex = r"\('([^']*)', '([^']*)'\)"
+    matches = eachmatch(regex, input)
+    output = [(match.captures[1], match.captures[2]) for match in matches]
+    return output
+end
+
+"""
+    parse_abc(filename::String; verbose::Bool = false)
+
+Read an abc-model process from the given file. If `verbose` is set to true, print some progress information to stdout.
+
+Returns a valid [`DAG`](@ref).
+"""
+function parse_abc(filename::String, verbose::Bool = false)
+    file = open(filename, "r")
+
+    if (verbose)
+        println("Opened file")
+    end
+    nodes_string = readline(file)
+    nodes = parse_nodes(nodes_string)
+
+    close(file)
+    if (verbose)
+        println("Read file")
+    end
+
+    graph = DAG()
+
+    # estimate total number of nodes
+    # try to slightly overestimate so no resizing is necessary
+    # data nodes are not included in length(nodes) and there are a few more than compute nodes
+    estimate_no_nodes = round(Int, length(nodes) * 4)
+    if (verbose)
+        println("Estimating ", estimate_no_nodes, " Nodes")
+    end
+    sizehint!(graph.nodes, estimate_no_nodes)
+
+    sum_node = insert_node!(graph, make_node(ComputeTaskSum()), false, false)
+    global_data_out = insert_node!(graph, make_node(DataTask(10)), false, false)
+    insert_edge!(graph, sum_node, global_data_out, false, false)
+
+    # remember the data out nodes for connection
+    dataOutNodes = Dict()
+
+    if (verbose)
+        println("Building graph")
+    end
+    noNodes = 0
+    nodesToRead = length(nodes)
+    while !isempty(nodes)
+        node = popfirst!(nodes)
+        noNodes += 1
+        if (noNodes % 100 == 0)
+            if (verbose)
+                percent = string(
+                    round(100.0 * noNodes / nodesToRead, digits = 2),
+                    "%",
+                )
+                print("\rReading Nodes... $percent")
+            end
+        end
+        if occursin(regex_a, node)
+            # add nodes and edges for the state reading to u(P(Particle))
+            data_in = insert_node!(graph, make_node(DataTask(4)), false, false) # read particle data node
+            compute_P =
+                insert_node!(graph, make_node(ComputeTaskP()), false, false) # compute P node
+            data_Pu = insert_node!(graph, make_node(DataTask(6)), false, false) # transfer data from P to u
+            compute_u =
+                insert_node!(graph, make_node(ComputeTaskU()), false, false) # compute U node
+            data_out = insert_node!(graph, make_node(DataTask(3)), false, false) # transfer data out from u
+
+            insert_edge!(graph, data_in, compute_P, false, false)
+            insert_edge!(graph, compute_P, data_Pu, false, false)
+            insert_edge!(graph, data_Pu, compute_u, false, false)
+            insert_edge!(graph, compute_u, data_out, false, false)
+
+            # remember the data_out node for future edges
+            dataOutNodes[node] = data_out
+        elseif occursin(regex_c, node)
+            capt = match(regex_c, node)
+
+            in1 = capt.captures[1]
+            in2 = capt.captures[2]
+
+            compute_v =
+                insert_node!(graph, make_node(ComputeTaskV()), false, false)
+            data_out = insert_node!(graph, make_node(DataTask(5)), false, false)
+
+            if (occursin(regex_c, in1))
+                # put an S node after this input
+                compute_S = insert_node!(
+                    graph,
+                    make_node(ComputeTaskS1()),
+                    false,
+                    false,
+                )
+                data_S_v =
+                    insert_node!(graph, make_node(DataTask(5)), false, false)
+
+                insert_edge!(graph, dataOutNodes[in1], compute_S, false, false)
+                insert_edge!(graph, compute_S, data_S_v, false, false)
+
+                insert_edge!(graph, data_S_v, compute_v, false, false)
+            else
+                insert_edge!(graph, dataOutNodes[in1], compute_v, false, false)
+            end
+
+            if (occursin(regex_c, in2))
+                # i think the current generator only puts the combined particles in the first space, so this case might never be entered
+                # put an S node after this input
+                compute_S = insert_node!(
+                    graph,
+                    make_node(ComputeTaskS1()),
+                    false,
+                    false,
+                )
+                data_S_v =
+                    insert_node!(graph, make_node(DataTask(5)), false, false)
+
+                insert_edge!(graph, dataOutNodes[in2], compute_S, false, false)
+                insert_edge!(graph, compute_S, data_S_v, false, false)
+
+                insert_edge!(graph, data_S_v, compute_v, false, false)
+            else
+                insert_edge!(graph, dataOutNodes[in2], compute_v, false, false)
+            end
+
+            insert_edge!(graph, compute_v, data_out, false, false)
+            dataOutNodes[node] = data_out
+
+        elseif occursin(regex_m, node)
+            # assume for now that only the first particle of the three is combined and the other two are "original" ones
+            capt = match(regex_m, node)
+            in1 = capt.captures[1]
+            in2 = capt.captures[2]
+            in3 = capt.captures[3]
+
+            # in2 + in3 with a v
+            compute_v =
+                insert_node!(graph, make_node(ComputeTaskV()), false, false)
+            data_v = insert_node!(graph, make_node(DataTask(5)), false, false)
+
+            insert_edge!(graph, dataOutNodes[in2], compute_v, false, false)
+            insert_edge!(graph, dataOutNodes[in3], compute_v, false, false)
+            insert_edge!(graph, compute_v, data_v, false, false)
+
+            # combine with the v of the combined other input
+            compute_S2 =
+                insert_node!(graph, make_node(ComputeTaskS2()), false, false)
+            data_out =
+                insert_node!(graph, make_node(DataTask(10)), false, false)
+
+            insert_edge!(graph, data_v, compute_S2, false, false)
+            insert_edge!(graph, dataOutNodes[in1], compute_S2, false, false)
+            insert_edge!(graph, compute_S2, data_out, false, false)
+
+            insert_edge!(graph, data_out, sum_node, false, false)
+        elseif occursin(regex_plus, node)
+            if (verbose)
+                println("\rReading Nodes Complete    ")
+                println("Added ", length(graph.nodes), " nodes")
+            end
+        else
+            @assert false (
+                "Unknown node '$node' while reading from file $filename"
+            )
+        end
+    end
+
+    #put all nodes into dirty nodes set
+    graph.dirtyNodes = copy(graph.nodes)
+
+    # don't actually need to read the edges
+    return graph
+end

src/models/abc/properties.jl

@@ -0,0 +1,102 @@
+"""
+    compute_effort(t::ComputeTaskS1)
+
+Return the compute effort of an S1 task.
+"""
+compute_effort(t::ComputeTaskS1) = 10
+
+"""
+    compute_effort(t::ComputeTaskS2)
+
+Return the compute effort of an S2 task.
+"""
+compute_effort(t::ComputeTaskS2) = 10
+
+"""
+    compute_effort(t::ComputeTaskU)
+
+Return the compute effort of a U task.
+"""
+compute_effort(t::ComputeTaskU) = 6
+
+"""
+    compute_effort(t::ComputeTaskV)
+
+Return the compute effort of a V task.
+"""
+compute_effort(t::ComputeTaskV) = 20
+
+"""
+    compute_effort(t::ComputeTaskP)
+
+Return the compute effort of a P task.
+"""
+compute_effort(t::ComputeTaskP) = 15
+
+"""
+    compute_effort(t::ComputeTaskSum)
+
+Return the compute effort of a Sum task. 
+
+Note: This is a constant compute effort, even though sum scales with the number of its inputs. Since there is only ever a single sum node in a graph generated from the ABC-Model,
+this doesn't matter.
+"""
+compute_effort(t::ComputeTaskSum) = 1
+
+"""
+    show(io::IO, t::DataTask)
+
+Print the data task to io.
+"""
+function show(io::IO, t::DataTask)
+    return print(io, "Data", t.data)
+end
+
+"""
+    show(io::IO, t::ComputeTaskS1)
+
+Print the S1 task to io.
+"""
+show(io::IO, t::ComputeTaskS1) = print("ComputeS1")
+
+"""
+    show(io::IO, t::ComputeTaskS2)
+
+Print the S2 task to io.
+"""
+show(io::IO, t::ComputeTaskS2) = print("ComputeS2")
+
+"""
+    show(io::IO, t::ComputeTaskP)
+
+Print the P task to io.
+"""
+show(io::IO, t::ComputeTaskP) = print("ComputeP")
+
+"""
+    show(io::IO, t::ComputeTaskU)
+
+Print the U task to io.
+"""
+show(io::IO, t::ComputeTaskU) = print("ComputeU")
+
+"""
+    show(io::IO, t::ComputeTaskV)
+
+Print the V task to io.
+"""
+show(io::IO, t::ComputeTaskV) = print("ComputeV")
+
+"""
+    show(io::IO, t::ComputeTaskSum)
+
+Print the sum task to io.
+"""
+show(io::IO, t::ComputeTaskSum) = print("ComputeSum")
+
+"""
+    copy(t::DataTask)
+
+Copy the data task and return it.
+"""
+copy(t::DataTask) = DataTask(t.data)

src/models/abc/types.jl

@@ -0,0 +1,65 @@
+"""
+    DataTask <: AbstractDataTask
+
+Task representing a specific data transfer in the ABC Model.
+"""
+struct DataTask <: AbstractDataTask
+    data::UInt64
+end
+
+"""
+    ComputeTaskS1 <: AbstractComputeTask
+
+S task with a single child.
+"""
+struct ComputeTaskS1 <: AbstractComputeTask end
+
+"""
+    ComputeTaskS2 <: AbstractComputeTask
+
+S task with two children.
+"""
+struct ComputeTaskS2 <: AbstractComputeTask end
+
+"""
+    ComputeTaskP <: AbstractComputeTask
+
+P task with no children.
+"""
+struct ComputeTaskP <: AbstractComputeTask end
+
+"""
+    ComputeTaskV <: AbstractComputeTask
+
+v task with two children.
+"""
+struct ComputeTaskV <: AbstractComputeTask end
+
+"""
+    ComputeTaskU <: AbstractComputeTask
+
+u task with a single child.
+"""
+struct ComputeTaskU <: AbstractComputeTask end
+
+"""
+    ComputeTaskSum <: AbstractComputeTask
+
+Task that sums all its inputs, n children.
+"""
+struct ComputeTaskSum <: AbstractComputeTask end
+
+"""
+    ABC_TASKS
+
+Constant vector of all tasks of the ABC-Model.
+"""
+ABC_TASKS = [
+    DataTask,
+    ComputeTaskS1,
+    ComputeTaskS2,
+    ComputeTaskP,
+    ComputeTaskV,
+    ComputeTaskU,
+    ComputeTaskSum,
+]

src/node/compare.jl

@@ -0,0 +1,35 @@
+"""
+    ==(e1::Edge, e2::Edge)
+
+Equality comparison between two edges.
+"""
+function ==(e1::Edge, e2::Edge)
+    return e1.edge[1] == e2.edge[1] && e1.edge[2] == e2.edge[2]
+end
+
+"""
+    ==(n1::Node, n2::Node)
+
+Fallback equality comparison between two nodes. For equal node types, the more specific versions of this function will be called.
+"""
+function ==(n1::Node, n2::Node)
+    return false
+end
+
+"""
+    ==(n1::ComputeTaskNode, n2::ComputeTaskNode)
+
+Equality comparison between two [`ComputeTaskNode`](@ref)s.
+"""
+function ==(n1::ComputeTaskNode, n2::ComputeTaskNode)
+    return n1.id == n2.id
+end
+
+"""
+    ==(n1::DataTaskNode, n2::DataTaskNode)
+
+Equality comparison between two [`DataTaskNode`](@ref)s.
+"""
+function ==(n1::DataTaskNode, n2::DataTaskNode)
+    return n1.id == n2.id
+end

src/node/create.jl

@@ -0,0 +1,53 @@
+"""
+    make_node(t::AbstractTask)
+
+Fallback implementation of `make_node` for an [`AbstractTask`](@ref), throwing an error.
+"""
+function make_node(t::AbstractTask)
+    return error("Cannot make a node from this task type")
+end
+
+"""
+    make_node(t::AbstractDataTask)
+
+Construct and return a new [`DataTaskNode`](@ref) with the given task.
+"""
+function make_node(t::AbstractDataTask)
+    return DataTaskNode(t)
+end
+
+"""
+    make_node(t::AbstractComputeTask)
+
+Construct and return a new [`ComputeTaskNode`](@ref) with the given task.
+"""
+function make_node(t::AbstractComputeTask)
+    return ComputeTaskNode(t)
+end
+
+"""
+    make_edge(n1::Node, n2::Node)
+
+Fallback implementation of `make_edge` throwing an error. If you got this error it likely means you tried to construct an edge between two nodes of the same type.
+"""
+function make_edge(n1::Node, n2::Node)
+    return error("Can only create edges from compute to data node or reverse")
+end
+
+"""
+    make_edge(n1::ComputeTaskNode, n2::DataTaskNode)
+
+Construct and return a new [`Edge`](@ref) pointing from `n1` (child) to `n2` (parent).
+"""
+function make_edge(n1::ComputeTaskNode, n2::DataTaskNode)
+    return Edge((n1, n2))
+end
+
+"""
+    make_edge(n1::DataTaskNode, n2::ComputeTaskNode)
+
+Construct and return a new [`Edge`](@ref) pointing from `n1` (child) to `n2` (parent).
+"""
+function make_edge(n1::DataTaskNode, n2::ComputeTaskNode)
+    return Edge((n1, n2))
+end

src/node/print.jl

@@ -0,0 +1,17 @@
+"""
+    show(io::IO, n::Node)
+
+Print a short string representation of the node to io.
+"""
+function show(io::IO, n::Node)
+    return print(io, "Node(", n.task, ")")
+end
+
+"""
+    show(io::IO, e::Edge)
+
+Print a short string representation of the edge to io.
+"""
+function show(io::IO, e::Edge)
+    return print(io, "Edge(", e.edge[1], ", ", e.edge[2], ")")
+end

src/node/properties.jl

@@ -0,0 +1,103 @@
+"""
+    is_entry_node(node::Node)
+
+Return whether this node is an entry node in its graph, i.e., it has no children.
+"""
+is_entry_node(node::Node) = length(node.children) == 0
+
+"""
+    is_exit_node(node::Node)
+
+Return whether this node is an exit node of its graph, i.e., it has no parents.
+"""
+is_exit_node(node::Node) = length(node.parents) == 0
+
+"""
+    children(node::Node)
+
+Return a copy of the node's children so it can safely be muted without changing the node in the graph.
+
+A node's children are its prerequisite nodes, nodes that need to execute before the task of this node.
+"""
+function children(node::Node)
+    return copy(node.children)
+end
+
+"""
+    parents(node::Node)
+
+Return a copy of the node's parents so it can safely be muted without changing the node in the graph.
+
+A node's parents are its subsequent nodes, nodes that need this node to execute.
+"""
+function parents(node::Node)
+    return copy(node.parents)
+end
+
+"""
+    siblings(node::Node)
+
+Return a vector of all siblings of this node. 
+
+A node's siblings are all children of any of its parents. The result contains no duplicates and includes the node itself.
+"""
+function siblings(node::Node)
+    result = Set{Node}()
+    push!(result, node)
+    for parent in node.parents
+        union!(result, parent.children)
+    end
+
+    return result
+end
+
+"""
+    partners(node::Node)
+
+Return a vector of all partners of this node. 
+
+A node's partners are all parents of any of its children. The result contains no duplicates and includes the node itself.
+
+Note: This is very slow when there are multiple children with many parents. 
+This is less of a problem in [`siblings(node::Node)`](@ref) because (depending on the model) there are no nodes with a large number of children, or only a single one.
+"""
+function partners(node::Node)
+    result = Set{Node}()
+    push!(result, node)
+    for child in node.children
+        union!(result, child.parents)
+    end
+
+    return result
+end
+
+"""
+    partners(node::Node, set::Set{Node})
+
+Alternative version to [`partners(node::Node)`](@ref), avoiding allocation of a new set. Works on the given set and returns `nothing`.
+"""
+function partners(node::Node, set::Set{Node})
+    push!(set, node)
+    for child in node.children
+        union!(set, child.parents)
+    end
+    return nothing
+end
+
+"""
+    is_parent(potential_parent::Node, node::Node)
+
+Return whether the `potential_parent` is a parent of `node`.
+"""
+function is_parent(potential_parent::Node, node::Node)
+    return potential_parent in node.parents
+end
+
+"""
+    is_child(potential_child::Node, node::Node)
+
+Return whether the `potential_child` is a child of `node`.
+"""
+function is_child(potential_child::Node, node::Node)
+    return potential_child in node.children
+end

src/node/type.jl

@@ -0,0 +1,138 @@
+using Random
+using UUIDs
+using Base.Threads
+
+# TODO: reliably find out how many threads we're running with (nthreads() returns 1 when precompiling :/)
+rng = [Random.MersenneTwister(0) for _ in 1:32]
+
+"""
+    Node
+
+The abstract base type of every node.
+
+See [`DataTaskNode`](@ref), [`ComputeTaskNode`](@ref) and [`make_node`](@ref).
+"""
+abstract type Node end
+
+# declare this type here because it's needed
+# the specific operations are declared in graph.jl
+abstract type Operation end
+
+"""
+    DataTaskNode <: Node
+    
+Any node that transfers data and does no computation.
+
+# Fields
+`.task`: The node's data task type. Usually [`DataTask`](@ref).\\
+`.parents`: A vector of the node's parents (i.e. nodes that depend on this one).\\
+`.children`: A vector of the node's children (i.e. nodes that this one depends on).\\
+`.id`: The node's id. Improves the speed of comparisons.\\
+`.nodeReduction`: Either this node's [`NodeReduction`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeSplit`: Either this node's [`NodeSplit`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeFusion`: Either this node's [`NodeFusion`](@ref) or `missing`, if none. There can only be at most one for DataTaskNodes.
+"""
+mutable struct DataTaskNode <: Node
+    task::AbstractDataTask
+
+    # use vectors as sets have way too much memory overhead
+    parents::Vector{Node}
+    children::Vector{Node}
+
+    # need a unique identifier unique to every *constructed* node
+    # however, it can be copied when splitting a node
+    id::Base.UUID
+
+    # the NodeReduction involving this node, if it exists
+    # Can't use the NodeReduction type here because it's not yet defined
+    nodeReduction::Union{Operation, Missing}
+
+    # the NodeSplit involving this node, if it exists
+    nodeSplit::Union{Operation, Missing}
+
+    # the node fusion involving this node, if it exists
+    nodeFusion::Union{Operation, Missing}
+end
+
+"""
+    ComputeTaskNode <: Node
+    
+Any node that transfers data and does no computation.
+
+# Fields
+`.task`: The node's data task type. Usually [`DataTask`](@ref).\\
+`.parents`: A vector of the node's parents (i.e. nodes that depend on this one).\\
+`.children`: A vector of the node's children (i.e. nodes that this one depends on).\\
+`.id`: The node's id. Improves the speed of comparisons.\\
+`.nodeReduction`: Either this node's [`NodeReduction`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeSplit`: Either this node's [`NodeSplit`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeFusion`: A vector of this node's [`NodeFusion`](@ref)s. For a ComputeTaskNode there can be any number of these, unlike the DataTaskNodes.
+"""
+mutable struct ComputeTaskNode <: Node
+    task::AbstractComputeTask
+    parents::Vector{Node}
+    children::Vector{Node}
+    id::Base.UUID
+
+    nodeReduction::Union{Operation, Missing}
+    nodeSplit::Union{Operation, Missing}
+
+    # for ComputeTasks there can be multiple fusions, unlike the DataTasks
+    nodeFusions::Vector{Operation}
+end
+
+DataTaskNode(t::AbstractDataTask) = DataTaskNode(
+    t,
+    Vector{Node}(),
+    Vector{Node}(),
+    UUIDs.uuid1(rng[threadid()]),
+    missing,
+    missing,
+    missing,
+)
+ComputeTaskNode(t::AbstractComputeTask) = ComputeTaskNode(
+    t,
+    Vector{Node}(),
+    Vector{Node}(),
+    UUIDs.uuid1(rng[threadid()]),
+    missing,
+    missing,
+    Vector{NodeFusion}(),
+)
+
+"""
+    Edge
+
+Type of an edge in the graph. Edges can only exist between a [`DataTaskNode`](@ref) and a [`ComputeTaskNode`](@ref) or vice versa, not between two of the same type of node.
+
+An edge always points from child to parent: `child = e.edge[1]` and `parent = e.edge[2]`.
+
+The child is the prerequisite node of the parent.
+"""
+struct Edge
+    # edge points from child to parent
+    edge::Union{
+        Tuple{DataTaskNode, ComputeTaskNode},
+        Tuple{ComputeTaskNode, DataTaskNode},
+    }
+end
+
+copy(m::Missing) = missing
+copy(n::ComputeTaskNode) = ComputeTaskNode(
+    copy(n.task),
+    copy(n.parents),
+    copy(n.children),
+    UUIDs.uuid1(rng[threadid()]),
+    copy(n.nodeReduction),
+    copy(n.nodeSplit),
+    copy(n.nodeFusions),
+)
+copy(n::DataTaskNode) = DataTaskNode(
+    copy(n.task),
+    copy(n.parents),
+    copy(n.children),
+    UUIDs.uuid1(rng[threadid()]),
+    copy(n.nodeReduction),
+    copy(n.nodeSplit),
+    copy(n.nodeFusion),
+)

src/node/validate.jl

@@ -0,0 +1,64 @@
+"""
+    is_valid_node(graph::DAG, node::Node)
+
+Verify that a given node is valid in the graph. Call like `@test is_valid_node(g, n)`. Uses `@assert` to fail if something is invalid but also provide an error message.
+
+This function is very performance intensive and should only be used when testing or debugging.
+
+See also this function's specific versions for the concrete Node types [`is_valid(graph::DAG, node::ComputeTaskNode)`](@ref) and [`is_valid(graph::DAG, node::DataTaskNode)`](@ref).
+"""
+function is_valid_node(graph::DAG, node::Node)
+    @assert node in graph "Node is not part of the given graph!"
+
+    for parent in node.parents
+        @assert typeof(parent) != typeof(node) "Node's type is the same as its parent's!"
+        @assert parent in graph "Node's parent is not in the same graph!"
+        @assert node in parent.children "Node is not a child of its parent!"
+    end
+
+    for child in node.children
+        @assert typeof(child) != typeof(node) "Node's type is the same as its child's!"
+        @assert child in graph "Node's child is not in the same graph!"
+        @assert node in child.parents "Node is not a parent of its child!"
+    end
+
+    if !ismissing(node.nodeReduction)
+        @assert is_valid(graph, node.nodeReduction)
+    end
+    if !ismissing(node.nodeSplit)
+        @assert is_valid(graph, node.nodeSplit)
+    end
+    return true
+end
+
+"""
+    is_valid(graph::DAG, node::ComputeTaskNode)
+
+Verify that the given compute node is valid in the graph. Call with `@assert` or `@test` when testing or debugging.
+
+This also calls [`is_valid_node(graph::DAG, node::Node)`](@ref).
+"""
+function is_valid(graph::DAG, node::ComputeTaskNode)
+    @assert is_valid_node(graph, node)
+
+    for nf in node.nodeFusions
+        @assert is_valid(graph, nf)
+    end
+    return true
+end
+
+"""
+    is_valid(graph::DAG, node::DataTaskNode)
+
+Verify that the given compute node is valid in the graph. Call with `@assert` or `@test` when testing or debugging.
+
+This also calls [`is_valid_node(graph::DAG, node::Node)`](@ref).
+"""
+function is_valid(graph::DAG, node::DataTaskNode)
+    @assert is_valid_node(graph, node)
+
+    if !ismissing(node.nodeFusion)
+        @assert is_valid(graph, node.nodeFusion)
+    end
+    return true
+end

src/node_functions.jl

@@ -1,50 +0,0 @@
-function make_node(t::AbstractTask)
-    error("Cannot make a node from this task type")
-end
-
-function make_node(t::AbstractDataTask)
-    return DataTaskNode(t)
-end
-
-function make_node(t::AbstractComputeTask)
-    return ComputeTaskNode(t)
-end
-
-function make_edge(n1::Node, n2::Node)
-    error("Can only create edges from compute to data node or reverse")
-end
-
-function make_edge(n1::ComputeTaskNode, n2::DataTaskNode)
-    return Edge((n1, n2))
-end
-
-function make_edge(n1::DataTaskNode, n2::ComputeTaskNode)
-    return Edge((n1, n2))
-end
-
-function show(io::IO, n::Node)
-    print(io, "Node(", n.task, ")")
-end
-
-function show(io::IO, e::Edge)
-    print(io, "Edge(", e.edge[1], ", ", e.edge[2], ")")
-end
-
-function ==(e1::Edge, e2::Edge)
-    return e1.edge[1] == e2.edge[1] && e1.edge[2] == e2.edge[2]
-end
-
-function ==(n1::Node, n2::Node)
-    return false
-end
-
-function ==(n1::ComputeTaskNode, n2::ComputeTaskNode)
-    return n1.id == n2.id
-end
-
-function ==(n1::DataTaskNode, n2::DataTaskNode)
-    return n1.id == n2.id
-end
-
-copy(n::ComputeTaskNode) =  ComputeTaskNode(copy(n.task), copy(n.parents), copy(n.children), UUIDs.uuid1(rng), copy(n.operations))
-copy(n::DataTaskNode) = DataTaskNode(copy(n.task), copy(n.parents), copy(n.children), UUIDs.uuid1(rng), copy(n.operations))

src/nodes.jl

@@ -1,42 +0,0 @@
-using Random
-using UUIDs
-
-rng = Random.MersenneTwister(0)
-
-abstract type Node end
-
-# declare this type here because it's needed
-# the specific operations are declared in graph.jl
-abstract type Operation end
-
-struct DataTaskNode <: Node
-   task::AbstractDataTask
-   
-   # use vectors as sets have way too much memory overhead
-   parents::Vector{Node}
-   children::Vector{Node}
-
-   # need a unique identifier unique to every *constructed* node
-   # however, it can be copied when splitting a node
-   id::Base.UUID
-
-   # a vector holding references to the graph operations involving this node
-   operations::Vector{Operation}
-end
-
-# same as DataTaskNode
-struct ComputeTaskNode <: Node
-   task::AbstractComputeTask
-   parents::Vector{Node}
-   children::Vector{Node}
-   id::Base.UUID
-   operations::Vector{Operation}
-end
-
-DataTaskNode(t::AbstractDataTask) = DataTaskNode(t, Vector{Node}(), Vector{Node}(), UUIDs.uuid1(rng), Vector{Operation}())
-ComputeTaskNode(t::AbstractComputeTask) = ComputeTaskNode(t, Vector{Node}(), Vector{Node}(), UUIDs.uuid1(rng), Vector{Operation}())
-
-struct Edge
-   # edge points from child to parent
-   edge::Union{Tuple{DataTaskNode, ComputeTaskNode}, Tuple{ComputeTaskNode, DataTaskNode}}
-end

src/operation/apply.jl

@@ -0,0 +1,281 @@
+"""
+    apply_all!(graph::DAG)
+
+Apply all unapplied operations in the DAG. Is automatically called in all functions that require the latest state of the [`DAG`](@ref).
+"""
+function apply_all!(graph::DAG)
+    while !isempty(graph.operationsToApply)
+        # get next operation to apply from front of the deque
+        op = popfirst!(graph.operationsToApply)
+
+        # apply it
+        appliedOp = apply_operation!(graph, op)
+
+        # push to the end of the appliedOperations deque
+        push!(graph.appliedOperations, appliedOp)
+    end
+    return nothing
+end
+
+"""
+    apply_operation!(graph::DAG, operation::Operation)
+
+Fallback implementation of apply_operation! for unimplemented operation types, throwing an error.
+"""
+function apply_operation!(graph::DAG, operation::Operation)
+    return error("Unknown operation type!")
+end
+
+"""
+    apply_operation!(graph::DAG, operation::NodeFusion)
+
+Apply the given [`NodeFusion`](@ref) to the graph. Generic wrapper around [`node_fusion!`](@ref).
+
+Return an [`AppliedNodeFusion`](@ref) object generated from the graph's [`Diff`](@ref).
+"""
+function apply_operation!(graph::DAG, operation::NodeFusion)
+    diff = node_fusion!(
+        graph,
+        operation.input[1],
+        operation.input[2],
+        operation.input[3],
+    )
+    return AppliedNodeFusion(operation, diff)
+end
+
+"""
+    apply_operation!(graph::DAG, operation::NodeReduction)
+
+Apply the given [`NodeReduction`](@ref) to the graph. Generic wrapper around [`node_reduction!`](@ref).
+
+Return an [`AppliedNodeReduction`](@ref) object generated from the graph's [`Diff`](@ref).
+"""
+function apply_operation!(graph::DAG, operation::NodeReduction)
+    diff = node_reduction!(graph, operation.input)
+    return AppliedNodeReduction(operation, diff)
+end
+
+"""
+    apply_operation!(graph::DAG, operation::NodeSplit)
+
+Apply the given [`NodeSplit`](@ref) to the graph. Generic wrapper around [`node_split!`](@ref).
+
+Return an [`AppliedNodeSplit`](@ref) object generated from the graph's [`Diff`](@ref).
+"""
+function apply_operation!(graph::DAG, operation::NodeSplit)
+    diff = node_split!(graph, operation.input)
+    return AppliedNodeSplit(operation, diff)
+end
+
+"""
+    revert_operation!(graph::DAG, operation::AppliedOperation)
+
+Fallback implementation of operation reversion for unimplemented operation types, throwing an error.
+"""
+function revert_operation!(graph::DAG, operation::AppliedOperation)
+    return error("Unknown operation type!")
+end
+
+"""
+    revert_operation!(graph::DAG, operation::AppliedNodeFusion)
+
+Revert the applied node fusion on the graph. Return the original [`NodeFusion`](@ref) operation.
+"""
+function revert_operation!(graph::DAG, operation::AppliedNodeFusion)
+    revert_diff!(graph, operation.diff)
+    return operation.operation
+end
+
+"""
+    revert_operation!(graph::DAG, operation::AppliedNodeReduction)
+
+Revert the applied node fusion on the graph. Return the original [`NodeReduction`](@ref) operation.
+"""
+function revert_operation!(graph::DAG, operation::AppliedNodeReduction)
+    revert_diff!(graph, operation.diff)
+    return operation.operation
+end
+
+"""
+    revert_operation!(graph::DAG, operation::AppliedNodeSplit)
+
+Revert the applied node fusion on the graph. Return the original [`NodeSplit`](@ref) operation.
+"""
+function revert_operation!(graph::DAG, operation::AppliedNodeSplit)
+    revert_diff!(graph, operation.diff)
+    return operation.operation
+end
+
+"""
+    revert_diff!(graph::DAG, diff::Diff)
+
+Revert the given diff on the graph. Used to revert the individual [`AppliedOperation`](@ref)s with [`revert_operation!`](@ref).
+"""
+function revert_diff!(graph::DAG, diff::Diff)
+    # add removed nodes, remove added nodes, same for edges
+    # note the order
+    for edge in diff.addedEdges
+        remove_edge!(graph, edge.edge[1], edge.edge[2], false)
+    end
+    for node in diff.addedNodes
+        remove_node!(graph, node, false)
+    end
+
+    for node in diff.removedNodes
+        insert_node!(graph, node, false)
+    end
+    for edge in diff.removedEdges
+        insert_edge!(graph, edge.edge[1], edge.edge[2], false)
+    end
+    return nothing
+end
+
+"""
+    node_fusion!(graph::DAG, n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+
+Fuse nodes n1 -> n2 -> n3 together into one node, return the applied difference to the graph.
+
+For details see [`NodeFusion`](@ref).
+"""
+function node_fusion!(
+    graph::DAG,
+    n1::ComputeTaskNode,
+    n2::DataTaskNode,
+    n3::ComputeTaskNode,
+)
+    # @assert is_valid_node_fusion_input(graph, n1, n2, n3)
+
+    # clear snapshot
+    get_snapshot_diff(graph)
+
+
+    # save children and parents
+    n1_children = children(n1)
+    n3_parents = parents(n3)
+    n3_children = children(n3)
+
+    # remove the edges and nodes that will be replaced by the fused node
+    remove_edge!(graph, n1, n2)
+    remove_edge!(graph, n2, n3)
+    remove_node!(graph, n1)
+    remove_node!(graph, n2)
+
+    # get n3's children now so it automatically excludes n2
+    n3_children = children(n3)
+    remove_node!(graph, n3)
+
+    # create new node with the fused compute task
+    new_node =
+        ComputeTaskNode(FusedComputeTask{typeof(n1.task), typeof(n3.task)}())
+    insert_node!(graph, new_node)
+
+    # use a set for combined children of n1 and n3 to not get duplicates
+    n1and3_children = Set{Node}()
+
+    # remove edges from n1 children to n1
+    for child in n1_children
+        remove_edge!(graph, child, n1)
+        push!(n1and3_children, child)
+    end
+
+    # remove edges from n3 children to n3
+    for child in n3_children
+        remove_edge!(graph, child, n3)
+        push!(n1and3_children, child)
+    end
+
+    for child in n1and3_children
+        insert_edge!(graph, child, new_node)
+    end
+
+    # "repoint" parents of n3 from new node
+    for parent in n3_parents
+        remove_edge!(graph, n3, parent)
+        insert_edge!(graph, new_node, parent)
+    end
+
+    return get_snapshot_diff(graph)
+end
+
+"""
+    node_reduction!(graph::DAG, nodes::Vector{Node})
+
+Reduce the given nodes together into one node, return the applied difference to the graph.
+
+For details see [`NodeReduction`](@ref).
+"""
+function node_reduction!(graph::DAG, nodes::Vector{Node})
+    # @assert is_valid_node_reduction_input(graph, nodes)
+
+    # clear snapshot
+    get_snapshot_diff(graph)
+
+    n1 = nodes[1]
+    n1_children = children(n1)
+
+    n1_parents = Set(n1.parents)
+    new_parents = Set{Node}()
+
+    # remove all of the nodes' parents and children and the nodes themselves (except for first node)
+    for i in 2:length(nodes)
+        n = nodes[i]
+        for child in n1_children
+            remove_edge!(graph, child, n)
+        end
+
+        for parent in parents(n)
+            remove_edge!(graph, n, parent)
+
+            # collect all parents
+            push!(new_parents, parent)
+        end
+
+        remove_node!(graph, n)
+    end
+
+    setdiff!(new_parents, n1_parents)
+
+    for parent in new_parents
+        # now add parents of all input nodes to n1 without duplicates
+        insert_edge!(graph, n1, parent)
+    end
+
+    return get_snapshot_diff(graph)
+end
+
+"""
+    node_split!(graph::DAG, n1::Node)
+
+Split the given node into one node per parent, return the applied difference to the graph.
+
+For details see [`NodeSplit`](@ref).
+"""
+function node_split!(graph::DAG, n1::Node)
+    # @assert is_valid_node_split_input(graph, n1)
+
+    # clear snapshot
+    get_snapshot_diff(graph)
+
+    n1_parents = parents(n1)
+    n1_children = children(n1)
+
+    for parent in n1_parents
+        remove_edge!(graph, n1, parent)
+    end
+    for child in n1_children
+        remove_edge!(graph, child, n1)
+    end
+    remove_node!(graph, n1)
+
+    for parent in n1_parents
+        n_copy = copy(n1)
+        insert_node!(graph, n_copy)
+        insert_edge!(graph, n_copy, parent)
+
+        for child in n1_children
+            insert_edge!(graph, child, n_copy)
+        end
+    end
+
+    return get_snapshot_diff(graph)
+end

src/operation/clean.jl

@@ -0,0 +1,140 @@
+# These are functions for "cleaning" nodes, i.e. regenerating the possible operations for a node
+
+"""
+    find_fusions!(graph::DAG, node::DataTaskNode)
+
+Find node fusions involving the given data node. The function pushes the found [`NodeFusion`](@ref) (if any) everywhere it needs to be and returns nothing.
+
+Does nothing if the node already has a node fusion set. Since it's a data node, only one node fusion can be possible with it.
+"""
+function find_fusions!(graph::DAG, node::DataTaskNode)
+    # if there is already a fusion here, skip to avoid duplicates
+    if !ismissing(node.nodeFusion)
+        return nothing
+    end
+
+    if length(node.parents) != 1 || length(node.children) != 1
+        return nothing
+    end
+
+    child_node = first(node.children)
+    parent_node = first(node.parents)
+
+    if !(child_node in graph) || !(parent_node in graph)
+        error("Parents/Children that are not in the graph!!!")
+    end
+
+    if length(child_node.parents) != 1
+        return nothing
+    end
+
+    nf = NodeFusion((child_node, node, parent_node))
+    push!(graph.possibleOperations.nodeFusions, nf)
+    push!(child_node.nodeFusions, nf)
+    node.nodeFusion = nf
+    push!(parent_node.nodeFusions, nf)
+
+    return nothing
+end
+
+"""
+    find_fusions!(graph::DAG, node::ComputeTaskNode)
+
+Find node fusions involving the given compute node. The function pushes the found [`NodeFusion`](@ref)s (if any) everywhere they need to be and returns nothing.
+"""
+function find_fusions!(graph::DAG, node::ComputeTaskNode)
+    # just find fusions in neighbouring DataTaskNodes
+    for child in node.children
+        find_fusions!(graph, child)
+    end
+
+    for parent in node.parents
+        find_fusions!(graph, parent)
+    end
+
+    return nothing
+end
+
+"""
+    find_reductions!(graph::DAG, node::Node)
+
+Find node reductions involving the given node. The function pushes the found [`NodeReduction`](@ref) (if any) everywhere it needs to be and returns nothing.
+"""
+function find_reductions!(graph::DAG, node::Node)
+    # there can only be one reduction per node, avoid adding duplicates
+    if !ismissing(node.nodeReduction)
+        return nothing
+    end
+
+    reductionVector = nothing
+    # possible reductions are with nodes that are partners, i.e. parents of children
+    partners_ = partners(node)
+    delete!(partners_, node)
+    for partner in partners_
+        if partner ∉ graph.nodes
+            error("Partner is not part of the graph")
+        end
+
+        if can_reduce(node, partner)
+            if Set(node.children) != Set(partner.children)
+                error("Not equal children")
+            end
+            if reductionVector === nothing
+                # only when there's at least one reduction partner, insert the vector
+                reductionVector = Vector{Node}()
+                push!(reductionVector, node)
+            end
+
+            push!(reductionVector, partner)
+        end
+    end
+
+    if reductionVector !== nothing
+        nr = NodeReduction(reductionVector)
+        push!(graph.possibleOperations.nodeReductions, nr)
+        for node in reductionVector
+            if !ismissing(node.nodeReduction)
+                # it can happen that the dirty node becomes part of an existing NodeReduction and overrides those ones now
+                # this is only a problem insofar the existing NodeReduction has to be deleted and replaced also in the possibleOperations
+                invalidate_caches!(graph, node.nodeReduction)
+            end
+            node.nodeReduction = nr
+        end
+    end
+
+    return nothing
+end
+
+"""
+    find_splits!(graph::DAG, node::Node)
+
+Find the node split of the given node. The function pushes the found [`NodeSplit`](@ref) (if any) everywhere it needs to be and returns nothing.
+"""
+function find_splits!(graph::DAG, node::Node)
+    if !ismissing(node.nodeSplit)
+        return nothing
+    end
+
+    if (can_split(node))
+        ns = NodeSplit(node)
+        push!(graph.possibleOperations.nodeSplits, ns)
+        node.nodeSplit = ns
+    end
+
+    return nothing
+end
+
+"""
+    clean_node!(graph::DAG, node::Node)
+
+Sort this node's parent and child sets, then find fusions, reductions and splits involving it. Needs to be called after the node was changed in some way.
+"""
+function clean_node!(graph::DAG, node::Node)
+    sort_node!(node)
+
+    find_fusions!(graph, node)
+    find_reductions!(graph, node)
+    find_splits!(graph, node)
+
+    return nothing
+end

src/operation/find.jl

@@ -0,0 +1,264 @@
+# functions that find operations on the inital graph
+
+using Base.Threads
+
+"""
+    insert_operation!(nf::NodeFusion, locks::Dict{ComputeTaskNode, SpinLock})
+
+Insert the given node fusion into its input nodes' operation caches. For the compute nodes, locking via the given `locks` is employed to have safe multi-threading. For a large set of nodes, contention on the locks should be very small.
+"""
+function insert_operation!(
+    nf::NodeFusion,
+    locks::Dict{ComputeTaskNode, SpinLock},
+)
+    n1 = nf.input[1]
+    n2 = nf.input[2]
+    n3 = nf.input[3]
+
+    lock(locks[n1]) do
+        return push!(nf.input[1].nodeFusions, nf)
+    end
+    n2.nodeFusion = nf
+    lock(locks[n3]) do
+        return push!(nf.input[3].nodeFusions, nf)
+    end
+    return nothing
+end
+
+"""
+    insert_operation!(nf::NodeReduction)
+
+Insert the given node reduction into its input nodes' operation caches. This is thread-safe.
+"""
+function insert_operation!(nr::NodeReduction)
+    for n in nr.input
+        n.nodeReduction = nr
+    end
+    return nothing
+end
+
+"""
+    insert_operation!(nf::NodeSplit)
+
+Insert the given node split into its input node's operation cache. This is thread-safe.
+"""
+function insert_operation!(ns::NodeSplit)
+    ns.input.nodeSplit = ns
+    return nothing
+end
+
+"""
+    nr_insertion!(operations::PossibleOperations, nodeReductions::Vector{Vector{NodeReduction}})
+
+Insert the node reductions into the graph and the nodes' caches. Employs multithreading for speedup.
+"""
+function nr_insertion!(
+    operations::PossibleOperations,
+    nodeReductions::Vector{Vector{NodeReduction}},
+)
+    total_len = 0
+    for vec in nodeReductions
+        total_len += length(vec)
+    end
+    sizehint!(operations.nodeReductions, total_len)
+
+    t = @task for vec in nodeReductions
+        union!(operations.nodeReductions, Set(vec))
+    end
+    schedule(t)
+
+    @threads for vec in nodeReductions
+        for op in vec
+            insert_operation!(op)
+        end
+    end
+
+    wait(t)
+
+    return nothing
+end
+
+"""
+    nf_insertion!(graph::DAG, operations::PossibleOperations, nodeFusions::Vector{Vector{NodeFusion}})
+
+Insert the node fusions into the graph and the nodes' caches. Employs multithreading for speedup.
+"""
+function nf_insertion!(
+    graph::DAG,
+    operations::PossibleOperations,
+    nodeFusions::Vector{Vector{NodeFusion}},
+)
+    total_len = 0
+    for vec in nodeFusions
+        total_len += length(vec)
+    end
+    sizehint!(operations.nodeFusions, total_len)
+
+    t = @task for vec in nodeFusions
+        union!(operations.nodeFusions, Set(vec))
+    end
+    schedule(t)
+
+    locks = Dict{ComputeTaskNode, SpinLock}()
+    for n in graph.nodes
+        if (typeof(n) <: ComputeTaskNode)
+            locks[n] = SpinLock()
+        end
+    end
+
+    @threads for vec in nodeFusions
+        for op in vec
+            insert_operation!(op, locks)
+        end
+    end
+
+    wait(t)
+
+    return nothing
+end
+
+"""
+    ns_insertion!(operations::PossibleOperations, nodeSplits::Vector{Vector{NodeSplits}})
+
+Insert the node splits into the graph and the nodes' caches. Employs multithreading for speedup.
+"""
+function ns_insertion!(
+    operations::PossibleOperations,
+    nodeSplits::Vector{Vector{NodeSplit}},
+)
+    total_len = 0
+    for vec in nodeSplits
+        total_len += length(vec)
+    end
+    sizehint!(operations.nodeSplits, total_len)
+
+    t = @task for vec in nodeSplits
+        union!(operations.nodeSplits, Set(vec))
+    end
+    schedule(t)
+
+    @threads for vec in nodeSplits
+        for op in vec
+            insert_operation!(op)
+        end
+    end
+
+    wait(t)
+
+    return nothing
+end
+
+"""
+    generate_operations(graph::DAG)
+
+Generate all possible operations on the graph. Used initially when the graph is freshly assembled or parsed. Uses multithreading for speedup.
+
+Safely inserts all the found operations into the graph and its nodes.
+"""
+function generate_operations(graph::DAG)
+    generatedFusions = [Vector{NodeFusion}() for _ in 1:nthreads()]
+    generatedReductions = [Vector{NodeReduction}() for _ in 1:nthreads()]
+    generatedSplits = [Vector{NodeSplit}() for _ in 1:nthreads()]
+
+    # make sure the graph is fully generated through
+    apply_all!(graph)
+
+    nodeArray = collect(graph.nodes)
+
+    # sort all nodes
+    @threads for node in nodeArray
+        sort_node!(node)
+    end
+
+    checkedNodes = Set{Node}()
+    checkedNodesLock = SpinLock()
+    # --- find possible node reductions ---
+    @threads for node in nodeArray
+        # we're looking for nodes with multiple parents, those parents can then potentially reduce with one another
+        if (length(node.parents) <= 1)
+            continue
+        end
+
+        candidates = node.parents
+
+        # sort into equivalence classes
+        trie = NodeTrie()
+
+        for candidate in candidates
+            # insert into trie
+            insert!(trie, candidate)
+        end
+
+        nodeReductions = collect(trie)
+
+        for nrVec in nodeReductions
+            # parent sets are ordered and any node can only be part of one nodeReduction, so a NodeReduction is uniquely identifiable by its first element
+            # this prevents duplicate nodeReductions being generated
+            lock(checkedNodesLock)
+            if (nrVec[1] in checkedNodes)
+                unlock(checkedNodesLock)
+                continue
+            else
+                push!(checkedNodes, nrVec[1])
+            end
+            unlock(checkedNodesLock)
+
+            push!(generatedReductions[threadid()], NodeReduction(nrVec))
+        end
+    end
+
+
+    # launch thread for node reduction insertion
+    # remove duplicates
+    nr_task = @task nr_insertion!(graph.possibleOperations, generatedReductions)
+    schedule(nr_task)
+
+    # --- find possible node fusions ---
+    @threads for node in nodeArray
+        if (typeof(node) <: DataTaskNode)
+            if length(node.parents) != 1
+                # data node can only have a single parent
+                continue
+            end
+            parent_node = first(node.parents)
+
+            if length(node.children) != 1
+                # this node is an entry node or has multiple children which should not be possible
+                continue
+            end
+            child_node = first(node.children)
+            if (length(child_node.parents) != 1)
+                continue
+            end
+
+            push!(
+                generatedFusions[threadid()],
+                NodeFusion((child_node, node, parent_node)),
+            )
+        end
+    end
+
+    # launch thread for node fusion insertion
+    nf_task =
+        @task nf_insertion!(graph, graph.possibleOperations, generatedFusions)
+    schedule(nf_task)
+
+    # find possible node splits
+    @threads for node in nodeArray
+        if (can_split(node))
+            push!(generatedSplits[threadid()], NodeSplit(node))
+        end
+    end
+
+    # launch thread for node split insertion
+    ns_task = @task ns_insertion!(graph.possibleOperations, generatedSplits)
+    schedule(ns_task)
+
+    empty!(graph.dirtyNodes)
+
+    wait(nr_task)
+    wait(nf_task)
+    wait(ns_task)
+
+    return nothing
+end

src/operation/get.jl

@@ -0,0 +1,23 @@
+# function to return the possible operations of a graph
+
+using Base.Threads
+
+"""
+    get_operations(graph::DAG)
+
+Return the [`PossibleOperations`](@ref) of the graph at the current state.
+"""
+function get_operations(graph::DAG)
+    apply_all!(graph)
+
+    if isempty(graph.possibleOperations)
+        generate_operations(graph)
+    end
+
+    for node in graph.dirtyNodes
+        clean_node!(graph, node)
+    end
+    empty!(graph.dirtyNodes)
+
+    return graph.possibleOperations
+end

src/operation/print.jl

@@ -0,0 +1,58 @@
+"""
+    show(io::IO, ops::PossibleOperations)
+
+Print a string representation of the set of possible operations to io.
+"""
+function show(io::IO, ops::PossibleOperations)
+    print(io, length(ops.nodeFusions))
+    println(io, " Node Fusions: ")
+    for nf in ops.nodeFusions
+        println(io, "  - ", nf)
+    end
+    print(io, length(ops.nodeReductions))
+    println(io, " Node Reductions: ")
+    for nr in ops.nodeReductions
+        println(io, "  - ", nr)
+    end
+    print(io, length(ops.nodeSplits))
+    println(io, " Node Splits: ")
+    for ns in ops.nodeSplits
+        println(io, "  - ", ns)
+    end
+end
+
+"""
+    show(io::IO, op::NodeReduction)
+
+Print a string representation of the node reduction to io.
+"""
+function show(io::IO, op::NodeReduction)
+    print(io, "NR: ")
+    print(io, length(op.input))
+    print(io, "x")
+    return print(io, op.input[1].task)
+end
+
+"""
+    show(io::IO, op::NodeSplit)
+
+Print a string representation of the node split to io.
+"""
+function show(io::IO, op::NodeSplit)
+    print(io, "NS: ")
+    return print(io, op.input.task)
+end
+
+"""
+    show(io::IO, op::NodeFusion)
+
+Print a string representation of the node fusion to io.
+"""
+function show(io::IO, op::NodeFusion)
+    print(io, "NF: ")
+    print(io, op.input[1].task)
+    print(io, "->")
+    print(io, op.input[2].task)
+    print(io, "->")
+    return print(io, op.input[3].task)
+end

src/operation/type.jl

@@ -0,0 +1,117 @@
+"""
+    Operation
+
+An abstract base class for operations. An operation can be applied to a [`DAG`](@ref), changing its nodes and edges.
+
+Possible operations on a [`DAG`](@ref) can be retrieved using [`get_operations`](@ref).
+
+See also: [`push_operation!`](@ref), [`pop_operation!`](@ref)
+"""
+abstract type Operation end
+
+"""
+    AppliedOperation
+
+An abstract base class for already applied operations.
+An applied operation can be reversed iff it is the last applied operation on the DAG.
+Every applied operation stores a [`Diff`](@ref) from when it was initially applied to be able to revert the operation.
+
+See also: [`revert_operation!`](@ref).
+"""
+abstract type AppliedOperation end
+
+"""
+    NodeFusion <: Operation
+
+The NodeFusion operation. Represents the fusing of a chain of compute node -> data node -> compute node.
+
+After the node fusion is applied, the graph has 2 fewer nodes and edges, and a new [`FusedComputeTask`](@ref) with the two input compute nodes as parts.
+
+# Requirements for successful application
+
+A chain of (n1, n2, n3) can be fused if:
+- All nodes are in the graph.
+- (n1, n2) is an edge in the graph.
+- (n2, n3) is an edge in the graph.
+- n2 has exactly one parent (n3) and exactly one child (n1).
+- n1 has exactly one parent (n2).
+
+[`is_valid_node_fusion_input`](@ref) can be used to `@assert` these requirements.
+
+See also: [`can_fuse`](@ref)
+"""
+struct NodeFusion <: Operation
+    input::Tuple{ComputeTaskNode, DataTaskNode, ComputeTaskNode}
+end
+
+"""
+    AppliedNodeFusion <: AppliedOperation
+
+The applied version of the [`NodeFusion`](@ref).
+"""
+struct AppliedNodeFusion <: AppliedOperation
+    operation::NodeFusion
+    diff::Diff
+end
+
+"""
+    NodeReduction <: Operation
+
+The NodeReduction operation. Represents the reduction of two or more nodes with one another.
+Only one of the input nodes is kept, while all others are deleted and their parents are accumulated in the kept node's parents instead.
+
+After the node reduction is applied, the graph has `length(nr.input) - 1` fewer nodes.
+
+# Requirements for successful application
+
+A vector of nodes can be reduced if:
+- All nodes are in the graph.
+- All nodes have the same task type.
+- All nodes have the same set of children.
+
+[`is_valid_node_reduction_input`](@ref) can be used to `@assert` these requirements.
+
+See also: [`can_reduce`](@ref)
+"""
+struct NodeReduction <: Operation
+    input::Vector{Node}
+end
+
+"""
+    AppliedNodeReduction <: AppliedOperation
+
+The applied version of the [`NodeReduction`](@ref).
+"""
+struct AppliedNodeReduction <: AppliedOperation
+    operation::NodeReduction
+    diff::Diff
+end
+
+"""
+    NodeSplit <: Operation
+
+The NodeSplit operation. Represents the split of its input node into one node for each of its parents. It is the reverse operation to the [`NodeReduction`](@ref).
+
+# Requirements for successful application
+
+A node can be split if:
+- It is in the graph.
+- It has at least 2 parents.
+
+[`is_valid_node_split_input`](@ref) can be used to `@assert` these requirements.
+
+See also: [`can_split`](@ref)
+"""
+struct NodeSplit <: Operation
+    input::Node
+end
+
+"""
+    AppliedNodeSplit <: AppliedOperation
+
+The applied version of the [`NodeSplit`](@ref).
+"""
+struct AppliedNodeSplit <: AppliedOperation
+    operation::NodeSplit
+    diff::Diff
+end

src/operation/utility.jl

@@ -0,0 +1,167 @@
+"""
+    isempty(operations::PossibleOperations)
+
+Return whether `operations` is empty, i.e. all of its fields are empty.
+"""
+function isempty(operations::PossibleOperations)
+    return isempty(operations.nodeFusions) &&
+           isempty(operations.nodeReductions) &&
+           isempty(operations.nodeSplits)
+end
+
+"""
+    length(operations::PossibleOperations)
+
+Return a named tuple with the number of each of the operation types as a named tuple. The fields are named the same as the [`PossibleOperations`](@ref)'.
+"""
+function length(operations::PossibleOperations)
+    return (
+        nodeFusions = length(operations.nodeFusions),
+        nodeReductions = length(operations.nodeReductions),
+        nodeSplits = length(operations.nodeSplits),
+    )
+end
+
+"""
+    delete!(operations::PossibleOperations, op::NodeFusion)
+
+Delete the given node fusion from the possible operations.
+"""
+function delete!(operations::PossibleOperations, op::NodeFusion)
+    delete!(operations.nodeFusions, op)
+    return operations
+end
+
+"""
+    delete!(operations::PossibleOperations, op::NodeReduction)
+
+Delete the given node reduction from the possible operations.
+"""
+function delete!(operations::PossibleOperations, op::NodeReduction)
+    delete!(operations.nodeReductions, op)
+    return operations
+end
+
+"""
+    delete!(operations::PossibleOperations, op::NodeSplit)
+
+Delete the given node split from the possible operations.
+"""
+function delete!(operations::PossibleOperations, op::NodeSplit)
+    delete!(operations.nodeSplits, op)
+    return operations
+end
+
+"""
+    can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+
+Return whether the given nodes can be fused. See [`NodeFusion`](@ref) for the requirements.
+"""
+function can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+    if !is_child(n1, n2) || !is_child(n2, n3)
+        # the checks are redundant but maybe a good sanity check
+        return false
+    end
+
+    if length(n2.parents) != 1 ||
+       length(n2.children) != 1 ||
+       length(n1.parents) != 1
+        return false
+    end
+
+    return true
+end
+
+"""
+    can_reduce(n1::Node, n2::Node)
+
+Return whether the given two nodes can be reduced. See [`NodeReduction`](@ref) for the requirements.
+"""
+function can_reduce(n1::Node, n2::Node)
+    if (n1.task != n2.task)
+        return false
+    end
+
+    n1_length = length(n1.children)
+    n2_length = length(n2.children)
+
+    if (n1_length != n2_length)
+        return false
+    end
+
+    # this seems to be the most common case so do this first
+    # doing it manually is a lot faster than using the sets for a general solution
+    if (n1_length == 2)
+        if (n1.children[1] != n2.children[1])
+            if (n1.children[1] != n2.children[2])
+                return false
+            end
+            # 1_1 == 2_2
+            if (n1.children[2] != n2.children[1])
+                return false
+            end
+            return true
+        end
+
+        # 1_1 == 2_1
+        if (n1.children[2] != n2.children[2])
+            return false
+        end
+        return true
+    end
+
+    # this is simple
+    if (n1_length == 1)
+        return n1.children[1] == n2.children[1]
+    end
+
+    # this takes a long time
+    return Set(n1.children) == Set(n2.children)
+end
+
+"""
+    can_split(n1::Node)
+
+Return whether the given node can be split. See [`NodeSplit`](@ref) for the requirements.
+"""
+function can_split(n::Node)
+    return length(parents(n)) > 1
+end
+
+"""
+    ==(op1::Operation, op2::Operation)
+
+Fallback implementation of operation equality. Return false. Actual comparisons are done by the overloads of same type operation comparisons.
+"""
+function ==(op1::Operation, op2::Operation)
+    return false
+end
+
+"""
+    ==(op1::NodeFusion, op2::NodeFusion)
+
+Equality comparison between two node fusions. Two node fusions are considered equal if they have the same inputs.
+"""
+function ==(op1::NodeFusion, op2::NodeFusion)
+    # there can only be one node fusion on a given data task, so if the data task is the same, the fusion is the same
+    return op1.input[2] == op2.input[2]
+end
+
+"""
+    ==(op1::NodeReduction, op2::NodeReduction)
+
+Equality comparison between two node reductions. Two node reductions are considered equal when they have the same inputs.
+"""
+function ==(op1::NodeReduction, op2::NodeReduction)
+    # node reductions are equal exactly if their first input is the same
+    return op1.input[1].id == op2.input[1].id
+end
+
+"""
+    ==(op1::NodeSplit, op2::NodeSplit)
+
+Equality comparison between two node splits. Two node splits are considered equal if they have the same input node.
+"""
+function ==(op1::NodeSplit, op2::NodeSplit)
+    return op1.input == op2.input
+end

src/operation/validate.jl

@@ -0,0 +1,174 @@
+# functions to throw assertion errors for inconsistent or wrong node operations
+# should be called with @assert
+# the functions throw their own errors though, to still have helpful error messages
+
+"""
+    is_valid_node_fusion_input(graph::DAG, n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+
+Assert for a gven node fusion input whether the nodes can be fused. For the requirements of a node fusion see [`NodeFusion`](@ref).
+
+Intended for use with `@assert` or `@test`.
+"""
+function is_valid_node_fusion_input(
+    graph::DAG,
+    n1::ComputeTaskNode,
+    n2::DataTaskNode,
+    n3::ComputeTaskNode,
+)
+    if !(n1 in graph) || !(n2 in graph) || !(n3 in graph)
+        throw(
+            AssertionError(
+                "[Node Fusion] The given nodes are not part of the given graph",
+            ),
+        )
+    end
+
+    if !is_child(n1, n2) ||
+       !is_child(n2, n3) ||
+       !is_parent(n3, n2) ||
+       !is_parent(n2, n1)
+        throw(
+            AssertionError(
+                "[Node Fusion] The given nodes are not connected by edges which is required for node fusion",
+            ),
+        )
+    end
+
+    if length(n2.parents) > 1
+        throw(
+            AssertionError(
+                "[Node Fusion] The given data node has more than one parent",
+            ),
+        )
+    end
+    if length(n2.children) > 1
+        throw(
+            AssertionError(
+                "[Node Fusion] The given data node has more than one child",
+            ),
+        )
+    end
+    if length(n1.parents) > 1
+        throw(
+            AssertionError(
+                "[Node Fusion] The given n1 has more than one parent",
+            ),
+        )
+    end
+
+    return true
+end
+
+"""
+    is_valid_node_reduction_input(graph::DAG, nodes::Vector{Node})
+
+Assert for a gven node reduction input whether the nodes can be reduced. For the requirements of a node reduction see [`NodeReduction`](@ref).
+
+Intended for use with `@assert` or `@test`.
+"""
+function is_valid_node_reduction_input(graph::DAG, nodes::Vector{Node})
+    for n in nodes
+        if n ∉ graph
+            throw(
+                AssertionError(
+                    "[Node Reduction] The given nodes are not part of the given graph",
+                ),
+            )
+        end
+    end
+
+    t = typeof(nodes[1].task)
+    for n in nodes
+        if typeof(n.task) != t
+            throw(
+                AssertionError(
+                    "[Node Reduction] The given nodes are not of the same type",
+                ),
+            )
+        end
+    end
+
+    n1_children = nodes[1].children
+    for n in nodes
+        if Set(n1_children) != Set(n.children)
+            throw(
+                AssertionError(
+                    "[Node Reduction] The given nodes do not have equal prerequisite nodes which is required for node reduction",
+                ),
+            )
+        end
+    end
+
+    return true
+end
+
+"""
+    is_valid_node_split_input(graph::DAG, n1::Node)
+
+Assert for a gven node split input whether the node can be split. For the requirements of a node split see [`NodeSplit`](@ref).
+
+Intended for use with `@assert` or `@test`.
+"""
+function is_valid_node_split_input(graph::DAG, n1::Node)
+    if n1 ∉ graph
+        throw(
+            AssertionError(
+                "[Node Split] The given node is not part of the given graph",
+            ),
+        )
+    end
+
+    if length(n1.parents) <= 1
+        throw(
+            AssertionError(
+                "[Node Split] The given node does not have multiple parents which is required for node split",
+            ),
+        )
+    end
+
+    return true
+end
+
+"""
+    is_valid(graph::DAG, nr::NodeReduction)
+
+Assert for a given [`NodeReduction`](@ref) whether it is a valid operation in the graph.
+
+Intended for use with `@assert` or `@test`.
+"""
+function is_valid(graph::DAG, nr::NodeReduction)
+    @assert is_valid_node_reduction_input(graph, nr.input)
+    @assert nr in graph.possibleOperations.nodeReductions "NodeReduction is not part of the graph's possible operations!"
+    return true
+end
+
+"""
+    is_valid(graph::DAG, nr::NodeSplit)
+
+Assert for a given [`NodeSplit`](@ref) whether it is a valid operation in the graph.
+
+Intended for use with `@assert` or `@test`.
+"""
+function is_valid(graph::DAG, ns::NodeSplit)
+    @assert is_valid_node_split_input(graph, ns.input)
+    @assert ns in graph.possibleOperations.nodeSplits "NodeSplit is not part of the graph's possible operations!"
+    return true
+end
+
+"""
+    is_valid(graph::DAG, nr::NodeFusion)
+
+Assert for a given [`NodeFusion`](@ref) whether it is a valid operation in the graph.
+
+Intended for use with `@assert` or `@test`.
+"""
+function is_valid(graph::DAG, nf::NodeFusion)
+    @assert is_valid_node_fusion_input(
+        graph,
+        nf.input[1],
+        nf.input[2],
+        nf.input[3],
+    )
+    @assert nf in graph.possibleOperations.nodeFusions "NodeFusion is not part of the graph's possible operations!"
+    return true
+end

src/task/compare.jl

@@ -0,0 +1,26 @@
+"""
+    ==(t1::AbstractTask, t2::AbstractTask)
+
+Fallback implementation of equality comparison between two abstract tasks. Always returns false. For equal specific types of t1 and t2, a more specific comparison is called instead, doing an actual comparison.
+"""
+function ==(t1::AbstractTask, t2::AbstractTask)
+    return false
+end
+
+"""
+    ==(t1::AbstractComputeTask, t2::AbstractComputeTask)
+
+Equality comparison between two compute tasks.
+"""
+function ==(t1::AbstractComputeTask, t2::AbstractComputeTask)
+    return typeof(t1) == typeof(t2)
+end
+
+"""
+    ==(t1::AbstractDataTask, t2::AbstractDataTask)
+
+Equality comparison between two data tasks.
+"""
+function ==(t1::AbstractDataTask, t2::AbstractDataTask)
+    return data(t1) == data(t2)
+end

src/task/create.jl

@@ -0,0 +1,14 @@
+"""
+    copy(t::AbstractDataTask)
+
+Fallback implementation of the copy of an abstract data task, throwing an error.
+"""
+copy(t::AbstractDataTask) =
+    error("Need to implement copying for your data tasks!")
+
+"""
+    copy(t::AbstractComputeTask)
+
+Return a copy of the given compute task.
+"""
+copy(t::AbstractComputeTask) = typeof(t)()

src/task/print.jl

@@ -0,0 +1,9 @@
+"""
+    show(io::IO, t::FusedComputeTask)
+    
+Print a string representation of the fused compute task to io.
+"""
+function show(io::IO, t::FusedComputeTask)
+    (T1, T2) = get_types(t)
+    return print(io, "ComputeFuse(", T1(), ", ", T2(), ")")
+end

src/task/properties.jl

@@ -0,0 +1,75 @@
+"""
+    compute(t::AbstractTask; data...)
+
+Fallback implementation of the compute function of a compute task, throwing an error.
+"""
+function compute(t::AbstractTask; data...)
+    return error("Need to implement compute()")
+end
+
+"""
+    compute_effort(t::AbstractTask)
+
+Fallback implementation of the compute effort of a task, throwing an error.
+"""
+function compute_effort(t::AbstractTask)
+    # default implementation using compute
+    return error("Need to implement compute_effort()")
+end
+
+"""
+    data(t::AbstractTask)
+
+Fallback implementation of the data of a task, throwing an error.
+"""
+function data(t::AbstractTask)
+    return error("Need to implement data()")
+end
+
+"""
+    compute_effort(t::AbstractDataTask)
+
+Return the compute effort of a data task, always zero, regardless of the specific task.
+"""
+compute_effort(t::AbstractDataTask) = 0
+
+"""
+    compute(t::AbstractDataTask; data...)
+
+The compute function of a data task, always the identity function, regardless of the specific task.
+"""
+compute(t::AbstractDataTask; data...) = data
+
+"""
+    data(t::AbstractDataTask)
+
+Return the data of a data task. Given by the task's `.data` field.
+"""
+data(t::AbstractDataTask) = getfield(t, :data)
+
+"""
+    data(t::AbstractComputeTask)
+
+Return the data of a compute task, always zero, regardless of the specific task.
+"""
+data(t::AbstractComputeTask) = 0
+
+"""
+    compute_effort(t::FusedComputeTask)
+
+Return the compute effort of a fused compute task. 
+"""
+function compute_effort(t::FusedComputeTask)
+    (T1, T2) = collect(typeof(t).parameters)
+    return compute_effort(T1()) + compute_effort(T2())
+end
+
+# actual compute functions for the tasks can stay undefined for now
+# compute(t::ComputeTaskU, data::Any) = mycomputation(data)
+
+"""
+    get_types(::FusedComputeTask{T1, T2})
+
+Return a tuple of a the fused compute task's components' types.
+"""
+get_types(::FusedComputeTask{T1, T2}) where {T1, T2} = (T1, T2)

src/task/type.jl

@@ -0,0 +1,30 @@
+"""
+    AbstractTask
+
+The shared base type for any task.
+"""
+abstract type AbstractTask end
+
+"""
+    AbstractComputeTask <: AbstractTask
+
+The shared base type for any compute task.
+"""
+abstract type AbstractComputeTask <: AbstractTask end
+
+"""
+    AbstractDataTask <: AbstractTask
+
+The shared base type for any data task.
+"""
+abstract type AbstractDataTask <: AbstractTask end
+
+"""
+    FusedComputeTask{T1 <: AbstractComputeTask, T2 <: AbstractComputeTask} <: AbstractComputeTask
+
+A fused compute task made up of the computation of first `T1` and then `T2`.
+
+Also see: [`get_types`](@ref).
+"""
+struct FusedComputeTask{T1 <: AbstractComputeTask, T2 <: AbstractComputeTask} <:
+       AbstractComputeTask end

src/task_functions.jl

@@ -1,53 +0,0 @@
-function compute(t::AbstractTask; data...)
-   error("Need to implement compute()")
-end
-
-function compute_effort(t::AbstractTask)
-   # default implementation using compute
-   error("Need to implement compute_effort()")
-end
-
-function data(t::AbstractTask)
-   error("Need to implement data()")
-end
-
-compute_effort(t::AbstractDataTask) = 0
-compute(t::AbstractDataTask; data...) = data
-data(t::AbstractDataTask) = getfield(t, :data)
-
-data(t::AbstractComputeTask) = 0
-
-function compute_effort(t::FusedComputeTask)
-   (T1, T2) = collect(typeof(t).parameters)
-   return compute_effort(T1()) + compute_effort(T2())
-end
-
-# actual compute functions for the tasks can stay undefined for now
-# compute(t::ComputeTaskU, data::Any) = mycomputation(data)
-
-function compute_intensity(t::AbstractTask)::UInt64
-   if data(t) == 0
-      return typemax(UInt64)
-   end
-   return compute_effort(t) / data(t)
-end
-
-function show(io::IO, t::FusedComputeTask)
-   (T1, T2) = get_types(t)
-   print(io, "ComputeFuse(", T1(), ", ", T2(), ")")
-end
-
-function ==(t1::AbstractTask, t2::AbstractTask)
-   return false
-end
-
-function ==(t1::AbstractComputeTask, t2::AbstractComputeTask)
-   return typeof(t1) == typeof(t2)
-end
-
-function ==(t1::AbstractDataTask, t2::AbstractDataTask)
-   return data(t1) == data(t2)
-end
-
-copy(t::AbstractDataTask) = error("Need to implement copying for your data tasks!")
-copy(t::AbstractComputeTask) = typeof(t)()

src/tasks.jl

@@ -1,9 +0,0 @@
-abstract type AbstractTask end
-
-abstract type AbstractComputeTask  <: AbstractTask end
-abstract type AbstractDataTask     <: AbstractTask end
-
-struct FusedComputeTask{T1<:AbstractComputeTask, T2<:AbstractComputeTask} <: AbstractComputeTask
-end
-
-get_types(::FusedComputeTask{T1, T2}) where {T1, T2} = (T1, T2)

src/trie.jl

@@ -0,0 +1,102 @@
+"""
+    NodeIdTrie
+
+Helper struct for [`NodeTrie`](@ref). After the Trie's first level, every Trie level contains the vector of nodes that had children up to that level, and the TrieNode's children by UUID of the node's children.
+"""
+mutable struct NodeIdTrie
+    value::Vector{Node}
+    children::Dict{UUID, NodeIdTrie}
+end
+
+"""
+    NodeTrie
+
+Trie data structure for node reduction, inserts nodes by children.
+Assumes that given nodes have ordered vectors of children (see [`sort_node!`](@ref)).
+First insertion level is the node's own task type and thus does not have a value (every node has a task type).
+
+See also: [`insert!`](@ref) and [`collect`](@ref)
+"""
+mutable struct NodeTrie
+    children::Dict{DataType, NodeIdTrie}
+end
+
+"""
+    NodeTrie()
+
+Constructor for an empty [`NodeTrie`](@ref).
+"""
+function NodeTrie()
+    return NodeTrie(Dict{DataType, NodeIdTrie}())
+end
+
+"""
+    NodeIdTrie()
+
+Constructor for an empty [`NodeIdTrie`](@ref).
+"""
+function NodeIdTrie()
+    return NodeIdTrie(Vector{Node}(), Dict{UUID, NodeIdTrie}())
+end
+
+"""
+    insert_helper!(trie::NodeIdTrie, node::Node, depth::Int)
+
+Insert the given node into the trie. The depth is used to iterate through the trie layers, while the function calls itself recursively until it ran through all children of the node.
+"""
+function insert_helper!(trie::NodeIdTrie, node::Node, depth::Int)
+    if (length(node.children) == depth)
+        push!(trie.value, node)
+        return nothing
+    end
+
+    depth = depth + 1
+    id = node.children[depth].id
+
+    if (!haskey(trie.children, id))
+        trie.children[id] = NodeIdTrie()
+    end
+    return insert_helper!(trie.children[id], node, depth)
+end
+
+"""
+    insert!(trie::NodeTrie, node::Node)
+
+Insert the given node into the trie. It's sorted by its type in the first layer, then by its children in the following layers.
+"""
+function insert!(trie::NodeTrie, node::Node)
+    t = typeof(node.task)
+    if (!haskey(trie.children, t))
+        trie.children[t] = NodeIdTrie()
+    end
+    return insert_helper!(trie.children[typeof(node.task)], node, 0)
+end
+
+"""
+    collect_helper(trie::NodeIdTrie, acc::Set{Vector{Node}})
+
+Collects the Vectors of this [`NodeIdTrie`](@ref) node and all its children and puts them in the `acc` argument.
+"""
+function collect_helper(trie::NodeIdTrie, acc::Set{Vector{Node}})
+    if (length(trie.value) >= 2)
+        push!(acc, trie.value)
+    end
+
+    for (id, child) in trie.children
+        collect_helper(child, acc)
+    end
+    return nothing
+end
+
+"""
+    collect(trie::NodeTrie)
+
+Return all sets of at least 2 [`Node`](@ref)s that have accumulated in leaves of the trie.
+"""
+function collect(trie::NodeTrie)
+    acc = Set{Vector{Node}}()
+    for (t, child) in trie.children
+        collect_helper(child, acc)
+    end
+    return acc
+end

src/utility.jl

@@ -1,9 +1,89 @@
-function bytes_to_human_readable(bytes::Int64)
+"""
+    bytes_to_human_readable(bytes)
+
+Return a human readable string representation of the given number.
+
+```jldoctest
+julia> using MetagraphOptimization
+
+julia> bytes_to_human_readable(4096)
+"4.0 KiB"
+```
+"""
+function bytes_to_human_readable(bytes)
     units = ["B", "KiB", "MiB", "GiB", "TiB"]
     unit_index = 1
     while bytes >= 1024 && unit_index < length(units)
         bytes /= 1024
         unit_index += 1
     end
-    return string(round(bytes, sigdigits=4), " ", units[unit_index])
+    return string(round(bytes, sigdigits = 4), " ", units[unit_index])
+end
+
+"""
+    lt_nodes(n1::Node, n2::Node)
+
+Less-Than comparison between nodes. Uses the nodes' ids to sort.
+"""
+function lt_nodes(n1::Node, n2::Node)
+    return n1.id < n2.id
+end
+
+"""
+    sort_node!(node::Node)
+
+Sort the nodes' parents and children vectors. The vectors are mostly very short so sorting does not take a lot of time.
+Sorted nodes are required to make the finding of [`NodeReduction`](@ref)s a lot faster using the [`NodeTrie`](@ref) data structure.
+"""
+function sort_node!(node::Node)
+    sort!(node.children, lt = lt_nodes)
+    return sort!(node.parents, lt = lt_nodes)
+end
+
+"""
+    mem(graph::DAG)
+
+Return the memory footprint of the graph in Byte. Should be the same result as `Base.summarysize(graph)` but a lot faster.
+"""
+function mem(graph::DAG)
+    size = 0
+    size += Base.summarysize(graph.nodes, exclude = Union{Node})
+    for n in graph.nodes
+        size += mem(n)
+    end
+
+    size += sizeof(graph.appliedOperations)
+    size += sizeof(graph.operationsToApply)
+
+    size += sizeof(graph.possibleOperations)
+    for op in graph.possibleOperations.nodeFusions
+        size += mem(op)
+    end
+    for op in graph.possibleOperations.nodeReductions
+        size += mem(op)
+    end
+    for op in graph.possibleOperations.nodeSplits
+        size += mem(op)
+    end
+
+    size += Base.summarysize(graph.dirtyNodes, exclude = Union{Node})
+    return size += sizeof(diff)
+end
+
+"""
+    mem(op::Operation)
+
+Return the memory footprint of the operation in Byte. Used in [`mem(graph::DAG)`](@ref). Unlike `Base.summarysize()` this doesn't follow all references which would yield (almost) the size of the entire graph.
+"""
+function mem(op::Operation)
+    return Base.summarysize(op, exclude = Union{Node})
+end
+
+"""
+    mem(op::Operation)
+
+Return the memory footprint of the node in Byte. Used in [`mem(graph::DAG)`](@ref). Unlike `Base.summarysize()` this doesn't follow all references which would yield (almost) the size of the entire graph.
+"""
+function mem(node::Node)
+    return Base.summarysize(node, exclude = Union{Node, Operation})
 end

test/known_graphs.jl

@@ -1,85 +1,89 @@
 using Random
 
-function test_known_graph(name::String, n, fusion_test=true)
-@testset "Test $name Graph ($n)" begin
-    graph = parse_abc(joinpath(@__DIR__, "..", "examples", "$name.txt"))
-    props = graph_properties(graph)
+function test_known_graph(name::String, n, fusion_test = true)
+    @testset "Test $name Graph ($n)" begin
+        graph = parse_abc(joinpath(@__DIR__, "..", "input", "$name.txt"))
+        props = graph_properties(graph)
 
-    if (fusion_test) 
-        test_node_fusion(graph)
+        if (fusion_test)
+            test_node_fusion(graph)
+        end
+        test_random_walk(graph, n)
     end
-    test_random_walk(graph, n)
-end
 end
 
 function test_node_fusion(g::DAG)
-@testset "Test Node Fusion" begin
-    props = graph_properties(g)
-
-    options = get_operations(g)
-
-    nodes_number = length(g.nodes)
-    data = props.data
-    compute_effort = props.compute_effort
-
-    while !isempty(options.nodeFusions)
-        fusion = first(options.nodeFusions)
-
-        @test typeof(fusion) <: NodeFusion
-
-        push_operation!(g, fusion)
-
+    @testset "Test Node Fusion" begin
         props = graph_properties(g)
-        @test props.data < data
-        @test props.compute_effort == compute_effort
+
+        options = get_operations(g)
 
         nodes_number = length(g.nodes)
         data = props.data
         compute_effort = props.compute_effort
 
-        options = get_operations(g)
+        while !isempty(options.nodeFusions)
+            fusion = first(options.nodeFusions)
+
+            @test typeof(fusion) <: NodeFusion
+
+            push_operation!(g, fusion)
+
+            props = graph_properties(g)
+            @test props.data < data
+            @test props.compute_effort == compute_effort
+
+            nodes_number = length(g.nodes)
+            data = props.data
+            compute_effort = props.compute_effort
+
+            options = get_operations(g)
+        end
     end
 end
-end
 
 function test_random_walk(g::DAG, n::Int64)
-@testset "Test Random Walk ($n)" begin
-    # the purpose here is to do "random" operations and reverse them again and validate that the graph stays the same and doesn't diverge
-    reset_graph!(g)
+    @testset "Test Random Walk ($n)" begin
+        # the purpose here is to do "random" operations and reverse them again and validate that the graph stays the same and doesn't diverge
+        reset_graph!(g)
 
-    properties = graph_properties(g)
+        @test is_valid(g)
 
-    for i = 1:n
-        # choose push or pop
-        if rand(Bool)
-            # push
-            opt = get_operations(g)
+        properties = graph_properties(g)
 
-            # choose one of fuse/split/reduce
-            option = rand(1:3)
-            if option == 1 && !isempty(opt.nodeFusions)
-                push_operation!(g, rand(collect(opt.nodeFusions)))
-            elseif option == 2 && !isempty(opt.nodeReductions)
-                push_operation!(g, rand(collect(opt.nodeReductions)))
-            elseif option == 3 && !isempty(opt.nodeSplits)
-                push_operation!(g, rand(collect(opt.nodeSplits)))
+        for i in 1:n
+            # choose push or pop
+            if rand(Bool)
+                # push
+                opt = get_operations(g)
+
+                # choose one of fuse/split/reduce
+                option = rand(1:3)
+                if option == 1 && !isempty(opt.nodeFusions)
+                    push_operation!(g, rand(collect(opt.nodeFusions)))
+                elseif option == 2 && !isempty(opt.nodeReductions)
+                    push_operation!(g, rand(collect(opt.nodeReductions)))
+                elseif option == 3 && !isempty(opt.nodeSplits)
+                    push_operation!(g, rand(collect(opt.nodeSplits)))
+                else
+                    i = i - 1
+                end
             else
-                i = i - 1
-            end
-        else
-            # pop
-            if (can_pop(g))
-                pop_operation!(g)
-            else
-                i = i - 1
+                # pop
+                if (can_pop(g))
+                    pop_operation!(g)
+                else
+                    i = i - 1
+                end
             end
         end
+
+        reset_graph!(g)
+
+        @test is_valid(g)
+
+        @test properties == graph_properties(g)
     end
-
-    reset_graph!(g)
-
-    @test properties == graph_properties(g)
-end
 end
 
 Random.seed!(0)

test/node_reduction.jl

@@ -0,0 +1,99 @@
+import MetagraphOptimization.insert_node!
+import MetagraphOptimization.insert_edge!
+import MetagraphOptimization.make_node
+
+@testset "Unit Tests Node Reduction" begin
+    graph = MetagraphOptimization.DAG()
+
+    d_exit = insert_node!(graph, make_node(DataTask(10)), false)
+
+    s0 = insert_node!(graph, make_node(ComputeTaskS2()), false)
+
+    ED = insert_node!(graph, make_node(DataTask(3)), false)
+    FD = insert_node!(graph, make_node(DataTask(3)), false)
+
+    EC = insert_node!(graph, make_node(ComputeTaskV()), false)
+    FC = insert_node!(graph, make_node(ComputeTaskV()), false)
+
+    A1D = insert_node!(graph, make_node(DataTask(4)), false)
+    B1D_1 = insert_node!(graph, make_node(DataTask(4)), false)
+    B1D_2 = insert_node!(graph, make_node(DataTask(4)), false)
+    C1D = insert_node!(graph, make_node(DataTask(4)), false)
+
+    A1C = insert_node!(graph, make_node(ComputeTaskU()), false)
+    B1C_1 = insert_node!(graph, make_node(ComputeTaskU()), false)
+    B1C_2 = insert_node!(graph, make_node(ComputeTaskU()), false)
+    C1C = insert_node!(graph, make_node(ComputeTaskU()), false)
+
+    AD = insert_node!(graph, make_node(DataTask(5)), false)
+    BD = insert_node!(graph, make_node(DataTask(5)), false)
+    CD = insert_node!(graph, make_node(DataTask(5)), false)
+
+    insert_edge!(graph, s0, d_exit, false)
+    insert_edge!(graph, ED, s0, false)
+    insert_edge!(graph, FD, s0, false)
+    insert_edge!(graph, EC, ED, false)
+    insert_edge!(graph, FC, FD, false)
+
+    insert_edge!(graph, A1D, EC, false)
+    insert_edge!(graph, B1D_1, EC, false)
+
+    insert_edge!(graph, B1D_2, FC, false)
+    insert_edge!(graph, C1D, FC, false)
+
+    insert_edge!(graph, A1C, A1D, false)
+    insert_edge!(graph, B1C_1, B1D_1, false)
+    insert_edge!(graph, B1C_2, B1D_2, false)
+    insert_edge!(graph, C1C, C1D, false)
+
+    insert_edge!(graph, AD, A1C, false)
+    insert_edge!(graph, BD, B1C_1, false)
+    insert_edge!(graph, BD, B1C_2, false)
+    insert_edge!(graph, CD, C1C, false)
+
+    @test is_valid(graph)
+
+    @test is_exit_node(d_exit)
+    @test is_entry_node(AD)
+    @test is_entry_node(BD)
+    @test is_entry_node(CD)
+
+    opt = get_operations(graph)
+
+    @test length(opt) == (nodeFusions = 6, nodeReductions = 1, nodeSplits = 1)
+
+    #println("Initial State:\n", opt)
+
+    nr = first(opt.nodeReductions)
+    @test Set(nr.input) == Set([B1C_1, B1C_2])
+    push_operation!(graph, nr)
+    opt = get_operations(graph)
+
+    @test length(opt) == (nodeFusions = 4, nodeReductions = 1, nodeSplits = 1)
+    #println("After 1 Node Reduction:\n", opt)
+
+    nr = first(opt.nodeReductions)
+    @test Set(nr.input) == Set([B1D_1, B1D_2])
+    push_operation!(graph, nr)
+    opt = get_operations(graph)
+
+    @test is_valid(graph)
+
+    @test length(opt) == (nodeFusions = 4, nodeReductions = 0, nodeSplits = 1)
+    #println("After 2 Node Reductions:\n", opt)
+
+    pop_operation!(graph)
+
+    opt = get_operations(graph)
+    @test length(opt) == (nodeFusions = 4, nodeReductions = 1, nodeSplits = 1)
+    #println("After reverting the second Node Reduction:\n", opt)
+
+    reset_graph!(graph)
+
+    opt = get_operations(graph)
+    @test length(opt) == (nodeFusions = 6, nodeReductions = 1, nodeSplits = 1)
+    #println("After reverting to the initial state:\n", opt)
+
+    @test is_valid(graph)
+end
+println("Node Reduction Unit Tests Complete!")

test/runtests.jl

@@ -2,10 +2,11 @@ using MetagraphOptimization
 using Test
 
 @testset "MetagraphOptimization Tests" begin
-   include("unit_tests_utility.jl")
-   include("unit_tests_tasks.jl")
-   include("unit_tests_nodes.jl")
-   include("unit_tests_graph.jl")
+    include("unit_tests_utility.jl")
+    include("unit_tests_tasks.jl")
+    include("unit_tests_nodes.jl")
+    include("node_reduction.jl")
+    include("unit_tests_graph.jl")
 
-   include("known_graphs.jl")
+    include("known_graphs.jl")
 end

test/unit_tests_graph.jl

@@ -1,210 +1,221 @@
 import MetagraphOptimization.insert_node!
 import MetagraphOptimization.insert_edge!
 import MetagraphOptimization.make_node
-import MetagraphOptimization.make_edge
 import MetagraphOptimization.siblings
 import MetagraphOptimization.partners
 
 @testset "Unit Tests Graph" begin
-   graph = MetagraphOptimization.DAG()
+    graph = MetagraphOptimization.DAG()
 
-   @test length(graph.nodes) == 0
-   @test length(graph.appliedOperations) == 0
-   @test length(graph.operationsToApply) == 0
-   @test length(graph.dirtyNodes) == 0
-   @test length(graph.diff) == (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
-   @test length(get_operations(graph)) == (nodeFusions = 0, nodeReductions = 0, nodeSplits = 0)
+    @test length(graph.nodes) == 0
+    @test length(graph.appliedOperations) == 0
+    @test length(graph.operationsToApply) == 0
+    @test length(graph.dirtyNodes) == 0
+    @test length(graph.diff) ==
+          (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
+    @test length(get_operations(graph)) ==
+          (nodeFusions = 0, nodeReductions = 0, nodeSplits = 0)
 
-   # s to output (exit node)
-   d_exit = insert_node!(graph, make_node(DataTask(10)), false)
+    # s to output (exit node)
+    d_exit = insert_node!(graph, make_node(DataTask(10)), false)
 
-   @test length(graph.nodes) == 1
-   @test length(graph.dirtyNodes) == 1
+    @test length(graph.nodes) == 1
+    @test length(graph.dirtyNodes) == 1
 
-   # final s compute
-   s0 = insert_node!(graph, make_node(ComputeTaskS2()), false)
+    # final s compute
+    s0 = insert_node!(graph, make_node(ComputeTaskS2()), false)
 
-   @test length(graph.nodes) == 2
-   @test length(graph.dirtyNodes) == 2
+    @test length(graph.nodes) == 2
+    @test length(graph.dirtyNodes) == 2
 
-   # data from v0 and v1 to s0
-   d_v0_s0 = insert_node!(graph, make_node(DataTask(5)), false)
-   d_v1_s0 = insert_node!(graph, make_node(DataTask(5)), false)
+    # data from v0 and v1 to s0
+    d_v0_s0 = insert_node!(graph, make_node(DataTask(5)), false)
+    d_v1_s0 = insert_node!(graph, make_node(DataTask(5)), false)
 
-   # v0 and v1 compute
-   v0 = insert_node!(graph, make_node(ComputeTaskV()), false)
-   v1 = insert_node!(graph, make_node(ComputeTaskV()), false)
+    # v0 and v1 compute
+    v0 = insert_node!(graph, make_node(ComputeTaskV()), false)
+    v1 = insert_node!(graph, make_node(ComputeTaskV()), false)
 
-   # data from uB, uA, uBp and uAp to v0 and v1
-   d_uB_v0 = insert_node!(graph, make_node(DataTask(3)), false)
-   d_uA_v0 = insert_node!(graph, make_node(DataTask(3)), false)
-   d_uBp_v1 = insert_node!(graph, make_node(DataTask(3)), false)
-   d_uAp_v1 = insert_node!(graph, make_node(DataTask(3)), false)
+    # data from uB, uA, uBp and uAp to v0 and v1
+    d_uB_v0 = insert_node!(graph, make_node(DataTask(3)), false)
+    d_uA_v0 = insert_node!(graph, make_node(DataTask(3)), false)
+    d_uBp_v1 = insert_node!(graph, make_node(DataTask(3)), false)
+    d_uAp_v1 = insert_node!(graph, make_node(DataTask(3)), false)
 
-   # uB, uA, uBp and uAp computes
-   uB = insert_node!(graph, make_node(ComputeTaskU()), false)
-   uA = insert_node!(graph, make_node(ComputeTaskU()), false)
-   uBp = insert_node!(graph, make_node(ComputeTaskU()), false)
-   uAp = insert_node!(graph, make_node(ComputeTaskU()), false)
+    # uB, uA, uBp and uAp computes
+    uB = insert_node!(graph, make_node(ComputeTaskU()), false)
+    uA = insert_node!(graph, make_node(ComputeTaskU()), false)
+    uBp = insert_node!(graph, make_node(ComputeTaskU()), false)
+    uAp = insert_node!(graph, make_node(ComputeTaskU()), false)
 
-   # data from PB, PA, PBp and PAp to uB, uA, uBp and uAp
-   d_PB_uB = insert_node!(graph, make_node(DataTask(6)), false)
-   d_PA_uA = insert_node!(graph, make_node(DataTask(6)), false)
-   d_PBp_uBp = insert_node!(graph, make_node(DataTask(6)), false)
-   d_PAp_uAp = insert_node!(graph, make_node(DataTask(6)), false)
+    # data from PB, PA, PBp and PAp to uB, uA, uBp and uAp
+    d_PB_uB = insert_node!(graph, make_node(DataTask(6)), false)
+    d_PA_uA = insert_node!(graph, make_node(DataTask(6)), false)
+    d_PBp_uBp = insert_node!(graph, make_node(DataTask(6)), false)
+    d_PAp_uAp = insert_node!(graph, make_node(DataTask(6)), false)
 
-   # P computes PB, PA, PBp and PAp
-   PB = insert_node!(graph, make_node(ComputeTaskP()), false)
-   PA = insert_node!(graph, make_node(ComputeTaskP()), false)
-   PBp = insert_node!(graph, make_node(ComputeTaskP()), false)
-   PAp = insert_node!(graph, make_node(ComputeTaskP()), false)
+    # P computes PB, PA, PBp and PAp
+    PB = insert_node!(graph, make_node(ComputeTaskP()), false)
+    PA = insert_node!(graph, make_node(ComputeTaskP()), false)
+    PBp = insert_node!(graph, make_node(ComputeTaskP()), false)
+    PAp = insert_node!(graph, make_node(ComputeTaskP()), false)
 
-   # entry nodes getting data for P computes
-   d_PB = insert_node!(graph, make_node(DataTask(4)), false)
-   d_PA = insert_node!(graph, make_node(DataTask(4)), false)
-   d_PBp = insert_node!(graph, make_node(DataTask(4)), false)
-   d_PAp = insert_node!(graph, make_node(DataTask(4)), false)
+    # entry nodes getting data for P computes
+    d_PB = insert_node!(graph, make_node(DataTask(4)), false)
+    d_PA = insert_node!(graph, make_node(DataTask(4)), false)
+    d_PBp = insert_node!(graph, make_node(DataTask(4)), false)
+    d_PAp = insert_node!(graph, make_node(DataTask(4)), false)
 
-   @test length(graph.nodes) == 26
-   @test length(graph.dirtyNodes) == 26
+    @test length(graph.nodes) == 26
+    @test length(graph.dirtyNodes) == 26
 
-   # now for all the edgese
-   insert_edge!(graph, make_edge(d_PB, PB), false)
-   insert_edge!(graph, make_edge(d_PA, PA), false)
-   insert_edge!(graph, make_edge(d_PBp, PBp), false)
-   insert_edge!(graph, make_edge(d_PAp, PAp), false)
+    # now for all the edgese
+    insert_edge!(graph, d_PB, PB, false)
+    insert_edge!(graph, d_PA, PA, false)
+    insert_edge!(graph, d_PBp, PBp, false)
+    insert_edge!(graph, d_PAp, PAp, false)
 
-   insert_edge!(graph, make_edge(PB, d_PB_uB), false)
-   insert_edge!(graph, make_edge(PA, d_PA_uA), false)
-   insert_edge!(graph, make_edge(PBp, d_PBp_uBp), false)
-   insert_edge!(graph, make_edge(PAp, d_PAp_uAp), false)
-   
-   insert_edge!(graph, make_edge(d_PB_uB, uB), false)
-   insert_edge!(graph, make_edge(d_PA_uA, uA), false)
-   insert_edge!(graph, make_edge(d_PBp_uBp, uBp), false)
-   insert_edge!(graph, make_edge(d_PAp_uAp, uAp), false)
+    insert_edge!(graph, PB, d_PB_uB, false)
+    insert_edge!(graph, PA, d_PA_uA, false)
+    insert_edge!(graph, PBp, d_PBp_uBp, false)
+    insert_edge!(graph, PAp, d_PAp_uAp, false)
 
-   insert_edge!(graph, make_edge(uB, d_uB_v0), false)
-   insert_edge!(graph, make_edge(uA, d_uA_v0), false)
-   insert_edge!(graph, make_edge(uBp, d_uBp_v1), false)
-   insert_edge!(graph, make_edge(uAp, d_uAp_v1), false)
+    insert_edge!(graph, d_PB_uB, uB, false)
+    insert_edge!(graph, d_PA_uA, uA, false)
+    insert_edge!(graph, d_PBp_uBp, uBp, false)
+    insert_edge!(graph, d_PAp_uAp, uAp, false)
 
-   insert_edge!(graph, make_edge(d_uB_v0, v0), false)
-   insert_edge!(graph, make_edge(d_uA_v0, v0), false)
-   insert_edge!(graph, make_edge(d_uBp_v1, v1), false)
-   insert_edge!(graph, make_edge(d_uAp_v1, v1), false)
+    insert_edge!(graph, uB, d_uB_v0, false)
+    insert_edge!(graph, uA, d_uA_v0, false)
+    insert_edge!(graph, uBp, d_uBp_v1, false)
+    insert_edge!(graph, uAp, d_uAp_v1, false)
 
-   insert_edge!(graph, make_edge(v0, d_v0_s0), false)
-   insert_edge!(graph, make_edge(v1, d_v1_s0), false)
+    insert_edge!(graph, d_uB_v0, v0, false)
+    insert_edge!(graph, d_uA_v0, v0, false)
+    insert_edge!(graph, d_uBp_v1, v1, false)
+    insert_edge!(graph, d_uAp_v1, v1, false)
 
-   insert_edge!(graph, make_edge(d_v0_s0, s0), false)
-   insert_edge!(graph, make_edge(d_v1_s0, s0), false)
+    insert_edge!(graph, v0, d_v0_s0, false)
+    insert_edge!(graph, v1, d_v1_s0, false)
 
-   insert_edge!(graph, make_edge(s0, d_exit), false)
+    insert_edge!(graph, d_v0_s0, s0, false)
+    insert_edge!(graph, d_v1_s0, s0, false)
 
-   @test length(graph.nodes) == 26
-   @test length(graph.appliedOperations) == 0
-   @test length(graph.operationsToApply) == 0
-   @test length(graph.dirtyNodes) == 26
-   @test length(graph.diff) == (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
+    insert_edge!(graph, s0, d_exit, false)
 
-   @test is_entry_node(d_PB)
-   @test is_entry_node(d_PA)
-   @test is_entry_node(d_PBp)
-   @test is_entry_node(d_PBp)
-   @test !is_entry_node(PB)
-   @test !is_entry_node(v0)
-   @test !is_entry_node(d_exit)
+    @test length(graph.nodes) == 26
+    @test length(graph.appliedOperations) == 0
+    @test length(graph.operationsToApply) == 0
+    @test length(graph.dirtyNodes) == 26
+    @test length(graph.diff) ==
+          (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
 
-   @test is_exit_node(d_exit)
-   @test !is_exit_node(d_uB_v0)
-   @test !is_exit_node(v0)
-   
-   @test length(children(v0)) == 2
-   @test length(children(v1)) == 2
-   @test length(parents(v0)) == 1
-   @test length(parents(v1)) == 1
+    @test is_valid(graph)
 
-   @test MetagraphOptimization.get_exit_node(graph) == d_exit
+    @test is_entry_node(d_PB)
+    @test is_entry_node(d_PA)
+    @test is_entry_node(d_PBp)
+    @test is_entry_node(d_PBp)
+    @test !is_entry_node(PB)
+    @test !is_entry_node(v0)
+    @test !is_entry_node(d_exit)
 
-   @test length(partners(s0)) == 0
-   @test length(siblings(s0)) == 0
+    @test is_exit_node(d_exit)
+    @test !is_exit_node(d_uB_v0)
+    @test !is_exit_node(v0)
 
-   operations = get_operations(graph)
-   @test length(operations) == (nodeFusions = 10, nodeReductions = 0, nodeSplits = 0)
-   @test length(graph.dirtyNodes) == 0
+    @test length(children(v0)) == 2
+    @test length(children(v1)) == 2
+    @test length(parents(v0)) == 1
+    @test length(parents(v1)) == 1
 
-   @test operations == get_operations(graph)
-   nf = first(operations.nodeFusions)
+    @test MetagraphOptimization.get_exit_node(graph) == d_exit
 
-   properties = graph_properties(graph)
-   @test properties.compute_effort == 134
-   @test properties.data == 62
-   @test properties.compute_intensity ≈ 134/62
-   @test properties.nodes == 26
-   @test properties.edges == 25
+    @test length(partners(s0)) == 1
+    @test length(siblings(s0)) == 1
 
-   push_operation!(graph, nf)
-   # **does not immediately apply the operation**
+    operations = get_operations(graph)
+    @test length(operations) ==
+          (nodeFusions = 10, nodeReductions = 0, nodeSplits = 0)
+    @test length(graph.dirtyNodes) == 0
 
-   @test length(graph.nodes) == 26
-   @test length(graph.appliedOperations) == 0
-   @test length(graph.operationsToApply) == 1
-   @test first(graph.operationsToApply) == nf
-   @test length(graph.dirtyNodes) == 0
-   @test length(graph.diff) == (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
+    @test operations == get_operations(graph)
+    nf = first(operations.nodeFusions)
 
-   # this applies pending operations
-   properties = graph_properties(graph)
+    properties = graph_properties(graph)
+    @test properties.compute_effort == 134
+    @test properties.data == 62
+    @test properties.compute_intensity ≈ 134 / 62
+    @test properties.nodes == 26
+    @test properties.edges == 25
 
-   @test length(graph.nodes) == 24
-   @test length(graph.appliedOperations) == 1
-   @test length(graph.operationsToApply) == 0
-   @test length(graph.dirtyNodes) != 0
-   @test properties.nodes == 24
-   @test properties.edges == 23
-   @test properties.compute_effort == 134
-   @test properties.data < 62
-   @test properties.compute_intensity > 134/62
+    push_operation!(graph, nf)
+    # **does not immediately apply the operation**
 
-   operations = get_operations(graph)
-   @test length(graph.dirtyNodes) == 0
-   
-   @test length(operations) == (nodeFusions = 9, nodeReductions = 0, nodeSplits = 0)
-   @test !isempty(operations)
+    @test length(graph.nodes) == 26
+    @test length(graph.appliedOperations) == 0
+    @test length(graph.operationsToApply) == 1
+    @test first(graph.operationsToApply) == nf
+    @test length(graph.dirtyNodes) == 0
+    @test length(graph.diff) ==
+          (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
 
-   possibleNF = 9
-   while !isempty(operations.nodeFusions)
-      push_operation!(graph, first(operations.nodeFusions))
-      operations = get_operations(graph)
-      possibleNF = possibleNF - 1
-      @test length(operations) == (nodeFusions = possibleNF, nodeReductions = 0, nodeSplits = 0)
-   end
+    # this applies pending operations
+    properties = graph_properties(graph)
 
-   @test isempty(operations)
+    @test length(graph.nodes) == 24
+    @test length(graph.appliedOperations) == 1
+    @test length(graph.operationsToApply) == 0
+    @test length(graph.dirtyNodes) != 0
+    @test properties.nodes == 24
+    @test properties.edges == 23
+    @test properties.compute_effort == 134
+    @test properties.data < 62
+    @test properties.compute_intensity > 134 / 62
 
-   @test length(operations) == (nodeFusions = 0, nodeReductions = 0, nodeSplits = 0)
-   @test length(graph.dirtyNodes) == 0
-   @test length(graph.nodes) == 6
-   @test length(graph.appliedOperations) == 10
-   @test length(graph.operationsToApply) == 0
+    operations = get_operations(graph)
+    @test length(graph.dirtyNodes) == 0
 
-   reset_graph!(graph)
+    @test length(operations) ==
+          (nodeFusions = 9, nodeReductions = 0, nodeSplits = 0)
+    @test !isempty(operations)
 
-   @test length(graph.dirtyNodes) == 26
-   @test length(graph.nodes) == 26
-   @test length(graph.appliedOperations) == 0
-   @test length(graph.operationsToApply) == 0
+    possibleNF = 9
+    while !isempty(operations.nodeFusions)
+        push_operation!(graph, first(operations.nodeFusions))
+        operations = get_operations(graph)
+        possibleNF = possibleNF - 1
+        @test length(operations) ==
+              (nodeFusions = possibleNF, nodeReductions = 0, nodeSplits = 0)
+    end
 
-   properties = graph_properties(graph)
-   @test properties.nodes == 26
-   @test properties.edges == 25
-   @test properties.compute_effort == 134
-   @test properties.data == 62
-   @test properties.compute_intensity ≈ 134/62
+    @test isempty(operations)
 
-   operations = get_operations(graph)
-   @test length(operations) == (nodeFusions = 10, nodeReductions = 0, nodeSplits = 0)
+    @test length(operations) ==
+          (nodeFusions = 0, nodeReductions = 0, nodeSplits = 0)
+    @test length(graph.dirtyNodes) == 0
+    @test length(graph.nodes) == 6
+    @test length(graph.appliedOperations) == 10
+    @test length(graph.operationsToApply) == 0
 
+    reset_graph!(graph)
+
+    @test length(graph.dirtyNodes) == 26
+    @test length(graph.nodes) == 26
+    @test length(graph.appliedOperations) == 0
+    @test length(graph.operationsToApply) == 0
+
+    properties = graph_properties(graph)
+    @test properties.nodes == 26
+    @test properties.edges == 25
+    @test properties.compute_effort == 134
+    @test properties.data == 62
+    @test properties.compute_intensity ≈ 134 / 62
+
+    operations = get_operations(graph)
+    @test length(operations) ==
+          (nodeFusions = 10, nodeReductions = 0, nodeSplits = 0)
+
+    @test is_valid(graph)
 end
 println("Graph Unit Tests Complete!")

test/unit_tests_nodes.jl

@@ -1,36 +1,37 @@
 
 @testset "Unit Tests Nodes" begin
-   nC1 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskU())
-   nC2 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskV())
-   nC3 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskP())
-   nC4 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskSum())
+    nC1 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskU())
+    nC2 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskV())
+    nC3 = MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskP())
+    nC4 =
+        MetagraphOptimization.make_node(MetagraphOptimization.ComputeTaskSum())
 
-   nD1 = MetagraphOptimization.make_node(MetagraphOptimization.DataTask(10))
-   nD2 = MetagraphOptimization.make_node(MetagraphOptimization.DataTask(20))
+    nD1 = MetagraphOptimization.make_node(MetagraphOptimization.DataTask(10))
+    nD2 = MetagraphOptimization.make_node(MetagraphOptimization.DataTask(20))
 
-   @test_throws ErrorException MetagraphOptimization.make_edge(nC1, nC2)
-   @test_throws ErrorException MetagraphOptimization.make_edge(nC1, nC1)
-   @test_throws ErrorException MetagraphOptimization.make_edge(nC3, nC4)
-   @test_throws ErrorException MetagraphOptimization.make_edge(nD1, nD2)
-   @test_throws ErrorException MetagraphOptimization.make_edge(nD1, nD1)
+    @test_throws ErrorException MetagraphOptimization.make_edge(nC1, nC2)
+    @test_throws ErrorException MetagraphOptimization.make_edge(nC1, nC1)
+    @test_throws ErrorException MetagraphOptimization.make_edge(nC3, nC4)
+    @test_throws ErrorException MetagraphOptimization.make_edge(nD1, nD2)
+    @test_throws ErrorException MetagraphOptimization.make_edge(nD1, nD1)
 
-   ed1 = MetagraphOptimization.make_edge(nC1, nD1)
-   ed2 = MetagraphOptimization.make_edge(nD1, nC2)
-   ed3 = MetagraphOptimization.make_edge(nC2, nD2)
-   ed4 = MetagraphOptimization.make_edge(nD2, nC3)
+    ed1 = MetagraphOptimization.make_edge(nC1, nD1)
+    ed2 = MetagraphOptimization.make_edge(nD1, nC2)
+    ed3 = MetagraphOptimization.make_edge(nC2, nD2)
+    ed4 = MetagraphOptimization.make_edge(nD2, nC3)
 
-   @test nC1 != nC2
-   @test nD1 != nD2
-   @test nC1 != nD1
-   @test nC3 != nC4
+    @test nC1 != nC2
+    @test nD1 != nD2
+    @test nC1 != nD1
+    @test nC3 != nC4
 
-   nC1_2 = copy(nC1)
-   @test nC1_2 != nC1
-   
-   nD1_2 = copy(nD1)
-   @test nD1_2 != nD1
+    nC1_2 = copy(nC1)
+    @test nC1_2 != nC1
 
-   nD1_c = MetagraphOptimization.make_node(MetagraphOptimization.DataTask(10))
-   @test nD1_c != nD1
+    nD1_2 = copy(nD1)
+    @test nD1_2 != nD1
+
+    nD1_c = MetagraphOptimization.make_node(MetagraphOptimization.DataTask(10))
+    @test nD1_c != nD1
 end
 println("Node Unit Tests Complete!")

test/unit_tests_tasks.jl

@@ -1,60 +1,51 @@
 
 @testset "Task Unit Tests" begin
-   S1 = MetagraphOptimization.ComputeTaskS1()
-   S2 = MetagraphOptimization.ComputeTaskS2()
-   U = MetagraphOptimization.ComputeTaskU()
-   V = MetagraphOptimization.ComputeTaskV()
-   P = MetagraphOptimization.ComputeTaskP()
-   Sum = MetagraphOptimization.ComputeTaskSum()
+    S1 = MetagraphOptimization.ComputeTaskS1()
+    S2 = MetagraphOptimization.ComputeTaskS2()
+    U = MetagraphOptimization.ComputeTaskU()
+    V = MetagraphOptimization.ComputeTaskV()
+    P = MetagraphOptimization.ComputeTaskP()
+    Sum = MetagraphOptimization.ComputeTaskSum()
 
-   Data10 = MetagraphOptimization.DataTask(10)
-   Data20 = MetagraphOptimization.DataTask(20)
+    Data10 = MetagraphOptimization.DataTask(10)
+    Data20 = MetagraphOptimization.DataTask(20)
 
-   @test MetagraphOptimization.compute_effort(S1) == 10
-   @test MetagraphOptimization.compute_effort(S2) == 10
-   @test MetagraphOptimization.compute_effort(U) == 6
-   @test MetagraphOptimization.compute_effort(V) == 20
-   @test MetagraphOptimization.compute_effort(P) == 15
-   @test MetagraphOptimization.compute_effort(Sum) == 1
-   @test MetagraphOptimization.compute_effort(Data10) == 0
-   @test MetagraphOptimization.compute_effort(Data20) == 0
+    @test MetagraphOptimization.compute_effort(S1) == 10
+    @test MetagraphOptimization.compute_effort(S2) == 10
+    @test MetagraphOptimization.compute_effort(U) == 6
+    @test MetagraphOptimization.compute_effort(V) == 20
+    @test MetagraphOptimization.compute_effort(P) == 15
+    @test MetagraphOptimization.compute_effort(Sum) == 1
+    @test MetagraphOptimization.compute_effort(Data10) == 0
+    @test MetagraphOptimization.compute_effort(Data20) == 0
 
-   @test MetagraphOptimization.data(S1) == 0
-   @test MetagraphOptimization.data(S2) == 0
-   @test MetagraphOptimization.data(U) == 0
-   @test MetagraphOptimization.data(V) == 0
-   @test MetagraphOptimization.data(P) == 0
-   @test MetagraphOptimization.data(Sum) == 0
-   @test MetagraphOptimization.data(Data10) == 10
-   @test MetagraphOptimization.data(Data20) == 20
+    @test MetagraphOptimization.data(S1) == 0
+    @test MetagraphOptimization.data(S2) == 0
+    @test MetagraphOptimization.data(U) == 0
+    @test MetagraphOptimization.data(V) == 0
+    @test MetagraphOptimization.data(P) == 0
+    @test MetagraphOptimization.data(Sum) == 0
+    @test MetagraphOptimization.data(Data10) == 10
+    @test MetagraphOptimization.data(Data20) == 20
 
-   @test MetagraphOptimization.compute_intensity(S1) == typemax(UInt64)
-   @test MetagraphOptimization.compute_intensity(S2) == typemax(UInt64)
-   @test MetagraphOptimization.compute_intensity(U) == typemax(UInt64)
-   @test MetagraphOptimization.compute_intensity(V) == typemax(UInt64)
-   @test MetagraphOptimization.compute_intensity(P) == typemax(UInt64)
-   @test MetagraphOptimization.compute_intensity(Sum) == typemax(UInt64)
-   @test MetagraphOptimization.compute_intensity(Data10) == 0
-   @test MetagraphOptimization.compute_intensity(Data20) == 0
+    @test S1 != S2
+    @test Data10 != Data20
 
-   @test S1 != S2
-   @test Data10 != Data20
-   
-   Data10_2 = MetagraphOptimization.DataTask(10)
+    Data10_2 = MetagraphOptimization.DataTask(10)
 
-   # two data tasks with same data are identical, their nodes need not be
-   @test Data10_2 == Data10
+    # two data tasks with same data are identical, their nodes need not be
+    @test Data10_2 == Data10
 
-   @test Data10 == Data10
-   @test S1 == S1
+    @test Data10 == Data10
+    @test S1 == S1
 
-   Data10_3 = copy(Data10)
+    Data10_3 = copy(Data10)
 
-   @test Data10_3 == Data10
+    @test Data10_3 == Data10
 
-   S1_2 = copy(S1)
+    S1_2 = copy(S1)
 
-   @test S1_2 == S1
-   @test S1 == MetagraphOptimization.ComputeTaskS1()
+    @test S1_2 == S1
+    @test S1 == MetagraphOptimization.ComputeTaskS1()
 end
 println("Task Unit Tests Complete!")

test/unit_tests_utility.jl

@@ -1,11 +1,13 @@
 
 @testset "Unit Tests Utility" begin
-   @test MetagraphOptimization.bytes_to_human_readable(0) == "0.0 B"
-   @test MetagraphOptimization.bytes_to_human_readable(1020) == "1020.0 B"
-   @test MetagraphOptimization.bytes_to_human_readable(1025) == "1.001 KiB"
-   @test MetagraphOptimization.bytes_to_human_readable(684235) == "668.2 KiB"
-   @test MetagraphOptimization.bytes_to_human_readable(86214576) == "82.22 MiB"
-   @test MetagraphOptimization.bytes_to_human_readable(9241457698) == "8.607 GiB"
-   @test MetagraphOptimization.bytes_to_human_readable(3218598654367) == "2.927 TiB"
+    @test MetagraphOptimization.bytes_to_human_readable(0) == "0.0 B"
+    @test MetagraphOptimization.bytes_to_human_readable(1020) == "1020.0 B"
+    @test MetagraphOptimization.bytes_to_human_readable(1025) == "1.001 KiB"
+    @test MetagraphOptimization.bytes_to_human_readable(684235) == "668.2 KiB"
+    @test MetagraphOptimization.bytes_to_human_readable(86214576) == "82.22 MiB"
+    @test MetagraphOptimization.bytes_to_human_readable(9241457698) ==
+          "8.607 GiB"
+    @test MetagraphOptimization.bytes_to_human_readable(3218598654367) ==
+          "2.927 TiB"
 end
 println("Utility Unit Tests Complete!")