Fix docs

Reviewed-on: Rubydragon/MetagraphOptimization.jl#7

.gitattributes

@@ -1,2 +1,2 @@
-examples/AB->ABBBBBBB.txt filter=lfs diff=lfs merge=lfs -text
-examples/AB->ABBBBBBBBB.txt filter=lfs diff=lfs merge=lfs -text
+input/AB->ABBBBBBBBB.txt filter=lfs diff=lfs merge=lfs -text
+input/AB->ABBBBBBB.txt filter=lfs diff=lfs merge=lfs -text

.gitea/workflows/julia-package-ci.yml

@@ -1,30 +1,110 @@
-name: Test
+name: MetagraphOptimization_CI
 
 on: [push]
 
+env:
+  # keep the depot directly in the repository for the cache
+  JULIA_DEPOT_PATH: './.julia'
+
 jobs:
-  test:
+  prepare:
     runs-on: arch-latest
 
     steps:
-      #- name: Get git-lfs
-      #  run: apt-get update && apt-get install git-lfs
-
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      #- name: Checkout LFS objects
-      #  run: git lfs checkout
-
       - name: Setup Julia environment
         uses: https://github.com/julia-actions/setup-julia@v1.9.2
         with:
           version: '1.9.2'
 
+      # needed for the file hashing, should be removed when ${{ hashFiles('**/Project.toml') }} is supported in gitea
+      - name: Setup go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Hash files
+        uses: https://gitea.com/actions/go-hashfiles@v0.0.1
+        id: get-hash
+        with: 
+          patterns: |-
+            **/Project.toml
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+      - name: Check cache hit
+        if: steps.cache-restore.outputs.cache-hit == 'true'
+        run: exit 0
+
       - name: Install dependencies
-        run: julia --project=./ -e 'import Pkg; Pkg.instantiate()'
+        run: |
+          julia --project=./        -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+          julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+          julia --project=docs/     -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Cache Julia packages
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+  test:
+    needs: prepare
+    runs-on: arch-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Julia environment
+        uses: https://github.com/julia-actions/setup-julia@v1.9.2
+        with:
+          version: '1.9.2'
+
+      # needed for the file hashing, should be removed when ${{ hashFiles('**/Project.toml') }} is supported in gitea
+      - name: Setup go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Hash files
+        uses: https://gitea.com/actions/go-hashfiles@v0.0.1
+        id: get-hash
+        with: 
+          patterns: |-
+            **/Project.toml  
+
+      - name: Restore cached Julia packages
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+      - name: Install dependencies
+        run: |
+          julia --project=./        -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+          julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+          julia --project=docs/     -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
 
       - name: Format check
         run: |
@@ -43,4 +123,63 @@ jobs:
         run: julia --project=./ -t 4 -e 'import Pkg; Pkg.test()' -O0
 
       - name: Run examples
-        run: julia --project=examples/ -t 4 -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); include("examples/import_bench.jl")' -O3
+        run: julia --project=examples/ -t 4 -e 'include("examples/import_bench.jl")' -O3
+
+  docs:
+    needs: prepare
+    runs-on: arch-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Julia environment
+        uses: https://github.com/julia-actions/setup-julia@v1.9.2
+        with:
+          version: '1.9.2'
+
+      # needed for the file hashing, should be removed when ${{ hashFiles('**/Project.toml') }} is supported in gitea
+      - name: Setup go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20'
+
+      - name: Hash files
+        uses: https://gitea.com/actions/go-hashfiles@v0.0.1
+        id: get-hash
+        with: 
+          patterns: |-
+            **/Project.toml  
+
+      - name: Restore cached Julia packages
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            .julia/artifacts
+            .julia/packages
+            .julia/registries
+          key: julia-${{ steps.get-hash.outputs.hash }}
+
+      - name: Install dependencies
+        run: |
+          julia --project=./        -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+          julia --project=examples/ -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+          julia --project=docs/     -e 'import Pkg; Pkg.develop(Pkg.PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Build docs
+        run: julia --project=docs/ docs/make.jl
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: web-doc
+          path: docs/build/
+
+      #- name: Webhook Trigger
+      #  uses: https://github.com/zzzze/webhook-trigger@master
+      #  continue-on-error: true
+      #  with:
+      #    data: "{\"event\":\"action_completed\", \"download_url\":\"deckardcain.local:8099/something\"}"
+      #    webhook_url: ${{ secrets.WEBHOOK_URL }}

Project.toml

@@ -4,9 +4,9 @@ authors = ["Anton Reinhard <anton.reinhard@proton.me>"]
 version = "0.1.0"
 
 [deps]
+AccurateArithmetic = "22286c92-06ac-501d-9306-4abd417d9753"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
-Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 

docs/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8"
+MetagraphOptimization = "3e869610-d48d-4942-ba70-c1b702a33ca4"

docs/make.jl

@@ -0,0 +1,33 @@
+using Documenter
+using MetagraphOptimization
+
+makedocs(
+    #format = Documenter.LaTeX(platform=""),
+
+    root = "docs",
+    source = "src",
+    build = "build",
+    warnonly = true,
+    clean = true,
+    doctest = true,
+    modules = Module[MetagraphOptimization],
+    #repo = "https://code.woubery.com/Rubydragon/MetagraphOptimization.jl/src/branch/{commit}{path}#L{line}",
+    remotes = nothing,
+    sitename = "MetagraphOptimization.jl",
+    pages = [
+        "index.md",
+        "Manual" => "manual.md",
+        "Library" => [
+            "Public" => "lib/public.md",
+            "Graph" => "lib/internals/graph.md",
+            "Node" => "lib/internals/node.md",
+            "Task" => "lib/internals/task.md",
+            "Operation" => "lib/internals/operation.md",
+            "Models" => "lib/internals/models.md",
+            "Diff" => "lib/internals/diff.md",
+            "Utility" => "lib/internals/utility.md",
+            "Code Generation" => "lib/internals/code_gen.md",
+        ],
+        "Contribution" => "contribution.md",
+    ],
+)

docs/src/contribution.md

@@ -0,0 +1,3 @@
+# Contribution
+
+This is currently in development for a diploma thesis and is therefore private and impossible to contribute to.

docs/src/index.md

@@ -0,0 +1,26 @@
+# MetagraphOptimization.jl
+
+*A domain-specific DAG-optimizer*
+
+## Package Features
+- Read a DAG from a file
+- Analyze its properties
+- Mute the graph using the operations NodeFusion, NodeReduction and NodeSplit
+
+## Coming Soon:
+- Add Code Generation from finished DAG
+- Add optimization algorithms and strategies
+
+## Library Outline
+
+```@contents
+Pages = [
+    "lib/public.md",
+    "lib/internals.md"
+]
+```
+
+### [Index](@id main-index)
+```@index
+Pages = ["lib/public.md"]
+```

docs/src/lib/internals/code_gen.md

@@ -0,0 +1,8 @@
+# Code Generation
+
+## Main
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["code_gen/main.jl"]
+Order = [:function]
+```

docs/src/lib/internals/diff.md

@@ -0,0 +1,22 @@
+# Diff
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["diff/type.jl"]
+Order   = [:type]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["diff/properties.jl"]
+Order   = [:function]
+```
+
+## Printing
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["diff/print.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/graph.md

@@ -0,0 +1,50 @@
+# Graph
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/type.jl"]
+Order   = [:type]
+```
+
+## Interface
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/interface.jl"]
+Order   = [:function]
+```
+
+## Compare
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/compare.jl"]
+Order   = [:function]
+```
+
+## Mute
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/mute.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/print.jl"]
+Order   = [:function]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/properties.jl"]
+Order   = [:function]
+```
+
+## Validate
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["graph/validate.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/models.md

@@ -0,0 +1,49 @@
+# Models
+
+## ABC-Model
+
+### Types
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/types.jl"]
+Order   = [:type, :constant]
+```
+
+### Particle
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/particle.jl"]
+Order   = [:type, :constant, :function]
+```
+
+### Parse
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/parse.jl"]
+Order   = [:function]
+```
+
+### Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/properties.jl"]
+Order   = [:function]
+```
+
+### Create
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/create.jl"]
+Order = [:function]
+```
+
+### Compute
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["models/abc/compute.jl"]
+Order = [:function]
+```
+
+## QED-Model
+
+*To be added*

docs/src/lib/internals/node.md

@@ -0,0 +1,43 @@
+# Node
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/type.jl"]
+Order   = [:type]
+```
+
+## Create
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/create.jl"]
+Order   = [:function]
+```
+
+## Compare
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/compare.jl"]
+Order   = [:function]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/properties.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/print.jl"]
+Order   = [:function]
+```
+
+## Validate
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["node/validate.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/operation.md

@@ -0,0 +1,57 @@
+# Operation
+
+## Types
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/type.jl"]
+Order   = [:type]
+```
+
+## Find
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/find.jl"]
+Order   = [:function]
+```
+
+## Apply
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/apply.jl"]
+Order   = [:function]
+```
+
+## Get
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/get.jl"]
+Order   = [:function]
+```
+
+## Clean
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/clean.jl"]
+Order   = [:function]
+```
+
+## Utility
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/utility.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/print.jl"]
+Order   = [:function]
+```
+
+## Validate
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["operation/validate.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/properties.md

@@ -0,0 +1,22 @@
+# Properties
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["properties/type.jl"]
+Order   = [:type]
+```
+
+## Create
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["properties/create.jl"]
+Order   = [:function]
+```
+
+## Utility
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["properties/utility.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/task.md

@@ -0,0 +1,36 @@
+# Task
+
+## Type
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/type.jl"]
+Order   = [:type]
+```
+
+## Create
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/create.jl"]
+Order   = [:function]
+```
+
+## Compare
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/compare.jl"]
+Order   = [:function]
+```
+
+## Properties
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/properties.jl"]
+Order   = [:function]
+```
+
+## Print
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages = ["task/print.jl"]
+Order   = [:function]
+```

docs/src/lib/internals/utility.md

@@ -0,0 +1,17 @@
+# Utility
+
+## Helper Functions
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages   = ["./utility.jl"]
+Order   = [:type, :function]
+```
+
+## Trie Helper
+This is a simple implementation of a [Trie Data Structure](https://en.wikipedia.org/wiki/Trie) to greatly improve the performance of the Node Reduction search.
+
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages   = ["trie.jl"]
+Order   = [:type, :function]
+```

docs/src/lib/public.md

@@ -0,0 +1,24 @@
+# Public Documentation
+
+Documentation for `MetagraphOptimization.jl`'s public interface.
+
+See the Internals section of the manual for documentation of everything else.
+
+```@autodocs
+Modules = [MetagraphOptimization]
+Pages   = ["MetagraphOptimization.jl"]
+Order   = [:module]
+```
+
+## Contents
+
+```@contents
+Pages = ["public.md"]
+Depth = 2
+```
+
+## Index
+
+```@index
+Pages = ["public.md"]
+```

docs/src/manual.md

@@ -0,0 +1,3 @@
+# Manual
+
+This will become a manual.

examples/Project.toml

@@ -1,7 +1,3 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 MetagraphOptimization = "3e869610-d48d-4942-ba70-c1b702a33ca4"
-PProf = "e4faabce-9ead-11e9-39d9-4379958e3056"
-Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
-ProfileView = "c46f51b8-102a-5cf2-8d2c-8597cb0e0da7"
-Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"

examples/plot_chain.jl

@@ -41,9 +41,9 @@ function gen_plot(filepath)
             i = i - 1
         end
 
-        props = graph_properties(g)
+        props = get_properties(g)
         push!(x, props.data)
-        push!(y, props.compute_effort)
+        push!(y, props.computeEffort)
     end
 
     println("\rDone.")

examples/plot_star.jl

@@ -44,9 +44,9 @@ function gen_plot(filepath)
 
 
 
-    props = graph_properties(g)
+    props = get_properties(g)
     x0 = props.data
-    y0 = props.compute_effort
+    y0 = props.computeEffort
 
     x = Vector{Float64}()
     y = Vector{Float64}()
@@ -55,9 +55,9 @@ function gen_plot(filepath)
     opt = get_operations(g)
     for op in opt.nodeFusions
         push_operation!(g, op)
-        props = graph_properties(g)
+        props = get_properties(g)
         push!(x, props.data)
-        push!(y, props.compute_effort)
+        push!(y, props.computeEffort)
         pop_operation!(g)
 
         push!(
@@ -65,15 +65,15 @@ function gen_plot(filepath)
             "NF: (" *
             string(props.data) *
             ", " *
-            string(props.compute_effort) *
+            string(props.computeEffort) *
             ")",
         )
     end
     for op in opt.nodeReductions
         push_operation!(g, op)
-        props = graph_properties(g)
+        props = get_properties(g)
         push!(x, props.data)
-        push!(y, props.compute_effort)
+        push!(y, props.computeEffort)
         pop_operation!(g)
 
         push!(
@@ -81,15 +81,15 @@ function gen_plot(filepath)
             "NR: (" *
             string(props.data) *
             ", " *
-            string(props.compute_effort) *
+            string(props.computeEffort) *
             ")",
         )
     end
     for op in opt.nodeSplits
         push_operation!(g, op)
-        props = graph_properties(g)
+        props = get_properties(g)
         push!(x, props.data)
-        push!(y, props.compute_effort)
+        push!(y, props.computeEffort)
         pop_operation!(g)
 
         push!(
@@ -97,7 +97,7 @@ function gen_plot(filepath)
             "NS: (" *
             string(props.data) *
             ", " *
-            string(props.compute_effort) *
+            string(props.computeEffort) *
             ")",
         )
     end

examples/profiling_utilities.jl

@@ -3,7 +3,7 @@ function test_random_walk(g::DAG, n::Int64)
     # the purpose here is to do "random" operations and reverse them again and validate that the graph stays the same and doesn't diverge
     reset_graph!(g)
 
-    properties = graph_properties(g)
+    properties = get_properties(g)
 
     for i in 1:n
         # choose push or pop
@@ -34,3 +34,26 @@ function test_random_walk(g::DAG, n::Int64)
 
     return reset_graph!(g)
 end
+
+function reduce_all!(g::DAG)
+    reset_graph!(g)
+
+    opt = get_operations(g)
+    while (!isempty(opt.nodeReductions))
+        push_operation!(g, pop!(opt.nodeReductions))
+
+        if (isempty(opt.nodeReductions))
+            opt = get_operations(g)
+        end
+    end
+    return nothing
+end
+
+function reduce_one!(g::DAG)
+    opt = get_operations(g)
+    if !isempty(opt.nodeReductions)
+        push_operation!(g, pop!(opt.nodeReductions))
+    end
+    opt = get_operations(g)
+    return nothing
+end

src/MetagraphOptimization.jl

@@ -1,35 +1,59 @@
+"""
+    MetagraphOptimization
+
+A module containing tools to work on DAGs.
+"""
 module MetagraphOptimization
 
-export Node, Edge, ComputeTaskNode, DataTaskNode, DAG
-export AbstractTask,
-    AbstractComputeTask, AbstractDataTask, DataTask, FusedComputeTask
-export make_node,
-    make_edge,
-    insert_node,
-    insert_edge,
-    is_entry_node,
-    is_exit_node,
-    parents,
-    children,
-    compute,
-    graph_properties,
-    get_exit_node,
-    is_valid
-export NodeFusion,
-    NodeReduction,
-    NodeSplit,
-    push_operation!,
-    pop_operation!,
-    can_pop,
-    reset_graph!,
-    get_operations
-export parse_abc,
-    ComputeTaskP,
-    ComputeTaskS1,
-    ComputeTaskS2,
-    ComputeTaskV,
-    ComputeTaskU,
-    ComputeTaskSum
+export DAG
+export Node
+export Edge
+export ComputeTaskNode
+export DataTaskNode
+export AbstractTask
+export AbstractComputeTask
+export AbstractDataTask
+export DataTask
+export FusedComputeTask
+export PossibleOperations
+export GraphProperties
+
+export make_node
+export make_edge
+export insert_node
+export insert_edge
+export is_entry_node
+export is_exit_node
+export parents
+export children
+export compute
+export get_properties
+export get_exit_node
+export is_valid
+
+export Operation
+export AppliedOperation
+export NodeFusion
+export NodeReduction
+export NodeSplit
+export push_operation!
+export pop_operation!
+export can_pop
+export reset_graph!
+export get_operations
+
+export parse_abc
+export ComputeTaskP
+export ComputeTaskS1
+export ComputeTaskS2
+export ComputeTaskV
+export ComputeTaskU
+export ComputeTaskSum
+
+export execute
+export gen_particles
+export ParticleValue
+export Particle
 
 export ==, in, show, isempty, delete!, length
 
@@ -38,6 +62,8 @@ export bytes_to_human_readable
 import Base.length
 import Base.show
 import Base.==
+import Base.+
+import Base.-
 import Base.in
 import Base.copy
 import Base.isempty
@@ -49,6 +75,7 @@ import Base.collect
 include("task/type.jl")
 include("node/type.jl")
 include("diff/type.jl")
+include("properties/type.jl")
 include("operation/type.jl")
 include("graph/type.jl")
 
@@ -79,12 +106,21 @@ include("operation/get.jl")
 include("operation/print.jl")
 include("operation/validate.jl")
 
+include("properties/create.jl")
+include("properties/utility.jl")
+
+include("task/create.jl")
 include("task/compare.jl")
 include("task/print.jl")
 include("task/properties.jl")
 
 include("models/abc/types.jl")
+include("models/abc/particle.jl")
+include("models/abc/compute.jl")
+include("models/abc/create.jl")
 include("models/abc/properties.jl")
 include("models/abc/parse.jl")
 
+include("code_gen/main.jl")
+
 end # module MetagraphOptimization

src/code_gen/main.jl

@@ -0,0 +1,126 @@
+using DataStructures
+
+"""
+    gen_code(graph::DAG)
+
+Generate the code for a given graph. The return value is a tuple of:
+
+- `code::Expr`: The julia expression containing the code for the whole graph.
+- `inputSymbols::Dict{String, Symbol}`: A dictionary of symbols mapping the names of the input nodes of the graph to the symbols their inputs should be provided on.
+- `outputSymbol::Symbol`: The symbol of the final calculated value
+
+See also: [`execute`](@ref)
+"""
+function gen_code(graph::DAG)
+    code = Vector{Expr}()
+    sizehint!(code, length(graph.nodes))
+
+    nodeQueue = PriorityQueue{Node, Int}()
+    inputSyms = Dict{String, Symbol}()
+
+    # use a priority equal to the number of unseen children -> 0 are nodes that can be added
+    for node in get_entry_nodes(graph)
+        enqueue!(nodeQueue, node => 0)
+        push!(inputSyms, node.name => Symbol("data_$(to_var_name(node.id))_in"))
+    end
+
+    node = nothing
+    while !isempty(nodeQueue)
+        @assert peek(nodeQueue)[2] == 0
+        node = dequeue!(nodeQueue)
+
+        push!(code, get_expression(node))
+        for parent in node.parents
+            # reduce the priority of all parents by one
+            if (!haskey(nodeQueue, parent))
+                enqueue!(nodeQueue, parent => length(parent.children) - 1)
+            else
+                nodeQueue[parent] = nodeQueue[parent] - 1
+            end
+        end
+    end
+
+    # node is now the last node we looked at -> the output node
+    outSym = Symbol("data_$(to_var_name(node.id))")
+
+    return (
+        code = Expr(:block, code...),
+        inputSymbols = inputSyms,
+        outputSymbol = outSym,
+    )
+end
+
+"""
+    execute(generated_code, input::Dict{ParticleType, Vector{Particle}})
+
+Execute the given `generated_code` (as returned by [`gen_code`](@ref)) on the given input particles.
+"""
+function execute(generated_code, input::Dict{ParticleType, Vector{Particle}})
+    (code, inputSymbols, outputSymbol) = generated_code
+
+    assignInputs = Vector{Expr}()
+    for (name, symbol) in inputSymbols
+        type = nothing
+        if startswith(name, "A")
+            type = A
+        elseif startswith(name, "B")
+            type = B
+        else
+            type = C
+        end
+        index = parse(Int, name[2:end])
+
+        push!(
+            assignInputs,
+            Meta.parse(
+                "$(symbol) = ParticleValue(Particle($(input[type][index]).P0, $(input[type][index]).P1, $(input[type][index]).P2, $(input[type][index]).P3, $(type)), 1.0)",
+            ),
+        )
+    end
+
+    assignInputs = Expr(:block, assignInputs...)
+    eval(assignInputs)
+    eval(code)
+
+    eval(Meta.parse("result = $outputSymbol"))
+    return result
+end
+
+"""
+    execute(graph::DAG, input::Dict{ParticleType, Vector{Particle}})
+
+Execute the given `generated_code` (as returned by [`gen_code`](@ref)) on the given input particles.
+The input particles should be sorted correctly into the dictionary to their according [`ParticleType`](@ref)s.
+
+See also: [`gen_particles`](@ref)
+"""
+function execute(graph::DAG, input::Dict{ParticleType, Vector{Particle}})
+    (code, inputSymbols, outputSymbol) = gen_code(graph)
+
+    assignInputs = Vector{Expr}()
+    for (name, symbol) in inputSymbols
+        type = nothing
+        if startswith(name, "A")
+            type = A
+        elseif startswith(name, "B")
+            type = B
+        else
+            type = C
+        end
+        index = parse(Int, name[2:end])
+
+        push!(
+            assignInputs,
+            Meta.parse(
+                "$(symbol) = ParticleValue(Particle($(input[type][index]).P0, $(input[type][index]).P1, $(input[type][index]).P2, $(input[type][index]).P3, $(type)), 1.0)",
+            ),
+        )
+    end
+
+    assignInputs = Expr(:block, assignInputs...)
+    eval(assignInputs)
+    eval(code)
+
+    eval(Meta.parse("result = $outputSymbol"))
+    return result
+end

src/diff/print.jl

@@ -1,3 +1,8 @@
+"""
+    show(io::IO, diff::Diff)
+
+Pretty-print a [`Diff`](@ref). Called via print, println and co.
+"""
 function show(io::IO, diff::Diff)
     print(io, "Nodes: ")
     print(io, length(diff.addedNodes) + length(diff.removedNodes))

src/diff/properties.jl

@@ -1,4 +1,9 @@
-# return a namedtuple of the lengths of the added/removed nodes/edges
+"""
+    length(diff::Diff)
+    
+Return a named tuple of the lengths of the added/removed nodes/edges.
+The fields are `.addedNodes`, `.addedEdges`, `.removedNodes` and `.removedEdges`.
+"""
 function length(diff::Diff)
     return (
         addedNodes = length(diff.addedNodes),

src/diff/type.jl

@@ -1,3 +1,8 @@
+"""
+    Diff
+
+A named tuple representing a difference of added and removed nodes and edges on a [`DAG`](@ref).
+"""
 const Diff = NamedTuple{
     (:addedNodes, :removedNodes, :addedEdges, :removedEdges),
     Tuple{Vector{Node}, Vector{Node}, Vector{Edge}, Vector{Edge}},

src/graph/compare.jl

@@ -1,7 +1,30 @@
+"""
+    in(node::Node, graph::DAG)
 
+Check whether the node is part of the graph.
+"""
 in(node::Node, graph::DAG) = node in graph.nodes
-in(edge::Edge, graph::DAG) = edge in graph.edges
 
+"""
+    in(edge::Edge, graph::DAG)
+
+Check whether the edge is part of the graph.
+"""
+function in(edge::Edge, graph::DAG)
+    n1 = edge.edge[1]
+    n2 = edge.edge[2]
+    if !(n1 in graph) || !(n2 in graph)
+        return false
+    end
+
+    return n1 in n2.children
+end
+
+"""
+    ==(n1::Node, n2::Node, g::DAG)
+
+Check equality of two nodes in a graph.
+"""
 function ==(n1::Node, n2::Node, g::DAG)
     if typeof(n1) != typeof(n2)
         return false

src/graph/interface.jl

@@ -1,6 +1,10 @@
-# user interface on the DAG
+"""
+    push_operation!(graph::DAG, operation::Operation)
 
-# applies a new operation to the end of the graph
+Apply a new operation to the graph.
+
+See also: [`DAG`](@ref), [`pop_operation!`](@ref)
+"""
 function push_operation!(graph::DAG, operation::Operation)
     # 1.: Add the operation to the DAG
     push!(graph.operationsToApply, operation)
@@ -8,7 +12,13 @@ function push_operation!(graph::DAG, operation::Operation)
     return nothing
 end
 
-# reverts the latest applied operation, essentially like a ctrl+z for
+"""
+    pop_operation!(graph::DAG)
+
+Revert the latest applied operation on the graph.
+
+See also: [`DAG`](@ref), [`push_operation!`](@ref)
+"""
 function pop_operation!(graph::DAG)
     # 1.: Remove the operation from the appliedChain of the DAG
     if !isempty(graph.operationsToApply)
@@ -23,10 +33,19 @@ function pop_operation!(graph::DAG)
     return nothing
 end
 
+"""
+    can_pop(graph::DAG)
+
+Return `true` if [`pop_operation!`](@ref) is possible, `false` otherwise.
+"""
 can_pop(graph::DAG) =
     !isempty(graph.operationsToApply) || !isempty(graph.appliedOperations)
 
-# reset the graph to its initial state with no operations applied
+"""
+    reset_graph!(graph::DAG)
+
+Reset the graph to its initial state with no operations applied.
+"""
 function reset_graph!(graph::DAG)
     while (can_pop(graph))
         pop_operation!(graph)

src/graph/mute.jl

@@ -3,6 +3,18 @@
 # 2: keep track of what was changed for the diff (if track == true)
 # 3: invalidate operation caches
 
+"""
+    insert_node!(graph::DAG, node::Node; track = true, invalidate_cache = true)
+
+Insert the node into the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`remove_node!`](@ref), [`insert_edge!`](@ref), [`remove_edge!`](@ref)
+"""
 function insert_node!(
     graph::DAG,
     node::Node,
@@ -26,6 +38,18 @@ function insert_node!(
     return node
 end
 
+"""
+    insert_edge!(graph::DAG, node1::Node, node2::Node; track = true, invalidate_cache = true)
+
+Insert the edge between node1 (child) and node2 (parent) into the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`insert_node!`](@ref), [`remove_node!`](@ref), [`remove_edge!`](@ref)
+"""
 function insert_edge!(
     graph::DAG,
     node1::Node,
@@ -59,6 +83,18 @@ function insert_edge!(
     return nothing
 end
 
+"""
+    remove_node!(graph::DAG, node::Node; track = true, invalidate_cache = true)
+
+Remove the node from the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`insert_node!`](@ref), [`insert_edge!`](@ref), [`remove_edge!`](@ref)
+"""
 function remove_node!(
     graph::DAG,
     node::Node,
@@ -86,6 +122,18 @@ function remove_node!(
     return nothing
 end
 
+"""
+    remove_edge!(graph::DAG, node1::Node, node2::Node; track = true, invalidate_cache = true)
+
+Remove the edge between node1 (child) and node2 (parent) into the graph.
+
+## Keyword Arguments
+`track::Bool`: Whether to add the changes to the [`DAG`](@ref)'s [`Diff`](@ref). Should be set `false` in parsing or graph creation functions for performance.
+
+`invalidate_cache::Bool`: Whether to invalidate caches associated with the changes. Should also be turned off for graph creation or parsing.
+
+See also: [`insert_node!`](@ref), [`remove_node!`](@ref), [`insert_edge!`](@ref)
+"""
 function remove_edge!(
     graph::DAG,
     node1::Node,
@@ -96,6 +144,8 @@ function remove_edge!(
     # 1: mute
     pre_length1 = length(node1.parents)
     pre_length2 = length(node2.children)
+
+    #TODO: filter is very slow
     filter!(x -> x != node2, node1.parents)
     filter!(x -> x != node1, node2.children)
 
@@ -131,16 +181,29 @@ function remove_edge!(
     return nothing
 end
 
-# return the graph "difference" since last time this function was called
+"""
+    get_snapshot_diff(graph::DAG)
+
+Return the graph's [`Diff`](@ref) since last time this function was called.
+
+See also: [`revert_diff!`](@ref), [`AppliedOperation`](@ref) and [`revert_operation!`](@ref)
+"""
 function get_snapshot_diff(graph::DAG)
     return swapfield!(graph, :diff, Diff())
 end
 
-# function to invalidate the operation caches for a given NodeFusion
+"""
+    invalidate_caches!(graph::DAG, operation::NodeFusion)
+
+Invalidate the operation caches for a given [`NodeFusion`](@ref).
+
+This deletes the operation from the graph's possible operations and from the involved nodes' own operation caches.
+"""
 function invalidate_caches!(graph::DAG, operation::NodeFusion)
     delete!(graph.possibleOperations, operation)
 
     # delete the operation from all caches of nodes involved in the operation
+    # TODO: filter is very slow
     filter!(!=(operation), operation.input[1].nodeFusions)
     filter!(!=(operation), operation.input[3].nodeFusions)
 
@@ -149,7 +212,13 @@ function invalidate_caches!(graph::DAG, operation::NodeFusion)
     return nothing
 end
 
-# function to invalidate the operation caches for a given NodeReduction
+"""
+    invalidate_caches!(graph::DAG, operation::NodeReduction)
+
+Invalidate the operation caches for a given [`NodeReduction`](@ref).
+
+This deletes the operation from the graph's possible operations and from the involved nodes' own operation caches.
+"""
 function invalidate_caches!(graph::DAG, operation::NodeReduction)
     delete!(graph.possibleOperations, operation)
 
@@ -160,7 +229,13 @@ function invalidate_caches!(graph::DAG, operation::NodeReduction)
     return nothing
 end
 
-# function to invalidate the operation caches for a given NodeSplit
+"""
+    invalidate_caches!(graph::DAG, operation::NodeSplit)
+
+Invalidate the operation caches for a given [`NodeSplit`](@ref).
+
+This deletes the operation from the graph's possible operations and from the involved nodes' own operation caches.
+"""
 function invalidate_caches!(graph::DAG, operation::NodeSplit)
     delete!(graph.possibleOperations, operation)
 
@@ -171,7 +246,11 @@ function invalidate_caches!(graph::DAG, operation::NodeSplit)
     return nothing
 end
 
-# function to invalidate the operation caches of a ComputeTaskNode
+"""
+    invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode)
+
+Invalidate the operation caches of the given node through calls to the respective [`invalidate_caches!`](@ref) functions.
+"""
 function invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode)
     if !ismissing(node.nodeReduction)
         invalidate_caches!(graph, node.nodeReduction)
@@ -185,7 +264,11 @@ function invalidate_operation_caches!(graph::DAG, node::ComputeTaskNode)
     return nothing
 end
 
-# function to invalidate the operation caches of a DataTaskNode
+"""
+    invalidate_operation_caches!(graph::DAG, node::DataTaskNode)
+
+Invalidate the operation caches of the given node through calls to the respective [`invalidate_caches!`](@ref) functions.
+"""
 function invalidate_operation_caches!(graph::DAG, node::DataTaskNode)
     if !ismissing(node.nodeReduction)
         invalidate_caches!(graph, node.nodeReduction)

src/graph/print.jl

@@ -1,4 +1,9 @@
-function show_nodes(io, graph::DAG)
+"""
+    show_nodes(io::IO, graph::DAG)
+
+Print a graph's nodes. Should only be used for small graphs as it prints every node in a list.
+"""
+function show_nodes(io::IO, graph::DAG)
     print(io, "[")
     first = true
     for n in graph.nodes
@@ -12,7 +17,13 @@ function show_nodes(io, graph::DAG)
     return print(io, "]")
 end
 
+"""
+    show(io::IO, graph::DAG)
+
+Print the given graph to io. If there are too many nodes it will print only a summary of them.
+"""
 function show(io::IO, graph::DAG)
+    apply_all!(graph)
     println(io, "Graph:")
     print(io, "  Nodes: ")
 
@@ -48,12 +59,12 @@ function show(io::IO, graph::DAG)
     end
     println(io)
     println(io, "  Edges: ", noEdges)
-    properties = graph_properties(graph)
-    println(io, "  Total Compute Effort: ", properties.compute_effort)
+    properties = get_properties(graph)
+    println(io, "  Total Compute Effort: ", properties.computeEffort)
     println(io, "  Total Data Transfer: ", properties.data)
     return println(
         io,
         "  Total Compute Intensity: ",
-        properties.compute_intensity,
+        properties.computeIntensity,
     )
 end

src/graph/properties.jl

@@ -1,28 +1,24 @@
-function graph_properties(graph::DAG)
+"""
+    get_properties(graph::DAG)
+
+Return the graph's [`GraphProperties`](@ref).
+"""
+function get_properties(graph::DAG)
     # make sure the graph is fully generated
     apply_all!(graph)
 
-    d = 0
-    ce = 0
-    ed = 0
-    for node in graph.nodes
-        d += data(node.task) * length(node.parents)
-        ce += compute_effort(node.task)
-        ed += length(node.parents)
+    if (graph.properties.computeEffort == 0.0)
+        graph.properties = GraphProperties(graph)
     end
 
-    ci = ce / d
-
-    result = (
-        data = d,
-        compute_effort = ce,
-        compute_intensity = ci,
-        nodes = length(graph.nodes),
-        edges = ed,
-    )
-    return result
+    return graph.properties
 end
 
+"""
+    get_exit_node(graph::DAG)
+
+Return the graph's exit node. This assumes the graph only has a single exit node. If the graph has multiple exit nodes, the one encountered first will be returned.
+"""
 function get_exit_node(graph::DAG)
     for node in graph.nodes
         if (is_exit_node(node))
@@ -31,3 +27,18 @@ function get_exit_node(graph::DAG)
     end
     @assert false "The given graph has no exit node! It is either empty or not acyclic!"
 end
+
+"""
+    get_entry_nodes(graph::DAG)
+
+Return a vector of the graph's entry nodes.
+"""
+function get_entry_nodes(graph::DAG)
+    result = Vector{Node}()
+    for node in graph.nodes
+        if (is_entry_node(node))
+            push!(result, node)
+        end
+    end
+    return result
+end

src/graph/type.jl

@@ -1,21 +1,28 @@
 using DataStructures
 
+"""
+    PossibleOperations
+
+A struct storing all possible operations on a [`DAG`](@ref).
+To get the [`PossibleOperations`](@ref) on a [`DAG`](@ref), use [`get_operations`](@ref).
+"""
 mutable struct PossibleOperations
     nodeFusions::Set{NodeFusion}
     nodeReductions::Set{NodeReduction}
     nodeSplits::Set{NodeSplit}
 end
 
-function PossibleOperations()
-    return PossibleOperations(
-        Set{NodeFusion}(),
-        Set{NodeReduction}(),
-        Set{NodeSplit}(),
-    )
-end
+""" 
+    DAG
 
-# The actual state of the DAG is the initial state given by the set of nodes 
-# but with all the operations in appliedChain applied in order
+The representation of the graph as a set of [`Node`](@ref)s.
+
+A DAG can be loaded using the appropriate parse function, e.g. [`parse_abc`](@ref).
+
+[`Operation`](@ref)s can be applied on it using [`push_operation!`](@ref) and reverted using [`pop_operation!`](@ref) like a stack.
+To get the set of possible operations, use [`get_operations`](@ref).
+The members of the object should not be manually accessed, instead always use the provided interface functions.
+"""
 mutable struct DAG
     nodes::Set{Node}
 
@@ -34,8 +41,29 @@ mutable struct DAG
     # "snapshot" system: keep track of added/removed nodes/edges since last snapshot
     # these are muted in insert_node! etc.
     diff::Diff
+
+    # the cached properties of the DAG
+    properties::GraphProperties
 end
 
+"""
+    PossibleOperations()
+
+Construct and return an empty [`PossibleOperations`](@ref) object.
+"""
+function PossibleOperations()
+    return PossibleOperations(
+        Set{NodeFusion}(),
+        Set{NodeReduction}(),
+        Set{NodeSplit}(),
+    )
+end
+
+"""
+    DAG()
+
+Construct and return an empty [`DAG`](@ref).
+"""
 function DAG()
     return DAG(
         Set{Node}(),
@@ -44,5 +72,6 @@ function DAG()
         PossibleOperations(),
         Set{Node}(),
         Diff(),
+        GraphProperties(),
     )
 end

src/graph/validate.jl

@@ -1,4 +1,8 @@
-# check whether the given graph is connected
+"""
+    is_connected(graph::DAG)
+
+Return whether the given graph is connected.
+"""
 function is_connected(graph::DAG)
     nodeQueue = Deque{Node}()
     push!(nodeQueue, get_exit_node(graph))
@@ -16,6 +20,11 @@ function is_connected(graph::DAG)
     return length(seenNodes) == length(graph.nodes)
 end
 
+"""
+    is_valid(graph::DAG)
+
+Validate the entire graph using asserts. Intended for testing with `@assert is_valid(graph)`.
+"""
 function is_valid(graph::DAG)
     for node in graph.nodes
         @assert is_valid(graph, node)

src/models/abc/compute.jl

@@ -0,0 +1,256 @@
+using AccurateArithmetic
+
+"""
+    compute(::ComputeTaskP, data::ParticleValue)
+
+Return the particle and value as is. 
+
+0 FLOP.
+"""
+function compute(::ComputeTaskP, data::ParticleValue)
+    return data
+end
+
+"""
+    compute(::ComputeTaskU, data::ParticleValue)
+
+Compute an outer edge. Return the particle value with the same particle and the value multiplied by an outer_edge factor.
+
+1 FLOP.
+"""
+function compute(::ComputeTaskU, data::ParticleValue)
+    return ParticleValue(data.p, data.v * outer_edge(data.p))
+end
+
+"""
+    compute(::ComputeTaskV, data1::ParticleValue, data2::ParticleValue)
+
+Compute a vertex. Preserve momentum and particle types (AB->C etc.) to create resulting particle, multiply values together and times a vertex factor.
+
+6 FLOP.
+"""
+function compute(::ComputeTaskV, data1::ParticleValue, data2::ParticleValue)
+    p3 = preserve_momentum(data1.p, data2.p)
+    dataOut = ParticleValue(p3, data1.v * vertex() * data2.v)
+    return dataOut
+end
+
+"""
+    compute(::ComputeTaskS2, data1::ParticleValue, data2::ParticleValue)
+
+Compute a final inner edge (2 input particles, no output particle).
+
+For valid inputs, both input particles should have the same momenta at this point.
+
+12 FLOP.
+"""
+function compute(::ComputeTaskS2, data1::ParticleValue, data2::ParticleValue)
+    return data1.v * inner_edge(data1.p) * data2.v
+end
+
+"""
+    compute(::ComputeTaskS1, data::ParticleValue)
+
+Compute inner edge (1 input particle, 1 output particle).
+
+11 FLOP.
+"""
+function compute(::ComputeTaskS1, data::ParticleValue)
+    return ParticleValue(data.p, data.v * inner_edge(data.p))
+end
+
+"""
+    compute(::ComputeTaskSum, data::Vector{Float64})
+
+Compute a sum over the vector. Use an algorithm that accounts for accumulated errors in long sums with potentially large differences in magnitude of the summands.
+
+Linearly many FLOP with growing data.
+"""
+function compute(::ComputeTaskSum, data::Vector{Float64})
+    return sum_kbn(data)
+end
+
+"""
+    compute(t::FusedComputeTask, data)
+
+Compute a [`FusedComputeTask`](@ref). This simply asserts false and should not be called. Fused Compute Tasks generate their expressions directly through the other tasks instead.
+"""
+function compute(t::FusedComputeTask, data)
+    @assert false "This is not implemented and should never be called"
+end
+
+"""
+    get_expression(::ComputeTaskP, inSymbol::Symbol, outSymbol::Symbol)
+
+Generate and return code evaluating [`ComputeTaskP`](@ref) on `inSymbol`, providing the output on `outSymbol`.
+"""
+function get_expression(::ComputeTaskP, inSymbol::Symbol, outSymbol::Symbol)
+    return Meta.parse("$outSymbol = compute(ComputeTaskP(), $inSymbol)")
+end
+
+"""
+    get_expression(::ComputeTaskU, inSymbol::Symbol, outSymbol::Symbol)
+
+Generate code evaluating [`ComputeTaskU`](@ref) on `inSymbol`, providing the output on `outSymbol`.
+`inSymbol` should be of type [`ParticleValue`](@ref), `outSymbol` will be of type [`ParticleValue`](@ref).
+"""
+function get_expression(::ComputeTaskU, inSymbol::Symbol, outSymbol::Symbol)
+    return Meta.parse("$outSymbol = compute(ComputeTaskU(), $inSymbol)")
+end
+
+"""
+    get_expression(::ComputeTaskV, inSymbol1::Symbol, inSymbol2::Symbol, outSymbol::Symbol)
+
+Generate code evaluating [`ComputeTaskV`](@ref) on `inSymbol1` and `inSymbol2`, providing the output on `outSymbol`.
+`inSymbol1` and `inSymbol2` should be of type [`ParticleValue`](@ref), `outSymbol` will be of type [`ParticleValue`](@ref).
+"""
+function get_expression(
+    ::ComputeTaskV,
+    inSymbol1::Symbol,
+    inSymbol2::Symbol,
+    outSymbol::Symbol,
+)
+    return Meta.parse(
+        "$outSymbol = compute(ComputeTaskV(), $inSymbol1, $inSymbol2)",
+    )
+end
+
+"""
+    get_expression(::ComputeTaskS2, inSymbol1::Symbol, inSymbol2::Symbol, outSymbol::Symbol)
+
+Generate code evaluating [`ComputeTaskS2`](@ref) on `inSymbol1` and `inSymbol2`, providing the output on `outSymbol`.
+`inSymbol1` and `inSymbol2` should be of type [`ParticleValue`](@ref), `outSymbol` will be of type `Float64`.
+"""
+function get_expression(
+    ::ComputeTaskS2,
+    inSymbol1::Symbol,
+    inSymbol2::Symbol,
+    outSymbol::Symbol,
+)
+    return Meta.parse(
+        "$outSymbol = compute(ComputeTaskS2(), $inSymbol1, $inSymbol2)",
+    )
+end
+
+"""
+    get_expression(::ComputeTaskS1, inSymbol::Symbol, outSymbol::Symbol)
+
+Generate code evaluating [`ComputeTaskS1`](@ref) on `inSymbol`, providing the output on `outSymbol`.
+`inSymbol` should be of type [`ParticleValue`](@ref), `outSymbol` will be of type [`ParticleValue`](@ref).
+"""
+function get_expression(::ComputeTaskS1, inSymbol::Symbol, outSymbol::Symbol)
+    return Meta.parse("$outSymbol = compute(ComputeTaskS1(), $inSymbol)")
+end
+
+"""
+    get_expression(::ComputeTaskSum, inSymbols::Vector{Symbol}, outSymbol::Symbol)
+
+Generate code evaluating [`ComputeTaskSum`](@ref) on `inSymbols`, providing the output on `outSymbol`.
+`inSymbols` should be of type [`Float64`], `outSymbol` will be of type [`Float64`].
+"""
+function get_expression(
+    ::ComputeTaskSum,
+    inSymbols::Vector{Symbol},
+    outSymbol::Symbol,
+)
+    return quote
+        $outSymbol = compute(ComputeTaskSum(), [$(inSymbols...)])
+    end
+end
+
+"""
+    get_expression(t::FusedComputeTask, inSymbols::Vector{Symbol}, outSymbol::Symbol)
+
+Generate code evaluating a [`FusedComputeTask`](@ref) on `inSymbols`, providing the output on `outSymbol`.
+`inSymbols` should be of the correct types and may be heterogeneous. `outSymbol` will be of the type of the output of `T2` of t.
+"""
+function get_expression(
+    t::FusedComputeTask,
+    inSymbols::Vector{Symbol},
+    outSymbol::Symbol,
+)
+    (T1, T2) = get_types(t)
+    c1 = children(T1())
+    c2 = children(T2())
+
+    expr1 = nothing
+    expr2 = nothing
+
+    # TODO need to figure out how to know which inputs belong to which subtask
+    # since we order the vectors with the child nodes we can't just split
+    if (c1 == 1)
+        expr1 = get_expression(T1(), inSymbols[begin], :intermediate)
+    elseif (c1 == 2)
+        expr1 =
+            get_expression(T1(), inSymbols[begin], inSymbols[2], :intermediate)
+    else
+        expr1 = get_expression(T1(), inSymbols[begin:c1], :intermediate)
+    end
+
+    if (c2 == 1)
+        expr2 = get_expression(T2(), :intermediate, outSymbol)
+    elseif c2 == 2
+        expr2 =
+            get_expression(T2(), :intermediate, inSymbols[c1 + 1], outSymbol)
+    else
+        expr2 = get_expression(
+            T2(),
+            :intermediate * inSymbols[(c1 + 1):end],
+            outSymbol,
+        )
+    end
+
+    return Expr(:block, expr1, expr2)
+end
+
+"""
+    get_expression(node::ComputeTaskNode)
+
+Generate and return code for a given [`ComputeTaskNode`](@ref).
+"""
+function get_expression(node::ComputeTaskNode)
+    t = typeof(node.task)
+    @assert length(node.children) == children(node.task) || t <: ComputeTaskSum
+
+    if (t <: ComputeTaskU || t <: ComputeTaskP || t <: ComputeTaskS1) # single input
+        symbolIn = Symbol("data_$(to_var_name(node.children[1].id))")
+        symbolOut = Symbol("data_$(to_var_name(node.id))")
+        return get_expression(t(), symbolIn, symbolOut)
+    elseif (t <: ComputeTaskS2 || t <: ComputeTaskV) # double input
+        symbolIn1 = Symbol("data_$(to_var_name(node.children[1].id))")
+        symbolIn2 = Symbol("data_$(to_var_name(node.children[2].id))")
+        symbolOut = Symbol("data_$(to_var_name(node.id))")
+        return get_expression(t(), symbolIn1, symbolIn2, symbolOut)
+    elseif (t <: ComputeTaskSum || t <: FusedComputeTask) # vector input
+        inSymbols = Vector{Symbol}()
+        for child in node.children
+            push!(inSymbols, Symbol("data_$(to_var_name(child.id))"))
+        end
+        outSymbol = Symbol("data_$(to_var_name(node.id))")
+        return get_expression(t(), inSymbols, outSymbol)
+    else
+        error("Unknown compute task")
+    end
+end
+
+"""
+    get_expression(node::DataTaskNode)
+
+Generate and return code for a given [`DataTaskNode`](@ref).
+"""
+function get_expression(node::DataTaskNode)
+    # TODO: do things to transport data from/to gpu, between numa nodes, etc.
+    @assert length(node.children) <= 1
+
+    inSymbol = nothing
+    if (length(node.children) == 1)
+        inSymbol = Symbol("data_$(to_var_name(node.children[1].id))")
+    else
+        inSymbol = Symbol("data_$(to_var_name(node.id))_in")
+    end
+    outSymbol = Symbol("data_$(to_var_name(node.id))")
+
+    dataTransportExp = Meta.parse("$outSymbol = $inSymbol")
+
+    return dataTransportExp
+end

src/models/abc/create.jl

@@ -0,0 +1,74 @@
+
+"""
+    Particle(rng)
+
+Return a randomly generated particle.
+"""
+function Particle(rng, type::ParticleType)
+
+    p1 = rand(rng, Float64)
+    p2 = rand(rng, Float64)
+    p3 = rand(rng, Float64)
+    m = mass(type)
+
+    # keep the momenta of the particles on-shell
+    p4 = sqrt(p1^2 + p2^2 + p3^2 + m^2)
+
+    return Particle(p1, p2, p3, p4, type)
+end
+
+"""
+    gen_particles(n::Int)
+
+Return a Vector of `n` randomly generated [`Particle`](@ref)s.
+
+Note: This does not take into account the preservation of momenta required for an actual valid process!
+"""
+function gen_particles(ns::Dict{ParticleType, Int})
+    particles = Dict{ParticleType, Vector{Particle}}()
+    rng = MersenneTwister(0)
+
+
+    if ns == Dict((A => 2), (B => 2))
+        rho = 1.0
+
+        omega = rand(rng, Float64)
+        theta = rand(rng, Float64) * π
+        phi = rand(rng, Float64) * π
+
+        particles[A] = Vector{Particle}()
+        particles[B] = Vector{Particle}()
+
+        push!(particles[A], Particle(omega, 0, 0, omega, A))
+        push!(particles[B], Particle(omega, 0, 0, -omega, B))
+        push!(
+            particles[A],
+            Particle(
+                omega,
+                rho * cos(theta) * cos(phi),
+                rho * cos(theta) * sin(phi),
+                rho * sin(theta),
+                A,
+            ),
+        )
+        push!(
+            particles[B],
+            Particle(
+                omega,
+                -rho * cos(theta) * cos(phi),
+                -rho * cos(theta) * sin(phi),
+                -rho * sin(theta),
+                B,
+            ),
+        )
+        return particles
+    end
+
+    for (type, n) in ns
+        particles[type] = Vector{Particle}()
+        for i in 1:n
+            push!(particles[type], Particle(rng, type))
+        end
+    end
+    return particles
+end

src/models/abc/parse.jl

@@ -1,11 +1,17 @@
-using Printf
-
 # functions for importing DAGs from a file
 regex_a = r"^[A-C]\d+$"                     # Regex for the initial particles
 regex_c = r"^[A-C]\(([^']*),([^']*)\)$"     # Regex for the combinations of 2 particles
 regex_m = r"^M\(([^']*),([^']*),([^']*)\)$" # Regex for the combinations of 3 particles
 regex_plus = r"^\+$"                        # Regex for the sum
 
+const PARTICLE_VALUE_SIZE::Int = 48
+const FLOAT_SIZE::Int = 8
+
+"""
+    parse_nodes(input::AbstractString)
+
+Parse the given string into a vector of strings containing each node.
+"""
 function parse_nodes(input::AbstractString)
     regex = r"'([^']*)'"
     matches = eachmatch(regex, input)
@@ -13,6 +19,11 @@ function parse_nodes(input::AbstractString)
     return output
 end
 
+"""
+    parse_edges(input::AbstractString)
+
+Parse the given string into a vector of strings containing each edge. Currently unused since the entire graph can be read from just the node names.
+"""
 function parse_edges(input::AbstractString)
     regex = r"\('([^']*)', '([^']*)'\)"
     matches = eachmatch(regex, input)
@@ -20,7 +31,13 @@ function parse_edges(input::AbstractString)
     return output
 end
 
-# reads an abc-model process from the given file
+"""
+    parse_abc(filename::String; verbose::Bool = false)
+
+Read an abc-model process from the given file. If `verbose` is set to true, print some progress information to stdout.
+
+Returns a valid [`DAG`](@ref).
+"""
 function parse_abc(filename::String, verbose::Bool = false)
     file = open(filename, "r")
 
@@ -47,7 +64,8 @@ function parse_abc(filename::String, verbose::Bool = false)
     sizehint!(graph.nodes, estimate_no_nodes)
 
     sum_node = insert_node!(graph, make_node(ComputeTaskSum()), false, false)
-    global_data_out = insert_node!(graph, make_node(DataTask(10)), false, false)
+    global_data_out =
+        insert_node!(graph, make_node(DataTask(FLOAT_SIZE)), false, false)
     insert_edge!(graph, sum_node, global_data_out, false, false)
 
     # remember the data out nodes for connection
@@ -63,20 +81,37 @@ function parse_abc(filename::String, verbose::Bool = false)
         noNodes += 1
         if (noNodes % 100 == 0)
             if (verbose)
-                @printf "\rReading Nodes... %.2f%%" (
-                    100.0 * noNodes / nodesToRead
+                percent = string(
+                    round(100.0 * noNodes / nodesToRead, digits = 2),
+                    "%",
                 )
+                print("\rReading Nodes... $percent")
             end
         end
         if occursin(regex_a, node)
             # add nodes and edges for the state reading to u(P(Particle))
-            data_in = insert_node!(graph, make_node(DataTask(4)), false, false) # read particle data node
+            data_in = insert_node!(
+                graph,
+                make_node(DataTask(PARTICLE_VALUE_SIZE), string(node)),
+                false,
+                false,
+            ) # read particle data node
             compute_P =
                 insert_node!(graph, make_node(ComputeTaskP()), false, false) # compute P node
-            data_Pu = insert_node!(graph, make_node(DataTask(6)), false, false) # transfer data from P to u
+            data_Pu = insert_node!(
+                graph,
+                make_node(DataTask(PARTICLE_VALUE_SIZE)),
+                false,
+                false,
+            ) # transfer data from P to u (one ParticleValue object)
             compute_u =
                 insert_node!(graph, make_node(ComputeTaskU()), false, false) # compute U node
-            data_out = insert_node!(graph, make_node(DataTask(3)), false, false) # transfer data out from u
+            data_out = insert_node!(
+                graph,
+                make_node(DataTask(PARTICLE_VALUE_SIZE)),
+                false,
+                false,
+            ) # transfer data out from u (one ParticleValue object)
 
             insert_edge!(graph, data_in, compute_P, false, false)
             insert_edge!(graph, compute_P, data_Pu, false, false)
@@ -93,7 +128,12 @@ function parse_abc(filename::String, verbose::Bool = false)
 
             compute_v =
                 insert_node!(graph, make_node(ComputeTaskV()), false, false)
-            data_out = insert_node!(graph, make_node(DataTask(5)), false, false)
+            data_out = insert_node!(
+                graph,
+                make_node(DataTask(PARTICLE_VALUE_SIZE)),
+                false,
+                false,
+            )
 
             if (occursin(regex_c, in1))
                 # put an S node after this input
@@ -103,8 +143,12 @@ function parse_abc(filename::String, verbose::Bool = false)
                     false,
                     false,
                 )
-                data_S_v =
-                    insert_node!(graph, make_node(DataTask(5)), false, false)
+                data_S_v = insert_node!(
+                    graph,
+                    make_node(DataTask(PARTICLE_VALUE_SIZE)),
+                    false,
+                    false,
+                )
 
                 insert_edge!(graph, dataOutNodes[in1], compute_S, false, false)
                 insert_edge!(graph, compute_S, data_S_v, false, false)
@@ -123,8 +167,12 @@ function parse_abc(filename::String, verbose::Bool = false)
                     false,
                     false,
                 )
-                data_S_v =
-                    insert_node!(graph, make_node(DataTask(5)), false, false)
+                data_S_v = insert_node!(
+                    graph,
+                    make_node(DataTask(PARTICLE_VALUE_SIZE)),
+                    false,
+                    false,
+                )
 
                 insert_edge!(graph, dataOutNodes[in2], compute_S, false, false)
                 insert_edge!(graph, compute_S, data_S_v, false, false)
@@ -147,7 +195,12 @@ function parse_abc(filename::String, verbose::Bool = false)
             # in2 + in3 with a v
             compute_v =
                 insert_node!(graph, make_node(ComputeTaskV()), false, false)
-            data_v = insert_node!(graph, make_node(DataTask(5)), false, false)
+            data_v = insert_node!(
+                graph,
+                make_node(DataTask(PARTICLE_VALUE_SIZE)),
+                false,
+                false,
+            )
 
             insert_edge!(graph, dataOutNodes[in2], compute_v, false, false)
             insert_edge!(graph, dataOutNodes[in3], compute_v, false, false)
@@ -156,8 +209,12 @@ function parse_abc(filename::String, verbose::Bool = false)
             # combine with the v of the combined other input
             compute_S2 =
                 insert_node!(graph, make_node(ComputeTaskS2()), false, false)
-            data_out =
-                insert_node!(graph, make_node(DataTask(10)), false, false)
+            data_out = insert_node!(
+                graph,
+                make_node(DataTask(FLOAT_SIZE)),
+                false,
+                false,
+            ) # output of a S2 task is only a float
 
             insert_edge!(graph, data_v, compute_S2, false, false)
             insert_edge!(graph, dataOutNodes[in1], compute_S2, false, false)
@@ -179,6 +236,14 @@ function parse_abc(filename::String, verbose::Bool = false)
     #put all nodes into dirty nodes set
     graph.dirtyNodes = copy(graph.nodes)
 
+    if (verbose)
+        println("Generating the graph's properties")
+    end
+    graph.properties = GraphProperties(graph)
+
+    if (verbose)
+        println("Done")
+    end
     # don't actually need to read the edges
     return graph
 end

src/models/abc/particle.jl

@@ -0,0 +1,130 @@
+"""
+    ParticleType
+
+A Particle Type in the ABC Model as an enum, with types `A`, `B` and `C`.
+"""
+@enum ParticleType A = 1 B = 2 C = 3
+
+"""
+    PARTICLE_MASSES
+
+A constant dictionary containing the masses of the different [`ParticleType`](@ref)s.
+"""
+const PARTICLE_MASSES =
+    Dict{ParticleType, Float64}(A => 1.0, B => 1.0, C => 0.0)
+
+"""
+    Particle
+
+A struct describing a particle of the ABC-Model. It has the 4 momentum parts P0...P3 and a [`ParticleType`](@ref).
+
+`sizeof(Particle())` = 40 Byte
+"""
+struct Particle
+    P0::Float64
+    P1::Float64
+    P2::Float64
+    P3::Float64
+
+    type::ParticleType
+end
+
+"""
+    ParticleValue
+
+A struct describing a particle during a calculation of a Feynman Diagram, together with the value that's being calculated.
+
+`sizeof(ParticleValue())` = 48 Byte
+"""
+struct ParticleValue
+    p::Particle
+    v::Float64
+end
+
+"""
+    mass(t::ParticleType)
+    
+Return the mass (at rest) of the given particle type.
+"""
+mass(t::ParticleType) = PARTICLE_MASSES[t]
+
+"""
+    remaining_type(t1::ParticleType, t2::ParticleType)
+
+For 2 given (non-equal) particle types, return the third of ABC.
+"""
+function remaining_type(t1::ParticleType, t2::ParticleType)
+    @assert t1 != t2
+    if t1 != A && t2 != A
+        return A
+    elseif t1 != B && t2 != B
+        return B
+    else
+        return C
+    end
+end
+
+"""
+    square(p::Particle)
+
+Return the square of the particle's momentum as a `Float` value.
+
+Takes 7 effective FLOP.
+"""
+function square(p::Particle)
+    return p.P0 * p.P0 - p.P1 * p.P1 - p.P2 * p.P2 - p.P3 * p.P3
+end
+
+"""
+    inner_edge(p::Particle)
+
+Return the factor of the inner edge with the given (virtual) particle.
+
+Takes 10 effective FLOP. (3 here + 10 in square(p))
+"""
+function inner_edge(p::Particle)
+    return 1.0 / (square(p) - mass(p.type) * mass(p.type))
+end
+
+"""
+    outer_edge(p::Particle)
+
+Return the factor of the outer edge with the given (real) particle.
+
+Takes 0 effective FLOP.
+"""
+function outer_edge(p::Particle)
+    return 1.0
+end
+
+"""
+    vertex()
+
+Return the factor of a vertex.
+
+Takes 0 effective FLOP since it's constant.
+"""
+function vertex()
+    i = 1.0
+    lambda = 1.0 / 137.0
+    return i * lambda
+end
+
+"""
+    preserve_momentum(p1::Particle, p2::Particle)
+
+Calculate and return a new particle from two given interacting ones at a vertex.
+
+Takes 4 effective FLOP.
+"""
+function preserve_momentum(p1::Particle, p2::Particle)
+    p3 = Particle(
+        p1.P0 + p2.P0,
+        p1.P1 + p2.P1,
+        p1.P2 + p2.P2,
+        p1.P3 + p2.P3,
+        remaining_type(p1.type, p2.type),
+    )
+
+    return p3
+end

src/models/abc/properties.jl

@@ -1,21 +1,165 @@
-# define compute_efforts tasks computation
-# put some "random" numbers here for now
-compute_effort(t::ComputeTaskS1) = 10
-compute_effort(t::ComputeTaskS2) = 10
-compute_effort(t::ComputeTaskU) = 6
-compute_effort(t::ComputeTaskV) = 20
-compute_effort(t::ComputeTaskP) = 15
+"""
+    compute_effort(t::ComputeTaskS1)
+
+Return the compute effort of an S1 task.
+"""
+compute_effort(t::ComputeTaskS1) = 11
+
+"""
+    compute_effort(t::ComputeTaskS2)
+
+Return the compute effort of an S2 task.
+"""
+compute_effort(t::ComputeTaskS2) = 12
+
+"""
+    compute_effort(t::ComputeTaskU)
+
+Return the compute effort of a U task.
+"""
+compute_effort(t::ComputeTaskU) = 1
+
+"""
+    compute_effort(t::ComputeTaskV)
+
+Return the compute effort of a V task.
+"""
+compute_effort(t::ComputeTaskV) = 6
+
+"""
+    compute_effort(t::ComputeTaskP)
+
+Return the compute effort of a P task.
+"""
+compute_effort(t::ComputeTaskP) = 0
+
+"""
+    compute_effort(t::ComputeTaskSum)
+
+Return the compute effort of a Sum task. 
+
+Note: This is a constant compute effort, even though sum scales with the number of its inputs. Since there is only ever a single sum node in a graph generated from the ABC-Model,
+this doesn't matter.
+"""
 compute_effort(t::ComputeTaskSum) = 1
 
+"""
+    show(io::IO, t::DataTask)
+
+Print the data task to io.
+"""
 function show(io::IO, t::DataTask)
     return print(io, "Data", t.data)
 end
 
+"""
+    show(io::IO, t::ComputeTaskS1)
+
+Print the S1 task to io.
+"""
 show(io::IO, t::ComputeTaskS1) = print("ComputeS1")
+
+"""
+    show(io::IO, t::ComputeTaskS2)
+
+Print the S2 task to io.
+"""
 show(io::IO, t::ComputeTaskS2) = print("ComputeS2")
+
+"""
+    show(io::IO, t::ComputeTaskP)
+
+Print the P task to io.
+"""
 show(io::IO, t::ComputeTaskP) = print("ComputeP")
+
+"""
+    show(io::IO, t::ComputeTaskU)
+
+Print the U task to io.
+"""
 show(io::IO, t::ComputeTaskU) = print("ComputeU")
+
+"""
+    show(io::IO, t::ComputeTaskV)
+
+Print the V task to io.
+"""
 show(io::IO, t::ComputeTaskV) = print("ComputeV")
+
+"""
+    show(io::IO, t::ComputeTaskSum)
+
+Print the sum task to io.
+"""
 show(io::IO, t::ComputeTaskSum) = print("ComputeSum")
 
+"""
+    copy(t::DataTask)
+
+Copy the data task and return it.
+"""
 copy(t::DataTask) = DataTask(t.data)
+
+"""
+    children(::DataTask)
+
+Return the number of children of a data task (always 1).
+"""
+children(::DataTask) = 1
+
+"""
+    children(::ComputeTaskS1)
+
+Return the number of children of a ComputeTaskS1 (always 1).
+"""
+children(::ComputeTaskS1) = 1
+
+"""
+    children(::ComputeTaskS2)
+
+Return the number of children of a ComputeTaskS2 (always 2).
+"""
+children(::ComputeTaskS2) = 2
+
+"""
+    children(::ComputeTaskP)
+
+Return the number of children of a ComputeTaskP (always 1).
+"""
+children(::ComputeTaskP) = 1
+
+"""
+    children(::ComputeTaskU)
+
+Return the number of children of a ComputeTaskU (always 1).
+"""
+children(::ComputeTaskU) = 1
+
+"""
+    children(::ComputeTaskV)
+
+Return the number of children of a ComputeTaskV (always 2).
+"""
+children(::ComputeTaskV) = 2
+
+
+"""
+    children(::ComputeTaskSum)
+
+Return the number of children of a ComputeTaskSum, since this is variable and the task doesn't know
+how many children it will sum over, return a wildcard -1.
+
+TODO: this is kind of bad because it means we can't fuse with a sum task
+"""
+children(::ComputeTaskSum) = -1
+
+"""
+    children(t::FusedComputeTask)
+
+Return the number of children of a FusedComputeTask. It's the sum of the children of both tasks minus one.
+"""
+function children(t::FusedComputeTask)
+    (T1, T2) = get_types(t)
+    return children(T1()) + children(T2()) - 1 # one of the inputs is the output of T1 and thus not a child of the node
+end

src/models/abc/types.jl

@@ -1,25 +1,59 @@
+"""
+    DataTask <: AbstractDataTask
+
+Task representing a specific data transfer in the ABC Model.
+"""
 struct DataTask <: AbstractDataTask
     data::UInt64
 end
 
-# S task with 1 child
+"""
+    ComputeTaskS1 <: AbstractComputeTask
+
+S task with a single child.
+"""
 struct ComputeTaskS1 <: AbstractComputeTask end
 
-# S task with 2 children
+"""
+    ComputeTaskS2 <: AbstractComputeTask
+
+S task with two children.
+"""
 struct ComputeTaskS2 <: AbstractComputeTask end
 
-# P task with 0 children
+"""
+    ComputeTaskP <: AbstractComputeTask
+
+P task with no children.
+"""
 struct ComputeTaskP <: AbstractComputeTask end
 
-# v task with 2 children
+"""
+    ComputeTaskV <: AbstractComputeTask
+
+v task with two children.
+"""
 struct ComputeTaskV <: AbstractComputeTask end
 
-# u task with 1 child
+"""
+    ComputeTaskU <: AbstractComputeTask
+
+u task with a single child.
+"""
 struct ComputeTaskU <: AbstractComputeTask end
 
-# task that sums all its inputs, n children
+"""
+    ComputeTaskSum <: AbstractComputeTask
+
+Task that sums all its inputs, n children.
+"""
 struct ComputeTaskSum <: AbstractComputeTask end
 
+"""
+    ABC_TASKS
+
+Constant vector of all tasks of the ABC-Model.
+"""
 ABC_TASKS = [
     DataTask,
     ComputeTaskS1,

src/node/compare.jl

@@ -1,15 +1,35 @@
+"""
+    ==(e1::Edge, e2::Edge)
+
+Equality comparison between two edges.
+"""
 function ==(e1::Edge, e2::Edge)
     return e1.edge[1] == e2.edge[1] && e1.edge[2] == e2.edge[2]
 end
 
+"""
+    ==(n1::Node, n2::Node)
+
+Fallback equality comparison between two nodes. For equal node types, the more specific versions of this function will be called.
+"""
 function ==(n1::Node, n2::Node)
     return false
 end
 
+"""
+    ==(n1::ComputeTaskNode, n2::ComputeTaskNode)
+
+Equality comparison between two [`ComputeTaskNode`](@ref)s.
+"""
 function ==(n1::ComputeTaskNode, n2::ComputeTaskNode)
     return n1.id == n2.id
 end
 
+"""
+    ==(n1::DataTaskNode, n2::DataTaskNode)
+
+Equality comparison between two [`DataTaskNode`](@ref)s.
+"""
 function ==(n1::DataTaskNode, n2::DataTaskNode)
     return n1.id == n2.id
 end

src/node/create.jl

@@ -1,23 +1,95 @@
+
+DataTaskNode(t::AbstractDataTask, name = "") = DataTaskNode(
+    t,
+    Vector{Node}(),
+    Vector{Node}(),
+    UUIDs.uuid1(rng[threadid()]),
+    missing,
+    missing,
+    missing,
+    name,
+)
+ComputeTaskNode(t::AbstractComputeTask) = ComputeTaskNode(
+    t,
+    Vector{Node}(),
+    Vector{Node}(),
+    UUIDs.uuid1(rng[threadid()]),
+    missing,
+    missing,
+    Vector{NodeFusion}(),
+)
+
+copy(m::Missing) = missing
+copy(n::ComputeTaskNode) = ComputeTaskNode(
+    copy(n.task),
+    copy(n.parents),
+    copy(n.children),
+    UUIDs.uuid1(rng[threadid()]),
+    copy(n.nodeReduction),
+    copy(n.nodeSplit),
+    copy(n.nodeFusions),
+)
+copy(n::DataTaskNode) = DataTaskNode(
+    copy(n.task),
+    copy(n.parents),
+    copy(n.children),
+    UUIDs.uuid1(rng[threadid()]),
+    copy(n.nodeReduction),
+    copy(n.nodeSplit),
+    copy(n.nodeFusion),
+    n.name,
+)
+
+"""
+    make_node(t::AbstractTask)
+
+Fallback implementation of `make_node` for an [`AbstractTask`](@ref), throwing an error.
+"""
 function make_node(t::AbstractTask)
     return error("Cannot make a node from this task type")
 end
 
-function make_node(t::AbstractDataTask)
-    return DataTaskNode(t)
+"""
+    make_node(t::AbstractDataTask)
+
+Construct and return a new [`DataTaskNode`](@ref) with the given task.
+"""
+function make_node(t::AbstractDataTask, name::String = "")
+    return DataTaskNode(t, name)
 end
 
+"""
+    make_node(t::AbstractComputeTask)
+
+Construct and return a new [`ComputeTaskNode`](@ref) with the given task.
+"""
 function make_node(t::AbstractComputeTask)
     return ComputeTaskNode(t)
 end
 
+"""
+    make_edge(n1::Node, n2::Node)
+
+Fallback implementation of `make_edge` throwing an error. If you got this error it likely means you tried to construct an edge between two nodes of the same type.
+"""
 function make_edge(n1::Node, n2::Node)
     return error("Can only create edges from compute to data node or reverse")
 end
 
+"""
+    make_edge(n1::ComputeTaskNode, n2::DataTaskNode)
+
+Construct and return a new [`Edge`](@ref) pointing from `n1` (child) to `n2` (parent).
+"""
 function make_edge(n1::ComputeTaskNode, n2::DataTaskNode)
     return Edge((n1, n2))
 end
 
+"""
+    make_edge(n1::DataTaskNode, n2::ComputeTaskNode)
+
+Construct and return a new [`Edge`](@ref) pointing from `n1` (child) to `n2` (parent).
+"""
 function make_edge(n1::DataTaskNode, n2::ComputeTaskNode)
     return Edge((n1, n2))
 end

src/node/print.jl

@@ -1,7 +1,26 @@
+"""
+    show(io::IO, n::Node)
+
+Print a short string representation of the node to io.
+"""
 function show(io::IO, n::Node)
     return print(io, "Node(", n.task, ")")
 end
 
+"""
+    show(io::IO, e::Edge)
+
+Print a short string representation of the edge to io.
+"""
 function show(io::IO, e::Edge)
     return print(io, "Edge(", e.edge[1], ", ", e.edge[2], ")")
 end
+
+"""
+    to_var_name(id::UUID)
+
+Return the uuid as a string usable as a variable name in code generation.
+"""
+function to_var_name(id::UUID)
+    return replace(string(id), "-" => "_")
+end

src/node/properties.jl

@@ -1,17 +1,58 @@
+"""
+    is_entry_node(node::Node)
+
+Return whether this node is an entry node in its graph, i.e., it has no children.
+"""
 is_entry_node(node::Node) = length(node.children) == 0
+
+"""
+    is_exit_node(node::Node)
+
+Return whether this node is an exit node of its graph, i.e., it has no parents.
+"""
 is_exit_node(node::Node) = length(node.parents) == 0
 
-# children = prerequisite nodes, nodes that need to execute before the task, edges point into this task
+"""
+    data(edge::Edge)
+
+Return the data transfered by this edge, i.e., 0 if the child is a [`ComputeTaskNode`](@ref), otherwise the child's `data()`.
+"""
+function data(edge::Edge)
+    if typeof(edge.edge[1]) <: DataTaskNode
+        return data(edge.edge[1].task)
+    end
+    return 0.0
+end
+
+"""
+    children(node::Node)
+
+Return a copy of the node's children so it can safely be muted without changing the node in the graph.
+
+A node's children are its prerequisite nodes, nodes that need to execute before the task of this node.
+"""
 function children(node::Node)
     return copy(node.children)
 end
 
-# parents = subsequent nodes, nodes that need this node to execute, edges point from this task
+"""
+    parents(node::Node)
+
+Return a copy of the node's parents so it can safely be muted without changing the node in the graph.
+
+A node's parents are its subsequent nodes, nodes that need this node to execute.
+"""
 function parents(node::Node)
     return copy(node.parents)
 end
 
-# siblings = all children of any parents, no duplicates, includes the node itself
+"""
+    siblings(node::Node)
+
+Return a vector of all siblings of this node. 
+
+A node's siblings are all children of any of its parents. The result contains no duplicates and includes the node itself.
+"""
 function siblings(node::Node)
     result = Set{Node}()
     push!(result, node)
@@ -22,7 +63,16 @@ function siblings(node::Node)
     return result
 end
 
-# partners = all parents of any children, no duplicates, includes the node itself
+"""
+    partners(node::Node)
+
+Return a vector of all partners of this node. 
+
+A node's partners are all parents of any of its children. The result contains no duplicates and includes the node itself.
+
+Note: This is very slow when there are multiple children with many parents. 
+This is less of a problem in [`siblings(node::Node)`](@ref) because (depending on the model) there are no nodes with a large number of children, or only a single one.
+"""
 function partners(node::Node)
     result = Set{Node}()
     push!(result, node)
@@ -33,8 +83,11 @@ function partners(node::Node)
     return result
 end
 
-# alternative version to partners(Node), avoiding allocation of a new set
-# works on the given set and returns nothing
+"""
+    partners(node::Node, set::Set{Node})
+
+Alternative version to [`partners(node::Node)`](@ref), avoiding allocation of a new set. Works on the given set and returns `nothing`.
+"""
 function partners(node::Node, set::Set{Node})
     push!(set, node)
     for child in node.children
@@ -43,10 +96,20 @@ function partners(node::Node, set::Set{Node})
     return nothing
 end
 
-function is_parent(potential_parent, node)
+"""
+    is_parent(potential_parent::Node, node::Node)
+
+Return whether the `potential_parent` is a parent of `node`.
+"""
+function is_parent(potential_parent::Node, node::Node)
     return potential_parent in node.parents
 end
 
-function is_child(potential_child, node)
+"""
+    is_child(potential_child::Node, node::Node)
+
+Return whether the `potential_child` is a child of `node`.
+"""
+function is_child(potential_child::Node, node::Node)
     return potential_child in node.children
 end

src/node/type.jl

@@ -5,12 +5,33 @@ using Base.Threads
 # TODO: reliably find out how many threads we're running with (nthreads() returns 1 when precompiling :/)
 rng = [Random.MersenneTwister(0) for _ in 1:32]
 
+"""
+    Node
+
+The abstract base type of every node.
+
+See [`DataTaskNode`](@ref), [`ComputeTaskNode`](@ref) and [`make_node`](@ref).
+"""
 abstract type Node end
 
 # declare this type here because it's needed
 # the specific operations are declared in graph.jl
 abstract type Operation end
 
+"""
+    DataTaskNode <: Node
+    
+Any node that transfers data and does no computation.
+
+# Fields
+`.task`: The node's data task type. Usually [`DataTask`](@ref).\\
+`.parents`: A vector of the node's parents (i.e. nodes that depend on this one).\\
+`.children`: A vector of the node's children (i.e. nodes that this one depends on).\\
+`.id`: The node's id. Improves the speed of comparisons.\\
+`.nodeReduction`: Either this node's [`NodeReduction`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeSplit`: Either this node's [`NodeSplit`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeFusion`: Either this node's [`NodeFusion`](@ref) or `missing`, if none. There can only be at most one for DataTaskNodes.
+"""
 mutable struct DataTaskNode <: Node
     task::AbstractDataTask
 
@@ -31,9 +52,25 @@ mutable struct DataTaskNode <: Node
 
     # the node fusion involving this node, if it exists
     nodeFusion::Union{Operation, Missing}
+
+    # for input nodes we need a name for the node to distinguish between them
+    name::String
 end
 
-# same as DataTaskNode
+"""
+    ComputeTaskNode <: Node
+    
+Any node that transfers data and does no computation.
+
+# Fields
+`.task`: The node's data task type. Usually [`DataTask`](@ref).\\
+`.parents`: A vector of the node's parents (i.e. nodes that depend on this one).\\
+`.children`: A vector of the node's children (i.e. nodes that this one depends on).\\
+`.id`: The node's id. Improves the speed of comparisons.\\
+`.nodeReduction`: Either this node's [`NodeReduction`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeSplit`: Either this node's [`NodeSplit`](@ref) or `missing`, if none. There can only be at most one.\\
+`.nodeFusion`: A vector of this node's [`NodeFusion`](@ref)s. For a ComputeTaskNode there can be any number of these, unlike the DataTaskNodes.
+"""
 mutable struct ComputeTaskNode <: Node
     task::AbstractComputeTask
     parents::Vector{Node}
@@ -47,25 +84,15 @@ mutable struct ComputeTaskNode <: Node
     nodeFusions::Vector{Operation}
 end
 
-DataTaskNode(t::AbstractDataTask) = DataTaskNode(
-    t,
-    Vector{Node}(),
-    Vector{Node}(),
-    UUIDs.uuid1(rng[threadid()]),
-    missing,
-    missing,
-    missing,
-)
-ComputeTaskNode(t::AbstractComputeTask) = ComputeTaskNode(
-    t,
-    Vector{Node}(),
-    Vector{Node}(),
-    UUIDs.uuid1(rng[threadid()]),
-    missing,
-    missing,
-    Vector{NodeFusion}(),
-)
+"""
+    Edge
 
+Type of an edge in the graph. Edges can only exist between a [`DataTaskNode`](@ref) and a [`ComputeTaskNode`](@ref) or vice versa, not between two of the same type of node.
+
+An edge always points from child to parent: `child = e.edge[1]` and `parent = e.edge[2]`.
+
+The child is the prerequisite node of the parent.
+"""
 struct Edge
     # edge points from child to parent
     edge::Union{
@@ -73,23 +100,3 @@ struct Edge
         Tuple{ComputeTaskNode, DataTaskNode},
     }
 end
-
-copy(m::Missing) = missing
-copy(n::ComputeTaskNode) = ComputeTaskNode(
-    copy(n.task),
-    copy(n.parents),
-    copy(n.children),
-    UUIDs.uuid1(rng[threadid()]),
-    copy(n.nodeReduction),
-    copy(n.nodeSplit),
-    copy(n.nodeFusions),
-)
-copy(n::DataTaskNode) = DataTaskNode(
-    copy(n.task),
-    copy(n.parents),
-    copy(n.children),
-    UUIDs.uuid1(rng[threadid()]),
-    copy(n.nodeReduction),
-    copy(n.nodeSplit),
-    copy(n.nodeFusion),
-)

src/node/validate.jl

@@ -1,3 +1,12 @@
+"""
+    is_valid_node(graph::DAG, node::Node)
+
+Verify that a given node is valid in the graph. Call like `@test is_valid_node(g, n)`. Uses `@assert` to fail if something is invalid but also provide an error message.
+
+This function is very performance intensive and should only be used when testing or debugging.
+
+See also this function's specific versions for the concrete Node types [`is_valid(graph::DAG, node::ComputeTaskNode)`](@ref) and [`is_valid(graph::DAG, node::DataTaskNode)`](@ref).
+"""
 function is_valid_node(graph::DAG, node::Node)
     @assert node in graph "Node is not part of the given graph!"
 
@@ -22,7 +31,13 @@ function is_valid_node(graph::DAG, node::Node)
     return true
 end
 
-# call with @assert
+"""
+    is_valid(graph::DAG, node::ComputeTaskNode)
+
+Verify that the given compute node is valid in the graph. Call with `@assert` or `@test` when testing or debugging.
+
+This also calls [`is_valid_node(graph::DAG, node::Node)`](@ref).
+"""
 function is_valid(graph::DAG, node::ComputeTaskNode)
     @assert is_valid_node(graph, node)
 
@@ -32,7 +47,13 @@ function is_valid(graph::DAG, node::ComputeTaskNode)
     return true
 end
 
-# call with @assert
+"""
+    is_valid(graph::DAG, node::DataTaskNode)
+
+Verify that the given compute node is valid in the graph. Call with `@assert` or `@test` when testing or debugging.
+
+This also calls [`is_valid_node(graph::DAG, node::Node)`](@ref).
+"""
 function is_valid(graph::DAG, node::DataTaskNode)
     @assert is_valid_node(graph, node)
 

src/operation/apply.jl

@@ -1,6 +1,8 @@
-# functions that apply graph operations
+"""
+    apply_all!(graph::DAG)
 
-# applies all unapplied operations in the DAG
+Apply all unapplied operations in the DAG. Is automatically called in all functions that require the latest state of the [`DAG`](@ref).
+"""
 function apply_all!(graph::DAG)
     while !isempty(graph.operationsToApply)
         # get next operation to apply from front of the deque
@@ -15,10 +17,22 @@ function apply_all!(graph::DAG)
     return nothing
 end
 
+"""
+    apply_operation!(graph::DAG, operation::Operation)
+
+Fallback implementation of apply_operation! for unimplemented operation types, throwing an error.
+"""
 function apply_operation!(graph::DAG, operation::Operation)
     return error("Unknown operation type!")
 end
 
+"""
+    apply_operation!(graph::DAG, operation::NodeFusion)
+
+Apply the given [`NodeFusion`](@ref) to the graph. Generic wrapper around [`node_fusion!`](@ref).
+
+Return an [`AppliedNodeFusion`](@ref) object generated from the graph's [`Diff`](@ref).
+"""
 function apply_operation!(graph::DAG, operation::NodeFusion)
     diff = node_fusion!(
         graph,
@@ -26,40 +40,86 @@ function apply_operation!(graph::DAG, operation::NodeFusion)
         operation.input[2],
         operation.input[3],
     )
+
+    graph.properties += GraphProperties(diff)
+
     return AppliedNodeFusion(operation, diff)
 end
 
+"""
+    apply_operation!(graph::DAG, operation::NodeReduction)
+
+Apply the given [`NodeReduction`](@ref) to the graph. Generic wrapper around [`node_reduction!`](@ref).
+
+Return an [`AppliedNodeReduction`](@ref) object generated from the graph's [`Diff`](@ref).
+"""
 function apply_operation!(graph::DAG, operation::NodeReduction)
     diff = node_reduction!(graph, operation.input)
+
+    graph.properties += GraphProperties(diff)
+
     return AppliedNodeReduction(operation, diff)
 end
 
+"""
+    apply_operation!(graph::DAG, operation::NodeSplit)
+
+Apply the given [`NodeSplit`](@ref) to the graph. Generic wrapper around [`node_split!`](@ref).
+
+Return an [`AppliedNodeSplit`](@ref) object generated from the graph's [`Diff`](@ref).
+"""
 function apply_operation!(graph::DAG, operation::NodeSplit)
     diff = node_split!(graph, operation.input)
+
+    graph.properties += GraphProperties(diff)
+
     return AppliedNodeSplit(operation, diff)
 end
 
+"""
+    revert_operation!(graph::DAG, operation::AppliedOperation)
 
+Fallback implementation of operation reversion for unimplemented operation types, throwing an error.
+"""
 function revert_operation!(graph::DAG, operation::AppliedOperation)
     return error("Unknown operation type!")
 end
 
+"""
+    revert_operation!(graph::DAG, operation::AppliedNodeFusion)
+
+Revert the applied node fusion on the graph. Return the original [`NodeFusion`](@ref) operation.
+"""
 function revert_operation!(graph::DAG, operation::AppliedNodeFusion)
     revert_diff!(graph, operation.diff)
     return operation.operation
 end
 
+"""
+    revert_operation!(graph::DAG, operation::AppliedNodeReduction)
+
+Revert the applied node fusion on the graph. Return the original [`NodeReduction`](@ref) operation.
+"""
 function revert_operation!(graph::DAG, operation::AppliedNodeReduction)
     revert_diff!(graph, operation.diff)
     return operation.operation
 end
 
+"""
+    revert_operation!(graph::DAG, operation::AppliedNodeSplit)
+
+Revert the applied node fusion on the graph. Return the original [`NodeSplit`](@ref) operation.
+"""
 function revert_operation!(graph::DAG, operation::AppliedNodeSplit)
     revert_diff!(graph, operation.diff)
     return operation.operation
 end
 
+"""
+    revert_diff!(graph::DAG, diff::Diff)
 
+Revert the given diff on the graph. Used to revert the individual [`AppliedOperation`](@ref)s with [`revert_operation!`](@ref).
+"""
 function revert_diff!(graph::DAG, diff::Diff)
     # add removed nodes, remove added nodes, same for edges
     # note the order
@@ -76,9 +136,19 @@ function revert_diff!(graph::DAG, diff::Diff)
     for edge in diff.removedEdges
         insert_edge!(graph, edge.edge[1], edge.edge[2], false)
     end
+
+    graph.properties -= GraphProperties(diff)
+
+    return nothing
 end
 
-# Fuse nodes n1 -> n2 -> n3 together into one node, return the applied difference to the graph
+"""
+    node_fusion!(graph::DAG, n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+
+Fuse nodes n1 -> n2 -> n3 together into one node, return the applied difference to the graph.
+
+For details see [`NodeFusion`](@ref).
+"""
 function node_fusion!(
     graph::DAG,
     n1::ComputeTaskNode,
@@ -90,7 +160,6 @@ function node_fusion!(
     # clear snapshot
     get_snapshot_diff(graph)
 
-
     # save children and parents
     n1_children = children(n1)
     n3_parents = parents(n3)
@@ -111,26 +180,18 @@ function node_fusion!(
         ComputeTaskNode(FusedComputeTask{typeof(n1.task), typeof(n3.task)}())
     insert_node!(graph, new_node)
 
-    # use a set for combined children of n1 and n3 to not get duplicates
-    n1and3_children = Set{Node}()
-
-    # remove edges from n1 children to n1
     for child in n1_children
         remove_edge!(graph, child, n1)
-        push!(n1and3_children, child)
-    end
-
-    # remove edges from n3 children to n3
-    for child in n3_children
-        remove_edge!(graph, child, n3)
-        push!(n1and3_children, child)
-    end
-
-    for child in n1and3_children
         insert_edge!(graph, child, new_node)
     end
 
-    # "repoint" parents of n3 from new node
+    for child in n3_children
+        remove_edge!(graph, child, n3)
+        if !(child in n1_children)
+            insert_edge!(graph, child, new_node)
+        end
+    end
+
     for parent in n3_parents
         remove_edge!(graph, n3, parent)
         insert_edge!(graph, new_node, parent)
@@ -139,6 +200,13 @@ function node_fusion!(
     return get_snapshot_diff(graph)
 end
 
+"""
+    node_reduction!(graph::DAG, nodes::Vector{Node})
+
+Reduce the given nodes together into one node, return the applied difference to the graph.
+
+For details see [`NodeReduction`](@ref).
+"""
 function node_reduction!(graph::DAG, nodes::Vector{Node})
     # @assert is_valid_node_reduction_input(graph, nodes)
 
@@ -178,6 +246,13 @@ function node_reduction!(graph::DAG, nodes::Vector{Node})
     return get_snapshot_diff(graph)
 end
 
+"""
+    node_split!(graph::DAG, n1::Node)
+
+Split the given node into one node per parent, return the applied difference to the graph.
+
+For details see [`NodeSplit`](@ref).
+"""
 function node_split!(graph::DAG, n1::Node)
     # @assert is_valid_node_split_input(graph, n1)
 

src/operation/clean.jl

@@ -1,9 +1,14 @@
-# functions for "cleaning" nodes, i.e. regenerating the possible operations for a node
+# These are functions for "cleaning" nodes, i.e. regenerating the possible operations for a node
 
-# function to find node fusions involving the given node if it's a data node
-# pushes the found fusion everywhere it needs to be and returns nothing
+"""
+    find_fusions!(graph::DAG, node::DataTaskNode)
+
+Find node fusions involving the given data node. The function pushes the found [`NodeFusion`](@ref) (if any) everywhere it needs to be and returns nothing.
+
+Does nothing if the node already has a node fusion set. Since it's a data node, only one node fusion can be possible with it.
+"""
 function find_fusions!(graph::DAG, node::DataTaskNode)
-    # if there is already a fusion here, skip
+    # if there is already a fusion here, skip to avoid duplicates
     if !ismissing(node.nodeFusion)
         return nothing
     end
@@ -32,7 +37,11 @@ function find_fusions!(graph::DAG, node::DataTaskNode)
     return nothing
 end
 
+"""
+    find_fusions!(graph::DAG, node::ComputeTaskNode)
 
+Find node fusions involving the given compute node. The function pushes the found [`NodeFusion`](@ref)s (if any) everywhere they need to be and returns nothing.
+"""
 function find_fusions!(graph::DAG, node::ComputeTaskNode)
     # just find fusions in neighbouring DataTaskNodes
     for child in node.children
@@ -46,6 +55,11 @@ function find_fusions!(graph::DAG, node::ComputeTaskNode)
     return nothing
 end
 
+"""
+    find_reductions!(graph::DAG, node::Node)
+
+Find node reductions involving the given node. The function pushes the found [`NodeReduction`](@ref) (if any) everywhere it needs to be and returns nothing.
+"""
 function find_reductions!(graph::DAG, node::Node)
     # there can only be one reduction per node, avoid adding duplicates
     if !ismissing(node.nodeReduction)
@@ -57,14 +71,8 @@ function find_reductions!(graph::DAG, node::Node)
     partners_ = partners(node)
     delete!(partners_, node)
     for partner in partners_
-        if partner ∉ graph.nodes
-            error("Partner is not part of the graph")
-        end
-
+        @assert partner in graph.nodes
         if can_reduce(node, partner)
-            if Set(node.children) != Set(partner.children)
-                error("Not equal children")
-            end
             if reductionVector === nothing
                 # only when there's at least one reduction partner, insert the vector
                 reductionVector = Vector{Node}()
@@ -91,6 +99,11 @@ function find_reductions!(graph::DAG, node::Node)
     return nothing
 end
 
+"""
+    find_splits!(graph::DAG, node::Node)
+
+Find the node split of the given node. The function pushes the found [`NodeSplit`](@ref) (if any) everywhere it needs to be and returns nothing.
+"""
 function find_splits!(graph::DAG, node::Node)
     if !ismissing(node.nodeSplit)
         return nothing
@@ -105,11 +118,17 @@ function find_splits!(graph::DAG, node::Node)
     return nothing
 end
 
-# "clean" the operations on a dirty node
+"""
+    clean_node!(graph::DAG, node::Node)
+
+Sort this node's parent and child sets, then find fusions, reductions and splits involving it. Needs to be called after the node was changed in some way.
+"""
 function clean_node!(graph::DAG, node::Node)
     sort_node!(node)
 
     find_fusions!(graph, node)
     find_reductions!(graph, node)
-    return find_splits!(graph, node)
+    find_splits!(graph, node)
+
+    return nothing
 end

src/operation/find.jl

@@ -2,6 +2,11 @@
 
 using Base.Threads
 
+"""
+    insert_operation!(nf::NodeFusion, locks::Dict{ComputeTaskNode, SpinLock})
+
+Insert the given node fusion into its input nodes' operation caches. For the compute nodes, locking via the given `locks` is employed to have safe multi-threading. For a large set of nodes, contention on the locks should be very small.
+"""
 function insert_operation!(
     nf::NodeFusion,
     locks::Dict{ComputeTaskNode, SpinLock},
@@ -20,6 +25,11 @@ function insert_operation!(
     return nothing
 end
 
+"""
+    insert_operation!(nf::NodeReduction)
+
+Insert the given node reduction into its input nodes' operation caches. This is thread-safe.
+"""
 function insert_operation!(nr::NodeReduction)
     for n in nr.input
         n.nodeReduction = nr
@@ -27,11 +37,21 @@ function insert_operation!(nr::NodeReduction)
     return nothing
 end
 
+"""
+    insert_operation!(nf::NodeSplit)
+
+Insert the given node split into its input node's operation cache. This is thread-safe.
+"""
 function insert_operation!(ns::NodeSplit)
     ns.input.nodeSplit = ns
     return nothing
 end
 
+"""
+    nr_insertion!(operations::PossibleOperations, nodeReductions::Vector{Vector{NodeReduction}})
+
+Insert the node reductions into the graph and the nodes' caches. Employs multithreading for speedup.
+"""
 function nr_insertion!(
     operations::PossibleOperations,
     nodeReductions::Vector{Vector{NodeReduction}},
@@ -58,6 +78,11 @@ function nr_insertion!(
     return nothing
 end
 
+"""
+    nf_insertion!(graph::DAG, operations::PossibleOperations, nodeFusions::Vector{Vector{NodeFusion}})
+
+Insert the node fusions into the graph and the nodes' caches. Employs multithreading for speedup.
+"""
 function nf_insertion!(
     graph::DAG,
     operations::PossibleOperations,
@@ -92,6 +117,11 @@ function nf_insertion!(
     return nothing
 end
 
+"""
+    ns_insertion!(operations::PossibleOperations, nodeSplits::Vector{Vector{NodeSplits}})
+
+Insert the node splits into the graph and the nodes' caches. Employs multithreading for speedup.
+"""
 function ns_insertion!(
     operations::PossibleOperations,
     nodeSplits::Vector{Vector{NodeSplit}},
@@ -118,8 +148,14 @@ function ns_insertion!(
     return nothing
 end
 
-# function to generate all possible operations on the graph
-function generate_options(graph::DAG)
+"""
+    generate_operations(graph::DAG)
+
+Generate all possible operations on the graph. Used initially when the graph is freshly assembled or parsed. Uses multithreading for speedup.
+
+Safely inserts all the found operations into the graph and its nodes.
+"""
+function generate_operations(graph::DAG)
     generatedFusions = [Vector{NodeFusion}() for _ in 1:nthreads()]
     generatedReductions = [Vector{NodeReduction}() for _ in 1:nthreads()]
     generatedSplits = [Vector{NodeSplit}() for _ in 1:nthreads()]

src/operation/get.jl

@@ -2,11 +2,16 @@
 
 using Base.Threads
 
+"""
+    get_operations(graph::DAG)
+
+Return the [`PossibleOperations`](@ref) of the graph at the current state.
+"""
 function get_operations(graph::DAG)
     apply_all!(graph)
 
     if isempty(graph.possibleOperations)
-        generate_options(graph)
+        generate_operations(graph)
     end
 
     for node in graph.dirtyNodes

src/operation/print.jl

@@ -1,3 +1,8 @@
+"""
+    show(io::IO, ops::PossibleOperations)
+
+Print a string representation of the set of possible operations to io.
+"""
 function show(io::IO, ops::PossibleOperations)
     print(io, length(ops.nodeFusions))
     println(io, " Node Fusions: ")
@@ -16,6 +21,11 @@ function show(io::IO, ops::PossibleOperations)
     end
 end
 
+"""
+    show(io::IO, op::NodeReduction)
+
+Print a string representation of the node reduction to io.
+"""
 function show(io::IO, op::NodeReduction)
     print(io, "NR: ")
     print(io, length(op.input))
@@ -23,11 +33,21 @@ function show(io::IO, op::NodeReduction)
     return print(io, op.input[1].task)
 end
 
+"""
+    show(io::IO, op::NodeSplit)
+
+Print a string representation of the node split to io.
+"""
 function show(io::IO, op::NodeSplit)
     print(io, "NS: ")
     return print(io, op.input.task)
 end
 
+"""
+    show(io::IO, op::NodeFusion)
+
+Print a string representation of the node fusion to io.
+"""
 function show(io::IO, op::NodeFusion)
     print(io, "NF: ")
     print(io, op.input[1].task)

src/operation/type.jl

@@ -1,33 +1,116 @@
-# An abstract base class for operations
-# an operation can be applied to a DAG
+"""
+    Operation
+
+An abstract base class for operations. An operation can be applied to a [`DAG`](@ref), changing its nodes and edges.
+
+Possible operations on a [`DAG`](@ref) can be retrieved using [`get_operations`](@ref).
+
+See also: [`push_operation!`](@ref), [`pop_operation!`](@ref)
+"""
 abstract type Operation end
 
-# An abstract base class for already applied operations
-# an applied operation can be reversed iff it is the last applied operation on the DAG
+"""
+    AppliedOperation
+
+An abstract base class for already applied operations.
+An applied operation can be reversed iff it is the last applied operation on the DAG.
+Every applied operation stores a [`Diff`](@ref) from when it was initially applied to be able to revert the operation.
+
+See also: [`revert_operation!`](@ref).
+"""
 abstract type AppliedOperation end
 
+"""
+    NodeFusion <: Operation
+
+The NodeFusion operation. Represents the fusing of a chain of compute node -> data node -> compute node.
+
+After the node fusion is applied, the graph has 2 fewer nodes and edges, and a new [`FusedComputeTask`](@ref) with the two input compute nodes as parts.
+
+# Requirements for successful application
+
+A chain of (n1, n2, n3) can be fused if:
+- All nodes are in the graph.
+- (n1, n2) is an edge in the graph.
+- (n2, n3) is an edge in the graph.
+- n2 has exactly one parent (n3) and exactly one child (n1).
+- n1 has exactly one parent (n2).
+
+[`is_valid_node_fusion_input`](@ref) can be used to `@assert` these requirements.
+
+See also: [`can_fuse`](@ref)
+"""
 struct NodeFusion <: Operation
     input::Tuple{ComputeTaskNode, DataTaskNode, ComputeTaskNode}
 end
 
+"""
+    AppliedNodeFusion <: AppliedOperation
+
+The applied version of the [`NodeFusion`](@ref).
+"""
 struct AppliedNodeFusion <: AppliedOperation
     operation::NodeFusion
     diff::Diff
 end
 
+"""
+    NodeReduction <: Operation
+
+The NodeReduction operation. Represents the reduction of two or more nodes with one another.
+Only one of the input nodes is kept, while all others are deleted and their parents are accumulated in the kept node's parents instead.
+
+After the node reduction is applied, the graph has `length(nr.input) - 1` fewer nodes.
+
+# Requirements for successful application
+
+A vector of nodes can be reduced if:
+- All nodes are in the graph.
+- All nodes have the same task type.
+- All nodes have the same set of children.
+
+[`is_valid_node_reduction_input`](@ref) can be used to `@assert` these requirements.
+
+See also: [`can_reduce`](@ref)
+"""
 struct NodeReduction <: Operation
     input::Vector{Node}
 end
 
+"""
+    AppliedNodeReduction <: AppliedOperation
+
+The applied version of the [`NodeReduction`](@ref).
+"""
 struct AppliedNodeReduction <: AppliedOperation
     operation::NodeReduction
     diff::Diff
 end
 
+"""
+    NodeSplit <: Operation
+
+The NodeSplit operation. Represents the split of its input node into one node for each of its parents. It is the reverse operation to the [`NodeReduction`](@ref).
+
+# Requirements for successful application
+
+A node can be split if:
+- It is in the graph.
+- It has at least 2 parents.
+
+[`is_valid_node_split_input`](@ref) can be used to `@assert` these requirements.
+
+See also: [`can_split`](@ref)
+"""
 struct NodeSplit <: Operation
     input::Node
 end
 
+"""
+    AppliedNodeSplit <: AppliedOperation
+
+The applied version of the [`NodeSplit`](@ref).
+"""
 struct AppliedNodeSplit <: AppliedOperation
     operation::NodeSplit
     diff::Diff

src/operation/utility.jl

@@ -1,10 +1,19 @@
+"""
+    isempty(operations::PossibleOperations)
 
+Return whether `operations` is empty, i.e. all of its fields are empty.
+"""
 function isempty(operations::PossibleOperations)
     return isempty(operations.nodeFusions) &&
            isempty(operations.nodeReductions) &&
            isempty(operations.nodeSplits)
 end
 
+"""
+    length(operations::PossibleOperations)
+
+Return a named tuple with the number of each of the operation types as a named tuple. The fields are named the same as the [`PossibleOperations`](@ref)'.
+"""
 function length(operations::PossibleOperations)
     return (
         nodeFusions = length(operations.nodeFusions),
@@ -13,22 +22,41 @@ function length(operations::PossibleOperations)
     )
 end
 
+"""
+    delete!(operations::PossibleOperations, op::NodeFusion)
+
+Delete the given node fusion from the possible operations.
+"""
 function delete!(operations::PossibleOperations, op::NodeFusion)
     delete!(operations.nodeFusions, op)
     return operations
 end
 
+"""
+    delete!(operations::PossibleOperations, op::NodeReduction)
+
+Delete the given node reduction from the possible operations.
+"""
 function delete!(operations::PossibleOperations, op::NodeReduction)
     delete!(operations.nodeReductions, op)
     return operations
 end
 
+"""
+    delete!(operations::PossibleOperations, op::NodeSplit)
+
+Delete the given node split from the possible operations.
+"""
 function delete!(operations::PossibleOperations, op::NodeSplit)
     delete!(operations.nodeSplits, op)
     return operations
 end
 
+"""
+    can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
 
+Return whether the given nodes can be fused. See [`NodeFusion`](@ref) for the requirements.
+"""
 function can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
     if !is_child(n1, n2) || !is_child(n2, n3)
         # the checks are redundant but maybe a good sanity check
@@ -44,6 +72,11 @@ function can_fuse(n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
     return true
 end
 
+"""
+    can_reduce(n1::Node, n2::Node)
+
+Return whether the given two nodes can be reduced. See [`NodeReduction`](@ref) for the requirements.
+"""
 function can_reduce(n1::Node, n2::Node)
     if (n1.task != n2.task)
         return false
@@ -86,26 +119,49 @@ function can_reduce(n1::Node, n2::Node)
     return Set(n1.children) == Set(n2.children)
 end
 
+"""
+    can_split(n1::Node)
+
+Return whether the given node can be split. See [`NodeSplit`](@ref) for the requirements.
+"""
 function can_split(n::Node)
     return length(parents(n)) > 1
 end
 
+"""
+    ==(op1::Operation, op2::Operation)
+
+Fallback implementation of operation equality. Return false. Actual comparisons are done by the overloads of same type operation comparisons.
+"""
 function ==(op1::Operation, op2::Operation)
     return false
 end
 
+"""
+    ==(op1::NodeFusion, op2::NodeFusion)
+
+Equality comparison between two node fusions. Two node fusions are considered equal if they have the same inputs.
+"""
 function ==(op1::NodeFusion, op2::NodeFusion)
     # there can only be one node fusion on a given data task, so if the data task is the same, the fusion is the same
     return op1.input[2] == op2.input[2]
 end
 
+"""
+    ==(op1::NodeReduction, op2::NodeReduction)
+
+Equality comparison between two node reductions. Two node reductions are considered equal when they have the same inputs.
+"""
 function ==(op1::NodeReduction, op2::NodeReduction)
     # node reductions are equal exactly if their first input is the same
     return op1.input[1].id == op2.input[1].id
 end
 
+"""
+    ==(op1::NodeSplit, op2::NodeSplit)
+
+Equality comparison between two node splits. Two node splits are considered equal if they have the same input node.
+"""
 function ==(op1::NodeSplit, op2::NodeSplit)
     return op1.input == op2.input
 end
-
-copy(id::UUID) = UUID(id.value)

src/operation/validate.jl

@@ -2,6 +2,13 @@
 # should be called with @assert
 # the functions throw their own errors though, to still have helpful error messages
 
+"""
+    is_valid_node_fusion_input(graph::DAG, n1::ComputeTaskNode, n2::DataTaskNode, n3::ComputeTaskNode)
+
+Assert for a gven node fusion input whether the nodes can be fused. For the requirements of a node fusion see [`NodeFusion`](@ref).
+
+Intended for use with `@assert` or `@test`.
+"""
 function is_valid_node_fusion_input(
     graph::DAG,
     n1::ComputeTaskNode,
@@ -52,6 +59,13 @@ function is_valid_node_fusion_input(
     return true
 end
 
+"""
+    is_valid_node_reduction_input(graph::DAG, nodes::Vector{Node})
+
+Assert for a gven node reduction input whether the nodes can be reduced. For the requirements of a node reduction see [`NodeReduction`](@ref).
+
+Intended for use with `@assert` or `@test`.
+"""
 function is_valid_node_reduction_input(graph::DAG, nodes::Vector{Node})
     for n in nodes
         if n ∉ graph
@@ -88,6 +102,13 @@ function is_valid_node_reduction_input(graph::DAG, nodes::Vector{Node})
     return true
 end
 
+"""
+    is_valid_node_split_input(graph::DAG, n1::Node)
+
+Assert for a gven node split input whether the node can be split. For the requirements of a node split see [`NodeSplit`](@ref).
+
+Intended for use with `@assert` or `@test`.
+"""
 function is_valid_node_split_input(graph::DAG, n1::Node)
     if n1 ∉ graph
         throw(
@@ -108,18 +129,39 @@ function is_valid_node_split_input(graph::DAG, n1::Node)
     return true
 end
 
+"""
+    is_valid(graph::DAG, nr::NodeReduction)
+
+Assert for a given [`NodeReduction`](@ref) whether it is a valid operation in the graph.
+
+Intended for use with `@assert` or `@test`.
+"""
 function is_valid(graph::DAG, nr::NodeReduction)
     @assert is_valid_node_reduction_input(graph, nr.input)
     @assert nr in graph.possibleOperations.nodeReductions "NodeReduction is not part of the graph's possible operations!"
     return true
 end
 
+"""
+    is_valid(graph::DAG, nr::NodeSplit)
+
+Assert for a given [`NodeSplit`](@ref) whether it is a valid operation in the graph.
+
+Intended for use with `@assert` or `@test`.
+"""
 function is_valid(graph::DAG, ns::NodeSplit)
     @assert is_valid_node_split_input(graph, ns.input)
     @assert ns in graph.possibleOperations.nodeSplits "NodeSplit is not part of the graph's possible operations!"
     return true
 end
 
+"""
+    is_valid(graph::DAG, nr::NodeFusion)
+
+Assert for a given [`NodeFusion`](@ref) whether it is a valid operation in the graph.
+
+Intended for use with `@assert` or `@test`.
+"""
 function is_valid(graph::DAG, nf::NodeFusion)
     @assert is_valid_node_fusion_input(
         graph,

src/properties/create.jl

@@ -0,0 +1,73 @@
+"""
+   GraphProperties()
+
+Create an empty [`GraphProperties`](@ref) object.
+"""
+function GraphProperties()
+    return (
+        data = 0.0,
+        computeEffort = 0.0,
+        computeIntensity = 0.0,
+        cost = 0.0,
+        noNodes = 0,
+        noEdges = 0,
+    )::GraphProperties
+end
+
+"""
+   GraphProperties(graph::DAG)
+
+Calculate the graph's properties and return the constructed [`GraphProperties`](@ref) object.
+"""
+function GraphProperties(graph::DAG)
+    # make sure the graph is fully generated
+    apply_all!(graph)
+
+    d = 0.0
+    ce = 0.0
+    ed = 0
+    for node in graph.nodes
+        d += data(node.task) * length(node.parents)
+        ce += compute_effort(node.task)
+        ed += length(node.parents)
+    end
+
+    return (
+        data = d,
+        computeEffort = ce,
+        computeIntensity = (d == 0) ? 0.0 : ce / d,
+        cost = 0.0, # TODO
+        noNodes = length(graph.nodes),
+        noEdges = ed,
+    )::GraphProperties
+end
+
+"""
+   GraphProperties(diff::Diff)
+
+Create the graph properties difference from a given [`Diff`](@ref).
+The graph's properties after applying the [`Diff`](@ref) will be `get_properties(graph) + GraphProperties(diff)`.
+For reverting a diff, it's `get_properties(graph) - GraphProperties(diff)`.
+"""
+function GraphProperties(diff::Diff)
+    d = 0.0
+    ce = 0.0
+    c = 0.0 # TODO
+
+    ce =
+        reduce(+, compute_effort(n.task) for n in diff.addedNodes; init = 0.0) -
+        reduce(+, compute_effort(n.task) for n in diff.removedNodes; init = 0.0)
+
+    d =
+        reduce(+, data(e) for e in diff.addedEdges; init = 0.0) -
+        reduce(+, data(e) for e in diff.removedEdges; init = 0.0)
+
+    return (
+        data = d,
+        computeEffort = ce,
+        computeIntensity = (d == 0) ? 0.0 : ce / d,
+        cost = c,
+        noNodes = length(diff.addedNodes) - length(diff.removedNodes),
+        noEdges = length(diff.addedEdges) - length(diff.removedEdges),
+    )::GraphProperties
+end

src/properties/type.jl

@@ -0,0 +1,17 @@
+"""
+   GraphProperties
+
+Representation of a [`DAG`](@ref)'s properties.
+
+# Fields:
+`.data`: The total data transfer.\\
+`.computeEffort`: The total compute effort.\\
+`.computeIntensity`: The compute intensity, will always equal `.computeEffort / .data`.\\
+`.cost`: The estimated cost.\\
+`.noNodes`: Number of [`Node`](@ref)s.\\
+`.noEdges`: Number of [`Edge`](@ref)s.
+"""
+const GraphProperties = NamedTuple{
+    (:data, :computeEffort, :computeIntensity, :cost, :noNodes, :noEdges),
+    Tuple{Float64, Float64, Float64, Float64, Int, Int},
+}

src/properties/utility.jl

@@ -0,0 +1,59 @@
+"""
+    -(prop1::GraphProperties, prop2::GraphProperties)
+
+Subtract `prop1` from `prop2` and return the result as a new [`GraphProperties`](@ref).
+Also take care to keep consistent compute intensity.
+"""
+function -(prop1::GraphProperties, prop2::GraphProperties)
+    return (
+        data = prop1.data - prop2.data,
+        computeEffort = prop1.computeEffort - prop2.computeEffort,
+        computeIntensity = if (prop1.data - prop2.data == 0)
+            0.0
+        else
+            (prop1.computeEffort - prop2.computeEffort) /
+            (prop1.data - prop2.data)
+        end,
+        cost = prop1.cost - prop2.cost,
+        noNodes = prop1.noNodes - prop2.noNodes,
+        noEdges = prop1.noEdges - prop2.noEdges,
+    )::GraphProperties
+end
+
+"""
+    +(prop1::GraphProperties, prop2::GraphProperties)
+
+Add `prop1` and `prop2` and return the result as a new [`GraphProperties`](@ref).
+Also take care to keep consistent compute intensity.
+"""
+function +(prop1::GraphProperties, prop2::GraphProperties)
+    return (
+        data = prop1.data + prop2.data,
+        computeEffort = prop1.computeEffort + prop2.computeEffort,
+        computeIntensity = if (prop1.data + prop2.data == 0)
+            0.0
+        else
+            (prop1.computeEffort + prop2.computeEffort) /
+            (prop1.data + prop2.data)
+        end,
+        cost = prop1.cost + prop2.cost,
+        noNodes = prop1.noNodes + prop2.noNodes,
+        noEdges = prop1.noEdges + prop2.noEdges,
+    )::GraphProperties
+end
+
+"""
+    -(prop::GraphProperties)
+
+Unary negation of the graph properties. `.computeIntensity` will not be negated because `.data` and `.computeEffort` both are.
+"""
+function -(prop::GraphProperties)
+    return (
+        data = -prop.data,
+        computeEffort = -prop.computeEffort,
+        computeIntensity = prop.computeIntensity,   # no negation here!
+        cost = -prop.cost,
+        noNodes = -prop.noNodes,
+        noEdges = -prop.noEdges,
+    )::GraphProperties
+end

src/task/compare.jl

@@ -1,11 +1,26 @@
+"""
+    ==(t1::AbstractTask, t2::AbstractTask)
+
+Fallback implementation of equality comparison between two abstract tasks. Always returns false. For equal specific types of t1 and t2, a more specific comparison is called instead, doing an actual comparison.
+"""
 function ==(t1::AbstractTask, t2::AbstractTask)
     return false
 end
 
+"""
+    ==(t1::AbstractComputeTask, t2::AbstractComputeTask)
+
+Equality comparison between two compute tasks.
+"""
 function ==(t1::AbstractComputeTask, t2::AbstractComputeTask)
     return typeof(t1) == typeof(t2)
 end
 
+"""
+    ==(t1::AbstractDataTask, t2::AbstractDataTask)
+
+Equality comparison between two data tasks.
+"""
 function ==(t1::AbstractDataTask, t2::AbstractDataTask)
     return data(t1) == data(t2)
 end

src/task/create.jl

@@ -0,0 +1,14 @@
+"""
+    copy(t::AbstractDataTask)
+
+Fallback implementation of the copy of an abstract data task, throwing an error.
+"""
+copy(t::AbstractDataTask) =
+    error("Need to implement copying for your data tasks!")
+
+"""
+    copy(t::AbstractComputeTask)
+
+Return a copy of the given compute task.
+"""
+copy(t::AbstractComputeTask) = typeof(t)()

src/task/print.jl

@@ -1,3 +1,8 @@
+"""
+    show(io::IO, t::FusedComputeTask)
+    
+Print a string representation of the fused compute task to io.
+"""
 function show(io::IO, t::FusedComputeTask)
     (T1, T2) = get_types(t)
     return print(io, "ComputeFuse(", T1(), ", ", T2(), ")")

src/task/properties.jl

@@ -1,33 +1,110 @@
+"""
+    compute(t::AbstractTask; data...)
+
+Fallback implementation of the compute function of a compute task, throwing an error.
+"""
 function compute(t::AbstractTask; data...)
     return error("Need to implement compute()")
 end
 
+"""
+    compute(t::FusedComputeTask; data...)
+
+Compute a fused compute task.
+"""
+function compute(t::FusedComputeTask; data...)
+    (T1, T2) = collect(typeof(t).parameters)
+
+    return compute(T2(), compute(T1(), data))
+end
+
+"""
+    compute(t::AbstractDataTask; data...)
+
+The compute function of a data task, always the identity function, regardless of the specific task.
+"""
+compute(t::AbstractDataTask; data...) = data
+
+"""
+    compute_effort(t::AbstractTask)
+
+Fallback implementation of the compute effort of a task, throwing an error.
+"""
 function compute_effort(t::AbstractTask)
     # default implementation using compute
     return error("Need to implement compute_effort()")
 end
 
+"""
+    data(t::AbstractTask)
+
+Fallback implementation of the data of a task, throwing an error.
+"""
 function data(t::AbstractTask)
     return error("Need to implement data()")
 end
 
+"""
+    compute_effort(t::AbstractDataTask)
+
+Return the compute effort of a data task, always zero, regardless of the specific task.
+"""
 compute_effort(t::AbstractDataTask) = 0
-compute(t::AbstractDataTask; data...) = data
+
+"""
+    data(t::AbstractDataTask)
+
+Return the data of a data task. Given by the task's `.data` field.
+"""
 data(t::AbstractDataTask) = getfield(t, :data)
 
+"""
+    data(t::AbstractComputeTask)
+
+Return the data of a compute task, always zero, regardless of the specific task.
+"""
 data(t::AbstractComputeTask) = 0
 
+"""
+    compute_effort(t::FusedComputeTask)
+
+Return the compute effort of a fused compute task. 
+"""
 function compute_effort(t::FusedComputeTask)
     (T1, T2) = collect(typeof(t).parameters)
     return compute_effort(T1()) + compute_effort(T2())
 end
 
-# actual compute functions for the tasks can stay undefined for now
-# compute(t::ComputeTaskU, data::Any) = mycomputation(data)
+"""
+    get_types(::FusedComputeTask{T1, T2})
 
-function compute_intensity(t::AbstractTask)::UInt64
-    if data(t) == 0
-        return typemax(UInt64)
+Return a tuple of a the fused compute task's components' types.
+"""
+get_types(::FusedComputeTask{T1, T2}) where {T1, T2} = (T1, T2)
+
+"""
+    get_expression(t::AbstractTask)
+
+Return an expression evaluating the given task on the :dataIn symbol
+"""
+function get_expression(t::AbstractTask)
+    return quote
+        dataOut = compute($t, dataIn)
     end
-    return compute_effort(t) / data(t)
+end
+
+"""
+    get_expression()
+"""
+function get_expression(
+    t::FusedComputeTask,
+    inSymbol::Symbol,
+    outSymbol::Symbol,
+)
+    #TODO
+    computeExp = quote
+        $outSymbol = compute($t, $inSymbol)
+    end
+
+    return computeExp
 end

src/task/type.jl

@@ -1,13 +1,30 @@
+"""
+    AbstractTask
+
+The shared base type for any task.
+"""
 abstract type AbstractTask end
 
+"""
+    AbstractComputeTask <: AbstractTask
+
+The shared base type for any compute task.
+"""
 abstract type AbstractComputeTask <: AbstractTask end
+
+"""
+    AbstractDataTask <: AbstractTask
+
+The shared base type for any data task.
+"""
 abstract type AbstractDataTask <: AbstractTask end
 
+"""
+    FusedComputeTask{T1 <: AbstractComputeTask, T2 <: AbstractComputeTask} <: AbstractComputeTask
+
+A fused compute task made up of the computation of first `T1` and then `T2`.
+
+Also see: [`get_types`](@ref).
+"""
 struct FusedComputeTask{T1 <: AbstractComputeTask, T2 <: AbstractComputeTask} <:
        AbstractComputeTask end
-
-get_types(::FusedComputeTask{T1, T2}) where {T1, T2} = (T1, T2)
-
-copy(t::AbstractDataTask) =
-    error("Need to implement copying for your data tasks!")
-copy(t::AbstractComputeTask) = typeof(t)()

src/trie.jl

@@ -1,26 +1,49 @@
+"""
+    NodeIdTrie
 
-# helper struct for NodeTrie
+Helper struct for [`NodeTrie`](@ref). After the Trie's first level, every Trie level contains the vector of nodes that had children up to that level, and the TrieNode's children by UUID of the node's children.
+"""
 mutable struct NodeIdTrie
     value::Vector{Node}
     children::Dict{UUID, NodeIdTrie}
 end
 
-# Trie data structure for node reduction, inserts nodes by children
-# Assumes that given nodes have ordered vectors of children (see sort_node)
-# First level is the task type and thus does not have a value
-# Should be constructed with all Types that will be used
+"""
+    NodeTrie
+
+Trie data structure for node reduction, inserts nodes by children.
+Assumes that given nodes have ordered vectors of children (see [`sort_node!`](@ref)).
+First insertion level is the node's own task type and thus does not have a value (every node has a task type).
+
+See also: [`insert!`](@ref) and [`collect`](@ref)
+"""
 mutable struct NodeTrie
     children::Dict{DataType, NodeIdTrie}
 end
 
+"""
+    NodeTrie()
+
+Constructor for an empty [`NodeTrie`](@ref).
+"""
 function NodeTrie()
     return NodeTrie(Dict{DataType, NodeIdTrie}())
 end
 
+"""
+    NodeIdTrie()
+
+Constructor for an empty [`NodeIdTrie`](@ref).
+"""
 function NodeIdTrie()
     return NodeIdTrie(Vector{Node}(), Dict{UUID, NodeIdTrie}())
 end
 
+"""
+    insert_helper!(trie::NodeIdTrie, node::Node, depth::Int)
+
+Insert the given node into the trie. The depth is used to iterate through the trie layers, while the function calls itself recursively until it ran through all children of the node.
+"""
 function insert_helper!(trie::NodeIdTrie, node::Node, depth::Int)
     if (length(node.children) == depth)
         push!(trie.value, node)
@@ -36,6 +59,11 @@ function insert_helper!(trie::NodeIdTrie, node::Node, depth::Int)
     return insert_helper!(trie.children[id], node, depth)
 end
 
+"""
+    insert!(trie::NodeTrie, node::Node)
+
+Insert the given node into the trie. It's sorted by its type in the first layer, then by its children in the following layers.
+"""
 function insert!(trie::NodeTrie, node::Node)
     t = typeof(node.task)
     if (!haskey(trie.children, t))
@@ -44,6 +72,11 @@ function insert!(trie::NodeTrie, node::Node)
     return insert_helper!(trie.children[typeof(node.task)], node, 0)
 end
 
+"""
+    collect_helper(trie::NodeIdTrie, acc::Set{Vector{Node}})
+
+Collects the Vectors of this [`NodeIdTrie`](@ref) node and all its children and puts them in the `acc` argument.
+"""
 function collect_helper(trie::NodeIdTrie, acc::Set{Vector{Node}})
     if (length(trie.value) >= 2)
         push!(acc, trie.value)
@@ -55,7 +88,11 @@ function collect_helper(trie::NodeIdTrie, acc::Set{Vector{Node}})
     return nothing
 end
 
-# returns all sets of multiple nodes that have accumulated in leaves
+"""
+    collect(trie::NodeTrie)
+
+Return all sets of at least 2 [`Node`](@ref)s that have accumulated in leaves of the trie.
+"""
 function collect(trie::NodeTrie)
     acc = Set{Vector{Node}}()
     for (t, child) in trie.children

src/utility.jl

@@ -1,3 +1,15 @@
+"""
+    bytes_to_human_readable(bytes)
+
+Return a human readable string representation of the given number.
+
+```jldoctest
+julia> using MetagraphOptimization
+
+julia> bytes_to_human_readable(4096)
+"4.0 KiB"
+```
+"""
 function bytes_to_human_readable(bytes)
     units = ["B", "KiB", "MiB", "GiB", "TiB"]
     unit_index = 1
@@ -8,15 +20,31 @@ function bytes_to_human_readable(bytes)
     return string(round(bytes, sigdigits = 4), " ", units[unit_index])
 end
 
+"""
+    lt_nodes(n1::Node, n2::Node)
+
+Less-Than comparison between nodes. Uses the nodes' ids to sort.
+"""
 function lt_nodes(n1::Node, n2::Node)
     return n1.id < n2.id
 end
 
+"""
+    sort_node!(node::Node)
+
+Sort the nodes' parents and children vectors. The vectors are mostly very short so sorting does not take a lot of time.
+Sorted nodes are required to make the finding of [`NodeReduction`](@ref)s a lot faster using the [`NodeTrie`](@ref) data structure.
+"""
 function sort_node!(node::Node)
     sort!(node.children, lt = lt_nodes)
     return sort!(node.parents, lt = lt_nodes)
 end
 
+"""
+    mem(graph::DAG)
+
+Return the memory footprint of the graph in Byte. Should be the same result as `Base.summarysize(graph)` but a lot faster.
+"""
 function mem(graph::DAG)
     size = 0
     size += Base.summarysize(graph.nodes, exclude = Union{Node})
@@ -42,12 +70,20 @@ function mem(graph::DAG)
     return size += sizeof(diff)
 end
 
-# calculate the size of this operation in Byte
+"""
+    mem(op::Operation)
+
+Return the memory footprint of the operation in Byte. Used in [`mem(graph::DAG)`](@ref). Unlike `Base.summarysize()` this doesn't follow all references which would yield (almost) the size of the entire graph.
+"""
 function mem(op::Operation)
     return Base.summarysize(op, exclude = Union{Node})
 end
 
-# calculate the size of this node in Byte
+"""
+    mem(op::Operation)
+
+Return the memory footprint of the node in Byte. Used in [`mem(graph::DAG)`](@ref). Unlike `Base.summarysize()` this doesn't follow all references which would yield (almost) the size of the entire graph.
+"""
 function mem(node::Node)
     return Base.summarysize(node, exclude = Union{Node, Operation})
 end

test/known_graphs.jl

@@ -3,7 +3,7 @@ using Random
 function test_known_graph(name::String, n, fusion_test = true)
     @testset "Test $name Graph ($n)" begin
         graph = parse_abc(joinpath(@__DIR__, "..", "input", "$name.txt"))
-        props = graph_properties(graph)
+        props = get_properties(graph)
 
         if (fusion_test)
             test_node_fusion(graph)
@@ -14,13 +14,13 @@ end
 
 function test_node_fusion(g::DAG)
     @testset "Test Node Fusion" begin
-        props = graph_properties(g)
+        props = get_properties(g)
 
         options = get_operations(g)
 
         nodes_number = length(g.nodes)
         data = props.data
-        compute_effort = props.compute_effort
+        compute_effort = props.computeEffort
 
         while !isempty(options.nodeFusions)
             fusion = first(options.nodeFusions)
@@ -29,13 +29,13 @@ function test_node_fusion(g::DAG)
 
             push_operation!(g, fusion)
 
-            props = graph_properties(g)
+            props = get_properties(g)
             @test props.data < data
-            @test props.compute_effort == compute_effort
+            @test props.computeEffort == compute_effort
 
             nodes_number = length(g.nodes)
             data = props.data
-            compute_effort = props.compute_effort
+            compute_effort = props.computeEffort
 
             options = get_operations(g)
         end
@@ -49,7 +49,7 @@ function test_random_walk(g::DAG, n::Int64)
 
         @test is_valid(g)
 
-        properties = graph_properties(g)
+        properties = get_properties(g)
 
         for i in 1:n
             # choose push or pop
@@ -82,7 +82,7 @@ function test_random_walk(g::DAG, n::Int64)
 
         @test is_valid(g)
 
-        @test properties == graph_properties(g)
+        @test properties == get_properties(g)
     end
 end
 

test/runtests.jl

@@ -5,8 +5,10 @@ using Test
     include("unit_tests_utility.jl")
     include("unit_tests_tasks.jl")
     include("unit_tests_nodes.jl")
+    include("unit_tests_properties.jl")
     include("node_reduction.jl")
     include("unit_tests_graph.jl")
+    include("unit_tests_execution.jl")
 
     include("known_graphs.jl")
 end

test/unit_tests_execution.jl

@@ -0,0 +1,31 @@
+import MetagraphOptimization.A
+import MetagraphOptimization.B
+import MetagraphOptimization.ParticleType
+
+@testset "Unit Tests Graph" begin
+    particles = Dict{ParticleType, Vector{Particle}}(
+        (
+            A => [
+                Particle(0.823648, 0.0, 0.0, 0.823648, A),
+                Particle(0.823648, -0.835061, -0.474802, 0.277915, A),
+            ]
+        ),
+        (
+            B => [
+                Particle(0.823648, 0.0, 0.0, -0.823648, B),
+                Particle(0.823648, 0.835061, 0.474802, -0.277915, B),
+            ]
+        ),
+    )
+
+    expected_result = 5.5320567694746876e-5
+
+    for _ in 1:10   # test in a loop because graph layout should not change the result
+        graph = parse_abc(joinpath(@__DIR__, "..", "input", "AB->AB.txt"))
+        @test isapprox(execute(graph, particles), expected_result; rtol = 0.001)
+
+        code = MetagraphOptimization.gen_code(graph)
+        @test isapprox(execute(code, particles), expected_result; rtol = 0.001)
+    end
+end
+println("Execution Unit Tests Complete!")

test/unit_tests_graph.jl

@@ -69,7 +69,7 @@ import MetagraphOptimization.partners
     @test length(graph.nodes) == 26
     @test length(graph.dirtyNodes) == 26
 
-    # now for all the edgese
+    # now for all the edges
     insert_edge!(graph, d_PB, PB, false)
     insert_edge!(graph, d_PA, PA, false)
     insert_edge!(graph, d_PBp, PBp, false)
@@ -142,12 +142,12 @@ import MetagraphOptimization.partners
     @test operations == get_operations(graph)
     nf = first(operations.nodeFusions)
 
-    properties = graph_properties(graph)
-    @test properties.compute_effort == 134
+    properties = get_properties(graph)
+    @test properties.computeEffort == 28
     @test properties.data == 62
-    @test properties.compute_intensity ≈ 134 / 62
-    @test properties.nodes == 26
-    @test properties.edges == 25
+    @test properties.computeIntensity ≈ 28 / 62
+    @test properties.noNodes == 26
+    @test properties.noEdges == 25
 
     push_operation!(graph, nf)
     # **does not immediately apply the operation**
@@ -161,17 +161,17 @@ import MetagraphOptimization.partners
           (addedNodes = 0, removedNodes = 0, addedEdges = 0, removedEdges = 0)
 
     # this applies pending operations
-    properties = graph_properties(graph)
+    properties = get_properties(graph)
 
     @test length(graph.nodes) == 24
     @test length(graph.appliedOperations) == 1
     @test length(graph.operationsToApply) == 0
     @test length(graph.dirtyNodes) != 0
-    @test properties.nodes == 24
-    @test properties.edges == 23
-    @test properties.compute_effort == 134
+    @test properties.noNodes == 24
+    @test properties.noEdges == 23
+    @test properties.computeEffort == 28
     @test properties.data < 62
-    @test properties.compute_intensity > 134 / 62
+    @test properties.computeIntensity > 28 / 62
 
     operations = get_operations(graph)
     @test length(graph.dirtyNodes) == 0
@@ -205,12 +205,12 @@ import MetagraphOptimization.partners
     @test length(graph.appliedOperations) == 0
     @test length(graph.operationsToApply) == 0
 
-    properties = graph_properties(graph)
-    @test properties.nodes == 26
-    @test properties.edges == 25
-    @test properties.compute_effort == 134
+    properties = get_properties(graph)
+    @test properties.noNodes == 26
+    @test properties.noEdges == 25
+    @test properties.computeEffort == 28
     @test properties.data == 62
-    @test properties.compute_intensity ≈ 134 / 62
+    @test properties.computeIntensity ≈ 28 / 62
 
     operations = get_operations(graph)
     @test length(operations) ==

test/unit_tests_properties.jl

@@ -0,0 +1,52 @@
+
+@testset "GraphProperties Unit Tests" begin
+    prop = GraphProperties()
+
+    @test prop.data == 0.0
+    @test prop.computeEffort == 0.0
+    @test prop.computeIntensity == 0.0
+    @test prop.cost == 0.0
+    @test prop.noNodes == 0.0
+    @test prop.noEdges == 0.0
+
+    prop2 = (
+        data = 5.0,
+        computeEffort = 6.0,
+        computeIntensity = 6.0 / 5.0,
+        cost = 0.0,
+        noNodes = 2,
+        noEdges = 3,
+    )::GraphProperties
+
+    @test prop + prop2 == prop2
+    @test prop2 - prop == prop2
+
+    negProp = -prop2
+    @test negProp.data == -5.0
+    @test negProp.computeEffort == -6.0
+    @test negProp.computeIntensity == 6.0 / 5.0
+    @test negProp.cost == 0.0
+    @test negProp.noNodes == -2
+    @test negProp.noEdges == -3
+
+    @test negProp + prop2 == GraphProperties()
+
+    prop3 = (
+        data = 7.0,
+        computeEffort = 3.0,
+        computeIntensity = 7.0 / 3.0,
+        cost = 0.0,
+        noNodes = -3,
+        noEdges = 2,
+    )::GraphProperties
+
+    propSum = prop2 + prop3
+
+    @test propSum.data == 12.0
+    @test propSum.computeEffort == 9.0
+    @test propSum.computeIntensity == 9.0 / 12.0
+    @test propSum.cost == 0.0
+    @test propSum.noNodes == -1
+    @test propSum.noEdges == 5
+end
+println("GraphProperties Unit Tests Complete!")

test/unit_tests_tasks.jl

@@ -10,11 +10,11 @@
     Data10 = MetagraphOptimization.DataTask(10)
     Data20 = MetagraphOptimization.DataTask(20)
 
-    @test MetagraphOptimization.compute_effort(S1) == 10
-    @test MetagraphOptimization.compute_effort(S2) == 10
-    @test MetagraphOptimization.compute_effort(U) == 6
-    @test MetagraphOptimization.compute_effort(V) == 20
-    @test MetagraphOptimization.compute_effort(P) == 15
+    @test MetagraphOptimization.compute_effort(S1) == 11
+    @test MetagraphOptimization.compute_effort(S2) == 12
+    @test MetagraphOptimization.compute_effort(U) == 1
+    @test MetagraphOptimization.compute_effort(V) == 6
+    @test MetagraphOptimization.compute_effort(P) == 0
     @test MetagraphOptimization.compute_effort(Sum) == 1
     @test MetagraphOptimization.compute_effort(Data10) == 0
     @test MetagraphOptimization.compute_effort(Data20) == 0
@@ -28,15 +28,6 @@
     @test MetagraphOptimization.data(Data10) == 10
     @test MetagraphOptimization.data(Data20) == 20
 
-    @test MetagraphOptimization.compute_intensity(S1) == typemax(UInt64)
-    @test MetagraphOptimization.compute_intensity(S2) == typemax(UInt64)
-    @test MetagraphOptimization.compute_intensity(U) == typemax(UInt64)
-    @test MetagraphOptimization.compute_intensity(V) == typemax(UInt64)
-    @test MetagraphOptimization.compute_intensity(P) == typemax(UInt64)
-    @test MetagraphOptimization.compute_intensity(Sum) == typemax(UInt64)
-    @test MetagraphOptimization.compute_intensity(Data10) == 0
-    @test MetagraphOptimization.compute_intensity(Data20) == 0
-
     @test S1 != S2
     @test Data10 != Data20