From 493ab0c6b4478956bf8f559a59530ea6b8c33b54 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 14:01:04 -0500 Subject: [PATCH 1/9] add basic CI Benchmarking --- .github/workflows/benchmark.yml | 32 ++++++++++++++++++++++++++++++++ benchmark/benchmarks.jl | 8 ++++++++ 2 files changed, 40 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/benchmarks.jl diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..06bde175 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,32 @@ +name: Run benchmarks + +on: + pull_request: + +jobs: + Benchmark: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: 1 + + ## Setup + - name: Default TESTCMD + run: echo "TESTCMD=julia" >> $GITHUB_ENV + - name: Ubuntu TESTCMD + if: startsWith(matrix.os,'ubuntu') + run: echo "TESTCMD=xvfb-run --auto-servernum julia" >> $GITHUB_ENV + - name: Install Plots dependencies + uses: julia-actions/julia-buildpkg@latest + - name: Install Benchmarking dependencies + run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' + + - name: Run benchmarks + run: $TESTCMD -e 'using BenchmarkCI; BenchmarkCI.judge()' + - name: Post results + run: julia -e 'using BenchmarkCI; BenchmarkCI.postjudge()' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 00000000..b4994a6c --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,8 @@ +using BenchmarkTools + +const SUITE = BenchmarkGroup() + +# numbered to enforce sequence +SUITE["1_load"] = @benchmarkable @eval(using Plots) +SUITE["2_plot"] = @benchmarkable p = plot(1:0.1:10, sin.(1:0.1:10)) +SUITE["3_display"] = @benchmarkable display(p) setup=(p = plot(1:0.1:10, sin.(1:0.1:10))) From 8efe35f5f858ae4d11b071d6b4972ce55529c386 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 14:14:04 -0500 Subject: [PATCH 2/9] add benchmark/Project.toml --- benchmark/Project.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 benchmark/Project.toml diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 00000000..d002907c --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,4 @@ +[deps] +BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b" +PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" From 6332aecc60714a6c1c9eff843e44010e6f9ec95e Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 14:18:18 -0500 Subject: [PATCH 3/9] Update Project.toml --- benchmark/Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/Project.toml b/benchmark/Project.toml index d002907c..d0c35ad5 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,4 +1,5 @@ [deps] BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" From 535d95334c6a3d58bbe4d234eaccf1fec4d8ac39 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 14:24:11 -0500 Subject: [PATCH 4/9] fix to use Ubuntu TESTCMD --- .github/workflows/benchmark.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 06bde175..e9afc61c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -14,10 +14,7 @@ jobs: version: 1 ## Setup - - name: Default TESTCMD - run: echo "TESTCMD=julia" >> $GITHUB_ENV - name: Ubuntu TESTCMD - if: startsWith(matrix.os,'ubuntu') run: echo "TESTCMD=xvfb-run --auto-servernum julia" >> $GITHUB_ENV - name: Install Plots dependencies uses: julia-actions/julia-buildpkg@latest From ac2fbadc07e0b9ee2a7609376c648a38ab0fded7 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 14:31:27 -0500 Subject: [PATCH 5/9] add missing GKS_ENCODING env var --- .github/workflows/benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e9afc61c..7047da27 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -6,6 +6,8 @@ on: jobs: Benchmark: if: "!contains(github.event.head_commit.message, '[skip ci]')" + env: + GKS_ENCODING: "utf8" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 0b8755db4b779f2f1c5a3890d3e36cfc89c7b069 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 14:48:37 -0500 Subject: [PATCH 6/9] add benchmark gitignores --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c653453c..544fc7cb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ deps/deps.jl Manifest.toml dev/ test/tmpplotsave.hdf5 +/.benchmarkci +/benchmark/*.json From 309f2db3b14802fc3beffe239528cee247dd3a63 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 15:21:23 -0500 Subject: [PATCH 7/9] run entire load_plot_display in separate julia instance --- benchmark/benchmarks.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index b4994a6c..1f8f9b61 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,8 +1,11 @@ using BenchmarkTools const SUITE = BenchmarkGroup() +julia_cmd = get(ENV, "TESTCMD", Base.JLOptions().julia_bin) # numbered to enforce sequence +SUITE["0_load_plot_display"] = @benchmarkable run(`$(julia_cmd) -e "using Plots; display(plot(1:0.1:10, sin.(1:0.1:10))))"`) + SUITE["1_load"] = @benchmarkable @eval(using Plots) SUITE["2_plot"] = @benchmarkable p = plot(1:0.1:10, sin.(1:0.1:10)) SUITE["3_display"] = @benchmarkable display(p) setup=(p = plot(1:0.1:10, sin.(1:0.1:10))) From 3c9580fb2faacfd6eeea714555518bb353a925a2 Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 15:45:49 -0500 Subject: [PATCH 8/9] fixes --- benchmark/benchmarks.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 1f8f9b61..4a478286 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -4,8 +4,8 @@ const SUITE = BenchmarkGroup() julia_cmd = get(ENV, "TESTCMD", Base.JLOptions().julia_bin) # numbered to enforce sequence -SUITE["0_load_plot_display"] = @benchmarkable run(`$(julia_cmd) -e "using Plots; display(plot(1:0.1:10, sin.(1:0.1:10))))"`) +SUITE["1_load_plot_display"] = @benchmarkable run(`sh -c $("$julia_cmd --startup-file=no -e 'using Plots; display(plot(1:0.1:10, sin.(1:0.1:10))))'")`) -SUITE["1_load"] = @benchmarkable @eval(using Plots) -SUITE["2_plot"] = @benchmarkable p = plot(1:0.1:10, sin.(1:0.1:10)) -SUITE["3_display"] = @benchmarkable display(p) setup=(p = plot(1:0.1:10, sin.(1:0.1:10))) +SUITE["2_load"] = @benchmarkable @eval(using Plots) +SUITE["3_plot"] = @benchmarkable p = plot(1:0.1:10, sin.(1:0.1:10)) +SUITE["4_display"] = @benchmarkable display(p) setup=(p = plot(1:0.1:10, sin.(1:0.1:10))) From 8a908c0de82e6865c608ed46afbc678e6153980d Mon Sep 17 00:00:00 2001 From: Ian Date: Sun, 24 Jan 2021 15:54:18 -0500 Subject: [PATCH 9/9] do load test in separate session --- benchmark/benchmarks.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 4a478286..398bf50e 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -5,7 +5,6 @@ julia_cmd = get(ENV, "TESTCMD", Base.JLOptions().julia_bin) # numbered to enforce sequence SUITE["1_load_plot_display"] = @benchmarkable run(`sh -c $("$julia_cmd --startup-file=no -e 'using Plots; display(plot(1:0.1:10, sin.(1:0.1:10))))'")`) - -SUITE["2_load"] = @benchmarkable @eval(using Plots) -SUITE["3_plot"] = @benchmarkable p = plot(1:0.1:10, sin.(1:0.1:10)) -SUITE["4_display"] = @benchmarkable display(p) setup=(p = plot(1:0.1:10, sin.(1:0.1:10))) +SUITE["2_load"] = @benchmarkable run(`sh -c $("$julia_cmd --startup-file=no -e 'using Plots'")`) +SUITE["3_plot"] = @benchmarkable p = plot(1:0.1:10, sin.(1:0.1:10)) setup(@eval(using Plots)) +SUITE["4_display"] = @benchmarkable display(p) setup=(@eval(using Plots); p = plot(1:0.1:10, sin.(1:0.1:10)))