switch tests to use ExamplePlots and VisualRegressionTests

.travis.yml

@@ -11,8 +11,10 @@ notifications:
 # uncomment the following lines to override the default test script
 script:
  - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
- - julia -e 'Pkg.clone("https://github.com/tbreloff/Images.jl.git"); Pkg.checkout("Images","tom_imgcompare");'
+ # - julia -e 'Pkg.clone("https://github.com/tbreloff/Images.jl.git"); Pkg.checkout("Images","tom_imgcompare");'
  - julia -e 'Pkg.clone("ImageMagick"); Pkg.build("ImageMagick")'
+ - julia -e 'Pkg.clone("https://github.com/tbreloff/VisualRegressionTests.jl.git");'
+ - julia -e 'Pkg.clone("https://github.com/tbreloff/ExamplePlots.jl.git");'
  - julia -e 'Pkg.clone("Cairo"); Pkg.build("Cairo")'
  - julia -e 'ENV["PYTHON"] = ""; Pkg.clone("PyPlot"); Pkg.build("PyPlot")'
  - julia -e 'Pkg.clone(pwd()); Pkg.build("Plots")'

src/recipes.jl

@@ -14,8 +14,17 @@ plot(recipe::PlotRecipe, args...; kw...) = plot(getRecipeXY(recipe)..., args...;
 plot!(recipe::PlotRecipe, args...; kw...) = plot!(getRecipeXY(recipe)..., args...; getRecipeArgs(recipe)..., kw...)
 plot!(plt::Plot, recipe::PlotRecipe, args...; kw...) = plot!(getRecipeXY(recipe)..., args...; getRecipeArgs(recipe)..., kw...)
 
+num_series(x::AMat) = size(x,2)
+num_series(x) = 1
+
 # if it's not a recipe, just do nothing and return the args
-_apply_recipe(d::Dict, args...; kw...) = args
+function _apply_recipe(d::Dict, args...; issubplot=false, kw...)
+    if issubplot && !haskey(d, :n) && !haskey(d, :layout)
+        # put in a sensible default
+        d[:n] = maximum(map(num_series, args))
+    end
+    args
+end
 
 # # -------------------------------------------------
 

test/imgcomp.jl

@@ -1,10 +1,13 @@
 
-# include this first to help with crashing??
-try
-  @eval using Gtk
-catch err
-  warn("Gtk not loaded. err: $err")
-end
+# # include this first to help with crashing??
+# try
+#   @eval using Gtk
+# catch err
+#   warn("Gtk not loaded. err: $err")
+# end
+
+using VisualRegressionTests
+using ExamplePlots
 
 # don't let pyplot use a gui... it'll crash
 # note: Agg will set gui -> :none in PyPlot
@@ -14,11 +17,11 @@ try
   info("Matplotlib version: $(PyPlot.matplotlib[:__version__])")
 end
 
-include("../docs/example_generation.jl")
+# include("../docs/example_generation.jl")
 
 
 using Plots, FactCheck
-import Images, ImageMagick
+# import Images, ImageMagick
 
 # if !isdefined(ImageMagick, :init_deps)
 #   function ImageMagick.init_deps()
@@ -26,68 +29,75 @@ import Images, ImageMagick
 #   end
 # end
 
-function makeImageWidget(fn)
-  img = Gtk.GtkImageLeaf(fn)
-  vbox = Gtk.GtkBoxLeaf(:v)
-  push!(vbox, Gtk.GtkLabelLeaf(fn))
-  push!(vbox, img)
-  show(img)
-  vbox
-end
+# function makeImageWidget(fn)
+#   img = Gtk.GtkImageLeaf(fn)
+#   vbox = Gtk.GtkBoxLeaf(:v)
+#   push!(vbox, Gtk.GtkLabelLeaf(fn))
+#   push!(vbox, img)
+#   show(img)
+#   vbox
+# end
 
-function replaceReferenceImage(tmpfn, reffn)
-  cmd = `cp $tmpfn $reffn`
-  run(cmd)
-  info("Replaced reference image with: $cmd")
-end
+# function replaceReferenceImage(tmpfn, reffn)
+#   cmd = `cp $tmpfn $reffn`
+#   run(cmd)
+#   info("Replaced reference image with: $cmd")
+# end
 
-"Show a Gtk popup with both images and a confirmation whether we should replace the new image with the old one"
-function compareToReferenceImage(tmpfn, reffn)
+# "Show a Gtk popup with both images and a confirmation whether we should replace the new image with the old one"
+# function compareToReferenceImage(tmpfn, reffn)
 
-  # add the images
-  imgbox = Gtk.GtkBoxLeaf(:h)
-  push!(imgbox, makeImageWidget(tmpfn))
-  push!(imgbox, makeImageWidget(reffn))
+#   # add the images
+#   imgbox = Gtk.GtkBoxLeaf(:h)
+#   push!(imgbox, makeImageWidget(tmpfn))
+#   push!(imgbox, makeImageWidget(reffn))
 
-  win = Gtk.GtkWindowLeaf("Should we make this the new reference image?")
-  push!(win, Gtk.GtkFrameLeaf(imgbox))
+#   win = Gtk.GtkWindowLeaf("Should we make this the new reference image?")
+#   push!(win, Gtk.GtkFrameLeaf(imgbox))
 
-  showall(win)
+#   showall(win)
 
-  # now ask the question
-  if Gtk.ask_dialog("Should we make this the new reference image?", "No", "Yes")
-    replaceReferenceImage(tmpfn, reffn)
-  end
+#   # now ask the question
+#   if Gtk.ask_dialog("Should we make this the new reference image?", "No", "Yes")
+#     replaceReferenceImage(tmpfn, reffn)
+#   end
 
-  destroy(win)
-end
+#   destroy(win)
+# end
 
 
 # TODO: use julia's Condition type and the wait() and notify() functions to initialize a Window, then wait() on a condition that 
 #       is referenced in a button press callback (the button clicked callback will call notify() on that condition)
 
-function image_comparison_tests(pkg::Symbol, idx::Int; debug = false, sigma = [1,1], eps = 1e-2)
+function image_comparison_tests(pkg::Symbol, idx::Int; debug = false, popup = isinteractive(), sigma = [1,1], eps = 1e-2)
   
   # first 
   Plots._debugMode.on = debug
-  info("Testing plot: $pkg:$idx:$(PlotExamples.examples[idx].header)")
+  example = ExamplePlots.examples[idx]
+  info("Testing plot: $pkg:$idx:$(example.header)")
   backend(pkg)
   backend()
 
   # ensure consistent results
   srand(1234)
 
+  # test function
+  func = (fn, idx) -> begin
+    map(eval, example.exprs)
+    png(fn)
+  end
+
   # run the example
-  map(eval, PlotExamples.examples[idx].exprs)
+  # map(eval, PlotExamples.examples[idx].exprs)
 
-  # save the png
-  tmpfn = tempname() * ".png"
-  png(tmpfn)
+  # # save the png
+  # tmpfn = tempname() * ".png"
+  # png(tmpfn)
 
-  # load the saved png
-  tmpimg = Images.load(tmpfn)
+  # # load the saved png
+  # tmpimg = Images.load(tmpfn)
 
-  # reference image location
+  # reference image directory setup
   refdir = joinpath(Pkg.dir("Plots"), "test", "refimg", string(pkg))
   try
     run(`mkdir -p $refdir`)
@@ -96,46 +106,50 @@ function image_comparison_tests(pkg::Symbol, idx::Int; debug = false, sigma = [1
   end
   reffn = joinpath(refdir, "ref$idx.png")
 
-  try
+  # the test
+  vtest = VisualTest(func, reffn, idx)
+  test_images(vtest, popup=popup, sigma=sigma, eps=eps)
 
-    # info("Comparing $tmpfn to reference $reffn")
+  # try
+
+  #   # info("Comparing $tmpfn to reference $reffn")
   
-    # load the reference image
-    refimg = Images.load(reffn)
+  #   # load the reference image
+  #   refimg = Images.load(reffn)
 
-    # run the comparison test... a difference will throw an error
-    # NOTE: sigma is a 2-length vector with x/y values for the number of pixels
-    #       to blur together when comparing images
-    diffpct = Images.test_approx_eq_sigma_eps(tmpimg, refimg, sigma, eps)
+  #   # run the comparison test... a difference will throw an error
+  #   # NOTE: sigma is a 2-length vector with x/y values for the number of pixels
+  #   #       to blur together when comparing images
+  #   diffpct = Images.test_approx_eq_sigma_eps(tmpimg, refimg, sigma, eps)
 
-    # we passed!
-    info("Reference image $reffn matches.  Difference: $diffpct")
-    return true
+  #   # we passed!
+  #   info("Reference image $reffn matches.  Difference: $diffpct")
+  #   return true
 
-  catch err
-    warn("Image did not match reference image $reffn. err: $err")
-    # showerror(Base.STDERR, err)
+  # catch err
+  #   warn("Image did not match reference image $reffn. err: $err")
+  #   # showerror(Base.STDERR, err)
     
-    if isinteractive()
+  #   if isinteractive()
 
-      # if we're in interactive mode, open a popup and give us a chance to examine the images
-      warn("Should we make this the new reference image?")
-      compareToReferenceImage(tmpfn, reffn)
-      # println("exited")
-      return
+  #     # if we're in interactive mode, open a popup and give us a chance to examine the images
+  #     warn("Should we make this the new reference image?")
+  #     compareToReferenceImage(tmpfn, reffn)
+  #     # println("exited")
+  #     return
       
-    else
+  #   else
 
-      # if we rejected the image, or if we're in automated tests, throw the error
-      rethrow(err)
-    end
+  #     # if we rejected the image, or if we're in automated tests, throw the error
+  #     rethrow(err)
+  #   end
 
-  end
+  # end
 end
 
-function image_comparison_tests(pkg::Symbol; skip = [], debug = false, sigma = [1,1], eps = 1e-2)
-  for i in 1:length(PlotExamples.examples)
+function image_comparison_facts(pkg::Symbol; skip = [], debug = false, sigma = [1,1], eps = 1e-2)
+  for i in 1:length(ExamplePlots.examples)
     i in skip && continue
-    @fact image_comparison_tests(pkg, i, debug=debug, sigma=sigma, eps=eps) --> true
+    @fact image_comparison_tests(pkg, i, debug=debug, sigma=sigma, eps=eps) |> success --> true
   end
 end

test/runtests.jl

@@ -1,23 +1,23 @@
 module PlotsTests
 
 
-# don't let pyplot use a gui... it'll crash
-# note: Agg will set gui -> :none in PyPlot
-ENV["MPLBACKEND"] = "Agg"
-try
-    @eval import PyPlot
-catch err
-    warn("Couldn't import PyPlot: $err")
-end
+# # don't let pyplot use a gui... it'll crash
+# # note: Agg will set gui -> :none in PyPlot
+# ENV["MPLBACKEND"] = "Agg"
+# try
+#     @eval import PyPlot
+# catch err
+#     warn("Couldn't import PyPlot: $err")
+# end
 
 
-using Plots
-using FactCheck
+# using Plots
+# using FactCheck
 
-# note: wrap first include in a try block because of the ImageMagick init_deps bug
-try
-    include("imgcomp.jl")
-end
+# # note: wrap first include in a try block because of the ImageMagick init_deps bug
+# try
+#     include("imgcomp.jl")
+# end
 include("imgcomp.jl")
 
 # don't actually show the plots
@@ -49,18 +49,14 @@ facts("Gadfly") do
     # plot(x::AMat, y::AMat; kw...)              # multiple lines (one per column of x/y... will assert size(x) == size(y))
     @fact plot!(rand(10,3), rand(10,3)) --> not(nothing)
 
-    if VERSION >= v"0.4-"
-        image_comparison_tests(:gadfly, skip=[4,19], eps=img_eps)
-    end
+    image_comparison_facts(:gadfly, skip=[4,19,23,24], eps=img_eps)
 end
 
 
-if VERSION >= v"0.4-"
-    facts("PyPlot") do
-        @fact pyplot() --> Plots.PyPlotPackage()
-        @fact backend() --> Plots.PyPlotPackage()
-        image_comparison_tests(:pyplot, skip=[19,21], eps=img_eps)
-    end
+facts("PyPlot") do
+    @fact pyplot() --> Plots.PyPlotPackage()
+    @fact backend() --> Plots.PyPlotPackage()
+    image_comparison_facts(:pyplot, skip=[19,21,23], eps=img_eps)
 end
 
 
@@ -117,7 +113,7 @@ end
     #     @fact backend() --> Plots.PyPlotPackage()
     #     @fact typeof(plot(1:10)) --> Plots.Plot{Plots.PyPlotPackage}
 
-    #     # image_comparison_tests(:pyplot, skip=[19])
+    #     # image_comparison_facts(:pyplot, skip=[19])
     # end
 # catch err
 #     warn("Skipped PyPlot due to: ", string(err))