diff --git a/Project.toml b/Project.toml index c4d6f59..81cff91 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ReferenceTests" uuid = "324d217c-45ce-50fc-942e-d289b448e8cf" authors = ["Christof Stocker ", "Lyndon White "] -version = "0.8.1" +version = "0.8.2" [deps] DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6" diff --git a/src/ReferenceTests.jl b/src/ReferenceTests.jl index ebfe0f4..ef7c750 100644 --- a/src/ReferenceTests.jl +++ b/src/ReferenceTests.jl @@ -17,8 +17,8 @@ export include("utils.jl") include("test_reference.jl") -include("core.jl") -include("handlers.jl") +include("fileio.jl") include("equality_metrics.jl") +include("render.jl") end # module diff --git a/src/core.jl b/src/core.jl deleted file mode 100644 index 19c2094..0000000 --- a/src/core.jl +++ /dev/null @@ -1,122 +0,0 @@ -################################################# -# Rendering -# This controls how failures are displayed -abstract type RenderMode end -struct Diff <: RenderMode end - -abstract type BeforeAfter <: RenderMode end -struct BeforeAfterLimited <: BeforeAfter end -struct BeforeAfterFull <: BeforeAfter end -struct BeforeAfterImage <: BeforeAfter end - -render_item(::RenderMode, item) = println(item) -function render_item(::BeforeAfterLimited, item) - show(IOContext(stdout, :limit=>true, :displaysize=>(20,80)), "text/plain", item) - println() -end -function render_item(::BeforeAfterImage, item) - str_item = @withcolor ImageInTerminal.encodeimg(ImageInTerminal.SmallBlocks(), ImageInTerminal.TermColor256(), item, 20, 40)[1] - println("eltype: ", eltype(item)) - println("size: ", map(length, axes(item))) - println("thumbnail:") - println.(str_item) -end - -## 2 arg form render for comparing -function render(mode::BeforeAfter, reference, actual) - println("- REFERENCE -------------------") - render_item(mode, reference) - println("-------------------------------") - println("- ACTUAL ----------------------") - render_item(mode, actual) - println("-------------------------------") -end -function render(::Diff, reference, actual) - println("- DIFF ------------------------") - @withcolor println(deepdiff(reference, actual)) - println("-------------------------------") -end - -## 1 arg form render for new content -function render(mode::RenderMode, actual) - println("- NEW CONTENT -----------------") - render_item(mode, actual) - println("-------------------------------") -end - -####################################### -# IO -# Right now this basically just extends FileIO to support some things as text files - -const TextFile = Union{File{format"TXT"}, File{format"SHA256"}} - -function loadfile(T, file::File) - T(load(file)) # Fallback to FileIO -end - -function loadfile(T, file::TextFile) - read(file.filename, String) -end - -function loadfile(::Type{<:Number}, file::File{format"TXT"}) - parse(Float64, loadfile(String, file)) -end - -function savefile(file::File, content) - save(file, content) # Fallback to FileIO -end - -function savefile(file::TextFile, content) - write(file.filename, content) -end - -########################################## - -# Final function -# all other functions should hit one of this eventually -# Which handles the actual testing and user prompting - -function _test_reference(equiv, rendermode, file::File, actual::T) where T - path = file.filename - dir, filename = splitdir(path) - - # preprocessing when reference file doesn't exists - if !isfile(path) - println("Reference file for \"$filename\" does not exist.") - render(rendermode, actual) - - if !isinteractive() - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") - end - - if !input_bool("Create reference file with above content (path: $path)?") - @test false - else - mkpath(dir) - savefile(file, actual) - @info("Please run the tests again for any changes to take effect") - end - - return nothing # skip current test case - end - - reference = loadfile(T, file) - if equiv(reference, actual) - @test true # to increase test counter if reached - else - # post-processing when test fails - println("Test for \"$filename\" failed.") - render(rendermode, reference, actual) - - if !isinteractive() - error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") - end - - if !input_bool("Replace reference with actual result (path: $path)?") - @test false - else - savefile(file, actual) - @info("Please run the tests again for any changes to take effect") - end - end -end diff --git a/src/equality_metrics.jl b/src/equality_metrics.jl index bb7923e..c011573 100644 --- a/src/equality_metrics.jl +++ b/src/equality_metrics.jl @@ -1,8 +1,22 @@ +""" + default_equality(reference, actual) -> f + +Infer a suitable equality comparison method `f` according to input types. + +`f` is a function that satisfies signature `f(reference, actual)::Bool`. If `f` outputs +`true`, it indicates that `reference` and `actual` are "equal" in the sense of `f`. +""" +default_equality(reference, actual) = isequal +function default_equality( + reference::AbstractArray{<:Colorant}, + actual::AbstractArray{<:Colorant}) + + return psnr_equality() +end + # --------------------------------- # Image -default_image_equality(reference, actual) = psnr_equality()(reference, actual) - """ psnr_equality(threshold=25) -> f diff --git a/src/fileio.jl b/src/fileio.jl new file mode 100644 index 0000000..fe63d63 --- /dev/null +++ b/src/fileio.jl @@ -0,0 +1,75 @@ +####################################### +# IO +# Right now this basically just extends FileIO to support some things as text files + +const TextFile = Union{File{format"TXT"}, File{format"SHA256"}} + +function loadfile(T, file::File) + T(load(file)) # Fallback to FileIO +end + +function loadfile(T, file::TextFile) + read(file.filename, String) +end + +function loadfile(::Type{<:Number}, file::File{format"TXT"}) + parse(Float64, loadfile(String, file)) +end + +function savefile(file::File, content) + save(file, content) # Fallback to FileIO +end + +function savefile(file::TextFile, content) + write(file.filename, content) +end + +function query_extended(filename) + file, ext = splitext(filename) + # TODO: make this less hacky + if uppercase(ext) == ".SHA256" + res = File{format"SHA256"}(filename) + else + res = query(filename) + if res isa File{DataFormat{:UNKNOWN}} + res = File{format"TXT"}(filename) + end + end + res +end + +""" + _convert(T::Type{<:DataFormat}, x; kw...) -> out + +Convert `x` to a validate content for file data format `T`. +""" +_convert(::Type{<:DataFormat}, x; kw...) = x + +# plain TXT +_convert(::Type{DataFormat{:TXT}}, x; kw...) = string(x) +_convert(::Type{DataFormat{:TXT}}, x::Number; kw...) = x +function _convert(::Type{DataFormat{:TXT}}, x::AbstractArray{<:AbstractString}; kw...) + return join(x, '\n') +end +function _convert( + ::Type{DataFormat{:TXT}}, img::AbstractArray{<:Colorant}; + size = (20,40), kw...) + + # encode image into string + strs = @withcolor ImageInTerminal.encodeimg( + ImageInTerminal.SmallBlocks(), + ImageInTerminal.TermColor256(), + img, + size...)[1] + return join(strs,'\n') +end + +# SHA256 +_convert(::Type{DataFormat{:SHA256}}, x; kw...) = bytes2hex(sha256(string(x))) +function _convert(::Type{DataFormat{:SHA256}}, img::AbstractArray{<:Colorant}; kw...) + # encode image into SHA256 + size_str = bytes2hex(sha256(reinterpret(UInt8,[map(Int64,size(img))...]))) + img_str = bytes2hex(sha256(reinterpret(UInt8,vec(rawview(channelview(img)))))) + + return size_str * img_str +end diff --git a/src/handlers.jl b/src/handlers.jl deleted file mode 100644 index f766ba8..0000000 --- a/src/handlers.jl +++ /dev/null @@ -1,65 +0,0 @@ -# -------------------------------------------------------------------- -# plain TXT - -function test_reference(file::File{format"TXT"}, actual; by = isequal, render = Diff()) - _test_reference(by, render, file, string(actual)) -end - -function test_reference(file::File{format"TXT"}, actual::Number; by = isequal, render = BeforeAfterFull()) - _test_reference(by, render, file, actual) -end - -function test_reference(file::File{format"TXT"}, actual::AbstractArray{<:AbstractString}; by = isequal, render = Diff()) - str = join(actual, '\n') - _test_reference(by, render, file, str) -end - -# --------------------------------- -# Image - -function test_reference(file::File, actual::AbstractArray{<:Colorant}; by = default_image_equality, render = BeforeAfterImage()) - _test_reference(by, render, file, actual) -end - -# Image as txt using ImageInTerminal -function test_reference(file::File{format"TXT"}, actual::AbstractArray{<:Colorant}; size = (20,40), by = isequal, render = BeforeAfterFull()) - strs = @withcolor ImageInTerminal.encodeimg(ImageInTerminal.SmallBlocks(), ImageInTerminal.TermColor256(), actual, size...)[1] - str = join(strs,'\n') - _test_reference(by, render, file, str) -end - -# -------------------------------------------------------------------- -# SHA as string - -function test_reference(file::File{format"SHA256"}, actual; by = nothing, render = BeforeAfterFull()) - test_reference(file, string(actual); render = render) -end - -function test_reference(file::File{format"SHA256"}, actual::Union{AbstractString,Vector{UInt8}}; by = nothing, render = BeforeAfterFull()) - str = bytes2hex(sha256(actual)) - _test_reference(isequal, render, file, str) -end - -function test_reference(file::File{format"SHA256"}, actual::AbstractArray{<:Colorant}; by = nothing, render = BeforeAfterFull()) - size_str = bytes2hex(sha256(reinterpret(UInt8,[map(Int64,size(actual))...]))) - img_str = bytes2hex(sha256(reinterpret(UInt8,vec(rawview(channelview(actual)))))) - _test_reference(isequal, render, file, size_str * img_str) -end - -# -------------------------------------------------------------------- - -# Fallback -function test_reference(file::File, actual; by = isequal, render = nothing) - if !(render === nothing) - _test_reference(by, render, file, actual) - else - if actual isa AbstractString - # we don't use dispatch for this as it is very ambiguous - # specialization will remove this conditional regardless - - _test_reference(by, Diff(), file, actual) - else - _test_reference(by, BeforeAfterLimited(), file, actual) - end - end -end diff --git a/src/render.jl b/src/render.jl new file mode 100644 index 0000000..267d141 --- /dev/null +++ b/src/render.jl @@ -0,0 +1,65 @@ +################################################# +# Rendering +# This controls how failures are displayed +abstract type RenderMode end +struct Diff <: RenderMode end + +abstract type BeforeAfter <: RenderMode end +struct BeforeAfterLimited <: BeforeAfter end +struct BeforeAfterFull <: BeforeAfter end +struct BeforeAfterImage <: BeforeAfter end + +render_item(::RenderMode, item) = println(item) +function render_item(::BeforeAfterLimited, item) + show(IOContext(stdout, :limit=>true, :displaysize=>(20,80)), "text/plain", item) + println() +end +function render_item(::BeforeAfterImage, item) + str_item = @withcolor ImageInTerminal.encodeimg(ImageInTerminal.SmallBlocks(), ImageInTerminal.TermColor256(), item, 20, 40)[1] + println("eltype: ", eltype(item)) + println("size: ", map(length, axes(item))) + println("thumbnail:") + println.(str_item) +end + +## 2 arg form render for comparing +function render(mode::BeforeAfter, reference, actual) + println("- REFERENCE -------------------") + render_item(mode, reference) + println("-------------------------------") + println("- ACTUAL ----------------------") + render_item(mode, actual) + println("-------------------------------") +end +function render(::Diff, reference, actual) + println("- DIFF ------------------------") + @withcolor println(deepdiff(reference, actual)) + println("-------------------------------") +end + +## 1 arg form render for new content +function render(mode::RenderMode, actual) + println("- NEW CONTENT -----------------") + render_item(mode, actual) + println("-------------------------------") +end + +""" + default_rendermode(::DataFormat, actual) + +Infer the most appropriate render mode according to type of reference file and `actual`. +""" +default_rendermode(::Type{<:DataFormat}, ::Any) = BeforeAfterLimited() +default_rendermode(::Type{<:DataFormat}, ::AbstractString) = Diff() +default_rendermode(::Type{<:DataFormat}, ::AbstractArray{<:Colorant}) = BeforeAfterImage() + +# plain TXTs +default_rendermode(::Type{DataFormat{:TXT}}, ::Any) = Diff() +default_rendermode(::Type{DataFormat{:TXT}}, ::AbstractString) = Diff() +default_rendermode(::Type{DataFormat{:TXT}}, ::Number) = BeforeAfterFull() +default_rendermode(::Type{DataFormat{:TXT}}, ::AbstractArray{<:Colorant}) = BeforeAfterFull() + +# SHA256 +default_rendermode(::Type{DataFormat{:SHA256}}, ::Any) = BeforeAfterFull() +default_rendermode(::Type{DataFormat{:SHA256}}, ::AbstractString) = BeforeAfterFull() +default_rendermode(::Type{DataFormat{:SHA256}}, ::AbstractArray{<:Colorant}) = BeforeAfterFull() diff --git a/src/test_reference.jl b/src/test_reference.jl index 8bbfccd..d9cdb31 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -83,20 +83,75 @@ macro test_reference(reference, actual, kws...) expr end -function test_reference(filename::AbstractString, actual; kw...) - test_reference(query_extended(filename), actual; kw...) +function test_reference( + filename::AbstractString, raw_actual; + by = nothing, render = nothing, kw...) + + test_reference(query_extended(filename), raw_actual, by, render; kw...) end -function query_extended(filename) - file, ext = splitext(filename) - # TODO: make this less hacky - if uppercase(ext) == ".SHA256" - res = File{format"SHA256"}(filename) +function test_reference( + file::File{F}, + raw_actual::T, + equiv=nothing, + rendermode=nothing; + kw...) where {F <: DataFormat, T} + + path = file.filename + dir, filename = splitdir(path) + + # infer the default rendermode here + # since `nothing` is always passed to this method from + # test_reference(filename::AbstractString, raw_actual; kw...) + if rendermode === nothing + rendermode = default_rendermode(F, raw_actual) + end + + # preprocessing when reference file doesn't exists + if !isfile(path) + println("Reference file for \"$filename\" does not exist.") + render(rendermode, raw_actual) + + if !isinteractive() + error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to create new reference images") + end + + if !input_bool("Create reference file with above content (path: $path)?") + @test false + else + mkpath(dir) + savefile(file, raw_actual) + @info("Please run the tests again for any changes to take effect") + end + + return nothing # skip current test case + end + + # file exists + actual = _convert(F, raw_actual; kw...) + reference = loadfile(T, file) + + if equiv === nothing + # generally, `reference` and `actual` are of the same type after preprocessing + equiv = default_equality(reference, actual) + end + + if equiv(reference, actual) + @test true # to increase test counter if reached else - res = query(filename) - if res isa File{DataFormat{:UNKNOWN}} - res = File{format"TXT"}(filename) + # post-processing when test fails + println("Test for \"$filename\" failed.") + render(rendermode, reference, actual) + + if !isinteractive() + error("You need to run the tests interactively with 'include(\"test/runtests.jl\")' to update reference images") + end + + if !input_bool("Replace reference with actual result (path: $path)?") + @test false + else + savefile(file, actual) + @info("Please run the tests again for any changes to take effect") end end - res end