From 4b6ce981330071c2f9505e156c212820593c22ef Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jan 2025 16:40:00 +0100 Subject: [PATCH 1/5] Revert back to our hand-rolled test runner. --- test/Project.toml | 8 +- test/{aqua_tests.jl => aqua.jl} | 4 - test/{bpf_tests.jl => bpf.jl} | 6 - test/{examples_tests.jl => examples.jl} | 4 - test/{gcn_tests.jl => gcn.jl} | 12 - test/{bpf_testsetup.jl => helpers/bpf.jl} | 9 +- test/{gcn_testsetup.jl => helpers/gcn.jl} | 9 +- test/{metal_testsetup.jl => helpers/metal.jl} | 9 +- .../native.jl} | 10 +- .../precompile.jl} | 26 +- test/{ptx_testsetup.jl => helpers/ptx.jl} | 9 +- test/{ => helpers}/runtime.jl | 0 test/{spirv_testsetup.jl => helpers/spirv.jl} | 9 +- test/{helper_testsetup.jl => helpers/test.jl} | 8 - test/{metal_tests.jl => metal.jl} | 9 - test/{native_tests.jl => native.jl} | 36 +- test/{ptx_tests.jl => ptx.jl} | 29 +- test/runtests.jl | 419 ++++++++++++++++-- test/setup.jl | 68 +++ test/{spirv_tests.jl => spirv.jl} | 10 - test/{util_tests.jl => utils.jl} | 56 --- 21 files changed, 520 insertions(+), 230 deletions(-) rename test/{aqua_tests.jl => aqua.jl} (57%) rename test/{bpf_tests.jl => bpf.jl} (89%) rename test/{examples_tests.jl => examples.jl} (95%) rename test/{gcn_tests.jl => gcn.jl} (95%) rename test/{bpf_testsetup.jl => helpers/bpf.jl} (89%) rename test/{gcn_testsetup.jl => helpers/gcn.jl} (91%) rename test/{metal_testsetup.jl => helpers/metal.jl} (92%) rename test/{native_testsetup.jl => helpers/native.jl} (95%) rename test/{precompile_testsetup.jl => helpers/precompile.jl} (77%) rename test/{ptx_testsetup.jl => helpers/ptx.jl} (95%) rename test/{ => helpers}/runtime.jl (100%) rename test/{spirv_testsetup.jl => helpers/spirv.jl} (92%) rename test/{helper_testsetup.jl => helpers/test.jl} (95%) rename test/{metal_tests.jl => metal.jl} (96%) rename test/{native_tests.jl => native.jl} (96%) rename test/{ptx_tests.jl => ptx.jl} (95%) create mode 100644 test/setup.jl rename test/{spirv_tests.jl => spirv.jl} (90%) rename test/{util_tests.jl => utils.jl} (58%) diff --git a/test/Project.toml b/test/Project.toml index 9e6307f1..0a577a22 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,11 +1,14 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SPIRV_LLVM_Translator_unified_jll = "85f0d8ed-5b39-5caa-b1ae-7472de402361" SPIRV_Tools_jll = "6ac6d60f-d740-5983-97d7-a4482c0689f4" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" @@ -14,3 +17,6 @@ demumble_jll = "1e29f10c-031c-5a83-9565-69cddfc27673" [compat] Aqua = "0.8" + +[extras] +GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55" diff --git a/test/aqua_tests.jl b/test/aqua.jl similarity index 57% rename from test/aqua_tests.jl rename to test/aqua.jl index 7244df57..08cd9b51 100644 --- a/test/aqua_tests.jl +++ b/test/aqua.jl @@ -1,7 +1,3 @@ -@testitem "Aqua" begin - using Aqua Aqua.test_all(GPUCompiler) - -end diff --git a/test/bpf_tests.jl b/test/bpf.jl similarity index 89% rename from test/bpf_tests.jl rename to test/bpf.jl index 677c7bf5..d94b0d32 100644 --- a/test/bpf_tests.jl +++ b/test/bpf.jl @@ -1,7 +1,3 @@ -@testitem "BPF" setup=[BPF, Helpers] begin - -############################################################################################ - @testset "No-op" begin kernel() = 0 @@ -40,5 +36,3 @@ end @test_throws GPUCompiler.InvalidIRError BPF.code_execution(kernel, (Int,)) end end - -end diff --git a/test/examples_tests.jl b/test/examples.jl similarity index 95% rename from test/examples_tests.jl rename to test/examples.jl index 095648cf..aef6d4cc 100644 --- a/test/examples_tests.jl +++ b/test/examples.jl @@ -1,5 +1,3 @@ -@testitem "examples" begin - function find_sources(path::String, sources=String[]) if isdir(path) for entry in readdir(path) @@ -23,5 +21,3 @@ cd(dir) do @test success(pipeline(`$cmd $example`, stderr=stderr)) end end - -end diff --git a/test/gcn_tests.jl b/test/gcn.jl similarity index 95% rename from test/gcn_tests.jl rename to test/gcn.jl index fe408337..c9df660f 100644 --- a/test/gcn_tests.jl +++ b/test/gcn.jl @@ -1,11 +1,3 @@ -@testitem "GCN" setup=[GCN, Helpers] begin - -@inline sink_gcn(i) = sink(i, Val(5)) - -@test GCNCompilerTarget(dev_isa="gfx900") == GCNCompilerTarget("gfx900") - -############################################################################################ - @testset "IR" begin @testset "kernel calling convention" begin @@ -222,7 +214,3 @@ end end end - -############################################################################################ - -end diff --git a/test/bpf_testsetup.jl b/test/helpers/bpf.jl similarity index 89% rename from test/bpf_testsetup.jl rename to test/helpers/bpf.jl index 0570a6e0..49d9d6e4 100644 --- a/test/bpf_testsetup.jl +++ b/test/helpers/bpf.jl @@ -1,11 +1,8 @@ -@testsetup module BPF +module BPF -using GPUCompiler +using ..GPUCompiler +import ..TestRuntime - -# create a native test compiler, and generate reflection methods for it - -include("runtime.jl") struct CompilerParams <: AbstractCompilerParams end GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime diff --git a/test/gcn_testsetup.jl b/test/helpers/gcn.jl similarity index 91% rename from test/gcn_testsetup.jl rename to test/helpers/gcn.jl index 846db4b6..2cb371e9 100644 --- a/test/gcn_testsetup.jl +++ b/test/helpers/gcn.jl @@ -1,11 +1,8 @@ -@testsetup module GCN +module GCN -using GPUCompiler +using ..GPUCompiler +import ..TestRuntime - -# create a GCN-based test compiler, and generate reflection methods for it - -include("runtime.jl") struct CompilerParams <: AbstractCompilerParams end GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime diff --git a/test/metal_testsetup.jl b/test/helpers/metal.jl similarity index 92% rename from test/metal_testsetup.jl rename to test/helpers/metal.jl index 0055cb18..c45ba4c4 100644 --- a/test/metal_testsetup.jl +++ b/test/helpers/metal.jl @@ -1,11 +1,8 @@ -@testsetup module Metal +module Metal -using GPUCompiler +using ..GPUCompiler +import ..TestRuntime - -# create a Metal test compiler, and generate reflection methods for it - -include("runtime.jl") struct CompilerParams <: AbstractCompilerParams end GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime diff --git a/test/native_testsetup.jl b/test/helpers/native.jl similarity index 95% rename from test/native_testsetup.jl rename to test/helpers/native.jl index 3406276c..c1c39ba0 100644 --- a/test/native_testsetup.jl +++ b/test/helpers/native.jl @@ -1,11 +1,7 @@ -@testsetup module Native +module Native -using GPUCompiler - - -# create a native test compiler, and generate reflection methods for it - -include("runtime.jl") +using ..GPUCompiler +import ..TestRuntime # local method table for device functions Base.Experimental.@MethodTable(test_method_table) diff --git a/test/precompile_testsetup.jl b/test/helpers/precompile.jl similarity index 77% rename from test/precompile_testsetup.jl rename to test/helpers/precompile.jl index ee3bec06..daa1b57b 100644 --- a/test/precompile_testsetup.jl +++ b/test/helpers/precompile.jl @@ -1,10 +1,3 @@ -@testsetup module Precompile - -using Test -using ReTestItems - -export precompile_test_harness, check_presence, create_standalone - function precompile_test_harness(@nospecialize(f), testset::String) @testset "$testset" begin precompile_test_harness(f, true) @@ -50,19 +43,18 @@ function check_presence(mi, token) end function create_standalone(load_path, name::String, file) - cp(joinpath(@__DIR__, "runtime.jl"), joinpath(load_path, "runtime.jl"), force=true) + code = :( + module $(Symbol(name)) - TS = include(file) - code = TS.code - if code.head == :begin - code.head = :block - end - @assert code.head == :block - code = Expr(:module, true, Symbol(name), code) + using GPUCompiler + + include($(joinpath(@__DIR__, "runtime.jl"))) + include($(joinpath(@__DIR__, file))) + + end + ) # Write out the test setup as a micro package write(joinpath(load_path, "$name.jl"), string(code)) Base.compilecache(Base.PkgId(name)) end - -end # testsetup diff --git a/test/ptx_testsetup.jl b/test/helpers/ptx.jl similarity index 95% rename from test/ptx_testsetup.jl rename to test/helpers/ptx.jl index ed5026f1..a9f58871 100644 --- a/test/ptx_testsetup.jl +++ b/test/helpers/ptx.jl @@ -1,11 +1,8 @@ -@testsetup module PTX +module PTX -using GPUCompiler +using ..GPUCompiler +import ..TestRuntime - -# create a PTX-based test compiler, and generate reflection methods for it - -include("runtime.jl") struct CompilerParams <: AbstractCompilerParams end PTXCompilerJob = CompilerJob{PTXCompilerTarget,CompilerParams} diff --git a/test/runtime.jl b/test/helpers/runtime.jl similarity index 100% rename from test/runtime.jl rename to test/helpers/runtime.jl diff --git a/test/spirv_testsetup.jl b/test/helpers/spirv.jl similarity index 92% rename from test/spirv_testsetup.jl rename to test/helpers/spirv.jl index f1221545..6bd00077 100644 --- a/test/spirv_testsetup.jl +++ b/test/helpers/spirv.jl @@ -1,11 +1,8 @@ -@testsetup module SPIRV +module SPIRV -using GPUCompiler +using ..GPUCompiler +import ..TestRuntime - -# create a SPIRV-based test compiler, and generate reflection methods for it - -include("runtime.jl") struct CompilerParams <: AbstractCompilerParams end GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime diff --git a/test/helper_testsetup.jl b/test/helpers/test.jl similarity index 95% rename from test/helper_testsetup.jl rename to test/helpers/test.jl index cf15bc73..eb60c9a1 100644 --- a/test/helper_testsetup.jl +++ b/test/helpers/test.jl @@ -1,9 +1,3 @@ -@testsetup module Helpers - -using Test - -export @test_throws_message, sink - # @test_throw, with additional testing for the exception message macro test_throws_message(f, typ, ex...) quote @@ -41,5 +35,3 @@ end ret i64 %value""" return :(Base.llvmcall($llvmcall_str, T, Tuple{T}, i)) end - -end diff --git a/test/metal_tests.jl b/test/metal.jl similarity index 96% rename from test/metal_tests.jl rename to test/metal.jl index 113868aa..db626435 100644 --- a/test/metal_tests.jl +++ b/test/metal.jl @@ -1,9 +1,3 @@ -@testitem "Metal" setup=[Metal, Helpers] begin - -using LLVM - -############################################################################################ - @testset "IR" begin @testset "kernel functions" begin @@ -110,7 +104,6 @@ end return end - ir = sprint(io->Metal.code_llvm(io, kernel1, Tuple{Core.LLVMPtr{Float32,1}}; validate=true)) @test occursin("@metal_os_log", ir) @@ -145,5 +138,3 @@ end end end - -end diff --git a/test/native_tests.jl b/test/native.jl similarity index 96% rename from test/native_tests.jl rename to test/native.jl index d4a07442..d88ec86a 100644 --- a/test/native_tests.jl +++ b/test/native.jl @@ -1,9 +1,3 @@ -@testitem "native" setup=[Native, Helpers] begin - -using Test - -############################################################################################ - @testset "reflection" begin job, _ = Native.create_job(identity, (Int,)) @@ -233,7 +227,7 @@ end @test "We did not crash!" != "" end -@testset "CUDAnative.jl#278" begin +@testset "CUDAjl#278" begin # codegen idempotency # NOTE: this isn't fixed, but surfaces here due to bad inference of checked_sub # NOTE: with the fix to print_to_string this doesn't error anymore, @@ -563,18 +557,17 @@ end ["jl_invoke", "apply_iterate", "inttoptr", "apply_type"]) end -end # testitem -@testitem "native precompile" setup=[Precompile,] begin +############################################################################################ + +@testset "precompile" begin -using Test precompile_test_harness("Inference caching") do load_path # Write out the Native test setup as a micro package - create_standalone(load_path, "NativeCompiler", "native_testsetup.jl") + create_standalone(load_path, "TestCompiler", "native.jl") write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching - import NativeCompiler - import GPUCompiler + import TestCompiler using PrecompileTools function kernel(A, x) @@ -583,13 +576,13 @@ precompile_test_harness("Inference caching") do load_path end let - job, _ = NativeCompiler.create_job(kernel, (Vector{Int}, Int)) + job, _ = TestCompiler.Native.create_job(kernel, (Vector{Int}, Int)) precompile(job) end # identity is foreign @setup_workload begin - job, _ = NativeCompiler.create_job(identity, (Int,)) + job, _ = TestCompiler.Native.create_job(identity, (Int,)) @compile_workload begin precompile(job) end @@ -598,13 +591,13 @@ precompile_test_harness("Inference caching") do load_path Base.compilecache(Base.PkgId("InferenceCaching")) @eval let - import NativeCompiler + import TestCompiler # Check that no cached entry is present identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) token = let - job, _ = NativeCompiler.create_job(identity, (Int,)) + job, _ = TestCompiler.Native.create_job(identity, (Int,)) GPUCompiler.ci_cache_token(job) end @test !check_presence(identity_mi, token) @@ -624,7 +617,7 @@ precompile_test_harness("Inference caching") do load_path GPUCompiler.enable_disk_cache!() @test GPUCompiler.disk_cache_enabled() == true - job, _ = NativeCompiler.create_job(InferenceCaching.kernel, (Vector{Int}, Int)) + job, _ = TestCompiler.Native.create_job(InferenceCaching.kernel, (Vector{Int}, Int)) @assert job.source == kernel_mi ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) @assert ci !== nothing @@ -632,13 +625,14 @@ precompile_test_harness("Inference caching") do load_path path = GPUCompiler.cache_file(ci, job.config) @test path !== nothing @test !ispath(path) - NativeCompiler.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int)) + TestCompiler.Native.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int)) @test ispath(path) GPUCompiler.clear_disk_cache!() @test !ispath(path) + + GPUCompiler.enable_disk_cache!(false) + @test GPUCompiler.disk_cache_enabled() == false end end -############################################################################################ - end diff --git a/test/ptx_tests.jl b/test/ptx.jl similarity index 95% rename from test/ptx_tests.jl rename to test/ptx.jl index 70f67bbb..46f0e673 100644 --- a/test/ptx_tests.jl +++ b/test/ptx.jl @@ -1,9 +1,3 @@ -@testitem "PTX" setup=[PTX, Helpers] begin - -using LLVM - -############################################################################################ - @testset "IR" begin @testset "exceptions" begin @@ -349,16 +343,17 @@ end end end -end # testitem -@testitem "PTX precompile" setup=[Precompile,] begin +############################################################################################ + +@testset "precompile" begin + precompile_test_harness("Inference caching") do load_path - # Write out the PTX test setup as a micro package - create_standalone(load_path, "PTXCompiler", "ptx_testsetup.jl") + # Write out the PTX test helpers as a micro package + create_standalone(load_path, "TestCompiler", "ptx.jl") write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching - import PTXCompiler - import GPUCompiler + import TestCompiler using PrecompileTools function kernel() @@ -366,13 +361,13 @@ precompile_test_harness("Inference caching") do load_path end let - job, _ = PTXCompiler.create_job(kernel, ()) + job, _ = TestCompiler.PTX.create_job(kernel, ()) precompile(job) end # identity is foreign @setup_workload begin - job, _ = PTXCompiler.create_job(identity, (Int,)) + job, _ = TestCompiler.PTX.create_job(identity, (Int,)) @compile_workload begin precompile(job) end @@ -381,13 +376,13 @@ precompile_test_harness("Inference caching") do load_path Base.compilecache(Base.PkgId("InferenceCaching")) @eval let - import PTXCompiler + import TestCompiler # Check that no cached entry is present identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) token = let - job, _ = PTXCompiler.create_job(identity, (Int,)) + job, _ = PTX.create_job(identity, (Int,)) GPUCompiler.ci_cache_token(job) end ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing @@ -407,6 +402,4 @@ precompile_test_harness("Inference caching") do load_path end end -############################################################################################ - end diff --git a/test/runtests.jl b/test/runtests.jl index 199e641f..cfb96f9a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,42 +1,407 @@ -using GPUCompiler, LLVM -GPUCompiler.reset_runtime() +using Distributed +using Dates +import REPL +using Printf: @sprintf +using Base.Filesystem: path_separator -using InteractiveUtils -@info "System information:\n" * sprint(io->versioninfo(io; verbose=true)) +# parse some command-line arguments +function extract_flag!(args, flag, default=nothing) + for f in args + if startswith(f, flag) + # Check if it's just `--flag` or if it's `--flag=foo` + if f != flag + val = split(f, '=')[2] + if default !== nothing && !(typeof(default) <: AbstractString) + val = parse(typeof(default), val) + end + else + val = default + end -import SPIRV_LLVM_Translator_unified_jll -import SPIRV_Tools_jll + # Drop this value from our args + filter!(x -> x != f, args) + return (true, val) + end + end + return (false, default) +end +do_help, _ = extract_flag!(ARGS, "--help") +if do_help + println(""" + Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...] + + --help Show this text. + --list List all available tests. + --quickfail Fail the entire run as soon as a single test errored. + --jobs=N Launch `N` processes to perform tests (default: Sys.CPU_THREADS). + + Remaining arguments filter the tests that will be executed.""") + exit(0) +end +_, jobs = extract_flag!(ARGS, "--jobs", Sys.CPU_THREADS) +do_quickfail, _ = extract_flag!(ARGS, "--quickfail") + +include("setup.jl") # make sure everything is precompiled + +@info "Running $jobs tests in parallel. If this is too many, specify the `--jobs` argument to the tests, or set the JULIA_CPU_THREADS environment variable." -using ReTestItems -runtests(GPUCompiler; nworkers=min(Sys.CPU_THREADS,4), nworker_threads=1, - testitem_timeout=120) do ti - if ti.name == "GCN" && LLVM.is_asserts() - # XXX: GCN's non-0 stack address space triggers LLVM assertions due to Julia bugs - return false +# discover tests +const all_tests = [] +const test_runners = Dict() +## files in the test folder +for (rootpath, dirs, files) in walkdir(@__DIR__) + # find Julia files + filter!(files) do file + endswith(file, ".jl") && file !== "setup.jl" && file !== "runtests.jl" + end + isempty(files) && continue + basename(rootpath) == "helpers" && continue + + # strip extension + files = map(files) do file + file[1:end-3] + end + + # prepend subdir + subdir = relpath(rootpath, @__DIR__) + if subdir != "." + files = map(files) do file + joinpath(subdir, file) end + end - @dispose ctx=Context() begin - # XXX: some back-ends do not support opaque pointers - if ti.name in ["Metal"] && !supports_typed_pointers(ctx) - return false - end + # unify path separators + files = map(files) do file + replace(file, path_separator => '/') + end + + append!(all_tests, files) + for file in files + test_runners[file] = ()->include("$(@__DIR__)/$file.jl") + end +end + +# parse some more command-line arguments +## --list to list all available tests +do_list, _ = extract_flag!(ARGS, "--list") +if do_list + println("Available tests:") + for test in sort(all_tests) + println(" - $test") end + exit(0) +end +## no options should remain +optlike_args = filter(startswith("-"), ARGS) +if !isempty(optlike_args) + error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)") +end +## the remaining args filter tests +tests = copy(all_tests) +if !isempty(ARGS) + filter!(tests) do test + any(arg->startswith(test, arg), ARGS) + end +end + +# determine tests to skip based on the environment +skip_tests = [] +if LLVM.is_asserts() + # XXX: GCN's non-0 stack address space triggers LLVM assertions due to Julia bugs + push!(skip_tests, "gcn") +end +@dispose ctx=Context() begin + # XXX: some back-ends do not support opaque pointers + if !supports_typed_pointers(ctx) + push!(skip_tests, "metal") + end +end +if Sys.isapple() + # support for AMDGPU and NVTX on macOS has been removed from Julia's LLVM build + append!(skip_tests, ["ptx", "gcn"]) +end +if !(SPIRV_LLVM_Translator_unified_jll.is_available() && SPIRV_Tools_jll.is_available()) + # SPIRV needs it's tools to be available + push!(skip_tests, "spirv") +end +## finalize +skip_tests = filter(test->any(skip->occursin(skip,test), skip_tests), tests) +if !isempty(skip_tests) + @info "Skipping the following tests: $(join(skip_tests, ", "))" + filter!(!in(skip_tests), tests) +end +sort!(tests; by=(file)->stat("$(@__DIR__)/$file.jl").size, rev=true) +unique!(tests) - if ti.name in ["PTX", "GCN", "PTX precompile"] && Sys.isapple() - # support for AMDGPU and NVTX on macOS has been removed from Julia's LLVM build - return false +# add workers +const test_exeflags = Base.julia_cmd() +filter!(test_exeflags.exec) do c + return !(startswith(c, "--depwarn") || startswith(c, "--check-bounds")) +end +push!(test_exeflags.exec, "--check-bounds=yes") +push!(test_exeflags.exec, "--startup-file=no") +push!(test_exeflags.exec, "--depwarn=yes") +push!(test_exeflags.exec, "--project=$(Base.active_project())") +const test_exename = popfirst!(test_exeflags.exec) +function addworker(X; kwargs...) + withenv("JULIA_NUM_THREADS" => 1, "OPENBLAS_NUM_THREADS" => 1) do + procs = addprocs(X; exename=test_exename, exeflags=test_exeflags, kwargs...) + @everywhere procs include($(joinpath(@__DIR__, "setup.jl"))) + procs end +end +addworker(min(jobs, length(tests))) + +# pretty print information about gc and mem usage +testgroupheader = "Test" +workerheader = "(Worker)" +name_align = maximum([textwidth(testgroupheader) + textwidth(" ") + + textwidth(workerheader); map(x -> textwidth(x) + + 3 + ndigits(nworkers()), tests)]) +elapsed_align = textwidth("Time (s)") +gc_align = textwidth("GC (s)") +percent_align = textwidth("GC %") +alloc_align = textwidth("Alloc (MB)") +rss_align = textwidth("RSS (MB)") +printstyled(" "^(name_align + textwidth(testgroupheader) - 3), " | ") +printstyled(" | ---------------- CPU ---------------- |\n", color=:white) +printstyled(testgroupheader, color=:white) +printstyled(lpad(workerheader, name_align - textwidth(testgroupheader) + 1), " | ", color=:white) +printstyled("Time (s) | GC (s) | GC % | Alloc (MB) | RSS (MB) |\n", color=:white) +print_lock = stdout isa Base.LibuvStream ? stdout.lock : ReentrantLock() +if stderr isa Base.LibuvStream + stderr.lock = print_lock +end +function print_testworker_stats(test, wrkr, resp) + @nospecialize resp + lock(print_lock) + try + printstyled(test, color=:white) + printstyled(lpad("($wrkr)", name_align - textwidth(test) + 1, " "), " | ", color=:white) + time_str = @sprintf("%7.2f",resp[2]) + printstyled(lpad(time_str, elapsed_align, " "), " | ", color=:white) + + cpu_gc_str = @sprintf("%5.2f", resp[4]) + printstyled(lpad(cpu_gc_str, gc_align, " "), " | ", color=:white) + # since there may be quite a few digits in the percentage, + # the left-padding here is less to make sure everything fits + cpu_percent_str = @sprintf("%4.1f", 100 * resp[4] / resp[2]) + printstyled(lpad(cpu_percent_str, percent_align, " "), " | ", color=:white) + cpu_alloc_str = @sprintf("%5.2f", resp[3] / 2^20) + printstyled(lpad(cpu_alloc_str, alloc_align, " "), " | ", color=:white) + cpu_rss_str = @sprintf("%5.2f", resp[6] / 2^20) + printstyled(lpad(cpu_rss_str, rss_align, " "), " |\n", color=:white) + finally + unlock(print_lock) + end +end +global print_testworker_started = (name, wrkr)->begin +end +function print_testworker_errored(name, wrkr) + lock(print_lock) + try + printstyled(name, color=:red) + printstyled(lpad("($wrkr)", name_align - textwidth(name) + 1, " "), " |", + " "^elapsed_align, " failed at $(now())\n", color=:red) + finally + unlock(print_lock) + end +end - if ti.name in ["SPIRV"] && !(SPIRV_LLVM_Translator_unified_jll.is_available() && SPIRV_Tools_jll.is_available()) - # SPIRV needs it's tools to be available - return false +# run tasks +t0 = now() +results = [] +all_tasks = Task[] +try + # Monitor stdin and kill this task on ^C + # but don't do this on Windows, because it may deadlock in the kernel + t = current_task() + running_tests = Dict{String, DateTime}() + if !Sys.iswindows() && isa(stdin, Base.TTY) + stdin_monitor = @async begin + term = REPL.Terminals.TTYTerminal("xterm", stdin, stdout, stderr) + try + REPL.Terminals.raw!(term, true) + while true + c = read(term, Char) + if c == '\x3' + Base.throwto(t, InterruptException()) + break + elseif c == '?' + println("Currently running: ") + tests = sort(collect(running_tests), by=x->x[2]) + foreach(tests) do (test, date) + println(test, " (running for ", round(now()-date, Minute), ")") + end + end + end + catch e + isa(e, InterruptException) || rethrow() + finally + REPL.Terminals.raw!(term, false) + end + end end + @sync begin + function recycle_worker(p) + rmprocs(p, waitfor=30) + return nothing + end + + for p in workers() + @async begin + push!(all_tasks, current_task()) + while length(tests) > 0 + test = popfirst!(tests) - if ti.name in ["PTX precompile", "native precompile"] && VERSION < v"1.11-" - # precompile needs v1.11 - return false + # sometimes a worker failed, and we need to spawn a new one + if p === nothing + p = addworker(1)[1] + end + wrkr = p + + local resp + + # run the test + running_tests[test] = now() + try + resp = remotecall_fetch(runtests, wrkr, test_runners[test], test) + catch e + isa(e, InterruptException) && return + resp = Any[e] + end + delete!(running_tests, test) + push!(results, (test, resp)) + + # act on the results + if resp[1] isa Exception + print_testworker_errored(test, wrkr) + do_quickfail && Base.throwto(t, InterruptException()) + + # the worker encountered some failure, recycle it + # so future tests get a fresh environment + p = recycle_worker(p) + else + print_testworker_stats(test, wrkr, resp) + + cpu_rss = resp[6] + if haskey(ENV, "CI") && cpu_rss > 3*2^30 + # XXX: collecting garbage + # after each test, we are leaking CPU memory somewhere. + # this is a problem on CI, where2 we don't have much RAM. + # work around this by periodically recycling the worker. + p = recycle_worker(p) + end + end + end + + if p !== nothing + recycle_worker(p) + end + end + end end +catch e + isa(e, InterruptException) || rethrow() + # If the test suite was merely interrupted, still print the + # summary, which can be useful to diagnose what's going on + foreach(task -> begin + istaskstarted(task) || return + istaskdone(task) && return + try + schedule(task, InterruptException(); error=true) + catch ex + @error "InterruptException" exception=ex,catch_backtrace() + end + end, all_tasks) + for t in all_tasks + # NOTE: we can't just wait, but need to discard the exception, + # because the throwto for --quickfail also kills the worker. + try + wait(t) + catch e + showerror(stderr, e) + end + end +finally + if @isdefined stdin_monitor + schedule(stdin_monitor, InterruptException(); error=true) + end +end +t1 = now() +elapsed = canonicalize(Dates.CompoundPeriod(t1-t0)) +println("Testing finished in $elapsed") - true +# construct a testset to render the test results +o_ts = Test.DefaultTestSet("Overall") +Test.push_testset(o_ts) +completed_tests = Set{String}() +for (testname, (resp,)) in results + push!(completed_tests, testname) + if isa(resp, Test.DefaultTestSet) + Test.push_testset(resp) + Test.record(o_ts, resp) + Test.pop_testset() + elseif isa(resp, Tuple{Int,Int}) + fake = Test.DefaultTestSet(testname) + for i in 1:resp[1] + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) + end + for i in 1:resp[2] + Test.record(fake, Test.Broken(:test, nothing)) + end + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() + elseif isa(resp, RemoteException) && isa(resp.captured.ex, Test.TestSetException) + println("Worker $(resp.pid) failed running test $(testname):") + Base.showerror(stdout, resp.captured) + println() + fake = Test.DefaultTestSet(testname) + for i in 1:resp.captured.ex.pass + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, nothing)) + end + for i in 1:resp.captured.ex.broken + Test.record(fake, Test.Broken(:test, nothing)) + end + for t in resp.captured.ex.errors_and_fails + Test.record(fake, t) + end + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() + else + if !isa(resp, Exception) + resp = ErrorException(string("Unknown result type : ", typeof(resp))) + end + # If this test raised an exception that is not a remote testset exception, + # i.e. not a RemoteException capturing a TestSetException that means + # the test runner itself had some problem, so we may have hit a segfault, + # deserialization errors or something similar. Record this testset as Errored. + fake = Test.DefaultTestSet(testname) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1))) + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() + end end +for test in all_tests + (test in completed_tests) && continue + fake = Test.DefaultTestSet(test) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, + [("skipped", [])], LineNumberNode(1))) + Test.push_testset(fake) + Test.record(o_ts, fake) + Test.pop_testset() +end +println() +Test.print_test_results(o_ts, 1) +if !o_ts.anynonpass + println(" \033[32;1mSUCCESS\033[0m") +else + println(" \033[31;1mFAILURE\033[0m\n") + Test.print_test_errors(o_ts) + throw(Test.FallbackTestSetException("Test run finished with errors")) +end + diff --git a/test/setup.jl b/test/setup.jl new file mode 100644 index 00000000..fedde1b0 --- /dev/null +++ b/test/setup.jl @@ -0,0 +1,68 @@ +using Distributed, Test, GPUCompiler, LLVM + +using SPIRV_LLVM_Translator_unified_jll, SPIRV_Tools_jll + +# include all helpers +include(joinpath(@__DIR__, "helpers", "runtime.jl")) +for file in readdir(joinpath(@__DIR__, "helpers")) + if file != "runtime.jl" + include(joinpath(@__DIR__, "helpers", file)) + end +end + + +## entry point + +function runtests(f, name) + old_print_setting = Test.TESTSET_PRINT_ENABLE[] + Test.TESTSET_PRINT_ENABLE[] = false + + try + # generate a temporary module to execute the tests in + mod_name = Symbol("Test", rand(1:100), "Main_", replace(name, '/' => '_')) + mod = @eval(Main, module $mod_name end) + @eval(mod, using Test, Random, GPUCompiler) + + let id = myid() + wait(@spawnat 1 print_testworker_started(name, id)) + end + + ex = quote + GC.gc(true) + Random.seed!(1) + + @timed @testset $"$name" begin + $f() + end + end + data = Core.eval(mod, ex) + #data[1] is the testset + + # process results + cpu_rss = Sys.maxrss() + if VERSION >= v"1.11.0-DEV.1529" + tc = Test.get_test_counts(data[1]) + passes,fails,error,broken,c_passes,c_fails,c_errors,c_broken = + tc.passes, tc.fails, tc.errors, tc.broken, tc.cumulative_passes, + tc.cumulative_fails, tc.cumulative_errors, tc.cumulative_broken + else + passes,fails,errors,broken,c_passes,c_fails,c_errors,c_broken = + Test.get_test_counts(data[1]) + end + if data[1].anynonpass == false + data = ((passes+c_passes,broken+c_broken), + data[2], + data[3], + data[4], + data[5]) + end + res = vcat(collect(data), cpu_rss) + + GC.gc(true) + res + finally + Test.TESTSET_PRINT_ENABLE[] = old_print_setting + end +end + +nothing # File is loaded via a remotecall to "include". Ensure it returns "nothing". diff --git a/test/spirv_tests.jl b/test/spirv.jl similarity index 90% rename from test/spirv_tests.jl rename to test/spirv.jl index 9de9a2cc..9d8d36bf 100644 --- a/test/spirv_tests.jl +++ b/test/spirv.jl @@ -1,9 +1,3 @@ -@testitem "SPIRV" setup=[SPIRV, Helpers] begin - -using SPIRV_LLVM_Translator_unified_jll, SPIRV_Tools_jll - -############################################################################################ - @testset "IR" begin @testset "kernel functions" begin @@ -93,7 +87,3 @@ end end end - -############################################################################################ - -end diff --git a/test/util_tests.jl b/test/utils.jl similarity index 58% rename from test/util_tests.jl rename to test/utils.jl index 12a9e331..52db84be 100644 --- a/test/util_tests.jl +++ b/test/utils.jl @@ -1,5 +1,3 @@ -@testitem "util" begin - @testset "split_kwargs" begin kwargs = [:(a=1), :(b=2), :(c=3), :(d=4)] groups = GPUCompiler.split_kwargs(kwargs, [:a], [:b, :c]) @@ -47,57 +45,3 @@ end # problematic examples @test mangle(identity, String, Matrix{Float32}, Broadcast.Broadcasted{Broadcast.ArrayStyle{Matrix{Float32}}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}, typeof(Base.literal_pow), Tuple{Base.RefValue{typeof(sin)}, Broadcast.Extruded{Matrix{Float32}, Tuple{Bool, Bool}, Tuple{Int64, Int64}}}}) == "identity(String, Array, Broadcasted>, Tuple, OneTo>, literal_pow, Tuple, Extruded, Tuple, Tuple>>>)" end - -@testset "safe loggers" begin - using Logging: Logging - - struct YieldingLogger <: Logging.AbstractLogger - logger::Logging.AbstractLogger - YieldingLogger() = new(Logging.current_logger()) - end - - function Logging.handle_message(logger::YieldingLogger, args...) - yield() - return Logging.handle_message(logger.logger, args...) - end - - Logging.shouldlog(::YieldingLogger, ::Any...) = true - Logging.min_enabled_level(::YieldingLogger) = Logging.Debug - - GPUCompiler.@locked function f() - GPUCompiler.@safe_debug "safe_debug" - GPUCompiler.@safe_info "safe_info" - GPUCompiler.@safe_warn "safe_warn" - GPUCompiler.@safe_error "safe_error" - GPUCompiler.@safe_show "safe_show" - end - - @test begin - @sync begin - Threads.@spawn begin - sleep(0.1) - @debug "debug" - sleep(0.1) - @info "info" - sleep(0.1) - @warn "warn" - sleep(0.1) - @error "error" - sleep(0.1) - @show "show" - sleep(0.1) - end - pipe = Pipe() - Base.link_pipe!(pipe; reader_supports_async=true, writer_supports_async=true) - Threads.@spawn print(stdout, read(pipe, String)) - Threads.@spawn Logging.with_logger(YieldingLogger()) do - sleep(0.1) - redirect_stdout(f, pipe) - close(pipe) - end - end - true - end -end - -end From cbe6d4941cb1c6933135dd961f57c73f214e46a7 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jan 2025 16:59:26 +0100 Subject: [PATCH 2/5] Split off and skip precompilation tests on <1.11 --- test/native.jl | 79 --------------------------------------- test/native/precompile.jl | 74 ++++++++++++++++++++++++++++++++++++ test/ptx.jl | 60 ----------------------------- test/ptx/precompile.jl | 53 ++++++++++++++++++++++++++ test/runtests.jl | 3 ++ 5 files changed, 130 insertions(+), 139 deletions(-) create mode 100644 test/native/precompile.jl create mode 100644 test/ptx/precompile.jl diff --git a/test/native.jl b/test/native.jl index d88ec86a..4772c49e 100644 --- a/test/native.jl +++ b/test/native.jl @@ -557,82 +557,3 @@ end ["jl_invoke", "apply_iterate", "inttoptr", "apply_type"]) end - -############################################################################################ - -@testset "precompile" begin - -precompile_test_harness("Inference caching") do load_path - # Write out the Native test setup as a micro package - create_standalone(load_path, "TestCompiler", "native.jl") - - write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching - import TestCompiler - using PrecompileTools - - function kernel(A, x) - A[1] = x - return - end - - let - job, _ = TestCompiler.Native.create_job(kernel, (Vector{Int}, Int)) - precompile(job) - end - - # identity is foreign - @setup_workload begin - job, _ = TestCompiler.Native.create_job(identity, (Int,)) - @compile_workload begin - precompile(job) - end - end - end) |> string) - - Base.compilecache(Base.PkgId("InferenceCaching")) - @eval let - import TestCompiler - - # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) - - token = let - job, _ = TestCompiler.Native.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) - end - @test !check_presence(identity_mi, token) - - using InferenceCaching - - # Check that kernel survived - kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{Vector{Int}, Int}) - @test check_presence(kernel_mi, token) - - # check that identity survived - @test check_presence(identity_mi, token) broken=VERSION>=v"1.12.0-DEV.1268" - - GPUCompiler.clear_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == false - - GPUCompiler.enable_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == true - - job, _ = TestCompiler.Native.create_job(InferenceCaching.kernel, (Vector{Int}, Int)) - @assert job.source == kernel_mi - ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) - @assert ci !== nothing - @assert ci.inferred !== nothing - path = GPUCompiler.cache_file(ci, job.config) - @test path !== nothing - @test !ispath(path) - TestCompiler.Native.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int)) - @test ispath(path) - GPUCompiler.clear_disk_cache!() - @test !ispath(path) - - GPUCompiler.enable_disk_cache!(false) - @test GPUCompiler.disk_cache_enabled() == false - end -end - -end diff --git a/test/native/precompile.jl b/test/native/precompile.jl new file mode 100644 index 00000000..e9983a7e --- /dev/null +++ b/test/native/precompile.jl @@ -0,0 +1,74 @@ + + +precompile_test_harness("Inference caching") do load_path + # Write out the Native test setup as a micro package + create_standalone(load_path, "TestCompiler", "native.jl") + + write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching + import TestCompiler + using PrecompileTools + + function kernel(A, x) + A[1] = x + return + end + + let + job, _ = TestCompiler.Native.create_job(kernel, (Vector{Int}, Int)) + precompile(job) + end + + # identity is foreign + @setup_workload begin + job, _ = TestCompiler.Native.create_job(identity, (Int,)) + @compile_workload begin + precompile(job) + end + end + end) |> string) + + Base.compilecache(Base.PkgId("InferenceCaching")) + @eval let + import TestCompiler + + # Check that no cached entry is present + identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + + token = let + job, _ = TestCompiler.Native.create_job(identity, (Int,)) + GPUCompiler.ci_cache_token(job) + end + @test !check_presence(identity_mi, token) + + using InferenceCaching + + # Check that kernel survived + kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{Vector{Int}, Int}) + @test check_presence(kernel_mi, token) + + # check that identity survived + @test check_presence(identity_mi, token) broken=VERSION>=v"1.12.0-DEV.1268" + + GPUCompiler.clear_disk_cache!() + @test GPUCompiler.disk_cache_enabled() == false + + GPUCompiler.enable_disk_cache!() + @test GPUCompiler.disk_cache_enabled() == true + + job, _ = TestCompiler.Native.create_job(InferenceCaching.kernel, (Vector{Int}, Int)) + @assert job.source == kernel_mi + ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) + @assert ci !== nothing + @assert ci.inferred !== nothing + path = GPUCompiler.cache_file(ci, job.config) + @test path !== nothing + @test !ispath(path) + TestCompiler.Native.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int)) + @test ispath(path) + GPUCompiler.clear_disk_cache!() + @test !ispath(path) + + GPUCompiler.enable_disk_cache!(false) + @test GPUCompiler.disk_cache_enabled() == false + end +end diff --git a/test/ptx.jl b/test/ptx.jl index 46f0e673..32efc006 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -343,63 +343,3 @@ end end end - -############################################################################################ - -@testset "precompile" begin - -precompile_test_harness("Inference caching") do load_path - # Write out the PTX test helpers as a micro package - create_standalone(load_path, "TestCompiler", "ptx.jl") - - write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching - import TestCompiler - using PrecompileTools - - function kernel() - return - end - - let - job, _ = TestCompiler.PTX.create_job(kernel, ()) - precompile(job) - end - - # identity is foreign - @setup_workload begin - job, _ = TestCompiler.PTX.create_job(identity, (Int,)) - @compile_workload begin - precompile(job) - end - end - end) |> string) - - Base.compilecache(Base.PkgId("InferenceCaching")) - @eval let - import TestCompiler - - # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) - - token = let - job, _ = PTX.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) - end - ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing - while ci !== nothing - @test ci.owner !== token - ci = isdefined(ci, :next) ? ci.next : nothing - end - - using InferenceCaching - - # Check that kernel survived - kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{}) - @test check_presence(kernel_mi, token) - - # check that identity survived - @test check_presence(identity_mi, token) broken=VERSION>=v"1.12.0-DEV.1268" - end -end - -end diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl new file mode 100644 index 00000000..86441c2c --- /dev/null +++ b/test/ptx/precompile.jl @@ -0,0 +1,53 @@ +precompile_test_harness("Inference caching") do load_path + # Write out the PTX test helpers as a micro package + create_standalone(load_path, "TestCompiler", "ptx.jl") + + write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching + import TestCompiler + using PrecompileTools + + function kernel() + return + end + + let + job, _ = TestCompiler.PTX.create_job(kernel, ()) + precompile(job) + end + + # identity is foreign + @setup_workload begin + job, _ = TestCompiler.PTX.create_job(identity, (Int,)) + @compile_workload begin + precompile(job) + end + end + end) |> string) + + Base.compilecache(Base.PkgId("InferenceCaching")) + @eval let + import TestCompiler + + # Check that no cached entry is present + identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + + token = let + job, _ = PTX.create_job(identity, (Int,)) + GPUCompiler.ci_cache_token(job) + end + ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing + while ci !== nothing + @test ci.owner !== token + ci = isdefined(ci, :next) ? ci.next : nothing + end + + using InferenceCaching + + # Check that kernel survived + kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{}) + @test check_presence(kernel_mi, token) + + # check that identity survived + @test check_presence(identity_mi, token) broken=VERSION>=v"1.12.0-DEV.1268" + end +end diff --git a/test/runtests.jl b/test/runtests.jl index cfb96f9a..5a155905 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -124,6 +124,9 @@ if !(SPIRV_LLVM_Translator_unified_jll.is_available() && SPIRV_Tools_jll.is_avai # SPIRV needs it's tools to be available push!(skip_tests, "spirv") end +if VERSION < v"1.11" + append!(skip_tests, ["ptx/precompile", "native/precompile"]) +end ## finalize skip_tests = filter(test->any(skip->occursin(skip,test), skip_tests), tests) if !isempty(skip_tests) From 65cb9e905b6f3c30d2f7d383a0083e5cf6a9a60b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jan 2025 16:59:47 +0100 Subject: [PATCH 3/5] Ignore test preferences. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c28d320e..22936030 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ test/Manifest.toml +test/LocalPreferences.toml Manifest.toml Manifest-*.toml From 339330c313011a3dba455ba3137caa463100ec17 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jan 2025 17:06:05 +0100 Subject: [PATCH 4/5] Fix test skipping. --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 5a155905..f26d9adb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -389,7 +389,7 @@ for (testname, (resp,)) in results Test.pop_testset() end end -for test in all_tests +for test in tests (test in completed_tests) && continue fake = Test.DefaultTestSet(test) Test.record(fake, Test.Error(:test_interrupted, test, nothing, From 37d82984c395088167ea5b54d9ed14cede62d10a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 13 Jan 2025 19:44:28 +0100 Subject: [PATCH 5/5] Fix precompilation harness. --- test/helpers/precompile.jl | 23 +++++++---------------- test/native/precompile.jl | 25 +++++++++++++------------ test/ptx/precompile.jl | 19 ++++++++++--------- 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/test/helpers/precompile.jl b/test/helpers/precompile.jl index daa1b57b..74ab2b37 100644 --- a/test/helpers/precompile.jl +++ b/test/helpers/precompile.jl @@ -4,27 +4,18 @@ function precompile_test_harness(@nospecialize(f), testset::String) end end function precompile_test_harness(@nospecialize(f), separate::Bool) - load_path = mktempdir() - load_cache_path = separate ? mktempdir() : load_path + # XXX: clean-up may fail on Windows, because opened files are not deletable. + # fix this by running the harness in a separate process, such that the + # compilation cache files are not opened? + load_path = mktempdir(cleanup=true) + load_cache_path = separate ? mktempdir(cleanup=true) : load_path try pushfirst!(LOAD_PATH, load_path) pushfirst!(DEPOT_PATH, load_cache_path) f(load_path) finally - try - rm(load_path, force=true, recursive=true) - catch err - @show err - end - if separate - try - rm(load_cache_path, force=true, recursive=true) - catch err - @show err - end - end - filter!((≠)(load_path), LOAD_PATH) - separate && filter!((≠)(load_cache_path), DEPOT_PATH) + popfirst!(DEPOT_PATH) + popfirst!(LOAD_PATH) end nothing end diff --git a/test/native/precompile.jl b/test/native/precompile.jl index e9983a7e..c2648701 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -2,10 +2,11 @@ precompile_test_harness("Inference caching") do load_path # Write out the Native test setup as a micro package - create_standalone(load_path, "TestCompiler", "native.jl") + create_standalone(load_path, "NativeCompiler", "native.jl") - write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching - import TestCompiler + write(joinpath(load_path, "NativeBackend.jl"), :( + module NativeBackend + import NativeCompiler using PrecompileTools function kernel(A, x) @@ -14,36 +15,36 @@ precompile_test_harness("Inference caching") do load_path end let - job, _ = TestCompiler.Native.create_job(kernel, (Vector{Int}, Int)) + job, _ = NativeCompiler.Native.create_job(kernel, (Vector{Int}, Int)) precompile(job) end # identity is foreign @setup_workload begin - job, _ = TestCompiler.Native.create_job(identity, (Int,)) + job, _ = NativeCompiler.Native.create_job(identity, (Int,)) @compile_workload begin precompile(job) end end end) |> string) - Base.compilecache(Base.PkgId("InferenceCaching")) + Base.compilecache(Base.PkgId("NativeBackend")) @eval let - import TestCompiler + import NativeCompiler # Check that no cached entry is present identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) token = let - job, _ = TestCompiler.Native.create_job(identity, (Int,)) + job, _ = NativeCompiler.Native.create_job(identity, (Int,)) GPUCompiler.ci_cache_token(job) end @test !check_presence(identity_mi, token) - using InferenceCaching + using NativeBackend # Check that kernel survived - kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{Vector{Int}, Int}) + kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel), Tuple{Vector{Int}, Int}) @test check_presence(kernel_mi, token) # check that identity survived @@ -55,7 +56,7 @@ precompile_test_harness("Inference caching") do load_path GPUCompiler.enable_disk_cache!() @test GPUCompiler.disk_cache_enabled() == true - job, _ = TestCompiler.Native.create_job(InferenceCaching.kernel, (Vector{Int}, Int)) + job, _ = NativeCompiler.Native.create_job(NativeBackend.kernel, (Vector{Int}, Int)) @assert job.source == kernel_mi ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) @assert ci !== nothing @@ -63,7 +64,7 @@ precompile_test_harness("Inference caching") do load_path path = GPUCompiler.cache_file(ci, job.config) @test path !== nothing @test !ispath(path) - TestCompiler.Native.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int)) + NativeCompiler.Native.cached_execution(NativeBackend.kernel, (Vector{Int}, Int)) @test ispath(path) GPUCompiler.clear_disk_cache!() @test !ispath(path) diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl index 86441c2c..b5f980c9 100644 --- a/test/ptx/precompile.jl +++ b/test/ptx/precompile.jl @@ -1,9 +1,10 @@ precompile_test_harness("Inference caching") do load_path # Write out the PTX test helpers as a micro package - create_standalone(load_path, "TestCompiler", "ptx.jl") + create_standalone(load_path, "PTXCompiler", "ptx.jl") - write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching - import TestCompiler + write(joinpath(load_path, "PTXBackend.jl"), :( + module PTXBackend + import PTXCompiler using PrecompileTools function kernel() @@ -11,22 +12,22 @@ precompile_test_harness("Inference caching") do load_path end let - job, _ = TestCompiler.PTX.create_job(kernel, ()) + job, _ = PTXCompiler.PTX.create_job(kernel, ()) precompile(job) end # identity is foreign @setup_workload begin - job, _ = TestCompiler.PTX.create_job(identity, (Int,)) + job, _ = PTXCompiler.PTX.create_job(identity, (Int,)) @compile_workload begin precompile(job) end end end) |> string) - Base.compilecache(Base.PkgId("InferenceCaching")) + Base.compilecache(Base.PkgId("PTXBackend")) @eval let - import TestCompiler + import PTXCompiler # Check that no cached entry is present identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) @@ -41,10 +42,10 @@ precompile_test_harness("Inference caching") do load_path ci = isdefined(ci, :next) ? ci.next : nothing end - using InferenceCaching + using PTXBackend # Check that kernel survived - kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{}) + kernel_mi = GPUCompiler.methodinstance(typeof(PTXBackend.kernel), Tuple{}) @test check_presence(kernel_mi, token) # check that identity survived