Revert back to our hand-rolled test runner. (#658)

.gitignore

@@ -1,3 +1,4 @@
 test/Manifest.toml
+test/LocalPreferences.toml
 Manifest.toml
 Manifest-*.toml

test/Project.toml

@@ -1,11 +1,14 @@
 [deps]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
-ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SPIRV_LLVM_Translator_unified_jll = "85f0d8ed-5b39-5caa-b1ae-7472de402361"
 SPIRV_Tools_jll = "6ac6d60f-d740-5983-97d7-a4482c0689f4"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
@@ -14,3 +17,6 @@ demumble_jll = "1e29f10c-031c-5a83-9565-69cddfc27673"
 
 [compat]
 Aqua = "0.8"
+
+[extras]
+GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"

test/aqua_tests.jl → test/aqua.jl

@@ -1,7 +1,3 @@
-@testitem "Aqua" begin
-
 using Aqua
 
 Aqua.test_all(GPUCompiler)
-
-end

test/bpf_tests.jl → test/bpf.jl

@@ -1,7 +1,3 @@
-@testitem "BPF" setup=[BPF, Helpers] begin
-
-############################################################################################
-
 @testset "No-op" begin
     kernel() = 0
 
@@ -40,5 +36,3 @@ end
         @test_throws GPUCompiler.InvalidIRError BPF.code_execution(kernel, (Int,))
     end
 end
-
-end

test/examples_tests.jl → test/examples.jl

@@ -1,5 +1,3 @@
-@testitem "examples" begin
-
 function find_sources(path::String, sources=String[])
     if isdir(path)
         for entry in readdir(path)
@@ -23,5 +21,3 @@ cd(dir) do
         @test success(pipeline(`$cmd $example`, stderr=stderr))
     end
 end
-
-end

test/gcn_tests.jl → test/gcn.jl

@@ -1,11 +1,3 @@
-@testitem "GCN" setup=[GCN, Helpers] begin
-
-@inline sink_gcn(i) = sink(i, Val(5))
-
-@test GCNCompilerTarget(dev_isa="gfx900") == GCNCompilerTarget("gfx900")
-
-############################################################################################
-
 @testset "IR" begin
 
 @testset "kernel calling convention" begin
@@ -222,7 +214,3 @@ end
 end
 
 end
-
-############################################################################################
-
-end

test/bpf_testsetup.jl → test/helpers/bpf.jl

@@ -1,11 +1,8 @@
-@testsetup module BPF
+module BPF
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a native test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/gcn_testsetup.jl → test/helpers/gcn.jl

@@ -1,11 +1,8 @@
-@testsetup module GCN
+module GCN
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a GCN-based test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/metal_testsetup.jl → test/helpers/metal.jl

@@ -1,11 +1,8 @@
-@testsetup module Metal
+module Metal
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a Metal test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/native_testsetup.jl → test/helpers/native.jl

@@ -1,11 +1,7 @@
-@testsetup module Native
+module Native
 
-using GPUCompiler
-
-
-# create a native test compiler, and generate reflection methods for it
-
-include("runtime.jl")
+using ..GPUCompiler
+import ..TestRuntime
 
 # local method table for device functions
 Base.Experimental.@MethodTable(test_method_table)

test/precompile_testsetup.jl → test/helpers/precompile.jl

@@ -1,37 +1,21 @@
-@testsetup module Precompile
-
-using Test
-using ReTestItems
-
-export precompile_test_harness, check_presence, create_standalone
-
 function precompile_test_harness(@nospecialize(f), testset::String)
     @testset "$testset" begin
         precompile_test_harness(f, true)
     end
 end
 function precompile_test_harness(@nospecialize(f), separate::Bool)
-    load_path = mktempdir()
-    load_cache_path = separate ? mktempdir() : load_path
+    # XXX: clean-up may fail on Windows, because opened files are not deletable.
+    #      fix this by running the harness in a separate process, such that the
+    #      compilation cache files are not opened?
+    load_path = mktempdir(cleanup=true)
+    load_cache_path = separate ? mktempdir(cleanup=true) : load_path
     try
         pushfirst!(LOAD_PATH, load_path)
         pushfirst!(DEPOT_PATH, load_cache_path)
         f(load_path)
     finally
-        try
-            rm(load_path, force=true, recursive=true)
-        catch err
-            @show err
-        end
-        if separate
-            try
-                rm(load_cache_path, force=true, recursive=true)
-            catch err
-                @show err
-            end
-        end
-        filter!((≠)(load_path), LOAD_PATH)
-        separate && filter!((≠)(load_cache_path), DEPOT_PATH)
+        popfirst!(DEPOT_PATH)
+        popfirst!(LOAD_PATH)
     end
     nothing
 end
@@ -50,19 +34,18 @@ function check_presence(mi, token)
 end
 
 function create_standalone(load_path, name::String, file)
-    cp(joinpath(@__DIR__, "runtime.jl"), joinpath(load_path, "runtime.jl"), force=true)
+    code = :(
+        module $(Symbol(name))
 
-    TS = include(file)
-    code = TS.code
-    if code.head == :begin
-        code.head = :block
-    end
-    @assert code.head == :block
-    code = Expr(:module, true, Symbol(name), code)
+        using GPUCompiler
+
+        include($(joinpath(@__DIR__, "runtime.jl")))
+        include($(joinpath(@__DIR__, file)))
+
+        end
+    )
 
     # Write out the test setup as a micro package
     write(joinpath(load_path, "$name.jl"), string(code))
     Base.compilecache(Base.PkgId(name))
 end
-
-end # testsetup

test/ptx_testsetup.jl → test/helpers/ptx.jl

@@ -1,11 +1,8 @@
-@testsetup module PTX
+module PTX
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a PTX-based test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 
 PTXCompilerJob = CompilerJob{PTXCompilerTarget,CompilerParams}

test/spirv_testsetup.jl → test/helpers/spirv.jl

@@ -1,11 +1,8 @@
-@testsetup module SPIRV
+module SPIRV
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a SPIRV-based test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/helper_testsetup.jl → test/helpers/test.jl

@@ -1,9 +1,3 @@
-@testsetup module Helpers
-
-using Test
-
-export @test_throws_message, sink
-
 # @test_throw, with additional testing for the exception message
 macro test_throws_message(f, typ, ex...)
     quote
@@ -41,5 +35,3 @@ end
                      ret i64 %value"""
     return :(Base.llvmcall($llvmcall_str, T, Tuple{T}, i))
 end
-
-end

test/metal_tests.jl → test/metal.jl

@@ -1,9 +1,3 @@
-@testitem "Metal" setup=[Metal, Helpers] begin
-
-using LLVM
-
-############################################################################################
-
 @testset "IR" begin
 
 @testset "kernel functions" begin
@@ -110,7 +104,6 @@ end
         return
     end
 
-
     ir = sprint(io->Metal.code_llvm(io, kernel1, Tuple{Core.LLVMPtr{Float32,1}}; validate=true))
     @test occursin("@metal_os_log", ir)
 
@@ -145,5 +138,3 @@ end
 end
 
 end
-
-end

test/native_tests.jl → test/native.jl

@@ -1,9 +1,3 @@
-@testitem "native" setup=[Native, Helpers] begin
-
-using Test
-
-############################################################################################
-
 @testset "reflection" begin
     job, _ = Native.create_job(identity, (Int,))
 
@@ -233,7 +227,7 @@ end
     @test "We did not crash!" != ""
 end
 
-@testset "CUDAnative.jl#278" begin
+@testset "CUDAjl#278" begin
     # codegen idempotency
     # NOTE: this isn't fixed, but surfaces here due to bad inference of checked_sub
     # NOTE: with the fix to print_to_string this doesn't error anymore,
@@ -563,82 +557,3 @@ end
                ["jl_invoke", "apply_iterate",
                 "inttoptr", "apply_type"])
 end
-end # testitem
-
-@testitem "native precompile" setup=[Precompile,] begin
-
-using Test
-precompile_test_harness("Inference caching") do load_path
-    # Write out the Native test setup as a micro package
-    create_standalone(load_path, "NativeCompiler", "native_testsetup.jl")
-
-    write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching
-        import NativeCompiler
-        import GPUCompiler
-        using PrecompileTools
-
-        function kernel(A, x)
-            A[1] = x
-            return
-        end
-
-        let
-            job, _ = NativeCompiler.create_job(kernel, (Vector{Int}, Int))
-            precompile(job)
-        end
-
-        # identity is foreign
-        @setup_workload begin
-            job, _ = NativeCompiler.create_job(identity, (Int,))
-            @compile_workload begin
-                precompile(job)
-            end
-        end
-    end) |> string)
-
-    Base.compilecache(Base.PkgId("InferenceCaching"))
-    @eval let
-        import NativeCompiler
-
-        # Check that no cached entry is present
-        identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int})
-
-        token = let
-            job, _ = NativeCompiler.create_job(identity, (Int,))
-            GPUCompiler.ci_cache_token(job)
-        end
-        @test !check_presence(identity_mi, token)
-
-        using InferenceCaching
-
-        # Check that kernel survived
-        kernel_mi = GPUCompiler.methodinstance(typeof(InferenceCaching.kernel), Tuple{Vector{Int}, Int})
-        @test check_presence(kernel_mi, token)
-
-        # check that identity survived
-        @test check_presence(identity_mi, token) broken=VERSION>=v"1.12.0-DEV.1268"
-
-        GPUCompiler.clear_disk_cache!()
-        @test GPUCompiler.disk_cache_enabled() == false
-
-        GPUCompiler.enable_disk_cache!()
-        @test GPUCompiler.disk_cache_enabled() == true
-
-        job, _ = NativeCompiler.create_job(InferenceCaching.kernel, (Vector{Int}, Int))
-        @assert job.source == kernel_mi
-        ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world)
-        @assert ci !== nothing
-        @assert ci.inferred !== nothing
-        path = GPUCompiler.cache_file(ci, job.config)
-        @test path !== nothing
-        @test !ispath(path)
-        NativeCompiler.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int))
-        @test ispath(path)
-        GPUCompiler.clear_disk_cache!()
-        @test !ispath(path)
-    end
-end
-
-############################################################################################
-
-end