Merge pull request #353 from guilhermeleobas/numba54

Tests for Numba 0.54.0 RC3

.conda/environment.yml

@@ -7,6 +7,7 @@
 name: rbc
 channels:
   - conda-forge
+  - numba
   - defaults
 dependencies:
   - python>=3.7

.github/workflows/changelog.yml

@@ -1,11 +1,11 @@
-name: 'CI'
+name: 'ChangeLog'
 on:
   push:
     tags:
       - 'v*'
 
 jobs:
-  release:
+  changelog:
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:

.github/workflows/rbc_test.yml

@@ -9,6 +9,12 @@ on:
   pull_request:
     branches:
       - master
+  # workflow_dispatch:
+  #   inputs:
+  #     debug_enabled:
+  #       description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+  #       required: false
+  #       default: false
 
 jobs:
   lint:
@@ -32,16 +38,13 @@ jobs:
     name: ${{ matrix.os }} - Python v${{ matrix.python-version }} - Numba v${{ matrix.numba-version }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: [3.9, 3.8, 3.7]
-        numba-version: [0.53, 0.52]
-        exclude:
-          - python-version: 3.9
-            numba-version: 0.52
+        numba-version: [0.54, 0.53]
 
-    needs: lint
+    needs: [lint, omniscidb]
 
     steps:
       - name: Checkout code
@@ -94,18 +97,27 @@ jobs:
           mamba run -n rbc pytest -sv -r A rbc/ -x
 
   omniscidb:
-    name: OmnisciDB ${{ matrix.omniscidb-version }} - ${{ matrix.os }} - Python v${{ matrix.python-version }} [${{ matrix.omniscidb-from }}]
+    name: OmnisciDB ${{ matrix.omniscidb-version }} - ${{ matrix.os }} - Numba v${{matrix.numba-version}} - Python v${{ matrix.python-version }} [${{ matrix.omniscidb-from }}]
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
         python-version: [3.9, 3.8, 3.7]
+        numba-version: [0.54, 0.53]
         omniscidb-version: [5.5, 5.4, 5.2]
         omniscidb-from: [conda]
         include:
           - os: ubuntu-latest
             python-version: 3.8
+            numba-version: 0.53
+            omniscidb-version: dev
+            docker-image: omnisci/core-os-cpu-dev:latest
+            omniscidb-from: docker
+          - os: ubuntu-latest
+            python-version: 3.8
+            numba-version: 0.54
             omniscidb-version: dev
             docker-image: omnisci/core-os-cpu-dev:latest
             omniscidb-from: docker
@@ -178,6 +190,7 @@ jobs:
         shell: bash -l {0}
         run: |
           cat .conda/environment.yml > rbc_test.yaml
+          echo "  - numba=${{ matrix.numba-version }}" >> rbc_test.yaml
           echo "  - python=${{ matrix.python-version }}" >> rbc_test.yaml
           mamba env create  --file=rbc_test.yaml -n rbc
 
@@ -196,24 +209,30 @@ jobs:
         run: |
           mamba run -n rbc python setup.py develop
 
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      #   if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
       - name: Run rbc tests [docker]
         shell: bash -l {0}
         if: matrix.os == 'ubuntu-latest' && matrix.omniscidb-from == 'docker'
         env:
           OMNISCIDB_DEV_LABEL: ${{ 'docker-dev' }}
           EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }}
           EXPECTED_OMNISCIDB_VERSION: ${{ matrix.omniscidb-version }}
+          EXPECTED_NUMBA_VERSION: ${{ matrix.numba-version }}
         run: |
-          mamba run -n rbc pytest -sv -r A rbc/ -x
+          mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k omnisci
 
       - name: Run rbc tests
         shell: bash -l {0}
         if: ${{ ! ( matrix.os == 'ubuntu-latest' && matrix.omniscidb-from == 'docker' ) }}
         env:
           EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }}
           EXPECTED_OMNISCIDB_VERSION: ${{ matrix.omniscidb-version }}
+          EXPECTED_NUMBA_VERSION: ${{ matrix.numba-version }}
         run: |
-          mamba run -n rbc pytest -sv -r A rbc/ -x
+          mamba run -n rbc pytest -sv -r A rbc/tests/ -x -k omnisci
 
       - name: Show Omniscidb docker logs on failure [docker]
         shell: bash -l {0}

rbc/externals/stdio.py

@@ -2,6 +2,7 @@
 """
 
 import functools
+from rbc import irutils
 from llvmlite import ir
 from rbc.targetinfo import TargetInfo
 from numba.core import cgutils, extending
@@ -13,7 +14,7 @@
 def cg_fflush(builder):
     int8_t = ir.IntType(8)
     fflush_fnty = ir.FunctionType(int32_t, [int8_t.as_pointer()])
-    fflush_fn = builder.module.get_or_insert_function(fflush_fnty, name="fflush")
+    fflush_fn = irutils.get_or_insert_function(builder.module, fflush_fnty, name="fflush")
 
     builder.call(fflush_fn, [int8_t.as_pointer()(None)])
 

rbc/irtools.py

@@ -10,6 +10,7 @@
 import llvmlite.binding as llvm
 from .targetinfo import TargetInfo
 from .errors import UnsupportedError
+from .utils import get_version
 from . import libfuncs, structure_type
 from rbc import externals
 from rbc.omnisci_backend import mathimpl
@@ -230,10 +231,10 @@ def make_wrapper(fname, atypes, rtype, cres, target: TargetInfo, verbose=False):
     if return_as_first_argument:
         wrapty = ir.FunctionType(ir.VoidType(),
                                  [ll_return_type] + ll_argtypes)
-        wrapfn = module.add_function(wrapty, fname)
+        wrapfn = ir.Function(module, wrapty, fname)
         builder = ir.IRBuilder(wrapfn.append_basic_block('entry'))
         fnty = context.call_conv.get_function_type(rtype, atypes)
-        fn = builder.module.add_function(fnty, cres.fndesc.llvm_func_name)
+        fn = ir.Function(builder.module, fnty, cres.fndesc.llvm_func_name)
         status, out = context.call_conv.call_function(
             builder, fn, rtype, atypes, wrapfn.args[1:])
         with cgutils.if_unlikely(builder, status.is_error):
@@ -247,10 +248,10 @@ def make_wrapper(fname, atypes, rtype, cres, target: TargetInfo, verbose=False):
         builder.ret_void()
     else:
         wrapty = ir.FunctionType(ll_return_type, ll_argtypes)
-        wrapfn = module.add_function(wrapty, fname)
+        wrapfn = ir.Function(module, wrapty, fname)
         builder = ir.IRBuilder(wrapfn.append_basic_block('entry'))
         fnty = context.call_conv.get_function_type(rtype, atypes)
-        fn = builder.module.add_function(fnty, cres.fndesc.llvm_func_name)
+        fn = ir.Function(builder.module, fnty, cres.fndesc.llvm_func_name)
         status, out = context.call_conv.call_function(
             builder, fn, rtype, atypes, wrapfn.args)
         if verbose and target.is_cpu:
@@ -275,9 +276,14 @@ def compile_instance(func, sig,
     succesful.
     """
     flags = compiler.Flags()
-    flags.set('no_compile')
-    flags.set('no_cpython_wrapper')
-    flags.set('no_cfunc_wrapper')
+    if get_version('numba') >= (0, 54):
+        flags.no_compile = True
+        flags.no_cpython_wrapper = True
+        flags.no_cfunc_wrapper = True
+    else:
+        flags.set('no_compile')
+        flags.set('no_cpython_wrapper')
+        flags.set('no_cfunc_wrapper')
 
     fname = func.__name__ + sig.mangling()
     args, return_type = sigutils.normalize_signature(
@@ -467,6 +473,10 @@ def compile_IR(ir):
         ir, re.M).group('triple')
 
     # Create execution engine
+    llvm.initialize()
+    llvm.initialize_all_targets()
+    llvm.initialize_all_asmprinters()
+
     target = llvm.Target.from_triple(triple)
     target_machine = target.create_target_machine()
     backing_mod = llvm.parse_assembly("")

rbc/irutils.py

@@ -0,0 +1,31 @@
+from rbc.utils import get_version
+from numba.core import cgutils
+from llvmlite import ir
+
+
+int8_t = ir.IntType(8)
+int32_t = ir.IntType(32)
+int64_t = ir.IntType(64)
+void_t = ir.VoidType()
+fp32 = ir.FloatType()
+fp64 = ir.DoubleType()
+
+
+def get_or_insert_function(module, fnty, name=None):
+    if get_version('numba') >= (0, 54):
+        fn = cgutils.get_or_insert_function(
+            module, fnty, name=name)
+    else:
+        fn = module.get_or_insert_function(
+            fnty, name=name)
+    return fn
+
+
+def get_member_value(builder, data, idx):
+    if get_version('numba') >= (0, 54):
+        assert data.opname == 'insertvalue', data.opname
+        return builder.extract_value(data, [idx])
+    else:
+        assert data.opname == 'load', data.opname
+        struct = data.operands[0]
+        return builder.load(builder.gep(struct, [int32_t(0), int32_t(idx)]))

rbc/libfuncs.py

@@ -36,7 +36,14 @@ def check(self, fname):
         """
         Return True if library contains a function with given name.
         """
-        return fname in self._function_names
+        if fname in self._function_names:
+            return True
+
+        for func in self._function_names:
+            if func.endswith('.*') and fname.startswith(func[:-2]):
+                return True
+
+        return False
 
 
 class OmnisciDB(Library):
@@ -196,6 +203,15 @@ def check(self, fname):
     experimental.constrained.floor experimental.constrained.round
     experimental.constrained.roundeven experimental.constrained.lround
     experimental.constrained.llround experimental.constrained.trunc
+    experimental.gc.statepoint experimental.gc.result experimental.gc.relocate
+    experimental.gc.get.pointer.base experimental.gc.get.pointer.offset
+    experimental.vector.reduce.add.* experimental.vector.reduce.fadd.*
+    experimental.vector.reduce.mul.* experimental.vector.reduce.fmul.*
+    experimental.vector.reduce.and.* experimental.vector.reduce.or.*
+    experimental.vector.reduce.xor.* experimental.vector.reduce.smax.*
+    experimental.vector.reduce.smin.* experimental.vector.reduce.umax.*
+    experimental.vector.reduce.umin.* experimental.vector.reduce.fmax.*
+    experimental.vector.reduce.fmin.*
     flt.rounds var.annotation ptr.annotation annotation
     codeview.annotation trap debugtrap stackprotector stackguard
     objectsize expect expect.with.probability assume ssa_copy
@@ -211,7 +227,8 @@ def check(self, fname):
     objc.retainAutoreleaseReturnValue
     objc.retainAutoreleasedReturnValue objc.retainBlock
     objc.storeStrong objc.storeWeak preserve.array.access.index
-    preserve.union.access.index preserve.struct.access.index '''.strip().split())
+    preserve.union.access.index preserve.struct.access.index
+    masked.store.* '''.strip().split())
 
 
 class NVVMIntrinsics(Library):

rbc/omnisci_backend/omnisci_buffer.py

@@ -28,7 +28,7 @@
 from .omnisci_metatype import OmnisciMetaType
 from llvmlite import ir
 import numpy as np
-from rbc import typesystem
+from rbc import typesystem, irutils
 from rbc.targetinfo import TargetInfo
 from llvmlite import ir as llvm_ir
 from numba.core import datamodel, cgutils, extending, types
@@ -164,8 +164,7 @@ def omnisci_buffer_constructor(context, builder, sig, args):
 
     alloc_fnty = ir.FunctionType(int8_t.as_pointer(), [int64_t, int64_t])
 
-    alloc_fn = builder.module.get_or_insert_function(
-        alloc_fnty, name="allocate_varlen_buffer")
+    alloc_fn = irutils.get_or_insert_function(builder.module, alloc_fnty, "allocate_varlen_buffer")
     ptr8 = builder.call(alloc_fn, [element_count, element_size])
     # remember possible temporary allocations so that when leaving a
     # UDF/UDTF, these will be deallocated, see omnisci_pipeline.py.
@@ -196,7 +195,7 @@ def codegen(context, builder, signature, args):
         # TODO: using stdlib `free` that works only for CPU. For CUDA
         # devices, we need to use omniscidb provided deallocator.
         free_fnty = llvm_ir.FunctionType(void_t, [int8_t.as_pointer()])
-        free_fn = builder.module.get_or_insert_function(free_fnty, name="free")
+        free_fn = irutils.get_or_insert_function(builder.module, free_fnty, "free")
 
         # We skip the ret pointer iff we're returning a Buffer
         # otherwise, we free everything
@@ -302,10 +301,7 @@ def omnisci_buffer_len_(typingctx, data):
 
     def codegen(context, builder, signature, args):
         data, = args
-        assert data.opname == 'load'
-        struct = data.operands[0]
-        return builder.load(builder.gep(
-            struct, [int32_t(0), int32_t(1)]))
+        return irutils.get_member_value(builder, data, 1)
 
     return sig, codegen
 
@@ -341,11 +337,7 @@ def omnisci_buffer_getitem_(typingctx, data, index):
 
     def codegen(context, builder, signature, args):
         data, index = args
-        assert data.opname == 'load'
-        buf = data.operands[0]
-
-        ptr = builder.load(builder.gep(
-            buf, [int32_t(0), int32_t(0)]))
+        ptr = irutils.get_member_value(builder, data, 0)
         res = builder.load(builder.gep(ptr, [index]))
 
         return res
@@ -416,14 +408,8 @@ def omnisci_buffer_setitem_(typingctx, data, index, value):
     nb_value = value
 
     def codegen(context, builder, signature, args):
-        zero = int32_t(0)
-
         data, index, value = args
-
-        assert data.opname == 'load'
-        buf = data.operands[0]
-
-        ptr = builder.load(builder.gep(buf, [zero, zero]))
+        ptr = irutils.get_member_value(builder, data, 0)
         value = truncate_or_extend(builder, nb_value, eltype, value, ptr.type.pointee)
 
         builder.store(value, builder.gep(ptr, [index]))

rbc/omnisci_backend/omnisci_column.py

@@ -8,7 +8,7 @@
            'OmnisciCursorType']
 
 from llvmlite import ir
-from rbc import typesystem
+from rbc import typesystem, irutils
 from .omnisci_buffer import Buffer, OmnisciBufferType, BufferType
 from .column_list import OmnisciColumnListType
 from rbc.targetinfo import TargetInfo
@@ -55,14 +55,8 @@ def omnisci_column_set_null_(typingctx, col_var, row_idx):
     null_value = target_info.null_values[str(T)]
 
     def codegen(context, builder, signature, args):
-        zero = int32_t(0)
-
         data, index = args
-
-        assert data.opname == 'load'
-        buf = data.operands[0]
-
-        ptr = builder.load(builder.gep(buf, [zero, zero]))
+        ptr = irutils.get_member_value(builder, data, 0)
 
         ty = ptr.type.pointee
         nv = ir.Constant(ir.IntType(T.bitwidth), null_value)
@@ -90,12 +84,8 @@ def omnisci_column_is_null_(typingctx, col_var, row_idx):
     nv = ir.Constant(ir.IntType(T.bitwidth), null_value)
 
     def codegen(context, builder, signature, args):
-        zero = int32_t(0)
         data, index = args
-        assert data.opname == 'load'
-        buf = data.operands[0]
-
-        ptr = builder.load(builder.gep(buf, [zero, zero]))
+        ptr = irutils.get_member_value(builder, data, 0)
         res = builder.load(builder.gep(ptr, [index]))
 
         if isinstance(T, types.Float):