Use dlpack conda package, remove dlpack submodule, reorganize cython

.gitmodules

@@ -2,10 +2,6 @@
 	path = thirdparty/cub
 	url = https://github.com/rapidsai/thirdparty-cub.git
 	branch = cudf-cub
-[submodule "thirdparty/dlpack"]
-	path = thirdparty/dlpack
-	url = https://github.com/rapidsai/dlpack.git
-	branch=cudf
 [submodule "thirdparty/jitify"]
 	path = thirdparty/jitify
 	url = https://github.com/rapidsai/jitify.git

ci/gpu/build.sh

@@ -44,7 +44,7 @@ logger "Activate conda env..."
 source activate gdf
 conda install "rmm=$MINOR_VERSION.*" "nvstrings=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
               "dask>=2.1.0" "distributed>=2.1.0" "numpy>=1.16" "double-conversion" \
-              "rapidjson" "flatbuffers" "boost-cpp" "fsspec>=0.3.3"
+              "rapidjson" "flatbuffers" "boost-cpp" "fsspec>=0.3.3" "dlpack"
 
 # Install the master version of dask and distributed
 logger "pip install git+https://github.com/dask/distributed.git --upgrade --no-deps" 

conda/environments/cudf_dev_cuda10.0.yml

@@ -37,6 +37,7 @@ dependencies:
 - pre_commit
 - dask>=2.1.0
 - distributed>=2.1.0
+- dlpack
 - arrow-cpp=0.14.1
 - boost-cpp
 - double-conversion

conda/environments/cudf_dev_cuda9.2.yml

@@ -37,6 +37,7 @@ dependencies:
 - pre_commit
 - dask>=2.1.0
 - distributed>=2.1.0
+- dlpack
 - arrow-cpp=0.14.1
 - boost-cpp
 - double-conversion

conda/recipes/cudf/meta.yaml

@@ -22,6 +22,7 @@ requirements:
     - cython >=0.29,<0.30
     - setuptools
     - numba >=0.41,<0.45
+    - dlpack
     - libcudf {{ version }}
   run:
     - python
@@ -31,6 +32,7 @@ requirements:
     - rmm {{ minor_version }}.*
     - nvstrings {{ minor_version }}.*
     - cython >=0.29,<0.30
+    - dlpack
 
 test:
   commands:

conda/recipes/libcudf/meta.yaml

@@ -35,9 +35,11 @@ requirements:
     - rapidjson
     - flatbuffers
     - boost-cpp
+    - dlpack
   run:
     - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
     - arrow-cpp 0.14.1.*
+    - dlpack
 
 test:
   commands:

python/cudf/cudf/bindings/cudf_cpp.pxd

@@ -5,8 +5,6 @@
 # cython: embedsignature = True
 # cython: language_level = 3
 
-from cudf.bindings.dlpack cimport DLManagedTensor
-
 from libcpp cimport bool
 from libc.stdint cimport (  # noqa: E211
     uint8_t,
@@ -333,18 +331,6 @@ cdef extern from "cudf.h" nogil:
         gdf_index_type* out_indices
     ) except +
 
-    cdef gdf_error gdf_from_dlpack(
-        gdf_column** columns,
-        gdf_size_type* num_columns,
-        const DLManagedTensor* tensor
-    ) except +
-
-    cdef gdf_error gdf_to_dlpack(
-        DLManagedTensor* tensor,
-        const gdf_column** columns,
-        gdf_size_type num_columns
-    ) except +
-
     cdef gdf_error gdf_nvtx_range_push(
         const char* const name,
         gdf_color color

python/cudf/cudf/bindings/dlpack.pxd

@@ -12,7 +12,22 @@ from libc.stdint cimport int64_t
 from libc.stdint cimport uint64_t
 from libcpp.vector cimport vector
 
-cdef extern from "dlpack.h" nogil:
+cdef extern from "dlpack/dlpack.h" nogil:
 
     ctypedef struct DLManagedTensor:
         void(*deleter)(DLManagedTensor*)
+
+
+cdef extern from "cudf.h" nogil:
+
+    cdef gdf_error gdf_from_dlpack(
+        gdf_column** columns,
+        gdf_size_type* num_columns,
+        const DLManagedTensor* tensor
+    ) except +
+
+    cdef gdf_error gdf_to_dlpack(
+        DLManagedTensor* tensor,
+        const gdf_column** columns,
+        gdf_size_type num_columns
+    ) except +

python/cudf/cudf/bindings/dlpack.pyx

@@ -9,7 +9,7 @@
 
 from cudf.bindings.cudf_cpp cimport *
 from cudf.bindings.cudf_cpp import *
-from cudf.bindings.dlpack cimport DLManagedTensor
+from cudf.bindings.dlpack cimport *
 from librmm_cffi import librmm as rmm
 
 from libc.stdint cimport uintptr_t, int8_t

python/cudf/setup.py

@@ -15,11 +15,7 @@ extensions = [
     Extension(
         "*",
         sources=cython_files,
-        include_dirs=[
-            "../../cpp/include/cudf",
-            "../../cpp/include",
-            "../../cpp/thirdparty/dlpack/include/dlpack/",
-        ],
+        include_dirs=["../../cpp/include/cudf", "../../cpp/include"],
         library_dirs=[get_python_lib()],
         libraries=["cudf"],
         language="c++",