diff --git a/.bazelversion b/.bazelversion index 4a36342f..712bd5a6 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -3.0.0 +3.3.1 \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index ca89e7d8..6a6695de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,19 +1,24 @@ version: 2.1 commands: - install-bazel-linux-rbe: + install-bazel: steps: - - run: curl -OL https://raw.githubusercontent.com/graknlabs/build-tools/master/ci/install-bazel-linux.sh - - run: bash ./install-bazel-linux.sh && rm ./install-bazel-linux.sh - - run: curl -OL https://raw.githubusercontent.com/graknlabs/build-tools/master/ci/install-bazel-rbe.sh - - run: bash ./install-bazel-rbe.sh && rm ./install-bazel-rbe.sh + - run: curl -OL https://raw.githubusercontent.com/graknlabs/dependencies/master/tool/bazelinstall/linux.sh + - run: bash ./linux.sh && rm ./linux.sh + - run: curl -OL https://raw.githubusercontent.com/graknlabs/dependencies/master/tool/bazelinstall/rbe.sh + - run: bash ./rbe.sh && rm ./rbe.sh + + run-bazel: + parameters: + command: + type: string + steps: + - run: bazel run @graknlabs_dependencies//tool/bazelrun:rbe -- << parameters.command >> - run-grakn-server: + run-grakn: steps: - - install-bazel-linux-rbe - - install-python - - run: bazel build @graknlabs_grakn_core//:assemble-linux-targz - - run: mkdir dist && tar -xvzf bazel-bin/external/graknlabs_grakn_core/grakn-core-all-linux.tar.gz -C ./dist/ + - run-bazel: + command: bazel run //:grakn-extractor -- dist/grakn-core-all-linux - run: nohup ./dist/grakn-core-all-linux/grakn server start install-python: @@ -23,47 +28,55 @@ commands: - run: sudo unlink /usr/bin/python3 - run: sudo ln -s $(which python3) /usr/bin/python3 - run: sudo sed -i -e 's|python3|python|g' $(which lsb_release) + - run: sudo sed -i -e '/from optparse import OptionParser/ i from __future__ import print_function' $(which lsb_release) jobs: build: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - - run: bazel build //... + - run-bazel: + command: bazel build //... test: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - - run: bazel test //kglib/... --test_output=streamed --spawn_strategy=standalone + - run-bazel: + command: bazel test //kglib/... --test_output=streamed --spawn_strategy=standalone test-end-to-end: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - run: bazel test //tests/end_to_end/... --test_output=streamed --spawn_strategy=standalone test-markdown-link-health: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - checkout - run: find . -name \*.md | xargs -L1 npx markdown-link-check@3.8.0 -v deploy-pip-snapshot: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - run: | @@ -72,27 +85,30 @@ jobs: bazel run --define version=$(git rev-parse HEAD) //:deploy-pip -- snapshot test-deployment-pip: - machine: true - working_directory: ~/kglib - steps: - - checkout - - run-grakn-server - - run: - name: Run test-deployment-pip for kglib - command: | - echo -n "0.0.0-$CIRCLE_SHA1" > VERSION - sed -i -e "s/KGLIB_VERSION_MARKER/$(cat VERSION)/g" tests/deployment/requirements.txt - cat tests/deployment/requirements.txt - pip install -U pip setuptools - pip install -r tests/deployment/requirements.txt - ./dist/grakn-core-all-linux/grakn console -k diagnosis -f kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql - cd tests/deployment/ && python -m unittest kgcn.diagnosis + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/kglib + steps: + - install-bazel + - checkout + - run-grakn + - run: + name: Run test-deployment-pip for kglib + command: | + echo -n "0.0.0-$CIRCLE_SHA1" > VERSION + sed -i -e "s/KGLIB_VERSION_MARKER/$(cat VERSION)/g" tests/deployment/requirements.txt + cat tests/deployment/requirements.txt + pip install -U pip setuptools + pip install -r tests/deployment/requirements.txt + ./dist/grakn-core-all-linux/grakn console -k diagnosis -f kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql + cd tests/deployment/ && python -m unittest kgcn.diagnosis release-approval: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - run: | @@ -101,9 +117,10 @@ jobs: bazel run @graknlabs_build_tools//ci:release-approval release-validate: - machine: true + machine: + image: ubuntu-1604:201903-01 steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - run: | @@ -111,24 +128,26 @@ jobs: graknlabs_grakn_core graknlabs_client_python deploy-github: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - checkout - run: wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_386.tar.gz - run: tar xvf ghr_v0.12.0_linux_386.tar.gz - run: - name: "Publish Draft Release on GitHub" - command: | - VERSION_TAG=$(cat "VERSION") - curl -X POST --fail --data "{\"tag_name\": \"${VERSION_TAG}\",\"target_commitish\": \"${CIRCLE_SHA1}\", - \"name\": \"KGLIB ${VERSION_TAG}\",\"body\": \"\",\"draft\": true,\"prerelease\": false}" https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases?access_token=${GITHUB_DEPLOYMENT_TOKEN} + name: "Publish Draft Release on GitHub" + command: | + VERSION_TAG=$(cat "VERSION") + curl -X POST --fail --data "{\"tag_name\": \"${VERSION_TAG}\",\"target_commitish\": \"${CIRCLE_SHA1}\", + \"name\": \"KGLIB ${VERSION_TAG}\",\"body\": \"\",\"draft\": true,\"prerelease\": false}" https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases?access_token=${GITHUB_DEPLOYMENT_TOKEN} deploy-pip-release: - machine: true + machine: + image: ubuntu-1604:201903-01 working_directory: ~/kglib steps: - - install-bazel-linux-rbe + - install-bazel - checkout - install-python - run: | @@ -137,7 +156,8 @@ jobs: bazel run --define version=$(cat VERSION) //:deploy-pip -- release release-cleanup: - machine: true + machine: + image: ubuntu-1604:201903-01 steps: - checkout - run: git push --delete https://$REPO_GITHUB_TOKEN@github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME $CIRCLE_BRANCH @@ -165,7 +185,9 @@ workflows: - deploy-pip-snapshot: filters: branches: - only: master + only: + - master + - development requires: - test - test-end-to-end @@ -174,7 +196,9 @@ workflows: - test-deployment-pip: filters: branches: - only: master + only: + - master + - development requires: - deploy-pip-snapshot - release-approval: diff --git a/BUILD b/BUILD index 8d61af7b..5816fffe 100644 --- a/BUILD +++ b/BUILD @@ -1,10 +1,16 @@ -exports_files(["requirements.txt"]) +exports_files(["requirements.txt", "deployment.properties", "RELEASE_TEMPLATE.md"]) load("@rules_python//python:defs.bzl", "py_library", "py_test") -load("@pypi_dependencies//:requirements.bzl", "requirement") -load("@graknlabs_bazel_distribution_pip//:requirements.bzl", deployment_requirement = "requirement") + +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") load("@graknlabs_bazel_distribution//pip:rules.bzl", "assemble_pip", "deploy_pip") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") + +load("@graknlabs_bazel_distribution//github:rules.bzl", "deploy_github") +load("@graknlabs_dependencies//distribution/artifact:rules.bzl", "artifact_extractor") assemble_pip( name = "assemble-pip", @@ -29,7 +35,7 @@ assemble_pip( license = "Apache-2.0", install_requires=[ 'enum-compat==0.0.2', - 'grakn-client==1.7.2', + 'grakn-client==1.8.0', 'absl-py==0.8.0', 'astor==0.8.0', 'cloudpickle==1.2.2', @@ -41,7 +47,7 @@ assemble_pip( 'gast==0.3.1', 'google-pasta==0.1.7', 'graph-nets==1.0.4', - 'grpcio==1.23.0', + 'grpcio==1.24.1,<2', 'h5py==2.10.0', 'Keras-Applications==1.0.8', 'Keras-Preprocessing==1.1.0', @@ -51,13 +57,13 @@ assemble_pip( 'networkx==2.3', 'numpy==1.17.2', 'pandas==0.25.1', - 'protobuf==3.9.1', + 'protobuf==3.6.1', 'pyparsing==2.4.2', 'python-dateutil==2.8.0', 'pytz==2019.2', 'scipy==1.3.1', 'semantic-version==2.8.2', - 'six==1.12.0', + 'six>=1.11.0', 'tensorboard==1.14.0', 'tensorflow==1.14.0', 'tensorflow-estimator==1.14.0', @@ -76,5 +82,5 @@ assemble_pip( deploy_pip( name = "deploy-pip", target = ":assemble-pip", - deployment_properties = "@graknlabs_build_tools//:deployment.properties", + deployment_properties = "@graknlabs_dependencies//distribution:deployment.properties", ) diff --git a/README.md b/README.md index 3baaeff6..23409705 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ At present this repo contains one project: [*Knowledge Graph Convolutional Netwo - KGLIB installed via pip: `pip install grakn-kglib`. -- [Grakn Core 1.6.1](https://github.com/graknlabs/grakn/releases) running in the background +- [Grakn Core 1.8.0](https://github.com/graknlabs/grakn/releases) running in the background -- client-python 1.6.1 ([PyPi](https://pypi.org/project/grakn-client/), [GitHub release](https://github.com/graknlabs/client-python/releases)) +- client-python 1.8.0 ([PyPi](https://pypi.org/project/grakn-client/), [GitHub release](https://github.com/graknlabs/client-python/releases)) **Run** Take a look at [*Knowledge Graph Convolutional Networks* (KGCNs)](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn) to see a walkthrough of how to use the library. diff --git a/VERSION b/VERSION index 7dff5b89..ee1372d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.1 \ No newline at end of file +0.2.2 diff --git a/WORKSPACE b/WORKSPACE index b8a65f1c..f2c5666a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,221 +1,123 @@ +# +# Copyright (C) 2020 Grakn Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +workspace(name = "graknlabs_kglib") -workspace( - name = "kglib" -) - - -######################################################################################################################## -# Load Bazel Rules -######################################################################################################################## +################################ +# Load @graknlabs_dependencies # +################################ +load("//dependencies/graknlabs:repositories.bzl", "graknlabs_dependencies") +graknlabs_dependencies() -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +# Load Bazel +load("@graknlabs_dependencies//builder/bazel:deps.bzl", "bazel_toolchain") +bazel_toolchain() -# We require loading bazel_rules from the graknlabs fork, not from bazel, since we've patched the python rules to work with TensorFlow -load("//dependencies/python:dependencies.bzl", "rules_python") -rules_python() +# Load gRPC +load("@graknlabs_dependencies//builder/grpc:deps.bzl", grpc_deps = "deps") +grpc_deps() +load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", +com_github_grpc_grpc_deps = "grpc_deps") +com_github_grpc_grpc_deps() -## Only needed for PIP support: -load("@rules_python//python:repositories.bzl", "py_repositories") -py_repositories() +# Load Java +load("@graknlabs_dependencies//builder/java:deps.bzl", java_deps = "deps") +java_deps() +load("@graknlabs_dependencies//library/maven:rules.bzl", "maven") +# Load Python +load("@graknlabs_dependencies//builder/python:deps.bzl", python_deps = "deps") +python_deps(use_patched_version=True) load("@rules_python//python:pip.bzl", "pip_repositories", "pip3_import") pip_repositories() -######################################################################################################################## -# Load Build Tools -######################################################################################################################## - -load("//dependencies/graknlabs:dependencies.bzl", "graknlabs_build_tools") -graknlabs_build_tools() - - -######################################################################################################################## -# Load Pip Distribution Requirements -######################################################################################################################## +# Load graknlabs_dependencies_ci_pip (for @graknlabs_dependencies//tool/sync:dependencies) +pip3_import( + name = "graknlabs_dependencies_ci_pip", + requirements = "@graknlabs_dependencies//tool:requirements.txt", +) +load("@graknlabs_dependencies_ci_pip//:requirements.bzl", +graknlabs_dependencies_ci_pip_install = "pip_install") +graknlabs_dependencies_ci_pip_install() -load("@graknlabs_build_tools//distribution:dependencies.bzl", "graknlabs_bazel_distribution") -graknlabs_bazel_distribution() +##################################################################### +# Load @graknlabs_bazel_distribution (from @graknlabs_dependencies) # +##################################################################### +load("@graknlabs_dependencies//distribution:deps.bzl", distribution_deps = "deps") +distribution_deps() pip3_import( name = "graknlabs_bazel_distribution_pip", requirements = "@graknlabs_bazel_distribution//pip:requirements.txt", ) -load("@graknlabs_bazel_distribution_pip//:requirements.bzl", graknlabs_bazel_distribution_pip_install = "pip_install") +load("@graknlabs_bazel_distribution_pip//:requirements.bzl", +graknlabs_bazel_distribution_pip_install = "pip_install") graknlabs_bazel_distribution_pip_install() +load("@graknlabs_bazel_distribution//github:dependencies.bzl", "tcnksm_ghr") +tcnksm_ghr() + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +git_repository( + name = "io_bazel_skydoc", + remote = "https://github.com/graknlabs/skydoc.git", + branch = "experimental-skydoc-allow-dep-on-bazel-tools", +) + +load("@io_bazel_skydoc//:setup.bzl", "skydoc_repositories") +skydoc_repositories() + load("@graknlabs_bazel_distribution//common:dependencies.bzl", "bazelbuild_rules_pkg") bazelbuild_rules_pkg() -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") -rules_pkg_dependencies() +############################ +# Load @graknlabs_protocol # +############################ +load("//dependencies/graknlabs:repositories.bzl", "graknlabs_protocol") +graknlabs_protocol() -################################### -# Load Client Python Dependencies # -################################### +####################################### +# Load @graknlabs_grakn_core_artifact # +####################################### +load("//dependencies/graknlabs:artifacts.bzl", "graknlabs_grakn_core_artifact") +graknlabs_grakn_core_artifact() -load("//dependencies/graknlabs:dependencies.bzl", "graknlabs_client_python") +################################# +# Load @graknlabs_client_python # +################################# +load("//dependencies/graknlabs:repositories.bzl", "graknlabs_client_python") graknlabs_client_python() pip3_import( name = "graknlabs_client_python_pip", requirements = "@graknlabs_client_python//:requirements.txt", ) - load("@graknlabs_client_python_pip//:requirements.bzl", graknlabs_client_python_pip_install = "pip_install") graknlabs_client_python_pip_install() - -######################################################################################################################## -# Load KGLIB's PyPi requirements -######################################################################################################################## - -# Load PyPI dependencies for Python programs +######################### +# Load @graknlabs_kglib # +######################### pip3_import( - name = "pypi_dependencies", + name = "graknlabs_kglib_pip", requirements = "//:requirements.txt", ) - -load("@pypi_dependencies//:requirements.bzl", pip_install_kglib_requirements = "pip_install") -pip_install_kglib_requirements() - - -######################################################################################################################## -# Load Grakn -######################################################################################################################## - -load("//dependencies/graknlabs:dependencies.bzl", "graknlabs_grakn_core") -graknlabs_grakn_core() - -################################ -# Load Grakn Labs dependencies # -################################ - -load( - "@graknlabs_grakn_core//dependencies/graknlabs:dependencies.bzl", - "graknlabs_graql", - "graknlabs_common", - "graknlabs_protocol", - "graknlabs_client_java", - "graknlabs_console", - "graknlabs_grabl_tracing" -) -graknlabs_graql() -graknlabs_common() -graknlabs_protocol() -graknlabs_client_java() -graknlabs_console() -graknlabs_grabl_tracing() - -load("@graknlabs_grabl_tracing//dependencies/maven:dependencies.bzl", -graknlabs_grabl_tracing_maven_dependencies="maven_dependencies") -graknlabs_grabl_tracing_maven_dependencies() - -# Skip since these are already present above -#load("@graknlabs_build_tools//distribution:dependencies.bzl", "graknlabs_bazel_distribution") -#graknlabs_bazel_distribution() - -########################### -# Load Bazel dependencies # -########################### - -load("@graknlabs_build_tools//bazel:dependencies.bzl", "bazel_common", "bazel_deps", "bazel_toolchain") -bazel_common() -bazel_deps() -bazel_toolchain() - - -################################# -# Load Build Tools dependencies # -################################# - -load("@graknlabs_build_tools//checkstyle:dependencies.bzl", "checkstyle_dependencies") -checkstyle_dependencies() - -load("@graknlabs_build_tools//sonarcloud:dependencies.bzl", "sonarcloud_dependencies") -sonarcloud_dependencies() - - -pip3_import( - name = "graknlabs_build_tools_ci_pip", - requirements = "@graknlabs_build_tools//ci:requirements.txt", -) - -load("@graknlabs_build_tools_ci_pip//:requirements.bzl", -graknlabs_build_tools_ci_pip_install = "pip_install") -graknlabs_build_tools_ci_pip_install() - - -##################################### -# Load Java dependencies from Maven # -##################################### - -load("@graknlabs_grakn_core//dependencies/maven:dependencies.bzl", "maven_dependencies") -maven_dependencies() - - -########################### -# Load Graql dependencies # -########################### - -# Load ANTLR dependencies for Bazel -load("@graknlabs_graql//dependencies/compilers:dependencies.bzl", "antlr_dependencies") -antlr_dependencies() - -# Load ANTLR dependencies for ANTLR programs -load("@rules_antlr//antlr:deps.bzl", "antlr_dependencies") -antlr_dependencies() - -load("@graknlabs_graql//dependencies/maven:dependencies.bzl", -graknlabs_graql_maven_dependencies = "maven_dependencies") -graknlabs_graql_maven_dependencies() - - -####################################### -# Load compiler dependencies for GRPC # -####################################### - -load("@graknlabs_build_tools//grpc:dependencies.bzl", "grpc_dependencies") -grpc_dependencies() - -load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", -com_github_grpc_grpc_deps = "grpc_deps") -com_github_grpc_grpc_deps() - -load("@stackb_rules_proto//java:deps.bzl", "java_grpc_compile") -java_grpc_compile() - - -################################## -# Load Distribution dependencies # -################################## - -# Skip since these are already present above -# TODO: rename the macro we load here to deploy_github_dependencies -#load("@graknlabs_bazel_distribution//github:dependencies.bzl", "github_dependencies_for_deployment") -#github_dependencies_for_deployment() - -load("@graknlabs_build_tools//bazel:dependencies.bzl", "bazel_rules_docker") -bazel_rules_docker() - -load("@io_bazel_rules_docker//repositories:repositories.bzl", -bazel_rules_docker_repositories = "repositories") -bazel_rules_docker_repositories() - -load("@io_bazel_rules_docker//container:container.bzl", "container_pull") -container_pull( - name = "openjdk_image", - registry = "index.docker.io", - repository = "library/openjdk", - tag = "8" -) - -##################################### -# Load Bazel common workspace rules # -##################################### - -# TODO: Figure out why this cannot be loaded at earlier at the top of the file -load("@com_github_google_bazel_common//:workspace_defs.bzl", "google_common_workspace_rules") -google_common_workspace_rules() +load("@graknlabs_kglib_pip//:requirements.bzl", +graknlabs_kglib_pip_install = "pip_install") +graknlabs_kglib_pip_install() diff --git a/dependencies/graknlabs/BUILD b/dependencies/graknlabs/BUILD index e69de29b..fb0409e3 100644 --- a/dependencies/graknlabs/BUILD +++ b/dependencies/graknlabs/BUILD @@ -0,0 +1,16 @@ +# +# Copyright (C) 2020 Grakn Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/dependencies/graknlabs/artifacts.bzl b/dependencies/graknlabs/artifacts.bzl new file mode 100644 index 00000000..b0353f71 --- /dev/null +++ b/dependencies/graknlabs/artifacts.bzl @@ -0,0 +1,26 @@ +# +# Copyright (C) 2020 Grakn Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +load("@graknlabs_dependencies//distribution/artifact:rules.bzl", "artifact_file") + +def graknlabs_grakn_core_artifact(): + artifact_file( + name = "graknlabs_grakn_core_artifact", + group_name = "graknlabs_grakn_core", + artifact_name = "grakn-core-all-linux-{version}.tar.gz", + commit = "1c91c824aa208ad77c14ab7ee3ea6f6de7cff467", + ) \ No newline at end of file diff --git a/dependencies/graknlabs/dependencies.bzl b/dependencies/graknlabs/dependencies.bzl deleted file mode 100644 index 4531c0c1..00000000 --- a/dependencies/graknlabs/dependencies.bzl +++ /dev/null @@ -1,23 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def graknlabs_build_tools(): - git_repository( - name = "graknlabs_build_tools", - remote = "https://github.com/graknlabs/build-tools", - commit = "715e4802ea375abda44e24b2bb092ee14de72e42", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_build_tools - ) - - -def graknlabs_grakn_core(): - git_repository( - name = "graknlabs_grakn_core", - remote = "https://github.com/graknlabs/grakn", - tag = "1.7.1" # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_grakn_core - ) - -def graknlabs_client_python(): - git_repository( - name = "graknlabs_client_python", - remote = "https://github.com/graknlabs/client-python", - commit = "1382eedb01814ab0222d80e0c7d35d7265a802df" # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_client_python - ) diff --git a/dependencies/graknlabs/repositories.bzl b/dependencies/graknlabs/repositories.bzl new file mode 100644 index 00000000..e2122f2a --- /dev/null +++ b/dependencies/graknlabs/repositories.bzl @@ -0,0 +1,40 @@ + +# +# Copyright (C) 2020 Grakn Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def graknlabs_dependencies(): + git_repository( + name = "graknlabs_dependencies", + remote = "https://github.com/graknlabs/dependencies", + commit = "8e4200dfb556f1b64db78f2223488b3ced80df75", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_dependencies + ) + +def graknlabs_protocol(): + git_repository( + name = "graknlabs_protocol", + remote = "https://github.com/graknlabs/protocol", + commit = "ae12f6c3829cd5304f3bb4ba865f5fcc10022738", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_protocol + ) + +def graknlabs_client_python(): + git_repository( + name = "graknlabs_client_python", + remote = "https://github.com/graknlabs/client-python", + commit = "4b8dd3f865ef528cbae60a27eb7d67d9fe4663be", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_client_python + ) diff --git a/dependencies/python/BUILD b/dependencies/python/BUILD deleted file mode 100644 index e69de29b..00000000 diff --git a/dependencies/python/dependencies.bzl b/dependencies/python/dependencies.bzl deleted file mode 100644 index eb3abe6e..00000000 --- a/dependencies/python/dependencies.bzl +++ /dev/null @@ -1,10 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - - -def rules_python(): - git_repository( - name = "rules_python", - # Grakn python rules - remote = "https://github.com/graknlabs/rules_python.git", - commit = "ee519e17ed5265bdd2431937bd271e3b76ad5b0a" - ) diff --git a/kglib/BUILD b/kglib/BUILD index dc05ecbb..325f9f93 100644 --- a/kglib/BUILD +++ b/kglib/BUILD @@ -1,6 +1,7 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") load("@graknlabs_bazel_distribution_pip//:requirements.bzl", deployment_requirement = "requirement") + py_library( name = "kglib", srcs = [ diff --git a/kglib/kgcn/README.md b/kglib/kgcn/README.md index 3b5e78a7..e6598408 100644 --- a/kglib/kgcn/README.md +++ b/kglib/kgcn/README.md @@ -10,9 +10,9 @@ This project introduces a novel model: the *Knowledge Graph Convolutional Networ - KGLIB installed via pip: `pip install grakn-kglib`. -- [Grakn Core 1.6.1](https://github.com/graknlabs/grakn/releases) running in the background +- [Grakn Core 1.8.0](https://github.com/graknlabs/grakn/releases) running in the background -- client-python 1.6.1 ([PyPi](https://pypi.org/project/grakn-client/), [GitHub release](https://github.com/graknlabs/client-python/releases)) +- client-python 1.8.0 ([PyPi](https://pypi.org/project/grakn-client/), [GitHub release](https://github.com/graknlabs/client-python/releases)) See the [full example](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn/examples/diagnosis/diagnosis.py) for how to use a KGCN for [Relation](https://dev.grakn.ai/docs/schema/concepts#relation) prediction. You can use the example as a template to create a KGCN for your own Grakn data. If you need to customise the learning or model used, you'll need to make changes to your version of the [pipeline](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn/pipeline/pipeline.py). @@ -118,7 +118,7 @@ Here we identify the core components used to build a working KGCN pipeline. ### Application -e.g. [diagnosis example](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn/examples/diagnosis/diagnosis.py) +e.g. [diagnosis example](https://github.com/graknlabs/kglib/tree/master/kglib/kgcn/examples/diagnosis) 1. Fetch subgraphs, each subgraph is used as an *example* - This requires specifying queries that will retrieve Concepts from Grakn diff --git a/kglib/kgcn/examples/diagnosis/BUILD b/kglib/kgcn/examples/diagnosis/BUILD index 635d74b0..a959ed5c 100644 --- a/kglib/kgcn/examples/diagnosis/BUILD +++ b/kglib/kgcn/examples/diagnosis/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( name = "diagnosis_test", @@ -9,9 +10,9 @@ py_test( deps = [ "diagnosis", "//kglib/utils/graph/test", - requirement('numpy'), - requirement('networkx'), - requirement('decorator'), + graknlabs_kglib_requirement('numpy'), + graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('decorator'), ] ) diff --git a/kglib/kgcn/learn/BUILD b/kglib/kgcn/learn/BUILD index 56f41944..707dd9d2 100644 --- a/kglib/kgcn/learn/BUILD +++ b/kglib/kgcn/learn/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( name = "metrics_test", @@ -33,41 +34,41 @@ py_library( ], deps = [ # Networkx deps - requirement('networkx'), - requirement('decorator'), + graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('decorator'), # Graph nets deps - requirement('absl-py'), - requirement('cloudpickle'), - requirement('contextlib2'), - # requirement('decorator'), - requirement('dm-sonnet'), - requirement('future'), - requirement('graph-nets'), - # requirement('networkx'), - requirement('numpy'), - requirement('semantic-version'), - requirement('six'), - requirement('tensorflow-probability'), - requirement('wrapt'), + graknlabs_kglib_requirement('absl-py'), + graknlabs_kglib_requirement('cloudpickle'), + graknlabs_kglib_requirement('contextlib2'), + # graknlabs_kglib_requirement('decorator'), + graknlabs_kglib_requirement('dm-sonnet'), + graknlabs_kglib_requirement('future'), + graknlabs_kglib_requirement('graph-nets'), + # graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('numpy'), + graknlabs_kglib_requirement('semantic-version'), + graknlabs_kglib_requirement('six'), + graknlabs_kglib_requirement('tensorflow-probability'), + graknlabs_kglib_requirement('wrapt'), # Tensorflow deps - # requirement('absl-py'), - requirement('astor'), - requirement('gast'), - requirement('google-pasta'), - requirement('keras-applications'), - requirement('keras-preprocessing'), - # requirement('six'), - requirement('protobuf'), - requirement('tensorboard'), - requirement('tensorflow'), - requirement('tensorflow-estimator'), - requirement('termcolor'), - # requirement('wrapt'), + # graknlabs_kglib_requirement('absl-py'), + graknlabs_kglib_requirement('astor'), + graknlabs_kglib_requirement('gast'), + graknlabs_kglib_requirement('google-pasta'), + graknlabs_kglib_requirement('keras-applications'), + graknlabs_kglib_requirement('keras-preprocessing'), + # graknlabs_kglib_requirement('six'), + graknlabs_kglib_requirement('protobuf'), + graknlabs_kglib_requirement('tensorboard'), + graknlabs_kglib_requirement('tensorflow'), + graknlabs_kglib_requirement('tensorflow-estimator'), + graknlabs_kglib_requirement('termcolor'), + # graknlabs_kglib_requirement('wrapt'), # Scipy deps - requirement('scipy') + graknlabs_kglib_requirement('scipy') ], visibility=['//visibility:public'] diff --git a/kglib/kgcn/models/BUILD b/kglib/kgcn/models/BUILD index 5c9674ab..3bb8f4f8 100644 --- a/kglib/kgcn/models/BUILD +++ b/kglib/kgcn/models/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( name = "attribute_test", @@ -84,38 +85,38 @@ py_library( ], deps = [ # Networkx deps - requirement('networkx'), - requirement('decorator'), + graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('decorator'), # Graph nets deps - requirement('absl-py'), - requirement('cloudpickle'), - requirement('contextlib2'), - # requirement('decorator'), - requirement('dm-sonnet'), - requirement('future'), - requirement('graph-nets'), - # requirement('networkx'), - requirement('numpy'), - requirement('semantic-version'), - requirement('six'), - requirement('tensorflow-probability'), - requirement('wrapt'), + graknlabs_kglib_requirement('absl-py'), + graknlabs_kglib_requirement('cloudpickle'), + graknlabs_kglib_requirement('contextlib2'), + # graknlabs_kglib_requirement('decorator'), + graknlabs_kglib_requirement('dm-sonnet'), + graknlabs_kglib_requirement('future'), + graknlabs_kglib_requirement('graph-nets'), + # graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('numpy'), + graknlabs_kglib_requirement('semantic-version'), + graknlabs_kglib_requirement('six'), + graknlabs_kglib_requirement('tensorflow-probability'), + graknlabs_kglib_requirement('wrapt'), # Tensorflow deps - # requirement('absl-py'), - requirement('astor'), - requirement('gast'), - requirement('google-pasta'), - requirement('keras-applications'), - requirement('keras-preprocessing'), - # requirement('six'), - requirement('protobuf'), - requirement('tensorboard'), - requirement('tensorflow'), - requirement('tensorflow-estimator'), - requirement('termcolor'), - # requirement('wrapt'), + # graknlabs_kglib_requirement('absl-py'), + graknlabs_kglib_requirement('astor'), + graknlabs_kglib_requirement('gast'), + graknlabs_kglib_requirement('google-pasta'), + graknlabs_kglib_requirement('keras-applications'), + graknlabs_kglib_requirement('keras-preprocessing'), + # graknlabs_kglib_requirement('six'), + graknlabs_kglib_requirement('protobuf'), + graknlabs_kglib_requirement('tensorboard'), + graknlabs_kglib_requirement('tensorflow'), + graknlabs_kglib_requirement('tensorflow-estimator'), + graknlabs_kglib_requirement('termcolor'), + # graknlabs_kglib_requirement('wrapt'), ], visibility=['//visibility:public'] ) \ No newline at end of file diff --git a/kglib/kgcn/pipeline/BUILD b/kglib/kgcn/pipeline/BUILD index 85014332..ac61109e 100644 --- a/kglib/kgcn/pipeline/BUILD +++ b/kglib/kgcn/pipeline/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( name = "encode_test", @@ -31,14 +32,14 @@ py_library( 'utils.py', ], deps = [ - requirement('graph_nets'), - requirement('numpy'), - requirement('dm-sonnet'), - requirement('tensorflow'), - requirement('tensorflow-probability'), - requirement('semantic-version'), - requirement('contextlib2'), - requirement('wrapt'), + graknlabs_kglib_requirement('graph_nets'), + graknlabs_kglib_requirement('numpy'), + graknlabs_kglib_requirement('dm-sonnet'), + graknlabs_kglib_requirement('tensorflow'), + graknlabs_kglib_requirement('tensorflow-probability'), + graknlabs_kglib_requirement('semantic-version'), + graknlabs_kglib_requirement('contextlib2'), + graknlabs_kglib_requirement('wrapt'), "//kglib/utils/graph", ], visibility=['//visibility:public'] diff --git a/kglib/kgcn/plot/BUILD b/kglib/kgcn/plot/BUILD index 49b0a411..6597c90e 100644 --- a/kglib/kgcn/plot/BUILD +++ b/kglib/kgcn/plot/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( name = "plotting_test", @@ -19,45 +20,45 @@ py_library( ], deps = [ # Matplotlib deps - requirement('cycler'), - requirement('kiwisolver'), - requirement('matplotlib'), - requirement('pyparsing'), - requirement('python-dateutil'), + graknlabs_kglib_requirement('cycler'), + graknlabs_kglib_requirement('kiwisolver'), + graknlabs_kglib_requirement('matplotlib'), + graknlabs_kglib_requirement('pyparsing'), + graknlabs_kglib_requirement('python-dateutil'), # Networkx deps - requirement('networkx'), - requirement('decorator'), + graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('decorator'), # Graph nets deps - requirement('absl-py'), - requirement('cloudpickle'), - requirement('contextlib2'), - # requirement('decorator'), - requirement('dm-sonnet'), - requirement('future'), - requirement('graph-nets'), - # requirement('networkx'), - requirement('numpy'), - requirement('semantic-version'), - requirement('six'), - requirement('tensorflow-probability'), - requirement('wrapt'), + graknlabs_kglib_requirement('absl-py'), + graknlabs_kglib_requirement('cloudpickle'), + graknlabs_kglib_requirement('contextlib2'), + # graknlabs_kglib_requirement('decorator'), + graknlabs_kglib_requirement('dm-sonnet'), + graknlabs_kglib_requirement('future'), + graknlabs_kglib_requirement('graph-nets'), + # graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('numpy'), + graknlabs_kglib_requirement('semantic-version'), + graknlabs_kglib_requirement('six'), + graknlabs_kglib_requirement('tensorflow-probability'), + graknlabs_kglib_requirement('wrapt'), # Tensorflow deps - # requirement('absl-py'), - requirement('astor'), - requirement('gast'), - requirement('google-pasta'), - requirement('keras-applications'), - requirement('keras-preprocessing'), - # requirement('six'), - requirement('protobuf'), - requirement('tensorboard'), - requirement('tensorflow'), - requirement('tensorflow-estimator'), - requirement('termcolor'), - # requirement('wrapt'), + # graknlabs_kglib_requirement('absl-py'), + graknlabs_kglib_requirement('astor'), + graknlabs_kglib_requirement('gast'), + graknlabs_kglib_requirement('google-pasta'), + graknlabs_kglib_requirement('keras-applications'), + graknlabs_kglib_requirement('keras-preprocessing'), + # graknlabs_kglib_requirement('six'), + graknlabs_kglib_requirement('protobuf'), + graknlabs_kglib_requirement('tensorboard'), + graknlabs_kglib_requirement('tensorflow'), + graknlabs_kglib_requirement('tensorflow-estimator'), + graknlabs_kglib_requirement('termcolor'), + # graknlabs_kglib_requirement('wrapt'), ], visibility=['//visibility:public'] diff --git a/kglib/utils/grakn/BUILD b/kglib/utils/grakn/BUILD index 6904ab6d..d88e5cf2 100644 --- a/kglib/utils/grakn/BUILD +++ b/kglib/utils/grakn/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_library( diff --git a/kglib/utils/grakn/object/BUILD b/kglib/utils/grakn/object/BUILD index 46bfacd7..6ddab780 100644 --- a/kglib/utils/grakn/object/BUILD +++ b/kglib/utils/grakn/object/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_library( diff --git a/kglib/utils/grakn/object/thing.py b/kglib/utils/grakn/object/thing.py index 903c5e8f..5be22bed 100644 --- a/kglib/utils/grakn/object/thing.py +++ b/kglib/utils/grakn/object/thing.py @@ -19,23 +19,23 @@ from kglib.utils.grakn.object.comparable import PropertyComparable -DATA_TYPE_NAMES = ('long', 'double', 'boolean', 'date', 'string') +VALUE_TYPE_NAMES = ('long', 'double', 'boolean', 'date', 'string') class Thing(PropertyComparable): - def __init__(self, id, type_label, base_type_label, data_type=None, value=None): + def __init__(self, id, type_label, base_type_label, value_type=None, value=None): self.id = id self.type_label = type_label self.base_type_label = base_type_label # TODO rename to base_type in line with Client Python # If the thing is an attribute - self.data_type = data_type + self.value_type = value_type self.value = value # TODO Make attribute a separate class if self.base_type_label == 'attribute': - if self.data_type is None: - raise ValueError('Attribute data_type must be provided') + if self.value_type is None: + raise ValueError('Attribute value_type must be provided') if self.value is None: raise ValueError('Attribute value must be provided') @@ -58,10 +58,10 @@ def build_thing(grakn_thing, tx): assert(base_type_label in ['entity', 'relation', 'attribute']) if base_type_label == 'attribute': - data_type = grakn_thing.as_remote(tx).type().data_type().name.lower() - assert data_type in DATA_TYPE_NAMES + value_type = grakn_thing.as_remote(tx).type().value_type().name.lower() + assert value_type in VALUE_TYPE_NAMES value = grakn_thing.value() - return Thing(id, type_label, base_type_label, data_type, value) + return Thing(id, type_label, base_type_label, value_type, value) return Thing(id, type_label, base_type_label) diff --git a/kglib/utils/grakn/synthetic/BUILD b/kglib/utils/grakn/synthetic/BUILD index 44ce7510..1c55a2f9 100644 --- a/kglib/utils/grakn/synthetic/BUILD +++ b/kglib/utils/grakn/synthetic/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_library( diff --git a/kglib/utils/grakn/synthetic/examples/BUILD b/kglib/utils/grakn/synthetic/examples/BUILD index aaf2e139..d2c54102 100644 --- a/kglib/utils/grakn/synthetic/examples/BUILD +++ b/kglib/utils/grakn/synthetic/examples/BUILD @@ -1,7 +1,14 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") -exports_files(["diagnosis/schema.gql"]) +filegroup( + name = "graql-schema", + srcs = [ + "diagnosis/schema.gql", + ], + visibility = ["//visibility:public"], +) py_library( name = "examples", @@ -10,7 +17,7 @@ py_library( ], deps = [ "@graknlabs_client_python//:client_python", - requirement('numpy'), + graknlabs_kglib_requirement('numpy'), ], visibility=['//visibility:public'] ) \ No newline at end of file diff --git a/kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql b/kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql index ead45b4b..f756b9a6 100644 --- a/kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql +++ b/kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql @@ -1,19 +1,19 @@ define example-id sub attribute, - datatype long; + value long; age sub attribute, - datatype long; + value long; name sub attribute, - datatype string; + value string; severity sub attribute, - datatype double; + value double; units-per-week sub attribute, - datatype long; + value long; person sub entity, key example-id, @@ -56,13 +56,13 @@ candidate-diagnosis sub relation, relates candidate-diagnosed-disease; probability-exists sub attribute, - datatype double; + value double; probability-non-exists sub attribute, - datatype double; + value double; probability-preexists sub attribute, - datatype double; + value double; kgcn sub entity, plays diagnoser; diff --git a/kglib/utils/grakn/synthetic/statistics/BUILD b/kglib/utils/grakn/synthetic/statistics/BUILD index d16af701..f727f101 100644 --- a/kglib/utils/grakn/synthetic/statistics/BUILD +++ b/kglib/utils/grakn/synthetic/statistics/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( name = "pmf_test", @@ -17,11 +18,11 @@ py_library( 'pmf.py', ], deps = [ - requirement('numpy'), - requirement('pandas'), - requirement('pytz'), - requirement('python-dateutil'), - requirement('six') + graknlabs_kglib_requirement('numpy'), + graknlabs_kglib_requirement('pandas'), + graknlabs_kglib_requirement('pytz'), + graknlabs_kglib_requirement('python-dateutil'), + graknlabs_kglib_requirement('six') ], visibility=['//visibility:public'] ) \ No newline at end of file diff --git a/kglib/utils/grakn/test/base.py b/kglib/utils/grakn/test/base.py index fae2cdaa..17258418 100644 --- a/kglib/utils/grakn/test/base.py +++ b/kglib/utils/grakn/test/base.py @@ -17,9 +17,12 @@ # under the License. # +from __future__ import print_function + import os import shutil import subprocess as sp +import tarfile import tempfile import zipfile @@ -39,10 +42,10 @@ def _extract_member(self, member, targetpath, pwd): class GraknServer(object): - DISTRIBUTION_LOCATION = 'external/graknlabs_grakn_core/grakn-core-all-mac.zip' - DISTRIBUTION_ROOT_DIR = 'grakn-core-all-mac' - def __init__(self): + def __init__(self, distribution_location): + self.__distribution_location = distribution_location + self.__distribution_root_dir = None self.__unpacked_dir = None def __enter__(self): @@ -56,27 +59,26 @@ def start(self): self._unpack() sp.check_call([ 'grakn', 'server', 'start' - ], cwd=os.path.join(self.__unpacked_dir, GraknServer.DISTRIBUTION_ROOT_DIR)) - return self + ], cwd=os.path.join(self.__unpacked_dir, self.__distribution_root_dir)) def stop(self): sp.check_call([ 'grakn', 'server', 'stop' - ], cwd=os.path.join(self.__unpacked_dir, GraknServer.DISTRIBUTION_ROOT_DIR)) + ], cwd=os.path.join(self.__unpacked_dir, self.__distribution_root_dir)) shutil.rmtree(self.__unpacked_dir) - return self def _unpack(self): self.__unpacked_dir = tempfile.mkdtemp(prefix='grakn') - with ZipFile(GraknServer.DISTRIBUTION_LOCATION) as zf: - zf.extractall(self.__unpacked_dir) + with tarfile.open(self.__distribution_location) as tf: + tf.extractall(self.__unpacked_dir) + self.__distribution_root_dir = os.path.commonpath(tf.getnames()[1:]) def load_graql_file(self, keyspace, graql_file_path): sp.check_call([ 'grakn', 'console', '-k', keyspace, '-f', - os.getenv("TEST_SRCDIR") + graql_file_path + os.getenv("TEST_SRCDIR") + "/" + os.getenv("TEST_WORKSPACE") + "/" + graql_file_path ], cwd=self.grakn_binary_location) @property def grakn_binary_location(self): - return os.path.join(self.__unpacked_dir, GraknServer.DISTRIBUTION_ROOT_DIR) + return os.path.join(self.__unpacked_dir, self.__distribution_root_dir) diff --git a/kglib/utils/grakn/test/mock/concept.py b/kglib/utils/grakn/test/mock/concept.py index ff943473..b6580d03 100644 --- a/kglib/utils/grakn/test/mock/concept.py +++ b/kglib/utils/grakn/test/mock/concept.py @@ -34,19 +34,19 @@ def label(self): return self._label -class DataType: +class ValueType: def __init__(self, name): self.name = name class MockAttributeType(MockType): - def __init__(self, id, label, base_type, data_type): + def __init__(self, id, label, base_type, value_type): super().__init__(id, label, base_type) - assert data_type in {'STRING', 'LONG', 'DOUBLE', 'DATE', 'BOOLEAN'} - self._data_type = DataType(data_type) + assert value_type in {'STRING', 'LONG', 'DOUBLE', 'DATETIME', 'BOOLEAN'} + self._value_type = ValueType(value_type) - def data_type(self): - return self._data_type + def value_type(self): + return self._value_type class MockThing(MockConcept): diff --git a/kglib/utils/grakn/type/BUILD b/kglib/utils/grakn/type/BUILD index 4b723eea..4afa91b6 100644 --- a/kglib/utils/grakn/type/BUILD +++ b/kglib/utils/grakn/type/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_library( diff --git a/kglib/utils/graph/query/BUILD b/kglib/utils/graph/query/BUILD index b632894a..e6307e75 100644 --- a/kglib/utils/graph/query/BUILD +++ b/kglib/utils/graph/query/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( @@ -18,8 +19,8 @@ py_library( 'query_graph.py', ], deps = [ - requirement('networkx'), - requirement('decorator'), + graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('decorator'), ], visibility=['//visibility:public'] ) \ No newline at end of file diff --git a/kglib/utils/graph/thing/BUILD b/kglib/utils/graph/thing/BUILD index e373255a..3f5fc393 100644 --- a/kglib/utils/graph/thing/BUILD +++ b/kglib/utils/graph/thing/BUILD @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_test", "py_library") -load("@pypi_dependencies//:requirements.bzl", "requirement") +load("@graknlabs_kglib_pip//:requirements.bzl", + graknlabs_kglib_requirement = "requirement") py_test( @@ -25,9 +26,8 @@ py_test( "//kglib/utils/graph/test", "@graknlabs_client_python//:client_python", ], - data = [ - "@graknlabs_grakn_core//:assemble-mac-zip", - ] + data = ["@graknlabs_grakn_core_artifact//file"], + args = ["$(location @graknlabs_grakn_core_artifact//file)"], ) @@ -48,8 +48,8 @@ py_library( 'queries_to_graph.py'], deps = [ "//kglib/utils/grakn/object", - requirement('networkx'), - requirement('decorator'), + graknlabs_kglib_requirement('networkx'), + graknlabs_kglib_requirement('decorator'), ], visibility=['//visibility:public'] ) \ No newline at end of file diff --git a/kglib/utils/graph/thing/concept_dict_to_graph.py b/kglib/utils/graph/thing/concept_dict_to_graph.py index 41c6112c..3d4dac1f 100644 --- a/kglib/utils/graph/thing/concept_dict_to_graph.py +++ b/kglib/utils/graph/thing/concept_dict_to_graph.py @@ -49,7 +49,7 @@ def concept_dict_to_graph(concept_dict, variable_graph): data = variable_graph.nodes[variable] data.update(type=thing.type_label) if thing.base_type_label == 'attribute': - data.update(datatype=thing.data_type, value=thing.value) + data.update(value_type=thing.value_type, value=thing.value) grakn_graph.add_node(thing, **data) diff --git a/kglib/utils/graph/thing/concept_dict_to_graph_test.py b/kglib/utils/graph/thing/concept_dict_to_graph_test.py index 20170c3c..895e91a8 100644 --- a/kglib/utils/graph/thing/concept_dict_to_graph_test.py +++ b/kglib/utils/graph/thing/concept_dict_to_graph_test.py @@ -43,12 +43,12 @@ def test_single_attribute_graph_is_as_expected(self): variable_graph = nx.MultiDiGraph() variable_graph.add_node('x') - name = Thing('V123', 'name', 'attribute', data_type='string', value='Bob') + name = Thing('V123', 'name', 'attribute', value_type='string', value='Bob') concept_dict = {'x': name} grakn_graph = concept_dict_to_graph(concept_dict, variable_graph) expected_grakn_graph = nx.MultiDiGraph() - expected_grakn_graph.add_node(name, type='name', datatype='string', value='Bob') + expected_grakn_graph.add_node(name, type='name', value_type='string', value='Bob') self.assertGraphsEqual(expected_grakn_graph, grakn_graph) @@ -132,7 +132,7 @@ def test_edge_starting_from_attribute_throws_exception(self): variable_graph.add_node('y') variable_graph.add_edge('x', 'y', type='employee') - name = Thing('V123', 'name', 'attribute', data_type='string', value='Bob') + name = Thing('V123', 'name', 'attribute', value_type='string', value='Bob') employment = Thing('V123', 'employment', 'relation') concept_dict = {'x': name, 'y': employment} diff --git a/kglib/utils/graph/thing/queries_to_graph_it.py b/kglib/utils/graph/thing/queries_to_graph_it.py index 1207201f..df8c68f5 100644 --- a/kglib/utils/graph/thing/queries_to_graph_it.py +++ b/kglib/utils/graph/thing/queries_to_graph_it.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. # +import sys import unittest import networkx as nx @@ -88,10 +89,10 @@ def test_graph_is_built_as_expected(self): person_exp = Thing('V123', 'person', 'entity') parentship_exp = Thing('V567', 'parentship', 'relation') - name_exp = Thing('V987', 'name', 'attribute', data_type='string', value='Bob') + name_exp = Thing('V987', 'name', 'attribute', value_type='string', value='Bob') expected_combined_graph = nx.MultiDiGraph() expected_combined_graph.add_node(person_exp, type='person') - expected_combined_graph.add_node(name_exp, type='name', datatype='string', value='Bob') + expected_combined_graph.add_node(name_exp, type='name', value_type='string', value='Bob') expected_combined_graph.add_node(parentship_exp, type='parentship') expected_combined_graph.add_edge(parentship_exp, person_exp, type='child') expected_combined_graph.add_edge(parentship_exp, person_exp, type='parent') @@ -144,7 +145,7 @@ class ITBuildGraphFromQueriesWithRealGrakn(GraphTestCase): KEYSPACE = "it_build_graph_from_queries" SCHEMA = ("define " "person sub entity, has name, plays parent, plays child;" - "name sub attribute, datatype string;" + "name sub attribute, value string;" "parentship sub relation, relates parent, relates child;") DATA = ('insert ' '$p isa person, has name "Bob";' @@ -200,7 +201,7 @@ def test_graph_is_built_from_grakn_as_expected(self): expected_combined_graph = nx.MultiDiGraph() expected_combined_graph.add_node(person_exp, type='person') - expected_combined_graph.add_node(name_exp, type='name', datatype='string', value='Bob') + expected_combined_graph.add_node(name_exp, type='name', value_type='string', value='Bob') expected_combined_graph.add_node(parentship_exp, type='parentship') expected_combined_graph.add_edge(parentship_exp, person_exp, type='child') expected_combined_graph.add_edge(parentship_exp, person_exp, type='parent') @@ -211,5 +212,5 @@ def test_graph_is_built_from_grakn_as_expected(self): if __name__ == "__main__": - with GraknServer() as gs: + with GraknServer(sys.argv.pop()) as gs: unittest.main() diff --git a/kglib/utils/graph/thing/queries_to_graph_test.py b/kglib/utils/graph/thing/queries_to_graph_test.py index f6e3b6c2..dd9d7531 100644 --- a/kglib/utils/graph/thing/queries_to_graph_test.py +++ b/kglib/utils/graph/thing/queries_to_graph_test.py @@ -65,7 +65,7 @@ def test_graph_combined_as_expected(self): grakn_graph_a.add_edge(employment, person, type='employee') person_b = Thing('V123', 'person', 'entity') - name = Thing('V1234', 'name', 'attribute', data_type='string', value='Bob') + name = Thing('V1234', 'name', 'attribute', value_type='string', value='Bob') grakn_graph_b = nx.MultiDiGraph() grakn_graph_b.add_node(person_b) grakn_graph_b.add_node(name) @@ -75,7 +75,7 @@ def test_graph_combined_as_expected(self): person_ex = Thing('V123', 'person', 'entity') employment_ex = Thing('V567', 'employment', 'relation') - name_ex = Thing('V1234', 'name', 'attribute', data_type='string', value='Bob') + name_ex = Thing('V1234', 'name', 'attribute', value_type='string', value='Bob') expected_combined_graph = nx.MultiDiGraph() expected_combined_graph.add_node(person_ex) expected_combined_graph.add_node(name_ex) @@ -87,14 +87,14 @@ def test_graph_combined_as_expected(self): def test_when_graph_node_properties_are_mismatched_exception_is_raised(self): person_a = Thing('V123', 'person', 'entity') - name_a = Thing('V1234', 'name', 'attribute', data_type='string', value='Bob') + name_a = Thing('V1234', 'name', 'attribute', value_type='string', value='Bob') grakn_graph_a = nx.MultiDiGraph(name='a') grakn_graph_a.add_node(person_a, input=1, solution=1) grakn_graph_a.add_node(name_a, input=1, solution=1) grakn_graph_a.add_edge(person_a, name_a, type='has', input=0, solution=1) person_b = Thing('V123', 'person', 'entity') - name_b = Thing('V1234', 'name', 'attribute', data_type='string', value='Bob') + name_b = Thing('V1234', 'name', 'attribute', value_type='string', value='Bob') grakn_graph_b = nx.MultiDiGraph(name='b') grakn_graph_b.add_node(person_b, input=1, solution=1) grakn_graph_b.add_node(name_b, input=0, solution=1) @@ -110,14 +110,14 @@ def test_when_graph_node_properties_are_mismatched_exception_is_raised(self): def test_when_graph_edge_properties_are_mismatched_exception_is_raised(self): person_a = Thing('V123', 'person', 'entity') - name_a = Thing('V1234', 'name', 'attribute', data_type='string', value='Bob') + name_a = Thing('V1234', 'name', 'attribute', value_type='string', value='Bob') grakn_graph_a = nx.MultiDiGraph(name='a') grakn_graph_a.add_node(person_a, input=1, solution=1) grakn_graph_a.add_node(name_a, input=1, solution=1) grakn_graph_a.add_edge(person_a, name_a, type='has', input=0, solution=1) person_b = Thing('V123', 'person', 'entity') - name_b = Thing('V1234', 'name', 'attribute', data_type='string', value='Bob') + name_b = Thing('V1234', 'name', 'attribute', value_type='string', value='Bob') grakn_graph_b = nx.MultiDiGraph(name='b') grakn_graph_b.add_node(person_b, input=1, solution=1) grakn_graph_b.add_node(name_b, input=1, solution=1) diff --git a/requirements.txt b/requirements.txt index 6267b321..15276136 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ future==0.17.1 gast==0.3.1 google-pasta==0.1.7 graph-nets==1.0.4 -grpcio==1.23.0 +grpcio==1.24.1,<2 h5py==2.10.0 Keras-Applications==1.0.8 Keras-Preprocessing==1.1.0 @@ -19,13 +19,13 @@ matplotlib==3.1.1 networkx==2.3 numpy==1.17.2 pandas==0.25.1 -protobuf==3.9.1 +protobuf==3.6.1 pyparsing==2.4.2 python-dateutil==2.8.0 pytz==2019.2 scipy==1.3.1 semantic-version==2.8.2 -six==1.12.0 +six>=1.11.0 tensorboard==1.14.0 tensorflow==1.14.0 tensorflow-estimator==1.14.0 diff --git a/tests/end_to_end/BUILD b/tests/end_to_end/BUILD index 49fe281e..f79c27dd 100644 --- a/tests/end_to_end/BUILD +++ b/tests/end_to_end/BUILD @@ -2,7 +2,7 @@ load("@rules_python//python:defs.bzl", "py_test") py_test( name = "diagnosis", - size = "large", + size = "enormous", srcs = [ "kgcn/diagnosis.py" ], @@ -11,7 +11,11 @@ py_test( "//kglib/utils/grakn/test", ], data = [ - "@graknlabs_grakn_core//:assemble-mac-zip", - "//kglib/utils/grakn/synthetic/examples:diagnosis/schema.gql" - ] + "//kglib/utils/grakn/synthetic/examples:graql-schema", + "@graknlabs_grakn_core_artifact//file" + ], + args = [ + "$(locations //kglib/utils/grakn/synthetic/examples:graql-schema)", + "$(location @graknlabs_grakn_core_artifact//file)", + ], ) diff --git a/tests/end_to_end/kgcn/diagnosis.py b/tests/end_to_end/kgcn/diagnosis.py index bc9f5a69..4237b1bf 100644 --- a/tests/end_to_end/kgcn/diagnosis.py +++ b/tests/end_to_end/kgcn/diagnosis.py @@ -17,6 +17,7 @@ # under the License. # +import sys import unittest from kglib.kgcn.examples.diagnosis.diagnosis import diagnosis_example @@ -28,9 +29,11 @@ class TestDiagnosisExample(unittest.TestCase): def setUp(self): - self._gs = GraknServer() + self._gs = GraknServer(sys.argv.pop()) self._gs.start() - self._gs.load_graql_file(TEST_KEYSPACE, '/kglib/kglib/utils/grakn/synthetic/examples/diagnosis/schema.gql') + schema_location = sys.argv.pop() + print("Schema location: " + str(schema_location)) + self._gs.load_graql_file(TEST_KEYSPACE, schema_location) def tearDown(self): self._gs.stop() @@ -44,4 +47,6 @@ def test_learning_is_done(self): if __name__ == "__main__": - unittest.main() + # This handles the fact that additional arguments that are supplied by our py_test definition + # https://stackoverflow.com/a/38012249 + unittest.main(argv=['ignored-arg'], exit=False)