diff --git a/.circleci/config.yml b/.circleci/config.yml index e81aa66a..0aab2c56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,9 +5,9 @@ commands: - run: name: Bazel - Install command: | - curl -OL https://github.com/bazelbuild/bazel/releases/download/0.20.0/bazel-0.20.0-installer-linux-x86_64.sh - chmod +x bazel-0.20.0-installer-linux-x86_64.sh - sudo ./bazel-0.20.0-installer-linux-x86_64.sh + curl -OL https://github.com/bazelbuild/bazel/releases/download/0.25.2/bazel-0.25.2-installer-linux-x86_64.sh + chmod +x bazel-0.25.2-installer-linux-x86_64.sh + sudo ./bazel-0.25.2-installer-linux-x86_64.sh jobs: test: @@ -23,20 +23,21 @@ jobs: - run: unzip grakn-core-all-mac-1.5.3.zip - run: nohup grakn-core-all-mac-1.5.3/grakn server start - run: cd grakn-core-all-mac-1.5.3 && ./grakn console -k test_schema -f ../kglib/kgcn/test_data/schema.gql - - run: bazel test //kglib/... --test_output=streamed --force_python PY3 --python_path $(which python) + - run: bazel test //kglib/... --test_output=streamed --python_version PY3 --python_path $(which python) - test-deploy-pip: + deploy-pip-snapshot: machine: true working_directory: ~/kglib steps: - checkout - bazel_install - - run: sudo apt-get update - run: pyenv install 3.6.3 - run: pyenv global 3.6.3 - - run: date +%s > VERSION - - run: cat VERSION - - run: bazel run //:deploy-pip -- test $PYPI_TEST_SERVER_USERNAME $PYPI_TEST_SERVER_PASSWORD + - run: | + echo -n "$(cat VERSION)-$CIRCLE_SHA1" > VERSION + export DEPLOY_PIP_USERNAME=$REPO_GRAKN_USERNAME + export DEPLOY_PIP_PASSWORD=$REPO_GRAKN_PASSWORD + bazel run //:deploy-pip -- snapshot end-to-end-test: machine: true @@ -47,7 +48,7 @@ jobs: - run: sudo apt-get update - run: pyenv install 3.6.3 - run: pyenv global 3.6.3 - - run: bazel test //examples:test_pypi_end_to_end_test --test_output=streamed --force_python PY3 --python_path $(which python) --spawn_strategy=standalone + - run: bazel test //examples:test_pypi_end_to_end_test --test_output=streamed --python_version PY3 --python_path $(which python) --spawn_strategy=standalone deploy-git: machine: true @@ -62,28 +63,30 @@ jobs: VERSION_TAG="v"$(cat "VERSION") curl -X POST --fail --data "{\"tag_name\": \"${VERSION_TAG}\",\"target_commitish\": \"${CIRCLE_SHA1}\",\"name\": \"Draft\",\"body\": \"\",\"draft\": true,\"prerelease\": false}" https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases?access_token=${GITHUB_DEPLOYMENT_TOKEN} - deploy-pip: + deploy-pip-release: machine: true working_directory: ~/kglib steps: - checkout - bazel_install - - run: sudo apt-get update - run: pyenv install 3.6.3 - run: pyenv global 3.6.3 - - run: bazel run //:deploy-pip -- pypi $PYPI_USERNAME $PYPI_PASSWORD + - run: | + export DEPLOY_PIP_USERNAME=$REPO_PYPI_USERNAME + export DEPLOY_PIP_PASSWORD=$REPO_PYPI_PASSWORD + bazel run //:deploy-pip -- release workflows: version: 2 kglib: jobs: - test - - test-deploy-pip: + - deploy-pip-snapshot: requires: - test - end-to-end-test: requires: - - test-deploy-pip + - deploy-pip-snapshot - approve-deploy-git: type: approval requires: @@ -91,10 +94,10 @@ workflows: - deploy-git: requires: - approve-deploy-git - - approve-deploy-pip: + - approve-deploy-pip-release: type: approval requires: - deploy-git - - deploy-pip: + - deploy-pip-release: requires: - - approve-deploy-pip \ No newline at end of file + - approve-deploy-pip-release \ No newline at end of file diff --git a/BUILD b/BUILD index 2806920a..db8ce316 100644 --- a/BUILD +++ b/BUILD @@ -2,14 +2,15 @@ exports_files(["requirements.txt"]) load("@io_bazel_rules_python//python:python.bzl", "py_library", "py_test") load("@pypi_dependencies//:requirements.bzl", "requirement") -load("@pypi_deployment_dependencies//:requirements.bzl", deployment_requirement = "requirement") +load("@graknlabs_bazel_distribution_pip//:requirements.bzl", deployment_requirement = "requirement") -load("@graknlabs_bazel_distribution//pip:rules.bzl", "deploy_pip") +load("@graknlabs_bazel_distribution//pip:rules.bzl", "assemble_pip", "deploy_pip") -deploy_pip( - name = "deploy-pip", - package_name = "grakn-kglib", +assemble_pip( + name = "assemble-pip", + target = "//kglib:kglib", version_file = "//:VERSION", + package_name = "grakn-kglib", classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", @@ -35,27 +36,13 @@ deploy_pip( 'grpcio==1.16.0', 'protobuf==3.6.1', 'six==1.11.0', 'enum34==1.1.6', 'twine==1.12.1', 'requests==2.21.0'], keywords = ["machine learning", "logical reasoning", "knowledege graph", "grakn", "database", "graph", "knowledgebase", "knowledge-engineering"], - deployment_properties = "//:deployment.properties", + description = "A Machine Learning Library for the Grakn knowledge graph.", long_description_file = "//:README.md", - deps = [ - deployment_requirement("twine"), - deployment_requirement("setuptools"), - deployment_requirement("wheel"), - deployment_requirement("requests"), - deployment_requirement("urllib3"), - deployment_requirement("chardet"), - deployment_requirement("certifi"), - deployment_requirement("idna"), - deployment_requirement("tqdm"), - deployment_requirement("requests_toolbelt"), - deployment_requirement("pkginfo"), - deployment_requirement("readme_renderer"), - deployment_requirement("pygments"), - deployment_requirement("docutils"), - deployment_requirement("bleach"), - deployment_requirement("webencodings"), - deployment_requirement("six"), - ], - target = "//kglib:kglib" +) + +deploy_pip( + name = "deploy-pip", + target = ":assemble-pip", + deployment_properties = "@graknlabs_build_tools//:deployment.properties", ) diff --git a/WORKSPACE b/WORKSPACE index c8c2e353..23700607 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,17 @@ + workspace( name = "kglib" ) + +######################################################################################################################## +# Load Bazel Rules +######################################################################################################################## + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +# We require loading bazel_rules from the graknlabs fork, not from bazel, since we've patched the python rules to work with TensorFlow git_repository( name = "io_bazel_rules_python", # Grakn python rules @@ -11,37 +19,232 @@ git_repository( commit = "4443fa25feac79b0e4c7c63ca84f87a1d6032f49", ) +## Only needed for PIP support: +load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip3_import") +pip_repositories() + + +######################################################################################################################## +# Load Bazel Distribution +######################################################################################################################## + +# Load Bazel Distribution here, since it is required for kglib and for grakn git_repository( name="graknlabs_bazel_distribution", remote="https://github.com/graknlabs/bazel-distribution", - commit="27c8bf9e5d9f9b11b2a70dc2697da196d69f799c" + commit="62a9a6343e9f2a1aeed7c935e9092c0fd1e8e8ac" ) -## Only needed for PIP support: -load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip3_import") -pip_repositories() +# --- Load the dependencies of graknlabs_bazel_distribution --- + +load("@graknlabs_bazel_distribution//github:dependencies.bzl", "github_dependencies_for_deployment") +github_dependencies_for_deployment() + +#load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "io_bazel_skydoc", + remote = "https://github.com/graknlabs/skydoc.git", + branch = "experimental-skydoc-allow-dep-on-bazel-tools", +) + +load("@io_bazel_skydoc//:setup.bzl", "skydoc_repositories") +skydoc_repositories() + +load("@io_bazel_rules_sass//:package.bzl", "rules_sass_dependencies") +rules_sass_dependencies() + +load("@build_bazel_rules_nodejs//:defs.bzl", "node_repositories") +node_repositories() + +load("@io_bazel_rules_sass//:defs.bzl", "sass_repositories") +sass_repositories() + +# Skip these graknlabs_bazel_distribution dependencies since they are already present +#git_repository( +# name = "io_bazel_rules_python", +# remote = "https://github.com/bazelbuild/rules_python.git", +# commit = "fdbb17a4118a1728d19e638a5291b4c4266ea5b8", +#) + +#load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip3_import") +#pip_repositories() -# Load PyPI dependencies for Python programs pip3_import( - name = "pypi_dependencies", - requirements = "//:requirements.txt", + name = "graknlabs_bazel_distribution_pip", + requirements = "@graknlabs_bazel_distribution//pip:requirements.txt", ) +load("@graknlabs_bazel_distribution_pip//:requirements.bzl", graknlabs_bazel_distribution_pip_install = "pip_install") +graknlabs_bazel_distribution_pip_install() + -load("@pypi_dependencies//:requirements.bzl", "pip_install") -pip_install() +######################################################################################################################## +# Load KGLIB's PyPi requirements +######################################################################################################################## +# Load PyPI dependencies for Python programs pip3_import( - name = "pypi_deployment_dependencies", - requirements = "@graknlabs_bazel_distribution//pip:requirements.txt" + name = "pypi_dependencies", + requirements = "//:requirements.txt", ) -load("@pypi_deployment_dependencies//:requirements.bzl", "pip_install") -pip_install() +load("@pypi_dependencies//:requirements.bzl", pip_install_kglib_requirements = "pip_install") +pip_install_kglib_requirements() -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +######################################################################################################################## +# Load the pre-loaded Animal Trade Grakn distribution +######################################################################################################################## http_file( name = "animaltrade_dist", urls = ["https://storage.googleapis.com/kglib/grakn-core-all-mac-animaltrade1.5.3.zip", # TODO How to update to the latest relase each time? ] -) \ No newline at end of file +) + + +######################################################################################################################## +# Load Grakn +######################################################################################################################## + +git_repository( + name="graknlabs_grakn_core", + remote="https://github.com/graknlabs/grakn", + commit="2845bb009876a74896bd479a7e49955c7fa1c7ca" +) + + +################################ +# Load Grakn Labs dependencies # +################################ + +load( + "@graknlabs_grakn_core//dependencies/graknlabs:dependencies.bzl", + "graknlabs_build_tools", + "graknlabs_graql", + "graknlabs_protocol", + "graknlabs_client_java", + "graknlabs_benchmark" +) +graknlabs_build_tools() +graknlabs_graql() +graknlabs_protocol() +graknlabs_client_java() +graknlabs_benchmark() + +# Skip since these are already present above +#load("@graknlabs_build_tools//distribution:dependencies.bzl", "graknlabs_bazel_distribution") +#graknlabs_bazel_distribution() + +########################### +# Load Bazel dependencies # +########################### + +load("@graknlabs_build_tools//bazel:dependencies.bzl", "bazel_common", "bazel_deps", "bazel_toolchain") +bazel_common() +bazel_deps() +bazel_toolchain() + + +################################# +# Load Build Tools dependencies # +################################# + +load("@graknlabs_build_tools//checkstyle:dependencies.bzl", "checkstyle_dependencies") +checkstyle_dependencies() + +load("@graknlabs_build_tools//sonarcloud:dependencies.bzl", "sonarcloud_dependencies") +sonarcloud_dependencies() + +load("@graknlabs_build_tools//bazel:dependencies.bzl", "bazel_rules_python") +bazel_rules_python() + +pip3_import( + name = "graknlabs_build_tools_ci_pip", + requirements = "@graknlabs_build_tools//ci:requirements.txt", +) + +load("@graknlabs_build_tools_ci_pip//:requirements.bzl", +graknlabs_build_tools_ci_pip_install = "pip_install") +graknlabs_build_tools_ci_pip_install() + + +##################################### +# Load Java dependencies from Maven # +##################################### + +load("@graknlabs_grakn_core//dependencies/maven:dependencies.bzl", "maven_dependencies") +maven_dependencies() + + +########################### +# Load Graql dependencies # +########################### + +# Load ANTLR dependencies for Bazel +load("@graknlabs_graql//dependencies/compilers:dependencies.bzl", "antlr_dependencies") +antlr_dependencies() + +# Load ANTLR dependencies for ANTLR programs +load("@rules_antlr//antlr:deps.bzl", "antlr_dependencies") +antlr_dependencies() + +load("@graknlabs_graql//dependencies/maven:dependencies.bzl", +graknlabs_graql_maven_dependencies = "maven_dependencies") +graknlabs_graql_maven_dependencies() + + +########################### +# Load Benchmark dependencies # +########################### +load("@graknlabs_benchmark//dependencies/maven:dependencies.bzl", +graknlabs_benchmark_maven_dependencies = "maven_dependencies") +graknlabs_benchmark_maven_dependencies() + + +####################################### +# Load compiler dependencies for GRPC # +####################################### + +load("@graknlabs_build_tools//grpc:dependencies.bzl", "grpc_dependencies") +grpc_dependencies() + +load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", +com_github_grpc_grpc_deps = "grpc_deps") +com_github_grpc_grpc_deps() + +load("@stackb_rules_proto//java:deps.bzl", "java_grpc_compile") +java_grpc_compile() + + +################################## +# Load Distribution dependencies # +################################## + +# Skip since these are already present above +# TODO: rename the macro we load here to deploy_github_dependencies +#load("@graknlabs_bazel_distribution//github:dependencies.bzl", "github_dependencies_for_deployment") +#github_dependencies_for_deployment() + +load("@graknlabs_build_tools//bazel:dependencies.bzl", "bazel_rules_docker") +bazel_rules_docker() + +load("@io_bazel_rules_docker//repositories:repositories.bzl", +bazel_rules_docker_repositories = "repositories") +bazel_rules_docker_repositories() + +load("@io_bazel_rules_docker//container:container.bzl", "container_pull") +container_pull( + name = "openjdk_image", + registry = "index.docker.io", + repository = "library/openjdk", + tag = "8" +) + +##################################### +# Load Bazel common workspace rules # +##################################### + +# TODO: Figure out why this cannot be loaded at earlier at the top of the file +load("@com_github_google_bazel_common//:workspace_defs.bzl", "google_common_workspace_rules") +google_common_workspace_rules() diff --git a/deployment.properties b/deployment.properties index 43d2e19c..7d3c1f9b 100644 --- a/deployment.properties +++ b/deployment.properties @@ -16,10 +16,5 @@ # along with this program. If not, see . # -github.repository=grakn -maven.repository-url.snapshot=http://maven.grakn.ai/nexus/content/repositories/snapshots/ -maven.repository-url.release=http://maven.grakn.ai/nexus/content/repositories/releases/ -pip.repository-url.pypi=https://upload.pypi.org/legacy/ -pip.repository-url.test=https://test.pypi.org/legacy/ -npm.repository-url=https://registry.npmjs.org/ -maven.packages=common,server,console,protocol,client-java \ No newline at end of file +repo.github.organisation=graknlabs +repo.github.repository=kglib \ No newline at end of file diff --git a/kglib/BUILD b/kglib/BUILD index f1ba9225..e7fc8e66 100644 --- a/kglib/BUILD +++ b/kglib/BUILD @@ -1,181 +1,11 @@ load("@io_bazel_rules_python//python:python.bzl", "py_library", "py_test") -load("@pypi_dependencies//:requirements.bzl", "requirement") - - -py_test( - name = "ordered_test", - srcs = [ - "kgcn/core/ingest/traverse/data/sample/ordered_test.py" - ], - deps = [ - "kglib" - ], -) - -py_test( - name = "random_sampling_test", - srcs = [ - "kgcn/core/ingest/traverse/data/sample/random_sampling_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "label_extraction_test", - srcs = [ - "kgcn/learn/attribute/label_extraction_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "report_test", - srcs = [ - "kgcn/learn/metrics/report_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "tf_hub_test", - srcs = [ - "kgcn/core/ingest/encode/tf_hub_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "schema_test", - srcs = [ - "kgcn/core/ingest/encode/schema_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "encode_test", - srcs = [ - "kgcn/core/ingest/encode/encode_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "boolean_test", - srcs = [ - "kgcn/core/ingest/encode/boolean_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "builder_test", - srcs = [ - "kgcn/core/ingest/traverse/data/context/builder_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "builder_IT", - srcs = [ - "kgcn/core/ingest/traverse/data/context/builder_IT.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "neighbour_test", - srcs = [ - "kgcn/core/ingest/traverse/data/context/neighbour_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "schema_traversal_test", - main = "traversal_test.py", - srcs = [ - "kgcn/core/ingest/traverse/schema/traversal_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "array_test", - srcs = [ - "kgcn/core/ingest/traverse/data/context/array_test.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "array_IT", - srcs = [ - "kgcn/core/ingest/traverse/data/context/array_IT.py" - ], - deps = [ - "kglib", - ] -) - -py_test( - name = "model_test", - srcs = [ - "kgcn/core/model_test.py" - ], - deps = [ - "kglib", - ] -) +load("@graknlabs_bazel_distribution_pip//:requirements.bzl", deployment_requirement = "requirement") py_library( name = "kglib", - srcs = glob(['__init__.py', 'kgcn/**/*.py']), + srcs = glob(['__init__.py']), deps = [ - # Grakn deps - requirement('grakn-client'), - requirement('grpcio'), - - # TensorFlow deps - requirement('tensorflow'), - requirement('numpy'), - requirement('protobuf'), - requirement('six'), - requirement('absl-py'), - requirement('keras_applications'), - requirement('keras_preprocessing'), - requirement('gast'), - requirement('astor'), - requirement('termcolor'), - - requirement('tensorflow-hub'), - requirement('scikit-learn'), - requirement('scipy'), + '//kglib/kgcn', ], visibility=['//visibility:public'] -) \ No newline at end of file +) diff --git a/kglib/kgcn/BUILD b/kglib/kgcn/BUILD new file mode 100644 index 00000000..1a9dc5ff --- /dev/null +++ b/kglib/kgcn/BUILD @@ -0,0 +1,181 @@ +load("@io_bazel_rules_python//python:python.bzl", "py_test", "py_library") +load("@pypi_dependencies//:requirements.bzl", "requirement") + + +py_test( + name = "ordered_test", + srcs = [ + "core/ingest/traverse/data/sample/ordered_test.py" + ], + deps = [ + "kgcn" + ], +) + +py_test( + name = "random_sampling_test", + srcs = [ + "core/ingest/traverse/data/sample/random_sampling_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "label_extraction_test", + srcs = [ + "learn/attribute/label_extraction_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "report_test", + srcs = [ + "learn/metrics/report_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "tf_hub_test", + srcs = [ + "core/ingest/encode/tf_hub_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "schema_test", + srcs = [ + "core/ingest/encode/schema_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "encode_test", + srcs = [ + "core/ingest/encode/encode_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "boolean_test", + srcs = [ + "core/ingest/encode/boolean_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "builder_test", + srcs = [ + "core/ingest/traverse/data/context/builder_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "builder_IT", + srcs = [ + "core/ingest/traverse/data/context/builder_IT.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "neighbour_test", + srcs = [ + "core/ingest/traverse/data/context/neighbour_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "schema_traversal_test", + main = "traversal_test.py", + srcs = [ + "core/ingest/traverse/schema/traversal_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "array_test", + srcs = [ + "core/ingest/traverse/data/context/array_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "array_IT", + srcs = [ + "core/ingest/traverse/data/context/array_IT.py" + ], + deps = [ + "kgcn", + ] +) + +py_test( + name = "model_test", + srcs = [ + "core/model_test.py" + ], + deps = [ + "kgcn", + ] +) + +py_library( + name = "kgcn", + srcs = glob(['**/*.py']), + deps = [ + # Grakn deps + requirement('grakn-client'), + requirement('grpcio'), + + # TensorFlow deps + requirement('tensorflow'), + requirement('numpy'), + requirement('protobuf'), + requirement('six'), + requirement('absl-py'), + requirement('keras_applications'), + requirement('keras_preprocessing'), + requirement('gast'), + requirement('astor'), + requirement('termcolor'), + + requirement('tensorflow-hub'), + requirement('scikit-learn'), + requirement('scipy'), + ], + visibility=['//visibility:public'] +) \ No newline at end of file