From 16f5d255e0bd63a384b2200d2c5a1aa7560389a9 Mon Sep 17 00:00:00 2001 From: icywang86rui Date: Wed, 11 Jul 2018 12:26:58 -0700 Subject: [PATCH] Add Chainer 4.1.0 (#278) --- CHANGELOG.rst | 6 ++++++ README.rst | 4 ++-- setup.py | 2 +- src/sagemaker/chainer/defaults.py | 2 +- tests/conftest.py | 4 ++-- tests/integ/test_chainer_train.py | 29 +++++++++++++++++------------ 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b4bdb55bf..e004f4a2ea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ CHANGELOG ========= +1.6.0 +===== + +* feature: Add Chainer 4.1.0 support +* bug-fix: Use chainer_full_version fixture in Chainer integration tests + 1.5.4 ===== diff --git a/README.rst b/README.rst index bc257f7d79..cea4ed5940 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ You can install from source by cloning this repository and issuing a pip install git clone https://github.com/aws/sagemaker-python-sdk.git python setup.py sdist - pip install dist/sagemaker-1.5.4.tar.gz + pip install dist/sagemaker-1.6.0.tar.gz Supported Python versions ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -236,7 +236,7 @@ Chainer SageMaker Estimators With Chainer Estimators, you can train and host Chainer models on Amazon SageMaker. -Supported versions of Chainer: ``4.0.0`` +Supported versions of Chainer: ``4.0.0``, ``4.1.0``. You can visit the Chainer repository at https://github.com/chainer/chainer. diff --git a/setup.py b/setup.py index f43bb70d9b..1dcbb5db41 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(fname): setup(name="sagemaker", - version="1.5.4", + version="1.6.0", description="Open source library for training and deploying models on Amazon SageMaker.", packages=find_packages('src'), package_dir={'': 'src'}, diff --git a/src/sagemaker/chainer/defaults.py b/src/sagemaker/chainer/defaults.py index af40e1cfdf..954dbd5f87 100644 --- a/src/sagemaker/chainer/defaults.py +++ b/src/sagemaker/chainer/defaults.py @@ -12,4 +12,4 @@ # language governing permissions and limitations under the License. from __future__ import absolute_import -CHAINER_VERSION = '4.0.0' +CHAINER_VERSION = '4.1.0' diff --git a/tests/conftest.py b/tests/conftest.py index fc65d33c55..fc6b383d82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,7 +86,7 @@ def pytorch_version(request): return request.param -@pytest.fixture(scope='module', params=['4.0', '4.0.0']) +@pytest.fixture(scope='module', params=['4.0', '4.0.0', '4.1', '4.1.0']) def chainer_version(request): return request.param @@ -106,6 +106,6 @@ def pytorch_full_version(request): return request.param -@pytest.fixture(scope='module', params=['4.0.0']) +@pytest.fixture(scope='module', params=['4.0.0', '4.1.0']) def chainer_full_version(request): return request.param diff --git a/tests/integ/test_chainer_train.py b/tests/integ/test_chainer_train.py index 1032ddae1b..0055750c7c 100644 --- a/tests/integ/test_chainer_train.py +++ b/tests/integ/test_chainer_train.py @@ -18,6 +18,7 @@ import pytest import numpy +from sagemaker.chainer.defaults import CHAINER_VERSION from sagemaker.chainer.estimator import Chainer from sagemaker.chainer.model import ChainerModel from sagemaker.utils import sagemaker_timestamp @@ -26,25 +27,26 @@ @pytest.fixture(scope='module') -def chainer_training_job(sagemaker_session): - return _run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 1) +def chainer_training_job(sagemaker_session, chainer_full_version): + return _run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 1, chainer_full_version) -def test_distributed_cpu_training(sagemaker_session): - _run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 2) +def test_distributed_cpu_training(sagemaker_session, chainer_full_version): + _run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 2, chainer_full_version) -def test_distributed_gpu_training(sagemaker_session): - _run_mnist_training_job(sagemaker_session, "ml.p2.xlarge", 2) +def test_distributed_gpu_training(sagemaker_session, chainer_full_version): + _run_mnist_training_job(sagemaker_session, "ml.p2.xlarge", 2, chainer_full_version) -def test_training_with_additional_hyperparameters(sagemaker_session): +def test_training_with_additional_hyperparameters(sagemaker_session, chainer_full_version): with timeout(minutes=15): script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py') data_path = os.path.join(DATA_DIR, 'chainer_mnist') chainer = Chainer(entry_point=script_path, role='SageMakerRole', train_instance_count=1, train_instance_type="ml.c4.xlarge", + framework_version=chainer_full_version, sagemaker_session=sagemaker_session, hyperparameters={'epochs': 1}, use_mpi=True, num_processes=2, @@ -75,8 +77,7 @@ def test_deploy_model(chainer_training_job, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=chainer_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py') - model = ChainerModel(model_data, 'SageMakerRole', entry_point=script_path, - sagemaker_session=sagemaker_session) + model = ChainerModel(model_data, 'SageMakerRole', entry_point=script_path, sagemaker_session=sagemaker_session) predictor = model.deploy(1, "ml.m4.xlarge", endpoint_name=endpoint_name) _predict_and_assert(predictor) @@ -85,7 +86,8 @@ def test_async_fit(sagemaker_session): endpoint_name = 'test-chainer-attach-deploy-{}'.format(sagemaker_timestamp()) with timeout(minutes=5): - training_job_name = _run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 1, wait=False) + training_job_name = _run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 1, + chainer_full_version=CHAINER_VERSION, wait=False) print("Waiting to re-attach to the training job: %s" % training_job_name) time.sleep(20) @@ -97,12 +99,13 @@ def test_async_fit(sagemaker_session): _predict_and_assert(predictor) -def test_failed_training_job(sagemaker_session): +def test_failed_training_job(sagemaker_session, chainer_full_version): with timeout(minutes=15): script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'failure_script.py') data_path = os.path.join(DATA_DIR, 'chainer_mnist') chainer = Chainer(entry_point=script_path, role='SageMakerRole', + framework_version=chainer_full_version, train_instance_count=1, train_instance_type='ml.c4.xlarge', sagemaker_session=sagemaker_session) @@ -113,7 +116,8 @@ def test_failed_training_job(sagemaker_session): chainer.fit(train_input) -def _run_mnist_training_job(sagemaker_session, instance_type, instance_count, wait=True): +def _run_mnist_training_job(sagemaker_session, instance_type, instance_count, + chainer_full_version, wait=True): with timeout(minutes=15): script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py') if instance_type == 1 else \ @@ -122,6 +126,7 @@ def _run_mnist_training_job(sagemaker_session, instance_type, instance_count, wa data_path = os.path.join(DATA_DIR, 'chainer_mnist') chainer = Chainer(entry_point=script_path, role='SageMakerRole', + framework_version=chainer_full_version, train_instance_count=instance_count, train_instance_type=instance_type, sagemaker_session=sagemaker_session, hyperparameters={'epochs': 1})