From 00345ede2b5cdf894b209bcf89a715ec4f8807a4 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Mon, 25 Nov 2019 15:50:36 -0800 Subject: [PATCH 1/7] feat: Initialize a project from non-cookiecutter github repo or zip file (#1595) --- samcli/commands/deploy/command.py | 4 +- samcli/commands/init/init_generator.py | 4 +- samcli/lib/utils/osutils.py | 73 ++++++++++++++ samcli/lib/utils/temp_file_utils.py | 21 ----- samcli/local/init/__init__.py | 15 ++- samcli/local/init/arbitrary_project.py | 94 +++++++++++++++++++ samcli/local/init/exceptions.py | 6 +- tests/integration/init/test_init_command.py | 72 ++++++++++---- tests/unit/lib/utils/test_file_utils.py | 2 +- .../unit/local/init/test_arbitrary_project.py | 53 +++++++++++ tests/unit/local/init/test_init.py | 33 ++++++- 11 files changed, 330 insertions(+), 47 deletions(-) create mode 100644 samcli/local/init/arbitrary_project.py create mode 100644 tests/unit/local/init/test_arbitrary_project.py diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index 23b37b24a5..b9d3aa5750 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -7,7 +7,7 @@ import click from click.types import FuncParamType -from samcli.lib.utils import temp_file_utils +from samcli.lib.utils import osutils from samcli.cli.cli_config_file import configuration_option, TomlProvider from samcli.cli.context import get_cmd_names from samcli.cli.main import pass_context, common_options, aws_creds_options @@ -256,7 +256,7 @@ def do_cli( confirm_changeset=changeset_decision if guided else confirm_changeset, ) - with temp_file_utils.tempfile_platform_independent() as output_template_file: + with osutils.tempfile_platform_independent() as output_template_file: with PackageContext( template_file=template_file, diff --git a/samcli/commands/init/init_generator.py b/samcli/commands/init/init_generator.py index dc6bbbea8b..dd2ceb63ff 100644 --- a/samcli/commands/init/init_generator.py +++ b/samcli/commands/init/init_generator.py @@ -5,11 +5,11 @@ from samcli.commands.exceptions import UserException from samcli.local.init import generate_project -from samcli.local.init.exceptions import GenerateProjectFailedError +from samcli.local.init.exceptions import GenerateProjectFailedError, ArbitraryProjectDownloadFailed def do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context): try: generate_project(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) - except GenerateProjectFailedError as e: + except (GenerateProjectFailedError, ArbitraryProjectDownloadFailed) as e: raise UserException(str(e)) diff --git a/samcli/lib/utils/osutils.py b/samcli/lib/utils/osutils.py index e22f3edbbb..6a75175ef5 100644 --- a/samcli/lib/utils/osutils.py +++ b/samcli/lib/utils/osutils.py @@ -6,10 +6,15 @@ import os import shutil import tempfile +import logging +import contextlib from contextlib import contextmanager +LOG = logging.getLogger(__name__) + + @contextmanager def mkdir_temp(mode=0o755): """ @@ -62,3 +67,71 @@ def stderr(): Byte stream of stderr """ return sys.stderr.buffer + + +def remove(path): + if path: + try: + os.remove(path) + except OSError: + pass + + +@contextlib.contextmanager +def tempfile_platform_independent(): + # NOTE(TheSriram): Setting delete=False is specific to windows. + # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile + _tempfile = tempfile.NamedTemporaryFile(delete=False) + try: + yield _tempfile + finally: + _tempfile.close() + remove(_tempfile.name) + + +# NOTE: Py3.8 or higher has a ``dir_exist_ok=True`` parameter to provide this functionality. +# This method can be removed if we stop supporting Py37 +def copytree(source, destination, ignore=None): + """ + Similar to shutil.copytree except that it removes the limitation that the destination directory should + be present. + :type source: str + :param source: + Path to the source folder to copy + :type destination: str + :param destination: + Path to destination folder + :type ignore: function + :param ignore: + A function that returns a set of file names to ignore, given a list of available file names. Similar to the + ``ignore`` property of ``shutils.copytree`` method + """ + + if not os.path.exists(destination): + os.makedirs(destination) + + try: + # Let's try to copy the directory metadata from source to destination + shutil.copystat(source, destination) + except OSError as ex: + # Can't copy file access times in Windows + LOG.debug("Unable to copy file access times from %s to %s", source, destination, exc_info=ex) + + names = os.listdir(source) + if ignore is not None: + ignored_names = ignore(source, names) + else: + ignored_names = set() + + for name in names: + # Skip ignored names + if name in ignored_names: + continue + + new_source = os.path.join(source, name) + new_destination = os.path.join(destination, name) + + if os.path.isdir(new_source): + copytree(new_source, new_destination, ignore=ignore) + else: + shutil.copy2(new_source, new_destination) diff --git a/samcli/lib/utils/temp_file_utils.py b/samcli/lib/utils/temp_file_utils.py index 20f094b024..82c9ddb832 100644 --- a/samcli/lib/utils/temp_file_utils.py +++ b/samcli/lib/utils/temp_file_utils.py @@ -2,25 +2,4 @@ Helper functions for temporary files """ import os -import contextlib import tempfile - - -def remove(path): - if path: - try: - os.remove(path) - except OSError: - pass - - -@contextlib.contextmanager -def tempfile_platform_independent(): - # NOTE(TheSriram): Setting delete=False is specific to windows. - # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile - _tempfile = tempfile.NamedTemporaryFile(delete=False) - try: - yield _tempfile - finally: - _tempfile.close() - remove(_tempfile.name) diff --git a/samcli/local/init/__init__.py b/samcli/local/init/__init__.py index 3227d498ec..d1aa765286 100644 --- a/samcli/local/init/__init__.py +++ b/samcli/local/init/__init__.py @@ -4,11 +4,14 @@ import itertools import logging -from cookiecutter.exceptions import CookiecutterException +from pathlib import Path + +from cookiecutter.exceptions import CookiecutterException, RepositoryNotFound from cookiecutter.main import cookiecutter from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING from samcli.local.init.exceptions import GenerateProjectFailedError +from .arbitrary_project import generate_non_cookiecutter_project LOG = logging.getLogger(__name__) @@ -76,5 +79,15 @@ def generate_project( try: LOG.debug("Baking a new template with cookiecutter with all parameters") cookiecutter(**params) + except RepositoryNotFound as e: + # cookiecutter.json is not found in the template. Let's just clone it directly without using cookiecutter + # and call it done. + LOG.debug( + "Unable to find cookiecutter.json in the project. Downloading it directly without treating " + "it as a cookiecutter template" + ) + project_output_dir = str(Path(output_dir, name)) if name else output_dir + generate_non_cookiecutter_project(location=params["template"], output_dir=project_output_dir) + except CookiecutterException as e: raise GenerateProjectFailedError(project=name, provider_error=e) diff --git a/samcli/local/init/arbitrary_project.py b/samcli/local/init/arbitrary_project.py new file mode 100644 index 0000000000..2b1d7288a7 --- /dev/null +++ b/samcli/local/init/arbitrary_project.py @@ -0,0 +1,94 @@ +""" +Initialize an arbitrary project +""" + +import functools +import logging + +from cookiecutter import repository +from cookiecutter import config + +from samcli.lib.utils import osutils +from .exceptions import ArbitraryProjectDownloadFailed + + +LOG = logging.getLogger(__name__) + + +def generate_non_cookiecutter_project(location, output_dir): + """ + Uses Cookiecutter APIs to download a project at given ``location`` to the ``output_dir``. + This does *not* run cookiecutter on the downloaded project. + + Parameters + ---------- + location : str + Path to where the project is. This supports all formats of location cookiecutter supports + (ex: zip, git, ssh, hg, local zipfile) + + NOTE: This value *cannot* be a local directory. We didn't see a value in simply copying the directory + contents to ``output_dir`` without any processing. + + output_dir : str + Directory where the project should be downloaded to + + Returns + ------- + str + Name of the directory where the project was downloaded to. + + Raises + ------ + cookiecutter.exception.CookiecutterException if download failed for some reason + """ + + LOG.debug("Downloading project from %s to %s", location, output_dir) + + # Don't prompt ever + no_input = True + + # Expand abbreviations in URL such as gh:awslabs/aws-sam-cli + location = repository.expand_abbreviations(location, config.BUILTIN_ABBREVIATIONS) + + # If this is a zip file, download and unzip into output directory + if repository.is_zip_file(location): + LOG.debug("%s location is a zip file", location) + download_fn = functools.partial( + repository.unzip, zip_uri=location, is_url=repository.is_repo_url(location), no_input=no_input + ) + + # Else, treat it as a git/hg/ssh URL and try to clone + elif repository.is_repo_url(location): + LOG.debug("%s location is a source control repository", location) + download_fn = functools.partial(repository.clone, repo_url=location, no_input=no_input) + + else: + raise ArbitraryProjectDownloadFailed(msg="Unsupported location {location}".format(location=location)) + + return _download_and_copy(download_fn, output_dir) + + +def _download_and_copy(download_fn, output_dir): + """ + Runs the download function to download files into a temporary directory and then copy the files over to + the ``output_dir`` + + Parameters + ---------- + download_fn : function + Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be + set to the temporary directory + + output_dir : str + Path to the directory where files will be copied to + + Returns + ------- + output_dir + """ + + with osutils.mkdir_temp() as tempdir: + downloaded_dir = download_fn(clone_to_dir=tempdir) + osutils.copytree(downloaded_dir, output_dir) + + return output_dir diff --git a/samcli/local/init/exceptions.py b/samcli/local/init/exceptions.py index 7452f0021a..5fde33aaaf 100644 --- a/samcli/local/init/exceptions.py +++ b/samcli/local/init/exceptions.py @@ -13,4 +13,8 @@ def __init__(self, **kwargs): class GenerateProjectFailedError(InitErrorException): - fmt = "An error occurred while generating this {project}: {provider_error}" + fmt = "An error occurred while generating this project {project}: {provider_error}" + + +class ArbitraryProjectDownloadFailed(InitErrorException): + fmt = "An error occurred when downloading this project: {msg}" diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index cb1499ab83..527599a78e 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -1,8 +1,12 @@ from unittest import TestCase +from parameterized import parameterized from subprocess import Popen, TimeoutExpired import os +import shutil import tempfile +from pathlib import Path + TIMEOUT = 300 @@ -11,7 +15,7 @@ def test_init_command_passes_and_dir_created(self): with tempfile.TemporaryDirectory() as temp: process = Popen( [ - TestBasicInitCommand._get_command(), + _get_command(), "init", "--runtime", "nodejs10.x", @@ -33,13 +37,13 @@ def test_init_command_passes_and_dir_created(self): raise self.assertEqual(process.returncode, 0) - self.assertTrue(os.path.isdir(temp + "/sam-app")) + self.assertTrue(Path(temp, "sam-app").is_dir()) def test_init_new_app_template(self): with tempfile.TemporaryDirectory() as temp: process = Popen( [ - TestBasicInitCommand._get_command(), + _get_command(), "init", "--runtime", "nodejs10.x", @@ -61,13 +65,13 @@ def test_init_new_app_template(self): raise self.assertEqual(process.returncode, 0) - self.assertTrue(os.path.isdir(temp + "/qs-scratch")) + self.assertTrue(Path(temp, "qs-scratch").is_dir()) def test_init_command_java_maven(self): with tempfile.TemporaryDirectory() as temp: process = Popen( [ - TestBasicInitCommand._get_command(), + _get_command(), "init", "--runtime", "java8", @@ -89,13 +93,13 @@ def test_init_command_java_maven(self): raise self.assertEqual(process.returncode, 0) - self.assertTrue(os.path.isdir(temp + "/sam-app-maven")) + self.assertTrue(Path(temp, "sam-app-maven").is_dir()) def test_init_command_java_gradle(self): with tempfile.TemporaryDirectory() as temp: process = Popen( [ - TestBasicInitCommand._get_command(), + _get_command(), "init", "--runtime", "java8", @@ -117,13 +121,13 @@ def test_init_command_java_gradle(self): raise self.assertEqual(process.returncode, 0) - self.assertTrue(os.path.isdir(temp + "/sam-app-gradle")) + self.assertTrue(Path(temp, "sam-app-gradle").is_dir()) def test_init_command_with_extra_context_parameter(self): with tempfile.TemporaryDirectory() as temp: process = Popen( [ - TestBasicInitCommand._get_command(), + _get_command(), "init", "--runtime", "java8", @@ -147,12 +151,48 @@ def test_init_command_with_extra_context_parameter(self): raise self.assertEqual(process.returncode, 0) - self.assertTrue(os.path.isdir(temp + "/sam-app-maven")) + self.assertTrue(Path(temp, "sam-app-maven").is_dir()) + + +class TestInitWithArbitraryProject(TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + zipdata_folder = Path(self.tempdir, "zipdata") + zipdata_folder.mkdir(parents=True) + Path(zipdata_folder, "test.txt").write_text("hello world") + + zip_path_no_extension = str(Path(self.tempdir, "myfile")) + + self.zip_path = shutil.make_archive(zip_path_no_extension, "zip", root_dir=self.tempdir, base_dir="zipdata") + + def tearDown(self): + shutil.rmtree(self.tempdir) + + @parameterized.expand([(None,), ("project_name",)]) + def test_arbitrary_project(self, project_name): + with tempfile.TemporaryDirectory() as temp: + args = [_get_command(), "init", "--location", self.zip_path, "-o", temp] + if project_name: + args.extend(["--name", project_name]) + + process = Popen(args) + try: + process.communicate(timeout=TIMEOUT) + except TimeoutExpired: + process.kill() + raise + + expected_output_folder = Path(temp, project_name) if project_name else Path(temp) + self.assertEqual(process.returncode, 0) + self.assertTrue(expected_output_folder.exists()) + self.assertEqual(os.listdir(str(expected_output_folder)), ["test.txt"]) + self.assertEqual(Path(expected_output_folder, "test.txt").read_text(), "hello world") + - @staticmethod - def _get_command(): - command = "sam" - if os.getenv("SAM_CLI_DEV"): - command = "samdev" +def _get_command(): + command = "sam" + if os.getenv("SAM_CLI_DEV"): + command = "samdev" - return command + return command diff --git a/tests/unit/lib/utils/test_file_utils.py b/tests/unit/lib/utils/test_file_utils.py index c26eb3c7de..7e677a7e67 100644 --- a/tests/unit/lib/utils/test_file_utils.py +++ b/tests/unit/lib/utils/test_file_utils.py @@ -2,7 +2,7 @@ import tempfile from unittest import TestCase -from samcli.lib.utils.temp_file_utils import remove, tempfile_platform_independent +from samcli.lib.utils.osutils import remove, tempfile_platform_independent class TestFile(TestCase): diff --git a/tests/unit/local/init/test_arbitrary_project.py b/tests/unit/local/init/test_arbitrary_project.py new file mode 100644 index 0000000000..cbcadb6058 --- /dev/null +++ b/tests/unit/local/init/test_arbitrary_project.py @@ -0,0 +1,53 @@ +""" +Test arbitrary project init +""" + +from unittest import TestCase +from unittest.mock import Mock, patch, ANY +from parameterized import parameterized + +from pathlib import Path + +from samcli.local.init.arbitrary_project import generate_non_cookiecutter_project, repository +from samcli.local.init.exceptions import ArbitraryProjectDownloadFailed + + +class TestGenerateNonCookieCutterProject(TestCase): + def setUp(self): + self.output_dir = "output_dir" + + def tearDown(self): + pass + + @parameterized.expand([("https://example.com/file.zip", True), ("/path/to/file.zip", False)]) + @patch("samcli.local.init.arbitrary_project.osutils") + def test_support_zip_files(self, location, is_url, osutils_mock): + + with patch.object(repository, "unzip") as unzip_mock: + unzip_mock.return_value = "unzipped_dir" + + generate_non_cookiecutter_project(location, self.output_dir) + + unzip_mock.assert_called_with(zip_uri=location, is_url=is_url, no_input=True, clone_to_dir=ANY) + + osutils_mock.copytree.assert_called_with("unzipped_dir", self.output_dir) + + @patch("samcli.local.init.arbitrary_project.osutils") + def test_support_source_control_repos(self, osutils_mock): + abbreviated_location = "gh:awslabs/aws-sam-cli" + location = "https://github.com/awslabs/aws-sam-cli.git" + + with patch.object(repository, "clone") as clone_mock: + clone_mock.return_value = "cloned_dir" + + generate_non_cookiecutter_project(abbreviated_location, self.output_dir) + + clone_mock.assert_called_with(repo_url=location, no_input=True, clone_to_dir=ANY) + + osutils_mock.copytree.assert_called_with("cloned_dir", self.output_dir) + + def test_must_fail_on_local_folders(self): + location = str(Path("my", "folder")) + + with self.assertRaises(ArbitraryProjectDownloadFailed): + generate_non_cookiecutter_project(location, self.output_dir) diff --git a/tests/unit/local/init/test_init.py b/tests/unit/local/init/test_init.py index a799640e94..a1aacc1c51 100644 --- a/tests/unit/local/init/test_init.py +++ b/tests/unit/local/init/test_init.py @@ -1,7 +1,9 @@ from unittest import TestCase from unittest.mock import patch -from cookiecutter.exceptions import CookiecutterException +from pathlib import Path +from cookiecutter.exceptions import CookiecutterException, RepositoryNotFound + from samcli.local.init import generate_project from samcli.local.init import GenerateProjectFailedError from samcli.local.init import RUNTIME_DEP_TEMPLATE_MAPPING @@ -12,7 +14,7 @@ def setUp(self): self.location = None self.runtime = "python3.6" self.dependency_manager = "pip" - self.output_dir = "." + self.output_dir = "mydir" self.name = "testing project" self.no_input = True self.extra_context = {"project_name": "testing project", "runtime": self.runtime} @@ -63,7 +65,7 @@ def test_init_error_with_non_compatible_dependency_manager(self): no_input=self.no_input, ) self.assertEqual( - "An error occurred while generating this " + "An error occurred while generating this project " "testing project: Lambda Runtime python3.6 " "does not support dependency manager: gradle", str(ctx.exception), @@ -111,3 +113,28 @@ def test_must_not_set_extra_content(self, cookiecutter_patch): # THEN we should receive no errors cookiecutter_patch.assert_called_once_with(template=custom_location, no_input=False, output_dir=self.output_dir) + + @patch("samcli.local.init.cookiecutter") + @patch("samcli.local.init.generate_non_cookiecutter_project") + def test_init_arbitrary_project_with_location_is_not_cookiecutter( + self, generate_non_cookiecutter_project_mock, cookiecutter_mock + ): + + cookiecutter_mock.side_effect = RepositoryNotFound("msg") + + generate_project(location=self.location, output_dir=self.output_dir) + + generate_non_cookiecutter_project_mock.assert_called_with(location=self.location, output_dir=self.output_dir) + + @patch("samcli.local.init.cookiecutter") + @patch("samcli.local.init.generate_non_cookiecutter_project") + def test_init_arbitrary_project_with_named_folder(self, generate_non_cookiecutter_project_mock, cookiecutter_mock): + + cookiecutter_mock.side_effect = RepositoryNotFound("msg") + + generate_project(location=self.location, output_dir=self.output_dir, name=self.name) + + expected_output_dir = str(Path(self.output_dir, self.name)) + generate_non_cookiecutter_project_mock.assert_called_with( + location=self.location, output_dir=expected_output_dir + ) From 32b048e4379e22264a78518ede99b4bc7a54f8c1 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Mon, 25 Nov 2019 15:57:47 -0800 Subject: [PATCH 2/7] chore: version bump to 0.34.0 (#1597) --- samcli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/__init__.py b/samcli/__init__.py index f89d1e7af6..c73ff0022a 100644 --- a/samcli/__init__.py +++ b/samcli/__init__.py @@ -2,4 +2,4 @@ SAM CLI version """ -__version__ = "0.33.1" +__version__ = "0.34.0" From da5cf775c182ac28b6318c6bd838bbe75833d396 Mon Sep 17 00:00:00 2001 From: Yoshiaki Yoshida Date: Wed, 27 Nov 2019 01:50:49 +0900 Subject: [PATCH 3/7] Fixed typo of "sam deploy" option (#1601) --- samcli/commands/_utils/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/_utils/options.py b/samcli/commands/_utils/options.py index 72973ad43e..d29e2863f2 100644 --- a/samcli/commands/_utils/options.py +++ b/samcli/commands/_utils/options.py @@ -72,7 +72,7 @@ def guided_deploy_stack_name(ctx, param, provided_value): raise click.BadOptionUsage( option_name=param.name, ctx=ctx, - message="Missing option '--stack-name', 'sam deploy –guided' can " + message="Missing option '--stack-name', 'sam deploy --guided' can " "be used to provide and save needed parameters for future deploys.", ) From 7d2728a30b136fea3e4d4bf3c35dbe6e7d1c3760 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Tue, 26 Nov 2019 09:57:54 -0800 Subject: [PATCH 4/7] fix: Ignore .git when init from arbitrary github repo (#1599) --- samcli/local/init/arbitrary_project.py | 4 +++- tests/unit/local/init/test_arbitrary_project.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/samcli/local/init/arbitrary_project.py b/samcli/local/init/arbitrary_project.py index 2b1d7288a7..5f450a4afd 100644 --- a/samcli/local/init/arbitrary_project.py +++ b/samcli/local/init/arbitrary_project.py @@ -3,8 +3,10 @@ """ import functools +import shutil import logging +from pathlib import Path from cookiecutter import repository from cookiecutter import config @@ -89,6 +91,6 @@ def _download_and_copy(download_fn, output_dir): with osutils.mkdir_temp() as tempdir: downloaded_dir = download_fn(clone_to_dir=tempdir) - osutils.copytree(downloaded_dir, output_dir) + osutils.copytree(downloaded_dir, output_dir, ignore=shutil.ignore_patterns("*.git")) return output_dir diff --git a/tests/unit/local/init/test_arbitrary_project.py b/tests/unit/local/init/test_arbitrary_project.py index cbcadb6058..b4472bd870 100644 --- a/tests/unit/local/init/test_arbitrary_project.py +++ b/tests/unit/local/init/test_arbitrary_project.py @@ -30,7 +30,7 @@ def test_support_zip_files(self, location, is_url, osutils_mock): unzip_mock.assert_called_with(zip_uri=location, is_url=is_url, no_input=True, clone_to_dir=ANY) - osutils_mock.copytree.assert_called_with("unzipped_dir", self.output_dir) + osutils_mock.copytree.assert_called_with("unzipped_dir", self.output_dir, ignore=ANY) @patch("samcli.local.init.arbitrary_project.osutils") def test_support_source_control_repos(self, osutils_mock): @@ -44,7 +44,7 @@ def test_support_source_control_repos(self, osutils_mock): clone_mock.assert_called_with(repo_url=location, no_input=True, clone_to_dir=ANY) - osutils_mock.copytree.assert_called_with("cloned_dir", self.output_dir) + osutils_mock.copytree.assert_called_with("cloned_dir", self.output_dir, ignore=ANY) def test_must_fail_on_local_folders(self): location = str(Path("my", "folder")) From d8b84bb45bc7213080a8c43deeadbe4429aea809 Mon Sep 17 00:00:00 2001 From: Shreya Date: Tue, 26 Nov 2019 10:34:40 -0800 Subject: [PATCH 5/7] feat: Upgrade SAM Translator to 1.16.0 (#1596) --- requirements/base.txt | 2 +- ...l => function_with_event_source_mapping.yaml} | 16 ++++++++++++++-- .../validate/lib/test_sam_template_validator.py | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) rename tests/functional/commands/validate/lib/models/{function_with_batch_window.yaml => function_with_event_source_mapping.yaml} (79%) diff --git a/requirements/base.txt b/requirements/base.txt index fd6e4eef24..25e8fa332c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,7 +5,7 @@ boto3~=1.9, >=1.9.56 jmespath~=0.9.4 PyYAML~=5.1 cookiecutter~=1.6.0 -aws-sam-translator==1.15.1 +aws-sam-translator==1.16.0 docker~=4.0 dateparser~=0.7 python-dateutil~=2.6, <2.8.1 diff --git a/tests/functional/commands/validate/lib/models/function_with_batch_window.yaml b/tests/functional/commands/validate/lib/models/function_with_event_source_mapping.yaml similarity index 79% rename from tests/functional/commands/validate/lib/models/function_with_batch_window.yaml rename to tests/functional/commands/validate/lib/models/function_with_event_source_mapping.yaml index 606734458b..7f743f4f4c 100644 --- a/tests/functional/commands/validate/lib/models/function_with_batch_window.yaml +++ b/tests/functional/commands/validate/lib/models/function_with_event_source_mapping.yaml @@ -1,6 +1,5 @@ AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 -Description: EventSourceMapping example with MaximumBatchingWindowInSeconds property Parameters: MyBatchingWindowParam: @@ -23,6 +22,9 @@ Resources: } } Runtime: nodejs8.10 + Policies: + - SQSSendMessagePolicy: + QueueName: !GetAtt MySqsQueue.QueueName Events: Stream: Type: Kinesis @@ -42,7 +44,14 @@ Resources: Stream: !GetAtt DynamoDBTable.StreamArn BatchSize: 100 MaximumBatchingWindowInSeconds: !Ref MyBatchingWindowParam + ParallelizationFactor: 8 + MaximumRetryAttempts: 100 + BisectBatchOnFunctionError: true + MaximumRecordAgeInSeconds: 86400 StartingPosition: TRIM_HORIZON + DestinationConfig: + OnFailure: + Destination: !GetAtt MySqsQueue.Arn KinesisStream: Type: AWS::Kinesis::Stream @@ -66,4 +75,7 @@ Resources: ReadCapacityUnits: 5 WriteCapacityUnits: 5 StreamSpecification: - StreamViewType: NEW_IMAGE \ No newline at end of file + StreamViewType: NEW_IMAGE + + MySqsQueue: + Type: AWS::SQS::Queue \ No newline at end of file diff --git a/tests/functional/commands/validate/lib/test_sam_template_validator.py b/tests/functional/commands/validate/lib/test_sam_template_validator.py index a9610e46e8..284cc754a3 100644 --- a/tests/functional/commands/validate/lib/test_sam_template_validator.py +++ b/tests/functional/commands/validate/lib/test_sam_template_validator.py @@ -90,7 +90,6 @@ class TestValidate(TestCase): ("tests/functional/commands/validate/lib/models/function_with_alias.yaml"), ("tests/functional/commands/validate/lib/models/function_with_alias_and_event_sources.yaml"), ("tests/functional/commands/validate/lib/models/function_with_alias_intrinsics.yaml"), - ("tests/functional/commands/validate/lib/models/function_with_batch_window.yaml"), ("tests/functional/commands/validate/lib/models/function_with_condition.yaml"), ("tests/functional/commands/validate/lib/models/function_with_conditional_managed_policy.yaml"), ( @@ -113,6 +112,7 @@ class TestValidate(TestCase): ), ("tests/functional/commands/validate/lib/models/function_with_disabled_deployment_preference.yaml"), ("tests/functional/commands/validate/lib/models/function_with_dlq.yaml"), + ("tests/functional/commands/validate/lib/models/function_with_event_source_mapping.yaml"), ("tests/functional/commands/validate/lib/models/function_with_global_layers.yaml"), ("tests/functional/commands/validate/lib/models/function_with_kmskeyarn.yaml"), ("tests/functional/commands/validate/lib/models/function_with_layers.yaml"), From c2cdfab35deb24f1a1408188d5558c27a704a1e1 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Tue, 26 Nov 2019 14:08:41 -0600 Subject: [PATCH 6/7] fix: Add --no-fail-on-empty-changeset back (#1607) --- samcli/commands/deploy/command.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index b9d3aa5750..14dc274296 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -113,7 +113,8 @@ "executing the change set.", ) @click.option( - "--fail-on-empty-changeset", + "--fail-on-empty-changeset/--no-fail-on-empty-changeset", + default=True, required=False, is_flag=True, help="Specify if the CLI should return a non-zero exit code if there are no" From 1c7ea728722cd976774ba05ce41df8e67912bc03 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Tue, 26 Nov 2019 12:10:41 -0800 Subject: [PATCH 7/7] fix: pretty error msg when init location is invalid (#1606) * Pretty error msg when init location is invalid * stringify path for windows --- samcli/local/init/arbitrary_project.py | 19 +++++++++++++++++-- samcli/local/init/exceptions.py | 2 +- tests/integration/init/test_init_command.py | 13 +++++++++++++ .../unit/local/init/test_arbitrary_project.py | 10 ++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/samcli/local/init/arbitrary_project.py b/samcli/local/init/arbitrary_project.py index 5f450a4afd..eaeb13aed0 100644 --- a/samcli/local/init/arbitrary_project.py +++ b/samcli/local/init/arbitrary_project.py @@ -8,6 +8,7 @@ from pathlib import Path from cookiecutter import repository +from cookiecutter import exceptions from cookiecutter import config from samcli.lib.utils import osutils @@ -17,6 +18,16 @@ LOG = logging.getLogger(__name__) +BAD_LOCATION_ERROR_MSG = ( + "Please verify your location. The following types of location are supported:" + "\n\n* Github: gh:user/repo (or) https://github.com/user/repo (or) git@github.com:user/repo.git" + "\n For Git repositories, you must use location of the root of the repository." + "\n\n* Mercurial: hg+ssh://hg@bitbucket.org/repo" + "\n\n* Http(s): https://example.com/code.zip" + "\n\n* Local Path: /path/to/code.zip" +) + + def generate_non_cookiecutter_project(location, output_dir): """ Uses Cookiecutter APIs to download a project at given ``location`` to the ``output_dir``. @@ -65,9 +76,13 @@ def generate_non_cookiecutter_project(location, output_dir): download_fn = functools.partial(repository.clone, repo_url=location, no_input=no_input) else: - raise ArbitraryProjectDownloadFailed(msg="Unsupported location {location}".format(location=location)) + raise ArbitraryProjectDownloadFailed(msg=BAD_LOCATION_ERROR_MSG) - return _download_and_copy(download_fn, output_dir) + try: + return _download_and_copy(download_fn, output_dir) + except exceptions.RepositoryNotFound: + # Download failed because the zip or the repository was not found + raise ArbitraryProjectDownloadFailed(msg=BAD_LOCATION_ERROR_MSG) def _download_and_copy(download_fn, output_dir): diff --git a/samcli/local/init/exceptions.py b/samcli/local/init/exceptions.py index 5fde33aaaf..43c4c5aab7 100644 --- a/samcli/local/init/exceptions.py +++ b/samcli/local/init/exceptions.py @@ -17,4 +17,4 @@ class GenerateProjectFailedError(InitErrorException): class ArbitraryProjectDownloadFailed(InitErrorException): - fmt = "An error occurred when downloading this project: {msg}" + fmt = "{msg}" diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index 527599a78e..641fbc58b1 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -189,6 +189,19 @@ def test_arbitrary_project(self, project_name): self.assertEqual(os.listdir(str(expected_output_folder)), ["test.txt"]) self.assertEqual(Path(expected_output_folder, "test.txt").read_text(), "hello world") + def test_zip_not_exists(self): + with tempfile.TemporaryDirectory() as temp: + args = [_get_command(), "init", "--location", str(Path("invalid", "zip", "path")), "-o", temp] + + process = Popen(args) + try: + process.communicate(timeout=TIMEOUT) + except TimeoutExpired: + process.kill() + raise + + self.assertEqual(process.returncode, 1) + def _get_command(): command = "sam" diff --git a/tests/unit/local/init/test_arbitrary_project.py b/tests/unit/local/init/test_arbitrary_project.py index b4472bd870..87b1b2a8a6 100644 --- a/tests/unit/local/init/test_arbitrary_project.py +++ b/tests/unit/local/init/test_arbitrary_project.py @@ -8,6 +8,7 @@ from pathlib import Path +from cookiecutter.exceptions import RepositoryNotFound from samcli.local.init.arbitrary_project import generate_non_cookiecutter_project, repository from samcli.local.init.exceptions import ArbitraryProjectDownloadFailed @@ -51,3 +52,12 @@ def test_must_fail_on_local_folders(self): with self.assertRaises(ArbitraryProjectDownloadFailed): generate_non_cookiecutter_project(location, self.output_dir) + + def test_must_fail_when_repo_not_found(self): + location = str(Path("my", "folder")) + + with patch.object(repository, "unzip") as unzip_mock: + unzip_mock.side_effect = RepositoryNotFound("repo") + + with self.assertRaises(ArbitraryProjectDownloadFailed): + generate_non_cookiecutter_project(location, self.output_dir)