From 42586d41e6b6ea29dc9383519955decaa6582e52 Mon Sep 17 00:00:00 2001 From: Neha Gupta <56321128+gupta-n@users.noreply.github.com> Date: Wed, 6 Nov 2019 16:28:51 -0800 Subject: [PATCH 1/7] exposed extra_context CLI option to pass cookiecutter parameter (#1500) * exposed extra_context CLI option to pass cookiecutter parameter * bug fix: Remove logic to set extra_context as None * added unit test when extra_context is invalid json * disable lint too-many-statements for do_cli * moved extra_context merging to private method * simplified json merge logic --- samcli/commands/init/__init__.py | 46 +++++++- tests/integration/init/test_init_command.py | 26 +++++ tests/unit/commands/init/test_cli.py | 112 +++++++++++++++++++- 3 files changed, 176 insertions(+), 8 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index db92f30765..23c4537fb9 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -3,9 +3,12 @@ Init command to scaffold a project app from a template """ import logging +import json +from json import JSONDecodeError import click +from samcli.commands.exceptions import UserException from samcli.cli.main import pass_context, common_options, global_cfg from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS from samcli.lib.telemetry.metrics import track_command @@ -86,12 +89,32 @@ default=False, help="Disable Cookiecutter prompting and accept default values defined template config", ) +@click.option( + "--extra_context", + default=None, + help="Override any custom parameters in the template's cookiecutter.json configuration e.g. " + "" + '{"customParam1": "customValue1", "customParam2":"customValue2"}' + """ """, + required=False, +) @common_options @pass_context @track_command -def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input): +def cli( + ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input, extra_context +): do_cli( - ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input + ctx, + no_interactive, + location, + runtime, + dependency_manager, + output_dir, + name, + app_template, + no_input, + extra_context, ) # pragma: no cover @@ -106,9 +129,9 @@ def do_cli( name, app_template, no_input, + extra_context, auto_clone=True, ): - from samcli.commands.exceptions import UserException from samcli.commands.init.init_generator import do_generate from samcli.commands.init.init_templates import InitTemplates from samcli.commands.init.interactive_init_flow import do_interactive @@ -126,12 +149,15 @@ def do_cli( # check for required parameters if location or (name and runtime and dependency_manager and app_template): # need to turn app_template into a location before we generate - extra_context = None if app_template: templates = InitTemplates(no_interactive, auto_clone) location = templates.location_from_app_template(runtime, dependency_manager, app_template) no_input = True - extra_context = {"project_name": name, "runtime": runtime} + default_context = {"project_name": name, "runtime": runtime} + if extra_context is None: + extra_context = default_context + else: + extra_context = _merge_extra_context(default_context, extra_context) if not output_dir: output_dir = "." do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) @@ -149,3 +175,13 @@ def do_cli( else: # proceed to interactive state machine, which will call do_generate do_interactive(location, runtime, dependency_manager, output_dir, name, app_template, no_input) + + +def _merge_extra_context(default_context, extra_context): + try: + extra_context_dict = json.loads(extra_context) + except JSONDecodeError: + raise UserException( + "Parse error reading the --extra-content parameter. The value of this parameter must be valid JSON." + ) + return {**extra_context_dict, **default_context} diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index 97e50a240b..9ba5418bc1 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -101,6 +101,32 @@ def test_init_command_java_gradle(self): self.assertEqual(return_code, 0) self.assertTrue(os.path.isdir(temp + "/sam-app-gradle")) + def test_init_command_with_extra_context_parameter(self): + with tempfile.TemporaryDirectory() as temp: + process = Popen( + [ + TestBasicInitCommand._get_command(), + "init", + "--runtime", + "java8", + "--dependency-manager", + "maven", + "--app-template", + "hello-world", + "--name", + "sam-app-maven", + "--no-interactive", + "--extra_context", + '{"schema_name": "codedeploy", "schema_type": "aws"}', + "-o", + temp, + ] + ) + return_code = process.wait() + + self.assertEqual(return_code, 0) + self.assertTrue(os.path.isdir(temp + "/sam-app-maven")) + @staticmethod def _get_command(): command = "sam" diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index 0e9ebc364e..72125793d5 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -1,7 +1,6 @@ from unittest import TestCase from unittest.mock import patch, ANY -import click from click.testing import CliRunner from samcli.commands.init import cli as init_cmd @@ -21,7 +20,8 @@ def setUp(self): self.name = "testing project" self.app_template = "hello-world" self.no_input = False - self.extra_context = {"project_name": "testing project", "runtime": "python3.6"} + self.extra_context = '{"project_name": "testing project", "runtime": "python3.6"}' + self.extra_context_as_json = {"project_name": "testing project", "runtime": "python3.6"} @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @patch("samcli.commands.init.init_generator.generate_project") @@ -38,6 +38,7 @@ def test_init_cli(self, generate_project_patch, sd_mock): name=self.name, app_template=self.app_template, no_input=self.no_input, + extra_context=None, auto_clone=False, ) @@ -50,7 +51,7 @@ def test_init_cli(self, generate_project_patch, sd_mock): self.output_dir, self.name, True, - self.extra_context, + self.extra_context_as_json, ) @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @@ -68,6 +69,7 @@ def test_init_fails_invalid_template(self, sd_mock): name=self.name, app_template="wrong-and-bad", no_input=self.no_input, + extra_context=None, auto_clone=False, ) @@ -86,6 +88,7 @@ def test_init_fails_invalid_dep_mgr(self, sd_mock): name=self.name, app_template=self.app_template, no_input=self.no_input, + extra_context=None, auto_clone=False, ) @@ -224,6 +227,7 @@ def test_init_cli_missing_params_fails(self): name=None, app_template=None, no_input=True, + extra_context=None, auto_clone=False, ) @@ -241,6 +245,7 @@ def test_init_cli_mutually_exclusive_params_fails(self): name=self.name, app_template="fails-anyways", no_input=self.no_input, + extra_context=None, auto_clone=False, ) @@ -266,9 +271,110 @@ def test_init_cli_generate_project_fails(self, generate_project_patch, sd_mock): name=self.name, app_template=None, no_input=self.no_input, + extra_context=None, auto_clone=False, ) generate_project_patch.assert_called_with( self.location, self.runtime, self.dependency_manager, self.output_dir, self.name, self.no_input ) + + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_with_extra_context_parameter_not_passed(self, generate_project_patch): + # GIVEN generate_project successfully created a project + # WHEN a project name has been passed + init_cli( + ctx=self.ctx, + no_interactive=self.no_interactive, + location=self.location, + runtime=self.runtime, + dependency_manager=self.dependency_manager, + output_dir=self.output_dir, + name=self.name, + app_template=self.app_template, + no_input=self.no_input, + extra_context=None, + auto_clone=False, + ) + + # THEN we should receive no errors + generate_project_patch.assert_called_once_with( + ANY, self.runtime, self.dependency_manager, ".", self.name, True, self.extra_context_as_json + ) + + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_with_extra_context_parameter_passed(self, generate_project_patch): + # GIVEN generate_project successfully created a project + # WHEN a project name has been passed + init_cli( + ctx=self.ctx, + no_interactive=self.no_interactive, + location=self.location, + runtime=self.runtime, + dependency_manager=self.dependency_manager, + output_dir=self.output_dir, + name=self.name, + app_template=self.app_template, + no_input=self.no_input, + extra_context='{"schema_name":"events", "schema_type":"aws"}', + auto_clone=False, + ) + + # THEN we should receive no errors + generate_project_patch.assert_called_once_with( + ANY, + self.runtime, + self.dependency_manager, + ".", + self.name, + True, + {"project_name": "testing project", "runtime": "python3.6", "schema_name": "events", "schema_type": "aws"}, + ) + + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_with_extra_context_not_overriding_default_parameter(self, generate_project_patch): + # GIVEN generate_project successfully created a project + # WHEN a project name has been passed + init_cli( + ctx=self.ctx, + no_interactive=self.no_interactive, + location=self.location, + runtime=self.runtime, + dependency_manager=self.dependency_manager, + output_dir=self.output_dir, + name=self.name, + app_template=self.app_template, + no_input=self.no_input, + extra_context='{"project_name": "my_project", "runtime": "java8", "schema_name":"events", "schema_type": "aws"}', + auto_clone=False, + ) + + # THEN we should receive no errors + generate_project_patch.assert_called_once_with( + ANY, + self.runtime, + self.dependency_manager, + ".", + self.name, + True, + {"project_name": "testing project", "runtime": "python3.6", "schema_name": "events", "schema_type": "aws"}, + ) + + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_with_extra_context_input_as_wrong_json_raises_exception(self, generate_project_patch): + # GIVEN generate_project successfully created a project + # WHEN a project name has been passed + with self.assertRaises(UserException): + init_cli( + ctx=self.ctx, + no_interactive=self.no_interactive, + location=self.location, + runtime=self.runtime, + dependency_manager=self.dependency_manager, + output_dir=self.output_dir, + name=self.name, + app_template=self.app_template, + no_input=self.no_input, + extra_context='{"project_name", "my_project", "runtime": "java8", "schema_name":"events", "schema_type": "aws"}', + auto_clone=False, + ) From 6f6a8cfcfd4aaf1f6b107b79ddbfeaf1a218f441 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+TheSriram@users.noreply.github.com> Date: Thu, 7 Nov 2019 16:21:46 -0800 Subject: [PATCH 2/7] fix: zip files were ignored by .gitignore (#1514) - `sam package` integration tests --- .../aws-elasticbeanstalk-applicationversion.yaml | 2 +- .../integration/testdata/package/sourcebundle.zip | Bin 0 -> 175 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/integration/testdata/package/sourcebundle.zip diff --git a/tests/integration/testdata/package/aws-elasticbeanstalk-applicationversion.yaml b/tests/integration/testdata/package/aws-elasticbeanstalk-applicationversion.yaml index dffdff756a..db8a972584 100644 --- a/tests/integration/testdata/package/aws-elasticbeanstalk-applicationversion.yaml +++ b/tests/integration/testdata/package/aws-elasticbeanstalk-applicationversion.yaml @@ -8,4 +8,4 @@ Resources: Properties: ApplicationName: "my app" Description: "my sample version" - SourceBundle: ./sample.zip \ No newline at end of file + SourceBundle: ./sourcebundle.zip diff --git a/tests/integration/testdata/package/sourcebundle.zip b/tests/integration/testdata/package/sourcebundle.zip new file mode 100644 index 0000000000000000000000000000000000000000..f273516a86e1d7bf6e8cd1954e04372825ed3881 GIT binary patch literal 175 zcmWIWW@Zs#-~htl!gPNIB*4iaz!2o>=;G_DmzxqA!NV|P+L72eAP)U|rkdODjP41~ zP!2C&pOe0<3<2Kk92rXrcJc%D^8#^zHzSiMGs0YCIZg&1m@N)V8bM4XTR7mh1bDNu Pfg~A$&>2XFfjA5R$%!MJ literal 0 HcmV?d00001 From 6327d3373f355656c07eae4eda8244768279629d Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Mon, 11 Nov 2019 14:22:17 -0600 Subject: [PATCH 3/7] fix: Move validation of timeout is an int further down the stack (#1515) * fix: Move validation of timeout is an int further down the stack * Fix casting floats to ints * pywin32 needs to be under 226 to work with virtualenv module --- requirements/base.txt | 2 + .../local/lib/sam_function_provider.py | 13 +--- samcli/local/lambdafn/config.py | 14 +++++ .../local/lib/test_sam_function_provider.py | 19 +----- tests/unit/local/lambdafn/test_config.py | 62 +++++++++++++++++++ 5 files changed, 81 insertions(+), 29 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 31fb7b0c94..f6ad4daebe 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -12,3 +12,5 @@ python-dateutil~=2.6, <2.8.1 requests==2.22.0 serverlessrepo==0.1.9 aws_lambda_builders==0.5.0 +# https://github.com/mhammond/pywin32/issues/1439 +pywin32 < 226; sys_platform == 'win32' \ No newline at end of file diff --git a/samcli/commands/local/lib/sam_function_provider.py b/samcli/commands/local/lib/sam_function_provider.py index 6bc09c9fb7..3485cac760 100644 --- a/samcli/commands/local/lib/sam_function_provider.py +++ b/samcli/commands/local/lib/sam_function_provider.py @@ -1,11 +1,9 @@ """ Class that provides functions from a given SAM template """ - -import ast import logging -from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, InvalidSamTemplateException +from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn from .exceptions import InvalidLayerReference from .provider import FunctionProvider, Function, LayerVersion from .sam_base_provider import SamBaseProvider @@ -123,18 +121,11 @@ def _convert_sam_function_resource(name, resource_properties, layers): LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri) - timeout = resource_properties.get("Timeout") - if isinstance(timeout, str): - try: - timeout = ast.literal_eval(timeout) - except ValueError: - raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(timeout)) - return Function( name=name, runtime=resource_properties.get("Runtime"), memory=resource_properties.get("MemorySize"), - timeout=timeout, + timeout=resource_properties.get("Timeout"), handler=resource_properties.get("Handler"), codeuri=codeuri, environment=resource_properties.get("Environment"), diff --git a/samcli/local/lambdafn/config.py b/samcli/local/lambdafn/config.py index e438ab9355..1d74285988 100644 --- a/samcli/local/lambdafn/config.py +++ b/samcli/local/lambdafn/config.py @@ -1,7 +1,9 @@ """ Lambda Function configuration data required by the runtime """ +import ast +from samcli.commands.local.cli_common.user_exceptions import InvalidSamTemplateException from .env_vars import EnvironmentVariables @@ -44,8 +46,20 @@ def __init__(self, name, runtime, handler, code_abs_path, layers, memory=None, t self.code_abs_path = code_abs_path self.layers = layers self.memory = memory or self._DEFAULT_MEMORY + self.timeout = timeout or self._DEFAULT_TIMEOUT_SECONDS + if not isinstance(self.timeout, int): + try: + self.timeout = ast.literal_eval(self.timeout) + + # Guard against float values like "3.2" + if not isinstance(self.timeout, int): + raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(self.timeout)) + + except (ValueError, SyntaxError, TypeError): + raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(self.timeout)) + if not env_vars: env_vars = EnvironmentVariables(self.memory, self.timeout, self.handler) diff --git a/tests/unit/commands/local/lib/test_sam_function_provider.py b/tests/unit/commands/local/lib/test_sam_function_provider.py index 97c057dba8..95c7479f4e 100644 --- a/tests/unit/commands/local/lib/test_sam_function_provider.py +++ b/tests/unit/commands/local/lib/test_sam_function_provider.py @@ -255,7 +255,7 @@ def test_must_convert(self): name="myname", runtime="myruntime", memory="mymemorysize", - timeout=30, + timeout="30", handler="myhandler", codeuri="/usr/local", environment="myenvironment", @@ -267,23 +267,6 @@ def test_must_convert(self): self.assertEqual(expected, result) - def test_must_fail_with_InvalidSamTemplateException(self): - - name = "myname" - properties = { - "CodeUri": "/usr/local", - "Runtime": "myruntime", - "MemorySize": "mymemorysize", - "Timeout": "timeout", - "Handler": "myhandler", - "Environment": "myenvironment", - "Role": "myrole", - "Layers": ["Layer1", "Layer2"], - } - - with self.assertRaises(InvalidSamTemplateException): - SamFunctionProvider._convert_sam_function_resource(name, properties, ["Layer1", "Layer2"]) - def test_must_skip_non_existent_properties(self): name = "myname" diff --git a/tests/unit/local/lambdafn/test_config.py b/tests/unit/local/lambdafn/test_config.py index 07b240459b..e617c20f77 100644 --- a/tests/unit/local/lambdafn/test_config.py +++ b/tests/unit/local/lambdafn/test_config.py @@ -1,7 +1,10 @@ from unittest import TestCase from unittest.mock import Mock +from parameterized import parameterized + from samcli.local.lambdafn.config import FunctionConfig +from samcli.commands.local.cli_common.user_exceptions import InvalidSamTemplateException class TestFunctionConfig(TestCase): @@ -59,3 +62,62 @@ def test_init_without_optional_values(self): self.assertEqual(config.env_vars.handler, self.handler) self.assertEqual(config.env_vars.memory, self.DEFAULT_MEMORY) self.assertEqual(config.env_vars.timeout, self.DEFAULT_TIMEOUT) + + def test_init_with_timeout_of_int_string(self): + config = FunctionConfig( + self.name, + self.runtime, + self.handler, + self.code_path, + self.layers, + memory=self.memory, + timeout="34", + env_vars=self.env_vars_mock, + ) + + self.assertEqual(config.name, self.name) + self.assertEqual(config.runtime, self.runtime) + self.assertEqual(config.handler, self.handler) + self.assertEqual(config.code_abs_path, self.code_path) + self.assertEqual(config.layers, self.layers) + self.assertEqual(config.memory, self.memory) + self.assertEqual(config.timeout, 34) + self.assertEqual(config.env_vars, self.env_vars_mock) + + self.assertEqual(self.env_vars_mock.handler, self.handler) + self.assertEqual(self.env_vars_mock.memory, self.memory) + self.assertEqual(self.env_vars_mock.timeout, 34) + + +class TestFunctionConfigInvalidTimeouts(TestCase): + def setUp(self): + self.name = "name" + self.runtime = "runtime" + self.handler = "handler" + self.code_path = "codepath" + self.memory = 1234 + self.env_vars_mock = Mock() + self.layers = ["layer1"] + + @parameterized.expand( + [ + ("none int string",), + ({"dictionary": "is not a string either"},), + ("/local/lambda/timeout",), + ("3.2",), + ("4.2",), + ("0.123",), + ] + ) + def test_init_with_invalid_timeout_values(self, timeout): + with self.assertRaises(InvalidSamTemplateException): + FunctionConfig( + self.name, + self.runtime, + self.handler, + self.code_path, + self.layers, + memory=self.memory, + timeout=timeout, + env_vars=self.env_vars_mock, + ) From 9f08f81f146b22316581731a46db1e6b26ae1f15 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Tue, 12 Nov 2019 16:19:31 -0600 Subject: [PATCH 4/7] chore: Simplify how timeouts are casted to an int (#1518) --- samcli/local/lambdafn/config.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/samcli/local/lambdafn/config.py b/samcli/local/lambdafn/config.py index 1d74285988..fe80cd0650 100644 --- a/samcli/local/lambdafn/config.py +++ b/samcli/local/lambdafn/config.py @@ -1,8 +1,6 @@ """ Lambda Function configuration data required by the runtime """ -import ast - from samcli.commands.local.cli_common.user_exceptions import InvalidSamTemplateException from .env_vars import EnvironmentVariables @@ -51,13 +49,9 @@ def __init__(self, name, runtime, handler, code_abs_path, layers, memory=None, t if not isinstance(self.timeout, int): try: - self.timeout = ast.literal_eval(self.timeout) - - # Guard against float values like "3.2" - if not isinstance(self.timeout, int): - raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(self.timeout)) + self.timeout = int(self.timeout) - except (ValueError, SyntaxError, TypeError): + except (ValueError, TypeError): raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(self.timeout)) if not env_vars: From 8ee1c7449529cb73a87ea62df1166b26312e64cc Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Wed, 13 Nov 2019 13:22:04 -0600 Subject: [PATCH 5/7] chore: fix java build and ruby appveyor builds (#1524) Split java and ruby builds into two functions, one for in process and one for container. When running java builds on Windows Server 2019, the in process builds where hanging, reason is known. Splitting the tests allows us to run the container builds on Windows Server 2019, which is required due to needing docker that spins up a linux instance. We can then run in-process builds in Visual Studio images, which works for java. This commit will also update the ruby builds on windows but instead of running both in process and container builds, we will only run container builds. When running ruby in-process builds on Visual Studio images, we still see failures (which is assumed to be the same as when they are run on Windows Server 2019 which is an ssl issue). This will at least allow us to verify container builds for ruby, which will give us confidence that things are working as expected. --- ... appveyor-windows-build-java-container.yml | 13 +--- appveyor-windows-build-java-inprocess.yml | 58 ++++++++++++++++++ appveyor-windows-build-ruby.yml | 9 +-- tests/integration/buildcmd/test_build_cmd.py | 59 ++++++++++++++----- 4 files changed, 105 insertions(+), 34 deletions(-) rename appveyor-windows-build-java.yml => appveyor-windows-build-java-container.yml (90%) create mode 100644 appveyor-windows-build-java-inprocess.yml diff --git a/appveyor-windows-build-java.yml b/appveyor-windows-build-java-container.yml similarity index 90% rename from appveyor-windows-build-java.yml rename to appveyor-windows-build-java-container.yml index ab3b93b399..ce758b679b 100644 --- a/appveyor-windows-build-java.yml +++ b/appveyor-windows-build-java-container.yml @@ -27,20 +27,12 @@ environment: HOMEDRIVE: 'C:' HOMEPATH: 'C:\Users\appveyor' - matrix: - - optional_gate: true - -matrix: - allow_failures: - - optional_gate: true - init: # Uncomment this for RDP - - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - ps: gcim Win32_Processor | % { "$($_.NumberOfLogicalProcessors) logical CPUs" } - ps: gcim Win32_OperatingSystem | % { "$([int]($_.TotalVisibleMemorySize/1mb)) Gb" } - install: # Make sure the temp directory exists for Python to use. @@ -83,7 +75,8 @@ test_script: # Reactivate virtualenv before running tests - "venv\\Scripts\\activate" - "docker system prune -a -f" - - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k TestBuildCommand_Java" + - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java_in_container" + # Uncomment for RDP # on_finish: diff --git a/appveyor-windows-build-java-inprocess.yml b/appveyor-windows-build-java-inprocess.yml new file mode 100644 index 0000000000..8432ee4218 --- /dev/null +++ b/appveyor-windows-build-java-inprocess.yml @@ -0,0 +1,58 @@ +version: 1.0.{build} +image: Visual Studio 2017 +build: off + +environment: + AWS_DEFAULT_REGION: us-east-1 + SAM_CLI_DEV: 1 + APPVEYOR_CI_OVERRIDE: 1 + GRADLE_OPTS: -Dorg.gradle.daemon=false + + # MSI Installers only use Py3.6.6. It is sufficient to test with this version here. + PYTHON_HOME: "C:\\Python36-x64" + PYTHON_SCRIPTS: "C:\\Python36-x64\\Scripts" + PYTHON_EXE: "C:\\Python36-x64\\python.exe" + PYTHON_VERSION: '3.6.8' + PYTHON_ARCH: '64' + +install: + + - "SET PATH=%PYTHON_HOME%;%PATH%" + - "echo %PYTHON_HOME%" + - "echo %PATH%" + - "python --version" + + # Upgrade setuptools, wheel and virtualenv + - "python -m pip install --upgrade setuptools wheel virtualenv" + + # Create new virtual environment with chosen python version and activate it + - "python -m virtualenv venv" + - "venv\\Scripts\\activate" + - "python --version" + + # Actually install SAM CLI's dependencies + - "pip install -e \".[dev]\"" + + # setup Java, Maven and Gradle + - "refreshenv" + - "choco install maven -y --force" + - "refreshenv" + - "choco install gradle -y --force" + - "refreshenv" + - "java -version" + - "gradle -v" + - "mvn --version" + + # setup Ruby dependencies + - "set PATH=C:\\Ruby25-x64\\bin;%PATH%" + - "gem --version" + - "gem install bundler -v 1.17.3" + - "bundler --version" + + # Echo final Path + - "echo %PATH%" + +test_script: + # Reactivate virtualenv before running tests + - "venv\\Scripts\\activate" + - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java_in_process" diff --git a/appveyor-windows-build-ruby.yml b/appveyor-windows-build-ruby.yml index 18bbf966ec..5c60406b0b 100644 --- a/appveyor-windows-build-ruby.yml +++ b/appveyor-windows-build-ruby.yml @@ -27,13 +27,6 @@ environment: HOMEDRIVE: 'C:' HOMEPATH: 'C:\Users\appveyor' - matrix: - - optional_gate: true - -matrix: - allow_failures: - - optional_gate: true - init: # Uncomment this for RDP - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) @@ -81,7 +74,7 @@ test_script: # Reactivate virtualenv before running tests - "venv\\Scripts\\activate" - "docker system prune -a -f" - - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k TestBuildCommand_RubyFunctions" + - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_ruby_in_container" # Uncomment for RDP # on_finish: diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 5bc05fe315..fd96ed5066 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -5,6 +5,7 @@ from unittest import skipIf from pathlib import Path from parameterized import parameterized +import pytest from .build_integ_base import BuildIntegBase from tests.testing_utils import IS_WINDOWS, RUNNING_ON_CI, CI_OVERRIDE @@ -201,8 +202,17 @@ class TestBuildCommand_RubyFunctions(BuildIntegBase): FUNCTION_LOGICAL_ID = "Function" - @parameterized.expand([("ruby2.5", False), ("ruby2.5", "use_container")]) - def test_with_default_gemfile(self, runtime, use_container): + @pytest.mark.flaky(reruns=3) + @pytest.mark.timeout(timeout=300, method="thread") + @parameterized.expand([("ruby2.5")]) + def test_building_ruby_in_container(self, runtime): + self._test_with_default_gemfile(runtime, "use_container") + + @parameterized.expand([("ruby2.5")]) + def test_building_ruby_in_process(self, runtime): + self._test_with_default_gemfile(runtime, False) + + def _test_with_default_gemfile(self, runtime, use_container): overrides = {"Runtime": runtime, "CodeUri": "Ruby", "Handler": "ignored"} cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides) @@ -277,21 +287,36 @@ class TestBuildCommand_Java(BuildIntegBase): WINDOWS_LINE_ENDING = b"\r\n" UNIX_LINE_ENDING = b"\n" + @pytest.mark.flaky(reruns=3) + @pytest.mark.timeout(timeout=300, method="thread") + @parameterized.expand( + [ + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ] + ) + def test_building_java_in_container(self, runtime, code_path, expected_files): + self._test_with_building_java(runtime, code_path, expected_files, "use_container") + @parameterized.expand( [ - ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, False), - ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, False), - ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, False), - ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, False), - ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, "use_container"), - ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, "use_container"), - ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, "use_container"), - ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, "use_container"), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), ] ) - def test_with_building_java(self, runtime, code_path, expected_files, use_container): + def test_building_java_in_process(self, runtime, code_path, expected_files): + self._test_with_building_java(runtime, code_path, expected_files, False) + + def _test_with_building_java(self, runtime, code_path, expected_files, use_container): overrides = {"Runtime": runtime, "CodeUri": code_path, "Handler": "aws.example.Hello::myHandler"} cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides) + cmdlist += ["--skip-pull-image"] if code_path == self.USING_GRADLEW_PATH and use_container and IS_WINDOWS: self._change_to_unix_line_ending(os.path.join(self.test_data_path, self.USING_GRADLEW_PATH, "gradlew")) @@ -313,12 +338,14 @@ def test_with_building_java(self, runtime, code_path, expected_files, use_contai ), ) - expected = "Hello World" - self._verify_invoke_built_function( - self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected - ) + # If we are testing in the container, invoke the function as well. Otherwise we cannot guarantee docker is on appveyor + if use_container: + expected = "Hello World" + self._verify_invoke_built_function( + self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected + ) - self.verify_docker_container_cleanedup(runtime) + self.verify_docker_container_cleanedup(runtime) def _verify_built_artifact(self, build_dir, function_logical_id, expected_files, expected_modules): From 412590f198ef9956c61d286ad370acbbea7de571 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Wed, 13 Nov 2019 15:41:22 -0600 Subject: [PATCH 6/7] fix: compare dictionaries instead of json strings for regression test (#1527) --- Makefile | 4 ++++ tests/regression/package/regression_package_base.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/Makefile b/Makefile index a59f87b19d..6d8a611af2 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,10 @@ func-test: @echo Telemetry Status: $(SAM_CLI_TELEMETRY) pytest --cov samcli.local --cov samcli.commands.local --cov-report term-missing tests/functional/commands/validate tests/functional/commands/cli/test_global_config.py +regres-test: + @echo Telemetry Status: $(SAM_CLI_TELEMETRY) + SAM_CLI_DEV=1 pytest tests/regression + smoke-test: # Smoke tests run in parallel SAM_CLI_DEV=1 pytest -n 4 tests/smoke diff --git a/tests/regression/package/regression_package_base.py b/tests/regression/package/regression_package_base.py index c893a52ba8..7f80dbb1ba 100644 --- a/tests/regression/package/regression_package_base.py +++ b/tests/regression/package/regression_package_base.py @@ -94,4 +94,8 @@ def regression_check(self, args): self.assertEqual(process.returncode, 0) output_aws = output_template_file_aws.read() + if "use_json" in args and args.get("use_json"): + output_sam = json.loads(output_sam) + output_aws = json.loads(output_aws) + self.assertEqual(output_sam, output_aws) From 52708db40a140990e608606c50c3217fd04cf0cd Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Thu, 14 Nov 2019 09:21:47 -0600 Subject: [PATCH 7/7] chore: Bump version to 0.31.1 (#1523) --- samcli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/__init__.py b/samcli/__init__.py index dc38712618..6d6932a165 100644 --- a/samcli/__init__.py +++ b/samcli/__init__.py @@ -2,4 +2,4 @@ SAM CLI version """ -__version__ = "0.31.0" +__version__ = "0.31.1"