From 5f7c294f8ed8efe147d2f5a8db7f514afdf161f9 Mon Sep 17 00:00:00 2001 From: Travis Wichert Date: Thu, 11 Jan 2018 21:54:22 -0500 Subject: [PATCH] Release 2.6.0 (#404) * Update ECS CE Docker image with ECS 3.0.0 Hotfix 2 (#312) * Reorg base files (fix my glitch) * Fixed Dockerfile so it patches everything in-place. (cherry picked from commit 867189eb92e36c8388d389273f2942b9f277380e) * update release files to use new image. * Duct tape for #301 Update to ECS 3.0.0.2 (3.0.0 HF 2) (#314) * fix a small typo * fix package install issue because EPEL is between versions again. * quick fixes for crashing HF2 to unblock clients * Docs update (#313) * Adds FAQ page * installation troubleshooting * formatting * Addition of network troubleshooting * Addition of network troubleshooting * More Troubleshooting * sidebar implementation attempt * implements important links dropdown, disables page dropdown. * whoops * Adds migration page and some small updates. * Bunch of docs updates * bugfix-hf2 (#315) * multitail and dstat are coming in handy right now * change fact cache location to /var/cache/emc/ecs-install * log the state of Docker at the end of bootstrapping for troubleshooting help (those hashes are good to see!) * [Ansible] Stop templating, start regex replacing props/confs * release prep 2.3.0 (#316) * OVA prep (#318) * Configure Jenkins pipeline to test installation process * Get repository information from Jenkins SCM config * Fix env var * Obtain TF options from Jenkins params. Moved deprovision step to post action * Allow multi-node configuration * Update checkout step in jenkinsfile * add zerofill.sh to /tools (#324) (cherry picked from commit 58f7e8eea587c5a005eff3df2c2733ac2f5e1a9c) * Add slack notifications to jenkins pipeline * [WiP] Configure Jenkins server to build PR and provide feedback (#328) Configure Jenkins server to build PR and provide feedback * Docs pass 2 (#326) * Removal of deprecated procedures * Templates * OVA install guide added * Fixed broken links * preflight remove all bootstrap packages if installed (#330) * Non-PR Jenkins jobs do not provide URL and commit author (#331) Fix variables in Slack notifications * put yum actions in retry loop with timeout (#332) Put yum actions in retry loop with timeout * Implements #205 Installer must have public key initial auth capability (#270) * ECS-CommunityEdition-205 Installer must have public key initial auth capability (cherry picked from commit 6eea10b5db3985f960d7b313d2e705a0f913ba55) * More sausage for the initial ssh key auth (cherry picked from commit 8535ccb5430e89b79d253ea1e74390a39b8b20f3) * more sausage (cherry picked from commit edf961e0765cd9a06ccea1a6d1e2406816533f46) * deploy.yml change ideas (cherry picked from commit ef48e2cc57fa6d0a57aa30bc62ba816fc167aed9) * bootstrap.sh modifications (cherry picked from commit d0b3c630f0a2004fe23534aca7e4a95986dce383) * bootstrap.sh modifications (cherry picked from commit 86f897af9395a57af5b33c2162cd85422b4e6ded) * move generic help to generic_help.j2.yml file from config.yml * include shipit.lib.sh * build install paths early add copy action for ssh PKI material * fix a couple gitopt bugs * add create_install_tree() to plugin-defaults.sh * copy ssh keys in bootstrap.sh * more longopts adjustments * add loop delay in retry_with_timeout() * stop trying to autoremove curl, it'll always error. * key_vals need basename not full path set 0700 bits on ssh/ssl stores * remove optarg debugging * more ssh pubkey sausage * update reference.deploy.yml to include feature * jenkins changes * jenkins changes * jenkins changes * jenkins changes * jenkins changes * bump versions and move OVA download links. (#335) * open-vm-tools now has a cross dependency (#337) with open-vm-tools-desktop and yum fails to install open-vm-tools on remote nodes when open-vm-tools-desktop is not installed. * ECS-CommunityEdition-317 Make `ecsconfig ping -cx` loop when dtquery fails (#344) (cherry picked from commit f5c7810a3385352a7ceb3bd9af66f9f824927dca) * Change the way ecs-install is pushed to repo (#346) * invoke zerofill via bash rather than expecting exec bit (#343) * Remove Ansible verbosity flag from Jenkinsfile (#350) * OVA QoL Improvements (#351) * add `ova-step1` and `ova-step2` macros * add `ecsdeploy noop` for some ova macros to look better * make videploy more intelligent and play nice with update_deploy * Implement Ansible global OVA flag fact (#349) * implements Ansible global ova flag fact - custom fact in /etc/ansible/facts.d/ova.fact - ova conditional flags in playbooks * misaligned `when` * skip rebooting when using the OVA. * Upgrade Ansible to 2.3 (#347) * install ansible package from @edge_main for 2.3 * Ansible changes for Ansible 2.3 * ECS-CommunityEdition-235 Bump Ansible version to 2.3 * refactor Ansible task `when:` clauses to Ansible 2.3 spec * refactor node reboot actions for Ansible 2.3 Also resolve #342 * remove unused json_file plugin * must ignore_errors: True `needs-restarting -r` * refactor port-check `when`s to Ansible 2.3 spec * cleanup * add loop_control to path permissions entries * add loop_control labels to many iterators * add loop_control labels to many iterators * break out one directive per line * add loop_control labels to many iterators * incorrect `when` * speling * Switch to Alpine 3.6 release (#359) * Switch to Alpine 3.6 release Install Python 2 from APK * Changes to Rockerfile for python:2-alpine parity * Split steps out from Ansible to get realtime console logging (#358) * Split steps out from Ansible to get realtime console logging * use /tmp? * template out a script to run command on install node via IP * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * Jenkinsfile sausage * log environment info to file log only, never to console. * Add CentOS 7.4 support (#360) (cherry picked from commit 1a960873d5d6f0b1011caabcbb3987114723dd8a) * [WiP] Misc. 2.5.0 bugfixes (#352) * update reference deploy version * bugfix typo in ed25519 private key filename * fix ova flag implementation * fix ova flag implementation round 2 * fix ova flag implementation round 3 * update entrypoint.sh * bump version to 2.5.0b1 (#364) * [WiP] Update ecs-install Python requirements (#356) * update python requirements * pin python requirements to major versions rather than patches. * add python2-dev to temporary build environment * need cryptography>=1.9 * [WiP] ECS 3.1.0.0 Reduced GA and CE Support (#353) * make 3.0.0.2 to use 100% regex (cherry picked from commit 0689ce59ec9e61beea4082b1f23222031b17f62d) * prep 3.1.0.0 RC3 (cherry picked from commit 2c3d5b9e92bcec83e01362c674a00158e6a61289) * prep 3.1.0.0 RC3 (cherry picked from commit ea12c296204442adf1e5079d778171140dede101) * ECS 3.1 templates * local facts must be fully qualified? * interface roles should be defined in deploy.yml * actually use a comma in the jinja joiner() func * use ansible_fqdn for agent strings not ansible_hostname * ECS 3.1.0.0 RC4 * joiner() needs to be the prefix not the suffix * the infamous missing comma * no trailing comma * remove redundant spaces * set host: field in testing * make object-main_network.json.j2 VDC-aware + formatting * Set georeceiver initialBufferNumOnHeap to 10 * Mount /usr instead of /usr/local to capture new install path * [WiP] ECS 3.1 Full GA and CE support (#367) * Use nodeId instead of the node IP to create data store * Fix errors getting node ID * fix 3.1 patch again * migrate cm.object.properties/'MustHaveEnoughResources=false' into Dockerfile * Run cf_client in container for new low partition count vars * Run cf_client in container for new low partition count vars * migrate cf_client variable settings into Dockerfile * update comments in Dockerfile for 3.1.0.0 * release-2.5.0-prep (#370) * Update ECS-Installation.md (cherry picked from commit f8be70f53b55bf718e2f1bb32df206484a12e7e8) * Update ECS-Installation.md (cherry picked from commit b479b0722308aaa40345af524f5e4430d29b11ed) * bump versions * Check if object user is editable before continuing to S3 credential provisioning (#376) * Fix #371 * Cosmetic change to fix output * Add retry loops around funcs to add credentials * add uniqueness to hostnames of CI VMs (#378) * Enable build-help in bootstrap.sh (#377) * enable build help in bootstrap.sh * enable build help in bootstrap.sh * Switch to upstream ECS version 3.1.0.1 (#375) * Add Dockerfile ecs-object-reduced to CE image patch * bump ecs version * bump version files for install node 2.5.1 (#379) * Develop (#372) (#380) backmerge * Documentation Pass 2 (#384) * ECS-CommunityEdition-299 Add Community Documentation * ECS-CommunityEdition-299 Add Community Documentation * CoC * update issue template * update pull request template * remove contributing document * update .dockerignore * move old changelogs to docs/legacy * Reformat README * remove changelog.rst * Create Standard, Island, and OVA Installation guides * Update ECS-Installation.md to be ToC style page * wording * install node capitalization * create building docs * add build_image macro to run.sh * create building docs * create utilities.md doc * undo a whoops * add utilities references and address #362 * Switch to mainline versions of libressl, mktorrent, ansible. (#391) * Switch to mainline versions of libressl, mktorrent, ansible. Resolves #390 * add tool to remove all ecs-install instances * update testbook for this issue * remove clicmd_access_host run in ecsdeploy cache command * change playbook logic to prevent single node deployments with an install node from failing. * update mktorrent-borg exec path * update tests * Change test network (#389) * Change test network to Local VLAN 999 * Switch network name to "CI Network" * Bump version 2.5.1 -> 2.5.2 (#393) * feature-ova-enhancements (#397) * no longer check OVA for reboot requirement * ecsconfig ping now differentiates between pass and failed PONGs depending on dtquery status * add comment to point to new vars location * upstream-ecs-3.1.0.2 (#398) * Add Dockerfile for 3.1.0.2 * Add 3.1.0 entries to user-definable release.conf * Adjust configs for 3.1.0.2 * implement custom autonames (#402) closes #399 * update versioning for 2.6.0 (#403) --- docs/design/reference.deploy.yml | 17 +++++++++++--- patches/3.1.0.2/Dockerfile | 23 +++++++++++++++++++ release.conf | 6 +++++ .../installer_generate_ssh_keys/vars/main.yml | 2 ++ ui/ansible/templates/autonames.yml.j2 | 4 +++- ui/ecsconfig.py | 9 +++++--- ui/etc/config.yml | 4 ++-- ui/etc/release.conf | 8 +++---- ui/etc/schema.yml | 12 ++++++++++ ui/run.sh | 10 +++++++- ui/setup.py | 2 +- ui/tui/schema_functions.py | 6 ++++- 12 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 patches/3.1.0.2/Dockerfile diff --git a/docs/design/reference.deploy.yml b/docs/design/reference.deploy.yml index d25be15e..36752306 100644 --- a/docs/design/reference.deploy.yml +++ b/docs/design/reference.deploy.yml @@ -1,4 +1,4 @@ -# deploy.yml reference implementation v2.5.0 +# deploy.yml reference implementation v2.6.0 # [Optional] # By changing the license_accepted boolean value to "true" you are @@ -8,6 +8,15 @@ licensing: license_accepted: false +#autonames: +# custom: +# - ecs01 +# - ecs02 +# - ecs03 +# - ecs04 +# - ecs05 +# - ecs06 + # [Required] # Deployment facts reference facts: @@ -63,7 +72,7 @@ facts: # VFS path to source of randomness # Defaults to /dev/urandom for speed considerations. If you prefer /dev/random, put that here. # If you have a /dev/srandom implementation or special entropy hardware, you may use that too - # so long as it implements a /dev/random type device abstraction. + # so long as it implements a /dev/random type device. entropy_source: /dev/urandom # # [Optional] @@ -71,7 +80,9 @@ facts: # Available options: # - "moons" (ECS CE default) # - "cities" (ECS SKU-flavored) - autonaming: moons + # - "custom" (uncomment and use the top-level autonames block to define these) + # autonaming: custom + # # [Optional] # If your ECS comes with differing default credentials, you can specify those here diff --git a/patches/3.1.0.2/Dockerfile b/patches/3.1.0.2/Dockerfile new file mode 100644 index 00000000..7c7cb7c3 --- /dev/null +++ b/patches/3.1.0.2/Dockerfile @@ -0,0 +1,23 @@ +# Fixes to the default 3.1.0.2 reduced image. + +# Build on object-reduced image (GA release) +FROM emcvipr/object:3.1.0.2-95467.5ec917e-reduced + +# Increase memory for transformsvc +RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc + +# Fix disk partitioning script +RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh +RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh + +# Set VNets useSeperateThreadPools to True +RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f + +# Set georeceiver's initialBufferNumOnHeap to something smaller for CE +RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="10"' $f || sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="10"/' $f + +# Configure CM Object properties: Disable minimum storage device count +RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f || sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f + +# Allow allocation of different blocks of a chunk to be stored on the same partition +RUN sed -i 's###g' /opt/storageos/conf/ssm-cf-conf.xml diff --git a/release.conf b/release.conf index b0555f8d..04b0c4f9 100644 --- a/release.conf +++ b/release.conf @@ -11,6 +11,12 @@ # DO NOT specify an alternate registry here. Use the -r argument to # bootstrap.sh for that. # +### Older versions ( 3.1 ) +# release_artifact="emccorp/ecs-software-3.0.1" +# release_tag="3.1.0.0" +# release_tag="3.1.0.1" +# release_tag="latest" +# ### Older versions ( 3.0 ) # release_artifact="emccorp/ecs-software-3.0.0" # release_tag="3.0.0.1" diff --git a/ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml b/ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml index 7f2c1c27..178809a3 100644 --- a/ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml +++ b/ui/ansible/roles/installer_generate_ssh_keys/vars/main.yml @@ -1,3 +1,5 @@ +# This is all now in ui/ansible/templates/all.j2 + #keys_rsa: # keygen_cmd: ssh-keygen -N '' -t rsa -b 4096 -o -a 100 -f # priv_key: /opt/ssh/id_rsa diff --git a/ui/ansible/templates/autonames.yml.j2 b/ui/ansible/templates/autonames.yml.j2 index 9835138d..08a50b2b 100644 --- a/ui/ansible/templates/autonames.yml.j2 +++ b/ui/ansible/templates/autonames.yml.j2 @@ -1,5 +1,7 @@ +--- +# Selected autonames: {{ facts['node_defaults']['autonaming'] }} autonames: -{%- set autonaming = facts['autonaming'] | default('moons') -%} +{%- set autonaming = facts['node_defaults']['autonaming'] | default('moons') -%} {%- for name in autonames[autonaming] %} - {{ name -}} {% endfor %} diff --git a/ui/ecsconfig.py b/ui/ecsconfig.py index f09d8cbc..d7cb4b66 100755 --- a/ui/ecsconfig.py +++ b/ui/ecsconfig.py @@ -296,14 +296,17 @@ def do_ping(): if resp_dict is not None: if resp_dict['common_name'] is not None: dt_status = conf.diag_dt_status() - o('PONG: api_endpoint={} username={} {}'.format(conf.api_endpoint, - resp_dict['common_name'], - dt_status['text'])) if x: if dt_status['status'] is True: pinging = False + o('PONG: api_endpoint={} username={} {}'.format(conf.api_endpoint, + resp_dict['common_name'], + dt_status['text'])) else: pinging = True + o('WAIT: api_endpoint={} username={} {}'.format(conf.api_endpoint, + resp_dict['common_name'], + dt_status['text'])) else: raise ECSClientException("Unexpected response from API") except requests.ConnectionError or httplib.HTTPException: diff --git a/ui/etc/config.yml b/ui/etc/config.yml index 92343c25..c8e3abd6 100644 --- a/ui/etc/config.yml +++ b/ui/etc/config.yml @@ -13,7 +13,7 @@ --- ui: name: ECS Community Edition Install Node - version: 2.5.2 + version: 2.6.0 host_root_dir: /opt/emc/ecs-install state_file: /opt/state.yml deploy_file: /opt/deploy.yml @@ -32,7 +32,7 @@ ui: ffx_sem: /opt/ffx.sem product: name: ECS - version: 3.1.0.1 + version: 3.1.0.2 vendor: Dell EMC flavor: Community Edition slogan: Free and Frictionless diff --git a/ui/etc/release.conf b/ui/etc/release.conf index 445f42ca..6426f0f9 100644 --- a/ui/etc/release.conf +++ b/ui/etc/release.conf @@ -8,10 +8,10 @@ # it is provided by or on behalf of EMC. release_name="ECS Community Edition" -release_version="3.1.0.1" +release_version="3.1.0.2" release_product="ECS Software" release_artifact="emccorp/ecs-software-3.1.0" -release_tag="hf1" +release_tag="3.1.0.2" release_common_name="emccorp/ecs-software:latest" docker_host_root="/opt/emc/ecs-install" @@ -32,8 +32,8 @@ repo_name='emccorp' image_name='ecs-install' tag='latest' ver_maj='2' -ver_min='5' -ver_rev='2' +ver_min='6' +ver_rev='0' ver_tag='r' serial=0 diff --git a/ui/etc/schema.yml b/ui/etc/schema.yml index 556b8063..dd3a761c 100644 --- a/ui/etc/schema.yml +++ b/ui/etc/schema.yml @@ -9,6 +9,18 @@ mapping: license_accepted: type: bool + autonames: + type: map + required: False + mapping: + custom: + type: seq + required: True + sequence: + - type: str + required: True + func: valid_hostname + facts: type: map required: True diff --git a/ui/run.sh b/ui/run.sh index 631aa667..10d1dc2f 100755 --- a/ui/run.sh +++ b/ui/run.sh @@ -112,7 +112,7 @@ case "$(basename ${0})" in #run ecsdeploy load || exit $? run ecsdeploy cache || exit $? ;; - island-step2|ova-step1) + island-step2) #run ecsdeploy load || exit $? run ecsdeploy access || exit $? run ecsdeploy check || exit $? @@ -123,6 +123,14 @@ case "$(basename ${0})" in run ecsdeploy deploy || exit $? run ecsdeploy start || exit $? ;; + ova-step1) + #run ecsdeploy load || exit $? + run ecsdeploy access || exit $? + run ecsdeploy check || exit $? + run ecsdeploy bootstrap || exit $? + run ecsdeploy deploy || exit $? + run ecsdeploy start || exit $? + ;; step1) #run ecsdeploy load || exit $? run ecsdeploy access || exit $? diff --git a/ui/setup.py b/ui/setup.py index 73ada285..a406ad22 100755 --- a/ui/setup.py +++ b/ui/setup.py @@ -3,7 +3,7 @@ setup( name='ecsdeploy', - version='2.5.2', + version='2.6.0', packages=find_packages(), scripts=['ui.py', 'ecsdeploy.py', diff --git a/ui/tui/schema_functions.py b/ui/tui/schema_functions.py index 727d8fe2..fa682cb0 100644 --- a/ui/tui/schema_functions.py +++ b/ui/tui/schema_functions.py @@ -36,7 +36,7 @@ def alphanumeric(value, rule_obj, path): __validate_regex(AlphanumericRegex, value, "Not a valid name: '%s' (%s)" % (value, path)) return True -def docker_image(value, rule_obj,path): +def docker_image(value, rule_obj, path): __validate_regex(DockerImageRegex, value, "Not a valid Docker image: '%s' (%s)" % (value, path)) return True @@ -47,3 +47,7 @@ def s3_secret_key(value, rule_obj, path): def valid_crypto_method(value, rule_obj, path): __validate_regex(ValidCryptoMethodRegex, value, "Not a supported crypto method: '%s' (%s)" % (value, path)) return True + +def valid_hostname(value, rule_obj, path): + __validate_regex(HostnameRegex, value, "Not a valid hostname: '%s' (%s)" % (value, path)) + return True