From 3f2a33f902983c0d395e0480e8a8df40ed5da29c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 10 Sep 2019 12:50:29 +0200 Subject: [PATCH] ARROW-6480: [Crossbow] Summary report e-mailer with polling logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a rebased version of #5319. Crossbow had a major refactoring in #5296 so the two implementation have diverged. Closes #5332 from kszucs/crossbow-report and squashes the following commits: ed62de438 unused import 76a3269ae share output redirection; use pass_obj instead of pass_context 9a0069cf0 start incrementing from 0 16015d227 merge tests.yaml with tasks.yaml 00f576629 latest-prefix 162a0fbf8 polling 1a8790f89 Add polling logic to wait for job completion 6aa2a7eca Test disabling stop_if_pending 3e4580aed Implement e-mail report for a Crossbow job Lead-authored-by: Krisztián Szűcs Co-authored-by: Wes McKinney Signed-off-by: Krisztián Szűcs --- dev/tasks/crossbow.py | 454 +++++++++++++++++++++++++++++++---------- dev/tasks/tasks.yml | 456 ++++++++++++++++++++++++++++++++++++++++++ dev/tasks/tests.yml | 439 ---------------------------------------- 3 files changed, 804 insertions(+), 545 deletions(-) delete mode 100644 dev/tasks/tests.yml diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index 4e4dabb4d3212..d629d916f2143 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -22,10 +22,12 @@ import glob import time import mimetypes +import textwrap from io import StringIO from pathlib import Path from textwrap import dedent from datetime import datetime +from functools import partial from collections import namedtuple import click @@ -395,8 +397,7 @@ def file_contents(self, commit_id, file): return blob.data def _parse_github_user_repo(self): - url = self._remote_url if self._remote_url else self.remote_url - m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', url) + m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url) user, repo = m.group(1), m.group(2) return user, repo @@ -516,15 +517,28 @@ def github_overwrite_release_assets(self, tag_name, target_commitish, class Queue(Repo): - def _next_job_id(self, prefix): - """Auto increments the branch's identifier based on the prefix""" + def _latest_prefix_id(self, prefix): pattern = re.compile(r'[\w\/-]*{}-(\d+)'.format(prefix)) matches = list(filter(None, map(pattern.match, self.repo.branches))) if matches: latest = max(int(m.group(1)) for m in matches) else: - latest = 0 - return '{}-{}'.format(prefix, latest + 1) + latest = -1 + return latest + + def _next_job_id(self, prefix): + """Auto increments the branch's identifier based on the prefix""" + latest_id = self._latest_prefix_id(prefix) + return '{}-{}'.format(prefix, latest_id + 1) + + def latest_for_prefix(self, prefix): + latest_id = self._latest_prefix_id(prefix) + if latest_id < 0: + raise RuntimeError( + 'No job has been submitted with prefix {} yet'.format(prefix) + ) + job_name = '{}-{}'.format(prefix, latest_id) + return self.get(job_name) def get(self, job_name): branch_name = 'origin/{}'.format(job_name) @@ -654,6 +668,7 @@ def __init__(self, platform, ci, template, artifacts=None, params=None): self.branch = None # filled after adding to a queue self.commit = None # filled after adding to a queue self._queue = None # set by the queue object after put or get + self._status = None # status cache def render_files(self, **extra_params): path = CWD / self.template @@ -676,8 +691,12 @@ def filename(self): } return config_files[self.ci] - def status(self): - return self._queue.github_commit_status(self.commit) + def status(self, force_query=False): + # _status might be uninitialized because of yaml serialization + _status = getattr(self, '_status', None) + if force_query or _status is None: + self._status = self._queue.github_commit_status(self.commit) + return self._status def assets(self): assets = self._queue.github_release_assets(self.tag) @@ -691,13 +710,15 @@ def assets(self): num_matches = len(matches) # validate artifact pattern matches single asset - if num_matches > 1: + if num_matches == 0: + artifacts[artifact] = None + elif num_matches == 1: + artifacts[artifact] = assets[matches[0].group(0)] + else: raise ValueError( 'Only a single asset should match pattern `{}`, there are ' 'multiple ones: {}'.format(', '.join(matches)) ) - elif num_matches == 1: - artifacts[artifact] = assets[matches[0].group(0)] return artifacts @@ -778,6 +799,215 @@ def from_config(cls, config, target, task_whitelist=None, return cls(target=target, tasks=tasks) + def is_finished(self): + for task in self.tasks.values(): + status = task.status(force_query=True) + if status.state == 'pending': + return False + return True + + def wait_until_finished(self, poll_max_minutes=120, + poll_interval_minutes=10): + started_at = time.time() + while True: + if self.is_finished(): + break + + waited_for_minutes = (time.time() - started_at) / 60 + if waited_for_minutes > poll_max_minutes: + msg = ('Exceeded the maximum amount of time waiting for job ' + 'to finish, waited for {} minutes.') + raise RuntimeError(msg.format(waited_for_minutes)) + + # TODO(kszucs): use logging + click.echo('Waiting {} minutes and then checking again' + .format(poll_interval_minutes)) + time.sleep(poll_interval_minutes * 60) + + +class Report: + + def __init__(self, job): + self.job = job + + def show(self): + raise NotImplementedError() + + +class ConsoleReport(Report): + """Report the status of a Job to the console using click""" + + # output table's header template + HEADER = '[{state:>7}] {branch:<49} {content:>20}' + + # output table's row template for assets + ARTIFACT_NAME = '{artifact:>70} ' + ARTIFACT_STATE = '[{state:>7}]' + + # state color mapping to highlight console output + COLORS = { + # from CombinedStatus + 'error': 'red', + 'failure': 'red', + 'pending': 'yellow', + 'success': 'green', + # custom state messages + 'ok': 'green', + 'missing': 'red' + } + + def lead(self, state, branch, n_uploaded, n_expected): + line = self.HEADER.format( + state=state.upper(), + branch=branch, + content='uploaded {} / {}'.format(n_uploaded, n_expected) + ) + return click.style(line, fg=self.COLORS[state.lower()]) + + def header(self): + header = self.HEADER.format( + state='state', + branch='Task / Branch', + content='Artifacts' + ) + delimiter = '-' * len(header) + return '{}\n{}'.format(header, delimiter) + + def artifact(self, state, pattern, asset): + if asset is None: + artifact = pattern + state = 'pending' if state == 'pending' else 'missing' + else: + artifact = asset.name + state = 'ok' + + name_ = self.ARTIFACT_NAME.format(artifact=artifact) + state_ = click.style( + self.ARTIFACT_STATE.format(state=state.upper()), + self.COLORS[state] + ) + return name_ + state_ + + def show(self, outstream, asset_callback=None): + echo = partial(click.echo, file=outstream) + + # write table's header + echo(self.header()) + + # write table's body + for task_name, task in sorted(self.job.tasks.items()): + status = task.status() + assets = task.assets() + + # write summary of the uploaded vs total assets + n_expected = len(assets) + n_uploaded = len(list(filter(None, assets.values()))) + echo(self.lead(status.state, task.branch, n_uploaded, n_expected)) + + # write per asset status + for artifact, asset in task.assets().items(): + if asset_callback is not None: + asset_callback(task_name, task, asset) + # artifact is a pattern for the expected asset + echo(self.artifact(status.state, artifact, asset)) + + +class EmailReport(Report): + + HEADER = textwrap.dedent(""" + Crossbow Report for Job {job_name} + + All tasks: {all_tasks_url} + """) + + TASK = textwrap.dedent(""" + - {name}: + URL: {url} + """).strip() + + EMAIL = textwrap.dedent(""" + From: {sender_name} <{sender_email}> + To: {recipient_email} + Subject: {subject} + + {body} + """).strip() + + STATUS_HEADERS = { + # from CombinedStatus + 'error': 'Errored Tasks:', + 'failure': 'Failed Tasks:', + 'pending': 'Pending Tasks:', + 'success': 'Succeeded Tasks:', + } + + def __init__(self, job, sender_name, sender_email, recipient_email): + self.sender_name = sender_name + self.sender_email = sender_email + self.recipient_email = recipient_email + super().__init__(job) + + def url(self, query): + repo_url = self.job.queue.remote_url.strip('.git') + return '{}/branches/all?query={}'.format(repo_url, query) + + def listing(self, tasks): + entries = ( + self.TASK.format(name=task_name, url=self.url(task.branch)) + for task_name, task in tasks.items() + ) + return '\n'.join(entries) + + def header(self): + url = self.url(self.job.branch) + return self.HEADER.format(job_name=self.job.branch, all_tasks_url=url) + + def subject(self): + return "Crossbow Task Report for {}".format(self.job.branch) + + def body(self): + buffer = StringIO() + buffer.write(self.header()) + + tasks_by_state = toolz.groupby( + lambda tpl: tpl[1].status().state, + self.job.tasks.items() + ) + + for state in ('failure', 'error', 'pending', 'success'): + if state in tasks_by_state: + tasks = dict(tasks_by_state[state]) + buffer.write('\n') + buffer.write(self.STATUS_HEADERS[state]) + buffer.write('\n') + buffer.write(self.listing(tasks)) + buffer.write('\n') + + return buffer.getvalue() + + def email(self): + return self.EMAIL.format( + sender_name=self.sender_name, + sender_email=self.sender_email, + recipient_email=self.recipient_email, + subject=self.subject(), + body=self.body() + ) + + def show(self, outstream): + outstream.write(self.email()) + + def send(self, smtp_user, smtp_password, smtp_server, smtp_port): + import smtplib + + email = self.email() + + server = smtplib.SMTP_SSL(smtp_server, smtp_port) + server.ehlo() + server.login(smtp_user, smtp_password) + server.sendmail(smtp_user, self.recipient_email, email) + server.close() + # configure yaml serializer yaml = YAML() @@ -785,13 +1015,6 @@ def from_config(cls, config, target, task_whitelist=None, yaml.register_class(Task) yaml.register_class(Target) -# state color mapping to highlight console output -COLORS = {'ok': 'green', - 'error': 'red', - 'missing': 'red', - 'failure': 'red', - 'pending': 'yellow', - 'success': 'green'} # define default paths DEFAULT_CONFIG_PATH = CWD / 'tasks.yml' @@ -812,8 +1035,12 @@ def from_config(cls, config, target, task_whitelist=None, 'Defaults to crossbow directory placed next to arrow') @click.option('--queue-remote', '-qr', default=None, help='Force to use this remote URL for the Queue repository') +@click.option('--output-file', metavar='', + type=click.File('w', encoding='utf8'), default='-', + help='Capture output result into file.') @click.pass_context -def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote): +def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote, + output_file): if github_token is None: raise click.ClickException( 'Could not determine GitHub token. Please set the ' @@ -821,6 +1048,8 @@ def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote): 'valid GitHub access token or pass one to --github-token.' ) + ctx.ensure_object(dict) + ctx.obj['output'] = output_file ctx.obj['arrow'] = Repo(arrow_path) ctx.obj['queue'] = Queue(queue_path, remote_url=queue_remote, github_token=github_token, require_https=True) @@ -838,11 +1067,11 @@ def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote): @click.option('--jira-password', '-P', default=None, help='JIRA password') @click.option('--dry-run/--write', default=False, help='Just display the new changelog, don\'t write it') -@click.pass_context -def changelog(ctx, changelog_path, arrow_version, is_website, jira_username, +@click.pass_obj +def changelog(obj, changelog_path, arrow_version, is_website, jira_username, jira_password, dry_run): changelog_path = Path(changelog_path) - target = Target.from_repo(ctx.obj['arrow']) + target = Target.from_repo(obj['arrow']) version = arrow_version or target.version changelog = JiraChangelog(version, username=jira_username, @@ -882,13 +1111,11 @@ def changelog(ctx, changelog_path, arrow_version, is_website, jira_username, @click.option('--dry-run/--push', default=False, help='Just display the rendered CI configurations without ' 'submitting them') -@click.option('--output', metavar='', - type=click.File('w', encoding='utf8'), default='-', - help='Capture output result into file.') -@click.pass_context -def submit(ctx, tasks, groups, job_prefix, config_path, arrow_version, - arrow_remote, arrow_branch, arrow_sha, dry_run, output): - queue, arrow = ctx.obj['queue'], ctx.obj['arrow'] +@click.pass_obj +def submit(obj, tasks, groups, job_prefix, config_path, arrow_version, + arrow_remote, arrow_branch, arrow_sha, dry_run): + output = obj['output'] + queue, arrow = obj['queue'], obj['arrow'] # load available tasks configuration and groups from yaml with Path(config_path).open() as fp: @@ -920,45 +1147,85 @@ def submit(ctx, tasks, groups, job_prefix, config_path, arrow_version, @crossbow.command() @click.argument('job-name', required=True) -@click.option('--output', metavar='', - type=click.File('w', encoding='utf8'), default='-', - help='Capture output result into file.') -@click.pass_context -def status(ctx, job_name, output): - queue = ctx.obj['queue'] +@click.pass_obj +def status(obj, job_name): + output = obj['output'] + queue = obj['queue'] queue.fetch() - tpl = '[{:>7}] {:<49} {:>20}' - header = tpl.format('status', 'branch', 'artifacts') - click.echo(header, file=output) - click.echo('-' * len(header), file=output) - job = queue.get(job_name) + ConsoleReport(job).show(output) - for task_name, task in sorted(job.tasks.items()): - status = task.status() - assets = task.assets() - uploaded = 'uploaded {} / {}'.format( - sum(a in assets for a in task.artifacts), - len(task.artifacts) +@crossbow.command() +@click.argument('prefix', required=True) +@click.pass_obj +def latest_prefix(obj, prefix): + queue = obj['queue'] + queue.fetch() + + latest = queue.latest_for_prefix(prefix) + click.echo(latest.branch) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--sender-name', '-n', + help='Name to use for report e-mail.') +@click.option('--sender-email', '-e', + help='E-mail to use for report e-mail.') +@click.option('--recipient-email', '-r', + help='Where to send the e-mail report') +@click.option('--smtp-user', '-u', + help='E-mail address to use for SMTP login') +@click.option('--smtp-password', '-P', + help='SMTP password to use for report e-mail.') +@click.option('--smtp-server', '-s', default='smtp.gmail.com', + help='SMTP server to use for report e-mail.') +@click.option('--smtp-port', '-p', default=465, + help='SMTP port to use for report e-mail.') +@click.option('--poll/--no-poll', default=False, + help='Wait for completion if there are tasks pending') +@click.option('--poll-max-minutes', default=180, + help='Maximum amount of time waiting for job completion') +@click.option('--poll-interval-minutes', default=10, + help='Number of minutes to wait to check job status again') +@click.option('--send/--dry-run', default=False, + help='Just display the report, don\'t send it') +@click.pass_obj +def report(obj, job_name, sender_name, sender_email, recipient_email, + smtp_user, smtp_password, smtp_server, smtp_port, poll, + poll_max_minutes, poll_interval_minutes, send): + """ + Send an e-mail report showing success/failure of tasks in a Crossbow run + """ + output = obj['output'] + queue = obj['queue'] + queue.fetch() + + job = queue.get(job_name) + report = EmailReport( + job=job, + sender_name=sender_name, + sender_email=sender_email, + recipient_email=recipient_email + ) + + if poll: + job.wait_until_finished( + poll_max_minutes=poll_max_minutes, + poll_interval_minutes=poll_interval_minutes ) - leadline = tpl.format(status.state.upper(), task.branch, uploaded) - click.echo(click.style(leadline, fg=COLORS[status.state]), file=output) - - for artifact in task.artifacts: - try: - asset = assets[artifact] - except KeyError: - state = 'pending' if status.state == 'pending' else 'missing' - filename = '{:>70} '.format(artifact) - else: - state = 'ok' - filename = '{:>70} '.format(asset.name) - statemsg = '[{:>7}]'.format(state.upper()) - click.echo(filename + click.style(statemsg, fg=COLORS[state]), - file=output) + if send: + report.send( + smtp_user=smtp_user, + smtp_password=smtp_password, + smtp_server=smtp_server, + smtp_port=smtp_port + ) + else: + report.show(output) @crossbow.command() @@ -966,60 +1233,35 @@ def status(ctx, job_name, output): @click.option('-t', '--target-dir', default=DEFAULT_ARROW_PATH / 'packages', type=click.Path(file_okay=False, dir_okay=True), help='Directory to download the build artifacts') -@click.pass_context -def download_artifacts(ctx, job_name, target_dir): +@click.pass_obj +def download_artifacts(obj, job_name, target_dir): """Download and sign build artifacts from github releases""" + output = obj['output'] + # fetch the queue repository - queue = ctx.obj['queue'] + queue = obj['queue'] queue.fetch() # query the job's artifacts job = queue.get(job_name) + # create directory to download the assets to target_dir = Path(target_dir).absolute() / job_name target_dir.mkdir(parents=True, exist_ok=True) - click.echo('Download {}\'s artifacts to {}'.format(job_name, target_dir)) - tpl = '{:<10} {:>73}' + # download the assets while showing the job status + def asset_callback(task_name, task, asset): + if asset is not None: + path = target_dir / task_name / asset.name + path.parent.mkdir(exist_ok=True) + asset.download(path) - task_items = sorted(job.tasks.items()) - ntasks = len(task_items) + click.echo('Downloading {}\'s artifacts.'.format(job_name)) + click.echo('Destination directory is {}'.format(target_dir)) + click.echo() - for i, (task_name, task) in enumerate(task_items, start=1): - assets = task.assets() - artifact_dir = target_dir / task_name - artifact_dir.mkdir(exist_ok=True) - - basemsg = 'Downloading assets for task {}'.format( - click.style(task_name, bold=True) - ) - click.echo( - '\n{} {:>{size}}' .format( - basemsg, - click.style('{}/{}'.format(i, ntasks), bold=True), - size=89 - (len(basemsg) + 1) + 2 * len( - click.style('', bold=True)) - ) - ) - click.echo('-' * 89) - - for artifact in task.artifacts: - try: - asset = assets[artifact] - except KeyError: - msg = click.style('[{:>13}]'.format('MISSING'), - fg=COLORS['missing']) - click.echo(tpl.format(msg, artifact)) - else: - click.echo(click.style(artifact, bold=True)) - - # download artifact - artifact_path = artifact_dir / asset.name - asset.download(artifact_path) - - msg = click.style('[{:>13}]'.format('DOWNLOADED'), - fg=COLORS['ok']) - click.echo(tpl.format(msg, str(asset.name))) + report = ConsoleReport(job) + report.show(output, asset_callback=asset_callback) @crossbow.command() @@ -1027,9 +1269,9 @@ def download_artifacts(ctx, job_name, target_dir): @click.option('--tag', required=True, help='Target tag') @click.option('--pattern', '-p', 'patterns', required=True, multiple=True, help='File pattern to upload as assets') -@click.pass_context -def upload_artifacts(ctx, tag, sha, patterns): - queue = ctx.obj['queue'] +@click.pass_obj +def upload_artifacts(obj, tag, sha, patterns): + queue = obj['queue'] queue.github_overwrite_release_assets( tag_name=tag, target_commitish=sha, patterns=patterns ) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 5831685d669b3..cc429f176bf45 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -18,6 +18,9 @@ groups: # these groups are just for convenience # makes it easier to submit related tasks + + ############################# Packaging tasks ############################### + conda: - conda-linux-gcc-py27 - conda-linux-gcc-py36 @@ -27,6 +30,7 @@ groups: - conda-osx-clang-py37 - conda-win-vs2015-py36 - conda-win-vs2015-py37 + wheel: - wheel-manylinux1-cp27m - wheel-manylinux1-cp27mu @@ -45,6 +49,7 @@ groups: - wheel-win-cp35m - wheel-win-cp36m - wheel-win-cp37m + linux: - debian-stretch - debian-stretch-arm64 @@ -55,10 +60,129 @@ groups: - ubuntu-disco - centos-6 - centos-7 + gandiva: - gandiva-jar-trusty - gandiva-jar-osx + ############################# Testing tasks ################################# + + docker: + - docker-r + - docker-r-conda + - docker-rust + - docker-cpp + - docker-cpp-alpine + - docker-cpp-cmake32 + - docker-cpp-release + - docker-cpp-fuzzit + - docker-cpp-static-only + - docker-c_glib + - docker-go + - docker-python-2.7 + - docker-python-3.6 + - docker-python-3.7 + - docker-python-2.7-alpine + - docker-python-3.6-alpine + - docker-python-2.7-nopandas + - docker-python-3.6-nopandas + - docker-java + - docker-js + - docker-docs + - docker-lint + - docker-iwyu + - docker-clang-format + - docker-pandas-master + - docker-dask-integration + - docker-hdfs-integration + - docker-spark-integration + - docker-turbodbc-integration + + integration: + - docker-pandas-master + - docker-dask-integration + - docker-hdfs-integration + - docker-spark-integration + - docker-turbodbc-integration + + cpp-python: + - docker-cpp + - docker-cpp-alpine + - docker-cpp-cmake32 + - docker-python-2.7 + - docker-python-2.7-alpine + - docker-python-2.7-nopandas + - docker-python-3.6 + - docker-python-3.6-alpine + - docker-python-3.6-nopandas + - docker-python-3.7 + + ######################## Tasks to run regularly ############################# + + nightly: + - conda-linux-gcc-py27 + - conda-linux-gcc-py36 + - conda-linux-gcc-py37 + - conda-osx-clang-py27 + - conda-osx-clang-py36 + - conda-osx-clang-py37 + - conda-win-vs2015-py36 + - conda-win-vs2015-py37 + - wheel-manylinux1-cp27m + - wheel-manylinux1-cp27mu + - wheel-manylinux1-cp35m + - wheel-manylinux1-cp36m + - wheel-manylinux1-cp37m + - wheel-manylinux2010-cp27m + - wheel-manylinux2010-cp27mu + - wheel-manylinux2010-cp35m + - wheel-manylinux2010-cp36m + - wheel-manylinux2010-cp37m + - wheel-osx-cp27m + - wheel-osx-cp35m + - wheel-osx-cp36m + - wheel-osx-cp37m + - wheel-win-cp35m + - wheel-win-cp36m + - wheel-win-cp37m + - debian-stretch + - debian-stretch-arm64 + - debian-buster + - ubuntu-xenial + - ubuntu-bionic + - ubuntu-cosmic + - ubuntu-disco + - centos-6 + - centos-7 + - gandiva-jar-trusty + - gandiva-jar-osx + - docker-r + - docker-r-conda + - docker-rust + - docker-cpp + - docker-cpp-cmake32 + - docker-cpp-release + - docker-cpp-fuzzit + - docker-cpp-static-only + - docker-c_glib + - docker-go + - docker-python-2.7 + - docker-python-3.6 + - docker-python-3.7 + - docker-python-2.7-nopandas + - docker-python-3.6-nopandas + - docker-java + - docker-js + - docker-docs + - docker-lint + - docker-iwyu + - docker-clang-format + - docker-pandas-master + - docker-dask-integration + - docker-hdfs-integration + - docker-spark-integration + - docker-turbodbc-integration + tasks: # arbitrary_task_name: # platform: osx|linux|win @@ -820,3 +944,335 @@ tasks: template: gandiva-jars/travis.osx.yml artifacts: - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar + + ############################## Docker tests ################################# + + docker-r: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose build r + - docker-compose run r + + docker-r-conda: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose build r-conda + - docker-compose run r-conda + + docker-rust: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build rust + - docker-compose run rust + + docker-cpp: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose run cpp + + docker-cpp-release: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose run cpp-release + + docker-cpp-alpine: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp-alpine + - docker-compose run cpp-alpine + + docker-cpp-cmake32: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp-cmake32 + - docker-compose run cpp-cmake32 + + docker-cpp-fuzzit: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose build fuzzit + - docker-compose run fuzzit + + docker-cpp-static-only: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose run cpp-static-only + + docker-c_glib: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build cpp + - docker-compose build c_glib + - docker-compose run c_glib + + docker-go: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build go + - docker-compose run go + + docker-js: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build js + - docker-compose run js + + docker-java: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + commands: + - docker-compose build java + - docker-compose run java + + docker-python-2.7: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 2.7 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose run python + + docker-python-3.6: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose run python + + docker-python-3.7: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.7 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose run python + + docker-python-2.7-alpine: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 2.7 + commands: + - docker-compose build cpp-alpine + - docker-compose build python-alpine + - docker-compose run python-alpine + + docker-python-3.6-alpine: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp-alpine + - docker-compose build python-alpine + - docker-compose run python-alpine + + docker-python-2.7-nopandas: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 2.7 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build python-nopandas + - docker-compose run python-nopandas + + docker-python-3.6-nopandas: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build python-nopandas + - docker-compose run python-nopandas + + ###################### Documentation building tests ######################### + + docker-docs: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build docs + - docker-compose run docs + + ############################## Linter tests ################################# + + docker-lint: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build lint + - docker-compose run lint + + docker-iwyu: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build lint + - docker-compose run iwyu + + docker-clang-format: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build lint + - docker-compose run clang-format + + ############################## Integration tests ############################ + + docker-dask-integration: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build dask-integration + - docker-compose run dask-integration + + docker-hdfs-integration: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build hdfs-integration + - docker-compose run hdfs-integration + + docker-spark-integration: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build spark-integration + - docker-compose run spark-integration + + docker-turbodbc-integration: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build turbodbc-integration + - docker-compose run turbodbc-integration + + docker-pandas-master: + ci: circle + platform: linux + template: docker-tests/circle.linux.yml + params: + environment: + PYTHON_VERSION: 3.6 + commands: + - docker-compose build cpp + - docker-compose build python + - docker-compose build --no-cache pandas-master + - docker-compose run pandas-master diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml deleted file mode 100644 index 930e62446476a..0000000000000 --- a/dev/tasks/tests.yml +++ /dev/null @@ -1,439 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -groups: - # these groups are just for convenience - # makes it easier to submit related tasks - nightly: - - docker-r - - docker-r-conda - - docker-rust - - docker-cpp - - docker-cpp-cmake32 - - docker-cpp-release - - docker-cpp-fuzzit - - docker-cpp-static-only - - docker-c_glib - - docker-go - - docker-python-2.7 - - docker-python-3.6 - - docker-python-3.7 - - docker-python-2.7-nopandas - - docker-python-3.6-nopandas - - docker-java - - docker-js - - docker-docs - - docker-lint - - docker-iwyu - - docker-clang-format - - docker-pandas-master - - docker-dask-integration - - docker-hdfs-integration - - docker-spark-integration - - docker-turbodbc-integration - - docker: - - docker-r - - docker-r-conda - - docker-rust - - docker-cpp - - docker-cpp-alpine - - docker-cpp-cmake32 - - docker-cpp-release - - docker-cpp-fuzzit - - docker-cpp-static-only - - docker-c_glib - - docker-go - - docker-python-2.7 - - docker-python-3.6 - - docker-python-3.7 - - docker-python-2.7-alpine - - docker-python-3.6-alpine - - docker-python-2.7-nopandas - - docker-python-3.6-nopandas - - docker-java - - docker-js - - docker-docs - - docker-lint - - docker-iwyu - - docker-clang-format - - docker-pandas-master - - docker-dask-integration - - docker-hdfs-integration - - docker-spark-integration - - docker-turbodbc-integration - - integration: - - docker-pandas-master - - docker-dask-integration - - docker-hdfs-integration - - docker-spark-integration - - docker-turbodbc-integration - - cpp-python: - - docker-cpp - - docker-cpp-alpine - - docker-cpp-cmake32 - - docker-python-2.7 - - docker-python-2.7-alpine - - docker-python-2.7-nopandas - - docker-python-3.6 - - docker-python-3.6-alpine - - docker-python-3.6-nopandas - - docker-python-3.7 - -tasks: - # arbitrary_task_name: - # platform: osx|linux|win - # template: path of jinja2 templated yml - # params: optional extra parameters - # artifacts: list of regex patterns, each needs to match a single github - # release asset, version variable is replaced in the pattern - # e.g.: - # - pyarrow-{version}-py36(h[a-z0-9]+)_0-linux-64.tar.bz2 - - ############################## Language containers ######################### - - docker-r: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose build r - - docker-compose run r - - docker-r-conda: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose build r-conda - - docker-compose run r-conda - - docker-rust: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build rust - - docker-compose run rust - - docker-cpp: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose run cpp - - docker-cpp-release: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose run cpp-release - - docker-cpp-alpine: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp-alpine - - docker-compose run cpp-alpine - - docker-cpp-cmake32: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp-cmake32 - - docker-compose run cpp-cmake32 - - docker-cpp-fuzzit: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose build fuzzit - - docker-compose run fuzzit - - docker-cpp-static-only: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose run cpp-static-only - - docker-c_glib: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build cpp - - docker-compose build c_glib - - docker-compose run c_glib - - docker-go: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build go - - docker-compose run go - - docker-js: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build js - - docker-compose run js - - docker-java: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - commands: - - docker-compose build java - - docker-compose run java - - docker-python-2.7: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 2.7 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose run python - - docker-python-3.6: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose run python - - docker-python-3.7: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.7 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose run python - - docker-python-2.7-alpine: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 2.7 - commands: - - docker-compose build cpp-alpine - - docker-compose build python-alpine - - docker-compose run python-alpine - - docker-python-3.6-alpine: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp-alpine - - docker-compose build python-alpine - - docker-compose run python-alpine - - docker-python-2.7-nopandas: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 2.7 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build python-nopandas - - docker-compose run python-nopandas - - docker-python-3.6-nopandas: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build python-nopandas - - docker-compose run python-nopandas - - ###################### Documentation building tests ######################### - - docker-docs: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build docs - - docker-compose run docs - - ############################## Linter tests ################################# - - docker-lint: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build lint - - docker-compose run lint - - docker-iwyu: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build lint - - docker-compose run iwyu - - docker-clang-format: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build lint - - docker-compose run clang-format - - ############################## Integration tests ############################ - - docker-dask-integration: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build dask-integration - - docker-compose run dask-integration - - docker-hdfs-integration: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build hdfs-integration - - docker-compose run hdfs-integration - - docker-spark-integration: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build spark-integration - - docker-compose run spark-integration - - docker-turbodbc-integration: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build turbodbc-integration - - docker-compose run turbodbc-integration - - docker-pandas-master: - ci: circle - platform: linux - template: docker-tests/circle.linux.yml - params: - environment: - PYTHON_VERSION: 3.6 - commands: - - docker-compose build cpp - - docker-compose build python - - docker-compose build --no-cache pandas-master - - docker-compose run pandas-master