crossbow command

Author: kszucs

dev/archery/archery/bot.py

@@ -1,9 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 import shlex
 from functools import partial
 
 import click
 import github
 
+from .utils.crossbow import Crossbow
+from .utils.git import Git
+
 
 class EventError(Exception):
     pass
@@ -132,96 +152,39 @@ def bot(ctx):
     ctx.ensure_object(dict)
 
 
-# @ursabot.command()
-# @click.argument('baseline', type=str, metavar='[<baseline>]', default=None,
-#                 required=False)
-# @click.option('--suite-filter', metavar='<regex>', show_default=True,
-#               type=str, default=None, help='Regex filtering benchmark suites.')
-# @click.option('--benchmark-filter', metavar='<regex>', show_default=True,
-#               type=str, default=None,
-#               help='Regex filtering benchmarks.')
-# def benchmark(baseline, suite_filter, benchmark_filter):
-#     """Run the benchmark suite in comparison mode.
-
-#     This command will run the benchmark suite for tip of the branch commit
-#     against `<baseline>` (or master if not provided).
-
-#     Examples:
-
-#     \b
-#     # Run the all the benchmarks
-#     @ursabot benchmark
-
-#     \b
-#     # Compare only benchmarks where the name matches the /^Sum/ regex
-#     @ursabot benchmark --benchmark-filter=^Sum
-
-#     \b
-#     # Compare only benchmarks where the suite matches the /compute-/ regex.
-#     # A suite is the C++ binary.
-#     @ursabot benchmark --suite-filter=compute-
-
-#     \b
-#     # Sometimes a new optimization requires the addition of new benchmarks to
-#     # quantify the performance increase. When doing this be sure to add the
-#     # benchmark in a separate commit before introducing the optimization.
-#     #
-#     # Note that specifying the baseline is the only way to compare using a new
-#     # benchmark, since master does not contain the new benchmark and no
-#     # comparison is possible.
-#     #
-#     # The following command compares the results of matching benchmarks,
-#     # compiling against HEAD and the provided baseline commit, e.g. eaf8302.
-#     # You can use this to quantify the performance improvement of new
-#     # optimizations or to check for regressions.
-#     @ursabot benchmark --benchmark-filter=MyBenchmark eaf8302
-#     """
-#     # each command must return a dictionary which are set as build properties
-#     props = {'command': 'benchmark'}
-
-#     if baseline:
-#         props['benchmark_baseline'] = baseline
-
-#     opts = []
-#     if suite_filter:
-#         suite_filter = shlex.quote(suite_filter)
-#         opts.append(f'--suite-filter={suite_filter}')
-#     if benchmark_filter:
-#         benchmark_filter = shlex.quote(benchmark_filter)
-#         opts.append(f'--benchmark-filter={benchmark_filter}')
-
-#     if opts:
-#         props['benchmark_options'] = opts
-
-#     return props
-
-
 @bot.group()
-@click.option('--repo', '-r', default='ursa-labs/crossbow',
+@click.option('--arrow', '-a', default='apache/arrow',
+              help='Arrow repository on github to use')
+@click.option('--crossbow', '-c', default='ursa-labs/crossbow',
               help='Crossbow repository on github to use')
 @click.pass_obj
-def crossbow(props, repo):
+def crossbow(obj, arrow, crossbow):
     """Trigger crossbow builds for this pull request"""
-    # TODO(kszucs): validate the repo format
-    props['command'] = 'crossbow'
-    props['crossbow_repo'] = repo  # github user/repo
-    props['crossbow_repository'] = f'https://github.com/{repo}'  # git url
+    obj['arrow_repo'] = 'https://github.com/{}'.format(arrow)
+    obj['crossbow_repo'] = 'https://github.com/{}'.format(crossbow)
 
 
 @crossbow.command()
 @click.argument('task', nargs=-1, required=False)
 @click.option('--group', '-g', multiple=True,
               help='Submit task groups as defined in tests.yml')
+@click.option('--dry-run/--push', default=False,
+              help='Just display the new changelog, don\'t write it')
 @click.pass_obj
-def submit(props, task, group):
+def submit(obj, task, group, dry_run):
     """Submit crossbow testing tasks.
 
     See groups defined in arrow/dev/tasks/tests.yml
     """
-    args = ['-c', 'tasks.yml']
+    args = []
     for g in group:
         args.extend(['-g', g])
     for t in task:
         args.append(t)
 
-    return {'crossbow_args': args, **props}
+    git = Git()
+    xbow = Crossbow('arrow/dev/tasks/crossbow.py')
+
+    # git.clone(obj['arrow_repo'], 'arrow')
+    # git.clone(obj['crossbow_repo'], 'crossbow')
+    xbow.run('submit', *args)

dev/archery/archery/lang/python.py

@@ -63,7 +63,6 @@ def _convert_typehint(tokens):
                 # are not supported by _signature_fromstr
                 yield (names[1].type, names[1].string)
             elif len(names) > 2:
-                print(names)
                 raise ValueError('More than two NAME tokens follow each other')
             names = []
             yield (token.type, token.string)

dev/archery/archery/tests/test_bot.py

@@ -2,10 +2,12 @@
 from pathlib import Path
 from unittest.mock import Mock
 
-import click
 import pytest
 import responses as rsps
-from archery.bot import CommentBot, CommandError, group
+import click
+from click.testing import CliRunner
+
+from archery.bot import CommentBot, CommandError, group, bot
 
 
 @pytest.fixture
@@ -186,4 +188,13 @@ def handler(command, **kwargs):
     bot.handle('issue_comment', payload)
 
     post = responses.calls[3]
-    assert json.loads(post.request.body) == {'content': reaction}
+    assert json.loads(post.request.body) == {'content': reaction}
+
+
+# TODO(kszucs): properly mock it
+# def test_crossbow_submit():
+#     runner = CliRunner()
+#     result = runner.invoke(
+#         bot, ['crossbow', 'submit', '-g', 'wheel', '--dry-run']
+#     )
+#     assert result.exit_code == 0

dev/archery/archery/utils/crossbow.py

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from .command import Command, default_bin
+
+
+class Crossbow(Command):
+    def __init__(self, crossbow_bin=None):
+        self.bin = default_bin(crossbow_bin, "arrow/dev/tasks/crossbow.py")