From 674384669f0058a5a1fddd0dc0680e009373b1e8 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Sat, 11 Aug 2018 22:31:01 -0400 Subject: [PATCH] Add command to create new integrations --- datadog_checks_dev/README.md | 14 ++ .../dev/tooling/commands/__init__.py | 2 + .../dev/tooling/commands/create.py | 128 ++++++++++++++++++ .../dev/tooling/files/__init__.py | 35 +++++ .../dev/tooling/files/changelog.py | 21 +++ .../dev/tooling/files/example.py | 21 +++ .../dev/tooling/files/manifest.py | 78 +++++++++++ .../dev/tooling/files/metadata.py | 18 +++ .../dev/tooling/files/package.py | 70 ++++++++++ .../dev/tooling/files/readme.py | 67 +++++++++ .../datadog_checks/dev/tooling/files/reqs.py | 32 +++++ .../datadog_checks/dev/tooling/files/setup.py | 78 +++++++++++ .../datadog_checks/dev/tooling/files/test.py | 27 ++++ .../datadog_checks/dev/tooling/files/tox.py | 49 +++++++ .../datadog_checks/dev/tooling/files/utils.py | 22 +++ 15 files changed, 662 insertions(+) create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/commands/create.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/__init__.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/changelog.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/example.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/manifest.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/metadata.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/package.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/readme.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/reqs.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/setup.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/test.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/tox.py create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/files/utils.py diff --git a/datadog_checks_dev/README.md b/datadog_checks_dev/README.md index 6d57782911262..874a61d36c28b 100644 --- a/datadog_checks_dev/README.md +++ b/datadog_checks_dev/README.md @@ -25,6 +25,7 @@ and is available on Linux, macOS, and Windows, and supports Python 2.7/3.5+ and - [Set](#set) - [Show](#show) - [Update](#update) + - [Create](#create) - [Dep](#dep) - [Freeze](#freeze) - [Pin](#pin) @@ -217,6 +218,19 @@ Options: -h, --help Show this message and exit. ``` +#### Create + +```console +$ ddev create -h +Usage: ddev create [OPTIONS] NAME + + Create a new integration. + +Options: + -n, --dry-run Only show what would be created + -h, --help Show this message and exit. +``` + #### Dep ```console diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/__init__.py index eabc6503b2522..f0a25c4ae557e 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/__init__.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/__init__.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from .clean import clean from .config import config +from .create import create from .dep import dep from .manifest import manifest from .release import release @@ -11,6 +12,7 @@ ALL_COMMANDS = ( clean, config, + create, dep, manifest, release, diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/create.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/create.py new file mode 100644 index 0000000000000..0645522cbd800 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/create.py @@ -0,0 +1,128 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os +from collections import defaultdict + +import click + +from .utils import CONTEXT_SETTINGS, abort, echo_info, echo_success +from ..constants import get_root +from ..files import CHECK_FILES +from ..utils import normalize_package_name + +HYPHEN = b'\xe2\x94\x80\xe2\x94\x80'.decode('utf-8') +PIPE = b'\xe2\x94\x82'.decode('utf-8') +PIPE_MIDDLE = b'\xe2\x94\x9c'.decode('utf-8') +PIPE_END = b'\xe2\x94\x94'.decode('utf-8') + + +def tree(): + return defaultdict(tree) + + +def construct_output_info(path, depth, last, is_dir=False): + if depth == 0: + return u'', path, is_dir + else: + if depth == 1: + return ( + u'{}{} '.format( + PIPE_END if last else PIPE_MIDDLE, HYPHEN + ), + path, + is_dir + ) + else: + return ( + u'{} {}{}'.format( + PIPE, + u' ' * 4 * (depth - 2), + u'{}{} '.format(PIPE_END if last or is_dir else PIPE_MIDDLE, HYPHEN) + ), + path, + is_dir + ) + + +def path_tree_output(path_tree, depth=0): + # Avoid possible imposed recursion limits by using a generator. + # See https://en.wikipedia.org/wiki/Trampoline_(computing) + dirs = [] + files = [] + + for path in path_tree: + if len(path_tree[path]) > 0: + dirs.append(path) + else: + files.append(path) + + dirs.sort() + length = len(dirs) + + for i, path in enumerate(dirs, 1): + yield construct_output_info(path, depth, last=i == length and not files, is_dir=True) + + for info in path_tree_output(path_tree[path], depth + 1): + yield info + + files.sort() + length = len(files) + + for i, path in enumerate(files, 1): + yield construct_output_info(path, depth, last=i == length) + + +def display_path_tree(path_tree): + for indent, path, is_dir in path_tree_output(path_tree): + if indent: + echo_info(indent, nl=False) + + if is_dir: + echo_success(path) + else: + echo_info(path) + + +@click.command(context_settings=CONTEXT_SETTINGS, short_help='Create a new integration') +@click.argument('name') +@click.option('--dry-run', '-n', is_flag=True, help='Only show what would be created') +@click.pass_context +def create(ctx, name, dry_run): + """Create a new integration.""" + check_name = normalize_package_name(name) + root = get_root() + path_sep = os.path.sep + + check_dir = os.path.join(root, check_name) + if os.path.exists(check_dir): + abort('Path `{}` already exists!'.format(check_dir)) + + config = { + 'root': check_dir, + 'check_name': check_name, + 'check_name_cap': check_name.capitalize(), + 'check_class': '{}Check'.format(''.join(part.capitalize() for part in check_name.split('_'))), + 'repo_choice': ctx.obj['repo_choice'], + } + + files = [file(config) for file in CHECK_FILES] + file_paths = [file.file_path.replace('{}{}'.format(root, path_sep), '', 1) for file in files] + + path_tree = tree() + for file_path in file_paths: + branch = path_tree + + for part in file_path.split(path_sep): + branch = branch[part] + + if dry_run: + echo_info('Will create in `{}`:'.format(root)) + display_path_tree(path_tree) + return + + for file in files: + file.write() + + echo_info('Created in `{}`:'.format(root)) + display_path_tree(path_tree) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/__init__.py new file mode 100644 index 0000000000000..c39f662994f2b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/__init__.py @@ -0,0 +1,35 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .changelog import Changelog +from .example import ExampleConf +from .manifest import ManifestIn, ManifestJson +from .metadata import MetadataCsv +from .package import PackageAbout, PackageCheck, PackageInit, PackageNamespace +from .readme import Readme +from .reqs import ReqsDevTxt, ReqsIn, ReqsTxt +from .setup import Setup +from .test import TestCheck, TestConf, TestInit +from .tox import Tox + + +CHECK_FILES = ( + Changelog, + ExampleConf, + ManifestIn, + ManifestJson, + MetadataCsv, + PackageAbout, + PackageCheck, + PackageInit, + PackageNamespace, + Readme, + ReqsDevTxt, + ReqsIn, + ReqsTxt, + Setup, + TestCheck, + TestConf, + TestInit, + Tox, +) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/changelog.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/changelog.py new file mode 100644 index 0000000000000..1dc9b725e10b1 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/changelog.py @@ -0,0 +1,21 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE = """\ +# CHANGELOG - {check_name_cap} + +""" + + +class Changelog(File): + def __init__(self, config): + super(Changelog, self).__init__( + os.path.join(config['root'], 'CHANGELOG.md'), + TEMPLATE.format( + check_name_cap=config['check_name_cap'], + ) + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/example.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/example.py new file mode 100644 index 0000000000000..686a4dda22ea3 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/example.py @@ -0,0 +1,21 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE = """\ +init_config: + +instances: + - {} +""" + + +class ExampleConf(File): + def __init__(self, config): + super(ExampleConf, self).__init__( + os.path.join(config['root'], 'datadog_checks', config['check_name'], 'data', 'conf.yaml.example'), + TEMPLATE + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/manifest.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/manifest.py new file mode 100644 index 0000000000000..19ae53ebf827d --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/manifest.py @@ -0,0 +1,78 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os +import uuid + +from .utils import File + +TEMPLATE_IN = """\ +graft datadog_checks +graft tests + +include MANIFEST.in +include README.md +include requirements.in +include requirements.txt +include requirements-dev.txt +include manifest.json + +global-exclude *.py[cod] __pycache__ +""" + +TEMPLATE_JSON = """\ +{{ + "display_name": "{check_name_cap}", + "maintainer": "{maintainer}", + "manifest_version": "1.0.0", + "name": "{check_name}", + "metric_prefix": "{check_name}.", + "metric_to_check": "", + "creates_events": false, + "short_description": "", + "guid": "{guid}", + "support": "{support_type}", + "supported_os": [ + "linux", + "mac_os", + "windows" + ], + "public_title": "Datadog-{check_name_cap} Integration", + "categories": [ + "" + ], + "type": "check", + "doc_link": "https://docs.datadoghq.com/integrations/{check_name}/", + "is_public": true, + "has_logo": true +}} +""" + + +class ManifestIn(File): + def __init__(self, config): + super(ManifestIn, self).__init__( + os.path.join(config['root'], 'MANIFEST.in'), + TEMPLATE_IN + ) + + +class ManifestJson(File): + def __init__(self, config): + if config['repo_choice'] == 'core': + maintainer = 'help@datadoghq.com' + support_type = 'core' + else: + maintainer = '' + support_type = 'contrib' + + super(ManifestJson, self).__init__( + os.path.join(config['root'], 'manifest.json'), + TEMPLATE_JSON.format( + check_name=config['check_name'], + check_name_cap=config['check_name_cap'], + maintainer=maintainer, + support_type=support_type, + guid=uuid.uuid4(), + ) + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/metadata.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/metadata.py new file mode 100644 index 0000000000000..34a6250c5668d --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/metadata.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE = """\ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name +""" + + +class MetadataCsv(File): + def __init__(self, config): + super(MetadataCsv, self).__init__( + os.path.join(config['root'], 'metadata.csv'), + TEMPLATE + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/package.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/package.py new file mode 100644 index 0000000000000..039f7b5d1679b --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/package.py @@ -0,0 +1,70 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE_NAMESPACE = """\ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) +""" + +TEMPLATE_ABOUT = """\ +__version__ = '0.0.1' +""" + +TEMPLATE_INIT = """\ +from .__about__ import __version__ +from .{check_name} import {check_class} + +__all__ = [ + '__version__', + '{check_class}' +] +""" + +TEMPLATE_CHECK = """\ +from datadog_checks.checks import AgentCheck + + +class {check_class}(AgentCheck): + def check(self, instance): + pass +""" + + +class PackageNamespace(File): + def __init__(self, config): + super(PackageNamespace, self).__init__( + os.path.join(config['root'], 'datadog_checks', '__init__.py'), + TEMPLATE_NAMESPACE + ) + + +class PackageAbout(File): + def __init__(self, config): + super(PackageAbout, self).__init__( + os.path.join(config['root'], 'datadog_checks', config['check_name'], '__about__.py'), + TEMPLATE_ABOUT + ) + + +class PackageInit(File): + def __init__(self, config): + super(PackageInit, self).__init__( + os.path.join(config['root'], 'datadog_checks', config['check_name'], '__init__.py'), + TEMPLATE_INIT.format( + check_name=config['check_name'], + check_class=config['check_class'], + ) + ) + + +class PackageCheck(File): + def __init__(self, config): + super(PackageCheck, self).__init__( + os.path.join(config['root'], 'datadog_checks', config['check_name'], '{}.py'.format(config['check_name'])), + TEMPLATE_CHECK.format( + check_class=config['check_class'], + ) + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/readme.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/readme.py new file mode 100644 index 0000000000000..0fd42dd51625d --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/readme.py @@ -0,0 +1,67 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE = """\ +# Agent Check: {name_cap} +## Overview + +This check monitors [{name_cap}][1]. + +## Setup + +### Installation + +The {name_cap} check is included in the [Datadog Agent][2] package, so you don't +need to install anything else on your server. + +### Configuration + +1. Edit the `{name}.d/conf.yaml` file, in the `conf.d/` folder at the root of your + Agent's configuration directory to start collecting your {name} performance data. + See the [sample {name}.d/conf.yaml][3] for all available configuration options. + +2. [Restart the Agent][4] + +### Validation + +[Run the Agent's `status` subcommand][5] and look for `{name}` under the Checks section. + +## Data Collected +### Metrics + +The {name_cap} check does not include any metrics at this time. + +### Service Checks + +The {name_cap} check does not include any service checks at this time. + +### Events + +The {name_cap} check does not include any events at this time. + +## Troubleshooting + +Need help? Contact [Datadog Support][6]. + +[1]: link to integration's site +[2]: https://app.datadoghq.com/account/settings#agent +[3]: https://github.com/DataDog/integrations-core/blob/master/{name}/datadog_checks/{name}/data/conf.yaml.example +[4]: https://docs.datadoghq.com/agent/faq/agent-commands/#start-stop-restart-the-agent +[5]: https://docs.datadoghq.com/agent/faq/agent-commands/#agent-status-and-information +[6]: https://docs.datadoghq.com/help/ +""" + + +class Readme(File): + def __init__(self, config): + super(Readme, self).__init__( + os.path.join(config['root'], 'README.md'), + TEMPLATE.format( + name=config['check_name'], + name_cap=config['check_name_cap'], + ) + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/reqs.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/reqs.py new file mode 100644 index 0000000000000..9054b1f148625 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/reqs.py @@ -0,0 +1,32 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE_DEV = """\ +datadog_checks_dev +""" + + +class ReqsDevTxt(File): + def __init__(self, config): + super(ReqsDevTxt, self).__init__( + os.path.join(config['root'], 'requirements-dev.txt'), + TEMPLATE_DEV + ) + + +class ReqsIn(File): + def __init__(self, config): + super(ReqsIn, self).__init__( + os.path.join(config['root'], 'requirements.in'), + ) + + +class ReqsTxt(File): + def __init__(self, config): + super(ReqsTxt, self).__init__( + os.path.join(config['root'], 'requirements.txt'), + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/setup.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/setup.py new file mode 100644 index 0000000000000..76929a8698be4 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/setup.py @@ -0,0 +1,78 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE = """\ +from setuptools import setup +from codecs import open # To use a consistent encoding +from os import path + +HERE = path.dirname(path.abspath(__file__)) + +# Get version info +ABOUT = {{}} +with open(path.join(HERE, 'datadog_checks', '{check_name}', '__about__.py')) as f: + exec(f.read(), ABOUT) + +# Get the long description from the README file +with open(path.join(HERE, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + + +CHECKS_BASE_REQ = 'datadog_checks_base' + + +setup( + name='datadog-{check_name}', + version=ABOUT['__version__'], + description='The {check_name_cap} check', + long_description=long_description, + long_description_content_type='text/markdown', + keywords='datadog agent {check_name} check', + + # The project's main homepage. + url='https://github.com/DataDog/integrations-core', + + # Author details + author='Datadog', + author_email='packages@datadoghq.com', + + # License + license='BSD', + + # See https://pypi.org/classifiers + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Topic :: System :: Monitoring', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + + # The package we're going to ship + packages=['datadog_checks.{check_name}'], + + # Run-time dependencies + install_requires=[CHECKS_BASE_REQ], + + # Extra files to ship with the wheel package + include_package_data=True, +) +""" + + +class Setup(File): + def __init__(self, config): + super(Setup, self).__init__( + os.path.join(config['root'], 'setup.py'), + TEMPLATE.format( + check_name=config['check_name'], + check_name_cap=config['check_name_cap'], + ) + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/test.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/test.py new file mode 100644 index 0000000000000..1718fb197a8f8 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/test.py @@ -0,0 +1,27 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + + +class TestInit(File): + def __init__(self, config): + super(TestInit, self).__init__( + os.path.join(config['root'], 'tests', '__init__.py'), + ) + + +class TestCheck(File): + def __init__(self, config): + super(TestCheck, self).__init__( + os.path.join(config['root'], 'tests', 'test_{}.py'.format(config['check_name'])), + ) + + +class TestConf(File): + def __init__(self, config): + super(TestConf, self).__init__( + os.path.join(config['root'], 'tests', 'conftest.py'), + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/tox.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/tox.py new file mode 100644 index 0000000000000..ef86324790b05 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/tox.py @@ -0,0 +1,49 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from .utils import File + +TEMPLATE = """\ +[tox] +minversion = 2.0 +basepython = py27 +envlist = + {check_name} + flake8 + +[testenv] +platform = linux|darwin|win32 +deps = + ../datadog_checks_base + -r../datadog_checks_base/requirements.in + -rrequirements-dev.txt +passenv = + DOCKER* + COMPOSE* +commands = + pip install --require-hashes -r requirements.txt + pytest -v + +[testenv:{check_name}] + +[testenv:flake8] +skip_install = true +deps = flake8 +commands = flake8 . + +[flake8] +exclude = .eggs,.tox,build +max-line-length = 120 +""" + + +class Tox(File): + def __init__(self, config): + super(Tox, self).__init__( + os.path.join(config['root'], 'tox.ini'), + TEMPLATE.format( + check_name=config['check_name'], + ) + ) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/files/utils.py b/datadog_checks_dev/datadog_checks/dev/tooling/files/utils.py new file mode 100644 index 0000000000000..d9d46f56750d7 --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/files/utils.py @@ -0,0 +1,22 @@ +# (C) Datadog, Inc. 2018 +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datetime import datetime + +from ...utils import ensure_parent_dir_exists, write_file + +LICENSE_HEADER = """\ +# (C) Datadog, Inc. {year} +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +""".format(year=str(datetime.now().year)) + + +class File(object): + def __init__(self, file_path, contents=''): + self.file_path = file_path + self.contents = LICENSE_HEADER + contents if file_path.endswith('.py') else contents + + def write(self): + ensure_parent_dir_exists(self.file_path) + write_file(self.file_path, self.contents)