diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml new file mode 100644 index 0000000..b800826 --- /dev/null +++ b/.github/workflows/CI.yaml @@ -0,0 +1,82 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + schedule: + - cron: '0 18 * * *' + +concurrency: + group: CI-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + changes: + # Check if any file related to Drain3/ CI behavior is changed + # set outputs for other jobs to access for if conditions + name: Check Changes + runs-on: ubuntu-latest + # To prevent error when there's no base branch + if: github.event_name != 'schedule' + timeout-minutes: 10 + outputs: + drain3: ${{ steps.filter.outputs.drain3 }} + steps: + - uses: actions/checkout@v3 # required for push event + - name: Check for file changes + uses: getsentry/paths-filter@v2.11.1 + id: filter + with: + token: ${{ github.token }} + # The following filters indicate a category along with + # the files that should not be ignored by CI when modified. + filters: | + drain3: + - '.github/**/*.yaml' + - '**/*.py' + - '**/Dockerfile*' + - '**/Makefile' + - 'tests/**' + - '**/*.bat' + - '**/*.sh' + - '**/*.ps1' + - '**/pyproject.toml' + - '**/poetry.lock' + - '**/*.cfg' + - '**/*.ini' + list-files: json # logs matched files + build: + runs-on: ubuntu-latest + needs: [changes] + if: | + ( always() && ! cancelled() ) && + ((github.event_name == 'schedule' && github.repository == 'logpai/drain3') || needs.changes.outputs.drain3 == 'true') + + strategy: + matrix: + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + fail-fast: false + env: + PYTHON_VERSION: ${{ matrix.python-version }} + + steps: + - name: Check out Drain3 codebase + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Poetry + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade poetry + + - name: Install dependencies + run: poetry install + + - name: Test with unittest + run: poetry run python -m unittest discover --verbose --start-directory tests diff --git a/.github/workflows/python_310.yaml b/.github/workflows/python_310.yaml deleted file mode 100644 index c391e25..0000000 --- a/.github/workflows/python_310.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Python 3.10 - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - - name: Setup Pipenv - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade pipenv - - - name: Install dependencies - run: pipenv sync - - - name: Test with unittest - run: pipenv run python -m unittest discover --verbose --start-directory tests diff --git a/.github/workflows/python_36.yaml b/.github/workflows/python_36.yaml deleted file mode 100644 index a279e17..0000000 --- a/.github/workflows/python_36.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Python 3.6 - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.6" - - - name: Setup Pipenv - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade pipenv - - - name: Install dependencies - run: pipenv sync - - - name: Test with unittest - run: pipenv run python -m unittest discover --verbose --start-directory tests diff --git a/.github/workflows/python_38.yaml b/.github/workflows/python_38.yaml deleted file mode 100644 index 034dfa4..0000000 --- a/.github/workflows/python_38.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Python 3.8 - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Setup Pipenv - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade pipenv - - - name: Install dependencies - run: pipenv sync - - - name: Test with unittest - run: pipenv run python -m unittest discover --verbose --start-directory tests diff --git a/.github/workflows/python_39.yaml b/.github/workflows/python_39.yaml deleted file mode 100644 index 37e08d3..0000000 --- a/.github/workflows/python_39.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Python 3.9 - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - - name: Setup Pipenv - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade pipenv - - - name: Install dependencies - run: pipenv sync - - - name: Test with unittest - run: pipenv run python -m unittest discover --verbose --start-directory tests diff --git a/.gitignore b/.gitignore index 524d065..36cba61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ **/__pycache__/* MANIFEST dist/* -venv/* +*venv/* .idea/* .vscode/* drain3.egg-info/* diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 0462856..0000000 --- a/Pipfile +++ /dev/null @@ -1,15 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -jsonpickle = "*" -kafka-python = "*" -redis = "*" -cachetools = "*" - -[requires] -python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 5af44bf..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,53 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "914a9dc7849f00ddd0ec12913f1e659a16f87b168063aa5753fbc6ad4eb458b0" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "cachetools": { - "hashes": [ - "sha256:1d9d5f567be80f7c07d765e21b814326d78c61eb0c3a637dffc0e5d1796cb2e2", - "sha256:f469e29e7aa4cff64d8de4aad95ce76de8ea1125a16c68e0d93f65c3c3dc92e9" - ], - "index": "pypi", - "version": "==4.2.1" - }, - "jsonpickle": { - "hashes": [ - "sha256:0be49cba80ea6f87a168aa8168d717d00c6ca07ba83df3cec32d3b30bfe6fb9a", - "sha256:c1010994c1fbda87a48f8a56698605b598cb0fc6bb7e7927559fc1100e69aeac" - ], - "index": "pypi", - "version": "==2.0.0" - }, - "kafka-python": { - "hashes": [ - "sha256:04dfe7fea2b63726cd6f3e79a2d86e709d608d74406638c5da33a01d45a9d7e3", - "sha256:2d92418c7cb1c298fa6c7f0fb3519b520d0d7526ac6cb7ae2a4fc65a51a94b6e" - ], - "index": "pypi", - "version": "==2.0.2" - }, - "redis": { - "hashes": [ - "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", - "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" - ], - "index": "pypi", - "version": "==3.5.3" - } - }, - "develop": {} -} diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..23a392c --- /dev/null +++ b/poetry.lock @@ -0,0 +1,143 @@ +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] + +[package.dependencies] +typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} + +[[package]] +name = "cachetools" +version = "5.3.0" +description = "Extensible memoizing collections and decorators" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, + {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, +] + +[[package]] +name = "importlib-metadata" +version = "6.6.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, + {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, +] + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "jsonpickle" +version = "3.0.1" +description = "Python library for serializing any arbitrary object graph into JSON" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpickle-3.0.1-py2.py3-none-any.whl", hash = "sha256:130d8b293ea0add3845de311aaba55e6d706d0bb17bc123bd2c8baf8a39ac77c"}, + {file = "jsonpickle-3.0.1.tar.gz", hash = "sha256:032538804795e73b94ead410800ac387fdb6de98f8882ac957fcd247e3a85200"}, +] + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] +testing-libs = ["simplejson", "ujson"] + +[[package]] +name = "kafka-python" +version = "2.0.2" +description = "Pure Python client for Apache Kafka" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "kafka-python-2.0.2.tar.gz", hash = "sha256:04dfe7fea2b63726cd6f3e79a2d86e709d608d74406638c5da33a01d45a9d7e3"}, + {file = "kafka_python-2.0.2-py2.py3-none-any.whl", hash = "sha256:2d92418c7cb1c298fa6c7f0fb3519b520d0d7526ac6cb7ae2a4fc65a51a94b6e"}, +] + +[package.extras] +crc32c = ["crc32c"] + +[[package]] +name = "redis" +version = "4.5.4" +description = "Python client for Redis database and key-value store" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "redis-4.5.4-py3-none-any.whl", hash = "sha256:2c19e6767c474f2e85167909061d525ed65bea9301c0770bb151e041b7ac89a2"}, + {file = "redis-4.5.4.tar.gz", hash = "sha256:73ec35da4da267d6847e47f68730fdd5f62e2ca69e3ef5885c6a78a9374c3893"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.2", markers = "python_version <= \"3.11.2\""} +importlib-metadata = {version = ">=1.0", markers = "python_version < \"3.8\""} +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + +[[package]] +name = "typing-extensions" +version = "4.5.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, +] + +[[package]] +name = "zipp" +version = "3.15.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[extras] +all = ["kafka-python", "redis"] +kafka = ["kafka-python"] +redis = ["redis"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.7" +content-hash = "cb23187de36e9666c5f83118b9756f3b0a8c1cf272739a502d9a3fa68f851fe3" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..19db270 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[tool.poetry] +name = "drain3" +version = "0.9.11" +description = "Persistent & streaming log template miner" +authors = ["IBM Research Haifa "] +maintainers = ["Yihao Chen(Superskyyy) "] +readme = "README.md" +license = "MIT" +keywords=['drain', 'log', 'parser', 'IBM', 'template', 'logs', 'miner'] + +classifiers=[ + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + 'Topic :: System :: Monitoring', + "Topic :: Software Development :: Libraries", +] + +packages = [ + { include = "drain3" }, +] + +exclude = ['tests', 'examples'] + +[tool.poetry.build] +generate-setup-file = true + + +[tool.poetry.dependencies] +python = "^3.7" +jsonpickle = "*" +cachetools = "*" +redis = { version = "*", optional = true } +kafka-python = { version = "*", optional = true } + +[tool.poetry.extras] +all=[ + 'redis', + 'kafka-python', +] +kafka= [ + 'kafka-python', +] +redis=[ + 'redis', +] + +[tool.poetry.group.dev.dependencies] + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b49d088..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -jsonpickle==2.0.0 -kafka-python==2.0.2 -redis==3.5.3 -cachetools==4.2.1 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md diff --git a/setup.py b/setup.py deleted file mode 100644 index 5f80a7a..0000000 --- a/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: MIT - -from os import path - -from setuptools import setup - -this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='drain3', - packages=['drain3'], - version="0.9.11", - license='MIT', - description="Persistent & streaming log template miner", - long_description=long_description, - long_description_content_type="text/markdown", - author="IBM Research Haifa", - author_email="david.ohana@ibm.com", - url="https://github.com/IBM/Drain3", - download_url='https://github.com/IBM/Drain3/archive/v_01.tar.gz', - keywords=['drain', 'log', 'parser', 'IBM', 'template', 'logs', 'miner'], - install_requires=[ - 'jsonpickle==1.5.1', 'cachetools==4.2.1' - ], - extras_require={ - "kafka": ['kafka-python==2.0.1'], - "redis": ['redis==3.5.3'], - }, - classifiers=[ - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Topic :: Software Development :: Libraries", - ], -)