Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
name: deploy

on:
pull_request: []
push:
branches:
- master
Expand All @@ -9,17 +10,22 @@ jobs:
docker-deploy:
runs-on: ubuntu-latest
env:
CONTAINER: quay.io/vanessa/cdb
CONTAINER: ghcr.io/vsoch/cdb
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
- name: Build Docker Image
run: docker build -f Dockerfile.base -t "${CONTAINER}" .
- name: Log In to Quay.io
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login quay.io -u ${{ secrets.DOCKER_USER }} --password-stdin

- name: GHCR Login
if: (github.event_name != 'pull_request')
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Tag and Push images
if: (github.event_name != 'pull_request')
run: |
DOCKER_TAG=$(docker run "${CONTAINER}:latest" --version)
echo "Docker tag is: ${DOCKER_TAG}"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
formatting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4

- name: Setup black environment
run: conda create --quiet --name black black pyflakes
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM bitnami/minideb:stretch as generator
FROM bitnami/minideb as generator
# docker build -t data-container .
ENV PATH /opt/conda/bin:${PATH}
ENV LANG C.UTF-8
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile.base
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM bitnami/minideb:stretch as generator
FROM bitnami/minideb as generator
# docker build -t data-container .
ENV PATH /opt/conda/bin:${PATH}
ENV LANG C.UTF-8
Expand Down
20 changes: 13 additions & 7 deletions cdb/client/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,26 @@ def get_parser():

description = "actions for cdb"
subparsers = parser.add_subparsers(
help="cdb actions", title="actions", description=description, dest="command",
help="cdb actions",
title="actions",
description=description,
dest="command",
)

# print version and exit
subparsers.add_parser("version", help="show software version")

# Generate a key for the interface
generate = subparsers.add_parser(
"generate", help="generate a template go script for container entrypoint.",
"generate",
help="generate a template go script for container entrypoint.",
)

generate.add_argument(
"path", help="Path to dataset files.", nargs="?", default=".",
"path",
help="Path to dataset files.",
nargs="?",
default=".",
)

# Specify output file, defaults to db.go
Expand Down Expand Up @@ -91,14 +98,13 @@ def get_parser():


def main():
"""main entrypoint for cdb
"""
"""main entrypoint for cdb"""

parser = get_parser()

def help(return_code=0):
"""print help, including the software version and active client
and exit with return code.
"""print help, including the software version and active client
and exit with return code.
"""
version = cdb.__version__

Expand Down
1 change: 0 additions & 1 deletion cdb/client/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@


def main(args, extra):

# Create a Container database
db = ContainerDatabase(path=args.path)
db.generate(
Expand Down
3 changes: 1 addition & 2 deletions cdb/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@


def basic(filename):
"""Given a filename, return a dictionary with basic metadata about it
"""
"""Given a filename, return a dictionary with basic metadata about it"""
st = os.stat(filename)
return {"size": st.st_size, "sha256": get_file_hash(filename)}
55 changes: 26 additions & 29 deletions cdb/logger/message.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ def success(self, message):

def useColor(self):
"""useColor will determine if color should be added
to a print. Will check if being run in a terminal, and
if has support for asci
to a print. Will check if being run in a terminal, and
if has support for asci
"""
COLORIZE = get_user_color_preference()
if COLORIZE is not None:
Expand All @@ -94,7 +94,7 @@ def useColor(self):

def addColor(self, level, text):
"""addColor to the prompt (usually prefix) if terminal
supports, and specified to do so
supports, and specified to do so
"""
if self.colorize:
if level in self.colors:
Expand All @@ -103,7 +103,7 @@ def addColor(self, level, text):

def emitError(self, level):
"""determine if a level should print to
stderr, includes all levels but INFO and QUIET
stderr, includes all levels but INFO and QUIET
"""
if level in [
ABORT,
Expand All @@ -126,20 +126,19 @@ def emitOutput(self, level):
return False

def isEnabledFor(self, messageLevel):
"""check if a messageLevel is enabled to emit a level
"""
"""check if a messageLevel is enabled to emit a level"""
if messageLevel <= self.level:
return True
return False

def emit(self, level, message, prefix=None, color=None):
"""emit is the main function to print the message
optionally with a prefix
optionally with a prefix

Arguments:
- level (int) : the level of the message
- message (str) : the message to print
- prefix (str) : a prefix for the message
Arguments:
- level (int) : the level of the message
- message (str) : the message to print
- prefix (str) : a prefix for the message
"""
if color is None:
color = level
Expand Down Expand Up @@ -172,15 +171,15 @@ def emit(self, level, message, prefix=None, color=None):

def write(self, stream, message):
"""write will write a message to a stream,
first checking the encoding
first checking the encoding
"""
if isinstance(message, bytes):
message = message.decode("utf-8")
stream.write(message)

def get_logs(self, join_newline=True):
"""'get_logs will return the complete history, joined by newline
(default) or as is.
(default) or as is.
"""
if join_newline:
return "\n".join(self.history)
Expand All @@ -198,15 +197,14 @@ def show_progress(
symbol=None,
):
"""create a terminal progress bar, default bar shows for verbose+
Parameters
==========
iteration: current iteration (Int)
total: total iterations (Int)
length: character length of bar (Int)

Parameters
==========
iteration: current iteration (Int)
total: total iterations (Int)
length: character length of bar (Int)
"""
if not self.level == QUIET:

percent = 100 * (iteration / float(total))
progress = int(length * iteration // total)

Expand Down Expand Up @@ -289,18 +287,17 @@ def debug(self, message):
self.emit(DEBUG, message, "DEBUG")

def is_quiet(self):
"""is_quiet returns true if the level is under 1
"""
"""is_quiet returns true if the level is under 1"""
if self.level < 1:
return False
return True

# Terminal ------------------------------------------

def table(self, rows, col_width=2):
"""table will print a table of entries. If the rows is
a dictionary, the keys are interpreted as column names. if
not, a numbered list is used.
"""table will print a table of entries. If the rows is
a dictionary, the keys are interpreted as column names. if
not, a numbered list is used.
"""

labels = [str(x) for x in range(1, len(rows) + 1)]
Expand All @@ -317,9 +314,9 @@ def table(self, rows, col_width=2):

def get_logging_level():
"""get_logging_level will configure a logging to standard out based on
selected level, which should be in an environment variable called
MESSAGELEVEL. if MESSAGELEVEL is not set, the maximum level
(5) is assumed (all messages).
selected level, which should be in an environment variable called
MESSAGELEVEL. if MESSAGELEVEL is not set, the maximum level
(5) is assumed (all messages).
"""
level = os.environ.get("CDB_MESSAGELEVEL", INFO)

Expand Down Expand Up @@ -361,7 +358,7 @@ def get_user_color_preference():

def convert2boolean(arg):
"""convert2boolean is used for environmental variables that must be
returned as boolean
returned as boolean
"""
if not isinstance(arg, bool):
return arg.lower() in ("yes", "true", "t", "1", "y")
Expand Down
24 changes: 10 additions & 14 deletions cdb/main/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,30 +26,28 @@

class ContainerDatabase:
"""A container database metadata generator will take some input folder
of files, and recursively run a user-provided or cdb-provided function
to extract metadata for each one. If no custom function is provided,
we provide basic metadata about file versions.
of files, and recursively run a user-provided or cdb-provided function
to extract metadata for each one. If no custom function is provided,
we provide basic metadata about file versions.
"""

def __init__(self, path, pattern="*"):
"""extract metadata for some recursive set of data objects
"""
"""extract metadata for some recursive set of data objects"""
self.get_files(path, pattern)
self.path = path
self.pattern = pattern
self.metadata = {}

def get_files(self, path, pattern="*"):
"""return files, given update path and pattern
"""
"""return files, given update path and pattern"""
self.path = path
self.pattern = pattern
return self.files

@property
def files(self):
"""Given a path, check that it exists, and then create an iterator
to go over files.
to go over files.
"""
if self.path in [".", None]:
self.path = os.getcwd()
Expand All @@ -59,8 +57,8 @@ def files(self):

def generate(self, output=None, template="db.go", force=False, func=None):
"""Given an output file name and a template, iterate
through files and generate the in-memory database golang file to
compile into a container.
through files and generate the in-memory database golang file to
compile into a container.
"""
bot.debug(f"Loading template {template}")
template = get_template(template)
Expand Down Expand Up @@ -91,13 +89,11 @@ def generate(self, output=None, template="db.go", force=False, func=None):
return script

def export_dockerfile(self, name="Dockerfile"):
"""export a Dockerfile that will perform a multistage build to
"""
"""export a Dockerfile that will perform a multistage build to"""
pass

def get_function(self, funcname=None):
"""Given a function name, return it. Exit on error if not found.
"""
"""Given a function name, return it. Exit on error if not found."""
# Default to cdb.functions.basic
funcname = funcname or "basic"

Expand Down
4 changes: 2 additions & 2 deletions cdb/templates/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

def get_template(name, load=True):
"""Given the name of a template (a go file to use to generate the dataset
metadata) return the template path if it exists. Otherwise, exit on
error
metadata) return the template path if it exists. Otherwise, exit on
error
"""
# The user can provide a custom template, must have {{ updates }} to include
if not os.path.exists(name):
Expand Down
Loading