Adding tests and container builds

.github/ISSUE_TEMPLATE.md

@@ -1,4 +1,4 @@
-* QueueMe version:
+* cdb version:
 * Python version:
 * Operating System:
 

.github/workflows/deploy.yml

@@ -0,0 +1,28 @@
+name: deploy
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  docker-deploy:
+    runs-on: ubuntu-latest
+    env:
+      CONTAINER: quay.io/vanessa/cdb
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Docker Image
+        run: docker build -f Dockerfile.base -t "${CONTAINER}" .
+      - name: Log In to Quay.io
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login quay.io -u ${{ secrets.DOCKER_USER }} --password-stdin
+      - name: Tag and Push images
+        run: |
+          DOCKER_TAG=$(docker run "${CONTAINER}:latest" --version)
+          echo "Docker tag is: ${DOCKER_TAG}"
+          docker tag "${CONTAINER}:latest" "${CONTAINER}:${DOCKER_TAG}"
+          docker push "${CONTAINER}:latest"
+          docker push "${CONTAINER}:${DOCKER_TAG}"

.github/workflows/main.yml

@@ -0,0 +1,66 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches_ignore: []
+
+jobs:
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup black environment
+        run: conda create --quiet --name black black pyflakes
+
+      - name: Check formatting with black
+        run: |
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate black
+          black --check .
+
+      - name: Check imports with pyflakes
+        run: |
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate black
+          pyflakes .
+
+
+  testing:
+    needs: formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup testing environment
+        run: conda create --quiet --name testing pytest
+
+      - name: Run Tests
+        run: |
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate testing
+          pip install -e .[all]
+          pytest -sv tests/*.py
+
+      - name: Test Docker Build
+        run: docker build -t data-container .
+      - name: Run Container
+        run: |
+          docker run data-container
+
+  build:
+    needs: testing
+    runs-on: ubuntu-latest
+    env:
+      CONTAINER: quay.io/vanessa/cdb
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Docker Image
+        run: docker build -f Dockerfile.base -t ${CONTAINER} .
+      - name: Tag and Preview Container
+        run: |
+          DOCKER_TAG=$(docker run "${CONTAINER}:latest" --version)
+          echo "Docker tag is: ${DOCKER_TAG}"
+          docker run ${CONTAINER} --help

CHANGELOG.md

@@ -13,5 +13,5 @@ and **Merged pull requests**. Critical items to know are:
 
 The versions coincide with releases on pip.
 
-## [0.0.x](https://github.com/singularityhub/cdb/master) (0.0.x)
+## [0.0.x](https://github.com/vsoch/cdb/master) (0.0.x)
  - skeleton release (0.0.0)

Dockerfile

@@ -12,22 +12,22 @@ ADD . /tmp/repo
 WORKDIR /tmp/repo
 RUN pip install .[all]
 
-WORKDIR /scif/data
+WORKDIR /data
 COPY ./tests/data .
-RUN cdb generate /scif/data --out /db.go
+RUN cdb generate /data --out /db.go
 
 FROM golang:1.13-alpine3.10 as builder
 COPY --from=generator /db.go /db.go
-COPY --from=generator /scif/data /scif/data
+COPY --from=generator /data /data
 
 # Dependencies
 RUN apk add git && \
-    go get github.com/singularityhub/containerdb && \
+    go get github.com/vsoch/containerdb && \
     GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /db -i /db.go
 
 FROM scratch
 LABEL MAINTAINER @vsoch
-COPY --from=builder /scif/data /scif/data
+COPY --from=builder /data /data
 COPY --from=builder /db /db
 
 CMD ["/db"]

Dockerfile.base

@@ -0,0 +1,14 @@
+FROM bitnami/minideb:stretch as generator
+# docker build -t data-container .
+ENV PATH /opt/conda/bin:${PATH}
+ENV LANG C.UTF-8
+RUN /bin/bash -c "install_packages wget git ca-certificates && \
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \
+    rm Miniconda3-latest-Linux-x86_64.sh"
+
+ADD . /tmp/repo
+WORKDIR /tmp/repo
+RUN pip install .[all]
+ENTRYPOINT ["cdb"]
+CMD ["--help"]

README.md

@@ -1,6 +1,6 @@
 # Container Database (cdb)
 
-This is the Python support tool for [containerdb](https://github.com/singularityhub/containerdb)
+This is the Python support tool for [containerdb](https://github.com/vsoch/containerdb)
 to support generation of [data containers](https://github.com/singularityhub/data-container).
 Python is more friendly to generating arbitrary data structures, and is popular among the
 data science community, so I chose it for metadata generation instead of using GoLang.
@@ -12,7 +12,7 @@ Generation works as follows:
 
  1. The library will take as input some data folder
  2. The user defines a function to parse each file and generate a dataset, or a default function is used
- 3. A GoLang template is generated to be compiled along with [containerdb](https://github.com/singularityhub/containerdb) to generate a container entrypoint and in-memory database.
+ 3. A GoLang template is generated to be compiled along with [containerdb](https://github.com/vsoch/containerdb) to generate a container entrypoint and in-memory database.
 
 ## Docker Usage
 
@@ -52,7 +52,7 @@ $ pip install cdb
 or
 
 ```bash
-git clone git@github.com:singularityhub/cdb
+git clone git@github.com:vsoch/cdb
 cd cdb
 pip install -e .
 ```
@@ -80,7 +80,7 @@ or do it locally with your own GoLang install and then add to the container.
 For example, to compile:
 
 ```bash
-go get github.com/singularityhub/containerdb && \
+go get github.com/vsoch/containerdb && \
 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /db -i /db.go
 ```
 

cdb/client/generate.py

@@ -9,14 +9,12 @@
 """
 
 from cdb.main import ContainerDatabase
-from cdb.logger import bot
-import sys
 
 
 def main(args, extra):
 
     # Create a Container database
     db = ContainerDatabase(path=args.path)
-    output = db.generate(
+    db.generate(
         output=args.output, template=args.template, force=args.force, func=args.function
     )

cdb/logger/__init__.py

@@ -1,6 +1,8 @@
 import os
 from .message import bot
 
+assert bot
+
 # Shared logging import for both client and default.py (for headless)
 CDB_LOG_LEVEL = os.environ.get("CDB_LOG_LEVEL", "INFO")
 CDB_LOG_LEVELS = ["DEBUG", "CRITICAL", "ERROR", "WARNING", "INFO", "QUIET", "FATAL"]

cdb/main/__init__.py

@@ -9,7 +9,7 @@
 """
 
 
-from cdb.utils.file import read_file, recursive_find, write_file
+from cdb.utils.file import recursive_find, write_file
 from cdb.utils.module import import_module
 from cdb.templates import get_template
 
@@ -18,7 +18,8 @@
 import json
 import logging
 import os
-import re
+import sys
+import types
 
 bot = logging.getLogger("cdb.main")
 
@@ -34,17 +35,27 @@ def __init__(self, path, pattern="*"):
         """extract metadata for some recursive set of data objects
         """
         self.get_files(path, pattern)
+        self.path = path
+        self.pattern = pattern
         self.metadata = {}
 
     def get_files(self, path, pattern="*"):
+        """return files, given update path and pattern
+        """
+        self.path = path
+        self.pattern = pattern
+        return self.files
+
+    @property
+    def files(self):
         """Given a path, check that it exists, and then create an iterator
            to go over files.
         """
-        if path in [".", None]:
-            path = os.getcwd()
-        if not os.path.exists(path):
-            raise FileNotFoundError(f"{path} does not exist.")
-        self.files = recursive_find(path, pattern=pattern)
+        if self.path in [".", None]:
+            self.path = os.getcwd()
+        if not os.path.exists(self.path):
+            raise FileNotFoundError(f"{self.path} does not exist.")
+        return recursive_find(self.path, pattern=self.pattern)
 
     def generate(self, output=None, template="db.go", force=False, func=None):
         """Given an output file name and a template, iterate
@@ -55,7 +66,7 @@ def generate(self, output=None, template="db.go", force=False, func=None):
         template = get_template(template)
 
         # If the output file exists and force is false, exit early
-        if force and os.path.exists(output):
+        if not force and output and os.path.exists(output):
             sys.exit(f"{output} exists, use --force to overwrite.")
 
         # Retrieve the requested function
@@ -71,7 +82,6 @@ def generate(self, output=None, template="db.go", force=False, func=None):
         # Populate the template
         template = Template(template)
         script = template.render(updates=self.metadata)
-        print(script)
 
         if output is not None:
             return write_file(output, script)
@@ -87,6 +97,12 @@ def get_function(self, funcname=None):
         """
         # Default to cdb.functions.basic
         funcname = funcname or "basic"
+
+        # If it's already a function, return it
+        if isinstance(funcname, types.FunctionType):
+            return funcname
+
+        # Otherwise, try to import it
         try:
             module = import_module("cdb.functions")
             func = getattr(module, funcname)

cdb/templates/db.go

@@ -4,7 +4,7 @@ import (
 	"log"
 	"fmt"
 
-	"github.com/singularityhub/containerdb"
+	"github.com/vsoch/containerdb"
 )
 
 func main() {

cdb/utils/file.py

@@ -60,6 +60,7 @@ def write_file(filename, content):
     """
     with open(filename, "w") as fd:
         fd.write(content)
+    return filename
 
 
 def write_json(json_obj, filename, pretty=True):

cdb/version.py

@@ -12,7 +12,7 @@
 AUTHOR = "Vanessa Sochat"
 AUTHOR_EMAIL = "vsochat@stanford.edu"
 NAME = "cdb"
-PACKAGE_URL = "https://github.com/singularityhub/cdb"
+PACKAGE_URL = "https://github.com/vsoch/cdb"
 KEYWORDS = "containers, metadata, database"
 DESCRIPTION = "container database (cdb) metadata generation tool."
 LICENSE = "LICENSE"

setup.py

@@ -12,7 +12,6 @@
 
 
 from setuptools import setup, find_packages
-import codecs
 import os
 
 

tests/test_containerdb.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+
+Copyright (C) 2020 Vanessa Sochat.
+
+This Source Code Form is subject to the terms of the
+Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+
+import os
+import pytest
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+def test_generation(tmp_path):
+    """test generation with default settings
+    """
+    from cdb.main import ContainerDatabase
+
+    path = os.path.join(here, "data")
+
+    # Fail if path doesn't exist
+    with pytest.raises(FileNotFoundError):
+        db = ContainerDatabase(path="_data")
+
+    # Create a Container database
+    db = ContainerDatabase(path=path)
+
+    # File generator should be created on demand
+    assert len(list(db.files)) == 2
+    assert len(list(db.files)) == 2
+
+    # No output means we return a string (default template db.go, func=basic
+    script = db.generate()
+    output_file = os.path.join(str(tmp_path), "db.go")
+    script = db.generate(output=output_file)
+    assert os.path.exists(script)
+
+    # Shouldn't overwrite if force is false
+    with pytest.raises(SystemExit):
+        script = db.generate(output=output_file)
+
+    # Should work given force is True
+    script = db.generate(output=output_file, force=True)
+
+    # Should not work if function doesn't exist
+    with pytest.raises(RuntimeError):
+        script = db.generate(func="doesnotexist")
+
+    # Should work with custom function
+    def return_name(filename):
+        return filename
+
+    script = db.generate(func=return_name)