Travis and generate config tool

Author: nickvergessen

.travis.yml

@@ -0,0 +1,17 @@
+language: python
+
+python:
+  - "2.7"
+
+branches:
+  only:
+    - master
+    - /^stable\d+(\.\d+)?$/
+
+matrix:
+  fast_finish: true
+
+script:
+
+  # Run smashbox tests
+  - ./travis/check-syntax.sh

bin/config_gen

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+import sys, os.path
+# insert the path to cernafs based on the relative position of this scrip inside the service directory tree
+exeDir = os.path.abspath(os.path.normpath(os.path.dirname(sys.argv[0])))
+pythonDir = os.path.join(os.path.dirname(exeDir), 'python' )
+sys.path.insert(0, pythonDir)
+etcDir = os.path.join(os.path.dirname(exeDir), 'etc')
+defaultTemplateFile = os.path.join(etcDir, 'smashbox.conf.template')
+defaultOutputFile = os.path.join(etcDir, 'smashbox.conf')
+
+import smashbox.configgen.generator as generator
+import smashbox.configgen.processors as processors
+from smashbox.configgen.processors_hooks import LoggingHook
+import logging
+import argparse
+import json
+
+parser = argparse.ArgumentParser(description='Config generator for smashbox')
+parser.add_argument('-i', default=defaultTemplateFile, help='template file to be used', dest='input_file')
+parser.add_argument('-o', default=defaultOutputFile, help='output file', dest='output_file')
+group = parser.add_mutually_exclusive_group()
+group.add_argument('--no-ask', default=None, action='store_false', help='don\'t ask for required keys', dest='ask_keys')
+group.add_argument('--ask', default=None, action='store_true', help='ask for required keys', dest='ask_keys')
+parser.add_argument('-k', default=[], action='append', required=False, help='key=value pairs', dest='keys')
+parser.add_argument('-kt', default=[], action='append', required=False, help='key=type pairs', dest='key_types')
+parser.add_argument('--key-value-file', help='json file containing key-value pairs. The file format should something like {keyname: {value: value, type: type}, oc_server: {value: server.com, type: string}, oc_ssl_enable: {value: True, type: bool}}')
+parser.add_argument('--logfile', help='write logs in this file')
+args = parser.parse_args()
+
+global_vars = {}
+local_vars = {}
+with open(args.input_file) as ifile:
+    code = compile(ifile.read(), args.input_file, 'exec')
+    exec(code, global_vars, local_vars)
+
+overwrite_dict = {}
+
+if args.key_value_file:
+    with open(args.key_value_file, 'r') as f:
+        data = json.load(f)
+        if type(data) is dict:
+            for data_element in data:
+                key = data_element
+                value = str(data[data_element]['value'])
+                if 'type' in data[data_element]:
+                    value = processors.convert_string_to_type(value, data[data_element]['type'])
+                overwrite_dict[key] = value
+
+# convert the keys argument to a dictionary
+key_list = [item.split('=', 1) for item in args.keys]
+key_dict = dict(key_list)
+
+# convert the key_types to [[key, type],[key, type]] and change the type of the values
+key_type_list = [item.split('=', 1) for item in args.key_types]
+for keytype in key_type_list:
+    if keytype[0] in key_dict:
+        key_dict[keytype[0]] = processors.convert_string_to_type(key_dict[keytype[0]], keytype[1])
+overwrite_dict.update(key_dict)
+
+config_generator = generator.Generator()
+config_generator.set_processors_from_data(local_vars['_configgen'])
+
+if args.ask_keys is not None:
+    processor = config_generator.get_processor_by_name('RequiredKeysProcessor')
+    if processor is not None:
+        processor.set_ask(args.ask_keys)
+
+if overwrite_dict:
+    # we need to overwrite keys
+    processor2 = config_generator.get_processor_by_name('OverwritterProcessor')
+    if processor2 is not None:
+        processor2.set_dict_to_merge(overwrite_dict)
+
+# setup logging for each processor
+if args.logfile:
+    logging.basicConfig(level=logging.NOTSET, format='%(asctime)-15s %(levelname)s %(name)s : %(message)s', filename=args.logfile)
+    for p in config_generator.get_processor_list():
+        processor_name = p.get_name()
+        logger = logging.getLogger('%s.%s' % (__name__, processor_name))
+        p.register_observer('logger', LoggingHook(logger, logging.INFO))
+
+    logging.getLogger(__name__).info('ready to start the generation')
+
+# generate the config file
+config_generator.process_data_to_file(local_vars, args.output_file)
+

etc/smashbox.conf.template-owncloud

@@ -0,0 +1,160 @@
+#
+# The _open_SmashBox Project.
+#
+# Author: Jakub T. Moscicki, CERN, 2013
+# License: AGPL
+#
+# this is the main config file template: copy to smashbox.conf and adjust the settings
+#
+# this template should work without changes if you are running your tests directly on the owncloud application server
+#
+
+# this is the top directory where all local working files are kept (test working direcotires, test logs, test data, temporary filesets, ..)
+smashdir = "~/smashdir"
+
+# name of the account used for testing
+# if None then account name is chosen automatically (based on the test name)
+oc_account_name=None
+
+# default number of users for tests involving multiple users (user number is appended to the oc_account_name)
+# this only applies to the tests involving multiple users
+oc_number_test_users=3
+
+# name of the group used for testing
+oc_group_name=None
+
+# default number of groups for tests involving multiple groups (group number is appended to the oc_group_name)
+# this only applies to the tests involving multiple groups
+oc_number_test_groups=1
+
+# password for test accounts: all test account will have the same password
+# if not set then it's an error
+oc_account_password="demo"
+
+# owncloud test server
+# if left blank or "localhost" then the real hostname of the localhost will be set
+oc_server = ''
+
+
+# root of the owncloud installation as visible in the URL
+oc_root = 'owncloud'
+
+# webdav endpoint URI within the oc_server
+import os.path
+oc_webdav_endpoint = os.path.join(oc_root,'remote.php/webdav') # standard owncloud server
+
+# target folder on the server (this may not be compatible with all tests)
+oc_server_folder = ''
+
+# should we use protocols with SSL (https, ownclouds)
+oc_ssl_enabled = True
+
+# how to invoke shell commands on the server
+# for localhost there is no problem - leave it blank
+# for remote host it may be set like this: "ssh -t -l root $oc_server"
+# note: configure ssh for passwordless login 
+# note: -t option is to make it possible to run sudo
+oc_server_shell_cmd = ""
+
+# Data directory on the owncloud server.
+# 
+oc_server_datadirectory = os.path.join('/var/www/html',oc_root, 'data')
+
+# a path to server side tools (create_user.php, ...)  
+#
+# it may be specified as relative path "dir" and then resolves to
+# <smashbox>/dir where <smashbox> is the top-level of of the tree
+# containing THIS configuration file
+#
+
+oc_server_tools_path = "server-tools"
+
+# a path to ocsync command with options
+# this path should work for all client hosts
+#
+# it may be specified as relative path "dir" and then resolves to
+# <smashbox>/dir where <smashbox> is the top-level of of the tree
+# containing THIS configuration file
+#
+oc_sync_cmd = "client/build/mirall/bin/owncloudcmd --trust"
+
+# number of times to repeat ocsync run every time
+oc_sync_repeat = 1
+
+####################################
+
+# unique identifier of your test run
+# if None then the runid is chosen automatically (and stored in this variable)
+runid = None
+
+# if True then the local working directory path will have the runid added to it automatically
+workdir_runid_enabled=False
+
+# if True then the runid will be part of the oc_account_name automatically
+oc_account_runid_enabled=False
+
+####################################
+
+# this defines the default account cleanup procedure
+#   - "delete": delete account if exists and then create a new account with the same name
+#   - "keep": don't delete existing account but create one if needed
+#
+# these are not implemeted yet:
+#   - "sync_delete": delete all files via a sync run
+#   - "webdav_delete": delete all files via webdav DELETE request
+#   - "filesystem_delete": delete all files directly on the server's filesystem
+oc_account_reset_procedure = "delete"
+
+# this defined the default local run directory reset procedure
+#   - "delete": delete everything in the local run directory prior to running the test
+#   - "keep": keep all files (from the previous run)
+rundir_reset_procedure = "delete"
+
+web_user = "www-data"
+
+oc_admin_user = "at_admin"
+oc_admin_password = "admin"
+
+# cleanup imported namespaces
+del os
+
+# Verbosity of curl client. 
+# If none then verbosity is on when smashbox run in --debug mode.
+# set it to True or False to override
+# 
+pycurl_verbose = None
+
+# scp port to be used in scp commands, used primarily when copying over the server log file
+scp_port = 22
+
+# user that can r+w the owncloud.log file (needs to be configured for passwordless login)
+oc_server_log_user = "www-data"
+
+#
+# Reset the server log file and verify that no exceptions and other known errors have been logged
+#
+oc_check_server_log = False
+
+from collections import OrderedDict
+_configgen = OrderedDict([('KeyRemoverProcessor',
+                                    {'keylist': ('_configgen', 'oc_server', 'oc_ssl_enabled',
+                                    'oc_admin_user', 'oc_admin_password',
+                                    'oc_root', 'oc_webdav_endpoint', 'oc_server_shell_cmd',
+                                    'oc_sync_cmd', 'scp_port')}),
+                            ('OverwritterProcessor',
+                                    {'dict_to_merge': {}}),
+                            ('RequiredKeysProcessor',
+                                    {'keylist': [
+                                        {'name': 'oc_server', 'help_text': 'ip or hostname of the server where owncloud is located, including the port, such as "10.20.30.40:8080"'},
+                                        {'name': 'oc_ssl_enabled', 'type': 'bool', 'default': False, 'help_text': 'if you access to the server through https, set this to True'},
+                                        {'name': 'oc_root', 'help_text': 'the path for the url to be added after the server. To access to "http://server.com/owncloud" use "owncloud", leave it empty if you want to access to "http://server.com/"'},
+                                        {'name': 'oc_webdav_endpoint', 'help_text': 'the path for the webdav endpoint. If the webdav endpoint is in "http://server.com/owncloud/remote.php/webdav" use "owncloud/remote.php/webdav"', 'default': 'remote.php/webdav'},
+                                        {'name': 'oc_admin_user', 'default':'admin'},
+                                        {'name': 'oc_admin_password', 'default': 'Password'},
+                                        {'name': 'oc_server_shell_cmd', 'help_text': 'ssh command to connect to the server such as "ssh -t -l root <server_url>" (include the server). Leave it empty if the server is localhost'},
+                                        {'name': 'scp_port', 'type': 'int', 'default': 22, 'help_text': 'port for scp commands accessing the owncloud server'},
+                                        {'name': 'oc_sync_cmd', 'default': '/usr/bin/owncloudcmd --trust', 'help_text': 'owncloudcmd command. Use the absolute path to the app and any required option'},
+                                        ],
+                                    'ask': True}),
+                            ('SortProcessor', None)])
+del OrderedDict

python/smashbox/configgen/__init__.py

@@ -0,0 +1,110 @@
+import smashbox.configgen.processors as processors
+
+class Generator(object):
+    '''
+    Class to generate configuration files.
+
+    You need to set a processor chain in order to process the dict-like object
+    and write it into a file. If no processor is set, it will write the same object
+
+    Result may vary depending the processor chain being used
+    '''
+    def __init__(self, processor_list = None):
+        '''
+        Initialize the object with the processor chain set, or with an empty chain
+        if None
+        '''
+        self.processor_list = [] if processor_list == None else processor_list
+
+    def insert_processor(self, i, processor):
+        '''
+        Insert a new processor in the "i" position
+        Check list.insert for details
+        '''
+        self.processor_list.insert(i, processor)
+
+    def append_processor(self, processor):
+        '''
+        Append the processor to the end of the chain
+        '''
+        self.processor_list.append(processor)
+
+    def get_processor_list(self):
+        '''
+        Get the processor list / chain
+        '''
+        return self.processor_list
+
+    def get_processor_by_name(self, name):
+        '''
+        Get the processor by name or None if it's not found
+        '''
+        for p in self.processor_list:
+            if p.get_name() == name:
+                return p
+
+    def process_dict(self, local_dict):
+        '''
+        Process the dictionary. It will go through all the process chain and it will be
+        returned after that.
+        '''
+        for p in self.processor_list:
+            local_dict = p.do_process(local_dict)
+        return local_dict
+
+    def write_dict(self, output_file, local_dict):
+        '''
+        Write the dictionary into a file. It will be readable by using the execfile
+        function, which should be the same or similar format that the smashbox.conf.template
+        file has, and MUST be a valid smashbox.conf file
+        '''
+        with open(output_file, 'w') as f:
+            for key in local_dict:
+                f.write('%s = %s\n' % (key, repr(local_dict[key])))
+
+    def generate_new_config(self, input_file, output_file):
+        '''
+        Generate a new configuration file from the input_file. The input file should be
+        similar to the smashbox.conf.template. The processor chain must be set before
+        calling this function
+        '''
+        input_globals = {}
+        input_locals = {}
+        execfile(input_file, input_globals, input_locals)
+
+        input_locals = self.process_dict(input_locals)
+        self.write_dict(output_file, input_locals)
+
+    def set_processors_from_data(self, processor_data):
+        '''
+        Set the processor chain based on the data passed as parameter. Check the
+        _configgen variable in the smashbox.conf.template for working data
+
+        The processor_data should be a dictionary-like. Due to the order of the processor
+        matters, an OrderedDict is recommended.
+        The keys of the dictionary are
+        the class name of the processor that will be used (from the
+        smashbox.configgen.processors module). Currently there are only 4 processors available.
+        The value for the key should also be a dictionary to initialize the processor. Only
+        one parameter will be passed, that's why a dictionary is recommended, although
+        what you must pass depends on the specific processor.
+        '''
+        for key in processor_data:
+            if hasattr(processors, key):
+                processor_class = getattr(processors, key)
+                if not issubclass(processor_class, processors.BasicProcessor):
+                    continue
+                values = processor_data[key]
+                processor = processor_class(values)
+                self.append_processor(processor)
+            else:
+                pass
+
+    def process_data_to_file(self, data, output_file):
+        '''
+        Process the data passed as parameter through the chain and write the result
+        to the file
+        '''
+        data_to_output = self.process_dict(data)
+        self.write_dict(output_file, data_to_output)
+