V 1.3

requirements.txt

@@ -1,8 +1,9 @@
 begins==0.9
-lxml == 2.3.4
-pyquery == 1.2.11
-pyxform == 0.9.22
 cyordereddict==1.0.0
-PyExcelerate==0.6.7
+jsonschema==2.5.1
+lxml==2.3.4
 path.py==8.1.2
+PyExcelerate==0.6.7
+pyquery==1.2.11
+pyxform==0.9.22
 statistics==1.0.3.5

setup.py

@@ -36,7 +36,7 @@ def get_requirements(path):
 dev_requirements, dev_dep_links = get_requirements('dev-requirements.txt')
 
 setup(name='formpack',
-      version='1.2',
+      version='1.3',
       description='Manipulation tools for kobocat forms',
       author='Alex Dorey',
       author_email='alex.dorey@kobotoolbox.org',

src/formpack/errors.py

@@ -4,3 +4,7 @@
 
 class TranslationError(ValueError):
     pass
+
+
+class SchemaError(ValueError):
+    pass

src/formpack/pack.py

@@ -25,15 +25,16 @@
 class FormPack(object):
 
     # TODO: make a clear signature for __init__
-    def __init__(self, versions, title='Submissions', id_string=None,
+    def __init__(self, versions=None, title='Submissions', id_string=None,
                  default_version_id_key='__version__',
+                 strict_schema=False,
                  asset_type=None, submissions_xml=None):
 
         if not versions:
-            raise ValueError('A FormPack must contain at least one FormVersion')
+            versions = []
 
         # accept a single version, but normalize it to an iterable
-        if "content" in versions:
+        if isinstance(versions, dict):
             versions = [versions]
 
         self.versions = OrderedDict()
@@ -44,6 +45,7 @@ def __init__(self, versions, title='Submissions', id_string=None,
         self.id_string = id_string
 
         self.title = title
+        self.strict_schema = strict_schema
 
         if len(self.title) > 31: # excel sheet name size limit
             self.title = self.title[:28] + '...'
@@ -125,9 +127,9 @@ def load_version(self, schema):
         replace_aliases(schema['content'], in_place=True)
         expand_content(schema['content'], in_place=True)
 
-        # TODO: make that an alternative constructor from_json_schema ?
-        # this way we could get rid of the construct accepting a json schema
-        # and pass it the choices, repeat and fieds
+        if self.strict_schema:
+            FormVersion.verify_schema_structure(schema)
+
         form_version = FormVersion(self, schema)
 
         # NB: id_string are readable string unique to the form

src/formpack/schema/datadef.py

@@ -100,16 +100,21 @@ def all_from_json_definition(cls, definition, translation_list):
         all_choices = {}
         for choice_definition in definition:
             choice_name = choice_definition.get('name')
-            if not choice_name:
+            choice_key = choice_definition.get('list_name')
+            if not choice_name or not choice_key:
                 continue
-            choice_key = choice_definition['list_name']
-            try:
-                choices = all_choices[choice_key]
-            except KeyError:
-                choices = all_choices[choice_key] = cls(choice_key)
+
+            if choice_key not in all_choices:
+                all_choices[choice_key] = FormChoice(choice_key)
+            choices = all_choices[choice_key]
 
             option = choices.options[choice_name] = {}
-            _label = choice_definition['label']
+
+            # apparently choices dont need a label if they have an image
+            if 'label' in choice_definition:
+                _label = choice_definition['label']
+            else:
+                _label = choice_definition.get('image')
             if isinstance(_label, basestring):
                 _label = [_label]
             option['labels'] = OrderedDict(zip(translation_list, _label))

src/formpack/schema/fields.py

@@ -90,7 +90,14 @@ def _get_label(self, lang=UNSPECIFIED_TRANSLATION, group_sep='/',
         if hierarchy_in_labels:
             path = []
             for level in self.hierarchy[1:_hierarchy_end]:
-                path.append(level.labels.get(lang) or level.name)
+                _t = level.labels.get(lang)
+                if isinstance(_t, list) and len(_t) is 1:
+                    _t = _t[0]
+                # sometimes, level.labels returns a list
+                if _t:
+                    path.append(_t)
+                else:
+                    path.append(level.name)
             return group_sep.join(path)
 
         return self.labels.get(lang, self.name)

src/formpack/utils/expand_content.py

@@ -1,5 +1,9 @@
 # coding: utf-8
 
+# This module might be more appropriately named "standardize_content"
+# and pass content through to formpack.utils.replace_aliases during
+# the standardization step: expand_content_in_place(...)
+
 from __future__ import (unicode_literals, print_function,
                         absolute_import, division)
 
@@ -11,8 +15,11 @@
 from ..constants import UNTRANSLATED
 
 
-def _convert_special_label_col(content, row, col_shortname,
-                               special_column_details):
+REMOVE_EMPTY_STRINGS = True
+
+
+def _expand_translatable_content(content, row, col_shortname,
+                                 special_column_details):
     _scd = special_column_details
     if 'translation' in _scd:
         translations = content['translations']
@@ -46,23 +53,38 @@ def expand_content_in_place(content):
         content['translations'] = translations
 
     for row in content.get('survey', []):
+        if 'name' in row and row['name'] == None:
+            del row['name']
         if 'type' in row:
             _type = row['type']
             if isinstance(_type, basestring):
                 row.update(_expand_type_to_dict(row['type']))
             elif isinstance(_type, dict):
+                # legacy {'select_one': 'xyz'} format might
+                # still be on kobo-prod
                 row.update({u'type': _type.keys()[0],
                             u'select_from_list_name': _type.values()[0]})
         for key in EXPANDABLE_FIELD_TYPES:
             if key in row and isinstance(row[key], basestring):
                 row[key] = _expand_xpath_to_list(row[key])
         for (key, vals) in specials.iteritems():
             if key in row:
-                _convert_special_label_col(content, row, key, vals)
+                _expand_translatable_content(content, row, key, vals)
+
+        if REMOVE_EMPTY_STRINGS:
+            for (key, val) in row.items():
+                if val == "":
+                    del row[key]
     for row in content.get('choices', []):
         for (key, vals) in specials.iteritems():
             if key in row:
-                _convert_special_label_col(content, row, key, vals)
+                _expand_translatable_content(content, row, key, vals)
+
+    if 'settings' in content and isinstance(content['settings'], list):
+        if len(content['settings']) > 0:
+            content['settings'] = content['settings'][0]
+        else:
+            content['settings'] = {}
 
 
 def expand_content(content, in_place=False):
@@ -95,6 +117,11 @@ def _pluck_uniq_cols(sheet_name):
     _pluck_uniq_cols('choices')
 
     for column_name in uniq_cols.keys():
+        if column_name in ['label', 'hint']:
+            special[column_name] = {
+                'column': column_name,
+                'translation': UNTRANSLATED,
+            }
         if ':' not in column_name:
             continue
         if column_name.startswith('bind:'):
@@ -145,20 +172,23 @@ def _pluck_uniq_cols(sheet_name):
 
 def _expand_type_to_dict(type_str):
     for _re in [
-                '^(select_one)\s+(\w+)$',
-                '^(select_multiple)\s+(\w+)$',
-                '^(select_one_external)\s+(\w+)$',
+                '^(select_one_or_other)\s+(\S+)$',
+                '^(select_one)\s+(\S+)$',
+                '^(select_multiple_or_other)\s+(\S+)$',
+                '^(select_multiple)\s+(\S+)$',
+                '^(select_one_external)\s+(\S+)$',
                ]:
         match = re.match(_re, type_str)
         if match:
             (type_, list_name) = match.groups()
             return {u'type': type_,
                     u'select_from_list_name': list_name}
 
-    _or_other = re.match('^select_one\s+(\w+)\s+or_other$', type_str)
+    _or_other = re.match('^(select_one|select_multiple)\s+(\w+)\s+or.other$',
+                         type_str)
     if _or_other:
-        list_name = _or_other.groups()[0]
-        return {u'type': 'select_one_or_other',
+        (select_mult, list_name) = _or_other.groups()
+        return {u'type': '{}_or_other'.format(select_mult),
                 u'select_from_list_name': list_name}
 
     # if it does not expand, we return the original string

src/formpack/utils/flatten_content.py

@@ -21,6 +21,10 @@ def _iter_through_sheet(sheet_name):
                 _flatten_translated_fields(row, translations)
     _iter_through_sheet('survey')
     _iter_through_sheet('choices')
+
+    if isinstance(survey_content.get('settings'), dict):
+        survey_content['settings'] = [survey_content['settings']]
+
     # do not list translations when only default exists
     if len(translations) == 1 and translations[0] == UNTRANSLATED:
         del survey_content['translations']
@@ -68,6 +72,8 @@ def _flatten_translated_fields(row, translations):
             for i in xrange(0, len(translations)):
                 _t = translations[i]
                 value = items[i]
+                if value is None:
+                    continue
                 if _t is UNTRANSLATED:
                     row[key] = value
                 else:
@@ -80,11 +86,15 @@ def _flatten_survey_row(row):
             row[key] = array_to_xpath(row[key])
     if 'type' in row:
         _type = row['type']
+        if isinstance(row.get('required'), bool):
+            row['required'] = 'true' if row['required'] else 'false'
         if isinstance(_type, dict):
             row['type'] = _stringify_type__depr(_type)
         elif 'select_from_list_name' in row:
             _list_name = row.pop('select_from_list_name')
             if row['type'] == 'select_one_or_other':
                 row['type'] = 'select_one {} or_other'.format(_list_name)
+            elif row['type'] == 'select_multiple_or_other':
+                row['type'] = 'select_multiple {} or_other'.format(_list_name)
             else:
                 row['type'] = '{} {}'.format(_type, _list_name)