create table add global index

README.md

@@ -8,6 +8,7 @@ DynamoDB is not like other document-based databases you might know, and is very
 
 ```
 * boto3
+* six
 * pytz
 * dateutil
 * simplejson
@@ -25,7 +26,7 @@ pip install git+https://github.com/gusibi/dynamodb-py.git@master
 
 ynamodb-py has some sensible defaults for you when you create a new table, including the table name and the primary key column. But you can change those if you like on table creation.
 
-```
+```python
 from dynamodb.model import Model
 from dynamodb.fields import CharField, IntegerField, FloatField, DictField
 from dynamodb.table import Table
@@ -54,6 +55,36 @@ Table(Movies()).update()
 Table(Movies()).delete()
 ```
 
+### Global Secondary Indexes Example
+
+```python
+from dynamodb.model import Model
+from dynamodb.fields import CharField, IntegerField, FloatField, DictField
+from dynamodb.table import Table
+
+class GameScores(Model):
+
+    __table_name__ = 'GameScores'
+
+    ReadCapacityUnits = 10
+    WriteCapacityUnits = 10
+
+    __global_indexes__ = [
+        ('game_scores-index', ('title', 'top_score'), ['title', 'top_score', 'user_id']),
+    ]
+
+    user_id = IntegerField(name='user_id', hash_key=True)
+    title = CharField(name='title', range_key=True)
+    top_score = FloatField(name='top_score', indexed=True)
+    top_score_date = CharField(name='top_score_date')
+    wins = IntegerField(name='wins', indexed=True)
+    losses = IntegerField(name='losses', indexed=True)
+
+
+# query use global secondary indexes
+GameScores.global_query(index_name='game_scores-index').where(GameScores.title.eq("Puzzle Battle")).order_by(GameScores.top_score, asc=False).all()
+```
+
 ## Fields
 You'll have to define all the fields on the model and the data type of each field. Every field on the object must be included here; if you miss any they'll be completely bypassed during DynamoDB's initialization and will not appear on the model objects.
 

dynamodb/errors.py

@@ -24,6 +24,10 @@ class ParameterException(Exception):
     pass
 
 
+class GlobalSecondaryIndexesException(Exception):
+    pass
+
+
 class FieldValidationException(Exception):
 
     def __init__(self, errors, *args, **kwargs):

dynamodb/expression.py

@@ -10,7 +10,7 @@
 from boto3.dynamodb.conditions import Key, Attr
 
 from .errors import ValidationException
-from .helpers import smart_unicode
+from .helpers import smart_text
 
 __all__ = ['Expression']
 
@@ -145,24 +145,26 @@ def add(self, value, path=None, attr_label=None):
         return exp, exp_attr, 'ADD'
 
     def typecast_for_storage(self, value):
-        return smart_unicode(value)
+        return smart_text(value)
 
     def _expression_func(self, op, *values, **kwargs):
+        # print(op, values, kwargs)
         # for use by index ... bad
-        values = map(self.typecast_for_storage, values)
+        # list 是为了兼容py3 python3 中 map 返回的是class map
+        values = list(map(self.typecast_for_storage, values))
         self.op = op
         self.express_args = values
-        use_key = kwargs.get('use_key', False)
+        is_key = kwargs.get('is_key', False)
         if self.hash_key and op != 'eq':
             raise ValidationException('Query key condition not supported')
-        elif self.hash_key or self.range_key or use_key:
-            use_key = True
+        elif self.hash_key or self.range_key or is_key:
+            is_key = True
             func = getattr(Key(self.name), op, None)
         else:
             func = getattr(Attr(self.name), op, None)
         if not func:
             raise ValidationException('Query key condition not supported')
-        return self, func(*values), use_key
+        return self, func(*values), is_key
 
     def _expression(self, op, value):
         if self.use_decimal_types:

dynamodb/fields.py

@@ -9,9 +9,11 @@
 import decimal
 from datetime import datetime, date, timedelta
 
+import six
+
 from .json_import import json
 from .errors import FieldValidationException
-from .helpers import str_time, str_to_time, date2timestamp, timestamp2date, smart_unicode
+from .helpers import str_time, str_to_time, date2timestamp, timestamp2date, smart_text
 from .expression import Expression
 
 
@@ -20,9 +22,13 @@
            'BooleanField', 'DictField', 'SetField', 'ListField']
 
 # TODO
-# 完成index
 # 完成 query scan
 
+if six.PY3:
+    basestring = str
+    unicode = str
+    long = int
+
 
 class DecimalEncoder(json.JSONEncoder):
     # Helper class to convert a DynamoDB item to JSON.
@@ -107,7 +113,7 @@ def typecast_for_read(self, value):
     def typecast_for_storage(self, value):
         """Typecasts the value for storing to DynamoDB."""
         # default store unicode
-        return smart_unicode(value)
+        return smart_text(value)
 
     def value_type(self):
         return unicode
@@ -144,14 +150,14 @@ def __init__(self, min_length=0, max_length=None, **kwargs):
     def typecast_for_read(self, value):
         if value == 'None':
             return ''
-        return smart_unicode(value)
+        return smart_text(value)
 
     def typecast_for_storage(self, value):
         """Typecasts the value for storing to DynamoDB."""
         if value is None:
             return ''
         try:
-            return unicode(value)
+            return smart_text(value)
         except UnicodeError:
             return value.decode('utf-8')
 

dynamodb/helpers.py

@@ -194,6 +194,7 @@ def force_bytes(s, encoding='utf-8', strings_only=False, errors='strict'):
 
 if six.PY3:
     smart_str = smart_text
+    smart_unicode = smart_text
     force_str = force_text
 else:
     smart_str = smart_bytes
@@ -232,7 +233,7 @@ def fn(*args, **kwargs):
 
 
 def get_attribute_type(attribute):
-    from .fields import CharField, IntegerField, DateTimeField, FloatField, TimeField
+    from .fields import CharField, IntegerField, DateTimeField, FloatField, TimeField, BooleanField
     if isinstance(attribute, (CharField, DateTimeField)):
         return 'S'
     elif isinstance(attribute, (IntegerField, FloatField, TimeField)):

dynamodb/model.py

@@ -1,10 +1,12 @@
 #! -*- coding: utf-8 -*-
 import copy
 
+import six
+from six import with_metaclass
 from botocore.exceptions import ClientError
 
 from .table import Table
-from .query import Query
+from .query import Query, GlobalQuery
 from .fields import Attribute
 from .errors import FieldValidationException, ValidationException, ClientException
 from .helpers import get_items_for_storage, cache_for
@@ -21,10 +23,10 @@ def _initialize_attributes(model_class, name, bases, attrs):
     for parent in bases:
         if not isinstance(parent, ModelMetaclass):
             continue
-        for k, v in parent._attributes.iteritems():
+        for k, v in six.iteritems(parent._attributes):
             model_class._attributes[k] = v
 
-    for k, v in attrs.iteritems():
+    for k, v in six.iteritems(attrs):
         if isinstance(v, Attribute):
             model_class._attributes[k] = v
             v.name = v.name or k
@@ -36,18 +38,21 @@ def _initialize_indexes(model_class, name, bases, attrs):
     """
     model_class._local_indexed_fields = []
     model_class._local_indexes = {}
-    model_class._global_indexed_fields = model_class.__global_index__
-    model_class._global_indexes = {}
+    model_class._global_indexed_fields = []
+    model_class._global_indexes = model_class.__global_indexes__
+    model_class._global_secondary_indexes = []
     model_class._hash_key = None
     model_class._range_key = None
     for parent in bases:
         if not isinstance(parent, ModelMetaclass):
             continue
-        for k, v in parent._attributes.iteritems():
+    for k, v in six.iteritems(attrs):
+        if isinstance(v, (Attribute,)):
             if v.indexed:
                 model_class._local_indexed_fields.append(k)
 
-    for k, v in attrs.iteritems():
+    # setting hash_key and range_key and local indexes 
+    for k, v in six.iteritems(attrs):
         if isinstance(v, (Attribute,)):
             if v.indexed:
                 model_class._local_indexed_fields.append(k)
@@ -60,6 +65,38 @@ def _initialize_indexes(model_class, name, bases, attrs):
     if name not in ('ModelBase', 'Model') and not model_class._hash_key:
         raise ValidationException('hash_key is required')
 
+    # setting global_indexes
+    global_indexes = []
+    global_fields = set()
+    for gindex in model_class._global_indexes:
+        index = {}
+        name, primary_key, fields = gindex
+        index['name'] = name
+        # 处理主键
+        if len(primary_key) == 2:
+            hash_key, range_key = primary_key
+        elif len(primary_key) == 1:
+            hash_key, range_key = primary_key[0], None
+        else:
+            raise ValidationException('invalid primary key')
+        index['hash_key'] = hash_key
+        global_fields.add(hash_key)
+        index['range_key'] = range_key
+        if range_key:
+            global_fields.add(range_key)
+        if hash_key not in model_class._attributes:
+            raise ValidationException('invalid hash key: %s' % hash_key)
+        if range_key and range_key not in model_class._attributes:
+            raise ValidationException('invalid range key: %s' % range_key)
+        # 处理备份键
+        for field in fields:
+            if field not in model_class._attributes:
+                raise ValidationException('invalid include field: %s' % field)
+        index['include_fields'] = fields
+        global_indexes.append(index)
+    model_class._global_indexed_fields = global_fields
+    model_class._global_secondary_indexes = global_indexes
+
 
 class ModelMetaclass(type):
 
@@ -68,7 +105,7 @@ class ModelMetaclass(type):
     """
 
     __table_name__ = None
-    __global_index__ = []
+    __global_indexes__ = []
     __local_index__ = {}
 
     def __init__(cls, name, bases, attrs):
@@ -78,9 +115,9 @@ def __init__(cls, name, bases, attrs):
         _initialize_indexes(cls, name, bases, attrs)
 
 
-class ModelBase(object):
+class ModelBase(with_metaclass(ModelMetaclass, object)):
 
-    __metaclass__ = ModelMetaclass
+    #  __metaclass__ = ModelMetaclass
 
     @classmethod
     def create(cls, **kwargs):
@@ -129,7 +166,7 @@ def update(self, *args, **kwargs):
             ReturnConsumedCapacity=ReturnConsumedCapacity)
         if not self.validate_attrs(**kwargs):
             raise FieldValidationException(self._errors)
-        for k, v in kwargs.items():
+        for k, v in six.iteritems(kwargs):
             field = self.attributes[k]
             update_fields[k] = field.typecast_for_storage(v)
         # use storage value
@@ -177,6 +214,11 @@ def query(cls, *args):
         instance = cls()
         return Query(instance, *args)
 
+    @classmethod
+    def global_query(cls, index_name=None, *args):
+        instance = cls()
+        return GlobalQuery(instance, index_name, *args)
+
     @classmethod
     def scan(cls):
         instance = cls()
@@ -238,7 +280,7 @@ def is_valid(self):
 
     def validate_attrs(self, **kwargs):
         self._errors = []
-        for attr, value in kwargs.iteritems():
+        for attr, value in six.iteritems(kwargs):
             field = self.attributes.get(attr)
             if not field:
                 raise ValidationException('Field not found: %s' % attr)
@@ -306,7 +348,7 @@ def fields(self):
 
     def _get_values_for_read(self, values):
         read_values = {}
-        for att, value in values.iteritems():
+        for att, value in six.iteritems(values):
             if att not in self.attributes:
                 continue
             descriptor = self.attributes[att]
@@ -318,7 +360,7 @@ def _get_values_for_storage(self):
         data = {}
         if not self.is_valid():
             raise FieldValidationException(self.errors)
-        for attr, field in self.attributes.iteritems():
+        for attr, field in six.iteritems(self.attributes):
             value = getattr(self, attr)
             if value is not None:
                 data[attr] = field.typecast_for_storage(value)