Commits

Jean-Tiare Le Bigot committed 3cae696

add preliminary scan method, test, bugfixes

  • Participants
  • Parent commits 91900fe

Comments (0)

Files changed (8)

ddbmock/database/item.py

                     "Expected field '{}'' = '{}'. Got '{}'".format(
                     fieldname, condition[u'Value'], self[fieldname]))
 
+    def match(self, conditions):
+        for name, condition in conditions.iteritems():
+            if not self.field_match(name, condition):
+                return False
+
+        return True
+
     def field_match(self, name, condition):
         """Check if a field matches a condition. Return False when field not
         found, or do not match. If condition is None, it is considered to match.
 
         # read the item
         if name not in self:
-            return False
-        value = self[name]
+            value = None
+        else:
+            value = self[name]
 
         # Load the test operator from the comparison module. Thamks to input
         # validation, no try/except required

ddbmock/database/table.py

 
         hk_name = self.hash_key.read(hash_key)
         rk_name = self.range_key.name
-        data = self.data[hk_name]
+        results = []
 
-        return [item.filter(fields) for item in data.values()
-                if item.field_match(rk_name, rk_condition)], None
+        for item in self.data[hk_name].values():
+            if item.field_match(rk_name, rk_condition):
+                results.append(item.filter(fields))
+
+        return results, None
+
+    def scan(self, scan_conditions, fields, start, limit):
+        """Scans a whole table, no matter the structure, and return matches as
+        well as the the last_evaluated key if applicable and the actually scanned
+        item count.
+
+        :ivar scan_conditions: Dict of key:conditions to match items against. If None, all is returned.
+        :ivar fields: return only these fields is applicable
+        :ivar start: key structure. where to start iteration
+        :ivar limit: max number of items to parse in this batch
+        :return: results, last_key, scanned_count
+        """
+        #FIXME: naive implementation (too)
+        #TODO:
+        # - reverse
+        # - esk
+        # - limit
+        # - size limit
+        # - last evaluated key
+
+        scanned_count = 0
+        results = []
+
+        for outer in self.data.values():
+            for item in outer.values():
+                scanned_count += 1
+                if item.match(scan_conditions):
+                    results.append(item.filter(fields))
+
+        return results, None, scanned_count
+
 
     @classmethod
     def from_dict(cls, data):

ddbmock/tests/functional/boto/test_query.py

         from ddbmock.database.db import DynamoDB
 
         expected = {
-            "Count": 5,
-            "Items": [
-                {"relevant_data": {"S": "titi"}, "hash_key": {"N": "123"}, "range_key": {"S": "Waldo-3"}},
-                {"relevant_data": {"S": "tete"}, "hash_key": {"N": "123"}, "range_key": {"S": "Waldo-2"}},
-                {"relevant_data": {"S": "tata"}, "hash_key": {"N": "123"}, "range_key": {"S": "Waldo-1"}},
-                {"relevant_data": {"S": "tutu"}, "hash_key": {"N": "123"}, "range_key": {"S": "Waldo-5"}},
-                {"relevant_data": {"S": "toto"}, "hash_key": {"N": "123"}, "range_key": {"S": "Waldo-4"}},
+            u"Count": 5,
+            u"Items": [
+                {u"relevant_data": {u"S": u"titi"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-3"}},
+                {u"relevant_data": {u"S": u"tete"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-2"}},
+                {u"relevant_data": {u"S": u"tata"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-1"}},
+                {u"relevant_data": {u"S": u"tutu"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-5"}},
+                {u"relevant_data": {u"S": u"toto"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-4"}},
             ],
-            "ConsumedCapacityUnits": 2.5,
+            u"ConsumedCapacityUnits": 2.5,
         }
 
         db = connect_boto()

ddbmock/tests/functional/boto/test_scan.py

+# -*- coding: utf-8 -*-
+
+import unittest
+import boto
+
+TABLE_NAME = 'Table-HR'
+TABLE_NAME_404 = 'Waldo'
+TABLE_RT = 45
+TABLE_WT = 123
+TABLE_RT2 = 10
+TABLE_WT2 = 10
+TABLE_HK_NAME = u'hash_key'
+TABLE_HK_TYPE = u'N'
+TABLE_RK_NAME = u'range_key'
+TABLE_RK_TYPE = u'S'
+
+HK_VALUE1 = u'123'
+HK_VALUE2 = u'456'
+HK_VALUE3 = u'789'
+RK_VALUE1 = u'Waldo-1'
+RK_VALUE2 = u'Waldo-2'
+RK_VALUE3 = u'Waldo-3'
+RK_VALUE4 = u'Waldo-4'
+RK_VALUE5 = u'Waldo-5'
+
+
+ITEM1 = {
+    TABLE_HK_NAME: {TABLE_HK_TYPE: HK_VALUE1},
+    TABLE_RK_NAME: {TABLE_RK_TYPE: RK_VALUE1},
+    u'relevant_data': {u'S': u'tata'},
+}
+ITEM2 = {
+    TABLE_HK_NAME: {TABLE_HK_TYPE: HK_VALUE1},
+    TABLE_RK_NAME: {TABLE_RK_TYPE: RK_VALUE2},
+    u'relevant_data': {u'S': u'tete'},
+}
+ITEM3 = {
+    TABLE_HK_NAME: {TABLE_HK_TYPE: HK_VALUE2},
+    TABLE_RK_NAME: {TABLE_RK_TYPE: RK_VALUE3},
+    u'relevant_data': {u'S': u'titi'},
+}
+ITEM4 = {
+    TABLE_HK_NAME: {TABLE_HK_TYPE: HK_VALUE3},
+    TABLE_RK_NAME: {TABLE_RK_TYPE: RK_VALUE4},
+    u'relevant_data': {u'S': u'toto'},
+}
+ITEM5 = {
+    TABLE_HK_NAME: {TABLE_HK_TYPE: HK_VALUE3},
+    TABLE_RK_NAME: {TABLE_RK_TYPE: RK_VALUE5},
+    u'relevant_data': {u'S': u'tutu'},
+}
+
+# Please note that most query features are not yet implemented hence not tested
+class TestGetItem(unittest.TestCase):
+    def setUp(self):
+        from ddbmock.database.db import DynamoDB
+        from ddbmock.database.table import Table
+        from ddbmock.database.key import PrimaryKey
+
+        db = DynamoDB()
+        db.hard_reset()
+
+        hash_key = PrimaryKey(TABLE_HK_NAME, TABLE_HK_TYPE)
+        range_key = PrimaryKey(TABLE_RK_NAME, TABLE_RK_TYPE)
+
+        self.t1 = Table(TABLE_NAME, TABLE_RT, TABLE_WT, hash_key, range_key)
+
+        db.data[TABLE_NAME]  = self.t1
+
+        self.t1.put(ITEM1, {})
+        self.t1.put(ITEM2, {})
+        self.t1.put(ITEM3, {})
+        self.t1.put(ITEM4, {})
+        self.t1.put(ITEM5, {})
+
+    def tearDown(self):
+        from ddbmock.database.db import DynamoDB
+        DynamoDB().hard_reset()
+
+    def test_scan_all(self):
+        from ddbmock import connect_boto
+        from ddbmock.database.db import DynamoDB
+
+        expected = {
+            u"Count": 5,
+            u"ScannedCount": 5,
+            u"Items": [
+                {u"relevant_data": {u"S": u"tete"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-2"}},
+                {u"relevant_data": {u"S": u"tata"}, u"hash_key": {u"N": u"123"}, u"range_key": {u"S": u"Waldo-1"}},
+                {u"relevant_data": {u"S": u"tutu"}, u"hash_key": {u"N": u"789"}, u"range_key": {u"S": u"Waldo-5"}},
+                {u"relevant_data": {u"S": u"toto"}, u"hash_key": {u"N": u"789"}, u"range_key": {u"S": u"Waldo-4"}},
+                {u"relevant_data": {u"S": u"titi"}, u"hash_key": {u"N": u"456"}, u"range_key": {u"S": u"Waldo-3"}},
+            ],
+            u"ConsumedCapacityUnits": 2.5,
+        }
+
+        db = connect_boto()
+
+        ret = db.layer1.scan(TABLE_NAME, None)
+        self.assertEqual(expected, ret)
+
+    def test_scan_all_filter_fields(self):
+        from ddbmock import connect_boto
+        from ddbmock.database.db import DynamoDB
+
+        expected = {
+            u"Count": 5,
+            u"ScannedCount": 5,
+            u"Items": [
+                {u"relevant_data": {u"S": "tete"}},
+                {u"relevant_data": {u"S": "tata"}},
+                {u"relevant_data": {u"S": "tutu"}},
+                {u"relevant_data": {u"S": "toto"}},
+                {u"relevant_data": {u"S": "titi"}},
+            ],
+            u"ConsumedCapacityUnits": 2.5,
+        }
+        fields = [u'relevant_data']
+
+        db = connect_boto()
+
+        ret = db.layer1.scan(TABLE_NAME, None, fields)
+        self.assertEqual(expected, ret)
+
+    # No need to test all conditions/type mismatch as they are unit tested
+    def test_scan_condition_filter_fields_in(self):
+        from ddbmock import connect_boto
+        from ddbmock.database.db import DynamoDB
+
+        expected = {
+            u"Count": 3,
+            u"ScannedCount": 5,
+            u"Items": [
+                {u"relevant_data": {u"S": u"tata"}},
+                {u"relevant_data": {u"S": u"toto"}},
+                {u"relevant_data": {u"S": u"titi"}},
+            ],
+            u"ConsumedCapacityUnits": 2.5,
+        }
+
+        conditions = {
+            "relevant_data": {
+                "AttributeValueList": [{"S":"toto"},{"S":"titi"},{"S":"tata"}],
+                "ComparisonOperator": "IN",
+            }
+        }
+        fields = [u'relevant_data']
+
+        db = connect_boto()
+
+        ret = db.layer1.scan(TABLE_NAME, conditions, fields)
+        self.assertEqual(expected, ret)
+
+    def test_scan_condition_filter_fields_contains(self):
+        from ddbmock import connect_boto
+        from ddbmock.database.db import DynamoDB
+
+        expected = {
+            u"Count": 1,
+            u"ScannedCount": 5,
+            u"Items": [
+                {u"relevant_data": {u"S": u"toto"}},
+            ],
+            u"ConsumedCapacityUnits": 2.5,
+        }
+
+        conditions = {
+            "relevant_data": {
+                "AttributeValueList": [{"S":"to"}],
+                "ComparisonOperator": "CONTAINS",
+            }
+        }
+        fields = [u'relevant_data']
+
+        db = connect_boto()
+
+        ret = db.layer1.scan(TABLE_NAME, conditions, fields)
+        self.assertEqual(expected, ret)
+
+    def test_scan_validation_error(self):
+        from ddbmock import connect_boto
+        from ddbmock.database.db import DynamoDB
+        from boto.dynamodb.exceptions import DynamoDBValidationError
+
+        expected = {
+            u"Count": 1,
+            u"ScannedCount": 5,
+            u"Items": [
+                {u"relevant_data": {u"S": u"toto"}},
+            ],
+            u"ConsumedCapacityUnits": 2.5,
+        }
+
+        conditions = {
+            "relevant_data": {
+                "AttributeValueList": [{"S":"to"},{"S":"ta"}],
+                "ComparisonOperator": "CONTAINS",
+            }
+        }
+        fields = [u'relevant_data']
+
+        db = connect_boto()
+
+        self.assertRaises(DynamoDBValidationError, db.layer1.scan,
+            TABLE_NAME, conditions, fields
+        )

ddbmock/validators/scan.py

+# -*- coding: utf-8 -*-
+
+from .types import (
+    table_name, optional, item_schema, consistent_read, limit, scan_filter,
+    attributes_to_get_schema, key_field_value, boolean, get_key_schema)
+
+post = {
+    u'TableName': table_name,
+    optional(u'ScanFilter'): scan_filter,
+    optional(u'Count'): boolean,
+    optional(u'Limit'): limit,
+    optional(u'ExclusiveStartKey'): get_key_schema,
+    optional(u'AttributesToGet'): attributes_to_get_schema, #FIXME: handle default
+}

ddbmock/validators/types.py

     field_name: update_action_schema
 }
 
+# Conditions shared by query and scan
 range_key_condition = any(
     {
         u"ComparisonOperator": any(u"EQ", u"GT", u"GE", u"LT", u"LE", u"BETWEEN"),
         u"AttributeValueList": single_str_bin_list,
     },
 )
+
+# Conditions only implemented in scan
+scan_condition = any(
+    range_key_condition,
+    {
+        u"ComparisonOperator": any(u"NULL", u"NOT_NULL"),
+    },{
+        u"ComparisonOperator": any(u"CONTAINS", u"NOT_CONTAINS"),
+        u"AttributeValueList": single_str_num_bin_list,
+    },{
+        u"ComparisonOperator": u"IN",
+        u"AttributeValueList": [simple_field_value],
+    },
+)
+
+# Scan filter
+scan_filter = {
+    field_name: scan_condition,
+}

ddbmock/views/query.py

 @dynamodb_api_validate
 def query(post):
     #FIXME: this line is a temp workaround
-    if u'ReturnValues' not in post:
-        post[u'ReturnValues'] = u"NONE"
     if u'RangeKeyCondition' not in post:
         post[u'RangeKeyCondition'] = None
     if u'AttributesToGet' not in post:
 
     count = len(results)
 
-    return {
+    ret = {
         "Count": count,
-        "Items": results,
         "ConsumedCapacityUnits": 0.5*count, #FIXME: stub
         #TODO: last evaluated key where applicable
     }
 
+    if not post[u'Count']:
+        ret[u'Items'] = results
+
+    return ret
+
 # Pyramid route wrapper
 @view_config(route_name='query', renderer='json')
 def pyramid_query(request):

ddbmock/views/scan.py

+# -*- coding: utf-8 -*-
+
+from pyramid.view import view_config
+from ddbmock.database import DynamoDB
+from ddbmock.validators import dynamodb_api_validate
+from ddbmock.errors import wrap_exceptions, ResourceNotFoundException, ValidationException
+
+# Real work
+@wrap_exceptions
+@dynamodb_api_validate
+def scan(post):
+    #FIXME: this line is a temp workaround
+    if u'ScanFilter' not in post:
+        post[u'ScanFilter'] = {}
+    if u'AttributesToGet' not in post:
+        post[u'AttributesToGet'] = []
+    if u'Count' not in post:
+        post[u'Count'] = False
+    if u'Limit' not in post:
+        post[u'Limit'] = None
+    if u'ExclusiveStartKey' not in post:
+        post[u'ExclusiveStartKey'] = None
+
+    if post[u'AttributesToGet'] and post[u'Count']:
+        raise ValidationException("Can filter fields when only count is requested")
+
+    name = post[u'TableName']
+    table = DynamoDB().get_table(name)
+    if table is None:
+        raise ResourceNotFoundException("Table {} does not exist".format(name))
+
+    results, last_key, scanned_count = table.scan(
+        post[u'ScanFilter'],
+        post[u'AttributesToGet'],
+        post[u'ExclusiveStartKey'],
+        post[u'Limit'],
+    )
+
+    count = len(results)
+
+    ret = {
+        "Count": count,
+        "ScannedCount": scanned_count,
+        "ConsumedCapacityUnits": 0.5*scanned_count, #FIXME: stub
+        #TODO: last evaluated key where applicable
+    }
+
+    if not post[u'Count']:
+        ret[u'Items'] = results
+
+    return ret
+
+# Pyramid route wrapper
+@view_config(route_name='scan', renderer='json')
+def pyramid_scan(request):
+    return scan(request.json)