From 9846986956d0248d344be049c47e0589fd0a41bb Mon Sep 17 00:00:00 2001 From: Young Ryul Bae Date: Sun, 16 Jun 2019 09:49:00 +0900 Subject: [PATCH 1/3] GC support 1. Added Gc class to sync.py 2. Mounted to ExpirableDictStorage and PersistentDictStorage 3. dict interface changed --- ring/func/asyncio.py | 3 +- ring/func/sync.py | 89 ++++++++++++++++++++++++++++- tests/test_dict_gc.py | 127 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 tests/test_dict_gc.py diff --git a/ring/func/asyncio.py b/ring/func/asyncio.py index efe8fda..474f1f0 100644 --- a/ring/func/asyncio.py +++ b/ring/func/asyncio.py @@ -452,7 +452,7 @@ def set_many_values(self, keys, values, expire): def dict( obj, key_prefix=None, expire=None, coder=None, user_interface=CacheUserInterface, storage_class=None, - **kwargs): + maxsize=128, **kwargs): """:class:`dict` interface for :mod:`asyncio`. :see: :func:`ring.func.sync.dict` for common description. @@ -463,6 +463,7 @@ def dict( storage_class = fsync.PersistentDictStorage else: storage_class = fsync.ExpirableDictStorage + storage_class.maxsize = maxsize return fbase.factory( obj, key_prefix=key_prefix, on_manufactured=None, diff --git a/ring/func/sync.py b/ring/func/sync.py index 4ce4ccd..2206507 100644 --- a/ring/func/sync.py +++ b/ring/func/sync.py @@ -8,6 +8,7 @@ import time import re import hashlib +import threading, random, time from . import base as fbase, lru_cache as lru_mod @@ -205,11 +206,82 @@ def touch_value(self, key, expire): except KeyError: pass +class Gc(object): + + def __init__(self, backend, target_size, expire_f=None): + assert round(target_size) > 0, 'target_size has to be at least 1' + assert expire_f is None or callable(expire_f), 'expire_f has to be function or None' + assert isinstance(backend, type({})), 'backend has to be dict-like' + self._backend = backend + self._target_size = round(target_size) + self._expire_f = expire_f + self._mutex = threading.Lock() + + def run(self): + class WorkThread(threading.Thread): + DEBUG = False + + def __init__(self, outer_instance): + threading.Thread.__init__(self) + self._backend = outer_instance._backend + self._target_size = outer_instance._target_size + self._expire_f = outer_instance._expire_f + self._mutex = outer_instance._mutex + + def strategy_with_expire(self): + MAX_EXPIRE_RETRY_COUNT = 4 + now = time.time() + retry_count = 0 + keys = list(self._backend.keys()) + while (len(self._backend) > self._target_size) and (retry_count < MAX_EXPIRE_RETRY_COUNT): + k = keys.pop(random.randrange(len(keys))) + val = self._backend.get(k, None) + if val is None: + continue + expire = self._expire_f(val) + if expire < now: + self._backend.pop(k, None) + self.print_d('{} removed from strategy_with_expire => size {}'.format(k, len(self._backend))) + else: + retry_count += 1 + if len(keys) == 0: + keys = list(self._backend.keys()) + + def strategy_with_force(self): + keys = list(self._backend.keys()) + while (len(self._backend) > self._target_size) and len(keys) > 0: + k = keys.pop(random.randrange(len(keys))) + self._backend.pop(k, None) + self.print_d('{} removed from strategy_with_force => size {}'.format(k, len(self._backend))) + if len(keys) == 0: + keys = list(self._backend.keys()) + + def run(self): + try: + self.print_d('gc started in size:{}'.format(len(self._backend))) + if self._expire_f is not None: + self.strategy_with_expire() + self.strategy_with_force() + self.print_d('gc ended in size:{}'.format(len(self._backend))) + finally: + self._mutex.release() + + + def print_d(self, str_): + if self.DEBUG: + print(str_) + + self._mutex.acquire() + if (len(self._backend) > self._target_size): + WorkThread(self).start() + else: + self._mutex.release() class ExpirableDictStorage(fbase.CommonMixinStorage, fbase.StorageMixin): - + maxsize = 128 in_memory_storage = True now = time.time + _gc = None def get_value(self, key): _now = self.now() @@ -229,6 +301,11 @@ def set_value(self, key, value, expire): expired_time = _now + expire self.backend[key] = expired_time, value + if self.maxsize < len(self.backend): + if self._gc == None: + self._gc = Gc(self.backend, self.maxsize * 0.75, lambda x: x[0]) + self._gc.run() + def delete_value(self, key): try: del self.backend[key] @@ -252,8 +329,9 @@ def touch_value(self, key, expire): class PersistentDictStorage(fbase.CommonMixinStorage, fbase.StorageMixin): - in_memory_storage = True + maxsize = 128 + _gc = None def get_value(self, key): try: @@ -264,6 +342,10 @@ def get_value(self, key): def set_value(self, key, value, expire): self.backend[key] = value + if self.maxsize < len(self.backend): + if self._gc == None: + self._gc = Gc(self.backend, self.maxsize * 0.75) + self._gc.run() def delete_value(self, key): try: @@ -443,7 +525,7 @@ def lru( def dict( obj, key_prefix=None, expire=None, coder=None, user_interface=CacheUserInterface, storage_class=None, - **kwargs): + maxsize=128, **kwargs): """Basic Python :class:`dict` based cache. This backend is not designed for real products. Please carefully read the @@ -467,6 +549,7 @@ def dict( storage_class = PersistentDictStorage else: storage_class = ExpirableDictStorage + storage_class.maxsize = maxsize return fbase.factory( obj, key_prefix=key_prefix, on_manufactured=None, diff --git a/tests/test_dict_gc.py b/tests/test_dict_gc.py new file mode 100644 index 0000000..fc7ca51 --- /dev/null +++ b/tests/test_dict_gc.py @@ -0,0 +1,127 @@ + +import ring + + +def test_dict_gc_persistence_1(): + import time + cache = {} + + @ring.dict(cache, maxsize=1) + def f_persistent_1(i): + return i + + for i in range(100): + f_persistent_1(i) + time.sleep(0.02) + assert len(cache) <= 1 + +def test_dict_gc_persistence_default(): + import time + cache = {} + + @ring.dict(cache) + def f_persistent_default(i): + return i + + for i in range(1000): + f_persistent_default(i) + time.sleep(0.1) + assert len(cache) <= 128 + +def test_dict_gc_persistent_random_delete(): + import time + cache = {} + MAX_SIZE = 10 + + @ring.dict(cache, maxsize=MAX_SIZE) + def f_persistent_random_delete(i): + return i + + for i in range(1000): + f_persistent_random_delete(i) + if i % 17 == 0: + for pop_count in range(8): + try: + cache.popitem() + except KeyError: + pass + time.sleep(0.1) + assert len(cache) <= MAX_SIZE + +def test_dict_gc_expire_1(): + import time + cache = {} + MAX_SIZE = 1 + + @ring.dict(cache, maxsize=MAX_SIZE, expire=1) + def f_expire_1(i): + return i + + for i in range(MAX_SIZE * 100): + f_expire_1(i) + time.sleep(0.02) + assert len(cache) <= MAX_SIZE + +def test_dict_gc_expire_many(): + import time + cache = {} + MAX_SIZE = 50000 + + @ring.dict(cache, maxsize=MAX_SIZE, expire=1) + def f_expire_1(i): + return i + + for i in range(MAX_SIZE * 10): + f_expire_1(i) + + time.sleep(0.1) + assert len(cache) <= MAX_SIZE + +def test_dict_gc_expire_some(): + import time + cache = {} + + @ring.dict(cache, maxsize=150, expire=1) + def f_expire_some_expire(i): + return i + + for i in range(100): + f_expire_some_expire(i) + time.sleep(1) + for i in range(100, 200): + f_expire_some_expire(i) + assert len(cache) <= 150 + +def test_dict_gc_expire_default(): + import time + cache = {} + + @ring.dict(cache, expire=1) + def f_expire_default(i): + return i + + for i in range(1000): + f_expire_default(i) + time.sleep(0.1) + assert len(cache) <= 128 + +def test_dict_gc_expire_random(): + import time + cache = {} + + @ring.dict(cache, maxsize=10, expire=1) + def f_expire_random(i): + return i + + for i in range(1000): + f_expire_random(i) + time.sleep(1) + for i in range(1000, 2000): + f_expire_random(i) + if i % 17 == 0: + for pop_count in range(8): + try: + cache.popitem() + except KeyError: + pass + assert len(cache) <= 10 From bee4599b57716defe2778c1adaf9119e3d0f36e5 Mon Sep 17 00:00:00 2001 From: Young Ryul Bae Date: Sun, 23 Jun 2019 17:22:46 +0900 Subject: [PATCH 2/3] Removed threading & newly written test --- ring/func/base.py | 6 +- ring/func/sync.py | 126 +++++++++++++++--------------------- tests/test_dict_gc.py | 127 ------------------------------------ tests/test_dict_size.py | 140 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 203 deletions(-) delete mode 100644 tests/test_dict_gc.py create mode 100644 tests/test_dict_size.py diff --git a/ring/func/base.py b/ring/func/base.py index f3bb8c0..9701211 100644 --- a/ring/func/base.py +++ b/ring/func/base.py @@ -562,6 +562,7 @@ def factory( # keyword-only arguments from here # building blocks coder, miss_value, user_interface, storage_class, + maxsize=128, default_action=Ellipsis, coder_registry=Ellipsis, # callback @@ -638,7 +639,7 @@ class RingRope(RopeCore): def __init__(self, *args, **kwargs): super(RingRope, self).__init__(*args, **kwargs) self.user_interface = self.user_interface_class(self) - self.storage = self.storage_class(self, storage_backend) + self.storage = self.storage_class(self, storage_backend, maxsize) _ignorable_keys = suggest_ignorable_keys( self.callable, ignorable_keys) _key_prefix = suggest_key_prefix(self.callable, key_prefix) @@ -747,9 +748,10 @@ class BaseStorage(object): are mandatory; Otherwise not. """ - def __init__(self, rope, backend): + def __init__(self, rope, backend, maxsize): self.rope = rope self.backend = backend + self.maxsize = maxsize @abc.abstractmethod def get(self, key): # pragma: no cover diff --git a/ring/func/sync.py b/ring/func/sync.py index 2206507..c50c8bc 100644 --- a/ring/func/sync.py +++ b/ring/func/sync.py @@ -8,7 +8,6 @@ import time import re import hashlib -import threading, random, time from . import base as fbase, lru_cache as lru_mod @@ -206,82 +205,68 @@ def touch_value(self, key, expire): except KeyError: pass -class Gc(object): +class SizeMaintainer(object): + _DEBUG = False def __init__(self, backend, target_size, expire_f=None): + from collections import abc assert round(target_size) > 0, 'target_size has to be at least 1' assert expire_f is None or callable(expire_f), 'expire_f has to be function or None' - assert isinstance(backend, type({})), 'backend has to be dict-like' + assert isinstance(backend, abc.MutableMapping), 'backend has to be dict-like' self._backend = backend self._target_size = round(target_size) self._expire_f = expire_f - self._mutex = threading.Lock() def run(self): - class WorkThread(threading.Thread): - DEBUG = False - - def __init__(self, outer_instance): - threading.Thread.__init__(self) - self._backend = outer_instance._backend - self._target_size = outer_instance._target_size - self._expire_f = outer_instance._expire_f - self._mutex = outer_instance._mutex - - def strategy_with_expire(self): - MAX_EXPIRE_RETRY_COUNT = 4 - now = time.time() - retry_count = 0 - keys = list(self._backend.keys()) - while (len(self._backend) > self._target_size) and (retry_count < MAX_EXPIRE_RETRY_COUNT): - k = keys.pop(random.randrange(len(keys))) - val = self._backend.get(k, None) - if val is None: - continue - expire = self._expire_f(val) - if expire < now: - self._backend.pop(k, None) - self.print_d('{} removed from strategy_with_expire => size {}'.format(k, len(self._backend))) - else: - retry_count += 1 - if len(keys) == 0: - keys = list(self._backend.keys()) - - def strategy_with_force(self): - keys = list(self._backend.keys()) - while (len(self._backend) > self._target_size) and len(keys) > 0: - k = keys.pop(random.randrange(len(keys))) - self._backend.pop(k, None) - self.print_d('{} removed from strategy_with_force => size {}'.format(k, len(self._backend))) - if len(keys) == 0: - keys = list(self._backend.keys()) - - def run(self): - try: - self.print_d('gc started in size:{}'.format(len(self._backend))) - if self._expire_f is not None: - self.strategy_with_expire() - self.strategy_with_force() - self.print_d('gc ended in size:{}'.format(len(self._backend))) - finally: - self._mutex.release() - - - def print_d(self, str_): - if self.DEBUG: - print(str_) - - self._mutex.acquire() - if (len(self._backend) > self._target_size): - WorkThread(self).start() - else: - self._mutex.release() + if (len(self._backend) <= self._target_size): + return + + import random, time + def strategy_with_expire(): + MAX_EXPIRE_RETRY_COUNT = 4 + now = time.time() + retry_count = 0 + + keys = list(self._backend.keys()) + random.shuffle(keys) + key_index = 0 + while (len(self._backend) > self._target_size) and (retry_count < MAX_EXPIRE_RETRY_COUNT): + key = keys[key_index] + val = self._backend.get(key, None) + expire = self._expire_f(val) + if expire < now: + self._backend.pop(key, None) + key_index += 1 + if self._DEBUG: + print('{} removed from strategy_with_expire => size {}'.format(key, len(self._backend))) + else: + retry_count += 1 + + def strategy_with_force(): + keys = list(self._backend.keys()) + random.shuffle(keys) + key_index = 0 + while (len(self._backend) > self._target_size): + key = keys[key_index] + self._backend.pop(key, None) + key_index += 1 + if self._DEBUG: + print('{} removed from strategy_with_force => size {}'.format(key, len(self._backend))) + + if self._DEBUG: + print('gc started in size:{}'.format(len(self._backend))) + + if self._expire_f is not None: + strategy_with_expire() + strategy_with_force() + + if self._DEBUG: + print('gc ended in size:{}'.format(len(self._backend))) + class ExpirableDictStorage(fbase.CommonMixinStorage, fbase.StorageMixin): - maxsize = 128 in_memory_storage = True now = time.time - _gc = None def get_value(self, key): _now = self.now() @@ -302,9 +287,7 @@ def set_value(self, key, value, expire): self.backend[key] = expired_time, value if self.maxsize < len(self.backend): - if self._gc == None: - self._gc = Gc(self.backend, self.maxsize * 0.75, lambda x: x[0]) - self._gc.run() + SizeMaintainer(self.backend, self.maxsize * 0.75, lambda x: x[0]).run() def delete_value(self, key): try: @@ -330,8 +313,6 @@ def touch_value(self, key, expire): class PersistentDictStorage(fbase.CommonMixinStorage, fbase.StorageMixin): in_memory_storage = True - maxsize = 128 - _gc = None def get_value(self, key): try: @@ -343,9 +324,7 @@ def get_value(self, key): def set_value(self, key, value, expire): self.backend[key] = value if self.maxsize < len(self.backend): - if self._gc == None: - self._gc = Gc(self.backend, self.maxsize * 0.75) - self._gc.run() + SizeMaintainer(self.backend, self.maxsize * 0.75).run() def delete_value(self, key): try: @@ -549,11 +528,10 @@ def dict( storage_class = PersistentDictStorage else: storage_class = ExpirableDictStorage - storage_class.maxsize = maxsize return fbase.factory( obj, key_prefix=key_prefix, on_manufactured=None, - user_interface=user_interface, storage_class=storage_class, + user_interface=user_interface, storage_class=storage_class, maxsize=maxsize, miss_value=None, expire_default=expire, coder=coder, **kwargs) diff --git a/tests/test_dict_gc.py b/tests/test_dict_gc.py deleted file mode 100644 index fc7ca51..0000000 --- a/tests/test_dict_gc.py +++ /dev/null @@ -1,127 +0,0 @@ - -import ring - - -def test_dict_gc_persistence_1(): - import time - cache = {} - - @ring.dict(cache, maxsize=1) - def f_persistent_1(i): - return i - - for i in range(100): - f_persistent_1(i) - time.sleep(0.02) - assert len(cache) <= 1 - -def test_dict_gc_persistence_default(): - import time - cache = {} - - @ring.dict(cache) - def f_persistent_default(i): - return i - - for i in range(1000): - f_persistent_default(i) - time.sleep(0.1) - assert len(cache) <= 128 - -def test_dict_gc_persistent_random_delete(): - import time - cache = {} - MAX_SIZE = 10 - - @ring.dict(cache, maxsize=MAX_SIZE) - def f_persistent_random_delete(i): - return i - - for i in range(1000): - f_persistent_random_delete(i) - if i % 17 == 0: - for pop_count in range(8): - try: - cache.popitem() - except KeyError: - pass - time.sleep(0.1) - assert len(cache) <= MAX_SIZE - -def test_dict_gc_expire_1(): - import time - cache = {} - MAX_SIZE = 1 - - @ring.dict(cache, maxsize=MAX_SIZE, expire=1) - def f_expire_1(i): - return i - - for i in range(MAX_SIZE * 100): - f_expire_1(i) - time.sleep(0.02) - assert len(cache) <= MAX_SIZE - -def test_dict_gc_expire_many(): - import time - cache = {} - MAX_SIZE = 50000 - - @ring.dict(cache, maxsize=MAX_SIZE, expire=1) - def f_expire_1(i): - return i - - for i in range(MAX_SIZE * 10): - f_expire_1(i) - - time.sleep(0.1) - assert len(cache) <= MAX_SIZE - -def test_dict_gc_expire_some(): - import time - cache = {} - - @ring.dict(cache, maxsize=150, expire=1) - def f_expire_some_expire(i): - return i - - for i in range(100): - f_expire_some_expire(i) - time.sleep(1) - for i in range(100, 200): - f_expire_some_expire(i) - assert len(cache) <= 150 - -def test_dict_gc_expire_default(): - import time - cache = {} - - @ring.dict(cache, expire=1) - def f_expire_default(i): - return i - - for i in range(1000): - f_expire_default(i) - time.sleep(0.1) - assert len(cache) <= 128 - -def test_dict_gc_expire_random(): - import time - cache = {} - - @ring.dict(cache, maxsize=10, expire=1) - def f_expire_random(i): - return i - - for i in range(1000): - f_expire_random(i) - time.sleep(1) - for i in range(1000, 2000): - f_expire_random(i) - if i % 17 == 0: - for pop_count in range(8): - try: - cache.popitem() - except KeyError: - pass - assert len(cache) <= 10 diff --git a/tests/test_dict_size.py b/tests/test_dict_size.py new file mode 100644 index 0000000..806903d --- /dev/null +++ b/tests/test_dict_size.py @@ -0,0 +1,140 @@ + +import ring + + +def test_dict_size_persistence_1(): + cache = {} + MAX_SIZE = 1 + + @ring.dict(cache, maxsize=MAX_SIZE) + def f_persistent_1(i): + return i + + for i in range(MAX_SIZE * 100): + f_persistent_1(i) + assert len(cache) <= 1 + assert len(cache) <= 1 + +def test_dict_size_persistence_default(): + cache = {} + + @ring.dict(cache) + def f_persistent_default(i): + return i + + for i in range(1000): + f_persistent_default(i) + assert len(cache) <= 128 + assert len(cache) <= 128 + +def test_dict_size_persistence_1000(): + cache = {} + MAX_SIZE = 1000 + + @ring.dict(cache, maxsize=MAX_SIZE) + def f_persistent_default(i): + return i + + for i in range(MAX_SIZE * 100): + f_persistent_default(i) + assert len(cache) <= MAX_SIZE + assert len(cache) <= MAX_SIZE + +def test_dict_size_persistent_with_delete(): + cache = {} + MAX_SIZE = 10 + + @ring.dict(cache, maxsize=MAX_SIZE) + def f_persistent_with_delete(i): + return i + + for i in range(MAX_SIZE * 100): + f_persistent_with_delete(i) + assert len(cache) <= MAX_SIZE + if i % 17 == 0: + for pop_count in range(8): + if len(cache) > 0: + cache.popitem() + + + assert len(cache) <= MAX_SIZE + +def test_dict_size_expire_1(): + cache = {} + MAX_SIZE = 1 + + @ring.dict(cache, maxsize=MAX_SIZE, expire=1) + def f_expire_1(i): + return i + + for i in range(MAX_SIZE * 100): + f_expire_1(i) + assert len(cache) <= MAX_SIZE + assert len(cache) <= MAX_SIZE + +def test_dict_size_expire_default(): + cache = {} + + @ring.dict(cache, expire=1) + def f_expire_default(i): + return i + + for i in range(1000): + f_expire_default(i) + assert len(cache) <= 128 + assert len(cache) <= 128 + +def test_dict_size_expire_1000(): + cache = {} + MAX_SIZE = 1000 + + @ring.dict(cache, maxsize=MAX_SIZE, expire=1) + def f_expire_1(i): + return i + + for i in range(MAX_SIZE * 100): + f_expire_1(i) + assert len(cache) <= MAX_SIZE + + assert len(cache) <= MAX_SIZE + +def test_dict_size_expire_some(): + import time + cache = {} + MAX_SIZE = 150 + + @ring.dict(cache, maxsize=MAX_SIZE, expire=1) + def f_expire_some_expire(i): + return i + + for _ in range(5): + for i in range(100): + f_expire_some_expire(i) + assert len(cache) <= MAX_SIZE + time.sleep(1) + for i in range(100, 200): + f_expire_some_expire(i) + assert len(cache) <= MAX_SIZE + assert len(cache) <= MAX_SIZE + + +def test_dict_size_expire_with_delete(): + import time + cache = {} + + @ring.dict(cache, expire=1) + def f_expire_with_delete(i): + return i + + for i in range(1000): + f_expire_with_delete(i) + time.sleep(1) + for i in range(1000, 2000): + f_expire_with_delete(i) + assert len(cache) <= 128 + if i % 17 == 0: + for pop_count in range(8): + if len(cache) > 0: + cache.popitem() + + assert len(cache) <= 128 From 0e2719b23db311082fe35e5f6483be0f81c2d0aa Mon Sep 17 00:00:00 2001 From: Young Ryul Bae Date: Thu, 4 Jul 2019 21:59:51 +0900 Subject: [PATCH 3/3] Fixed test_redis_hash fail. Changed RedisHashStorage, AioredisHashStorage __init__ --- ring/func/asyncio.py | 4 ++-- ring/func/base.py | 2 +- ring/func/sync.py | 8 ++++---- tests/test_dict_size.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/ring/func/asyncio.py b/ring/func/asyncio.py index 474f1f0..ef6bf45 100644 --- a/ring/func/asyncio.py +++ b/ring/func/asyncio.py @@ -405,10 +405,10 @@ def set_many_values(self, keys, values, expire): class AioredisHashStorage(AioredisStorage): """Storage implementation for :class:`aioredis.Redis`.""" - def __init__(self, rope, backend): + def __init__(self, rope, backend, maxsize): storage_backend = backend[0] self.hash_key = backend[1] - super(AioredisHashStorage, self).__init__(rope, storage_backend) + super(AioredisHashStorage, self).__init__(rope, storage_backend, maxsize) @asyncio.coroutine def get_value(self, key): diff --git a/ring/func/base.py b/ring/func/base.py index 9701211..778ee91 100644 --- a/ring/func/base.py +++ b/ring/func/base.py @@ -562,7 +562,7 @@ def factory( # keyword-only arguments from here # building blocks coder, miss_value, user_interface, storage_class, - maxsize=128, + maxsize=None, default_action=Ellipsis, coder_registry=Ellipsis, # callback diff --git a/ring/func/sync.py b/ring/func/sync.py index c50c8bc..7551b9d 100644 --- a/ring/func/sync.py +++ b/ring/func/sync.py @@ -286,7 +286,7 @@ def set_value(self, key, value, expire): expired_time = _now + expire self.backend[key] = expired_time, value - if self.maxsize < len(self.backend): + if (self.maxsize is not None) and (self.maxsize < len(self.backend)): SizeMaintainer(self.backend, self.maxsize * 0.75, lambda x: x[0]).run() def delete_value(self, key): @@ -323,7 +323,7 @@ def get_value(self, key): def set_value(self, key, value, expire): self.backend[key] = value - if self.maxsize < len(self.backend): + if (self.maxsize is not None) and self.maxsize < len(self.backend): SizeMaintainer(self.backend, self.maxsize * 0.75).run() def delete_value(self, key): @@ -415,10 +415,10 @@ def set_many_values(self, keys, values, expire): class RedisHashStorage(RedisStorage): - def __init__(self, rope, backend): + def __init__(self, rope, backend, maxsize): storage_backend = backend[0] self.hash_key = backend[1] - super(RedisHashStorage, self).__init__(rope, storage_backend) + super(RedisHashStorage, self).__init__(rope, storage_backend, maxsize) def get_value(self, key): value = self.backend.hget(self.hash_key, key) diff --git a/tests/test_dict_size.py b/tests/test_dict_size.py index 806903d..97edb12 100644 --- a/tests/test_dict_size.py +++ b/tests/test_dict_size.py @@ -59,6 +59,20 @@ def f_persistent_with_delete(i): assert len(cache) <= MAX_SIZE +def test_dict_size_persistent_infinite(): + cache = {} + MAX_SIZE = None + + @ring.dict(cache, maxsize=MAX_SIZE) + def f_persistent_infinite(i): + return i + + for i in range(10000): + f_persistent_infinite(i) + assert len(cache) <= 10000 + + assert len(cache) == 10000 + def test_dict_size_expire_1(): cache = {} MAX_SIZE = 1 @@ -138,3 +152,17 @@ def f_expire_with_delete(i): cache.popitem() assert len(cache) <= 128 + +def test_dict_size_expire_infinite(): + cache = {} + MAX_SIZE = None + + @ring.dict(cache, maxsize=MAX_SIZE) + def f_expire_infinite(i): + return i + + for i in range(10000): + f_expire_infinite(i) + assert len(cache) <= 10000 + + assert len(cache) == 10000