Viewing file:  BeeDict.py (38.36 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
""" BeeDict - On-disk dictionary like databases
"""
import exceptions,UserDict
import BeeIndex,BeeStorage
from mx import Tools
freeze = Tools.freeze
from mx.Log import *
#import TraceCalls; TraceCalls.install()
### Constants
# Special keys
FirstKey = BeeIndex.FirstKey
LastKey = BeeIndex.LastKey
# Maximal cache size ( == size of transaction log; kept in memory !)
MAXCACHESIZE = 1000
### Errors
class Error(exceptions.StandardError):
pass
class RecreateIndexError(Error):
""" This error is raised in case the index for a dictionary was
not found and/or needs to be recreated by running recovery.
"""
pass
class RecoverError(Error):
""" This error is raised in case the storage for a dictionary was
found to be in an inconsistent state.
"""
pass
### Base classes
class BeeBaseDict:
""" Base class that collects some method that all dicts can usually
share without modification.
The base class implements a transaction based caching
mechanism. The cache stores data in the form
key:(state,value) where state is:
* 0 for read but not yet modified
* 1 for modified
* 2 for deleted
* 3 for key read, value still on disk
The .commit method has to implement the needed disk mechanism
to write the cache data to disk storage.
"""
name = 'NoName' # Name of the dict
index = None # Index object
storage = None # Storage object
closed = 0 # Are we closed ?
cache = None # Cache dict
readonly = 0 # Are we read-only ?
autocommit = 0 # Autocommit ? (this will
# perform a commit whenever
# the cache gets to full)
# Special keys that can be used for .cursor()
FirstKey = FirstKey
LastKey = LastKey
def __init__(self):
""" Inits the cache.
"""
self.cache = {}
def __len__(self):
return len(self.index)
def close(self):
""" Flush buffers and close.
This issues a .rollback(), so the current transaction is
rolled back.
"""
if not self.closed:
self.rollback()
for obj in (self.index,self.storage):
if obj is not None:
obj.flush()
obj.close()
self.closed = 1
def __del__(self,
AttributeError=AttributeError):
""" Make sure the object is closed.
"""
if not self.closed:
try:
self.close()
except AttributeError:
# Could occur during interpreter cleanup
pass
def flush(self):
""" Flush buffers to disk.
"""
if self.storage is not None:
self.storage.flush()
if self.index is not None:
self.index.flush()
def __repr__(self):
return '<%s instance for "%s" at 0x%x>' % (self.__class__.__name__,
self.name,
id(self))
def commit(self):
""" Commit all changes and start a new transaction.
This method does not implement any disk operation, but
takes care of managing the cache. Post-override.
"""
# Clear cache
self.cache.clear()
# Flush storage and index
self.flush()
def rollback(self):
""" Take back all changes and start a new transaction.
Overriding is normally not needed. This method only takes
care of managing the cache. Post-override.
"""
# Clear cache
self.cache.clear()
# Flush storage and index
self.flush()
def changed(self,
modified=(1,2)):
""" Return true in case the current transaction includes changes
to the database, false otherwise.
"""
if self.cache:
for key,(state,value) in self.cache.items():
if state in modified:
return 1
return 0
def free_cache(self,
len=len,MAXCACHESIZE=MAXCACHESIZE,unmodified=(0,3)):
""" Remove all unmodified entries from the cache.
If self.autocommit and the cache cleanup did not reduce
the size below the MAXCACHESIZE limit, a .commit() will be
executed automatically.
Otherwise the processing continues. If the cache size hits
2 * MAXCACHESIZE, a forced .rollback() is done and an
error raised.
"""
cache = self.cache
for key,(state,value) in cache.items():
if state in unmodified:
del cache[key]
size = len(cache)
if size > MAXCACHESIZE:
if self.autocommit:
self.commit()
return
log(SYSTEM_ERROR,
'Cache overflow: %i modified items in cache',size)
if size > 2 * MAXCACHESIZE:
# Force .rollback() and raise an error
log(SYSTEM_ERROR,
'Cache overflow: forced a .rollback()')
self.rollback()
raise Error,'cache overflow; last transaction rolled back'
def read(self,key,checkonly=0):
""" Read and return the value corresponding to key.
If checkonly is true, only the availability of a value
has to be checked.
Override this method with an implementation that accesses
the disk.
"""
raise KeyError,'key not found'
def __setitem__(self,key,value):
""" Save the item in the dictionary.
Note that it is written to the cache first. Use .commit()
to make sure it is written to disk.
"""
self.cache[key] = (1,value)
def __getitem__(self,key,
validstates=(0,1),len=len,KeyError=KeyError,
MAXCACHESIZE=MAXCACHESIZE):
""" Get an item from the dictionary.
This first tries to read the item from cache and reverts
to the disk storage if it is not found.
"""
try:
cache = self.cache
state,value = cache[key]
if state in validstates:
return value
except KeyError:
pass
else:
if state == 2:
raise KeyError,'key deleted'
# Read from disk
value = self.read(key)
# Cache the item
if len(cache) > MAXCACHESIZE:
self.free_cache()
cache[key] = (0,value)
return value
def __delitem__(self,key):
""" Delete an item.
The item is only marked for deletion. The next .commit()
will make the action permanent.
"""
cache = self.cache
if not cache.has_key(key):
self.read(key,1)
cache[key] = (2,None)
def has_key(self,key,
len=len,KeyError=KeyError,MAXCACHESIZE=MAXCACHESIZE):
""" Check if the dictionary has an item indexed by key.
Successfully found items are put in the cache for fast
subsequent access.
"""
try:
cache = self.cache
state,value = cache[key]
if state == 2:
return 0
return 1
except KeyError:
pass
# Read from disk
try:
value = self.read(key)
except KeyError:
return 0
# Cache the item
if len(cache) > MAXCACHESIZE:
self.free_cache()
cache[key] = (0,value)
return 1
def get(self,key,default=None,
len=len,KeyError=KeyError,MAXCACHESIZE=MAXCACHESIZE):
""" Get item indexed by key from the dictionary or default if
no such item exists.
This first tries to read the item from cache and reverts
to the disk storage if it is not found.
"""
try:
cache = self.cache
state,value = cache[key]
if state == 2:
return default
return value
except KeyError:
pass
# Read from disk
try:
value = self.read(key)
except KeyError:
return default
# Cache the item
if len(cache) > MAXCACHESIZE:
self.free_cache()
cache[key] = (0,value)
return value
def cursor(self,key,default=None):
""" Return a cursor instance for this kind of dictionary.
In case the key is not found, default is returned instead.
Note that cursors operate with the data on disk meaning
that any uncommitted changes will not be seen by the
cursor.
Has to be overridden to return a suitable cursor object.
"""
# Check for uncommitted changes
if self.cache and self.changed():
raise Error,'uncommitted data exists'
# Get the index cursor and create a dict cursor from it
cursor = self.index.cursor(key,None)
if cursor is None:
return default
return BeeBaseDictCursor(self,cursor)
def garbage(self):
""" Determine the amount of garbage in bytes that has accumulated
in the storage file.
This amount would be freed if .collect() were run.
"""
valid,old,invalid = self.storage.statistics()
return old + invalid
def collect(self):
""" Run the storage garbage collector.
Storage collection can only be done for writeable
dictionaries and then only if the current transaction does
not contain any pending changes.
This can take a while depending on the size of the
dictionary.
"""
if self.readonly:
raise Error,'dict is read-only'
# Check for uncommitted changes
if self.cache and self.changed():
raise Error,'uncommitted data exists'
self.flush()
log(SYSTEM_INFO,'Collecting %s',self)
# Run collector
self.storage.collect(self.collect_callback)
# End the storage transaction
self.storage.end_transaction()
def collect_callback(self,old_position,new_position,raw_data):
""" Internal callback used to update the index when
the storage garbage collector moves a record.
This method must be overridden to account for the
different indexing schemes.
"""
raise Error,'.collect_callback() not implemented'
def recover(self):
""" Recover all valid records and recreate the index.
"""
log(SYSTEM_INFO,'Recovering %s',self)
# Clear the index
self.index.clear()
# Run recovery and recreate the index through the callback
self.storage.recover(self.recover_callback)
# End the storage transaction
self.storage.end_transaction()
def recover_callback(self,old_position,new_position,raw_data):
""" Internal callback used to update the index when
the storage recovery collector finds a record.
old_position is only given as additional information,
new_position contains the position of the found record.
This method must be overridden to account for the
different indexing schemes.
"""
raise Error,'.recover_callback() not implemented'
def validate_index(self):
""" Checks the consistency of the index and raises an
RecreateIndexError in case the index is found to be
inconsistent.
Validation takes places for the data on disk. The in
memory data is not checked.
"""
log(SYSTEM_INFO,'Validating index for %s',self)
if not self.index.validate():
raise RecreateIndexError,'index is inconsistent'
def validate_storage(self):
""" Checks the consistency of the storage and raises an
RecoverError in case it is found to be
inconsistent.
Validation takes places for the data on disk. The in
memory data is not checked.
XXX This currently does nothing since storages don't
support validation yet.
"""
log(SYSTEM_INFO,'Validating storage for %s',self)
return
def backup(self,archive=None):
""" Issues a backup request to the storage using archive
which defaults to the storage filename + '.backup'.
This causes an implicit .rollback() to be done.
"""
self.rollback()
self.storage.backup(archive)
def restore(self,archive):
""" Restores the storage from an archive file.
XXX Currently does NOT work.
"""
raise SystemError,'.restore() not implemented !'
### Cursor base class
class BeeBaseDictCursor:
""" Cursor for BeeBaseDict subclasses.
The read/read_key/write methods assume that the dictionary
uses a BeeKeyValueStorage instances as storage facility.
Note that modifying the targetted dictionary while using a
cursor can cause the cursor to skip new entries or fail due to
deleted items. Especially deleting the key to which the cursor
currently points can cause errors to be raised. In all other
cases, the cursor will be repositioned.
"""
cursor = None
def __init__(self,dict,cursor):
self.dict = dict
self.cursor = cursor
self.key = cursor.key
self.value = cursor.value
def position(self,key,value=None):
""" Position the index cursor to index[key]. If value is given,
index[key] == value is assured.
key may also be FirstKey or LastKey (in which case value
has to be None).
"""
# Create and position the cursor
self.cursor = cursor = self.dict.index.cursor(key)
if value and cursor.value != value:
# assure that the cursor points to key,value
while cursor.next():
if cursor.value == value:
break
if cursor.key != key:
raise KeyError,'key not found'
else:
raise KeyError,'key not found'
self.key = key
self.value = value
def next(self):
""" Moves to the next entry in the dictionary.
Returns true on success, false if the end-of-data has been
reached.
"""
cursor = self.cursor
if not cursor.valid:
self.position(self.key,self.value)
cursor = self.cursor
rc = cursor.next()
self.key = cursor.key
self.value = cursor.value
return rc
def prev(self):
""" Moves to the previous entry in the dictionary.
Returns true on success, false if the end-of-data has been
reached.
"""
cursor = self.cursor
if not cursor.valid:
self.position(self.key,self.value)
cursor = self.cursor
rc = cursor.prev()
self.key = cursor.key
self.value = cursor.value
return rc
def read(self):
""" Reads the object from the dict to which the cursor
currently points.
This method assumes a BeeKeyValueStorage instance in
self.storage and an index that holds copies of the key
objects. Override if this is not the case.
"""
return self.dict[self.key]
def read_key(self):
""" Reads the key object from the dict to which the cursor
currently points.
This method is intended for dictionaries that store hash
values in the index. Normally, accessing the instance
variable .key would give the current key object.
This method assumes a BeeKeyValueStorage instance in
self.storage. Override if this is not the case.
"""
return self.dict.storage.read_key(self.value)
def write(self,object,minsize=0):
""" Writes the object to the dict under the key to which
the cursor currently points.
The new data is not written to disk until the dictionaries
current transaction is committed.
This method assumes a BeeKeyValueStorage instance in
self.storage and an index that holds copies of the key
objects. Override if this is not the case.
"""
self.dict[self.key] = object
### Actual implementations
class BeeDict(BeeBaseDict):
""" On-disk dictionary that uses a "hash to address" index.
Both Keys and values must be pickleable. Keys also have to be
hashable. Both can have arbitrary size (keys shouldn't be too
long though).
Hash collisions are treated by sequential reads of all records
with the same hash value and testing for equality of keys. This
can be expensive !
"""
def __init__(self,name,min_recordsize=0,readonly=0,recover=0,
autocommit=0,validate=0,
basemethod=BeeBaseDict.__init__):
""" Create an instance using name as basename for the
data and index files.
min_recordsize is passed to the BeeStorage as indicator of
the minimum size for data records. readonly can be set to
true to open the files in read-only mode, preventing any
disk modifications.
To open the dictionary in recovery mode, pass a keyword
recover=1. Then run .recover() and reopen using the normal
settings.
If autocommit is true the cache control will do an
automatic .commit() whenever the transaction log
overflows.
If validate is true, the dictionary will run a validation
check after having successfully opened storage and index.
RecreateIndexError or RecoverError exceptions could be
raised in case inconsistencies are found.
"""
self.name = name
self.storage = BeeStorage.BeeKeyValueStorage(
name + '.dat',
lock= not readonly,
cache=0,
min_recordsize=min_recordsize,
readonly=readonly,
recover=recover)
if readonly:
# Readonly mode
filemode = 1
else:
if self.storage.is_new or recover:
# Create a new file (overwriting a possibly existing one)
filemode = 2
else:
# Open a file, create is non-existent
filemode = 3
try:
self.index = BeeIndex.BeeIntegerIndex(
name + '.idx',
dupkeys=1,
filemode=filemode,
sectorsize=256)
except IOError:
raise RecreateIndexError,\
'Index for BeeDict "%s" is missing' % name
self.collisions = 0
self.readonly = readonly
self.autocommit = autocommit
if validate:
self.validate_index()
self.validate_storage()
# Call basemethod
basemethod(self)
def find_address(self,cursor,hashvalue,key):
""" Find the address of a key by scanning the hash value
set pointed to by the cursor.
Returns the address of the found key or None.
"""
while cursor.next():
if cursor.key != hashvalue:
# Left set of identical keys in index: not found
return None
if key == self.storage.read_key(cursor.value):
# Found
return cursor.value
# EOF reached index: not found
return None
def index_cursor(self,key,default=None):
""" Return an index cursor pointing to key.
In case the key is not found, default is returned.
XXX Should use this method in more places...
"""
if key is FirstKey or key is LastKey:
return self.index.cursor(key,default)
hashvalue = hash(key)
cursor = self.index.cursor(hashvalue,None)
if cursor is None:
return default
# Check that we have really found the key
address = cursor.value
if key != self.storage.read_key(address):
# Ah, a collision
address = self.find_address(cursor,hashvalue,key)
if address is None:
return default
return cursor
def commit(self,
basemethod=BeeBaseDict.commit):
""" Commit all changes and start a new transaction.
"""
if __debug__:
log(SYSTEM_DEBUG,'Committing all changes for "%s"...',self.name)
# Write all changed entries in the cache to disk
index = self.index
index_cursor = index.cursor
index_update = index.update
index_delete = index.delete
storage = self.storage
storage_read_key = storage.read_key
storage_write = storage.write
storage_delete = storage.delete
readonly = self.readonly
for key,(state,value) in self.cache.items():
if state == 1:
if __debug__:
log.object(SYSTEM_DEBUG,' writing key "%s":' % key,value)
if readonly:
raise Error,'dict is read-only'
hashvalue = hash(key)
cursor = index_cursor(hashvalue,None)
if cursor is not None:
# Update an existing entry (or maybe add a new one if there
# was a hash collision)
address = cursor.value
if key != storage_read_key(address):
self.collisions = self.collisions + 1
address = self.find_address(cursor,hashvalue,key)
# address may be None... meaning that we have to add a
# new record
new_address = storage_write(key,value,address)
if new_address == address:
continue
# Update index
if address is not None:
index_update(hashvalue,new_address,address)
else:
index[hashvalue] = new_address
else:
# Add a new entry
address = storage_write(key,value)
# Update index
index[hashvalue] = address
elif state == 2:
if __debug__:
log(SYSTEM_DEBUG,' deleting key "%s"',key)
if readonly:
raise Error,'dict is read-only'
hashvalue = hash(key)
cursor = index_cursor(hashvalue)
# Check that we have really found the key
address = cursor.value
if key != storage_read_key(address):
# Ah, a collision
address = self.find_address(cursor,hashvalue,key)
if address is None:
raise KeyError,'key not found'
storage_delete(address)
# Update index
index_delete(hashvalue,address)
# End the storage transaction
storage.end_transaction()
# Call basemethod
basemethod(self)
def read(self,key,checkonly=0):
# Load from disk
hashvalue = hash(key)
cursor = self.index.cursor(hashvalue)
# Check that we have really found the key
address = cursor.value
if key != self.storage.read_key(address):
# Ah, a collision
address = self.find_address(cursor,hashvalue,key)
if address is None:
raise KeyError,'key not found'
if checkonly:
return
else:
return self.storage.read(address)[1]
def keys(self):
""" Return a list of keys.
The method does not load any data into the cache, but does
take notice of uncommitted changes.
"""
l = []
read_key = self.storage.read_key
# First the cache entries that are not yet committed
for key,(state,value) in self.cache.items():
if state != 2:
l.append(key)
# Next the remaining entries read from disk
key_added = self.cache.has_key
for address in self.index.values():
key = read_key(address)
if not key_added(key):
l.append(key)
return l
def values(self):
""" Return a list of values.
The method does not load any data into the cache, but does
take notice of uncommitted changes.
"""
l = []
read = self.storage.read
# First the cache entries that are not yet committed
for key,(state,value) in self.cache.items():
if state != 2:
l.append(value)
# Next the remaining entries read from disk
key_added = self.cache.has_key
for address in self.index.values():
key,value = read(address)
if not key_added(key):
l.append(value)
return l
def items(self):
""" Return a list of items.
The method does not load any data into the cache, but does
take notice of uncommitted changes.
"""
l = []
read = self.storage.read
# First the cache entries that are not yet committed
for key,(state,value) in self.cache.items():
if state != 2:
l.append((key,value))
# Next the remaining entries read from disk
key_added = self.cache.has_key
for address in self.index.values():
key,value = read(address)
if not key_added(key):
l.append((key,value))
return l
def cursor(self,key,default=None):
""" Return a cursor instance for this kind of dictionary.
Note that cursors operate with the data on disk. The
method will raise an error in case there are uncommitted
changes pending.
In case the key is not found, default is returned instead.
"""
# Check for uncommitted changes
if self.cache and self.changed():
raise Error,'uncommitted data exists'
# Get the index cursor and create a dict cursor from it
cursor = self.index_cursor(key,None)
if cursor is None:
return default
return BeeDictCursor(self,cursor)
def collect_callback(self,old_position,new_position,raw_data):
""" Internal callback used to update the index when
the storage garbage collector moves a record.
"""
key = hash(self.storage.decode_key(raw_data))
try:
self.index.update(key,new_position,old_position)
except KeyError:
# Ok, then we'll add the key as new entry
log(SYSTEM_WARNING,
'Collect callback detected a missing key in '
'index: %i; restoring it',old_position)
self.index[key] = new_position
def recover_callback(self,old_position,new_position,raw_data):
""" Internal callback used to update the index when
the storage recovery collector finds a record.
old_position is only given as additional information.
new_position contains the position of the found record.
"""
self.index[hash(self.storage.decode_key(raw_data))] = new_position
###
class BeeDictCursor(BeeBaseDictCursor):
""" Cursor for BeeDict instances.
Since the keys are not kept directly in the index, .read_key()
must be used to obtain the real key objects instead of
querying the instance variable .key.
"""
def read(self):
""" Reads the object from the dict to which the cursor
currently points.
"""
keyobj = self.dict.storage.read_key(self.value)
return self.dict[keyobj]
def read_key(self):
""" Reads the key object from the dict to which the cursor
currently points.
"""
return self.dict.storage.read_key(self.value)
def write(self,object,minsize=0):
""" Writes the object to the dict under the key to which
the cursor currently points.
The new data is not written to disk until the dictionary's
current transaction is committed.
"""
keyobj = self.dict.storage.read_key(self.value)
self.dict[keyobj] = object
freeze(BeeDictCursor)
###
class BeeStringDict(BeeBaseDict):
""" A dictionary that is indexed by limited size strings.
Though the keys must be strings of limited size, the values
can be any pickleable object.
"""
def __init__(self,name,keysize=10,min_recordsize=0,readonly=0,recover=0,
autocommit=0,validate=0,
basemethod=BeeBaseDict.__init__):
""" Create an instance using name as dictionary name.
Two files will be created: name.dat and name.idx.
keysize gives the maximal size of the strings used as
index keys. min_recordsize gives a hint to the expected
typical size of (key,value) pickles: all records will have
at least this size.
To open the dictionary in recovery mode, pass a keyword
recover=1. Then run .recover() and reopen using the normal
settings.
If autocommit is true the cache control will do an
automatic .commit() whenever the transaction log
overflows.
If validate is true, the dictionary will run a validation
check after having successfully opened storage and index.
RecreateIndexError or RecoverError exceptions could be
raised in case inconsistencies are found.
XXX Save keysize in storage file header.
"""
self.name = name
self.storage = BeeStorage.BeeKeyValueStorage(\
name + '.dat',
lock= not readonly,
cache=0,
min_recordsize=min_recordsize,
readonly=readonly,
recover=recover)
if readonly:
filemode = 1
else:
if self.storage.is_new or recover:
filemode = 2
else:
filemode = 3
try:
self.index = BeeIndex.BeeStringIndex(\
name + '.idx',
dupkeys=0,
keysize=keysize,
filemode=filemode,
sectorsize=256)
except IOError:
raise RecreateIndexError,\
'Index for BeeStringDict "%s" could not be opened' % name
self.readonly = readonly
self.autocommit = autocommit
if validate:
self.validate_index()
self.validate_storage()
# Call basemethod
basemethod(self)
def commit(self,
basemethod=BeeBaseDict.commit):
""" Commit all changes and start a new transaction.
"""
if __debug__:
log(SYSTEM_DEBUG,'Committing all changes for "%s"...',self.name)
# Write all changed entries in the cache to disk
index = self.index
index_get = index.get
storage = self.storage
storage_write = storage.write
storage_delete = storage.delete
readonly = self.readonly
for key,(state,value) in self.cache.items():
if state == 1:
if __debug__:
log.object(SYSTEM_DEBUG,' writing key "%s":' % key,value)
address = index_get(key,None)
if address is not None:
# Update storage
new_addr = storage_write(key,value,address)
if new_addr == address:
continue
# Update index
index[key] = new_addr
else:
# Add to storage
address = storage_write(key,value)
# Add to the index
index[key] = address
elif state == 2:
if __debug__:
log(SYSTEM_DEBUG,' deleting key "%s"',key)
if readonly:
raise Error,'dict is read-only'
address = index_get(key,None)
if address is not None:
# Delete record
storage_delete(address)
# Update Index
del index[key]
else:
raise KeyError,'key not found'
# End the storage transaction
storage.end_transaction()
# Call basemethod
basemethod(self)
def read(self,key,checkonly=0):
address = self.index[key]
if checkonly:
return
else:
return self.storage.read(address)[1]
def keys(self):
""" Return a list of keys.
The method will raise an error if there are uncommitted
changes pending. Output is sorted ascending according to
keys.
"""
if self.cache and self.changed():
raise Error,'uncommitted data exists'
return self.index.keys()
def values(self):
""" Return a list of values.
The method will raise an error if there are uncommitted
changes pending. Output is sorted ascending according to
keys.
"""
if self.cache and self.changed():
raise Error,'uncommitted data exists'
l = []
read = self.storage.read
for address in self.index.values():
l.append(read(address)[1])
return l
def items(self):
""" Return a list of items.
The method will raise an error if there are uncommitted
changes pending. Output is sorted ascending according to
keys.
"""
if self.cache and self.changed():
raise Error,'uncommitted data exists'
l = []
read = self.storage.read
for address in self.index.values():
l.append(read(address))
return l
def cursor(self,key,default=None):
""" Return a cursor instance for this kind of dictionary.
Note that cursors operate with the data on disk meaning
that any uncommitted changes will not be seen by the
cursor.
In case the key is not found, default is returned instead.
"""
# Check for uncommitted changes
if self.cache and self.changed():
raise Error,'uncommitted data exists'
# Get the index cursor and create a dict cursor from it
cursor = self.index.cursor(key,None)
if cursor is None:
return default
return BeeStringDictCursor(self,cursor)
def collect_callback(self,old_position,new_position,raw_data):
""" Internal callback used to update the index when
the storage garbage collector moves a record.
"""
key = self.storage.decode_key(raw_data)
try:
self.index.update(key,new_position,old_position)
except KeyError:
# Ok, then we'll add the key as new entry
log(SYSTEM_WARNING,
'Collect callback detected a missing key in '
'index: %i; restoring it',old_position)
self.index[key] = new_position
def recover_callback(self,old_position,new_position,raw_data):
""" Internal callback used to update the index when
the storage recovery collector finds a record.
old_position is only given as additional information.
new_position contains the position of the found record.
XXX What if the recovery collector find two records
with the same key ?
"""
self.index[self.storage.decode_key(raw_data)] = new_position
freeze(BeeStringDict)
###
class BeeStringDictCursor(BeeBaseDictCursor):
""" Cursor for BeeStringDict instances.
"""
def read_key(self):
""" Reads the key object from the dict to which the cursor
currently points.
"""
return self.key
freeze(BeeStringDictCursor)
###
def AutoRecover(Class,*args,**kws):
""" Wrapper that runs recovery whenever needed.
This can still fail, but at least many "normal" failures can
be handled this way automatically.
"""
try:
d = apply(Class,args,kws)
return d
except BeeStorage.RunRecoveryError:
pass
except RecreateIndexError:
pass
except RecoverError:
pass
# Open in recovery mode
log(SYSTEM_ERROR,
'Database "%s" is corrupt; trying recovery...',args[0])
new_kws = kws.copy()
new_kws['readonly'] = 0
new_kws['validate'] = 0
#new_kws['lock'] = 1
new_kws['recover'] = 1
d = apply(Class,args,new_kws)
# Run recover & close
d.recover()
del d
# Reopen in original mode
d = apply(Class,args,kws)
log(SYSTEM_WARNING,
'Successfully recovered the database "%s"',args[0])
return d
###
if __name__ == '__main__':
s = BeeStringDict('test-BeeStringDict')
s['Marc'] = 'Sveta'
s['Thorsten'] = 'Petra'
s['Christian'] = 'Leesa'
d = BeeDict('test-BeeDict')
d['Marc'] = 'Sveta'
d['Thorsten'] = 'Petra'
d['Christian'] = 'Leesa'
del d,s
|