mirror of
https://github.com/python/cpython.git
synced 2025-11-04 03:44:55 +00:00
402 lines
12 KiB
Python
402 lines
12 KiB
Python
#!/usr/bin/env python
|
|
#------------------------------------------------------------------------
|
|
# Copyright (c) 1997-2001 by Total Control Software
|
|
# All Rights Reserved
|
|
#------------------------------------------------------------------------
|
|
#
|
|
# Module Name: dbShelve.py
|
|
#
|
|
# Description: A reimplementation of the standard shelve.py that
|
|
# forces the use of cPickle, and DB.
|
|
#
|
|
# Creation Date: 11/3/97 3:39:04PM
|
|
#
|
|
# License: This is free software. You may use this software for any
|
|
# purpose including modification/redistribution, so long as
|
|
# this header remains intact and that you do not claim any
|
|
# rights of ownership or authorship of this software. This
|
|
# software has been tested, but no warranty is expressed or
|
|
# implied.
|
|
#
|
|
# 13-Dec-2000: Updated to be used with the new bsddb3 package.
|
|
# Added DBShelfCursor class.
|
|
#
|
|
#------------------------------------------------------------------------
|
|
|
|
"""Manage shelves of pickled objects using bsddb database files for the
|
|
storage.
|
|
"""
|
|
|
|
#------------------------------------------------------------------------
|
|
|
|
import sys
|
|
absolute_import = (sys.version_info[0] >= 3)
|
|
if absolute_import :
|
|
# Because this syntaxis is not valid before Python 2.5
|
|
exec("from . import db")
|
|
else :
|
|
import db
|
|
|
|
if sys.version_info[0] >= 3 :
|
|
import cPickle # Will be converted to "pickle" by "2to3"
|
|
else :
|
|
if sys.version_info < (2, 6) :
|
|
import cPickle
|
|
else :
|
|
# When we drop support for python 2.3 and 2.4
|
|
# we could use: (in 2.5 we need a __future__ statement)
|
|
#
|
|
# with warnings.catch_warnings():
|
|
# warnings.filterwarnings(...)
|
|
# ...
|
|
#
|
|
# We can not use "with" as is, because it would be invalid syntax
|
|
# in python 2.3, 2.4 and (with no __future__) 2.5.
|
|
# Here we simulate "with" following PEP 343 :
|
|
import warnings
|
|
w = warnings.catch_warnings()
|
|
w.__enter__()
|
|
try :
|
|
warnings.filterwarnings('ignore',
|
|
message='the cPickle module has been removed in Python 3.0',
|
|
category=DeprecationWarning)
|
|
import cPickle
|
|
finally :
|
|
w.__exit__()
|
|
del w
|
|
|
|
#At version 2.3 cPickle switched to using protocol instead of bin
|
|
if sys.version_info >= (2, 3):
|
|
HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL
|
|
# In python 2.3.*, "cPickle.dumps" accepts no
|
|
# named parameters. "pickle.dumps" accepts them,
|
|
# so this seems a bug.
|
|
if sys.version_info < (2, 4):
|
|
def _dumps(object, protocol):
|
|
return cPickle.dumps(object, protocol)
|
|
else :
|
|
def _dumps(object, protocol):
|
|
return cPickle.dumps(object, protocol=protocol)
|
|
|
|
else:
|
|
HIGHEST_PROTOCOL = None
|
|
def _dumps(object, protocol):
|
|
return cPickle.dumps(object, bin=protocol)
|
|
|
|
|
|
if sys.version_info < (2, 6) :
|
|
try:
|
|
from UserDict import DictMixin
|
|
except ImportError:
|
|
# DictMixin is new in Python 2.3
|
|
class DictMixin: pass
|
|
MutableMapping = DictMixin
|
|
else :
|
|
import collections
|
|
MutableMapping = collections.MutableMapping
|
|
|
|
#------------------------------------------------------------------------
|
|
|
|
|
|
def open(filename, flags=db.DB_CREATE, mode=0660, filetype=db.DB_HASH,
|
|
dbenv=None, dbname=None):
|
|
"""
|
|
A simple factory function for compatibility with the standard
|
|
shleve.py module. It can be used like this, where key is a string
|
|
and data is a pickleable object:
|
|
|
|
from bsddb import dbshelve
|
|
db = dbshelve.open(filename)
|
|
|
|
db[key] = data
|
|
|
|
db.close()
|
|
"""
|
|
if type(flags) == type(''):
|
|
sflag = flags
|
|
if sflag == 'r':
|
|
flags = db.DB_RDONLY
|
|
elif sflag == 'rw':
|
|
flags = 0
|
|
elif sflag == 'w':
|
|
flags = db.DB_CREATE
|
|
elif sflag == 'c':
|
|
flags = db.DB_CREATE
|
|
elif sflag == 'n':
|
|
flags = db.DB_TRUNCATE | db.DB_CREATE
|
|
else:
|
|
raise db.DBError, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"
|
|
|
|
d = DBShelf(dbenv)
|
|
d.open(filename, dbname, filetype, flags, mode)
|
|
return d
|
|
|
|
#---------------------------------------------------------------------------
|
|
|
|
class DBShelveError(db.DBError): pass
|
|
|
|
|
|
class DBShelf(MutableMapping):
|
|
"""A shelf to hold pickled objects, built upon a bsddb DB object. It
|
|
automatically pickles/unpickles data objects going to/from the DB.
|
|
"""
|
|
def __init__(self, dbenv=None):
|
|
self.db = db.DB(dbenv)
|
|
self._closed = True
|
|
if HIGHEST_PROTOCOL:
|
|
self.protocol = HIGHEST_PROTOCOL
|
|
else:
|
|
self.protocol = 1
|
|
|
|
|
|
def __del__(self):
|
|
self.close()
|
|
|
|
|
|
def __getattr__(self, name):
|
|
"""Many methods we can just pass through to the DB object.
|
|
(See below)
|
|
"""
|
|
return getattr(self.db, name)
|
|
|
|
|
|
#-----------------------------------
|
|
# Dictionary access methods
|
|
|
|
def __len__(self):
|
|
return len(self.db)
|
|
|
|
|
|
def __getitem__(self, key):
|
|
data = self.db[key]
|
|
return cPickle.loads(data)
|
|
|
|
|
|
def __setitem__(self, key, value):
|
|
data = _dumps(value, self.protocol)
|
|
self.db[key] = data
|
|
|
|
|
|
def __delitem__(self, key):
|
|
del self.db[key]
|
|
|
|
|
|
def keys(self, txn=None):
|
|
if txn is not None:
|
|
return self.db.keys(txn)
|
|
else:
|
|
return self.db.keys()
|
|
|
|
if sys.version_info >= (2, 6) :
|
|
def __iter__(self) : # XXX: Load all keys in memory :-(
|
|
for k in self.db.keys() :
|
|
yield k
|
|
|
|
# Do this when "DB" support iteration
|
|
# Or is it enough to pass thru "getattr"?
|
|
#
|
|
# def __iter__(self) :
|
|
# return self.db.__iter__()
|
|
|
|
|
|
def open(self, *args, **kwargs):
|
|
self.db.open(*args, **kwargs)
|
|
self._closed = False
|
|
|
|
|
|
def close(self, *args, **kwargs):
|
|
self.db.close(*args, **kwargs)
|
|
self._closed = True
|
|
|
|
|
|
def __repr__(self):
|
|
if self._closed:
|
|
return '<DBShelf @ 0x%x - closed>' % (id(self))
|
|
else:
|
|
return repr(dict(self.iteritems()))
|
|
|
|
|
|
def items(self, txn=None):
|
|
if txn is not None:
|
|
items = self.db.items(txn)
|
|
else:
|
|
items = self.db.items()
|
|
newitems = []
|
|
|
|
for k, v in items:
|
|
newitems.append( (k, cPickle.loads(v)) )
|
|
return newitems
|
|
|
|
def values(self, txn=None):
|
|
if txn is not None:
|
|
values = self.db.values(txn)
|
|
else:
|
|
values = self.db.values()
|
|
|
|
return map(cPickle.loads, values)
|
|
|
|
#-----------------------------------
|
|
# Other methods
|
|
|
|
def __append(self, value, txn=None):
|
|
data = _dumps(value, self.protocol)
|
|
return self.db.append(data, txn)
|
|
|
|
def append(self, value, txn=None):
|
|
if self.get_type() == db.DB_RECNO:
|
|
return self.__append(value, txn=txn)
|
|
raise DBShelveError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"
|
|
|
|
|
|
def associate(self, secondaryDB, callback, flags=0):
|
|
def _shelf_callback(priKey, priData, realCallback=callback):
|
|
# Safe in Python 2.x because expresion short circuit
|
|
if sys.version_info[0] < 3 or isinstance(priData, bytes) :
|
|
data = cPickle.loads(priData)
|
|
else :
|
|
data = cPickle.loads(bytes(priData, "iso8859-1")) # 8 bits
|
|
return realCallback(priKey, data)
|
|
|
|
return self.db.associate(secondaryDB, _shelf_callback, flags)
|
|
|
|
|
|
#def get(self, key, default=None, txn=None, flags=0):
|
|
def get(self, *args, **kw):
|
|
# We do it with *args and **kw so if the default value wasn't
|
|
# given nothing is passed to the extension module. That way
|
|
# an exception can be raised if set_get_returns_none is turned
|
|
# off.
|
|
data = self.db.get(*args, **kw)
|
|
try:
|
|
return cPickle.loads(data)
|
|
except (EOFError, TypeError, cPickle.UnpicklingError):
|
|
return data # we may be getting the default value, or None,
|
|
# so it doesn't need unpickled.
|
|
|
|
def get_both(self, key, value, txn=None, flags=0):
|
|
data = _dumps(value, self.protocol)
|
|
data = self.db.get(key, data, txn, flags)
|
|
return cPickle.loads(data)
|
|
|
|
|
|
def cursor(self, txn=None, flags=0):
|
|
c = DBShelfCursor(self.db.cursor(txn, flags))
|
|
c.protocol = self.protocol
|
|
return c
|
|
|
|
|
|
def put(self, key, value, txn=None, flags=0):
|
|
data = _dumps(value, self.protocol)
|
|
return self.db.put(key, data, txn, flags)
|
|
|
|
|
|
def join(self, cursorList, flags=0):
|
|
raise NotImplementedError
|
|
|
|
|
|
#----------------------------------------------
|
|
# Methods allowed to pass-through to self.db
|
|
#
|
|
# close, delete, fd, get_byteswapped, get_type, has_key,
|
|
# key_range, open, remove, rename, stat, sync,
|
|
# upgrade, verify, and all set_* methods.
|
|
|
|
|
|
#---------------------------------------------------------------------------
|
|
|
|
class DBShelfCursor:
|
|
"""
|
|
"""
|
|
def __init__(self, cursor):
|
|
self.dbc = cursor
|
|
|
|
def __del__(self):
|
|
self.close()
|
|
|
|
|
|
def __getattr__(self, name):
|
|
"""Some methods we can just pass through to the cursor object. (See below)"""
|
|
return getattr(self.dbc, name)
|
|
|
|
|
|
#----------------------------------------------
|
|
|
|
def dup(self, flags=0):
|
|
c = DBShelfCursor(self.dbc.dup(flags))
|
|
c.protocol = self.protocol
|
|
return c
|
|
|
|
|
|
def put(self, key, value, flags=0):
|
|
data = _dumps(value, self.protocol)
|
|
return self.dbc.put(key, data, flags)
|
|
|
|
|
|
def get(self, *args):
|
|
count = len(args) # a method overloading hack
|
|
method = getattr(self, 'get_%d' % count)
|
|
method(*args)
|
|
|
|
def get_1(self, flags):
|
|
rec = self.dbc.get(flags)
|
|
return self._extract(rec)
|
|
|
|
def get_2(self, key, flags):
|
|
rec = self.dbc.get(key, flags)
|
|
return self._extract(rec)
|
|
|
|
def get_3(self, key, value, flags):
|
|
data = _dumps(value, self.protocol)
|
|
rec = self.dbc.get(key, flags)
|
|
return self._extract(rec)
|
|
|
|
|
|
def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT)
|
|
def first(self, flags=0): return self.get_1(flags|db.DB_FIRST)
|
|
def last(self, flags=0): return self.get_1(flags|db.DB_LAST)
|
|
def next(self, flags=0): return self.get_1(flags|db.DB_NEXT)
|
|
def prev(self, flags=0): return self.get_1(flags|db.DB_PREV)
|
|
def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME)
|
|
def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP)
|
|
def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP)
|
|
def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP)
|
|
|
|
|
|
def get_both(self, key, value, flags=0):
|
|
data = _dumps(value, self.protocol)
|
|
rec = self.dbc.get_both(key, flags)
|
|
return self._extract(rec)
|
|
|
|
|
|
def set(self, key, flags=0):
|
|
rec = self.dbc.set(key, flags)
|
|
return self._extract(rec)
|
|
|
|
def set_range(self, key, flags=0):
|
|
rec = self.dbc.set_range(key, flags)
|
|
return self._extract(rec)
|
|
|
|
def set_recno(self, recno, flags=0):
|
|
rec = self.dbc.set_recno(recno, flags)
|
|
return self._extract(rec)
|
|
|
|
set_both = get_both
|
|
|
|
def _extract(self, rec):
|
|
if rec is None:
|
|
return None
|
|
else:
|
|
key, data = rec
|
|
# Safe in Python 2.x because expresion short circuit
|
|
if sys.version_info[0] < 3 or isinstance(data, bytes) :
|
|
return key, cPickle.loads(data)
|
|
else :
|
|
return key, cPickle.loads(bytes(data, "iso8859-1")) # 8 bits
|
|
|
|
#----------------------------------------------
|
|
# Methods allowed to pass-through to self.dbc
|
|
#
|
|
# close, count, delete, get_recno, join_item
|
|
|
|
|
|
#---------------------------------------------------------------------------
|